From d3ec5a5138297a948654094a8cfe8cc734fb6bf2 Mon Sep 17 00:00:00 2001 From: wojtekt Date: Tue, 9 Mar 2021 21:15:34 +0100 Subject: [PATCH] Move non-provider specific upgrade tests logic to upgrades package --- test/e2e/cloud/gcp/cluster_upgrade.go | 144 ++-------------------- test/e2e/cloud/gcp/upgrade_mechanics.go | 6 +- test/e2e/upgrades/upgrade_suite.go | 151 ++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 135 deletions(-) create mode 100644 test/e2e/upgrades/upgrade_suite.go diff --git a/test/e2e/cloud/gcp/cluster_upgrade.go b/test/e2e/cloud/gcp/cluster_upgrade.go index e3b3981e4a3..584e89972f9 100644 --- a/test/e2e/cloud/gcp/cluster_upgrade.go +++ b/test/e2e/cloud/gcp/cluster_upgrade.go @@ -17,22 +17,15 @@ limitations under the License. package gcp import ( - "encoding/xml" "fmt" - "os" "path" - "path/filepath" - "regexp" "strings" - "sync" "time" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/discovery" - "k8s.io/kubernetes/test/e2e/chaosmonkey" "k8s.io/kubernetes/test/e2e/framework" e2econfig "k8s.io/kubernetes/test/e2e/framework/config" - e2eginkgowrapper "k8s.io/kubernetes/test/e2e/framework/ginkgowrapper" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/upgrades" "k8s.io/kubernetes/test/e2e/upgrades/apps" @@ -115,7 +108,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { testSuite.TestCases = append(testSuite.TestCases, masterUpgradeTest, nil) upgradeFunc := ControlPlaneUpgradeFunc(f, upgCtx, masterUpgradeTest, nil) - runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) @@ -129,7 +122,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { testSuite.TestCases = append(testSuite.TestCases, clusterUpgradeTest) upgradeFunc := ClusterUpgradeFunc(f, upgCtx, clusterUpgradeTest, nil, nil) - runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -148,7 +141,7 @@ var _ = ginkgo.Describe("Downgrade [Feature:Downgrade]", func() { testSuite.TestCases = append(testSuite.TestCases, clusterDowngradeTest) upgradeFunc := ClusterDowngradeFunc(f, upgCtx, clusterDowngradeTest, nil, nil) - runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -167,10 +160,10 @@ var _ = ginkgo.Describe("etcd Upgrade [Feature:EtcdUpgrade]", func() { upgradeFunc := func() { start := time.Now() - defer finalizeUpgradeTest(start, etcdTest) + defer upgrades.FinalizeUpgradeTest(start, etcdTest) framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion)) } - runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.EtcdUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.EtcdUpgrade, upgradeFunc) }) }) }) @@ -189,7 +182,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { testSuite.TestCases = append(testSuite.TestCases, gpuUpgradeTest) upgradeFunc := ControlPlaneUpgradeFunc(f, upgCtx, gpuUpgradeTest, nil) - runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) ginkgo.Describe("cluster upgrade", func() { @@ -202,7 +195,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { testSuite.TestCases = append(testSuite.TestCases, gpuUpgradeTest) upgradeFunc := ClusterUpgradeFunc(f, upgCtx, gpuUpgradeTest, nil, nil) - runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) ginkgo.Describe("cluster downgrade", func() { @@ -215,7 +208,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { testSuite.TestCases = append(testSuite.TestCases, gpuDowngradeTest) upgradeFunc := ClusterDowngradeFunc(f, upgCtx, gpuDowngradeTest, nil, nil) - runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -234,7 +227,7 @@ var _ = ginkgo.Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", testSuite.TestCases = append(testSuite.TestCases, statefulUpgradeTest) upgradeFunc := ClusterUpgradeFunc(f, upgCtx, statefulUpgradeTest, nil, nil) - runUpgradeSuite(upgCtx, statefulsetUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, statefulsetUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -261,7 +254,7 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati extraEnvs := kubeProxyDaemonSetExtraEnvs(true) upgradeFunc := ClusterUpgradeFunc(f, upgCtx, kubeProxyUpgradeTest, extraEnvs, extraEnvs) - runUpgradeSuite(upgCtx, kubeProxyUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, kubeProxyUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -279,7 +272,7 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati extraEnvs := kubeProxyDaemonSetExtraEnvs(false) upgradeFunc := ClusterDowngradeFunc(f, upgCtx, kubeProxyDowngradeTest, extraEnvs, extraEnvs) - runUpgradeSuite(upgCtx, kubeProxyDowngradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, kubeProxyDowngradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -302,124 +295,11 @@ var _ = ginkgo.Describe("[sig-auth] ServiceAccount admission controller migratio extraEnvs := []string{"KUBE_FEATURE_GATES=BoundServiceAccountTokenVolume=true"} upgradeFunc := ControlPlaneUpgradeFunc(f, upgCtx, serviceaccountAdmissionControllerMigrationTest, extraEnvs) - runUpgradeSuite(upgCtx, serviceaccountAdmissionControllerMigrationTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) + upgrades.RunUpgradeSuite(upgCtx, serviceaccountAdmissionControllerMigrationTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) }) -type chaosMonkeyAdapter struct { - test upgrades.Test - testReport *junit.TestCase - framework *framework.Framework - upgradeType upgrades.UpgradeType - upgCtx upgrades.UpgradeContext -} - -func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) { - start := time.Now() - var once sync.Once - ready := func() { - once.Do(func() { - sem.Ready() - }) - } - defer finalizeUpgradeTest(start, cma.testReport) - defer ready() - if skippable, ok := cma.test.(upgrades.Skippable); ok && skippable.Skip(cma.upgCtx) { - ginkgo.By("skipping test " + cma.test.Name()) - cma.testReport.Skipped = "skipping test " + cma.test.Name() - return - } - - defer cma.test.Teardown(cma.framework) - cma.test.Setup(cma.framework) - ready() - cma.test.Test(cma.framework, sem.StopCh, cma.upgradeType) -} - -func finalizeUpgradeTest(start time.Time, tc *junit.TestCase) { - tc.Time = time.Since(start).Seconds() - r := recover() - if r == nil { - return - } - - switch r := r.(type) { - case e2eginkgowrapper.FailurePanic: - tc.Failures = []*junit.Failure{ - { - Message: r.Message, - Type: "Failure", - Value: fmt.Sprintf("%s\n\n%s", r.Message, r.FullStackTrace), - }, - } - case e2eskipper.SkipPanic: - tc.Skipped = fmt.Sprintf("%s:%d %q", r.Filename, r.Line, r.Message) - default: - tc.Errors = []*junit.Error{ - { - Message: fmt.Sprintf("%v", r), - Type: "Panic", - Value: fmt.Sprintf("%v", r), - }, - } - } -} - -func createUpgradeFrameworks(tests []upgrades.Test) map[string]*framework.Framework { - nsFilter := regexp.MustCompile("[^[:word:]-]+") // match anything that's not a word character or hyphen - testFrameworks := map[string]*framework.Framework{} - for _, t := range tests { - ns := nsFilter.ReplaceAllString(t.Name(), "-") // and replace with a single hyphen - ns = strings.Trim(ns, "-") - testFrameworks[t.Name()] = framework.NewDefaultFramework(ns) - } - return testFrameworks -} - -func runUpgradeSuite( - upgCtx *upgrades.UpgradeContext, - tests []upgrades.Test, - testSuite *junit.TestSuite, - upgradeType upgrades.UpgradeType, - upgradeFunc func(), -) { - testFrameworks := createUpgradeFrameworks(tests) - - cm := chaosmonkey.New(upgradeFunc) - for _, t := range tests { - testCase := &junit.TestCase{ - Name: t.Name(), - Classname: "upgrade_tests", - } - testSuite.TestCases = append(testSuite.TestCases, testCase) - cma := chaosMonkeyAdapter{ - test: t, - testReport: testCase, - framework: testFrameworks[t.Name()], - upgradeType: upgradeType, - upgCtx: *upgCtx, - } - cm.Register(cma.Test) - } - - start := time.Now() - defer func() { - testSuite.Update() - testSuite.Time = time.Since(start).Seconds() - if framework.TestContext.ReportDir != "" { - fname := filepath.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%supgrades.xml", framework.TestContext.ReportPrefix)) - f, err := os.Create(fname) - if err != nil { - return - } - defer f.Close() - xml.NewEncoder(f).Encode(testSuite) - } - }() - cm.Do() -} - func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget, upgradeImage string) (*upgrades.UpgradeContext, error) { current, err := c.ServerVersion() if err != nil { diff --git a/test/e2e/cloud/gcp/upgrade_mechanics.go b/test/e2e/cloud/gcp/upgrade_mechanics.go index 3f2e721b2b4..2c1407d31e1 100644 --- a/test/e2e/cloud/gcp/upgrade_mechanics.go +++ b/test/e2e/cloud/gcp/upgrade_mechanics.go @@ -36,7 +36,7 @@ import ( func ControlPlaneUpgradeFunc(f *framework.Framework, upgCtx *upgrades.UpgradeContext, testCase *junit.TestCase, controlPlaneExtraEnvs []string) func() { return func() { start := time.Now() - defer finalizeUpgradeTest(start, testCase) + defer upgrades.FinalizeUpgradeTest(start, testCase) target := upgCtx.Versions[1].Version.String() framework.ExpectNoError(controlPlaneUpgrade(f, target, controlPlaneExtraEnvs)) framework.ExpectNoError(checkControlPlaneVersion(f.ClientSet, target)) @@ -47,7 +47,7 @@ func ControlPlaneUpgradeFunc(f *framework.Framework, upgCtx *upgrades.UpgradeCon func ClusterUpgradeFunc(f *framework.Framework, upgCtx *upgrades.UpgradeContext, testCase *junit.TestCase, controlPlaneExtraEnvs, nodeExtraEnvs []string) func() { return func() { start := time.Now() - defer finalizeUpgradeTest(start, testCase) + defer upgrades.FinalizeUpgradeTest(start, testCase) target := upgCtx.Versions[1].Version.String() image := upgCtx.Versions[1].NodeImage framework.ExpectNoError(controlPlaneUpgrade(f, target, controlPlaneExtraEnvs)) @@ -61,7 +61,7 @@ func ClusterUpgradeFunc(f *framework.Framework, upgCtx *upgrades.UpgradeContext, func ClusterDowngradeFunc(f *framework.Framework, upgCtx *upgrades.UpgradeContext, testCase *junit.TestCase, controlPlaneExtraEnvs, nodeExtraEnvs []string) func() { return func() { start := time.Now() - defer finalizeUpgradeTest(start, testCase) + defer upgrades.FinalizeUpgradeTest(start, testCase) target := upgCtx.Versions[1].Version.String() image := upgCtx.Versions[1].NodeImage // Yes this really is a downgrade. And nodes must downgrade first. diff --git a/test/e2e/upgrades/upgrade_suite.go b/test/e2e/upgrades/upgrade_suite.go new file mode 100644 index 00000000000..89f1a906842 --- /dev/null +++ b/test/e2e/upgrades/upgrade_suite.go @@ -0,0 +1,151 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrades + +import ( + "encoding/xml" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "k8s.io/kubernetes/test/e2e/chaosmonkey" + "k8s.io/kubernetes/test/e2e/framework" + e2eginkgowrapper "k8s.io/kubernetes/test/e2e/framework/ginkgowrapper" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" + "k8s.io/kubernetes/test/utils/junit" + + "github.com/onsi/ginkgo" +) + +type chaosMonkeyAdapter struct { + test Test + testReport *junit.TestCase + framework *framework.Framework + upgradeType UpgradeType + upgCtx UpgradeContext +} + +func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) { + start := time.Now() + var once sync.Once + ready := func() { + once.Do(func() { + sem.Ready() + }) + } + defer FinalizeUpgradeTest(start, cma.testReport) + defer ready() + if skippable, ok := cma.test.(Skippable); ok && skippable.Skip(cma.upgCtx) { + ginkgo.By("skipping test " + cma.test.Name()) + cma.testReport.Skipped = "skipping test " + cma.test.Name() + return + } + + defer cma.test.Teardown(cma.framework) + cma.test.Setup(cma.framework) + ready() + cma.test.Test(cma.framework, sem.StopCh, cma.upgradeType) +} + +// FinalizeUpgradeTest fills the necessary information about junit.TestCase. +func FinalizeUpgradeTest(start time.Time, tc *junit.TestCase) { + tc.Time = time.Since(start).Seconds() + r := recover() + if r == nil { + return + } + + switch r := r.(type) { + case e2eginkgowrapper.FailurePanic: + tc.Failures = []*junit.Failure{ + { + Message: r.Message, + Type: "Failure", + Value: fmt.Sprintf("%s\n\n%s", r.Message, r.FullStackTrace), + }, + } + case e2eskipper.SkipPanic: + tc.Skipped = fmt.Sprintf("%s:%d %q", r.Filename, r.Line, r.Message) + default: + tc.Errors = []*junit.Error{ + { + Message: fmt.Sprintf("%v", r), + Type: "Panic", + Value: fmt.Sprintf("%v", r), + }, + } + } +} + +func createUpgradeFrameworks(tests []Test) map[string]*framework.Framework { + nsFilter := regexp.MustCompile("[^[:word:]-]+") // match anything that's not a word character or hyphen + testFrameworks := map[string]*framework.Framework{} + for _, t := range tests { + ns := nsFilter.ReplaceAllString(t.Name(), "-") // and replace with a single hyphen + ns = strings.Trim(ns, "-") + testFrameworks[t.Name()] = framework.NewDefaultFramework(ns) + } + return testFrameworks +} + +// RunUpgradeSuite runs the actual upgrade tests. +func RunUpgradeSuite( + upgCtx *UpgradeContext, + tests []Test, + testSuite *junit.TestSuite, + upgradeType UpgradeType, + upgradeFunc func(), +) { + testFrameworks := createUpgradeFrameworks(tests) + + cm := chaosmonkey.New(upgradeFunc) + for _, t := range tests { + testCase := &junit.TestCase{ + Name: t.Name(), + Classname: "upgrade_tests", + } + testSuite.TestCases = append(testSuite.TestCases, testCase) + cma := chaosMonkeyAdapter{ + test: t, + testReport: testCase, + framework: testFrameworks[t.Name()], + upgradeType: upgradeType, + upgCtx: *upgCtx, + } + cm.Register(cma.Test) + } + + start := time.Now() + defer func() { + testSuite.Update() + testSuite.Time = time.Since(start).Seconds() + if framework.TestContext.ReportDir != "" { + fname := filepath.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%supgrades.xml", framework.TestContext.ReportPrefix)) + f, err := os.Create(fname) + if err != nil { + return + } + defer f.Close() + xml.NewEncoder(f).Encode(testSuite) + } + }() + cm.Do() +}