From 6d4a601b72cfdf40aa64d249845310cc3adc09c5 Mon Sep 17 00:00:00 2001 From: wojtekt Date: Sat, 6 Mar 2021 19:03:57 +0100 Subject: [PATCH 1/4] Compute upgradeContext once in upgrade tests --- test/e2e/cloud/gcp/cluster_upgrade.go | 32 +++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/e2e/cloud/gcp/cluster_upgrade.go b/test/e2e/cloud/gcp/cluster_upgrade.go index 15738d2d2a9..13b27bafa5f 100644 --- a/test/e2e/cloud/gcp/cluster_upgrade.go +++ b/test/e2e/cloud/gcp/cluster_upgrade.go @@ -159,7 +159,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } - runUpgradeSuite(f, upgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) @@ -181,7 +181,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } - runUpgradeSuite(f, upgradeTests, testSuite, upgrades.NodeUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.NodeUpgrade, upgradeFunc) }) }) @@ -202,7 +202,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } - runUpgradeSuite(f, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -230,7 +230,7 @@ var _ = ginkgo.Describe("Downgrade [Feature:Downgrade]", func() { framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } - runUpgradeSuite(f, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -240,6 +240,9 @@ var _ = ginkgo.Describe("etcd Upgrade [Feature:EtcdUpgrade]", func() { ginkgo.Describe("etcd upgrade", func() { ginkgo.It("should maintain a functioning cluster", func() { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + framework.ExpectNoError(err) + testSuite := &junit.TestSuite{Name: "Etcd upgrade"} etcdTest := &junit.TestCase{Name: "[sig-cloud-provider-gcp] etcd-upgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, etcdTest) @@ -249,7 +252,7 @@ var _ = ginkgo.Describe("etcd Upgrade [Feature:EtcdUpgrade]", func() { defer finalizeUpgradeTest(start, etcdTest) framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion)) } - runUpgradeSuite(f, upgradeTests, testSuite, upgrades.EtcdUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.EtcdUpgrade, upgradeFunc) }) }) }) @@ -273,7 +276,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } - runUpgradeSuite(f, gpuUpgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) ginkgo.Describe("cluster upgrade", func() { @@ -293,7 +296,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } - runUpgradeSuite(f, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) ginkgo.Describe("cluster downgrade", func() { @@ -313,7 +316,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } - runUpgradeSuite(f, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -339,7 +342,7 @@ var _ = ginkgo.Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } - runUpgradeSuite(f, statefulsetUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, statefulsetUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -373,7 +376,7 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati framework.ExpectNoError(nodeUpgradeGCEWithKubeProxyDaemonSet(f, target, *upgradeImage, true)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } - runUpgradeSuite(f, kubeProxyUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, kubeProxyUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -399,7 +402,7 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati framework.ExpectNoError(masterUpgradeGCEWithKubeProxyDaemonSet(target, false)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } - runUpgradeSuite(f, kubeProxyDowngradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, kubeProxyDowngradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) }) @@ -426,7 +429,7 @@ var _ = ginkgo.Describe("[sig-auth] ServiceAccount admission controller migratio target := upgCtx.Versions[1].Version.String() framework.ExpectNoError(masterUpgrade(f, target, []string{"KUBE_FEATURE_GATES=BoundServiceAccountTokenVolume=true"})) } - runUpgradeSuite(f, serviceaccountAdmissionControllerMigrationTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) + runUpgradeSuite(upgCtx, serviceaccountAdmissionControllerMigrationTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) }) @@ -502,15 +505,12 @@ func createUpgradeFrameworks(tests []upgrades.Test) map[string]*framework.Framew } func runUpgradeSuite( - f *framework.Framework, + upgCtx *upgrades.UpgradeContext, tests []upgrades.Test, testSuite *junit.TestSuite, upgradeType upgrades.UpgradeType, upgradeFunc func(), ) { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) - framework.ExpectNoError(err) - testFrameworks := createUpgradeFrameworks(tests) cm := chaosmonkey.New(upgradeFunc) From d1f721b73f48d2068d8fd2a7281c1c90d9844e0a Mon Sep 17 00:00:00 2001 From: wojtekt Date: Tue, 9 Mar 2021 08:35:53 +0100 Subject: [PATCH 2/4] Cleanup upgrade functions --- test/e2e/cloud/gcp/cluster_upgrade.go | 49 +++++++++++++++------------ 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/test/e2e/cloud/gcp/cluster_upgrade.go b/test/e2e/cloud/gcp/cluster_upgrade.go index 13b27bafa5f..964159f7b60 100644 --- a/test/e2e/cloud/gcp/cluster_upgrade.go +++ b/test/e2e/cloud/gcp/cluster_upgrade.go @@ -142,7 +142,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { // in a "Describe". ginkgo.Describe("master upgrade", func() { ginkgo.It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Master upgrade"} @@ -165,7 +165,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { ginkgo.Describe("node upgrade", func() { ginkgo.It("should maintain a functioning cluster [Feature:NodeUpgrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Node upgrade"} @@ -178,7 +178,8 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { start := time.Now() defer finalizeUpgradeTest(start, nodeUpgradeTest) target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) + image := upgCtx.Versions[1].NodeImage + framework.ExpectNoError(nodeUpgrade(f, target, image)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.NodeUpgrade, upgradeFunc) @@ -187,7 +188,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { ginkgo.Describe("cluster upgrade", func() { ginkgo.It("should maintain a functioning cluster [Feature:ClusterUpgrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Cluster upgrade"} @@ -197,9 +198,10 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { start := time.Now() defer finalizeUpgradeTest(start, clusterUpgradeTest) target := upgCtx.Versions[1].Version.String() + image := upgCtx.Versions[1].NodeImage framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) + framework.ExpectNoError(nodeUpgrade(f, target, image)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) @@ -213,7 +215,7 @@ var _ = ginkgo.Describe("Downgrade [Feature:Downgrade]", func() { ginkgo.Describe("cluster downgrade", func() { ginkgo.It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Cluster downgrade"} @@ -225,7 +227,8 @@ var _ = ginkgo.Describe("Downgrade [Feature:Downgrade]", func() { defer finalizeUpgradeTest(start, clusterDowngradeTest) // Yes this really is a downgrade. And nodes must downgrade first. target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) + image := upgCtx.Versions[1].NodeImage + framework.ExpectNoError(nodeUpgrade(f, target, image)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) @@ -240,7 +243,7 @@ var _ = ginkgo.Describe("etcd Upgrade [Feature:EtcdUpgrade]", func() { ginkgo.Describe("etcd upgrade", func() { ginkgo.It("should maintain a functioning cluster", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Etcd upgrade"} @@ -263,7 +266,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { ginkgo.Describe("master upgrade", func() { ginkgo.It("should NOT disrupt gpu pod [Feature:GPUMasterUpgrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "GPU master upgrade"} @@ -281,7 +284,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { }) ginkgo.Describe("cluster upgrade", func() { ginkgo.It("should be able to run gpu pod after upgrade [Feature:GPUClusterUpgrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "GPU cluster upgrade"} @@ -291,9 +294,10 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { start := time.Now() defer finalizeUpgradeTest(start, gpuUpgradeTest) target := upgCtx.Versions[1].Version.String() + image := upgCtx.Versions[1].NodeImage framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) + framework.ExpectNoError(nodeUpgrade(f, target, image)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) @@ -301,7 +305,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { }) ginkgo.Describe("cluster downgrade", func() { ginkgo.It("should be able to run gpu pod after downgrade [Feature:GPUClusterDowngrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "GPU cluster downgrade"} @@ -311,7 +315,8 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { start := time.Now() defer finalizeUpgradeTest(start, gpuDowngradeTest) target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) + image := upgCtx.Versions[1].NodeImage + framework.ExpectNoError(nodeUpgrade(f, target, image)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) @@ -327,7 +332,7 @@ var _ = ginkgo.Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", ginkgo.Describe("stateful upgrade", func() { ginkgo.It("should maintain a functioning cluster", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "Stateful upgrade"} @@ -337,9 +342,10 @@ var _ = ginkgo.Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", start := time.Now() defer finalizeUpgradeTest(start, statefulUpgradeTest) target := upgCtx.Versions[1].Version.String() + image := upgCtx.Versions[1].NodeImage framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) + framework.ExpectNoError(nodeUpgrade(f, target, image)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } runUpgradeSuite(upgCtx, statefulsetUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) @@ -357,7 +363,7 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati ginkgo.Describe("Upgrade kube-proxy from static pods to a DaemonSet", func() { ginkgo.It("should maintain a functioning cluster [Feature:KubeProxyDaemonSetUpgrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "kube-proxy upgrade"} @@ -371,9 +377,10 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati start := time.Now() defer finalizeUpgradeTest(start, kubeProxyUpgradeTest) target := upgCtx.Versions[1].Version.String() + image := upgCtx.Versions[1].NodeImage framework.ExpectNoError(masterUpgradeGCEWithKubeProxyDaemonSet(target, true)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgradeGCEWithKubeProxyDaemonSet(f, target, *upgradeImage, true)) + framework.ExpectNoError(nodeUpgradeGCEWithKubeProxyDaemonSet(f, target, image, true)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } runUpgradeSuite(upgCtx, kubeProxyUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) @@ -382,7 +389,7 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati ginkgo.Describe("Downgrade kube-proxy from a DaemonSet to static pods", func() { ginkgo.It("should maintain a functioning cluster [Feature:KubeProxyDaemonSetDowngrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "kube-proxy downgrade"} @@ -413,7 +420,7 @@ var _ = ginkgo.Describe("[sig-auth] ServiceAccount admission controller migratio ginkgo.Describe("master upgrade", func() { ginkgo.It("should maintain a functioning cluster", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget) + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) framework.ExpectNoError(err) testSuite := &junit.TestSuite{Name: "ServiceAccount admission controller migration"} @@ -547,7 +554,7 @@ func runUpgradeSuite( cm.Do() } -func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget string) (*upgrades.UpgradeContext, error) { +func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget, upgradeImage string) (*upgrades.UpgradeContext, error) { current, err := c.ServerVersion() if err != nil { return nil, err @@ -583,7 +590,7 @@ func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget string) (*u upgCtx.Versions = append(upgCtx.Versions, upgrades.VersionContext{ Version: *nextVer, - NodeImage: *upgradeImage, + NodeImage: upgradeImage, }) return upgCtx, nil From 04a196a3dcfcd104bf7a08700ec26c2b33d40054 Mon Sep 17 00:00:00 2001 From: wojtekt Date: Tue, 9 Mar 2021 08:50:49 +0100 Subject: [PATCH 3/4] Unify masterUpgrade and nodeUpgrade signatures --- test/e2e/cloud/gcp/cluster_upgrade.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/e2e/cloud/gcp/cluster_upgrade.go b/test/e2e/cloud/gcp/cluster_upgrade.go index 964159f7b60..b289e1688e1 100644 --- a/test/e2e/cloud/gcp/cluster_upgrade.go +++ b/test/e2e/cloud/gcp/cluster_upgrade.go @@ -179,7 +179,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { defer finalizeUpgradeTest(start, nodeUpgradeTest) target := upgCtx.Versions[1].Version.String() image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(nodeUpgrade(f, target, image)) + framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.NodeUpgrade, upgradeFunc) @@ -201,7 +201,7 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { image := upgCtx.Versions[1].NodeImage framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgrade(f, target, image)) + framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) @@ -228,7 +228,7 @@ var _ = ginkgo.Describe("Downgrade [Feature:Downgrade]", func() { // Yes this really is a downgrade. And nodes must downgrade first. target := upgCtx.Versions[1].Version.String() image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(nodeUpgrade(f, target, image)) + framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) @@ -297,7 +297,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { image := upgCtx.Versions[1].NodeImage framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgrade(f, target, image)) + framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) @@ -316,7 +316,7 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { defer finalizeUpgradeTest(start, gpuDowngradeTest) target := upgCtx.Versions[1].Version.String() image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(nodeUpgrade(f, target, image)) + framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) @@ -345,7 +345,7 @@ var _ = ginkgo.Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", image := upgCtx.Versions[1].NodeImage framework.ExpectNoError(masterUpgrade(f, target, nil)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgrade(f, target, image)) + framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) } runUpgradeSuite(upgCtx, statefulsetUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) @@ -676,12 +676,12 @@ func checkNodesVersions(cs clientset.Interface, want string) error { } // nodeUpgrade upgrades nodes on GCE/GKE. -func nodeUpgrade(f *framework.Framework, v string, img string) error { +func nodeUpgrade(f *framework.Framework, v string, img string, extraEnvs []string) error { // Perform the upgrade. var err error switch framework.TestContext.Provider { case "gce": - err = nodeUpgradeGCE(v, img, false) + err = nodeUpgradeGCE(v, img, extraEnvs) case "gke": err = nodeUpgradeGKE(f.Namespace.Name, v, img) default: @@ -697,16 +697,16 @@ func nodeUpgrade(f *framework.Framework, v string, img string) error { // TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default. func nodeUpgradeGCEWithKubeProxyDaemonSet(f *framework.Framework, v string, img string, enableKubeProxyDaemonSet bool) error { // Perform the upgrade. - if err := nodeUpgradeGCE(v, img, enableKubeProxyDaemonSet); err != nil { + if err := nodeUpgradeGCE(v, img, []string{fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet)}); err != nil { return err } return waitForNodesReadyAfterUpgrade(f) } // TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default. -func nodeUpgradeGCE(rawV, img string, enableKubeProxyDaemonSet bool) error { +func nodeUpgradeGCE(rawV, img string, extraEnvs []string) error { v := "v" + rawV - env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet)) + env := append(os.Environ(), extraEnvs...) if img != "" { env = append(env, "KUBE_NODE_OS_DISTRIBUTION="+img) _, _, err := framework.RunCmdEnv(env, framework.GCEUpgradeScript(), "-N", "-o", v) From bd3d1e7af02f23a593767f068261015b39f64eff Mon Sep 17 00:00:00 2001 From: wojtekt Date: Tue, 9 Mar 2021 10:06:12 +0100 Subject: [PATCH 4/4] Refactor upgrade tests to share more code --- test/e2e/cloud/gcp/cluster_upgrade.go | 187 ++++++++------------------ 1 file changed, 58 insertions(+), 129 deletions(-) diff --git a/test/e2e/cloud/gcp/cluster_upgrade.go b/test/e2e/cloud/gcp/cluster_upgrade.go index b289e1688e1..bda0ac5989d 100644 --- a/test/e2e/cloud/gcp/cluster_upgrade.go +++ b/test/e2e/cloud/gcp/cluster_upgrade.go @@ -109,10 +109,8 @@ func masterUpgrade(f *framework.Framework, v string, extraEnvs []string) error { } } -// masterUpgradeGCEWithKubeProxyDaemonSet upgrades master node on GCE with enabling/disabling the daemon set of kube-proxy. -// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default. -func masterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error { - return masterUpgradeGCE(v, []string{fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet)}) +func kubeProxyDaemonSetExtraEnvs(enableKubeProxyDaemonSet bool) []string { + return []string{fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet)} } func masterUpgradeGCE(rawV string, extraEnvs []string) error { @@ -134,6 +132,43 @@ func masterUpgradeGCE(rawV string, extraEnvs []string) error { return err } +func masterUpgradeFunc(f *framework.Framework, upgCtx *upgrades.UpgradeContext, testCase *junit.TestCase, masterExtraEnvs []string) func() { + return func() { + start := time.Now() + defer finalizeUpgradeTest(start, testCase) + target := upgCtx.Versions[1].Version.String() + framework.ExpectNoError(masterUpgrade(f, target, masterExtraEnvs)) + framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) + } +} + +func clusterUpgradeFunc(f *framework.Framework, upgCtx *upgrades.UpgradeContext, testCase *junit.TestCase, masterExtraEnvs, nodeExtraEnvs []string) func() { + return func() { + start := time.Now() + defer finalizeUpgradeTest(start, testCase) + target := upgCtx.Versions[1].Version.String() + image := upgCtx.Versions[1].NodeImage + framework.ExpectNoError(masterUpgrade(f, target, masterExtraEnvs)) + framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) + framework.ExpectNoError(nodeUpgrade(f, target, image, nodeExtraEnvs)) + framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) + } +} + +func clusterDowngradeFunc(f *framework.Framework, upgCtx *upgrades.UpgradeContext, testCase *junit.TestCase, masterExtraEnvs, nodeExtraEnvs []string) func() { + return func() { + start := time.Now() + defer finalizeUpgradeTest(start, testCase) + target := upgCtx.Versions[1].Version.String() + image := upgCtx.Versions[1].NodeImage + // Yes this really is a downgrade. And nodes must downgrade first. + framework.ExpectNoError(nodeUpgrade(f, target, image, nodeExtraEnvs)) + framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) + framework.ExpectNoError(masterUpgrade(f, target, masterExtraEnvs)) + framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) + } +} + // TODO(#98326): Split the test by SIGs, move to appropriate directories and use SIGDescribe. var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { f := framework.NewDefaultFramework("cluster-upgrade") @@ -150,42 +185,13 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { Name: "[sig-cloud-provider-gcp] master-upgrade", Classname: "upgrade_tests", } - testSuite.TestCases = append(testSuite.TestCases, masterUpgradeTest) + testSuite.TestCases = append(testSuite.TestCases, masterUpgradeTest, nil) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, masterUpgradeTest) - target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(masterUpgrade(f, target, nil)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - } + upgradeFunc := masterUpgradeFunc(f, upgCtx, masterUpgradeTest, nil) runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) - ginkgo.Describe("node upgrade", func() { - ginkgo.It("should maintain a functioning cluster [Feature:NodeUpgrade]", func() { - upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) - framework.ExpectNoError(err) - - testSuite := &junit.TestSuite{Name: "Node upgrade"} - nodeUpgradeTest := &junit.TestCase{ - Name: "node-upgrade", - Classname: "upgrade_tests", - } - - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, nodeUpgradeTest) - target := upgCtx.Versions[1].Version.String() - image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - } - runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.NodeUpgrade, upgradeFunc) - }) - }) - ginkgo.Describe("cluster upgrade", func() { ginkgo.It("should maintain a functioning cluster [Feature:ClusterUpgrade]", func() { upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), *upgradeTarget, *upgradeImage) @@ -194,16 +200,8 @@ var _ = ginkgo.Describe("Upgrade [Feature:Upgrade]", func() { testSuite := &junit.TestSuite{Name: "Cluster upgrade"} clusterUpgradeTest := &junit.TestCase{Name: "[sig-cloud-provider-gcp] cluster-upgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, clusterUpgradeTest) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, clusterUpgradeTest) - target := upgCtx.Versions[1].Version.String() - image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(masterUpgrade(f, target, nil)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - } + + upgradeFunc := clusterUpgradeFunc(f, upgCtx, clusterUpgradeTest, nil, nil) runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -222,17 +220,7 @@ var _ = ginkgo.Describe("Downgrade [Feature:Downgrade]", func() { clusterDowngradeTest := &junit.TestCase{Name: "[sig-cloud-provider-gcp] cluster-downgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, clusterDowngradeTest) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, clusterDowngradeTest) - // Yes this really is a downgrade. And nodes must downgrade first. - target := upgCtx.Versions[1].Version.String() - image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - framework.ExpectNoError(masterUpgrade(f, target, nil)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - } + upgradeFunc := clusterDowngradeFunc(f, upgCtx, clusterDowngradeTest, nil, nil) runUpgradeSuite(upgCtx, upgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -272,13 +260,8 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { testSuite := &junit.TestSuite{Name: "GPU master upgrade"} gpuUpgradeTest := &junit.TestCase{Name: "[sig-node] gpu-master-upgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, gpuUpgradeTest) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, gpuUpgradeTest) - target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(masterUpgrade(f, target, nil)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - } + + upgradeFunc := masterUpgradeFunc(f, upgCtx, gpuUpgradeTest, nil) runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) @@ -290,16 +273,8 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { testSuite := &junit.TestSuite{Name: "GPU cluster upgrade"} gpuUpgradeTest := &junit.TestCase{Name: "[sig-node] gpu-cluster-upgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, gpuUpgradeTest) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, gpuUpgradeTest) - target := upgCtx.Versions[1].Version.String() - image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(masterUpgrade(f, target, nil)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - } + + upgradeFunc := clusterUpgradeFunc(f, upgCtx, gpuUpgradeTest, nil, nil) runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -311,16 +286,8 @@ var _ = ginkgo.Describe("gpu Upgrade [Feature:GPUUpgrade]", func() { testSuite := &junit.TestSuite{Name: "GPU cluster downgrade"} gpuDowngradeTest := &junit.TestCase{Name: "[sig-node] gpu-cluster-downgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, gpuDowngradeTest) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, gpuDowngradeTest) - target := upgCtx.Versions[1].Version.String() - image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - framework.ExpectNoError(masterUpgrade(f, target, nil)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - } + + upgradeFunc := clusterDowngradeFunc(f, upgCtx, gpuDowngradeTest, nil, nil) runUpgradeSuite(upgCtx, gpuUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -338,16 +305,8 @@ var _ = ginkgo.Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", testSuite := &junit.TestSuite{Name: "Stateful upgrade"} statefulUpgradeTest := &junit.TestCase{Name: "[sig-apps] stateful-upgrade", Classname: "upgrade_tests"} testSuite.TestCases = append(testSuite.TestCases, statefulUpgradeTest) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, statefulUpgradeTest) - target := upgCtx.Versions[1].Version.String() - image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(masterUpgrade(f, target, nil)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgrade(f, target, image, nil)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - } + + upgradeFunc := clusterUpgradeFunc(f, upgCtx, statefulUpgradeTest, nil, nil) runUpgradeSuite(upgCtx, statefulsetUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -373,16 +332,8 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati } testSuite.TestCases = append(testSuite.TestCases, kubeProxyUpgradeTest) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, kubeProxyUpgradeTest) - target := upgCtx.Versions[1].Version.String() - image := upgCtx.Versions[1].NodeImage - framework.ExpectNoError(masterUpgradeGCEWithKubeProxyDaemonSet(target, true)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - framework.ExpectNoError(nodeUpgradeGCEWithKubeProxyDaemonSet(f, target, image, true)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - } + extraEnvs := kubeProxyDaemonSetExtraEnvs(true) + upgradeFunc := clusterUpgradeFunc(f, upgCtx, kubeProxyUpgradeTest, extraEnvs, extraEnvs) runUpgradeSuite(upgCtx, kubeProxyUpgradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -399,16 +350,8 @@ var _ = ginkgo.Describe("kube-proxy migration [Feature:KubeProxyDaemonSetMigrati } testSuite.TestCases = append(testSuite.TestCases, kubeProxyDowngradeTest) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, kubeProxyDowngradeTest) - // Yes this really is a downgrade. And nodes must downgrade first. - target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(nodeUpgradeGCEWithKubeProxyDaemonSet(f, target, *upgradeImage, false)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - framework.ExpectNoError(masterUpgradeGCEWithKubeProxyDaemonSet(target, false)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) - } + extraEnvs := kubeProxyDaemonSetExtraEnvs(false) + upgradeFunc := clusterDowngradeFunc(f, upgCtx, kubeProxyDowngradeTest, extraEnvs, extraEnvs) runUpgradeSuite(upgCtx, kubeProxyDowngradeTests, testSuite, upgrades.ClusterUpgrade, upgradeFunc) }) }) @@ -430,12 +373,8 @@ var _ = ginkgo.Describe("[sig-auth] ServiceAccount admission controller migratio } testSuite.TestCases = append(testSuite.TestCases, serviceaccountAdmissionControllerMigrationTest) - upgradeFunc := func() { - start := time.Now() - defer finalizeUpgradeTest(start, serviceaccountAdmissionControllerMigrationTest) - target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(masterUpgrade(f, target, []string{"KUBE_FEATURE_GATES=BoundServiceAccountTokenVolume=true"})) - } + extraEnvs := []string{"KUBE_FEATURE_GATES=BoundServiceAccountTokenVolume=true"} + upgradeFunc := masterUpgradeFunc(f, upgCtx, serviceaccountAdmissionControllerMigrationTest, extraEnvs) runUpgradeSuite(upgCtx, serviceaccountAdmissionControllerMigrationTests, testSuite, upgrades.MasterUpgrade, upgradeFunc) }) }) @@ -693,16 +632,6 @@ func nodeUpgrade(f *framework.Framework, v string, img string, extraEnvs []strin return waitForNodesReadyAfterUpgrade(f) } -// nodeUpgradeGCEWithKubeProxyDaemonSet upgrades nodes on GCE with enabling/disabling the daemon set of kube-proxy. -// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default. -func nodeUpgradeGCEWithKubeProxyDaemonSet(f *framework.Framework, v string, img string, enableKubeProxyDaemonSet bool) error { - // Perform the upgrade. - if err := nodeUpgradeGCE(v, img, []string{fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet)}); err != nil { - return err - } - return waitForNodesReadyAfterUpgrade(f) -} - // TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default. func nodeUpgradeGCE(rawV, img string, extraEnvs []string) error { v := "v" + rawV