From 80c19e4cef0864851e26113eb8fb13a428f90cc0 Mon Sep 17 00:00:00 2001 From: Kenichi Omichi Date: Tue, 14 Jul 2020 22:19:55 +0000 Subject: [PATCH] Move MasterUpgrade() out from e2e framework MasterUpgrade() is called only from cloud/gcp/cluster_upgrade.go. And the function depends on GCP, so it is nice to move this function from e2e framework. --- test/e2e/cloud/gcp/cluster_upgrade.go | 58 ++++++++++++++++++++++----- test/e2e/framework/nodes_util.go | 38 ------------------ 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/test/e2e/cloud/gcp/cluster_upgrade.go b/test/e2e/cloud/gcp/cluster_upgrade.go index 1abe01cd105..8e912e30032 100644 --- a/test/e2e/cloud/gcp/cluster_upgrade.go +++ b/test/e2e/cloud/gcp/cluster_upgrade.go @@ -52,6 +52,8 @@ var ( upgradeImage = e2econfig.Flags.String("upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.") ) +const etcdImage = "3.4.9-1" + var upgradeTests = []upgrades.Test{ &upgrades.ServiceUpgradeTest{}, &upgrades.SecretUpgradeTest{}, @@ -87,6 +89,44 @@ var kubeProxyDowngradeTests = []upgrades.Test{ &upgrades.ServiceUpgradeTest{}, } +// masterUpgrade upgrades master node on GCE/GKE. +func masterUpgrade(f *framework.Framework, v string) error { + switch framework.TestContext.Provider { + case "gce": + return masterUpgradeGCE(v, false) + case "gke": + return framework.MasterUpgradeGKE(f.Namespace.Name, v) + default: + return fmt.Errorf("masterUpgrade() is not implemented for provider %s", framework.TestContext.Provider) + } +} + +// masterUpgradeGCEWithKubeProxyDaemonSet upgrades master node on GCE with enabling/disabling the daemon set of kube-proxy. +// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default. +func masterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error { + return masterUpgradeGCE(v, enableKubeProxyDaemonSet) +} + +// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default. +func masterUpgradeGCE(rawV string, enableKubeProxyDaemonSet bool) error { + env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet)) + // TODO: Remove these variables when they're no longer needed for downgrades. + if framework.TestContext.EtcdUpgradeVersion != "" && framework.TestContext.EtcdUpgradeStorage != "" { + env = append(env, + "TEST_ETCD_VERSION="+framework.TestContext.EtcdUpgradeVersion, + "STORAGE_BACKEND="+framework.TestContext.EtcdUpgradeStorage, + "TEST_ETCD_IMAGE="+etcdImage) + } else { + // In e2e tests, we skip the confirmation prompt about + // implicit etcd upgrades to simulate the user entering "y". + env = append(env, "TEST_ALLOW_IMPLICIT_ETCD_UPGRADE=true") + } + + v := "v" + rawV + _, _, err := framework.RunCmdEnv(env, framework.GCEUpgradeScript(), "-M", v) + return err +} + var _ = SIGDescribe("Upgrade [Feature:Upgrade]", func() { f := framework.NewDefaultFramework("cluster-upgrade") @@ -109,7 +149,7 @@ var _ = SIGDescribe("Upgrade [Feature:Upgrade]", func() { start := time.Now() defer finalizeUpgradeTest(start, masterUpgradeTest) target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(framework.MasterUpgrade(f, target)) + framework.ExpectNoError(masterUpgrade(f, target)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } runUpgradeSuite(f, upgradeTests, testFrameworks, testSuite, upgrades.MasterUpgrade, upgradeFunc) @@ -150,7 +190,7 @@ var _ = SIGDescribe("Upgrade [Feature:Upgrade]", func() { start := time.Now() defer finalizeUpgradeTest(start, clusterUpgradeTest) target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(framework.MasterUpgrade(f, target)) + framework.ExpectNoError(masterUpgrade(f, target)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) @@ -183,7 +223,7 @@ var _ = SIGDescribe("Downgrade [Feature:Downgrade]", func() { target := upgCtx.Versions[1].Version.String() framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - framework.ExpectNoError(framework.MasterUpgrade(f, target)) + framework.ExpectNoError(masterUpgrade(f, target)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } runUpgradeSuite(f, upgradeTests, testFrameworks, testSuite, upgrades.ClusterUpgrade, upgradeFunc) @@ -231,7 +271,7 @@ var _ = SIGDescribe("gpu Upgrade [Feature:GPUUpgrade]", func() { start := time.Now() defer finalizeUpgradeTest(start, gpuUpgradeTest) target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(framework.MasterUpgrade(f, target)) + framework.ExpectNoError(masterUpgrade(f, target)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } runUpgradeSuite(f, gpuUpgradeTests, testFrameworks, testSuite, upgrades.MasterUpgrade, upgradeFunc) @@ -249,7 +289,7 @@ var _ = SIGDescribe("gpu Upgrade [Feature:GPUUpgrade]", func() { start := time.Now() defer finalizeUpgradeTest(start, gpuUpgradeTest) target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(framework.MasterUpgrade(f, target)) + framework.ExpectNoError(masterUpgrade(f, target)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) @@ -271,7 +311,7 @@ var _ = SIGDescribe("gpu Upgrade [Feature:GPUUpgrade]", func() { target := upgCtx.Versions[1].Version.String() framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - framework.ExpectNoError(framework.MasterUpgrade(f, target)) + framework.ExpectNoError(masterUpgrade(f, target)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } runUpgradeSuite(f, gpuUpgradeTests, testFrameworks, testSuite, upgrades.ClusterUpgrade, upgradeFunc) @@ -297,7 +337,7 @@ var _ = ginkgo.Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", start := time.Now() defer finalizeUpgradeTest(start, statefulUpgradeTest) target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(framework.MasterUpgrade(f, target)) + framework.ExpectNoError(masterUpgrade(f, target)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) framework.ExpectNoError(nodeUpgrade(f, target, *upgradeImage)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) @@ -332,7 +372,7 @@ var _ = SIGDescribe("kube-proxy migration [Feature:KubeProxyDaemonSetMigration]" start := time.Now() defer finalizeUpgradeTest(start, kubeProxyUpgradeTest) target := upgCtx.Versions[1].Version.String() - framework.ExpectNoError(framework.MasterUpgradeGCEWithKubeProxyDaemonSet(target, true)) + framework.ExpectNoError(masterUpgradeGCEWithKubeProxyDaemonSet(target, true)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) framework.ExpectNoError(nodeUpgradeGCEWithKubeProxyDaemonSet(f, target, *upgradeImage, true)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) @@ -362,7 +402,7 @@ var _ = SIGDescribe("kube-proxy migration [Feature:KubeProxyDaemonSetMigration]" target := upgCtx.Versions[1].Version.String() framework.ExpectNoError(nodeUpgradeGCEWithKubeProxyDaemonSet(f, target, *upgradeImage, false)) framework.ExpectNoError(checkNodesVersions(f.ClientSet, target)) - framework.ExpectNoError(framework.MasterUpgradeGCEWithKubeProxyDaemonSet(target, false)) + framework.ExpectNoError(masterUpgradeGCEWithKubeProxyDaemonSet(target, false)) framework.ExpectNoError(checkMasterVersion(f.ClientSet, target)) } runUpgradeSuite(f, kubeProxyDowngradeTests, testFrameworks, testSuite, upgrades.ClusterUpgrade, upgradeFunc) diff --git a/test/e2e/framework/nodes_util.go b/test/e2e/framework/nodes_util.go index 2dc622cd100..3180b76e578 100644 --- a/test/e2e/framework/nodes_util.go +++ b/test/e2e/framework/nodes_util.go @@ -46,18 +46,6 @@ func EtcdUpgrade(targetStorage, targetVersion string) error { } } -// MasterUpgrade upgrades master node on GCE/GKE. -func MasterUpgrade(f *Framework, v string) error { - switch TestContext.Provider { - case "gce": - return masterUpgradeGCE(v, false) - case "gke": - return MasterUpgradeGKE(f.Namespace.Name, v) - default: - return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", TestContext.Provider) - } -} - func etcdUpgradeGCE(targetStorage, targetVersion string) error { env := append( os.Environ(), @@ -69,32 +57,6 @@ func etcdUpgradeGCE(targetStorage, targetVersion string) error { return err } -// MasterUpgradeGCEWithKubeProxyDaemonSet upgrades master node on GCE with enabling/disabling the daemon set of kube-proxy. -// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default. -func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error { - return masterUpgradeGCE(v, enableKubeProxyDaemonSet) -} - -// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default. -func masterUpgradeGCE(rawV string, enableKubeProxyDaemonSet bool) error { - env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet)) - // TODO: Remove these variables when they're no longer needed for downgrades. - if TestContext.EtcdUpgradeVersion != "" && TestContext.EtcdUpgradeStorage != "" { - env = append(env, - "TEST_ETCD_VERSION="+TestContext.EtcdUpgradeVersion, - "STORAGE_BACKEND="+TestContext.EtcdUpgradeStorage, - "TEST_ETCD_IMAGE="+etcdImage) - } else { - // In e2e tests, we skip the confirmation prompt about - // implicit etcd upgrades to simulate the user entering "y". - env = append(env, "TEST_ALLOW_IMPLICIT_ETCD_UPGRADE=true") - } - - v := "v" + rawV - _, _, err := RunCmdEnv(env, GCEUpgradeScript(), "-M", v) - return err -} - // LocationParamGKE returns parameter related to location for gcloud command. func LocationParamGKE() string { if TestContext.CloudConfig.MultiMaster {