diff --git a/test/e2e/framework/nodes_util.go b/test/e2e/framework/nodes_util.go index 92d90309f4d..5eef8d3115d 100644 --- a/test/e2e/framework/nodes_util.go +++ b/test/e2e/framework/nodes_util.go @@ -42,7 +42,7 @@ func EtcdUpgrade(target_storage, target_version string) error { func MasterUpgrade(v string) error { switch TestContext.Provider { case "gce": - return masterUpgradeGCE(v) + return masterUpgradeGCE(v, false) case "gke": return masterUpgradeGKE(v) default: @@ -61,8 +61,14 @@ func etcdUpgradeGCE(target_storage, target_version string) error { return err } -func masterUpgradeGCE(rawV string) error { - env := os.Environ() +// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default. +func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error { + return masterUpgradeGCE(v, enableKubeProxyDaemonSet) +} + +// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default. +func masterUpgradeGCE(rawV string, enableKubeProxyDaemonSet bool) error { + env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet)) // TODO: Remove these variables when they're no longer needed for downgrades. if TestContext.EtcdUpgradeVersion != "" && TestContext.EtcdUpgradeStorage != "" { env = append(env, @@ -101,7 +107,7 @@ func NodeUpgrade(f *Framework, v string, img string) error { var err error switch TestContext.Provider { case "gce": - err = nodeUpgradeGCE(v, img) + err = nodeUpgradeGCE(v, img, false) case "gke": err = nodeUpgradeGKE(v, img) default: @@ -122,14 +128,30 @@ func NodeUpgrade(f *Framework, v string, img string) error { return nil } -func nodeUpgradeGCE(rawV, img string) error { +// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default. +func NodeUpgradeGCEWithKubeProxyDaemonSet(f *Framework, v string, enableKubeProxyDaemonSet bool) error { + // Perform the upgrade. + if err := nodeUpgradeGCE(v, "", enableKubeProxyDaemonSet); err != nil { + return err + } + // Wait for it to complete and validate nodes are healthy. + Logf("Waiting up to %v for all nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout) + if _, err := CheckNodesReady(f.ClientSet, RestartNodeReadyAgainTimeout, TestContext.CloudConfig.NumNodes); err != nil { + return err + } + return nil +} + +// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default. +func nodeUpgradeGCE(rawV, img string, enableKubeProxyDaemonSet bool) error { v := "v" + rawV + env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet)) if img != "" { - env := append(os.Environ(), "KUBE_NODE_OS_DISTRIBUTION="+img) + env = append(env, "KUBE_NODE_OS_DISTRIBUTION="+img) _, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v) return err } - _, _, err := RunCmd(gceUpgradeScript(), "-N", v) + _, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", v) return err } diff --git a/test/e2e/lifecycle/cluster_upgrade.go b/test/e2e/lifecycle/cluster_upgrade.go index d3efc6629b3..3d2787123e6 100644 --- a/test/e2e/lifecycle/cluster_upgrade.go +++ b/test/e2e/lifecycle/cluster_upgrade.go @@ -216,6 +216,69 @@ var _ = Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", func() }) }) +var _ = SIGDescribe("kube-proxy migration [Feature:KubeProxyDaemonSetMigration]", func() { + f := framework.NewDefaultFramework("kube-proxy-ds-migration") + + // Create the frameworks here because we can only create them + // in a "Describe". + testFrameworks := createUpgradeFrameworks(upgradeTests) + + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce") + }) + + Describe("Upgrade kube-proxy from static pods to a DaemonSet", func() { + It("should maintain a functioning cluster [Feature:KubeProxyDaemonSetUpgrade]", func() { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + + testSuite := &junit.TestSuite{Name: "kube-proxy upgrade"} + kubeProxyUpgradeTest := &junit.TestCase{ + Name: "kube-proxy-ds-upgrade", + Classname: "upgrade_tests", + } + testSuite.TestCases = append(testSuite.TestCases, kubeProxyUpgradeTest) + + upgradeFunc := func() { + start := time.Now() + defer finalizeUpgradeTest(start, kubeProxyUpgradeTest) + target := upgCtx.Versions[1].Version.String() + framework.ExpectNoError(framework.MasterUpgradeGCEWithKubeProxyDaemonSet(target, true)) + framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target)) + framework.ExpectNoError(framework.NodeUpgradeGCEWithKubeProxyDaemonSet(f, target, true)) + framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target)) + } + runUpgradeSuite(f, upgradeTests, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc) + }) + }) + + Describe("Downgrade kube-proxy from a DaemonSet to static pods", func() { + It("should maintain a functioning cluster [Feature:KubeProxyDaemonSetDowngrade]", func() { + upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + + testSuite := &junit.TestSuite{Name: "kube-proxy downgrade"} + kubeProxyDowngradeTest := &junit.TestCase{ + Name: "kube-proxy-ds-downgrade", + Classname: "upgrade_tests", + } + testSuite.TestCases = append(testSuite.TestCases, kubeProxyDowngradeTest) + + upgradeFunc := func() { + start := time.Now() + defer finalizeUpgradeTest(start, kubeProxyDowngradeTest) + // Yes this really is a downgrade. And nodes must downgrade first. + target := upgCtx.Versions[1].Version.String() + framework.ExpectNoError(framework.NodeUpgradeGCEWithKubeProxyDaemonSet(f, target, false)) + framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target)) + framework.ExpectNoError(framework.MasterUpgradeGCEWithKubeProxyDaemonSet(target, false)) + framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target)) + } + runUpgradeSuite(f, upgradeTests, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc) + }) + }) +}) + type chaosMonkeyAdapter struct { test upgrades.Test testReport *junit.TestCase