diff --git a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-upgrades.yaml b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-upgrades.yaml index 044323833a7..439e67f436a 100644 --- a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-upgrades.yaml +++ b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-upgrades.yaml @@ -152,7 +152,7 @@ export E2E_OPT="--check_version_skew=false" export E2E_UP="false" export E2E_DOWN="false" - export GINKGO_TEST_ARGS="--ginkgo.focus=\[Feature:Upgrade\].*upgrade-master" + export GINKGO_TEST_ARGS="--ginkgo.focus=\[Feature:MasterUpgrade\]" - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-{step}': step: 'step4-e2e-old' runner: '{runner-old}' @@ -169,7 +169,7 @@ export E2E_OPT="--check_version_skew=false" export E2E_UP="false" export E2E_DOWN="false" - export GINKGO_TEST_ARGS="--ginkgo.focus=\[Feature:Upgrade\].*upgrade-cluster" + export GINKGO_TEST_ARGS="--ginkgo.focus=\[Feature:NodeUpgrade\]" - 'kubernetes-upgrade-{provider}-{version-old}-{version-new}-{step}': step: 'step6-e2e-old' runner: '{runner-old}' diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index ea943eed512..e22c43eaa56 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -36,11 +36,6 @@ import ( . "github.com/onsi/gomega" ) -const ( - // version applies to upgrades; kube-push always pushes local binaries. - versionURLFmt = "https://storage.googleapis.com/kubernetes-release/%s/%s.txt" -) - // realVersion turns a version constant s into a version string deployable on // GKE. See hack/get-build.sh for more information. func realVersion(s string) (string, error) { @@ -84,12 +79,6 @@ func masterUpgradeGKE(v string) error { return err } -var masterPush = func(_ string) error { - // TODO(mikedanese): Make master push use the provided version. - _, _, err := runCmd(path.Join(testContext.RepoRoot, "hack/e2e-internal/e2e-push.sh"), "-m") - return err -} - var nodeUpgrade = func(f *Framework, replicas int, v string) error { // Perform the upgrade. var err error @@ -106,6 +95,9 @@ var nodeUpgrade = func(f *Framework, replicas int, v string) error { } // Wait for it to complete and validate nodes and pods are healthy. + // + // TODO(ihmccreery) We shouldn't have to wait for nodes to be ready in + // GKE; the operation shouldn't return until they all are. Logf("Waiting up to %v for all nodes to be ready after the upgrade", restartNodeReadyAgainTimeout) if _, err := checkNodesReady(f.Client, restartNodeReadyAgainTimeout, testContext.CloudConfig.NumNodes); err != nil { return err @@ -115,8 +107,10 @@ var nodeUpgrade = func(f *Framework, replicas int, v string) error { } func nodeUpgradeGCE(rawV string) error { + // TODO(ihmccreery) This code path should be identical to how a user + // would trigger a node update; right now it's very different. v := "v" + rawV - Logf("Preparing node upgarde by creating new instance template for %q", v) + Logf("Preparing node upgrade by creating new instance template for %q", v) stdout, _, err := runCmd(path.Join(testContext.RepoRoot, "hack/e2e-internal/e2e-upgrade.sh"), "-P", v) if err != nil { return err @@ -191,27 +185,17 @@ var _ = Describe("Upgrade [Feature:Upgrade]", func() { // - volumes // - persistent volumes }) + AfterEach(func() { w.Cleanup() }) - Describe("kube-push", func() { - It("of master should maintain responsive services", func() { + Describe("master upgrade", func() { + It("should maintain responsive services [Feature:MasterUpgrade]", func() { By("Validating cluster before master upgrade") expectNoError(validate(f, svcName, rcName, ingress, replicas)) By("Performing a master upgrade") - testMasterUpgrade(ip, v, masterPush) - By("Validating cluster after master upgrade") - expectNoError(validate(f, svcName, rcName, ingress, replicas)) - }) - }) - - Describe("upgrade-master", func() { - It("should maintain responsive services", func() { - By("Validating cluster before master upgrade") - expectNoError(validate(f, svcName, rcName, ingress, replicas)) - By("Performing a master upgrade") - testMasterUpgrade(ip, v, masterUpgrade) + testUpgrade(ip, v, masterUpgrade) By("Checking master version") expectNoError(checkMasterVersion(f.Client, v)) By("Validating cluster after master upgrade") @@ -219,7 +203,7 @@ var _ = Describe("Upgrade [Feature:Upgrade]", func() { }) }) - Describe("upgrade-cluster", func() { + Describe("node upgrade", func() { var tmplBefore, tmplAfter string BeforeEach(func() { if providerIs("gce") { @@ -256,18 +240,36 @@ var _ = Describe("Upgrade [Feature:Upgrade]", func() { } }) - It("should maintain a functioning cluster", func() { + It("should maintain a functioning cluster [Feature:NodeUpgrade]", func() { By("Validating cluster before node upgrade") expectNoError(validate(f, svcName, rcName, ingress, replicas)) By("Performing a node upgrade") - testNodeUpgrade(f, nodeUpgrade, replicas, v) + // Circumnavigate testUpgrade, since services don't necessarily stay up. + Logf("Starting upgrade") + expectNoError(nodeUpgrade(f, replicas, v)) + Logf("Upgrade complete") + By("Checking node versions") + expectNoError(checkNodesVersions(f.Client, v)) + By("Validating cluster after node upgrade") + expectNoError(validate(f, svcName, rcName, ingress, replicas)) + }) + + It("should maintain responsive services [Feature:ExperimentalNodeUpgrade]", func() { + By("Validating cluster before node upgrade") + expectNoError(validate(f, svcName, rcName, ingress, replicas)) + By("Performing a node upgrade") + testUpgrade(ip, v, func(v string) error { + return nodeUpgrade(f, replicas, v) + }) + By("Checking node versions") + expectNoError(checkNodesVersions(f.Client, v)) By("Validating cluster after node upgrade") expectNoError(validate(f, svcName, rcName, ingress, replicas)) }) }) }) -func testMasterUpgrade(ip, v string, mUp func(v string) error) { +func testUpgrade(ip, v string, upF func(v string) error) { Logf("Starting async validation") httpClient := http.Client{Timeout: 2 * time.Second} done := make(chan struct{}, 1) @@ -294,18 +296,18 @@ func testMasterUpgrade(ip, v string, mUp func(v string) error) { // because this validation runs in another goroutine. Without this, // a failure is very confusing to track down because from the logs // everything looks fine. - msg := fmt.Sprintf("Failed to contact service during master upgrade: %v", err) + msg := fmt.Sprintf("Failed to contact service during upgrade: %v", err) Logf(msg) Failf(msg) } }, 200*time.Millisecond, done) - Logf("Starting master upgrade") - expectNoError(mUp(v)) + Logf("Starting upgrade") + expectNoError(upF(v)) done <- struct{}{} Logf("Stopping async validation") wg.Wait() - Logf("Master upgrade complete") + Logf("Upgrade complete") } func checkMasterVersion(c *client.Client, want string) error { @@ -325,15 +327,6 @@ func checkMasterVersion(c *client.Client, want string) error { return nil } -func testNodeUpgrade(f *Framework, nUp func(f *Framework, n int, v string) error, replicas int, v string) { - Logf("Starting node upgrade") - expectNoError(nUp(f, replicas, v)) - Logf("Node upgrade complete") - By("Checking node versions") - expectNoError(checkNodesVersions(f.Client, v)) - Logf("All nodes are at version %s", v) -} - func checkNodesVersions(c *client.Client, want string) error { l := ListSchedulableNodesOrDie(c) for _, n := range l.Items {