diff --git a/hack/e2e-internal/e2e-push.sh b/hack/e2e-internal/e2e-push.sh index 02ded0aac50..ba1d4c486d8 100755 --- a/hack/e2e-internal/e2e-push.sh +++ b/hack/e2e-internal/e2e-push.sh @@ -31,4 +31,4 @@ source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" prepare-e2e -"${KUBE_VERSION_ROOT}/cluster/kube-push.sh" +"${KUBE_VERSION_ROOT}/cluster/kube-push.sh" $@ diff --git a/hack/e2e-internal/e2e-upgrade.sh b/hack/e2e-internal/e2e-upgrade.sh new file mode 100755 index 00000000000..aacd9cbc5e2 --- /dev/null +++ b/hack/e2e-internal/e2e-upgrade.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. + +: ${KUBE_VERSION_ROOT:=${KUBE_ROOT}} +: ${KUBECTL:=${KUBE_VERSION_ROOT}/cluster/kubectl.sh} +: ${KUBE_CONFIG_FILE:="config-test.sh"} + +export KUBECTL KUBE_CONFIG_FILE + +source "${KUBE_ROOT}/cluster/kube-env.sh" +source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" + +prepare-e2e + +"${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/upgrade.sh" $@ diff --git a/hack/upgrade-e2e-test.sh b/hack/upgrade-e2e-test.sh index a6d4213cd9f..197b85e546a 100755 --- a/hack/upgrade-e2e-test.sh +++ b/hack/upgrade-e2e-test.sh @@ -14,15 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Provide reasonable default for running the end-to-end tests against a recent -# stable release and then again after upgrading it to a version built from head. - +# This runs e2e, does a checked upgrade of the MASTER using the locally built release, then runs e2e again. go run "$(dirname $0)/e2e.go" -v -build -up -version="v0.14.0" -test -check_version_skew=false if [ $? -eq 0 ]; then - echo "Tests on initial version succeeded. Proceeding with push and second set of tests." - go run "$(dirname $0)/e2e.go" -v -push -version="" -test -check_version_skew=false + echo "Tests on initial version succeeded. Running the checked master upgrade." + go run "$(dirname $0)/e2e.go" -v --test --test_args='--ginkgo.focus=Skipped.*Cluster\supgrade.*gce-upgrade' -check_version_skew=false else echo "Tests on initial version failed. Skipping tests on second version." fi +echo "Master upgrade complete. Running e2e on the upgraded cluster." +go run "$(dirname $0)/e2e.go" -v -build -up -version="" -test -check_version_skew=false + exit $? diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go new file mode 100644 index 00000000000..5a097220366 --- /dev/null +++ b/test/e2e/cluster_upgrade.go @@ -0,0 +1,161 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path" + "sync" + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/fields" + "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Skipped", func() { + Describe("Cluster upgrade", func() { + svcName := "baz" + var podName string + framework := Framework{BaseName: "cluster-upgrade"} + var webserver *WebserverTest + + BeforeEach(func() { + framework.beforeEach() + webserver = NewWebserverTest(framework.Client, framework.Namespace.Name, svcName) + pod := webserver.CreateWebserverPod() + podName = pod.Name + svc := webserver.BuildServiceSpec() + svc.Spec.Type = api.ServiceTypeLoadBalancer + webserver.CreateService(svc) + }) + + AfterEach(func() { + framework.afterEach() + webserver.Cleanup() + }) + + Describe("kube-push", func() { + It("of master should maintain responsive services", func() { + testClusterUpgrade(framework, svcName, podName, func() { + runUpgradeScript("hack/e2e-internal/e2e-push.sh", "-m") + }) + }) + }) + + Describe("gce-upgrade", func() { + It("of master should maintain responsive services", func() { + if !providerIs("gce") { + By(fmt.Sprintf("Skippingt test, which is not implemented for %s", testContext.Provider)) + return + } + testClusterUpgrade(framework, svcName, podName, func() { + runUpgradeScript("hack/e2e-internal/e2e-upgrade.sh", "-M", "-l") + }) + }) + }) + }) +}) + +func testClusterUpgrade(framework Framework, svcName, podName string, upgrade func()) { + result, err := waitForLoadBalancerIngress(framework.Client, svcName, framework.Namespace.Name) + Expect(err).NotTo(HaveOccurred()) + ingresses := result.Status.LoadBalancer.Ingress + if len(ingresses) != 1 { + Failf("Was expecting only 1 ingress IP but got %d (%v): %v", len(ingresses), ingresses, result) + } + ingress := ingresses[0] + ip := ingress.IP + if ip == "" { + ip = ingress.Hostname + } + + By("Waiting for pod to become reachable") + testLoadBalancerReachable(ingress, 80) + validateClusterUpgrade(framework, svcName, podName) + + Logf("starting async validation") + httpClient := http.Client{Timeout: 2 * time.Second} + done := make(chan struct{}, 1) + // Let's make sure we've finished the heartbeat before shutting things down. + var wg sync.WaitGroup + go util.Until(func() { + defer GinkgoRecover() + wg.Add(1) + defer wg.Done() + + expectNoError(wait.Poll(poll, singleCallTimeout, func() (bool, error) { + r, err := httpClient.Get("http://" + ip) + if err != nil { + return false, nil + } + if r.StatusCode < http.StatusOK || r.StatusCode >= http.StatusNotFound { + return false, nil + } + return true, nil + })) + }, 200*time.Millisecond, done) + + By("Starting upgrade") + upgrade() + done <- struct{}{} + Logf("Stopping async validation") + wg.Wait() + Logf("Upgrade complete.") + + By("Validating post upgrade state") + validateClusterUpgrade(framework, svcName, podName) +} + +func runUpgradeScript(scriptPath string, args ...string) { + cmd := exec.Command(path.Join(testContext.RepoRoot, scriptPath), args...) + upgradeLogPath := path.Join(testContext.OutputDir, "upgrade-"+string(util.NewUUID())+".log") + Logf("Writing upgrade logs to %s", upgradeLogPath) + upgradeLog, err := os.Create(upgradeLogPath) + expectNoError(err) + + cmd.Stdout = io.MultiWriter(os.Stdout, upgradeLog) + cmd.Stderr = io.MultiWriter(os.Stderr, upgradeLog) + if err := cmd.Run(); err != nil { + Failf("Upgrade failed: %v", err) + } +} + +func validateClusterUpgrade(framework Framework, svcName, podName string) { + pods, err := framework.Client.Pods(framework.Namespace.Name).List(labels.Everything(), fields.Everything()) + Expect(err).NotTo(HaveOccurred()) + Expect(len(pods.Items) == 1).Should(BeTrue()) + if podName != pods.Items[0].Name { + Failf("pod name should not have changed") + } + _, err = podRunningReady(&pods.Items[0]) + Expect(err).NotTo(HaveOccurred()) + svc, err := framework.Client.Services(framework.Namespace.Name).Get(svcName) + Expect(err).NotTo(HaveOccurred()) + if svcName != svc.Name { + Failf("service name should not have changed") + } +} diff --git a/test/e2e/service.go b/test/e2e/service.go index e0b697b8ac0..08effe86647 100644 --- a/test/e2e/service.go +++ b/test/e2e/service.go @@ -1142,7 +1142,7 @@ func (t *WebserverTest) BuildServiceSpec() *api.Service { } // Create a pod with the well-known webserver configuration, and record it for cleanup -func (t *WebserverTest) CreateWebserverPod() { +func (t *WebserverTest) CreateWebserverPod() *api.Pod { name := t.ServiceName + "-" + strconv.Itoa(t.SequenceNext()) pod := &api.Pod{ TypeMeta: api.TypeMeta{ @@ -1162,11 +1162,12 @@ func (t *WebserverTest) CreateWebserverPod() { }, }, } - _, err := t.CreatePod(pod) + pod, err := t.CreatePod(pod) if err != nil { Failf("Failed to create pod %s: %v", pod.Name, err) } expectNoError(waitForPodRunningInNamespace(t.Client, pod.Name, t.Namespace)) + return pod } // Create a pod, and record it for cleanup