diff --git a/cluster/gce/gci/mounter/mounter.go b/cluster/gce/gci/mounter/mounter.go index ef37ce0601a..4224efab03b 100644 --- a/cluster/gce/gci/mounter/mounter.go +++ b/cluster/gce/gci/mounter/mounter.go @@ -63,7 +63,7 @@ func main() { // MountInChroot is to run mount within chroot with the passing root directory func mountInChroot(rootfsPath string, args []string) error { if _, err := os.Stat(rootfsPath); os.IsNotExist(err) { - return fmt.Errorf("Path <%s> does not exist.\n", rootfsPath) + return fmt.Errorf("path <%s> does not exist", rootfsPath) } args = append([]string{rootfsPath, mountCmd}, args...) output, err := exec.Command(chrootCmd, args...).CombinedOutput() @@ -73,7 +73,7 @@ func mountInChroot(rootfsPath string, args []string) error { if !strings.EqualFold(string(output), nfsRPCBindErrMsg) { // Mount failed but not because of RPC bind error - return fmt.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s\n", err, chrootCmd, args, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s", err, chrootCmd, args, string(output)) } // Mount failed because it is NFS V3 and we need to run rpcBind diff --git a/federation/cluster/upgrade.sh b/federation/cluster/upgrade.sh new file mode 100755 index 00000000000..81755aed808 --- /dev/null +++ b/federation/cluster/upgrade.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +# For `kube::log::status` function since it already sources +# "${KUBE_ROOT}/cluster/lib/logging.sh" +source "${KUBE_ROOT}/cluster/common.sh" +# For $FEDERATION_NAME, $FEDERATION_NAMESPACE, $HOST_CLUSTER_CONTEXT, +source "${KUBE_ROOT}/federation/cluster/common.sh" + +KUBE_VERSION="${1}" + +host_kubectl="${KUBE_ROOT}/cluster/kubectl.sh --context=${HOST_CLUSTER_CONTEXT} --namespace=${FEDERATION_NAMESPACE}" + +function upgrade() { + local -r project="${KUBE_PROJECT:-${PROJECT:-}}" + local -r kube_registry="${KUBE_REGISTRY:-gcr.io/${project}}" + local -r image_version="${kube_registry}/hyperkube-amd64:${KUBE_VERSION}" + + kube::log::status "Upgrading federation control plane ${FEDERATION_NAME} with image ${image_version}" + + # Upgrade apiserver image + ${host_kubectl} set image deployment/federation-apiserver apiserver=${image_version} + + # Upgrade controller-manager image + ${host_kubectl} set image deployment/federation-controller-manager controller-manager=${image_version} +} + +upgrade diff --git a/hack/.linted_packages b/hack/.linted_packages index 3fc4d3b4ed1..66d3afd446a 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -1,4 +1,5 @@ cluster/addons/fluentd-elasticsearch/es-image +cluster/gce/gci/mounter cluster/images/etcd/attachlease cluster/images/etcd/rollback cmd/clicheck @@ -385,6 +386,7 @@ staging/src/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1alpha1 staging/src/k8s.io/sample-apiserver staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install test/e2e/perftype +test/e2e_federation/upgrades test/e2e_node/runner/local test/images/clusterapi-tester test/images/entrypoint-tester diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index ec3361bafa4..8ea118c0f6e 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -262,6 +262,7 @@ federated-api-qps federated-kube-context federation-name federation-system-namespace +federation-upgrade-target file-check-frequency file_content_in_loop file-suffix diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index d494cb6aa69..1adbabce7b5 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -17,11 +17,6 @@ limitations under the License. package e2e import ( - "fmt" - "path" - "strings" - - "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/test/e2e/chaosmonkey" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/upgrades" @@ -55,10 +50,10 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { framework.KubeDescribe("master upgrade", func() { It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() { cm := chaosmonkey.New(func() { - v, err := realVersion(framework.TestContext.UpgradeTarget) + v, err := framework.RealVersion(framework.TestContext.UpgradeTarget) framework.ExpectNoError(err) framework.ExpectNoError(framework.MasterUpgrade(v)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, v)) + framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v)) }) for _, t := range upgradeTests { cm.RegisterInterface(&chaosMonkeyAdapter{ @@ -75,10 +70,10 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { framework.KubeDescribe("node upgrade", func() { It("should maintain a functioning cluster [Feature:NodeUpgrade]", func() { cm := chaosmonkey.New(func() { - v, err := realVersion(framework.TestContext.UpgradeTarget) + v, err := framework.RealVersion(framework.TestContext.UpgradeTarget) framework.ExpectNoError(err) framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, v)) + framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v)) }) for _, t := range upgradeTests { cm.RegisterInterface(&chaosMonkeyAdapter{ @@ -94,12 +89,12 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { framework.KubeDescribe("cluster upgrade", func() { It("should maintain a functioning cluster [Feature:ClusterUpgrade]", func() { cm := chaosmonkey.New(func() { - v, err := realVersion(framework.TestContext.UpgradeTarget) + v, err := framework.RealVersion(framework.TestContext.UpgradeTarget) framework.ExpectNoError(err) framework.ExpectNoError(framework.MasterUpgrade(v)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, v)) + framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v)) framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, v)) + framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v)) }) for _, t := range upgradeTests { cm.RegisterInterface(&chaosMonkeyAdapter{ @@ -127,12 +122,12 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() { It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() { cm := chaosmonkey.New(func() { // Yes this really is a downgrade. And nodes must downgrade first. - v, err := realVersion(framework.TestContext.UpgradeTarget) + v, err := framework.RealVersion(framework.TestContext.UpgradeTarget) framework.ExpectNoError(err) framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage)) - framework.ExpectNoError(checkNodesVersions(f.ClientSet, v)) + framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v)) framework.ExpectNoError(framework.MasterUpgrade(v)) - framework.ExpectNoError(checkMasterVersion(f.ClientSet, v)) + framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v)) }) for _, t := range upgradeTests { cm.RegisterInterface(&chaosMonkeyAdapter{ @@ -190,53 +185,3 @@ func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) { func (cma *chaosMonkeyAdapter) Teardown() { cma.test.Teardown(cma.framework) } - -// realVersion turns a version constant s into a version string deployable on -// GKE. See hack/get-build.sh for more information. -func realVersion(s string) (string, error) { - framework.Logf(fmt.Sprintf("Getting real version for %q", s)) - v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/get-build.sh"), "-v", s) - if err != nil { - return v, err - } - framework.Logf("Version for %q is %q", s, v) - return strings.TrimPrefix(strings.TrimSpace(v), "v"), nil -} - -func checkMasterVersion(c clientset.Interface, want string) error { - framework.Logf("Checking master version") - v, err := c.Discovery().ServerVersion() - if err != nil { - return fmt.Errorf("checkMasterVersion() couldn't get the master version: %v", err) - } - // We do prefix trimming and then matching because: - // want looks like: 0.19.3-815-g50e67d4 - // got looks like: v0.19.3-815-g50e67d4034e858-dirty - got := strings.TrimPrefix(v.GitVersion, "v") - if !strings.HasPrefix(got, want) { - return fmt.Errorf("master had kube-apiserver version %s which does not start with %s", - got, want) - } - framework.Logf("Master is at version %s", want) - return nil -} - -func checkNodesVersions(cs clientset.Interface, want string) error { - l := framework.GetReadySchedulableNodesOrDie(cs) - for _, n := range l.Items { - // We do prefix trimming and then matching because: - // want looks like: 0.19.3-815-g50e67d4 - // kv/kvp look like: v0.19.3-815-g50e67d4034e858-dirty - kv, kpv := strings.TrimPrefix(n.Status.NodeInfo.KubeletVersion, "v"), - strings.TrimPrefix(n.Status.NodeInfo.KubeProxyVersion, "v") - if !strings.HasPrefix(kv, want) { - return fmt.Errorf("node %s had kubelet version %s which does not start with %s", - n.ObjectMeta.Name, kv, want) - } - if !strings.HasPrefix(kpv, want) { - return fmt.Errorf("node %s had kube-proxy version %s which does not start with %s", - n.ObjectMeta.Name, kpv, want) - } - } - return nil -} diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index a3fa78a6ad7..51d8cb4943b 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -32,6 +32,7 @@ go_library( "service_util.go", "statefulset_utils.go", "test_context.go", + "upgrade_util.go", "util.go", ], tags = ["automanaged"], diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 8597965b012..acb3ff7d875 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -88,6 +88,8 @@ type TestContextType struct { NodeTestContextType // Federation e2e context FederatedKubeContext string + // Federation control plane version to upgrade to while doing upgrade tests + FederationUpgradeTarget string // Viper-only parameters. These will in time replace all flags. @@ -202,6 +204,7 @@ func RegisterClusterFlags() { flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.") flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.") flag.StringVar(&TestContext.GCEUpgradeScript, "gce-upgrade-script", "", "Script to use to upgrade a GCE cluster.") + flag.StringVar(&TestContext.FederationUpgradeTarget, "federation-upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an federation upgrade test.") flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.") flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.") flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", true, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.") diff --git a/test/e2e/framework/upgrade_util.go b/test/e2e/framework/upgrade_util.go new file mode 100644 index 00000000000..8e5f4609e05 --- /dev/null +++ b/test/e2e/framework/upgrade_util.go @@ -0,0 +1,75 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "path" + "strings" + + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" +) + +// RealVersion turns a version constants into a version string deployable on +// GKE. See hack/get-build.sh for more information. +func RealVersion(s string) (string, error) { + Logf("Getting real version for %q", s) + v, _, err := RunCmd(path.Join(TestContext.RepoRoot, "hack/get-build.sh"), "-v", s) + if err != nil { + return v, err + } + Logf("Version for %q is %q", s, v) + return strings.TrimPrefix(strings.TrimSpace(v), "v"), nil +} + +func CheckMasterVersion(c clientset.Interface, want string) error { + Logf("Checking master version") + v, err := c.Discovery().ServerVersion() + if err != nil { + return fmt.Errorf("CheckMasterVersion() couldn't get the master version: %v", err) + } + // We do prefix trimming and then matching because: + // want looks like: 0.19.3-815-g50e67d4 + // got looks like: v0.19.3-815-g50e67d4034e858-dirty + got := strings.TrimPrefix(v.GitVersion, "v") + if !strings.HasPrefix(got, want) { + return fmt.Errorf("master had kube-apiserver version %s which does not start with %s", + got, want) + } + Logf("Master is at version %s", want) + return nil +} + +func CheckNodesVersions(cs clientset.Interface, want string) error { + l := GetReadySchedulableNodesOrDie(cs) + for _, n := range l.Items { + // We do prefix trimming and then matching because: + // want looks like: 0.19.3-815-g50e67d4 + // kv/kvp look like: v0.19.3-815-g50e67d4034e858-dirty + kv, kpv := strings.TrimPrefix(n.Status.NodeInfo.KubeletVersion, "v"), + strings.TrimPrefix(n.Status.NodeInfo.KubeProxyVersion, "v") + if !strings.HasPrefix(kv, want) { + return fmt.Errorf("node %s had kubelet version %s which does not start with %s", + n.ObjectMeta.Name, kv, want) + } + if !strings.HasPrefix(kpv, want) { + return fmt.Errorf("node %s had kube-proxy version %s which does not start with %s", + n.ObjectMeta.Name, kpv, want) + } + } + return nil +} diff --git a/test/e2e_federation/BUILD b/test/e2e_federation/BUILD index f403d60b718..2fecaefc4a8 100644 --- a/test/e2e_federation/BUILD +++ b/test/e2e_federation/BUILD @@ -20,6 +20,7 @@ go_library( "replicaset.go", "secret.go", "service.go", + "upgrade.go", "util.go", ], tags = ["automanaged"], @@ -34,9 +35,11 @@ go_library( "//pkg/api/v1:go_default_library", "//pkg/apis/extensions/v1beta1:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", + "//test/e2e/chaosmonkey:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/e2e_federation/framework:go_default_library", + "//test/e2e_federation/upgrades:go_default_library", "//vendor:github.com/onsi/ginkgo", "//vendor:github.com/onsi/gomega", "//vendor:k8s.io/apimachinery/pkg/api/errors", @@ -63,6 +66,7 @@ filegroup( srcs = [ ":package-srcs", "//test/e2e_federation/framework:all-srcs", + "//test/e2e_federation/upgrades:all-srcs", ], tags = ["automanaged"], ) diff --git a/test/e2e_federation/framework/util.go b/test/e2e_federation/framework/util.go index 55f583b62aa..7cc79aea451 100644 --- a/test/e2e_federation/framework/util.go +++ b/test/e2e_federation/framework/util.go @@ -19,7 +19,9 @@ package framework import ( "fmt" "os" + "path" "regexp" + "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -110,3 +112,46 @@ func GetValidDNSSubdomainName(name string) (string, error) { } return name, nil } + +func FederationControlPlaneUpgrade(version string) error { + version = "v" + version + _, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "federation/cluster/upgrade.sh"), version) + return err +} + +func CheckFederationVersion(c federation_clientset.Interface, want string) error { + framework.Logf("Checking federation version") + v, err := c.Discovery().ServerVersion() + if err != nil { + return fmt.Errorf("CheckFederationVersion() couldn't get the master version: %v", err) + } + // We do prefix trimming and then matching because: + // want looks like: 0.19.3-815-g50e67d4 + // got looks like: v0.19.3-815-g50e67d4034e858-dirty + got := strings.TrimPrefix(v.GitVersion, "v") + if !strings.HasPrefix(got, want) { + return fmt.Errorf("federation had apiserver version %s which does not start with %s", + got, want) + } + framework.Logf("Federation is at version %s", want) + return nil +} + +func MasterUpgrade(context, version string) error { + switch framework.TestContext.Provider { + case "gce": + return masterUpgradeGCE(context, version) + default: + return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", framework.TestContext.Provider) + } +} + +func masterUpgradeGCE(context, rawVersion string) error { + version := "v" + rawVersion + // TODO: this breaks if we want to upgrade 2 clusters in same zone. use alternate methods in future to get zone of a cluster + zone := strings.TrimPrefix(context, "federation-e2e-"+framework.TestContext.Provider+"-") + + env := append(os.Environ(), "KUBE_CONTEXT="+context, "ZONE="+zone) + _, _, err := framework.RunCmdEnv(env, path.Join(framework.TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", version) + return err +} diff --git a/test/e2e_federation/upgrade.go b/test/e2e_federation/upgrade.go new file mode 100644 index 00000000000..ce24bf8b451 --- /dev/null +++ b/test/e2e_federation/upgrade.go @@ -0,0 +1,139 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_federation + +import ( + "k8s.io/kubernetes/test/e2e/chaosmonkey" + "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" + "k8s.io/kubernetes/test/e2e_federation/upgrades" + + . "github.com/onsi/ginkgo" +) + +var upgradeTests = []upgrades.Test{} + +var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { + f := fedframework.NewDefaultFederatedFramework("federation-upgrade") + + framework.KubeDescribe("Federation Control Plane upgrade", func() { + It("should maintain a functioning federation [Feature:FCPUpgrade]", func() { + fedframework.SkipUnlessFederated(f.ClientSet) + cm := chaosmonkey.New(func() { + federationControlPlaneUpgrade(f) + }) + for _, t := range upgradeTests { + cm.RegisterInterface(&chaosMonkeyAdapter{ + test: t, + framework: f, + upgradeType: upgrades.FCPUpgrade, + }) + } + cm.Do() + }) + }) + + framework.KubeDescribe("Federated clusters upgrade", func() { + It("should maintain a functioning federation [Feature:FederatedClustersUpgrade]", func() { + fedframework.SkipUnlessFederated(f.ClientSet) + cm := chaosmonkey.New(func() { + federatedClustersUpgrade(f) + }) + for _, t := range upgradeTests { + cm.RegisterInterface(&chaosMonkeyAdapter{ + test: t, + framework: f, + upgradeType: upgrades.FederatedClustersUpgrade, + }) + } + cm.Do() + }) + }) + + framework.KubeDescribe("FCP upgrade followed by federated clusters upgrade", func() { + It("should maintain a functioning federation [Feature:FCPUpgradeFollowedByFederatedClustersUpgrade]", func() { + fedframework.SkipUnlessFederated(f.ClientSet) + cm := chaosmonkey.New(func() { + federationControlPlaneUpgrade(f) + federatedClustersUpgrade(f) + }) + for _, t := range upgradeTests { + cm.RegisterInterface(&chaosMonkeyAdapter{ + test: t, + framework: f, + upgradeType: upgrades.FCPUpgradeFollowedByFederatedClustersUpgrade, + }) + } + cm.Do() + }) + }) + + framework.KubeDescribe("Federated clusters upgrade followed by FCP upgrade", func() { + It("should maintain a functioning federation [Feature:FederatedClustersUpgradeFollowedByFCPUpgrade]", func() { + fedframework.SkipUnlessFederated(f.ClientSet) + cm := chaosmonkey.New(func() { + federatedClustersUpgrade(f) + federationControlPlaneUpgrade(f) + }) + for _, t := range upgradeTests { + cm.RegisterInterface(&chaosMonkeyAdapter{ + test: t, + framework: f, + upgradeType: upgrades.FederatedClustersUpgradeFollowedByFCPUpgrade, + }) + } + cm.Do() + }) + }) +}) + +type chaosMonkeyAdapter struct { + test upgrades.Test + framework *fedframework.Framework + upgradeType upgrades.FederationUpgradeType +} + +func (cma *chaosMonkeyAdapter) Setup() { + cma.test.Setup(cma.framework) +} + +func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) { + cma.test.Test(cma.framework, stopCh, cma.upgradeType) +} + +func (cma *chaosMonkeyAdapter) Teardown() { + cma.test.Teardown(cma.framework) +} + +func federationControlPlaneUpgrade(f *fedframework.Framework) { + federationVersion, err := framework.RealVersion(framework.TestContext.FederationUpgradeTarget) + framework.ExpectNoError(err) + framework.ExpectNoError(fedframework.FederationControlPlaneUpgrade(federationVersion)) + framework.ExpectNoError(fedframework.CheckFederationVersion(f.FederationClientset, federationVersion)) +} + +func federatedClustersUpgrade(f *fedframework.Framework) { + k8sVersion, err := framework.RealVersion(framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + clusters, _ := getRegisteredClusters(UserAgentName, f) + for _, cluster := range clusters { + framework.ExpectNoError(fedframework.MasterUpgrade(cluster.name, k8sVersion)) + framework.ExpectNoError(framework.CheckMasterVersion(cluster.Clientset, k8sVersion)) + + // TODO: Need to add Node upgrade. Add once this framework is stable + } +} diff --git a/test/e2e_federation/upgrades/BUILD b/test/e2e_federation/upgrades/BUILD new file mode 100644 index 00000000000..4296f48f5f4 --- /dev/null +++ b/test/e2e_federation/upgrades/BUILD @@ -0,0 +1,28 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", +) + +go_library( + name = "go_default_library", + srcs = ["upgrade.go"], + tags = ["automanaged"], + deps = ["//test/e2e_federation/framework:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/test/e2e_federation/upgrades/upgrade.go b/test/e2e_federation/upgrades/upgrade.go new file mode 100644 index 00000000000..de9327c7736 --- /dev/null +++ b/test/e2e_federation/upgrades/upgrade.go @@ -0,0 +1,56 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package upgrades provides a framework for testing Kubernetes federation +// features before, during, and after different types of upgrades. +package upgrades + +import fedframework "k8s.io/kubernetes/test/e2e_federation/framework" + +// FederationUpgradeType represents different types of federation upgrades. +type FederationUpgradeType int + +const ( + // FCPUpgrade indicates that federation control plane is being upgraded. + FCPUpgrade FederationUpgradeType = iota + + // FederatedClustersUpgrade indicates that federated clusters are being upgraded. + FederatedClustersUpgrade + + // FCPUpgradeFollowedByFederatedClustersUpgrade indicates that federation control plane is upgraded + // followed by federated clusters upgrade. + FCPUpgradeFollowedByFederatedClustersUpgrade + + // FederatedClustersUpgradeFollowedByFCPUpgrade indicates that federated clusters are upgraded + // followed by federation control plane upgrade. + FederatedClustersUpgradeFollowedByFCPUpgrade +) + +// Test is an interface for federation upgrade tests. +type Test interface { + // Setup should create and verify whatever objects need to + // exist before the upgrade disruption starts. + Setup(f *fedframework.Framework) + + // Test will run during the upgrade. When the upgrade is + // complete, done will be closed and final validation can + // begin. + Test(f *fedframework.Framework, done <-chan struct{}, upgrade FederationUpgradeType) + + // TearDown should clean up any objects that are created that + // aren't already cleaned up by the framework. + Teardown(f *fedframework.Framework) +}