diff --git a/federation/cluster/upgrade.sh b/federation/cluster/upgrade.sh new file mode 100755 index 00000000000..81755aed808 --- /dev/null +++ b/federation/cluster/upgrade.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +# For `kube::log::status` function since it already sources +# "${KUBE_ROOT}/cluster/lib/logging.sh" +source "${KUBE_ROOT}/cluster/common.sh" +# For $FEDERATION_NAME, $FEDERATION_NAMESPACE, $HOST_CLUSTER_CONTEXT, +source "${KUBE_ROOT}/federation/cluster/common.sh" + +KUBE_VERSION="${1}" + +host_kubectl="${KUBE_ROOT}/cluster/kubectl.sh --context=${HOST_CLUSTER_CONTEXT} --namespace=${FEDERATION_NAMESPACE}" + +function upgrade() { + local -r project="${KUBE_PROJECT:-${PROJECT:-}}" + local -r kube_registry="${KUBE_REGISTRY:-gcr.io/${project}}" + local -r image_version="${kube_registry}/hyperkube-amd64:${KUBE_VERSION}" + + kube::log::status "Upgrading federation control plane ${FEDERATION_NAME} with image ${image_version}" + + # Upgrade apiserver image + ${host_kubectl} set image deployment/federation-apiserver apiserver=${image_version} + + # Upgrade controller-manager image + ${host_kubectl} set image deployment/federation-controller-manager controller-manager=${image_version} +} + +upgrade diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 94eb3638ac3..dce3eda8e27 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -263,6 +263,7 @@ federated-api-qps federated-kube-context federation-name federation-system-namespace +federation-upgrade-target file-check-frequency file_content_in_loop file-suffix diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 8597965b012..acb3ff7d875 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -88,6 +88,8 @@ type TestContextType struct { NodeTestContextType // Federation e2e context FederatedKubeContext string + // Federation control plane version to upgrade to while doing upgrade tests + FederationUpgradeTarget string // Viper-only parameters. These will in time replace all flags. @@ -202,6 +204,7 @@ func RegisterClusterFlags() { flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.") flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.") flag.StringVar(&TestContext.GCEUpgradeScript, "gce-upgrade-script", "", "Script to use to upgrade a GCE cluster.") + flag.StringVar(&TestContext.FederationUpgradeTarget, "federation-upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an federation upgrade test.") flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.") flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.") flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", true, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.") diff --git a/test/e2e_federation/framework/util.go b/test/e2e_federation/framework/util.go index 55f583b62aa..7cc79aea451 100644 --- a/test/e2e_federation/framework/util.go +++ b/test/e2e_federation/framework/util.go @@ -19,7 +19,9 @@ package framework import ( "fmt" "os" + "path" "regexp" + "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -110,3 +112,46 @@ func GetValidDNSSubdomainName(name string) (string, error) { } return name, nil } + +func FederationControlPlaneUpgrade(version string) error { + version = "v" + version + _, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "federation/cluster/upgrade.sh"), version) + return err +} + +func CheckFederationVersion(c federation_clientset.Interface, want string) error { + framework.Logf("Checking federation version") + v, err := c.Discovery().ServerVersion() + if err != nil { + return fmt.Errorf("CheckFederationVersion() couldn't get the master version: %v", err) + } + // We do prefix trimming and then matching because: + // want looks like: 0.19.3-815-g50e67d4 + // got looks like: v0.19.3-815-g50e67d4034e858-dirty + got := strings.TrimPrefix(v.GitVersion, "v") + if !strings.HasPrefix(got, want) { + return fmt.Errorf("federation had apiserver version %s which does not start with %s", + got, want) + } + framework.Logf("Federation is at version %s", want) + return nil +} + +func MasterUpgrade(context, version string) error { + switch framework.TestContext.Provider { + case "gce": + return masterUpgradeGCE(context, version) + default: + return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", framework.TestContext.Provider) + } +} + +func masterUpgradeGCE(context, rawVersion string) error { + version := "v" + rawVersion + // TODO: this breaks if we want to upgrade 2 clusters in same zone. use alternate methods in future to get zone of a cluster + zone := strings.TrimPrefix(context, "federation-e2e-"+framework.TestContext.Provider+"-") + + env := append(os.Environ(), "KUBE_CONTEXT="+context, "ZONE="+zone) + _, _, err := framework.RunCmdEnv(env, path.Join(framework.TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", version) + return err +} diff --git a/test/e2e_federation/upgrade.go b/test/e2e_federation/upgrade.go new file mode 100644 index 00000000000..ce24bf8b451 --- /dev/null +++ b/test/e2e_federation/upgrade.go @@ -0,0 +1,139 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_federation + +import ( + "k8s.io/kubernetes/test/e2e/chaosmonkey" + "k8s.io/kubernetes/test/e2e/framework" + fedframework "k8s.io/kubernetes/test/e2e_federation/framework" + "k8s.io/kubernetes/test/e2e_federation/upgrades" + + . "github.com/onsi/ginkgo" +) + +var upgradeTests = []upgrades.Test{} + +var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { + f := fedframework.NewDefaultFederatedFramework("federation-upgrade") + + framework.KubeDescribe("Federation Control Plane upgrade", func() { + It("should maintain a functioning federation [Feature:FCPUpgrade]", func() { + fedframework.SkipUnlessFederated(f.ClientSet) + cm := chaosmonkey.New(func() { + federationControlPlaneUpgrade(f) + }) + for _, t := range upgradeTests { + cm.RegisterInterface(&chaosMonkeyAdapter{ + test: t, + framework: f, + upgradeType: upgrades.FCPUpgrade, + }) + } + cm.Do() + }) + }) + + framework.KubeDescribe("Federated clusters upgrade", func() { + It("should maintain a functioning federation [Feature:FederatedClustersUpgrade]", func() { + fedframework.SkipUnlessFederated(f.ClientSet) + cm := chaosmonkey.New(func() { + federatedClustersUpgrade(f) + }) + for _, t := range upgradeTests { + cm.RegisterInterface(&chaosMonkeyAdapter{ + test: t, + framework: f, + upgradeType: upgrades.FederatedClustersUpgrade, + }) + } + cm.Do() + }) + }) + + framework.KubeDescribe("FCP upgrade followed by federated clusters upgrade", func() { + It("should maintain a functioning federation [Feature:FCPUpgradeFollowedByFederatedClustersUpgrade]", func() { + fedframework.SkipUnlessFederated(f.ClientSet) + cm := chaosmonkey.New(func() { + federationControlPlaneUpgrade(f) + federatedClustersUpgrade(f) + }) + for _, t := range upgradeTests { + cm.RegisterInterface(&chaosMonkeyAdapter{ + test: t, + framework: f, + upgradeType: upgrades.FCPUpgradeFollowedByFederatedClustersUpgrade, + }) + } + cm.Do() + }) + }) + + framework.KubeDescribe("Federated clusters upgrade followed by FCP upgrade", func() { + It("should maintain a functioning federation [Feature:FederatedClustersUpgradeFollowedByFCPUpgrade]", func() { + fedframework.SkipUnlessFederated(f.ClientSet) + cm := chaosmonkey.New(func() { + federatedClustersUpgrade(f) + federationControlPlaneUpgrade(f) + }) + for _, t := range upgradeTests { + cm.RegisterInterface(&chaosMonkeyAdapter{ + test: t, + framework: f, + upgradeType: upgrades.FederatedClustersUpgradeFollowedByFCPUpgrade, + }) + } + cm.Do() + }) + }) +}) + +type chaosMonkeyAdapter struct { + test upgrades.Test + framework *fedframework.Framework + upgradeType upgrades.FederationUpgradeType +} + +func (cma *chaosMonkeyAdapter) Setup() { + cma.test.Setup(cma.framework) +} + +func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) { + cma.test.Test(cma.framework, stopCh, cma.upgradeType) +} + +func (cma *chaosMonkeyAdapter) Teardown() { + cma.test.Teardown(cma.framework) +} + +func federationControlPlaneUpgrade(f *fedframework.Framework) { + federationVersion, err := framework.RealVersion(framework.TestContext.FederationUpgradeTarget) + framework.ExpectNoError(err) + framework.ExpectNoError(fedframework.FederationControlPlaneUpgrade(federationVersion)) + framework.ExpectNoError(fedframework.CheckFederationVersion(f.FederationClientset, federationVersion)) +} + +func federatedClustersUpgrade(f *fedframework.Framework) { + k8sVersion, err := framework.RealVersion(framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + clusters, _ := getRegisteredClusters(UserAgentName, f) + for _, cluster := range clusters { + framework.ExpectNoError(fedframework.MasterUpgrade(cluster.name, k8sVersion)) + framework.ExpectNoError(framework.CheckMasterVersion(cluster.Clientset, k8sVersion)) + + // TODO: Need to add Node upgrade. Add once this framework is stable + } +} diff --git a/test/e2e_federation/upgrades/upgrade.go b/test/e2e_federation/upgrades/upgrade.go new file mode 100644 index 00000000000..de9327c7736 --- /dev/null +++ b/test/e2e_federation/upgrades/upgrade.go @@ -0,0 +1,56 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package upgrades provides a framework for testing Kubernetes federation +// features before, during, and after different types of upgrades. +package upgrades + +import fedframework "k8s.io/kubernetes/test/e2e_federation/framework" + +// FederationUpgradeType represents different types of federation upgrades. +type FederationUpgradeType int + +const ( + // FCPUpgrade indicates that federation control plane is being upgraded. + FCPUpgrade FederationUpgradeType = iota + + // FederatedClustersUpgrade indicates that federated clusters are being upgraded. + FederatedClustersUpgrade + + // FCPUpgradeFollowedByFederatedClustersUpgrade indicates that federation control plane is upgraded + // followed by federated clusters upgrade. + FCPUpgradeFollowedByFederatedClustersUpgrade + + // FederatedClustersUpgradeFollowedByFCPUpgrade indicates that federated clusters are upgraded + // followed by federation control plane upgrade. + FederatedClustersUpgradeFollowedByFCPUpgrade +) + +// Test is an interface for federation upgrade tests. +type Test interface { + // Setup should create and verify whatever objects need to + // exist before the upgrade disruption starts. + Setup(f *fedframework.Framework) + + // Test will run during the upgrade. When the upgrade is + // complete, done will be closed and final validation can + // begin. + Test(f *fedframework.Framework, done <-chan struct{}, upgrade FederationUpgradeType) + + // TearDown should clean up any objects that are created that + // aren't already cleaned up by the framework. + Teardown(f *fedframework.Framework) +}