Add framework for federation upgrade tests

federation/cluster/upgrade.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+# For `kube::log::status` function since it already sources
+# "${KUBE_ROOT}/cluster/lib/logging.sh"
+source "${KUBE_ROOT}/cluster/common.sh"
+# For $FEDERATION_NAME, $FEDERATION_NAMESPACE, $HOST_CLUSTER_CONTEXT,
+source "${KUBE_ROOT}/federation/cluster/common.sh"
+
+KUBE_VERSION="${1}"
+
+host_kubectl="${KUBE_ROOT}/cluster/kubectl.sh --context=${HOST_CLUSTER_CONTEXT} --namespace=${FEDERATION_NAMESPACE}"
+
+function upgrade() {
+  local -r project="${KUBE_PROJECT:-${PROJECT:-}}"
+  local -r kube_registry="${KUBE_REGISTRY:-gcr.io/${project}}"
+  local -r image_version="${kube_registry}/hyperkube-amd64:${KUBE_VERSION}"
+
+  kube::log::status "Upgrading federation control plane ${FEDERATION_NAME} with image ${image_version}"
+
+  # Upgrade apiserver image
+  ${host_kubectl} set image deployment/federation-apiserver apiserver=${image_version}
+
+  # Upgrade controller-manager image
+  ${host_kubectl} set image deployment/federation-controller-manager controller-manager=${image_version}
+}
+
+upgrade

hack/verify-flags/known-flags.txt

@@ -263,6 +263,7 @@ federated-api-qps
 federated-kube-context
 federation-name
 federation-system-namespace
+federation-upgrade-target
 file-check-frequency
 file_content_in_loop
 file-suffix

test/e2e/framework/test_context.go

@@ -88,6 +88,8 @@ type TestContextType struct {
 	NodeTestContextType
 	// Federation e2e context
 	FederatedKubeContext string
+	// Federation control plane version to upgrade to while doing upgrade tests
+	FederationUpgradeTarget string
 
 	// Viper-only parameters.  These will in time replace all flags.
 
@@ -202,6 +204,7 @@ func RegisterClusterFlags() {
 	flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.")
 	flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.")
 	flag.StringVar(&TestContext.GCEUpgradeScript, "gce-upgrade-script", "", "Script to use to upgrade a GCE cluster.")
+	flag.StringVar(&TestContext.FederationUpgradeTarget, "federation-upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an federation upgrade test.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
 	flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", true, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")

test/e2e_federation/framework/util.go

@@ -19,7 +19,9 @@ package framework
 import (
 	"fmt"
 	"os"
+	"path"
 	"regexp"
+	"strings"
 	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -110,3 +112,46 @@ func GetValidDNSSubdomainName(name string) (string, error) {
 	}
 	return name, nil
 }
+
+func FederationControlPlaneUpgrade(version string) error {
+	version = "v" + version
+	_, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "federation/cluster/upgrade.sh"), version)
+	return err
+}
+
+func CheckFederationVersion(c federation_clientset.Interface, want string) error {
+	framework.Logf("Checking federation version")
+	v, err := c.Discovery().ServerVersion()
+	if err != nil {
+		return fmt.Errorf("CheckFederationVersion() couldn't get the master version: %v", err)
+	}
+	// We do prefix trimming and then matching because:
+	// want looks like:  0.19.3-815-g50e67d4
+	// got  looks like: v0.19.3-815-g50e67d4034e858-dirty
+	got := strings.TrimPrefix(v.GitVersion, "v")
+	if !strings.HasPrefix(got, want) {
+		return fmt.Errorf("federation had apiserver version %s which does not start with %s",
+			got, want)
+	}
+	framework.Logf("Federation is at version %s", want)
+	return nil
+}
+
+func MasterUpgrade(context, version string) error {
+	switch framework.TestContext.Provider {
+	case "gce":
+		return masterUpgradeGCE(context, version)
+	default:
+		return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", framework.TestContext.Provider)
+	}
+}
+
+func masterUpgradeGCE(context, rawVersion string) error {
+	version := "v" + rawVersion
+	// TODO: this breaks if we want to upgrade 2 clusters in same zone. use alternate methods in future to get zone of a cluster
+	zone := strings.TrimPrefix(context, "federation-e2e-"+framework.TestContext.Provider+"-")
+
+	env := append(os.Environ(), "KUBE_CONTEXT="+context, "ZONE="+zone)
+	_, _, err := framework.RunCmdEnv(env, path.Join(framework.TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", version)
+	return err
+}

test/e2e_federation/upgrade.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"k8s.io/kubernetes/test/e2e/chaosmonkey"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+	"k8s.io/kubernetes/test/e2e_federation/upgrades"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var upgradeTests = []upgrades.Test{}
+
+var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federation-upgrade")
+
+	framework.KubeDescribe("Federation Control Plane upgrade", func() {
+		It("should maintain a functioning federation [Feature:FCPUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federationControlPlaneUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FCPUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+
+	framework.KubeDescribe("Federated clusters upgrade", func() {
+		It("should maintain a functioning federation [Feature:FederatedClustersUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federatedClustersUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FederatedClustersUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+
+	framework.KubeDescribe("FCP upgrade followed by federated clusters upgrade", func() {
+		It("should maintain a functioning federation [Feature:FCPUpgradeFollowedByFederatedClustersUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federationControlPlaneUpgrade(f)
+				federatedClustersUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FCPUpgradeFollowedByFederatedClustersUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+
+	framework.KubeDescribe("Federated clusters upgrade followed by FCP upgrade", func() {
+		It("should maintain a functioning federation [Feature:FederatedClustersUpgradeFollowedByFCPUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federatedClustersUpgrade(f)
+				federationControlPlaneUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FederatedClustersUpgradeFollowedByFCPUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+})
+
+type chaosMonkeyAdapter struct {
+	test        upgrades.Test
+	framework   *fedframework.Framework
+	upgradeType upgrades.FederationUpgradeType
+}
+
+func (cma *chaosMonkeyAdapter) Setup() {
+	cma.test.Setup(cma.framework)
+}
+
+func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) {
+	cma.test.Test(cma.framework, stopCh, cma.upgradeType)
+}
+
+func (cma *chaosMonkeyAdapter) Teardown() {
+	cma.test.Teardown(cma.framework)
+}
+
+func federationControlPlaneUpgrade(f *fedframework.Framework) {
+	federationVersion, err := framework.RealVersion(framework.TestContext.FederationUpgradeTarget)
+	framework.ExpectNoError(err)
+	framework.ExpectNoError(fedframework.FederationControlPlaneUpgrade(federationVersion))
+	framework.ExpectNoError(fedframework.CheckFederationVersion(f.FederationClientset, federationVersion))
+}
+
+func federatedClustersUpgrade(f *fedframework.Framework) {
+	k8sVersion, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
+	framework.ExpectNoError(err)
+	clusters, _ := getRegisteredClusters(UserAgentName, f)
+	for _, cluster := range clusters {
+		framework.ExpectNoError(fedframework.MasterUpgrade(cluster.name, k8sVersion))
+		framework.ExpectNoError(framework.CheckMasterVersion(cluster.Clientset, k8sVersion))
+
+		// TODO: Need to add Node upgrade. Add once this framework is stable
+	}
+}

test/e2e_federation/upgrades/upgrade.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package upgrades provides a framework for testing Kubernetes federation
+// features before, during, and after different types of upgrades.
+package upgrades
+
+import fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+// FederationUpgradeType represents different types of federation upgrades.
+type FederationUpgradeType int
+
+const (
+	// FCPUpgrade indicates that federation control plane is being upgraded.
+	FCPUpgrade FederationUpgradeType = iota
+
+	// FederatedClustersUpgrade indicates that federated clusters are being upgraded.
+	FederatedClustersUpgrade
+
+	// FCPUpgradeFollowedByFederatedClustersUpgrade indicates that federation control plane is upgraded
+	// followed by federated clusters upgrade.
+	FCPUpgradeFollowedByFederatedClustersUpgrade
+
+	// FederatedClustersUpgradeFollowedByFCPUpgrade indicates that federated clusters are upgraded
+	// followed by federation control plane upgrade.
+	FederatedClustersUpgradeFollowedByFCPUpgrade
+)
+
+// Test is an interface for federation upgrade tests.
+type Test interface {
+	// Setup should create and verify whatever objects need to
+	// exist before the upgrade disruption starts.
+	Setup(f *fedframework.Framework)
+
+	// Test will run during the upgrade. When the upgrade is
+	// complete, done will be closed and final validation can
+	// begin.
+	Test(f *fedframework.Framework, done <-chan struct{}, upgrade FederationUpgradeType)
+
+	// TearDown should clean up any objects that are created that
+	// aren't already cleaned up by the framework.
+	Teardown(f *fedframework.Framework)
+}