Implement automated downgrade testing.

Node version cannot be higher than the master version, so we must
switch the node version first. Also, we must use the upgrade script
from the appropriate version for GCE.

hack/verify-flags/known-flags.txt

@@ -282,6 +282,7 @@ gather-metrics-at-teardown
 gather-resource-usage
 gce-project
 gce-service-account
+gce-upgrade-script
 gce-zone
 ginkgo-flags
 gke-cluster

test/e2e/cluster_upgrade.go

@@ -113,6 +113,39 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 	})
 })
 
+var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() {
+	f := framework.NewDefaultFramework("cluster-downgrade")
+
+	// Create the frameworks here because we can only create them
+	// in a "Describe".
+	testFrameworks := map[string]*framework.Framework{}
+	for _, t := range upgradeTests {
+		testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name())
+	}
+
+	framework.KubeDescribe("cluster downgrade", func() {
+		It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() {
+			cm := chaosmonkey.New(func() {
+				// Yes this really is a downgrade. And nodes must downgrade first.
+				v, err := realVersion(framework.TestContext.UpgradeTarget)
+				framework.ExpectNoError(err)
+				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
+				framework.ExpectNoError(checkNodesVersions(f.ClientSet, v))
+				framework.ExpectNoError(framework.MasterUpgrade(v))
+				framework.ExpectNoError(checkMasterVersion(f.ClientSet, v))
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   testFrameworks[t.Name()],
+					upgradeType: upgrades.ClusterUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+})
+
 var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
 	// Create the frameworks here because we can only create them
 	// in a "Describe".

test/e2e/framework/nodes_util.go

@@ -57,13 +57,13 @@ func etcdUpgradeGCE(target_storage, target_version string) error {
 		"STORAGE_BACKEND="+target_storage,
 		"TEST_ETCD_IMAGE=3.0.17")
 
-	_, _, err := RunCmdEnv(env, path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-l", "-M")
+	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-l", "-M")
 	return err
 }
 
 func masterUpgradeGCE(rawV string) error {
 	v := "v" + rawV
-	_, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", v)
+	_, _, err := RunCmd(gceUpgradeScript(), "-M", v)
 	return err
 }
 
@@ -111,10 +111,10 @@ func nodeUpgradeGCE(rawV, img string) error {
 	v := "v" + rawV
 	if img != "" {
 		env := append(os.Environ(), "KUBE_NODE_OS_DISTRIBUTION="+img)
-		_, _, err := RunCmdEnv(env, path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-N", "-o", v)
+		_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v)
 		return err
 	}
-	_, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-N", v)
+	_, _, err := RunCmd(gceUpgradeScript(), "-N", v)
 	return err
 }
 
@@ -254,3 +254,10 @@ func MigTemplate() (string, error) {
 	}
 	return templ, nil
 }
+
+func gceUpgradeScript() string {
+	if len(TestContext.GCEUpgradeScript) == 0 {
+		return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh")
+	}
+	return TestContext.GCEUpgradeScript
+}

test/e2e/framework/test_context.go

@@ -52,6 +52,7 @@ type TestContextType struct {
 	EtcdUpgradeStorage       string
 	EtcdUpgradeVersion       string
 	UpgradeImage             string
+	GCEUpgradeScript         string
 	PrometheusPushGateway    string
 	ContainerRuntime         string
 	MasterOSDistro           string
@@ -200,6 +201,7 @@ func RegisterClusterFlags() {
 	flag.StringVar(&TestContext.EtcdUpgradeStorage, "etcd-upgrade-storage", "", "The storage version to upgrade to (either 'etcdv2' or 'etcdv3') if doing an etcd upgrade test.")
 	flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.")
 	flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.")
+	flag.StringVar(&TestContext.GCEUpgradeScript, "gce-upgrade-script", "", "Script to use to upgrade a GCE cluster.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
 	flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", true, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")