Merge pull request #32588 from maisem/upgrade_images

Automatic merge from submit-queue

Adding support for upgrading testing across image types.

Adds support for upgrade testing across image types.

@spxtr @vishh @ixdy @pwittrock

This change only affects upgrade testing. This does not touch production code and hence should be safe for cherrypicks into the 1.4 release branch.

hack/verify-flags/known-flags.txt

@@ -512,6 +512,7 @@ udp-timeout
 unhealthy-zone-threshold
 unix-socket
 update-period
+upgrade-image
 upgrade-target
 use-kubernetes-cluster-service
 user-whitelist

test/e2e/cluster_upgrade.go

@@ -58,7 +58,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 			cm := chaosmonkey.New(func() {
 				v, err := realVersion(framework.TestContext.UpgradeTarget)
 				framework.ExpectNoError(err)
-				framework.ExpectNoError(framework.NodeUpgrade(f, v))
+				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
 				framework.ExpectNoError(checkNodesVersions(f.Client, v))
 			})
 			cm.Register(func(sem *chaosmonkey.Semaphore) {
@@ -72,7 +72,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 			cm := chaosmonkey.New(func() {
 				v, err := realVersion(framework.TestContext.UpgradeTarget)
 				framework.ExpectNoError(err)
-				framework.ExpectNoError(framework.NodeUpgrade(f, v))
+				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
 				framework.ExpectNoError(checkNodesVersions(f.Client, v))
 			})
 			cm.Register(func(sem *chaosmonkey.Semaphore) {
@@ -90,7 +90,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 				framework.ExpectNoError(err)
 				framework.ExpectNoError(framework.MasterUpgrade(v))
 				framework.ExpectNoError(checkMasterVersion(f.Client, v))
-				framework.ExpectNoError(framework.NodeUpgrade(f, v))
+				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
 				framework.ExpectNoError(checkNodesVersions(f.Client, v))
 			})
 			cm.Register(func(sem *chaosmonkey.Semaphore) {
@@ -106,7 +106,7 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 				framework.ExpectNoError(err)
 				framework.ExpectNoError(framework.MasterUpgrade(v))
 				framework.ExpectNoError(checkMasterVersion(f.Client, v))
-				framework.ExpectNoError(framework.NodeUpgrade(f, v))
+				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
 				framework.ExpectNoError(checkNodesVersions(f.Client, v))
 			})
 			cm.Register(func(sem *chaosmonkey.Semaphore) {

test/e2e/framework/nodes_util.go

@@ -61,14 +61,15 @@ func masterUpgradeGKE(v string) error {
 	return err
 }
 
-var NodeUpgrade = func(f *Framework, v string) error {
+var NodeUpgrade = func(f *Framework, v string, img string) error {
 	// Perform the upgrade.
 	var err error
 	switch TestContext.Provider {
 	case "gce":
+		// TODO(maisem): add GCE support for upgrading to different images.
 		err = nodeUpgradeGCE(v)
 	case "gke":
-		err = nodeUpgradeGKE(v)
+		err = nodeUpgradeGKE(v, img)
 	default:
 		err = fmt.Errorf("NodeUpgrade() is not implemented for provider %s", TestContext.Provider)
 	}
@@ -249,16 +250,22 @@ func cleanupNodeUpgradeGCE(tmplBefore string) {
 	}
 }
 
-func nodeUpgradeGKE(v string) error {
+func nodeUpgradeGKE(v string, img string) error {
-	Logf("Upgrading nodes to %q", v)
+	Logf("Upgrading nodes to version %q and image %q", v, img)
-	_, _, err := RunCmd("gcloud", "container",
+	args := []string{
+		"container",
 		"clusters",
 		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
 		fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
 		"upgrade",
 		TestContext.CloudConfig.Cluster,
 		fmt.Sprintf("--cluster-version=%s", v),
-		"--quiet")
+		"--quiet",
+	}
+	if len(img) > 0 {
+		args = append(args, fmt.Sprintf("--image-type=%s", img))
+	}
+	_, _, err := RunCmd("gcloud", args...)
 	return err
 }
 

test/e2e/framework/test_context.go

@@ -46,6 +46,7 @@ type TestContextType struct {
 	// Timeout for waiting for system pods to be running
 	SystemPodsStartupTimeout time.Duration
 	UpgradeTarget            string
+	UpgradeImage             string
 	PrometheusPushGateway    string
 	ContainerRuntime         string
 	MasterOSDistro           string
@@ -163,6 +164,7 @@ func RegisterClusterFlags() {
 	flag.IntVar(&TestContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.")
 	flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.")
 	flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
+	flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
 	flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", true, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")