Merge pull request #43820 from krousey/upgrades

Automatic merge from submit-queue (batch tested with PRs 42617, 43247, 43509, 43644, 43820)

Adding a SSH tunnel check to GKE upgrades

**What this PR does / why we need it**: After an upgrade, it takes some time fore the master to re-establish SSH tunnels to the nodes. This adds a wait for GKE since it runs in this configuration.

**Which issue this PR fixes**: fixes #43611, #43612

test/e2e/framework/nodes_util.go

@@ -87,7 +87,13 @@ func masterUpgradeGKE(v string) error {
 		"--master",
 		fmt.Sprintf("--cluster-version=%s", v),
 		"--quiet")
-	return err
+	if err != nil {
+		return err
+	}
+
+	waitForSSHTunnels()
+
+	return nil
 }
 
 func NodeUpgrade(f *Framework, v string, img string) error {
@@ -166,7 +172,14 @@ func nodeUpgradeGKE(v string, img string) error {
 		args = append(args, fmt.Sprintf("--image-type=%s", img))
 	}
 	_, _, err := RunCmd("gcloud", args...)
-	return err
+
+	if err != nil {
+		return err
+	}
+
+	waitForSSHTunnels()
+
+	return nil
 }
 
 // CheckNodesReady waits up to nt for expect nodes accessed by c to be ready,
@@ -270,3 +283,19 @@ func gceUpgradeScript() string {
 	}
 	return TestContext.GCEUpgradeScript
 }
+
+func waitForSSHTunnels() {
+	Logf("Waiting for SSH tunnels to establish")
+	RunKubectl("run", "ssh-tunnel-test",
+		"--image=gcr.io/google_containers/busybox:1.24",
+		"--restart=Never",
+		"--command", "--",
+		"echo", "Hello")
+	defer RunKubectl("delete", "pod", "ssh-tunnel-test")
+
+	// allow up to a minute for new ssh tunnels to establish
+	wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) {
+		_, err := RunKubectl("logs", "ssh-tunnel-test")
+		return err == nil, nil
+	})
+}