Merge pull request #62049 from nikhiljindal/removeClustersTest

Automatic merge from submit-queue (batch tested with PRs 62049, 62085). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Adding a test for kubemci remove-clusters

Ref https://github.com/GoogleCloudPlatform/k8s-multicluster-ingress/issues/58

Adding an e2e test for `kubemci remove-clusters` command. 
The test creates an ingress and then removes it from all clusters.

cc @G-Harmon

```release-note
NONE
```

test/e2e/framework/ingress_utils.go

@@ -1170,7 +1170,7 @@ func (j *IngressTestJig) runCreate(ing *extensions.Ingress) (*extensions.Ingress
 	if err := manifest.IngressToManifest(ing, filePath); err != nil {
 		return nil, err
 	}
-	_, err := runKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
+	_, err := RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
 	return ing, err
 }
 
@@ -1185,7 +1185,7 @@ func (j *IngressTestJig) runUpdate(ing *extensions.Ingress) (*extensions.Ingress
 	if err := manifest.IngressToManifest(ing, filePath); err != nil {
 		return nil, err
 	}
-	_, err := runKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force")
+	_, err := RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force")
 	return ing, err
 }
 
@@ -1273,7 +1273,7 @@ func (j *IngressTestJig) runDelete(ing *extensions.Ingress) error {
 	if err := manifest.IngressToManifest(ing, filePath); err != nil {
 		return err
 	}
-	_, err := runKubemciWithKubeconfig("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
+	_, err := RunKubemciWithKubeconfig("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
 	return err
 }
 
@@ -1281,7 +1281,7 @@ func (j *IngressTestJig) runDelete(ing *extensions.Ingress) error {
 // TODO(nikhiljindal): Update this to be able to return hostname as well.
 func getIngressAddressFromKubemci(name string) ([]string, error) {
 	var addresses []string
-	out, err := runKubemciCmd("get-status", name)
+	out, err := RunKubemciCmd("get-status", name)
 	if err != nil {
 		return addresses, err
 	}

test/e2e/framework/util.go

@@ -2251,17 +2251,17 @@ func RunKubectlOrDieInput(data string, args ...string) string {
 	return NewKubectlCommand(args...).WithStdinData(data).ExecOrDie()
 }
 
-// runKubemciWithKubeconfig is a convenience wrapper over runKubemciCmd
+// RunKubemciWithKubeconfig is a convenience wrapper over RunKubemciCmd
-func runKubemciWithKubeconfig(args ...string) (string, error) {
+func RunKubemciWithKubeconfig(args ...string) (string, error) {
 	if TestContext.KubeConfig != "" {
 		args = append(args, "--"+clientcmd.RecommendedConfigPathFlag+"="+TestContext.KubeConfig)
 	}
-	return runKubemciCmd(args...)
+	return RunKubemciCmd(args...)
 }
 
-// runKubemciCmd is a convenience wrapper over kubectlBuilder to run kubemci.
+// RunKubemciCmd is a convenience wrapper over kubectlBuilder to run kubemci.
 // It assumes that kubemci exists in PATH.
-func runKubemciCmd(args ...string) (string, error) {
+func RunKubemciCmd(args ...string) (string, error) {
 	// kubemci is assumed to be in PATH.
 	kubemci := "kubemci"
 	b := new(kubectlBuilder)

test/e2e/network/ingress.go

@@ -630,6 +630,23 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			executeStaticIPHttpsOnlyTest(f, jig, ipName, ipAddress)
 		})
 
+		It("should remove clusters as expected", func() {
+			ingAnnotations := map[string]string{
+				framework.IngressStaticIPKey: ipName,
+			}
+			ingFilePath := filepath.Join(framework.IngressManifestPath, "http")
+			jig.CreateIngress(ingFilePath, ns, ingAnnotations, map[string]string{})
+			jig.WaitForIngress(false /*waitForNodePort*/)
+			name := jig.Ingress.Name
+			// Verify that the ingress is spread to 1 cluster as expected.
+			verifyKubemciStatusHas(name, "is spread across 1 cluster")
+			// Reuse the ingress file created while creating the ingress.
+			filePath := filepath.Join(framework.TestContext.OutputDir, "mci.yaml")
+			if _, err := framework.RunKubemciWithKubeconfig("remove-clusters", name, "--ingress="+filePath); err != nil {
+				framework.Failf("unexpected error in running kubemci remove-clusters: %s", err)
+			}
+			verifyKubemciStatusHas(name, "is spread across 0 cluster")
+		})
 	})
 
 	// Time: borderline 5m, slow by design
@@ -684,6 +701,17 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 	})
 })
 
+// verifyKubemciStatusHas fails if kubemci get-status output for the given mci does not have the given expectedSubStr.
+func verifyKubemciStatusHas(name, expectedSubStr string) {
+	statusStr, err := framework.RunKubemciCmd("get-status", name)
+	if err != nil {
+		framework.Failf("unexpected error in running kubemci get-status %s: %s", name, err)
+	}
+	if !strings.Contains(statusStr, expectedSubStr) {
+		framework.Failf("expected status to have sub string %s, actual status: %s", expectedSubStr, statusStr)
+	}
+}
+
 func executePresharedCertTest(f *framework.Framework, jig *framework.IngressTestJig, staticIPName string) {
 	preSharedCertName := "test-pre-shared-cert"
 	By(fmt.Sprintf("Creating ssl certificate %q on GCE", preSharedCertName))