Merge pull request #59234 from nikhiljindal/kubemcie2e

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Adding kubemci e2e test for ingress spec conformance

**What this PR does / why we need it**:
Adding an e2e test case for kubemci to verify that it conforms to the ingress spec.
Not all tests will pass right now, but adding it will enable us to track the latest status.

```release-note
NONE
```

test/e2e/framework/ingress_utils.go

@@ -31,6 +31,7 @@ import (
 	"net/http"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -1127,10 +1128,39 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m
 		j.Ingress.Annotations[k] = v
 	}
 	j.Logger.Infof(fmt.Sprintf("creating" + j.Ingress.Name + " ingress"))
-	j.Ingress, err = j.Client.ExtensionsV1beta1().Ingresses(ns).Create(j.Ingress)
+	j.Ingress, err = j.RunCreate(j.Ingress)
 	ExpectNoError(err)
 }
 
+// RunCreate runs the required command to create the given ingress.
+func (j *IngressTestJig) RunCreate(ing *extensions.Ingress) (*extensions.Ingress, error) {
+	if j.Class != MulticlusterIngressClassValue {
+		return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Create(ing)
+	}
+	// Use kubemci to create a multicluster ingress.
+	filePath := TestContext.OutputDir + "/mci.yaml"
+	if err := manifest.IngressToManifest(ing, filePath); err != nil {
+		return nil, err
+	}
+	_, err := RunKubemciCmd("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
+	return ing, err
+}
+
+// RunUpdate runs the required command to update the given ingress.
+func (j *IngressTestJig) RunUpdate(ing *extensions.Ingress) (*extensions.Ingress, error) {
+	if j.Class != MulticlusterIngressClassValue {
+		return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Update(ing)
+	}
+	// Use kubemci to update a multicluster ingress.
+	// kubemci does not have an update command. We use "create --force" to update an existing ingress.
+	filePath := TestContext.OutputDir + "/mci.yaml"
+	if err := manifest.IngressToManifest(ing, filePath); err != nil {
+		return nil, err
+	}
+	_, err := RunKubemciCmd("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force")
+	return ing, err
+}
+
 // Update retrieves the ingress, performs the passed function, and then updates it.
 func (j *IngressTestJig) Update(update func(ing *extensions.Ingress)) {
 	var err error
@@ -1141,7 +1171,7 @@ func (j *IngressTestJig) Update(update func(ing *extensions.Ingress)) {
 			Failf("failed to get ingress %q: %v", name, err)
 		}
 		update(j.Ingress)
-		j.Ingress, err = j.Client.ExtensionsV1beta1().Ingresses(ns).Update(j.Ingress)
+		j.Ingress, err = j.RunUpdate(j.Ingress)
 		if err == nil {
 			DescribeIng(j.Ingress.Namespace)
 			return
@@ -1193,9 +1223,8 @@ func (j *IngressTestJig) TryDeleteIngress() {
 }
 
 func (j *IngressTestJig) TryDeleteGivenIngress(ing *extensions.Ingress) {
-	err := j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil)
-	if err != nil {
-		j.Logger.Infof("Error while deleting the ingress %v/%v: %v", ing.Namespace, ing.Name, err)
+	if err := j.RunDelete(ing, j.Class); err != nil {
+		j.Logger.Infof("Error while deleting the ingress %v/%v with class %s: %v", ing.Namespace, ing.Name, j.Class, err)
 	}
 }
 
@@ -1206,8 +1235,45 @@ func (j *IngressTestJig) TryDeleteGivenService(svc *v1.Service) {
 	}
 }
 
+// RunDelete runs the required command to delete the given ingress.
+func (j *IngressTestJig) RunDelete(ing *extensions.Ingress, class string) error {
+	if j.Class != MulticlusterIngressClassValue {
+		return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil)
+	}
+	// Use kubemci to delete a multicluster ingress.
+	filePath := TestContext.OutputDir + "/mci.yaml"
+	if err := manifest.IngressToManifest(ing, filePath); err != nil {
+		return err
+	}
+	_, err := RunKubemciCmd("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
+	return err
+}
+
+// getIngressAddressFromKubemci returns the IP address of the given multicluster ingress using kubemci.
+// TODO(nikhiljindal): Update this to be able to return hostname as well.
+func getIngressAddressFromKubemci(name string) ([]string, error) {
+	out, err := RunKubemciCmd("get-status", name)
+	if err != nil {
+		return []string{}, err
+	}
+	ip := findIPv4(out)
+	return []string{ip}, err
+}
+
+// findIPv4 returns the first IPv4 address found in the given string.
+func findIPv4(input string) string {
+	numBlock := "(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])"
+	regexPattern := numBlock + "\\." + numBlock + "\\." + numBlock + "\\." + numBlock
+
+	regEx := regexp.MustCompile(regexPattern)
+	return regEx.FindString(input)
+}
+
 // getIngressAddress returns the ips/hostnames associated with the Ingress.
-func getIngressAddress(client clientset.Interface, ns, name string) ([]string, error) {
+func getIngressAddress(client clientset.Interface, ns, name, class string) ([]string, error) {
+	if class == MulticlusterIngressClassValue {
+		return getIngressAddressFromKubemci(name)
+	}
 	ing, err := client.ExtensionsV1beta1().Ingresses(ns).Get(name, metav1.GetOptions{})
 	if err != nil {
 		return nil, err
@@ -1228,7 +1294,7 @@ func getIngressAddress(client clientset.Interface, ns, name string) ([]string, e
 func (j *IngressTestJig) WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout time.Duration) (string, error) {
 	var address string
 	err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
-		ipOrNameList, err := getIngressAddress(c, ns, ingName)
+		ipOrNameList, err := getIngressAddress(c, ns, ingName, j.Class)
 		if err != nil || len(ipOrNameList) == 0 {
 			j.Logger.Errorf("Waiting for Ingress %v to acquire IP, error %v", ingName, err)
 			if IsRetryableAPIError(err) {

test/e2e/framework/util.go

@@ -2177,6 +2177,21 @@ func RunKubectlOrDieInput(data string, args ...string) string {
 	return NewKubectlCommand(args...).WithStdinData(data).ExecOrDie()
 }
 
+// RunKubemciCmd is a convenience wrapper over kubectlBuilder to run kubemci.
+// It assumes that kubemci exists in PATH.
+func RunKubemciCmd(args ...string) (string, error) {
+	// kubemci is assumed to be in PATH.
+	kubemci := "kubemci"
+	b := new(kubectlBuilder)
+	if TestContext.KubeConfig != "" {
+		args = append(args, "--"+clientcmd.RecommendedConfigPathFlag+"="+TestContext.KubeConfig)
+	}
+	args = append(args, "--gcp-project="+TestContext.CloudConfig.ProjectID)
+
+	b.cmd = exec.Command(kubemci, args...)
+	return b.Exec()
+}
+
 func StartCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) {
 	stdout, err = cmd.StdoutPipe()
 	if err != nil {

test/e2e/manifest/BUILD

@@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"])
 load(
     "@io_bazel_rules_go//go:def.bzl",
     "go_library",
+    "go_test",
 )
 
 go_library(
@@ -10,6 +11,7 @@ go_library(
     srcs = ["manifest.go"],
     importpath = "k8s.io/kubernetes/test/e2e/manifest",
     deps = [
+        "//cmd/kubeadm/app/util:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//test/e2e/generated:go_default_library",
         "//vendor/k8s.io/api/apps/v1:go_default_library",
@@ -33,3 +35,11 @@ filegroup(
     srcs = [":package-srcs"],
     tags = ["automanaged"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["manifest_test.go"],
+    embed = [":go_default_library"],
+    importpath = "k8s.io/kubernetes/test/e2e/manifest",
+    deps = ["//vendor/k8s.io/api/extensions/v1beta1:go_default_library"],
+)

test/e2e/manifest/manifest.go

@@ -17,12 +17,16 @@ limitations under the License.
 package manifest
 
 import (
+	"fmt"
+	"io/ioutil"
+
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/test/e2e/generated"
 )
@@ -87,6 +91,20 @@ func IngressFromManifest(fileName string) (*extensions.Ingress, error) {
 	return &ing, nil
 }
 
+// IngressToManifest generates a yaml file in the given path with the given ingress.
+// Returns the file name and error, if there was any.
+func IngressToManifest(ing *extensions.Ingress, path string) error {
+	serialized, err := util.MarshalToYaml(ing, extensions.SchemeGroupVersion)
+	if err != nil {
+		return fmt.Errorf("failed to marshal ingress %v to YAML: %v", ing, err)
+	}
+
+	if err := ioutil.WriteFile(path, serialized, 0600); err != nil {
+		return fmt.Errorf("error in writing ingress to file: %s", err)
+	}
+	return nil
+}
+
 // StatefulSetFromManifest returns a StatefulSet from a manifest stored in fileName in the Namespace indicated by ns.
 func StatefulSetFromManifest(fileName, ns string) (*apps.StatefulSet, error) {
 	var ss apps.StatefulSet

test/e2e/manifest/manifest_test.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package manifest
+
+import (
+	"testing"
+
+	extensions "k8s.io/api/extensions/v1beta1"
+)
+
+func TestIngressToManifest(t *testing.T) {
+	ing := &extensions.Ingress{}
+	// Write the ingress to a file and ensure that there is no error.
+	if err := IngressToManifest(ing, "/tmp/ing.yaml"); err != nil {
+		t.Fatalf("Error in creating file: %s", err)
+	}
+	// Writing it again should not return an error.
+	if err := IngressToManifest(ing, "/tmp/ing.yaml"); err != nil {
+		t.Fatalf("Error in creating file: %s", err)
+	}
+}

test/e2e/network/ingress.go

@@ -566,6 +566,38 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		})
 	})
 
+	Describe("GCE [Slow] [Feature:kubemci]", func() {
+		// Platform specific setup
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce", "gke")
+			jig.Class = framework.MulticlusterIngressClassValue
+		})
+
+		// Platform specific cleanup
+		AfterEach(func() {
+			if CurrentGinkgoTestDescription().Failed {
+				framework.DescribeIng(ns)
+			}
+			if jig.Ingress == nil {
+				By("No ingress created, no cleanup necessary")
+				return
+			}
+			By("Deleting ingress")
+			jig.TryDeleteIngress()
+		})
+
+		It("should conform to Ingress spec", func() {
+			jig.PollInterval = 5 * time.Second
+			conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{})
+			for _, t := range conformanceTests {
+				By(t.EntryLog)
+				t.Execute()
+				By(t.ExitLog)
+				jig.WaitForIngress(true /*waitForNodePort*/)
+			}
+		})
+	})
+
 	// Time: borderline 5m, slow by design
 	Describe("[Slow] Nginx", func() {
 		var nginxController *framework.NginxIngressController