Adding kubemci e2e test for conformance

test/e2e/framework/ingress_utils.go

@@ -31,6 +31,7 @@ import (
 	"net/http"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -1127,10 +1128,39 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m
 		j.Ingress.Annotations[k] = v
 	}
 	j.Logger.Infof(fmt.Sprintf("creating" + j.Ingress.Name + " ingress"))
-	j.Ingress, err = j.Client.ExtensionsV1beta1().Ingresses(ns).Create(j.Ingress)
+	j.Ingress, err = j.RunCreate(j.Ingress)
 	ExpectNoError(err)
 }
 
+// RunCreate runs the required command to create the given ingress.
+func (j *IngressTestJig) RunCreate(ing *extensions.Ingress) (*extensions.Ingress, error) {
+	if j.Class != MulticlusterIngressClassValue {
+		return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Create(ing)
+	}
+	// Use kubemci to create a multicluster ingress.
+	filePath := TestContext.OutputDir + "/mci.yaml"
+	if err := manifest.IngressToManifest(ing, filePath); err != nil {
+		return nil, err
+	}
+	_, err := RunKubemciCmd("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
+	return ing, err
+}
+
+// RunUpdate runs the required command to update the given ingress.
+func (j *IngressTestJig) RunUpdate(ing *extensions.Ingress) (*extensions.Ingress, error) {
+	if j.Class != MulticlusterIngressClassValue {
+		return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Update(ing)
+	}
+	// Use kubemci to update a multicluster ingress.
+	// kubemci does not have an update command. We use "create --force" to update an existing ingress.
+	filePath := TestContext.OutputDir + "/mci.yaml"
+	if err := manifest.IngressToManifest(ing, filePath); err != nil {
+		return nil, err
+	}
+	_, err := RunKubemciCmd("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force")
+	return ing, err
+}
+
 // Update retrieves the ingress, performs the passed function, and then updates it.
 func (j *IngressTestJig) Update(update func(ing *extensions.Ingress)) {
 	var err error
@@ -1141,7 +1171,7 @@ func (j *IngressTestJig) Update(update func(ing *extensions.Ingress)) {
 			Failf("failed to get ingress %q: %v", name, err)
 		}
 		update(j.Ingress)
-		j.Ingress, err = j.Client.ExtensionsV1beta1().Ingresses(ns).Update(j.Ingress)
+		j.Ingress, err = j.RunUpdate(j.Ingress)
 		if err == nil {
 			DescribeIng(j.Ingress.Namespace)
 			return
@@ -1193,9 +1223,8 @@ func (j *IngressTestJig) TryDeleteIngress() {
 }
 
 func (j *IngressTestJig) TryDeleteGivenIngress(ing *extensions.Ingress) {
-	err := j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil)
+	if err := j.RunDelete(ing, j.Class); err != nil {
-	if err != nil {
+		j.Logger.Infof("Error while deleting the ingress %v/%v with class %s: %v", ing.Namespace, ing.Name, j.Class, err)
-		j.Logger.Infof("Error while deleting the ingress %v/%v: %v", ing.Namespace, ing.Name, err)
 	}
 }
 
@@ -1206,8 +1235,45 @@ func (j *IngressTestJig) TryDeleteGivenService(svc *v1.Service) {
 	}
 }
 
+// RunDelete runs the required command to delete the given ingress.
+func (j *IngressTestJig) RunDelete(ing *extensions.Ingress, class string) error {
+	if j.Class != MulticlusterIngressClassValue {
+		return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil)
+	}
+	// Use kubemci to delete a multicluster ingress.
+	filePath := TestContext.OutputDir + "/mci.yaml"
+	if err := manifest.IngressToManifest(ing, filePath); err != nil {
+		return err
+	}
+	_, err := RunKubemciCmd("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
+	return err
+}
+
+// getIngressAddressFromKubemci returns the IP address of the given multicluster ingress using kubemci.
+// TODO(nikhiljindal): Update this to be able to return hostname as well.
+func getIngressAddressFromKubemci(name string) ([]string, error) {
+	out, err := RunKubemciCmd("get-status", name)
+	if err != nil {
+		return []string{}, err
+	}
+	ip := findIPv4(out)
+	return []string{ip}, err
+}
+
+// findIPv4 returns the first IPv4 address found in the given string.
+func findIPv4(input string) string {
+	numBlock := "(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])"
+	regexPattern := numBlock + "\\." + numBlock + "\\." + numBlock + "\\." + numBlock
+
+	regEx := regexp.MustCompile(regexPattern)
+	return regEx.FindString(input)
+}
+
 // getIngressAddress returns the ips/hostnames associated with the Ingress.
-func getIngressAddress(client clientset.Interface, ns, name string) ([]string, error) {
+func getIngressAddress(client clientset.Interface, ns, name, class string) ([]string, error) {
+	if class == MulticlusterIngressClassValue {
+		return getIngressAddressFromKubemci(name)
+	}
 	ing, err := client.ExtensionsV1beta1().Ingresses(ns).Get(name, metav1.GetOptions{})
 	if err != nil {
 		return nil, err
@@ -1228,7 +1294,7 @@ func getIngressAddress(client clientset.Interface, ns, name string) ([]string, e
 func (j *IngressTestJig) WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout time.Duration) (string, error) {
 	var address string
 	err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
-		ipOrNameList, err := getIngressAddress(c, ns, ingName)
+		ipOrNameList, err := getIngressAddress(c, ns, ingName, j.Class)
 		if err != nil || len(ipOrNameList) == 0 {
 			j.Logger.Errorf("Waiting for Ingress %v to acquire IP, error %v", ingName, err)
 			if IsRetryableAPIError(err) {

test/e2e/framework/util.go

@@ -2177,6 +2177,21 @@ func RunKubectlOrDieInput(data string, args ...string) string {
 	return NewKubectlCommand(args...).WithStdinData(data).ExecOrDie()
 }
 
+// RunKubemciCmd is a convenience wrapper over kubectlBuilder to run kubemci.
+// It assumes that kubemci exists in PATH.
+func RunKubemciCmd(args ...string) (string, error) {
+	// kubemci is assumed to be in PATH.
+	kubemci := "kubemci"
+	b := new(kubectlBuilder)
+	if TestContext.KubeConfig != "" {
+		args = append(args, "--"+clientcmd.RecommendedConfigPathFlag+"="+TestContext.KubeConfig)
+	}
+	args = append(args, "--gcp-project="+TestContext.CloudConfig.ProjectID)
+
+	b.cmd = exec.Command(kubemci, args...)
+	return b.Exec()
+}
+
 func StartCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) {
 	stdout, err = cmd.StdoutPipe()
 	if err != nil {

test/e2e/manifest/manifest.go

@@ -17,12 +17,16 @@ limitations under the License.
 package manifest
 
 import (
+	"fmt"
+	"io/ioutil"
+
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/test/e2e/generated"
 )
@@ -87,6 +91,20 @@ func IngressFromManifest(fileName string) (*extensions.Ingress, error) {
 	return &ing, nil
 }
 
+// IngressToManifest generates a yaml file in the given path with the given ingress.
+// Returns the file name and error, if there was any.
+func IngressToManifest(ing *extensions.Ingress, path string) error {
+	serialized, err := util.MarshalToYaml(ing, extensions.SchemeGroupVersion)
+	if err != nil {
+		return fmt.Errorf("failed to marshal ingress %v to YAML: %v", ing, err)
+	}
+
+	if err := ioutil.WriteFile(path, serialized, 0600); err != nil {
+		return fmt.Errorf("error in writing ingress to file: %s", err)
+	}
+	return nil
+}
+
 // StatefulSetFromManifest returns a StatefulSet from a manifest stored in fileName in the Namespace indicated by ns.
 func StatefulSetFromManifest(fileName, ns string) (*apps.StatefulSet, error) {
 	var ss apps.StatefulSet

test/e2e/manifest/manifest_test.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package manifest
+
+import (
+	"testing"
+
+	extensions "k8s.io/api/extensions/v1beta1"
+)
+
+func TestIngressToManifest(t *testing.T) {
+	ing := &extensions.Ingress{}
+	// Write the ingress to a file and ensure that there is no error.
+	if err := IngressToManifest(ing, "/tmp/ing.yaml"); err != nil {
+		t.Fatalf("Error in creating file: %s", err)
+	}
+	// Writing it again should not return an error.
+	if err := IngressToManifest(ing, "/tmp/ing.yaml"); err != nil {
+		t.Fatalf("Error in creating file: %s", err)
+	}
+}

test/e2e/network/ingress.go

@@ -566,6 +566,38 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		})
 	})
 
+	Describe("GCE [Slow] [Feature:kubemci]", func() {
+		// Platform specific setup
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce", "gke")
+			jig.Class = framework.MulticlusterIngressClassValue
+		})
+
+		// Platform specific cleanup
+		AfterEach(func() {
+			if CurrentGinkgoTestDescription().Failed {
+				framework.DescribeIng(ns)
+			}
+			if jig.Ingress == nil {
+				By("No ingress created, no cleanup necessary")
+				return
+			}
+			By("Deleting ingress")
+			jig.TryDeleteIngress()
+		})
+
+		It("should conform to Ingress spec", func() {
+			jig.PollInterval = 5 * time.Second
+			conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{})
+			for _, t := range conformanceTests {
+				By(t.EntryLog)
+				t.Execute()
+				By(t.ExitLog)
+				jig.WaitForIngress(true /*waitForNodePort*/)
+			}
+		})
+	})
+
 	// Time: borderline 5m, slow by design
 	Describe("[Slow] Nginx", func() {
 		var nginxController *framework.NginxIngressController