diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index b36083b6d0f..bd14628f698 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -31,6 +31,7 @@ import ( "net/http" "os/exec" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -1127,10 +1128,39 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m j.Ingress.Annotations[k] = v } j.Logger.Infof(fmt.Sprintf("creating" + j.Ingress.Name + " ingress")) - j.Ingress, err = j.Client.ExtensionsV1beta1().Ingresses(ns).Create(j.Ingress) + j.Ingress, err = j.RunCreate(j.Ingress) ExpectNoError(err) } +// RunCreate runs the required command to create the given ingress. +func (j *IngressTestJig) RunCreate(ing *extensions.Ingress) (*extensions.Ingress, error) { + if j.Class != MulticlusterIngressClassValue { + return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Create(ing) + } + // Use kubemci to create a multicluster ingress. + filePath := TestContext.OutputDir + "/mci.yaml" + if err := manifest.IngressToManifest(ing, filePath); err != nil { + return nil, err + } + _, err := RunKubemciCmd("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath)) + return ing, err +} + +// RunUpdate runs the required command to update the given ingress. +func (j *IngressTestJig) RunUpdate(ing *extensions.Ingress) (*extensions.Ingress, error) { + if j.Class != MulticlusterIngressClassValue { + return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Update(ing) + } + // Use kubemci to update a multicluster ingress. + // kubemci does not have an update command. We use "create --force" to update an existing ingress. + filePath := TestContext.OutputDir + "/mci.yaml" + if err := manifest.IngressToManifest(ing, filePath); err != nil { + return nil, err + } + _, err := RunKubemciCmd("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force") + return ing, err +} + // Update retrieves the ingress, performs the passed function, and then updates it. func (j *IngressTestJig) Update(update func(ing *extensions.Ingress)) { var err error @@ -1141,7 +1171,7 @@ func (j *IngressTestJig) Update(update func(ing *extensions.Ingress)) { Failf("failed to get ingress %q: %v", name, err) } update(j.Ingress) - j.Ingress, err = j.Client.ExtensionsV1beta1().Ingresses(ns).Update(j.Ingress) + j.Ingress, err = j.RunUpdate(j.Ingress) if err == nil { DescribeIng(j.Ingress.Namespace) return @@ -1193,9 +1223,8 @@ func (j *IngressTestJig) TryDeleteIngress() { } func (j *IngressTestJig) TryDeleteGivenIngress(ing *extensions.Ingress) { - err := j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil) - if err != nil { - j.Logger.Infof("Error while deleting the ingress %v/%v: %v", ing.Namespace, ing.Name, err) + if err := j.RunDelete(ing, j.Class); err != nil { + j.Logger.Infof("Error while deleting the ingress %v/%v with class %s: %v", ing.Namespace, ing.Name, j.Class, err) } } @@ -1206,8 +1235,45 @@ func (j *IngressTestJig) TryDeleteGivenService(svc *v1.Service) { } } +// RunDelete runs the required command to delete the given ingress. +func (j *IngressTestJig) RunDelete(ing *extensions.Ingress, class string) error { + if j.Class != MulticlusterIngressClassValue { + return j.Client.ExtensionsV1beta1().Ingresses(ing.Namespace).Delete(ing.Name, nil) + } + // Use kubemci to delete a multicluster ingress. + filePath := TestContext.OutputDir + "/mci.yaml" + if err := manifest.IngressToManifest(ing, filePath); err != nil { + return err + } + _, err := RunKubemciCmd("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath)) + return err +} + +// getIngressAddressFromKubemci returns the IP address of the given multicluster ingress using kubemci. +// TODO(nikhiljindal): Update this to be able to return hostname as well. +func getIngressAddressFromKubemci(name string) ([]string, error) { + out, err := RunKubemciCmd("get-status", name) + if err != nil { + return []string{}, err + } + ip := findIPv4(out) + return []string{ip}, err +} + +// findIPv4 returns the first IPv4 address found in the given string. +func findIPv4(input string) string { + numBlock := "(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])" + regexPattern := numBlock + "\\." + numBlock + "\\." + numBlock + "\\." + numBlock + + regEx := regexp.MustCompile(regexPattern) + return regEx.FindString(input) +} + // getIngressAddress returns the ips/hostnames associated with the Ingress. -func getIngressAddress(client clientset.Interface, ns, name string) ([]string, error) { +func getIngressAddress(client clientset.Interface, ns, name, class string) ([]string, error) { + if class == MulticlusterIngressClassValue { + return getIngressAddressFromKubemci(name) + } ing, err := client.ExtensionsV1beta1().Ingresses(ns).Get(name, metav1.GetOptions{}) if err != nil { return nil, err @@ -1228,7 +1294,7 @@ func getIngressAddress(client clientset.Interface, ns, name string) ([]string, e func (j *IngressTestJig) WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout time.Duration) (string, error) { var address string err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) { - ipOrNameList, err := getIngressAddress(c, ns, ingName) + ipOrNameList, err := getIngressAddress(c, ns, ingName, j.Class) if err != nil || len(ipOrNameList) == 0 { j.Logger.Errorf("Waiting for Ingress %v to acquire IP, error %v", ingName, err) if IsRetryableAPIError(err) { diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index d57dfb913ef..170d6bcae76 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -2177,6 +2177,21 @@ func RunKubectlOrDieInput(data string, args ...string) string { return NewKubectlCommand(args...).WithStdinData(data).ExecOrDie() } +// RunKubemciCmd is a convenience wrapper over kubectlBuilder to run kubemci. +// It assumes that kubemci exists in PATH. +func RunKubemciCmd(args ...string) (string, error) { + // kubemci is assumed to be in PATH. + kubemci := "kubemci" + b := new(kubectlBuilder) + if TestContext.KubeConfig != "" { + args = append(args, "--"+clientcmd.RecommendedConfigPathFlag+"="+TestContext.KubeConfig) + } + args = append(args, "--gcp-project="+TestContext.CloudConfig.ProjectID) + + b.cmd = exec.Command(kubemci, args...) + return b.Exec() +} + func StartCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) { stdout, err = cmd.StdoutPipe() if err != nil { diff --git a/test/e2e/manifest/manifest.go b/test/e2e/manifest/manifest.go index b5e77860782..01d7aa1ff90 100644 --- a/test/e2e/manifest/manifest.go +++ b/test/e2e/manifest/manifest.go @@ -17,12 +17,16 @@ limitations under the License. package manifest import ( + "fmt" + "io/ioutil" + apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/test/e2e/generated" ) @@ -87,6 +91,20 @@ func IngressFromManifest(fileName string) (*extensions.Ingress, error) { return &ing, nil } +// IngressToManifest generates a yaml file in the given path with the given ingress. +// Returns the file name and error, if there was any. +func IngressToManifest(ing *extensions.Ingress, path string) error { + serialized, err := util.MarshalToYaml(ing, extensions.SchemeGroupVersion) + if err != nil { + return fmt.Errorf("failed to marshal ingress %v to YAML: %v", ing, err) + } + + if err := ioutil.WriteFile(path, serialized, 0600); err != nil { + return fmt.Errorf("error in writing ingress to file: %s", err) + } + return nil +} + // StatefulSetFromManifest returns a StatefulSet from a manifest stored in fileName in the Namespace indicated by ns. func StatefulSetFromManifest(fileName, ns string) (*apps.StatefulSet, error) { var ss apps.StatefulSet diff --git a/test/e2e/manifest/manifest_test.go b/test/e2e/manifest/manifest_test.go new file mode 100644 index 00000000000..ae2145218a0 --- /dev/null +++ b/test/e2e/manifest/manifest_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manifest + +import ( + "testing" + + extensions "k8s.io/api/extensions/v1beta1" +) + +func TestIngressToManifest(t *testing.T) { + ing := &extensions.Ingress{} + // Write the ingress to a file and ensure that there is no error. + if err := IngressToManifest(ing, "/tmp/ing.yaml"); err != nil { + t.Fatalf("Error in creating file: %s", err) + } + // Writing it again should not return an error. + if err := IngressToManifest(ing, "/tmp/ing.yaml"); err != nil { + t.Fatalf("Error in creating file: %s", err) + } +} diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index e91b8daac30..2d961ee9e61 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -566,6 +566,38 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { }) }) + Describe("GCE [Slow] [Feature:kubemci]", func() { + // Platform specific setup + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce", "gke") + jig.Class = framework.MulticlusterIngressClassValue + }) + + // Platform specific cleanup + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + framework.DescribeIng(ns) + } + if jig.Ingress == nil { + By("No ingress created, no cleanup necessary") + return + } + By("Deleting ingress") + jig.TryDeleteIngress() + }) + + It("should conform to Ingress spec", func() { + jig.PollInterval = 5 * time.Second + conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{}) + for _, t := range conformanceTests { + By(t.EntryLog) + t.Execute() + By(t.ExitLog) + jig.WaitForIngress(true /*waitForNodePort*/) + } + }) + }) + // Time: borderline 5m, slow by design Describe("[Slow] Nginx", func() { var nginxController *framework.NginxIngressController