diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index b7d937d7b1b..ec305a17e9d 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -79,7 +79,7 @@ func setupProviderConfig() error { managedZones = []string{zone} } - gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{}) + gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{gcecloud.AlphaFeatureNetworkEndpointGroup}) if err != nil { glog.Errorf("Encountered error for creating alpha feature gate: %v", err) } diff --git a/test/e2e/framework/ingress_utils.go b/test/e2e/framework/ingress_utils.go index bbb5223ebec..666fafe60b4 100644 --- a/test/e2e/framework/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -126,7 +126,7 @@ type IngressConformanceTests struct { // CreateIngressComformanceTests generates an slice of sequential test cases: // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname, // ingress https with a modified URLMap -func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConformanceTests { +func CreateIngressComformanceTests(jig *IngressTestJig, ns string, annotations map[string]string) []IngressConformanceTests { manifestPath := filepath.Join(IngressManifestPath, "http") // These constants match the manifests used in IngressManifestPath tlsHost := "foo.bar.com" @@ -138,7 +138,7 @@ func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConf return []IngressConformanceTests{ { fmt.Sprintf("should create a basic HTTP ingress"), - func() { jig.CreateIngress(manifestPath, ns, map[string]string{}) }, + func() { jig.CreateIngress(manifestPath, ns, annotations, annotations) }, fmt.Sprintf("waiting for urls on basic HTTP ingress"), }, { @@ -591,6 +591,39 @@ func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) { return msg } +func (cont *GCEIngressController) deleteNetworkEndpointGroup(del bool) (msg string) { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + // TODO: E2E cloudprovider has only 1 zone, but the cluster can have many. + // We need to poll on all NEGs across all zones. + negList, err := gceCloud.ListNetworkEndpointGroup(cont.Cloud.Zone) + if err != nil { + if cont.isHTTPErrorCode(err, http.StatusNotFound) { + return msg + } + // Do not return error as NEG is still alpha. + Logf("Failed to list network endpoint group: %v", err) + return msg + } + if len(negList) == 0 { + return msg + } + for _, neg := range negList { + if !cont.canDeleteNEG(neg.Name, neg.CreationTimestamp, del) { + continue + } + if del { + Logf("Deleting network-endpoint-group: %s", neg.Name) + if err := gceCloud.DeleteNetworkEndpointGroup(neg.Name, cont.Cloud.Zone); err != nil && + !cont.isHTTPErrorCode(err, http.StatusNotFound) { + msg += fmt.Sprintf("Failed to delete network endpoint group %v\n", neg.Name) + } + } else { + msg += fmt.Sprintf("%v (network-endpoint-group)\n", neg.Name) + } + } + return msg +} + // canDelete returns true if either the name ends in a suffix matching this // controller's UID, or the creationTimestamp exceeds the maxAge and del is set // to true. Always returns false if the name doesn't match that we expect for @@ -617,6 +650,28 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri if !delOldResources { return false } + return canDeleteWithTimestamp(resourceName, creationTimestamp) +} + +// canDeleteNEG returns true if either the name contains this controller's UID, +// or the creationTimestamp exceeds the maxAge and del is set to true. +func (cont *GCEIngressController) canDeleteNEG(resourceName, creationTimestamp string, delOldResources bool) bool { + if !strings.HasPrefix(resourceName, "k8s") { + return false + } + + if strings.Contains(resourceName, cont.UID) { + return true + } + + if !delOldResources { + return false + } + + return canDeleteWithTimestamp(resourceName, creationTimestamp) +} + +func canDeleteWithTimestamp(resourceName, creationTimestamp string) bool { createdTime, err := time.Parse(time.RFC3339, creationTimestamp) if err != nil { Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err) @@ -667,6 +722,44 @@ func (cont *GCEIngressController) isHTTPErrorCode(err error, code int) bool { return ok && apiErr.Code == code } +// BackendServiceUsingNEG returns true only if all global backend service with matching nodeports pointing to NEG as backend +func (cont *GCEIngressController) BackendServiceUsingNEG(nodeports []string) (bool, error) { + return cont.backendMode(nodeports, "networkEndpointGroups") +} + +// BackendServiceUsingIG returns true only if all global backend service with matching nodeports pointing to IG as backend +func (cont *GCEIngressController) BackendServiceUsingIG(nodeports []string) (bool, error) { + return cont.backendMode(nodeports, "instanceGroups") +} + +func (cont *GCEIngressController) backendMode(nodeports []string, keyword string) (bool, error) { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + beList, err := gceCloud.ListGlobalBackendServices() + if err != nil { + return false, fmt.Errorf("failed to list backend services: %v", err) + } + + matchingBackendService := 0 + for _, bs := range beList.Items { + match := false + for _, np := range nodeports { + // Warning: This assumes backend service naming convention includes nodeport in the name + if strings.Contains(bs.Name, np) { + match = true + matchingBackendService += 1 + } + } + if match { + for _, be := range bs.Backends { + if !strings.Contains(be.Group, keyword) { + return false, nil + } + } + } + } + return matchingBackendService == len(nodeports), nil +} + // Cleanup cleans up cloud resources. // If del is false, it simply reports existing resources without deleting them. // If dle is true, it deletes resources it finds acceptable (see canDelete func). @@ -683,6 +776,7 @@ func (cont *GCEIngressController) Cleanup(del bool) error { errMsg += cont.deleteHTTPHealthCheck(del) errMsg += cont.deleteInstanceGroup(del) + errMsg += cont.deleteNetworkEndpointGroup(del) errMsg += cont.deleteFirewallRule(del) errMsg += cont.deleteSSLCertificate(del) @@ -812,7 +906,9 @@ func GcloudComputeResourceCreate(resource, name, project string, args ...string) // Required: ing.yaml, rc.yaml, svc.yaml must exist in manifestPath // Optional: secret.yaml, ingAnnotations // If ingAnnotations is specified it will overwrite any annotations in ing.yaml -func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string) { +// If svcAnnotations is specified it will overwrite any annotations in svc.yaml +func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string, svcAnnotations map[string]string) { + var err error mkpath := func(file string) string { return filepath.Join(TestContext.RepoRoot, manifestPath, file) } @@ -822,13 +918,22 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m Logf("creating service") RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns)) + if len(svcAnnotations) > 0 { + svcList, err := j.Client.CoreV1().Services(ns).List(metav1.ListOptions{}) + ExpectNoError(err) + for _, svc := range svcList.Items { + svc.Annotations = svcAnnotations + _, err = j.Client.CoreV1().Services(ns).Update(&svc) + ExpectNoError(err) + } + } if exists, _ := utilfile.FileExists(mkpath("secret.yaml")); exists { Logf("creating secret") RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns)) } Logf("Parsing ingress from %v", filepath.Join(manifestPath, "ing.yaml")) - var err error + j.Ingress, err = manifest.IngressFromManifest(filepath.Join(manifestPath, "ing.yaml")) ExpectNoError(err) j.Ingress.Namespace = ns @@ -954,14 +1059,16 @@ func (j *IngressTestJig) pollServiceNodePort(ns, name string, port int) { ExpectNoError(PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false)) } -// GetIngressNodePorts returns all related backend services' nodePorts. +// GetIngressNodePorts returns related backend services' nodePorts. // Current GCE ingress controller allows traffic to the default HTTP backend -// by default, so retrieve its nodePort as well. -func (j *IngressTestJig) GetIngressNodePorts() []string { +// by default, so retrieve its nodePort if includeDefaultBackend is true. +func (j *IngressTestJig) GetIngressNodePorts(includeDefaultBackend bool) []string { nodePorts := []string{} - defaultSvc, err := j.Client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - nodePorts = append(nodePorts, strconv.Itoa(int(defaultSvc.Spec.Ports[0].NodePort))) + if includeDefaultBackend { + defaultSvc, err := j.Client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + nodePorts = append(nodePorts, strconv.Itoa(int(defaultSvc.Spec.Ports[0].NodePort))) + } backendSvcs := []string{} if j.Ingress.Spec.Backend != nil { @@ -982,7 +1089,7 @@ func (j *IngressTestJig) GetIngressNodePorts() []string { // ConstructFirewallForIngress returns the expected GCE firewall rule for the ingress resource func (j *IngressTestJig) ConstructFirewallForIngress(gceController *GCEIngressController, nodeTags []string) *compute.Firewall { - nodePorts := j.GetIngressNodePorts() + nodePorts := j.GetIngressNodePorts(true) fw := compute.Firewall{} fw.Name = gceController.GetFirewallRuleName() diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go index de16b0cf6e8..f408b325d74 100644 --- a/test/e2e/network/ingress.go +++ b/test/e2e/network/ingress.go @@ -30,6 +30,10 @@ import ( . "github.com/onsi/gomega" ) +const ( + NEGAnnotation = "alpha.cloud.google.com/load-balancer-neg" +) + var _ = SIGDescribe("Loadbalancing: L7", func() { defer GinkgoRecover() var ( @@ -96,7 +100,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { }) It("should conform to Ingress spec", func() { - conformanceTests = framework.CreateIngressComformanceTests(jig, ns) + conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{}) for _, t := range conformanceTests { By(t.EntryLog) t.Execute() @@ -113,7 +117,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{ "kubernetes.io/ingress.global-static-ip-name": ns, "kubernetes.io/ingress.allow-http": "false", - }) + }, map[string]string{}) By("waiting for Ingress to come up with ip: " + ip) httpClient := framework.BuildInsecureClient(framework.IngressReqTimeout) @@ -149,6 +153,53 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { // TODO: Implement a multizone e2e that verifies traffic reaches each // zone based on pod labels. }) + Describe("GCE [Slow] [Feature:NEG]", func() { + var gceController *framework.GCEIngressController + + // Platform specific setup + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce", "gke") + By("Initializing gce controller") + gceController = &framework.GCEIngressController{ + Ns: ns, + Client: jig.Client, + Cloud: framework.TestContext.CloudConfig, + } + gceController.Init() + }) + + // Platform specific cleanup + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + framework.DescribeIng(ns) + } + if jig.Ingress == nil { + By("No ingress created, no cleanup necessary") + return + } + By("Deleting ingress") + jig.TryDeleteIngress() + + By("Cleaning up cloud resources") + framework.CleanupGCEIngressController(gceController) + }) + + It("should conform to Ingress spec", func() { + jig.PollInterval = 5 * time.Second + conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{ + NEGAnnotation: "true", + }) + for _, t := range conformanceTests { + By(t.EntryLog) + t.Execute() + By(t.ExitLog) + jig.WaitForIngress(true) + usingNeg, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false)) + Expect(err).NotTo(HaveOccurred()) + Expect(usingNeg).To(BeTrue()) + } + }) + }) // Time: borderline 5m, slow by design Describe("[Slow] Nginx", func() { @@ -191,7 +242,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() { // Poll more frequently to reduce e2e completion time. // This test runs in presubmit. jig.PollInterval = 5 * time.Second - conformanceTests = framework.CreateIngressComformanceTests(jig, ns) + conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{}) for _, t := range conformanceTests { By(t.EntryLog) t.Execute() diff --git a/test/e2e/upgrades/ingress.go b/test/e2e/upgrades/ingress.go index 7ca9e3b7d13..d2d8c71369a 100644 --- a/test/e2e/upgrades/ingress.go +++ b/test/e2e/upgrades/ingress.go @@ -69,7 +69,7 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) { jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{ "kubernetes.io/ingress.global-static-ip-name": t.ipName, "kubernetes.io/ingress.allow-http": "false", - }) + }, map[string]string{}) By("waiting for Ingress to come up with ip: " + t.ip) framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false))