From 9dc98f529669c7152b17285c9675271603b746d4 Mon Sep 17 00:00:00 2001 From: Nick Sardo Date: Wed, 22 Feb 2017 18:08:37 -0800 Subject: [PATCH] upgrade tests: ingress/glbc --- test/e2e/BUILD | 2 - test/e2e/cluster_upgrade.go | 1 + test/e2e/framework/BUILD | 2 + test/e2e/{ => framework}/ingress_utils.go | 458 ++++++++++++---------- test/e2e/ingress.go | 106 ++--- test/e2e/upgrades/BUILD | 1 + test/e2e/upgrades/ingress.go | 122 ++++++ 7 files changed, 409 insertions(+), 283 deletions(-) rename test/e2e/{ => framework}/ingress_utils.go (59%) create mode 100644 test/e2e/upgrades/ingress.go diff --git a/test/e2e/BUILD b/test/e2e/BUILD index e619ccbc816..731820f1d5b 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -48,7 +48,6 @@ go_library( "ha_master.go", "horizontal_pod_autoscaling.go", "ingress.go", - "ingress_utils.go", "initial_resources.go", "job.go", "kibana_logging.go", @@ -179,7 +178,6 @@ go_library( "//vendor:golang.org/x/net/context", "//vendor:golang.org/x/net/websocket", "//vendor:golang.org/x/oauth2/google", - "//vendor:google.golang.org/api/compute/v1", "//vendor:google.golang.org/api/googleapi", "//vendor:google.golang.org/api/logging/v2beta1", "//vendor:k8s.io/apimachinery/pkg/api/errors", diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index 6b419867954..aba1c877790 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -39,6 +39,7 @@ var upgradeTests = []upgrades.Test{ &upgrades.HPAUpgradeTest{}, &upgrades.PersistentVolumeUpgradeTest{}, &upgrades.DaemonSetUpgradeTest{}, + &upgrades.IngressUpgradeTest{}, } var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 747ec870ef1..20c926cf215 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -18,6 +18,7 @@ go_library( "framework.go", "get-kubemark-resource-usage.go", "google_compute.go", + "ingress_utils.go", "jobs_util.go", "kubelet_stats.go", "log_size_monitoring.go", @@ -71,6 +72,7 @@ go_library( "//pkg/master/ports:go_default_library", "//pkg/metrics:go_default_library", "//pkg/ssh:go_default_library", + "//pkg/util:go_default_library", "//pkg/util/exec:go_default_library", "//pkg/util/labels:go_default_library", "//pkg/util/system:go_default_library", diff --git a/test/e2e/ingress_utils.go b/test/e2e/framework/ingress_utils.go similarity index 59% rename from test/e2e/ingress_utils.go rename to test/e2e/framework/ingress_utils.go index 7ec47f61824..b582e103ad9 100644 --- a/test/e2e/ingress_utils.go +++ b/test/e2e/framework/ingress_utils.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e +package framework import ( "bytes" @@ -31,16 +31,12 @@ import ( "math/big" "net" "net/http" - "os" "os/exec" "path/filepath" "strconv" "strings" "time" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/api/v1" - compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -51,11 +47,13 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/v1" extensions "k8s.io/kubernetes/pkg/apis/extensions/v1beta1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" + "k8s.io/kubernetes/pkg/util" utilexec "k8s.io/kubernetes/pkg/util/exec" - "k8s.io/kubernetes/test/e2e/framework" testutils "k8s.io/kubernetes/test/utils" . "github.com/onsi/ginkgo" @@ -80,59 +78,84 @@ const ( // Name of the default http backend service defaultBackendName = "default-http-backend" - // IP src range from which the GCE L7 performs health checks. + // GCEL7SrcRange is the IP src range from which the GCE L7 performs health checks. GCEL7SrcRange = "130.211.0.0/22" // Cloud resources created by the ingress controller older than this // are automatically purged to prevent running out of quota. // TODO(37335): write soak tests and bump this up to a week. maxAge = -48 * time.Hour + + // IngressManifestPath is the parent path to yaml test manifests. + IngressManifestPath = "test/e2e/testing-manifests/ingress" + + // IngressReqTimeout is the timeout on a single http request. + IngressReqTimeout = 10 * time.Second + + // healthz port used to verify glbc restarted correctly on the master. + glbcHealthzPort = 8086 + + // General cloud resource poll timeout (eg: create static ip, firewall etc) + cloudResourcePollTimeout = 5 * time.Minute + + // Name of the config-map and key the ingress controller stores its uid in. + uidConfigMap = "ingress-uid" + uidKey = "uid" + + // GCE only allows names < 64 characters, and the loadbalancer controller inserts + // a single character of padding. + nameLenLimit = 62 ) -type testJig struct { - client clientset.Interface - rootCAs map[string][]byte - address string - ing *extensions.Ingress +// IngressTestJig holds the relevant state and parameters of the ingress test. +type IngressTestJig struct { + Client clientset.Interface + RootCAs map[string][]byte + Address string + Ingress *extensions.Ingress // class is the value of the annotation keyed under // `kubernetes.io/ingress.class`. It's added to all ingresses created by // this jig. - class string + Class string // The interval used to poll urls - pollInterval time.Duration + PollInterval time.Duration } -type conformanceTests struct { - entryLog string - execute func() - exitLog string +// IngressConformanceTests contains a closure with an entry and exit log line. +type IngressConformanceTests struct { + EntryLog string + Execute func() + ExitLog string } -func createComformanceTests(jig *testJig, ns string) []conformanceTests { - manifestPath := filepath.Join(ingressManifestPath, "http") - // These constants match the manifests used in ingressManifestPath +// CreateIngressComformanceTests generates an slice of sequential test cases: +// a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname, +// ingress https with a modified URLMap +func CreateIngressComformanceTests(jig *IngressTestJig, ns string) []IngressConformanceTests { + manifestPath := filepath.Join(IngressManifestPath, "http") + // These constants match the manifests used in IngressManifestPath tlsHost := "foo.bar.com" tlsSecretName := "foo" updatedTLSHost := "foobar.com" updateURLMapHost := "bar.baz.com" updateURLMapPath := "/testurl" // Platform agnostic list of tests that must be satisfied by all controllers - return []conformanceTests{ + return []IngressConformanceTests{ { fmt.Sprintf("should create a basic HTTP ingress"), - func() { jig.createIngress(manifestPath, ns, map[string]string{}) }, + func() { jig.CreateIngress(manifestPath, ns, map[string]string{}) }, fmt.Sprintf("waiting for urls on basic HTTP ingress"), }, { fmt.Sprintf("should terminate TLS for host %v", tlsHost), - func() { jig.addHTTPS(tlsSecretName, tlsHost) }, + func() { jig.AddHTTPS(tlsSecretName, tlsHost) }, fmt.Sprintf("waiting for HTTPS updates to reflect in ingress"), }, { fmt.Sprintf("should update SSL certificate with modified hostname %v", updatedTLSHost), func() { - jig.update(func(ing *extensions.Ingress) { + jig.Update(func(ing *extensions.Ingress) { newRules := []extensions.IngressRule{} for _, rule := range ing.Spec.Rules { if rule.Host != tlsHost { @@ -146,7 +169,7 @@ func createComformanceTests(jig *testJig, ns string) []conformanceTests { } ing.Spec.Rules = newRules }) - jig.addHTTPS(tlsSecretName, updatedTLSHost) + jig.AddHTTPS(tlsSecretName, updatedTLSHost) }, fmt.Sprintf("Waiting for updated certificates to accept requests for host %v", updatedTLSHost), }, @@ -154,7 +177,7 @@ func createComformanceTests(jig *testJig, ns string) []conformanceTests { fmt.Sprintf("should update url map for host %v to expose a single url: %v", updateURLMapHost, updateURLMapPath), func() { var pathToFail string - jig.update(func(ing *extensions.Ingress) { + jig.Update(func(ing *extensions.Ingress) { newRules := []extensions.IngressRule{} for _, rule := range ing.Spec.Rules { if rule.Host != updateURLMapHost { @@ -180,8 +203,8 @@ func createComformanceTests(jig *testJig, ns string) []conformanceTests { ing.Spec.Rules = newRules }) By("Checking that " + pathToFail + " is not exposed by polling for failure") - route := fmt.Sprintf("http://%v%v", jig.address, pathToFail) - framework.ExpectNoError(framework.PollURL(route, updateURLMapHost, framework.LoadBalancerCleanupTimeout, jig.pollInterval, &http.Client{Timeout: reqTimeout}, true)) + route := fmt.Sprintf("http://%v%v", jig.Address, pathToFail) + ExpectNoError(PollURL(route, updateURLMapHost, LoadBalancerCleanupTimeout, jig.PollInterval, &http.Client{Timeout: IngressReqTimeout}, true)) }, fmt.Sprintf("Waiting for path updates to reflect in L7"), }, @@ -248,13 +271,13 @@ func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error { return nil } -// buildTransport creates a transport for use in executing HTTPS requests with +// buildTransportWithCA creates a transport for use in executing HTTPS requests with // the given certs. Note that the given rootCA must be configured with isCA=true. -func buildTransport(serverName string, rootCA []byte) (*http.Transport, error) { +func buildTransportWithCA(serverName string, rootCA []byte) (*http.Transport, error) { pool := x509.NewCertPool() ok := pool.AppendCertsFromPEM(rootCA) if !ok { - return nil, fmt.Errorf("Unable to load serverCA.") + return nil, fmt.Errorf("Unable to load serverCA") } return utilnet.SetTransportDefaults(&http.Transport{ TLSClientConfig: &tls.Config{ @@ -265,20 +288,20 @@ func buildTransport(serverName string, rootCA []byte) (*http.Transport, error) { }), nil } -// buildInsecureClient returns an insecure http client. Can be used for "curl -k". -func buildInsecureClient(timeout time.Duration) *http.Client { +// BuildInsecureClient returns an insecure http client. Can be used for "curl -k". +func BuildInsecureClient(timeout time.Duration) *http.Client { t := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} return &http.Client{Timeout: timeout, Transport: utilnet.SetTransportDefaults(t)} } -// createSecret creates a secret containing TLS certificates for the given Ingress. -// If a secret with the same name already exists in the namespace of the +// createIngressTLSSecret creates a secret containing TLS certificates for the given Ingress. +// If a secret with the same name already pathExists in the namespace of the // Ingress, it's updated. -func createSecret(kubeClient clientset.Interface, ing *extensions.Ingress) (host string, rootCA, privKey []byte, err error) { +func createIngressTLSSecret(kubeClient clientset.Interface, ing *extensions.Ingress) (host string, rootCA, privKey []byte, err error) { var k, c bytes.Buffer tls := ing.Spec.TLS[0] host = strings.Join(tls.Hosts, ",") - framework.Logf("Generating RSA cert for host %v", host) + Logf("Generating RSA cert for host %v", host) if err = generateRSACerts(host, true, &k, &c); err != nil { return @@ -297,20 +320,22 @@ func createSecret(kubeClient clientset.Interface, ing *extensions.Ingress) (host var s *v1.Secret if s, err = kubeClient.Core().Secrets(ing.Namespace).Get(tls.SecretName, metav1.GetOptions{}); err == nil { // TODO: Retry the update. We don't really expect anything to conflict though. - framework.Logf("Updating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name) + Logf("Updating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name) s.Data = secret.Data _, err = kubeClient.Core().Secrets(ing.Namespace).Update(s) } else { - framework.Logf("Creating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name) + Logf("Creating secret %v in ns %v with hosts %v for ingress %v", secret.Name, secret.Namespace, host, ing.Name) _, err = kubeClient.Core().Secrets(ing.Namespace).Create(secret) } return host, cert, key, err } -func cleanupGCE(gceController *GCEIngressController) { - pollErr := wait.Poll(5*time.Second, framework.LoadBalancerCleanupTimeout, func() (bool, error) { +// CleanupGCEIngressController calls the GCEIngressController.Cleanup(false) +// followed with deleting the static ip, and then a final GCEIngressController.Cleanup(true) +func CleanupGCEIngressController(gceController *GCEIngressController) { + pollErr := wait.Poll(5*time.Second, LoadBalancerCleanupTimeout, func() (bool, error) { if err := gceController.Cleanup(false); err != nil { - framework.Logf("Still waiting for glbc to cleanup:\n%v", err) + Logf("Still waiting for glbc to cleanup:\n%v", err) return false, nil } return true, nil @@ -320,9 +345,9 @@ func cleanupGCE(gceController *GCEIngressController) { // controller. Delete this IP only after the controller has had a chance // to cleanup or it might interfere with the controller, causing it to // throw out confusing events. - if ipErr := wait.Poll(5*time.Second, framework.LoadBalancerCleanupTimeout, func() (bool, error) { + if ipErr := wait.Poll(5*time.Second, LoadBalancerCleanupTimeout, func() (bool, error) { if err := gceController.deleteStaticIPs(); err != nil { - framework.Logf("Failed to delete static-ip: %v\n", err) + Logf("Failed to delete static-ip: %v\n", err) return false, nil } return true, nil @@ -342,7 +367,7 @@ func cleanupGCE(gceController *GCEIngressController) { // Fail if the controller didn't cleanup if pollErr != nil { - framework.Failf("L7 controller failed to delete all cloud resources on time. %v", pollErr) + Failf("L7 controller failed to delete all cloud resources on time. %v", pollErr) } } @@ -350,7 +375,7 @@ func (cont *GCEIngressController) deleteForwardingRule(del bool) string { msg := "" fwList := []compute.ForwardingRule{} for _, regex := range []string{fmt.Sprintf("%vfw-.*%v.*", k8sPrefix, clusterDelimiter), fmt.Sprintf("%vfws-.*%v.*", k8sPrefix, clusterDelimiter)} { - gcloudList("forwarding-rules", regex, cont.cloud.ProjectID, &fwList) + gcloudComputeResourceList("forwarding-rules", regex, cont.Cloud.ProjectID, &fwList) if len(fwList) == 0 { continue } @@ -360,7 +385,7 @@ func (cont *GCEIngressController) deleteForwardingRule(del bool) string { } msg += fmt.Sprintf("%v (forwarding rule)\n", f.Name) if del { - gcloudDelete("forwarding-rules", f.Name, cont.cloud.ProjectID, "--global") + GcloudComputeResourceDelete("forwarding-rules", f.Name, cont.Cloud.ProjectID, "--global") } } } @@ -371,7 +396,7 @@ func (cont *GCEIngressController) deleteAddresses(del bool) string { msg := "" ipList := []compute.Address{} regex := fmt.Sprintf("%vfw-.*%v.*", k8sPrefix, clusterDelimiter) - gcloudList("addresses", regex, cont.cloud.ProjectID, &ipList) + gcloudComputeResourceList("addresses", regex, cont.Cloud.ProjectID, &ipList) if len(ipList) != 0 { for _, ip := range ipList { if !cont.canDelete(ip.Name, ip.CreationTimestamp, del) { @@ -379,7 +404,7 @@ func (cont *GCEIngressController) deleteAddresses(del bool) string { } msg += fmt.Sprintf("%v (static-ip)\n", ip.Name) if del { - gcloudDelete("addresses", ip.Name, cont.cloud.ProjectID, "--global") + GcloudComputeResourceDelete("addresses", ip.Name, cont.Cloud.ProjectID, "--global") } } } @@ -390,7 +415,7 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string { msg := "" tpList := []compute.TargetHttpProxy{} regex := fmt.Sprintf("%vtp-.*%v.*", k8sPrefix, clusterDelimiter) - gcloudList("target-http-proxies", regex, cont.cloud.ProjectID, &tpList) + gcloudComputeResourceList("target-http-proxies", regex, cont.Cloud.ProjectID, &tpList) if len(tpList) != 0 { for _, t := range tpList { if !cont.canDelete(t.Name, t.CreationTimestamp, del) { @@ -398,13 +423,13 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string { } msg += fmt.Sprintf("%v (target-http-proxy)\n", t.Name) if del { - gcloudDelete("target-http-proxies", t.Name, cont.cloud.ProjectID) + GcloudComputeResourceDelete("target-http-proxies", t.Name, cont.Cloud.ProjectID) } } } tpsList := []compute.TargetHttpsProxy{} regex = fmt.Sprintf("%vtps-.*%v.*", k8sPrefix, clusterDelimiter) - gcloudList("target-https-proxies", regex, cont.cloud.ProjectID, &tpsList) + gcloudComputeResourceList("target-https-proxies", regex, cont.Cloud.ProjectID, &tpsList) if len(tpsList) != 0 { for _, t := range tpsList { if !cont.canDelete(t.Name, t.CreationTimestamp, del) { @@ -412,15 +437,15 @@ func (cont *GCEIngressController) deleteTargetProxy(del bool) string { } msg += fmt.Sprintf("%v (target-https-proxy)\n", t.Name) if del { - gcloudDelete("target-https-proxies", t.Name, cont.cloud.ProjectID) + GcloudComputeResourceDelete("target-https-proxies", t.Name, cont.Cloud.ProjectID) } } } return msg } -func (cont *GCEIngressController) deleteUrlMap(del bool) (msg string) { - gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud) +func (cont *GCEIngressController) deleteURLMap(del bool) (msg string) { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) umList, err := gceCloud.ListUrlMaps() if err != nil { if cont.isHTTPErrorCode(err, http.StatusNotFound) { @@ -447,7 +472,7 @@ func (cont *GCEIngressController) deleteUrlMap(del bool) (msg string) { } func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) { - gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud) + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) beList, err := gceCloud.ListBackendServices() if err != nil { if cont.isHTTPErrorCode(err, http.StatusNotFound) { @@ -456,7 +481,7 @@ func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) { return fmt.Sprintf("Failed to list backend services: %v", err) } if len(beList.Items) == 0 { - framework.Logf("No backend services found") + Logf("No backend services found") return msg } for _, be := range beList.Items { @@ -474,8 +499,8 @@ func (cont *GCEIngressController) deleteBackendService(del bool) (msg string) { return msg } -func (cont *GCEIngressController) deleteHttpHealthCheck(del bool) (msg string) { - gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud) +func (cont *GCEIngressController) deleteHTTPHealthCheck(del bool) (msg string) { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) hcList, err := gceCloud.ListHttpHealthChecks() if err != nil { if cont.isHTTPErrorCode(err, http.StatusNotFound) { @@ -502,7 +527,7 @@ func (cont *GCEIngressController) deleteHttpHealthCheck(del bool) (msg string) { } func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) { - gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud) + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) sslList, err := gceCloud.ListSslCertificates() if err != nil { if cont.isHTTPErrorCode(err, http.StatusNotFound) { @@ -528,10 +553,10 @@ func (cont *GCEIngressController) deleteSSLCertificate(del bool) (msg string) { } func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) { - gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud) + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) // TODO: E2E cloudprovider has only 1 zone, but the cluster can have many. // We need to poll on all IGs across all zones. - igList, err := gceCloud.ListInstanceGroups(cont.cloud.Zone) + igList, err := gceCloud.ListInstanceGroups(cont.Cloud.Zone) if err != nil { if cont.isHTTPErrorCode(err, http.StatusNotFound) { return msg @@ -547,7 +572,7 @@ func (cont *GCEIngressController) deleteInstanceGroup(del bool) (msg string) { } msg += fmt.Sprintf("%v (instance-group)\n", ig.Name) if del { - if err := gceCloud.DeleteInstanceGroup(ig.Name, cont.cloud.Zone); err != nil && + if err := gceCloud.DeleteInstanceGroup(ig.Name, cont.Cloud.Zone); err != nil && !cont.isHTTPErrorCode(err, http.StatusNotFound) { msg += fmt.Sprintf("Failed to delete instance group %v\n", ig.Name) } @@ -574,23 +599,25 @@ func (cont *GCEIngressController) canDelete(resourceName, creationTimestamp stri } createdTime, err := time.Parse(time.RFC3339, creationTimestamp) if err != nil { - framework.Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err) + Logf("WARNING: Failed to parse creation timestamp %v for %v: %v", creationTimestamp, resourceName, err) return false } if createdTime.Before(time.Now().Add(maxAge)) { - framework.Logf("%v created on %v IS too old", resourceName, creationTimestamp) + Logf("%v created on %v IS too old", resourceName, creationTimestamp) return true } return false } -func (cont *GCEIngressController) getFirewallRuleName() string { +// GetFirewallRuleName returns the name of the firewall used for the GCEIngressController. +func (cont *GCEIngressController) GetFirewallRuleName() string { return fmt.Sprintf("%vfw-l7%v%v", k8sPrefix, clusterDelimiter, cont.UID) } -func (cont *GCEIngressController) getFirewallRule() *compute.Firewall { - gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud) - fwName := cont.getFirewallRuleName() +// GetFirewallRule returns the firewall used by the GCEIngressController. +func (cont *GCEIngressController) GetFirewallRule() *compute.Firewall { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) + fwName := cont.GetFirewallRuleName() fw, err := gceCloud.GetFirewall(fwName) Expect(err).NotTo(HaveOccurred()) return fw @@ -599,7 +626,7 @@ func (cont *GCEIngressController) getFirewallRule() *compute.Firewall { func (cont *GCEIngressController) deleteFirewallRule(del bool) (msg string) { fwList := []compute.Firewall{} regex := fmt.Sprintf("%vfw-l7%v.*", k8sPrefix, clusterDelimiter) - gcloudList("firewall-rules", regex, cont.cloud.ProjectID, &fwList) + gcloudComputeResourceList("firewall-rules", regex, cont.Cloud.ProjectID, &fwList) if len(fwList) != 0 { for _, f := range fwList { if !cont.canDelete(f.Name, f.CreationTimestamp, del) { @@ -607,7 +634,7 @@ func (cont *GCEIngressController) deleteFirewallRule(del bool) (msg string) { } msg += fmt.Sprintf("%v (firewall rule)\n", f.Name) if del { - gcloudDelete("firewall-rules", f.Name, cont.cloud.ProjectID) + GcloudComputeResourceDelete("firewall-rules", f.Name, cont.Cloud.ProjectID) } } } @@ -631,9 +658,9 @@ func (cont *GCEIngressController) Cleanup(del bool) error { errMsg += cont.deleteAddresses(del) errMsg += cont.deleteTargetProxy(del) - errMsg += cont.deleteUrlMap(del) + errMsg += cont.deleteURLMap(del) errMsg += cont.deleteBackendService(del) - errMsg += cont.deleteHttpHealthCheck(del) + errMsg += cont.deleteHTTPHealthCheck(del) errMsg += cont.deleteInstanceGroup(del) errMsg += cont.deleteFirewallRule(del) @@ -648,63 +675,64 @@ func (cont *GCEIngressController) Cleanup(del bool) error { return fmt.Errorf(errMsg) } -func (cont *GCEIngressController) init() { +// Init initializes the GCEIngressController with an UID +func (cont *GCEIngressController) Init() { uid, err := cont.getL7AddonUID() Expect(err).NotTo(HaveOccurred()) cont.UID = uid // There's a name limit imposed by GCE. The controller will truncate. - testName := fmt.Sprintf("k8s-fw-foo-app-X-%v--%v", cont.ns, cont.UID) + testName := fmt.Sprintf("k8s-fw-foo-app-X-%v--%v", cont.Ns, cont.UID) if len(testName) > nameLenLimit { - framework.Logf("WARNING: test name including cluster UID: %v is over the GCE limit of %v", testName, nameLenLimit) + Logf("WARNING: test name including cluster UID: %v is over the GCE limit of %v", testName, nameLenLimit) } else { - framework.Logf("Detected cluster UID %v", cont.UID) + Logf("Detected cluster UID %v", cont.UID) } } -// createStaticIP allocates a random static ip with the given name. Returns a string +// CreateStaticIP allocates a random static ip with the given name. Returns a string // representation of the ip. Caller is expected to manage cleanup of the ip by // invoking deleteStaticIPs. -func (cont *GCEIngressController) createStaticIP(name string) string { - gceCloud := cont.cloud.Provider.(*gcecloud.GCECloud) +func (cont *GCEIngressController) CreateStaticIP(name string) string { + gceCloud := cont.Cloud.Provider.(*gcecloud.GCECloud) ip, err := gceCloud.ReserveGlobalStaticIP(name, "") if err != nil { if delErr := gceCloud.DeleteGlobalStaticIP(name); delErr != nil { if cont.isHTTPErrorCode(delErr, http.StatusNotFound) { - framework.Logf("Static ip with name %v was not allocated, nothing to delete", name) + Logf("Static ip with name %v was not allocated, nothing to delete", name) } else { - framework.Logf("Failed to delete static ip %v: %v", name, delErr) + Logf("Failed to delete static ip %v: %v", name, delErr) } } - framework.Failf("Failed to allocated static ip %v: %v", name, err) + Failf("Failed to allocated static ip %v: %v", name, err) } cont.staticIPName = ip.Name - framework.Logf("Reserved static ip %v: %v", cont.staticIPName, ip.Address) + Logf("Reserved static ip %v: %v", cont.staticIPName, ip.Address) return ip.Address } -// deleteStaticIPs deletes all static-ips allocated through calls to -// createStaticIP. +// deleteStaticIPs delets all static-ips allocated through calls to +// CreateStaticIP. func (cont *GCEIngressController) deleteStaticIPs() error { if cont.staticIPName != "" { - if err := gcloudDelete("addresses", cont.staticIPName, cont.cloud.ProjectID, "--global"); err == nil { + if err := GcloudComputeResourceDelete("addresses", cont.staticIPName, cont.Cloud.ProjectID, "--global"); err == nil { cont.staticIPName = "" } else { return err } } else { e2eIPs := []compute.Address{} - gcloudList("addresses", "e2e-.*", cont.cloud.ProjectID, &e2eIPs) + gcloudComputeResourceList("addresses", "e2e-.*", cont.Cloud.ProjectID, &e2eIPs) ips := []string{} for _, ip := range e2eIPs { ips = append(ips, ip.Name) } - framework.Logf("None of the remaining %d static-ips were created by this e2e: %v", len(ips), strings.Join(ips, ", ")) + Logf("None of the remaining %d static-ips were created by this e2e: %v", len(ips), strings.Join(ips, ", ")) } return nil } -// gcloudList unmarshals json output of gcloud into given out interface. -func gcloudList(resource, regex, project string, out interface{}) { +// gcloudComputeResourceList unmarshals json output of gcloud into given out interface. +func gcloudComputeResourceList(resource, regex, project string, out interface{}) { // gcloud prints a message to stderr if it has an available update // so we only look at stdout. command := []string{ @@ -724,204 +752,210 @@ func gcloudList(resource, regex, project string, out interface{}) { errMsg = fmt.Sprintf("%v, stderr %v", errMsg, string(osExitErr.Stderr)) } } - framework.Logf("Error running gcloud command 'gcloud %s': err: %v, output: %v, status: %d, msg: %v", strings.Join(command, " "), err, string(output), errCode, errMsg) + Logf("Error running gcloud command 'gcloud %s': err: %v, output: %v, status: %d, msg: %v", strings.Join(command, " "), err, string(output), errCode, errMsg) } if err := json.Unmarshal([]byte(output), out); err != nil { - framework.Logf("Error unmarshalling gcloud output for %v: %v, output: %v", resource, err, string(output)) + Logf("Error unmarshalling gcloud output for %v: %v, output: %v", resource, err, string(output)) } } -func gcloudDelete(resource, name, project string, args ...string) error { - framework.Logf("Deleting %v: %v", resource, name) +// GcloudComputeResourceDelete deletes the specified compute resource by name and project. +func GcloudComputeResourceDelete(resource, name, project string, args ...string) error { + Logf("Deleting %v: %v", resource, name) argList := append([]string{"compute", resource, "delete", name, fmt.Sprintf("--project=%v", project), "-q"}, args...) output, err := exec.Command("gcloud", argList...).CombinedOutput() if err != nil { - framework.Logf("Error deleting %v, output: %v\nerror: %+v", resource, string(output), err) + Logf("Error deleting %v, output: %v\nerror: %+v", resource, string(output), err) } return err } -func gcloudCreate(resource, name, project string, args ...string) error { - framework.Logf("Creating %v in project %v: %v", resource, project, name) +// GcloudComputeResourceCreate creates a compute resource with a name and arguments. +func GcloudComputeResourceCreate(resource, name, project string, args ...string) error { + Logf("Creating %v in project %v: %v", resource, project, name) argsList := append([]string{"compute", resource, "create", name, fmt.Sprintf("--project=%v", project)}, args...) - framework.Logf("Running command: gcloud %+v", strings.Join(argsList, " ")) + Logf("Running command: gcloud %+v", strings.Join(argsList, " ")) output, err := exec.Command("gcloud", argsList...).CombinedOutput() if err != nil { - framework.Logf("Error creating %v, output: %v\nerror: %+v", resource, string(output), err) + Logf("Error creating %v, output: %v\nerror: %+v", resource, string(output), err) } return err } -// createIngress creates the Ingress and associated service/rc. +// CreateIngress creates the Ingress and associated service/rc. // Required: ing.yaml, rc.yaml, svc.yaml must exist in manifestPath // Optional: secret.yaml, ingAnnotations // If ingAnnotations is specified it will overwrite any annotations in ing.yaml -func (j *testJig) createIngress(manifestPath, ns string, ingAnnotations map[string]string) { +func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string) { mkpath := func(file string) string { - return filepath.Join(framework.TestContext.RepoRoot, manifestPath, file) + return filepath.Join(TestContext.RepoRoot, manifestPath, file) } - framework.Logf("creating replication controller") - framework.RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", ns)) + Logf("creating replication controller") + RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", ns)) - framework.Logf("creating service") - framework.RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns)) + Logf("creating service") + RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns)) - if exists(mkpath("secret.yaml")) { - framework.Logf("creating secret") - framework.RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns)) + if exists, _ := util.FileExists(mkpath("secret.yaml")); exists { + Logf("creating secret") + RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns)) } - j.ing = ingFromManifest(mkpath("ing.yaml")) - j.ing.Namespace = ns - j.ing.Annotations = map[string]string{ingressClass: j.class} + j.Ingress = createIngressFromManifest(mkpath("ing.yaml")) + j.Ingress.Namespace = ns + j.Ingress.Annotations = map[string]string{ingressClass: j.Class} for k, v := range ingAnnotations { - j.ing.Annotations[k] = v + j.Ingress.Annotations[k] = v } - framework.Logf(fmt.Sprintf("creating" + j.ing.Name + " ingress")) + Logf(fmt.Sprintf("creating" + j.Ingress.Name + " ingress")) var err error - j.ing, err = j.client.Extensions().Ingresses(ns).Create(j.ing) - framework.ExpectNoError(err) + j.Ingress, err = j.Client.Extensions().Ingresses(ns).Create(j.Ingress) + ExpectNoError(err) } -func (j *testJig) update(update func(ing *extensions.Ingress)) { +// Update retrieves the ingress, performs the passed function, and then updates it. +func (j *IngressTestJig) Update(update func(ing *extensions.Ingress)) { var err error - ns, name := j.ing.Namespace, j.ing.Name + ns, name := j.Ingress.Namespace, j.Ingress.Name for i := 0; i < 3; i++ { - j.ing, err = j.client.Extensions().Ingresses(ns).Get(name, metav1.GetOptions{}) + j.Ingress, err = j.Client.Extensions().Ingresses(ns).Get(name, metav1.GetOptions{}) if err != nil { - framework.Failf("failed to get ingress %q: %v", name, err) + Failf("failed to get ingress %q: %v", name, err) } - update(j.ing) - j.ing, err = j.client.Extensions().Ingresses(ns).Update(j.ing) + update(j.Ingress) + j.Ingress, err = j.Client.Extensions().Ingresses(ns).Update(j.Ingress) if err == nil { - framework.DescribeIng(j.ing.Namespace) + DescribeIng(j.Ingress.Namespace) return } if !apierrs.IsConflict(err) && !apierrs.IsServerTimeout(err) { - framework.Failf("failed to update ingress %q: %v", name, err) + Failf("failed to update ingress %q: %v", name, err) } } - framework.Failf("too many retries updating ingress %q", name) + Failf("too many retries updating ingress %q", name) } -func (j *testJig) addHTTPS(secretName string, hosts ...string) { - j.ing.Spec.TLS = []extensions.IngressTLS{{Hosts: hosts, SecretName: secretName}} - // TODO: Just create the secret in getRootCAs once we're watching secrets in +// AddHTTPS updates the ingress to use this secret for these hosts. +func (j *IngressTestJig) AddHTTPS(secretName string, hosts ...string) { + j.Ingress.Spec.TLS = []extensions.IngressTLS{{Hosts: hosts, SecretName: secretName}} + // TODO: Just create the secret in GetRootCAs once we're watching secrets in // the ingress controller. - _, cert, _, err := createSecret(j.client, j.ing) - framework.ExpectNoError(err) - framework.Logf("Updating ingress %v to use secret %v for TLS termination", j.ing.Name, secretName) - j.update(func(ing *extensions.Ingress) { + _, cert, _, err := createIngressTLSSecret(j.Client, j.Ingress) + ExpectNoError(err) + Logf("Updating ingress %v to use secret %v for TLS termination", j.Ingress.Name, secretName) + j.Update(func(ing *extensions.Ingress) { ing.Spec.TLS = []extensions.IngressTLS{{Hosts: hosts, SecretName: secretName}} }) - j.rootCAs[secretName] = cert + j.RootCAs[secretName] = cert } -func (j *testJig) getRootCA(secretName string) (rootCA []byte) { +// GetRootCA returns a rootCA from the ingress test jig. +func (j *IngressTestJig) GetRootCA(secretName string) (rootCA []byte) { var ok bool - rootCA, ok = j.rootCAs[secretName] + rootCA, ok = j.RootCAs[secretName] if !ok { - framework.Failf("Failed to retrieve rootCAs, no recorded secret by name %v", secretName) + Failf("Failed to retrieve rootCAs, no recorded secret by name %v", secretName) } return } -func (j *testJig) deleteIngress() { - framework.ExpectNoError(j.client.Extensions().Ingresses(j.ing.Namespace).Delete(j.ing.Name, nil)) +// DeleteIngress deletes the ingress resource +func (j *IngressTestJig) DeleteIngress() { + ExpectNoError(j.Client.Extensions().Ingresses(j.Ingress.Namespace).Delete(j.Ingress.Name, nil)) } -// waitForIngress waits till the ingress acquires an IP, then waits for its +// WaitForIngress waits till the ingress acquires an IP, then waits for its // hosts/urls to respond to a protocol check (either http or https). If // waitForNodePort is true, the NodePort of the Service is verified before // verifying the Ingress. NodePort is currently a requirement for cloudprovider // Ingress. -func (j *testJig) waitForIngress(waitForNodePort bool) { +func (j *IngressTestJig) WaitForIngress(waitForNodePort bool) { // Wait for the loadbalancer IP. - address, err := framework.WaitForIngressAddress(j.client, j.ing.Namespace, j.ing.Name, framework.LoadBalancerPollTimeout) + address, err := WaitForIngressAddress(j.Client, j.Ingress.Namespace, j.Ingress.Name, LoadBalancerPollTimeout) if err != nil { - framework.Failf("Ingress failed to acquire an IP address within %v", framework.LoadBalancerPollTimeout) + Failf("Ingress failed to acquire an IP address within %v", LoadBalancerPollTimeout) } - j.address = address - framework.Logf("Found address %v for ingress %v", j.address, j.ing.Name) - timeoutClient := &http.Client{Timeout: reqTimeout} + j.Address = address + Logf("Found address %v for ingress %v", j.Address, j.Ingress.Name) + timeoutClient := &http.Client{Timeout: IngressReqTimeout} // Check that all rules respond to a simple GET. - for _, rules := range j.ing.Spec.Rules { + for _, rules := range j.Ingress.Spec.Rules { proto := "http" - if len(j.ing.Spec.TLS) > 0 { - knownHosts := sets.NewString(j.ing.Spec.TLS[0].Hosts...) + if len(j.Ingress.Spec.TLS) > 0 { + knownHosts := sets.NewString(j.Ingress.Spec.TLS[0].Hosts...) if knownHosts.Has(rules.Host) { - timeoutClient.Transport, err = buildTransport(rules.Host, j.getRootCA(j.ing.Spec.TLS[0].SecretName)) - framework.ExpectNoError(err) + timeoutClient.Transport, err = buildTransportWithCA(rules.Host, j.GetRootCA(j.Ingress.Spec.TLS[0].SecretName)) + ExpectNoError(err) proto = "https" } } for _, p := range rules.IngressRuleValue.HTTP.Paths { if waitForNodePort { - j.curlServiceNodePort(j.ing.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal)) + j.pollServiceNodePort(j.Ingress.Namespace, p.Backend.ServiceName, int(p.Backend.ServicePort.IntVal)) } route := fmt.Sprintf("%v://%v%v", proto, address, p.Path) - framework.Logf("Testing route %v host %v with simple GET", route, rules.Host) - framework.ExpectNoError(framework.PollURL(route, rules.Host, framework.LoadBalancerPollTimeout, j.pollInterval, timeoutClient, false)) + Logf("Testing route %v host %v with simple GET", route, rules.Host) + ExpectNoError(PollURL(route, rules.Host, LoadBalancerPollTimeout, j.PollInterval, timeoutClient, false)) } } } -// verifyURL polls for the given iterations, in intervals, and fails if the +// VerifyURL polls for the given iterations, in intervals, and fails if the // given url returns a non-healthy http code even once. -func (j *testJig) verifyURL(route, host string, iterations int, interval time.Duration, httpClient *http.Client) error { +func (j *IngressTestJig) VerifyURL(route, host string, iterations int, interval time.Duration, httpClient *http.Client) error { for i := 0; i < iterations; i++ { - b, err := framework.SimpleGET(httpClient, route, host) + b, err := SimpleGET(httpClient, route, host) if err != nil { - framework.Logf(b) + Logf(b) return err } - framework.Logf("Verfied %v with host %v %d times, sleeping for %v", route, host, i, interval) + Logf("Verfied %v with host %v %d times, sleeping for %v", route, host, i, interval) time.Sleep(interval) } return nil } -func (j *testJig) curlServiceNodePort(ns, name string, port int) { +func (j *IngressTestJig) pollServiceNodePort(ns, name string, port int) { // TODO: Curl all nodes? - u, err := framework.GetNodePortURL(j.client, ns, name, port) - framework.ExpectNoError(err) - framework.ExpectNoError(framework.PollURL(u, "", 30*time.Second, j.pollInterval, &http.Client{Timeout: reqTimeout}, false)) + u, err := GetNodePortURL(j.Client, ns, name, port) + ExpectNoError(err) + ExpectNoError(PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false)) } -// getIngressNodePorts returns all related backend services' nodePorts. +// GetIngressNodePorts returns all related backend services' nodePorts. // Current GCE ingress controller allows traffic to the default HTTP backend // by default, so retrieve its nodePort as well. -func (j *testJig) getIngressNodePorts() []string { +func (j *IngressTestJig) GetIngressNodePorts() []string { nodePorts := []string{} - defaultSvc, err := j.client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{}) + defaultSvc, err := j.Client.Core().Services(metav1.NamespaceSystem).Get(defaultBackendName, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) nodePorts = append(nodePorts, strconv.Itoa(int(defaultSvc.Spec.Ports[0].NodePort))) backendSvcs := []string{} - if j.ing.Spec.Backend != nil { - backendSvcs = append(backendSvcs, j.ing.Spec.Backend.ServiceName) + if j.Ingress.Spec.Backend != nil { + backendSvcs = append(backendSvcs, j.Ingress.Spec.Backend.ServiceName) } - for _, rule := range j.ing.Spec.Rules { + for _, rule := range j.Ingress.Spec.Rules { for _, ingPath := range rule.HTTP.Paths { backendSvcs = append(backendSvcs, ingPath.Backend.ServiceName) } } for _, svcName := range backendSvcs { - svc, err := j.client.Core().Services(j.ing.Namespace).Get(svcName, metav1.GetOptions{}) + svc, err := j.Client.Core().Services(j.Ingress.Namespace).Get(svcName, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) nodePorts = append(nodePorts, strconv.Itoa(int(svc.Spec.Ports[0].NodePort))) } return nodePorts } -// constructFirewallForIngress returns the expected GCE firewall rule for the ingress resource -func (j *testJig) constructFirewallForIngress(gceController *GCEIngressController) *compute.Firewall { - nodeTags := framework.GetNodeTags(j.client, gceController.cloud) - nodePorts := j.getIngressNodePorts() +// ConstructFirewallForIngress returns the expected GCE firewall rule for the ingress resource +func (j *IngressTestJig) ConstructFirewallForIngress(gceController *GCEIngressController) *compute.Firewall { + nodeTags := GetNodeTags(j.Client, gceController.Cloud) + nodePorts := j.GetIngressNodePorts() fw := compute.Firewall{} - fw.Name = gceController.getFirewallRuleName() + fw.Name = gceController.GetFirewallRuleName() fw.SourceRanges = []string{GCEL7SrcRange} fw.TargetTags = nodeTags.Items fw.Allowed = []*compute.FirewallAllowed{ @@ -933,23 +967,23 @@ func (j *testJig) constructFirewallForIngress(gceController *GCEIngressControlle return &fw } -// ingFromManifest reads a .json/yaml file and returns the rc in it. -func ingFromManifest(fileName string) *extensions.Ingress { +// createIngressFromManifest reads a .json/yaml file and returns the rc in it. +func createIngressFromManifest(fileName string) *extensions.Ingress { var ing extensions.Ingress - framework.Logf("Parsing ingress from %v", fileName) + Logf("Parsing ingress from %v", fileName) data, err := ioutil.ReadFile(fileName) - framework.ExpectNoError(err) + ExpectNoError(err) json, err := utilyaml.ToJSON(data) - framework.ExpectNoError(err) + ExpectNoError(err) - framework.ExpectNoError(runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &ing)) + ExpectNoError(runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &ing)) return &ing } func (cont *GCEIngressController) getL7AddonUID() (string, error) { - framework.Logf("Retrieving UID from config map: %v/%v", metav1.NamespaceSystem, uidConfigMap) - cm, err := cont.c.Core().ConfigMaps(metav1.NamespaceSystem).Get(uidConfigMap, metav1.GetOptions{}) + Logf("Retrieving UID from config map: %v/%v", metav1.NamespaceSystem, uidConfigMap) + cm, err := cont.Client.Core().ConfigMaps(metav1.NamespaceSystem).Get(uidConfigMap, metav1.GetOptions{}) if err != nil { return "", err } @@ -959,64 +993,54 @@ func (cont *GCEIngressController) getL7AddonUID() (string, error) { return "", fmt.Errorf("Could not find cluster UID for L7 addon pod") } -func exists(path string) bool { - _, err := os.Stat(path) - if err == nil { - return true - } - if os.IsNotExist(err) { - return false - } - framework.Failf("Failed to os.Stat path %v", path) - return false -} - // GCEIngressController manages implementation details of Ingress on GCE/GKE. type GCEIngressController struct { - ns string + Ns string rcPath string UID string staticIPName string rc *v1.ReplicationController svc *v1.Service - c clientset.Interface - cloud framework.CloudConfig + Client clientset.Interface + Cloud CloudConfig } -func newTestJig(c clientset.Interface) *testJig { - return &testJig{client: c, rootCAs: map[string][]byte{}, pollInterval: framework.LoadBalancerPollInterval} +// NewIngressTestJig instantiates struct with client +func NewIngressTestJig(c clientset.Interface) *IngressTestJig { + return &IngressTestJig{Client: c, RootCAs: map[string][]byte{}, PollInterval: LoadBalancerPollInterval} } // NginxIngressController manages implementation details of Ingress on Nginx. type NginxIngressController struct { - ns string + Ns string rc *v1.ReplicationController pod *v1.Pod - c clientset.Interface + Client clientset.Interface externalIP string } -func (cont *NginxIngressController) init() { +// Init initializes the NginxIngressController +func (cont *NginxIngressController) Init() { mkpath := func(file string) string { - return filepath.Join(framework.TestContext.RepoRoot, ingressManifestPath, "nginx", file) + return filepath.Join(TestContext.RepoRoot, IngressManifestPath, "nginx", file) } - framework.Logf("initializing nginx ingress controller") - framework.RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", cont.ns)) + Logf("initializing nginx ingress controller") + RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", cont.Ns)) - rc, err := cont.c.Core().ReplicationControllers(cont.ns).Get("nginx-ingress-controller", metav1.GetOptions{}) - framework.ExpectNoError(err) + rc, err := cont.Client.Core().ReplicationControllers(cont.Ns).Get("nginx-ingress-controller", metav1.GetOptions{}) + ExpectNoError(err) cont.rc = rc - framework.Logf("waiting for pods with label %v", rc.Spec.Selector) + Logf("waiting for pods with label %v", rc.Spec.Selector) sel := labels.SelectorFromSet(labels.Set(rc.Spec.Selector)) - framework.ExpectNoError(testutils.WaitForPodsWithLabelRunning(cont.c, cont.ns, sel)) - pods, err := cont.c.Core().Pods(cont.ns).List(metav1.ListOptions{LabelSelector: sel.String()}) - framework.ExpectNoError(err) + ExpectNoError(testutils.WaitForPodsWithLabelRunning(cont.Client, cont.Ns, sel)) + pods, err := cont.Client.Core().Pods(cont.Ns).List(metav1.ListOptions{LabelSelector: sel.String()}) + ExpectNoError(err) if len(pods.Items) == 0 { - framework.Failf("Failed to find nginx ingress controller pods with selector %v", sel) + Failf("Failed to find nginx ingress controller pods with selector %v", sel) } cont.pod = &pods.Items[0] - cont.externalIP, err = framework.GetHostExternalAddress(cont.c, cont.pod) - framework.ExpectNoError(err) - framework.Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP) + cont.externalIP, err = GetHostExternalAddress(cont.Client, cont.pod) + ExpectNoError(err) + Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP) } diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 151fe112b07..92eefcfecf1 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -30,48 +30,26 @@ import ( . "github.com/onsi/gomega" ) -const ( - // parent path to yaml test manifests. - ingressManifestPath = "test/e2e/testing-manifests/ingress" - - // timeout on a single http request. - reqTimeout = 10 * time.Second - - // healthz port used to verify glbc restarted correctly on the master. - glbcHealthzPort = 8086 - - // General cloud resource poll timeout (eg: create static ip, firewall etc) - cloudResourcePollTimeout = 5 * time.Minute - - // Name of the config-map and key the ingress controller stores its uid in. - uidConfigMap = "ingress-uid" - uidKey = "uid" - - // GCE only allows names < 64 characters, and the loadbalancer controller inserts - // a single character of padding. - nameLenLimit = 62 -) - var _ = framework.KubeDescribe("Loadbalancing: L7", func() { defer GinkgoRecover() var ( ns string - jig *testJig - conformanceTests []conformanceTests + jig *framework.IngressTestJig + conformanceTests []framework.IngressConformanceTests ) f := framework.NewDefaultFramework("ingress") BeforeEach(func() { f.BeforeEach() - jig = newTestJig(f.ClientSet) + jig = framework.NewIngressTestJig(f.ClientSet) ns = f.Namespace.Name // this test wants powerful permissions. Since the namespace names are unique, we can leave this // lying around so we don't have to race any caches - framework.BindClusterRole(jig.client.Rbac(), "cluster-admin", f.Namespace.Name, + framework.BindClusterRole(jig.Client.Rbac(), "cluster-admin", f.Namespace.Name, rbacv1beta1.Subject{Kind: rbacv1beta1.ServiceAccountKind, Namespace: f.Namespace.Name, Name: "default"}) - err := framework.WaitForAuthorizationUpdate(jig.client.AuthorizationV1beta1(), + err := framework.WaitForAuthorizationUpdate(jig.Client.AuthorizationV1beta1(), serviceaccount.MakeUsername(f.Namespace.Name, "default"), "", "create", schema.GroupResource{Resource: "pods"}, true) framework.ExpectNoError(err) @@ -85,18 +63,18 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() { // Slow by design ~10m for each "It" block dominated by loadbalancer setup time // TODO: write similar tests for nginx, haproxy and AWS Ingress. framework.KubeDescribe("GCE [Slow] [Feature:Ingress]", func() { - var gceController *GCEIngressController + var gceController *framework.GCEIngressController // Platform specific setup BeforeEach(func() { framework.SkipUnlessProviderIs("gce", "gke") By("Initializing gce controller") - gceController = &GCEIngressController{ - ns: ns, - c: jig.client, - cloud: framework.TestContext.CloudConfig, + gceController = &framework.GCEIngressController{ + Ns: ns, + Client: jig.Client, + Cloud: framework.TestContext.CloudConfig, } - gceController.init() + gceController.Init() }) // Platform specific cleanup @@ -104,51 +82,51 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() { if CurrentGinkgoTestDescription().Failed { framework.DescribeIng(ns) } - if jig.ing == nil { + if jig.Ingress == nil { By("No ingress created, no cleanup necessary") return } By("Deleting ingress") - jig.deleteIngress() + jig.DeleteIngress() By("Cleaning up cloud resources") - cleanupGCE(gceController) + framework.CleanupGCEIngressController(gceController) }) It("should conform to Ingress spec", func() { - conformanceTests = createComformanceTests(jig, ns) + conformanceTests = framework.CreateIngressComformanceTests(jig, ns) for _, t := range conformanceTests { - By(t.entryLog) - t.execute() - By(t.exitLog) - jig.waitForIngress(true) + By(t.EntryLog) + t.Execute() + By(t.ExitLog) + jig.WaitForIngress(true) } }) It("should create ingress with given static-ip", func() { - // ip released when the rest of lb resources are deleted in cleanupGCE - ip := gceController.createStaticIP(ns) + // ip released when the rest of lb resources are deleted in CleanupGCEIngressController + ip := gceController.CreateStaticIP(ns) By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ns, ip)) - jig.createIngress(filepath.Join(ingressManifestPath, "static-ip"), ns, map[string]string{ + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{ "kubernetes.io/ingress.global-static-ip-name": ns, "kubernetes.io/ingress.allow-http": "false", }) By("waiting for Ingress to come up with ip: " + ip) - httpClient := buildInsecureClient(reqTimeout) - framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", ip), "", framework.LoadBalancerPollTimeout, jig.pollInterval, httpClient, false)) + httpClient := framework.BuildInsecureClient(framework.IngressReqTimeout) + framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, httpClient, false)) By("should reject HTTP traffic") - framework.ExpectNoError(framework.PollURL(fmt.Sprintf("http://%v/", ip), "", framework.LoadBalancerPollTimeout, jig.pollInterval, httpClient, true)) + framework.ExpectNoError(framework.PollURL(fmt.Sprintf("http://%v/", ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, httpClient, true)) By("should have correct firewall rule for ingress") - fw := gceController.getFirewallRule() - expFw := jig.constructFirewallForIngress(gceController) + fw := gceController.GetFirewallRule() + expFw := jig.ConstructFirewallForIngress(gceController) // Passed the last argument as `true` to verify the backend ports is a subset // of the allowed ports in firewall rule, given there may be other existing // ingress resources and backends we are not aware of. - Expect(framework.VerifyFirewallRule(fw, expFw, gceController.cloud.Network, true)).NotTo(HaveOccurred()) + Expect(framework.VerifyFirewallRule(fw, expFw, gceController.Cloud.Network, true)).NotTo(HaveOccurred()) // TODO: uncomment the restart test once we have a way to synchronize // and know that the controller has resumed watching. If we delete @@ -167,51 +145,51 @@ var _ = framework.KubeDescribe("Loadbalancing: L7", func() { // Time: borderline 5m, slow by design framework.KubeDescribe("Nginx", func() { - var nginxController *NginxIngressController + var nginxController *framework.NginxIngressController BeforeEach(func() { framework.SkipUnlessProviderIs("gce", "gke") By("Initializing nginx controller") - jig.class = "nginx" - nginxController = &NginxIngressController{ns: ns, c: jig.client} + jig.Class = "nginx" + nginxController = &framework.NginxIngressController{Ns: ns, Client: jig.Client} // TODO: This test may fail on other platforms. We can simply skip it // but we want to allow easy testing where a user might've hand // configured firewalls. if framework.ProviderIs("gce", "gke") { - framework.ExpectNoError(gcloudCreate("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID, "--allow", "tcp:80,tcp:443", "--network", framework.TestContext.CloudConfig.Network)) + framework.ExpectNoError(framework.GcloudComputeResourceCreate("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID, "--allow", "tcp:80,tcp:443", "--network", framework.TestContext.CloudConfig.Network)) } else { framework.Logf("WARNING: Not running on GCE/GKE, cannot create firewall rules for :80, :443. Assuming traffic can reach the external ips of all nodes in cluster on those ports.") } - nginxController.init() + nginxController.Init() }) AfterEach(func() { if framework.ProviderIs("gce", "gke") { - framework.ExpectNoError(gcloudDelete("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID)) + framework.ExpectNoError(framework.GcloudComputeResourceDelete("firewall-rules", fmt.Sprintf("ingress-80-443-%v", ns), framework.TestContext.CloudConfig.ProjectID)) } if CurrentGinkgoTestDescription().Failed { framework.DescribeIng(ns) } - if jig.ing == nil { + if jig.Ingress == nil { By("No ingress created, no cleanup necessary") return } By("Deleting ingress") - jig.deleteIngress() + jig.DeleteIngress() }) It("should conform to Ingress spec", func() { // Poll more frequently to reduce e2e completion time. // This test runs in presubmit. - jig.pollInterval = 5 * time.Second - conformanceTests = createComformanceTests(jig, ns) + jig.PollInterval = 5 * time.Second + conformanceTests = framework.CreateIngressComformanceTests(jig, ns) for _, t := range conformanceTests { - By(t.entryLog) - t.execute() - By(t.exitLog) - jig.waitForIngress(false) + By(t.EntryLog) + t.Execute() + By(t.ExitLog) + jig.WaitForIngress(false) } }) }) diff --git a/test/e2e/upgrades/BUILD b/test/e2e/upgrades/BUILD index e939012ede7..742b3508345 100644 --- a/test/e2e/upgrades/BUILD +++ b/test/e2e/upgrades/BUILD @@ -14,6 +14,7 @@ go_library( "daemonsets.go", "deployments.go", "horizontal_pod_autoscalers.go", + "ingress.go", "job.go", "persistent_volumes.go", "secrets.go", diff --git a/test/e2e/upgrades/ingress.go b/test/e2e/upgrades/ingress.go new file mode 100644 index 00000000000..f1a060352d9 --- /dev/null +++ b/test/e2e/upgrades/ingress.go @@ -0,0 +1,122 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrades + +import ( + "fmt" + "net/http" + "path/filepath" + + . "github.com/onsi/ginkgo" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" +) + +// IngressUpgradeTest adapts the Ingress e2e for upgrade testing +type IngressUpgradeTest struct { + gceController *framework.GCEIngressController + jig *framework.IngressTestJig + httpClient *http.Client + ip string +} + +// Setup creates a GLBC, allocates an ip, and an ingress resource, +// then waits for a successful connectivity check to the ip. +func (t *IngressUpgradeTest) Setup(f *framework.Framework) { + framework.SkipUnlessProviderIs("gce", "gke") + + // jig handles all Kubernetes testing logic + jig := framework.NewIngressTestJig(f.ClientSet) + + ns, err := f.CreateNamespace("ingress-upgrade", nil) + framework.ExpectNoError(err) + + // gceController handles all cloud testing logic + gceController := &framework.GCEIngressController{ + Ns: ns.Name, + Client: jig.Client, + Cloud: framework.TestContext.CloudConfig, + } + gceController.Init() + + t.gceController = gceController + t.jig = jig + t.httpClient = framework.BuildInsecureClient(framework.IngressReqTimeout) + + // Allocate a static-ip for the Ingress, this IP is cleaned up via CleanupGCEIngressController + t.ip = t.gceController.CreateStaticIP(ns.Name) + + // Create a working basic Ingress + By(fmt.Sprintf("allocated static ip %v: %v through the GCE cloud provider", ns.Name, t.ip)) + jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{ + "kubernetes.io/ingress.global-static-ip-name": ns.Name, + "kubernetes.io/ingress.allow-http": "false", + }) + + By("waiting for Ingress to come up with ip: " + t.ip) + framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false)) +} + +// Test waits for the upgrade to complete, and then verifies +// with a connectvity check to the loadbalancer ip. +func (t *IngressUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade UpgradeType) { + switch upgrade { + case MasterUpgrade: + // Restarting the ingress controller shouldn't disrupt a steady state + // Ingress. Restarting the ingress controller and deleting ingresses + // while it's down will leak cloud resources, because the ingress + // controller doesn't checkpoint to disk. + t.verify(f, done, true) + default: + // Currently ingress gets disrupted across node upgrade, because endpoints + // get killed and we don't have any guarantees that 2 nodes don't overlap + // their upgrades (even on cloud platforms like GCE, because VM level + // rolling upgrades are not Kubernetes aware). + t.verify(f, done, false) + } +} + +// Teardown cleans up any remaining resources. +func (t *IngressUpgradeTest) Teardown(f *framework.Framework) { + if CurrentGinkgoTestDescription().Failed { + framework.DescribeIng(t.gceController.Ns) + } + if t.jig.Ingress == nil { + By("No ingress created, no cleanup necessary") + return + } + By("Deleting ingress") + t.jig.DeleteIngress() + + By("Cleaning up cloud resources") + framework.CleanupGCEIngressController(t.gceController) +} + +func (t *IngressUpgradeTest) verify(f *framework.Framework, done <-chan struct{}, testDuringDisruption bool) { + if testDuringDisruption { + By("continuously hitting the Ingress IP") + wait.Until(func() { + framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, t.jig.PollInterval, t.httpClient, false)) + }, t.jig.PollInterval, done) + } else { + By("waiting for upgrade to finish without checking if Ingress remains up") + <-done + } + By("hitting the Ingress IP " + t.ip) + framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, t.jig.PollInterval, t.httpClient, false)) +}