diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index a2039e63f85..41590b5557c 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -124,6 +124,7 @@ GKE_REQUIRED_SKIP_TESTS=( # Tests which cannot be run on AWS. AWS_REQUIRED_SKIP_TESTS=( "experimental\sresource\susage\stracking" # Expect --max-pods=100 + "GCE\sL7\sLoadBalancer\sController" # GCE L7 loadbalancing ) @@ -153,6 +154,11 @@ GCE_FLAKY_TESTS=( # comments below, and for poorly implemented tests, please quote the # issue number tracking speed improvements. GCE_SLOW_TESTS=( + # Before enabling this loadbalancer test in any other test list you must + # make sure the associated project has enough quota. At the time of this + # writing a GCE project is allowed 3 backend services by default. This + # test requires at least 5. + "GCE\sL7\sLoadBalancer\sController" # 10 min, file: ingress.go, slow by design "SchedulerPredicates\svalidates\sMaxPods\slimit " # 8 min, file: scheduler_predicates.go, PR: #13315 "Nodes\sResize" # 3 min 30 sec, file: resize_nodes.go, issue: #13323 "resource\susage\stracking" # 1 hour, file: kubelet_perf.go, slow by design @@ -164,6 +170,7 @@ GCE_SLOW_TESTS=( # Tests which are not able to be run in parallel. GCE_PARALLEL_SKIP_TESTS=( + "GCE\sL7\sLoadBalancer\sController" # TODO: This cannot run in parallel with other L4 tests till quota has been bumped up. "Nodes\sNetwork" "MaxPods" "Resource\susage\sof\ssystem\scontainers" diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go new file mode 100644 index 00000000000..0d378b95f26 --- /dev/null +++ b/test/e2e/ingress.go @@ -0,0 +1,321 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/json" + "fmt" + "net/http" + "os/exec" + "sort" + "time" + + compute "google.golang.org/api/compute/v1" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/extensions" + client "k8s.io/kubernetes/pkg/client/unversioned" + "k8s.io/kubernetes/pkg/fields" + "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/util" + "k8s.io/kubernetes/pkg/util/wait" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +// Before enabling this test you must make sure the associated project has +// enough quota. At the time of this writing GCE projects are allowed 3 +// backend services by default. This test requires at least 5. + +// This test exercises the GCE L7 loadbalancer controller cluster-addon. It +// will fail if the addon isn't running, or doesn't send traffic to the expected +// backend. Common failure modes include: +// * GCE L7 took too long to spin up +// * GCE L7 took too long to health check a backend +// * Repeated 404: +// - L7 is sending traffic to the default backend of the addon. +// - Backend is receiving /foo when it expects /bar. +// * Repeated 5xx: +// - Out of quota (describe ing should show you if this is the case) +// - Mismatched service/container port, or endpoints are dead. + +var ( + appPrefix = "foo-app-" + pathPrefix = "foo" + testImage = "gcr.io/google_containers/n-way-http:1.0" + httpContainerPort = 8080 + + expectedLBCreationTime = 7 * time.Minute + expectedLBHealthCheckTime = 7 * time.Minute + + // On average it takes ~6 minutes for a single backend to come online. + // We *don't* expect this poll to consistently take 15 minutes for every + // Ingress as GCE is creating/checking backends in parallel, but at the + // same time, we're not testing GCE startup latency. So give it enough + // time, and fail if the average is too high. + lbPollTimeout = 15 * time.Minute + lbPollInterval = 30 * time.Second + + // One can scale this test by tweaking numApps and numIng, the former will + // create more RCs/Services and add them to a single Ingress, while the latter + // will create smaller, more fragmented Ingresses. The numbers 4, 2 are chosen + // arbitrarity, we want to test more than a single Ingress, and it should have + // more than 1 url endpoint going to a service. + numApps = 4 + numIng = 2 +) + +// timeSlice allows sorting of time.Duration +type timeSlice []time.Duration + +func (p timeSlice) Len() int { + return len(p) +} + +func (p timeSlice) Less(i, j int) bool { + return p[i] < p[j] +} + +func (p timeSlice) Swap(i, j int) { + p[i], p[j] = p[j], p[i] +} + +// ruleByIndex returns an IngressRule for the given index. +func ruleByIndex(i int) extensions.IngressRule { + return extensions.IngressRule{ + Host: fmt.Sprintf("foo%d.bar.com", i), + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: fmt.Sprintf("/%v%d", pathPrefix, i), + Backend: extensions.IngressBackend{ + ServiceName: fmt.Sprintf("%v%d", appPrefix, i), + ServicePort: util.NewIntOrStringFromInt(httpContainerPort), + }, + }, + }, + }, + }, + } +} + +// createIngress creates an Ingress with num rules. Eg: +// start = 1 num = 2 will given you a single Ingress with 2 rules: +// Ingress { +// foo1.bar.com: /foo1 +// foo2.bar.com: /foo2 +// } +func createIngress(c *client.Client, ns string, start, num int) extensions.Ingress { + ing := extensions.Ingress{ + ObjectMeta: api.ObjectMeta{ + Name: fmt.Sprintf("%v%d", appPrefix, start), + Namespace: ns, + }, + Spec: extensions.IngressSpec{ + Backend: &extensions.IngressBackend{ + ServiceName: fmt.Sprintf("%v%d", appPrefix, start), + ServicePort: util.NewIntOrStringFromInt(httpContainerPort), + }, + Rules: []extensions.IngressRule{}, + }, + } + for i := start; i < start+num; i++ { + ing.Spec.Rules = append(ing.Spec.Rules, ruleByIndex(i)) + } + Logf("Creating ingress %v", start) + _, err := c.Extensions().Ingress(ns).Create(&ing) + Expect(err).NotTo(HaveOccurred()) + return ing +} + +// createApp will create a single RC and Svc. The Svc will match pods of the +// RC using the selector: 'name'= +func createApp(c *client.Client, ns string, i int) { + name := fmt.Sprintf("%v%d", appPrefix, i) + l := map[string]string{} + + Logf("Creating svc %v", name) + svc := svcByName(name, httpContainerPort) + svc.Spec.Type = api.ServiceTypeNodePort + _, err := c.Services(ns).Create(svc) + Expect(err).NotTo(HaveOccurred()) + + Logf("Creating rc %v", name) + rc := rcByNamePort(name, 1, testImage, httpContainerPort, l) + rc.Spec.Template.Spec.Containers[0].Args = []string{ + "--num=1", + fmt.Sprintf("--start=%d", i), + fmt.Sprintf("--prefix=%v", pathPrefix), + fmt.Sprintf("--port=%d", httpContainerPort), + } + _, err = c.ReplicationControllers(ns).Create(rc) + Expect(err).NotTo(HaveOccurred()) +} + +// gcloudUnmarshal unmarshals json output of gcloud into given out interface. +func gcloudUnmarshal(resource, regex string, out interface{}) { + output, err := exec.Command("gcloud", "compute", resource, "list", + fmt.Sprintf("--regex=%v", regex), "-q", "--format=json").CombinedOutput() + if err != nil { + Failf("Error unmarshalling gcloud output: %v", err) + } + if err := json.Unmarshal([]byte(output), out); err != nil { + Failf("Error unmarshalling gcloud output: %v", err) + } +} + +func checkLeakedResources() error { + msg := "" + // Check all resources #16636. + beList := []compute.BackendService{} + gcloudUnmarshal("backend-services", "k8s-be-[0-9]+", &beList) + if len(beList) != 0 { + for _, b := range beList { + msg += fmt.Sprintf("%v\n", b.Name) + } + return fmt.Errorf("Found backend services:\n%v", msg) + } + fwList := []compute.ForwardingRule{} + gcloudUnmarshal("forwarding-rules", "k8s-fw-.*", &fwList) + if len(fwList) != 0 { + for _, f := range fwList { + msg += fmt.Sprintf("%v\n", f.Name) + } + return fmt.Errorf("Found forwarding rules:\n%v", msg) + } + return nil +} + +var _ = Describe("GCE L7 LoadBalancer Controller", func() { + // These variables are initialized after framework's beforeEach. + var ns string + var client *client.Client + var responseTimes, creationTimes []time.Duration + + framework := Framework{BaseName: "glbc"} + + BeforeEach(func() { + // This test requires a GCE/GKE only cluster-addon + SkipUnlessProviderIs("gce", "gke") + framework.beforeEach() + client = framework.Client + ns = framework.Namespace.Name + Expect(waitForRCPodsRunning(client, "kube-system", "glbc")).NotTo(HaveOccurred()) + Expect(checkLeakedResources()).NotTo(HaveOccurred()) + responseTimes = []time.Duration{} + creationTimes = []time.Duration{} + }) + + AfterEach(func() { + framework.afterEach() + err := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) { + if err := checkLeakedResources(); err != nil { + Logf("Still waiting for glbc to cleanup: %v", err) + return false, nil + } + return true, nil + }) + Logf("Average creation time %+v, health check time %+v", creationTimes, responseTimes) + if err != nil { + Failf("Failed to cleanup GCE L7 resources.") + } + Logf("Successfully verified GCE L7 loadbalancer via Ingress.") + }) + + It("should create GCE L7 loadbalancers and verify Ingress", func() { + // Create numApps apps, exposed via numIng Ingress each with 2 paths. + // Eg with numApp=10, numIng=5: + // apps: {foo-app-(0-10)} + // ingress: {foo-app-(0, 2, 4, 6, 8)} + // paths: + // ingress foo-app-0: + // default1.bar.com + // foo0.bar.com: /foo0 + // foo1.bar.com: /foo1 + if numApps < numIng { + Failf("Need more apps than Ingress") + } + appsPerIngress := numApps / numIng + By(fmt.Sprintf("Creating %d rcs + svc, and %d apps per Ingress", numApps, appsPerIngress)) + for appID := 0; appID < numApps; appID = appID + appsPerIngress { + // Creates appsPerIngress apps, then creates one Ingress with paths to all the apps. + for j := appID; j < appID+appsPerIngress; j++ { + createApp(client, ns, j) + } + createIngress(client, ns, appID, appsPerIngress) + } + + ings, err := client.Extensions().Ingress(ns).List( + labels.Everything(), fields.Everything()) + Expect(err).NotTo(HaveOccurred()) + + for _, ing := range ings.Items { + // Wait for the loadbalancer IP. + start := time.Now() + address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout) + Expect(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online", + address, ing.Name, time.Since(start))) + creationTimes = append(creationTimes, time.Since(start)) + + // Check that all rules respond to a simple GET. + for _, rules := range ing.Spec.Rules { + // As of Kubernetes 1.1 we only support HTTP Ingress. + if rules.IngressRuleValue.HTTP == nil { + continue + } + for _, p := range rules.IngressRuleValue.HTTP.Paths { + route := fmt.Sprintf("http://%v%v", address, p.Path) + Logf("Testing route %v host %v with simple GET", route, rules.Host) + + GETStart := time.Now() + var lastBody string + pollErr := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) { + var err error + lastBody, err = simpleGET(http.DefaultClient, route, rules.Host) + if err != nil { + Logf("host %v path %v: %v", rules.Host, route, err) + return false, nil + } + return true, nil + }) + if pollErr != nil { + Failf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v", + lbPollTimeout, route, rules.Host, lastBody, pollErr) + } + rt := time.Since(GETStart) + By(fmt.Sprintf("Route %v host %v took %v to respond", route, rules.Host, rt)) + responseTimes = append(responseTimes, rt) + } + } + } + // In most cases slow loadbalancer creation/startup translates directly to + // GCE api sluggishness. However this might be because of something the + // controller is doing, eg: maxing out QPS by repeated polling. + sort.Sort(timeSlice(creationTimes)) + perc50 := creationTimes[len(creationTimes)/2] + if perc50 > expectedLBCreationTime { + Failf("Average creation time is too high: %+v", creationTimes) + } + sort.Sort(timeSlice(responseTimes)) + perc50 = responseTimes[len(responseTimes)/2] + if perc50 > expectedLBHealthCheckTime { + Failf("Average startup time is too high: %+v", responseTimes) + } + }) +}) diff --git a/test/e2e/resize_nodes.go b/test/e2e/resize_nodes.go index b7e36bad3ed..20538a1d1a1 100644 --- a/test/e2e/resize_nodes.go +++ b/test/e2e/resize_nodes.go @@ -41,6 +41,7 @@ const ( serveHostnameImage = "gcr.io/google_containers/serve_hostname:1.1" resizeNodeReadyTimeout = 2 * time.Minute resizeNodeNotReadyTimeout = 2 * time.Minute + testPort = 9376 ) func resizeGroup(size int) error { @@ -111,25 +112,26 @@ func waitForGroupSize(size int) error { return fmt.Errorf("timeout waiting %v for node instance group size to be %d", timeout, size) } -func svcByName(name string) *api.Service { +func svcByName(name string, port int) *api.Service { return &api.Service{ ObjectMeta: api.ObjectMeta{ - Name: "test-service", + Name: name, }, Spec: api.ServiceSpec{ + Type: api.ServiceTypeNodePort, Selector: map[string]string{ "name": name, }, Ports: []api.ServicePort{{ - Port: 9376, - TargetPort: util.NewIntOrStringFromInt(9376), + Port: port, + TargetPort: util.NewIntOrStringFromInt(port), }}, }, } } func newSVCByName(c *client.Client, ns, name string) error { - _, err := c.Services(ns).Create(svcByName(name)) + _, err := c.Services(ns).Create(svcByName(name, testPort)) return err } diff --git a/test/e2e/serviceloadbalancers.go b/test/e2e/serviceloadbalancers.go index b86d65f1ea2..1cce656b42b 100644 --- a/test/e2e/serviceloadbalancers.go +++ b/test/e2e/serviceloadbalancers.go @@ -195,7 +195,7 @@ func (s *ingManager) test(path string) error { url := fmt.Sprintf("%v/hostName", path) httpClient := &http.Client{} return wait.Poll(pollInterval, serviceRespondingTimeout, func() (bool, error) { - body, err := simpleGET(httpClient, url) + body, err := simpleGET(httpClient, url, "") if err != nil { Logf("%v\n%v\n%v", url, body, err) return false, nil @@ -240,8 +240,13 @@ var _ = Describe("ServiceLoadBalancer", func() { }) // simpleGET executes a get on the given url, returns error if non-200 returned. -func simpleGET(c *http.Client, url string) (string, error) { - res, err := c.Get(url) +func simpleGET(c *http.Client, url, host string) (string, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", err + } + req.Host = host + res, err := c.Do(req) if err != nil { return "", err } diff --git a/test/e2e/util.go b/test/e2e/util.go index afa61cb0b1f..4c74e0c4659 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -2157,3 +2157,36 @@ func OpenWebSocketForURL(url *url.URL, config *client.Config, protocols []string cfg.Protocol = protocols return websocket.DialConfig(cfg) } + +// getIngressAddress returns the ips/hostnames associated with the Ingress. +func getIngressAddress(client *client.Client, ns, name string) ([]string, error) { + ing, err := client.Extensions().Ingress(ns).Get(name) + if err != nil { + return nil, err + } + addresses := []string{} + for _, a := range ing.Status.LoadBalancer.Ingress { + if a.IP != "" { + addresses = append(addresses, a.IP) + } + if a.Hostname != "" { + addresses = append(addresses, a.Hostname) + } + } + return addresses, nil +} + +// waitForIngressAddress waits for the Ingress to acquire an address. +func waitForIngressAddress(c *client.Client, ns, ingName string, timeout time.Duration) (string, error) { + var address string + err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) { + ipOrNameList, err := getIngressAddress(c, ns, ingName) + if err != nil || len(ipOrNameList) == 0 { + Logf("Waiting for Ingress %v to acquire IP, error %v", ingName, err) + return false, nil + } + address = ipOrNameList[0] + return true, nil + }) + return address, err +} diff --git a/test/images/n-way-http/Dockerfile b/test/images/n-way-http/Dockerfile new file mode 100644 index 00000000000..46c120ceca0 --- /dev/null +++ b/test/images/n-way-http/Dockerfile @@ -0,0 +1,18 @@ +# Copyright 2015 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM busybox +MAINTAINER Prashanth B +ADD server server +ENTRYPOINT ["/server"] diff --git a/test/images/n-way-http/Makefile b/test/images/n-way-http/Makefile new file mode 100644 index 00000000000..864c66d01da --- /dev/null +++ b/test/images/n-way-http/Makefile @@ -0,0 +1,17 @@ +all: push + +# 0.0 shouldn't clobber any released builds +TAG = 0.0 +PREFIX = gcr.io/google_containers/n-way-http + +server: server.go + CGO_ENABLED=0 GOOS=linux godep go build -a -installsuffix cgo -ldflags '-w' -o server ./server.go + +container: server + docker build -t $(PREFIX):$(TAG) . + +push: container + gcloud docker push $(PREFIX):$(TAG) + +clean: + rm -f server diff --git a/test/images/n-way-http/server.go b/test/images/n-way-http/server.go new file mode 100644 index 00000000000..057cfef9f35 --- /dev/null +++ b/test/images/n-way-http/server.go @@ -0,0 +1,55 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// A webserver that runs n http handlers. Example invocation: +// - server -port 8080 -prefix foo -num 10 -start 0 +// Will given you 10 /foo(i) endpoints that simply echo foo(i) when requested. +// - server -start 3 -num 1 +// Will create just one endpoint, at /foo3 +package main + +import ( + "flag" + "fmt" + "log" + "net/http" +) + +var ( + port = flag.Int("port", 8080, "Port number for requests.") + prefix = flag.String("prefix", "foo", "String used as path prefix") + num = flag.Int("num", 10, "Number of endpoints to create.") + start = flag.Int("start", 0, "Index to start, only makes sense with --num") +) + +func main() { + flag.Parse() + // This container is used to test the GCE L7 controller which expects "/" + // to return a 200 response. + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "ok") + }) + for i := *start; i < *start+*num; i++ { + path := fmt.Sprintf("%v%d", *prefix, i) + http.HandleFunc(fmt.Sprintf("/%v", path), func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, path) + }) + } + log.Printf("server -port %d -prefix %v -num %d -start %d", *port, *prefix, *num, *start) + http.ListenAndServe(fmt.Sprintf(":%d", *port), nil) +}