Merge pull request #9698 from lavalamp/e2eTestFix

Add a proxy test

contrib/for-tests/porter/.gitignore

@@ -0,0 +1,2 @@
+porter
+.tag

contrib/for-tests/porter/Dockerfile

@@ -0,0 +1,18 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM scratch
+MAINTAINER Daniel Smith <dbsmith@google.com>
+ADD porter porter
+ENTRYPOINT ["/porter"]

contrib/for-tests/porter/Makefile

@@ -0,0 +1,32 @@
+# Use:
+#
+# `make porter` will build porter.
+# `make tag` will suggest a tag.
+# `make container` will build a container-- you must supply a tag.
+# `make push` will push the container-- you must supply a tag.
+
+REPO ?= gcr.io/google_containers
+
+porter: porter.go
+	CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' ./porter.go
+
+.tag: porter
+	md5sum porter | cut -d " " -f 1 > .tag
+
+tag: .tag
+	@echo "Suggest using TAG=$(shell cat .tag)"
+	@echo "$$ make container TAG=$(shell cat .tag)"
+	@echo "or"
+	@echo "$$ make push TAG=$(shell cat .tag)"
+
+container:
+	$(if $(TAG),,$(error TAG is not defined. Use 'make tag' to see a suggestion))
+	docker build -t $(REPO)/porter:$(TAG) .
+
+push:
+	$(if $(TAG),,$(error TAG is not defined. Use 'make tag' to see a suggestion))
+	gcloud preview docker push $(REPO)/porter:$(TAG)
+
+clean:
+	rm -f porter
+	rm -f .tag

contrib/for-tests/porter/README.md

@@ -0,0 +1,5 @@
+This directory contains go source, Dockerfile and Makefile for making a test
+container which serves requested data on ports specified in ENV variables.
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/contrib/for-tests/porter/README.md?pixel)]()

contrib/for-tests/porter/pod.json

@@ -0,0 +1,53 @@
+{
+  "kind": "Pod",
+  "apiVersion": "v1",
+  "metadata": {
+    "name": "kubectl-tester"
+  },
+  "spec": {
+    "containers": [
+      {
+        "name": "bb",
+        "image": "gcr.io/google_containers/busybox",
+        "command": [
+          "sh", "-c", "sleep 5; wget -O - ${KUBERNETES_RO_SERVICE_HOST}:${KUBERNETES_RO_SERVICE_PORT}/api/v1/pods/; sleep 10000"
+        ],
+        "ports": [
+          {
+            "containerPort": 8080
+          }
+        ],
+        "env": [
+          {
+            "name": "KUBERNETES_RO_SERVICE_HOST",
+            "value": "127.0.0.1"
+          },
+          {
+            "name": "KUBERNETES_RO_SERVICE_PORT",
+            "value": "8001"
+          }
+        ],
+        "volumeMounts": [
+          {
+            "name": "test-volume",
+            "mountPath": "/mount/test-volume"
+          }
+        ]
+      },
+      {
+        "name": "kubectl",
+        "image": "gcr.io/google_containers/kubectl:v0.18.0-120-gaeb4ac55ad12b1-dirty",
+        "imagePullPolicy": "Always",
+        "args": [
+          "proxy", "-p", "8001"
+        ]
+      }
+    ],
+    "volumes": [
+      {
+        "name": "test-volume",
+        "emptyDir": {}
+      }
+    ]
+  }
+}

contrib/for-tests/porter/porter.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// A tiny binary for testing ports.
+//
+// Reads env vars; for every var of the form SERVE_PORT_X, where X is a valid
+// port number, porter starts an HTTP server which serves the env var's value
+// in response to any query.
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+)
+
+const prefix = "SERVE_PORT_"
+
+func main() {
+	for _, vk := range os.Environ() {
+		parts := strings.Split(vk, "=")
+		key := parts[0]
+		value := parts[1]
+		if strings.HasPrefix(key, prefix) {
+			port := strings.TrimPrefix(key, prefix)
+			go servePort(port, value)
+		}
+	}
+
+	select {}
+}
+
+func servePort(port, value string) {
+	s := &http.Server{
+		Addr: "0.0.0.0:" + port,
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			fmt.Fprint(w, value)
+		}),
+	}
+	log.Printf("server on port %q failed: %v", port, s.ListenAndServe())
+}

test/e2e/framework.go

@@ -103,3 +103,49 @@ func (f *Framework) WaitForPodRunning(podName string) error {
 func (f *Framework) TestContainerOutput(scenarioName string, pod *api.Pod, expectedOutput []string) {
 	testContainerOutputInNamespace(scenarioName, f.Client, pod, expectedOutput, f.Namespace.Name)
 }
+
+// WaitForAnEndpoint waits for at least one endpoint to become available in the
+// service's corresponding endpoints object.
+func (f *Framework) WaitForAnEndpoint(serviceName string) error {
+	for {
+		// TODO: Endpoints client should take a field selector so we
+		// don't have to list everything.
+		list, err := f.Client.Endpoints(f.Namespace.Name).List(labels.Everything())
+		if err != nil {
+			return err
+		}
+		rv := list.ResourceVersion
+
+		isOK := func(e *api.Endpoints) bool {
+			return e.Name == serviceName && len(e.Subsets) > 0 && len(e.Subsets[0].Addresses) > 0
+		}
+		for i := range list.Items {
+			if isOK(&list.Items[i]) {
+				return nil
+			}
+		}
+
+		w, err := f.Client.Endpoints(f.Namespace.Name).Watch(
+			labels.Everything(),
+			fields.Set{"metadata.name": serviceName}.AsSelector(),
+			rv,
+		)
+		if err != nil {
+			return err
+		}
+		defer w.Stop()
+
+		for {
+			val, ok := <-w.ResultChan()
+			if !ok {
+				// reget and re-watch
+				break
+			}
+			if e, ok := val.Object.(*api.Endpoints); ok {
+				if isOK(e) {
+					return nil
+				}
+			}
+		}
+	}
+}

test/e2e/proxy.go

@@ -0,0 +1,186 @@
+/*
+Copyright 2014 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Proxy", func() {
+	for _, version := range []string{"v1beta3", "v1"} {
+		Context("version "+version, func() { proxyContext(version) })
+	}
+})
+
+func proxyContext(version string) {
+	f := NewFramework("proxy")
+	prefix := "/api/" + version
+
+	It("should proxy logs on node with explicit kubelet port", func() {
+		node, err := pickNode(f.Client)
+		Expect(err).NotTo(HaveOccurred())
+		// AbsPath preserves the trailing '/'.
+		body, err := f.Client.Get().AbsPath(prefix + "/proxy/nodes/" + node + ":10250/logs/").Do().Raw()
+		if len(body) > 0 {
+			if len(body) > 100 {
+				body = body[:100]
+				body = append(body, '.', '.', '.')
+			}
+			Logf("Got: %s", body)
+		}
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should proxy logs on node", func() {
+		node, err := pickNode(f.Client)
+		Expect(err).NotTo(HaveOccurred())
+		body, err := f.Client.Get().AbsPath(prefix + "/proxy/nodes/" + node + "/logs/").Do().Raw()
+		if len(body) > 0 {
+			if len(body) > 100 {
+				body = body[:100]
+				body = append(body, '.', '.', '.')
+			}
+			Logf("Got: %s", body)
+		}
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should proxy to cadvisor", func() {
+		node, err := pickNode(f.Client)
+		Expect(err).NotTo(HaveOccurred())
+		body, err := f.Client.Get().AbsPath(prefix + "/proxy/nodes/" + node + ":4194/containers/").Do().Raw()
+		if len(body) > 0 {
+			if len(body) > 100 {
+				body = body[:100]
+				body = append(body, '.', '.', '.')
+			}
+			Logf("Got: %s", body)
+		}
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should proxy through a service and a pod", func() {
+		labels := map[string]string{"proxy-service-target": "true"}
+		service, err := f.Client.Services(f.Namespace.Name).Create(&api.Service{
+			ObjectMeta: api.ObjectMeta{
+				GenerateName: "proxy-service-",
+			},
+			Spec: api.ServiceSpec{
+				Selector: labels,
+				Ports: []api.ServicePort{
+					{
+						Name:       "portname1",
+						Port:       80,
+						TargetPort: util.NewIntOrStringFromString("dest1"),
+					},
+					{
+						Name:       "portname2",
+						Port:       81,
+						TargetPort: util.NewIntOrStringFromInt(162),
+					},
+				},
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+		defer func(name string) {
+			err := f.Client.Services(f.Namespace.Name).Delete(name)
+			if err != nil {
+				Logf("Failed deleting service %v: %v", name, err)
+			}
+		}(service.Name)
+
+		// Make an RC with a single pod.
+		pods := []*api.Pod{}
+		cfg := RCConfig{
+			Client:       f.Client,
+			Image:        "gcr.io/google_containers/porter:91d46193649807d1340b46797774d8b2",
+			Name:         service.Name,
+			Namespace:    f.Namespace.Name,
+			Replicas:     1,
+			PollInterval: time.Second,
+			Env: map[string]string{
+				"SERVE_PORT_80":  "not accessible via service",
+				"SERVE_PORT_160": "foo",
+				"SERVE_PORT_162": "bar",
+			},
+			Ports: map[string]int{
+				"dest1": 160,
+				"dest2": 162,
+			},
+			Labels:      labels,
+			CreatedPods: &pods,
+		}
+		Expect(RunRC(cfg)).NotTo(HaveOccurred())
+		defer DeleteRC(f.Client, f.Namespace.Name, cfg.Name)
+
+		Expect(f.WaitForAnEndpoint(service.Name)).NotTo(HaveOccurred())
+
+		// Try proxying through the service and directly to through the pod.
+		svcPrefix := prefix + "/proxy/namespaces/" + f.Namespace.Name + "/services/" + service.Name
+		podPrefix := prefix + "/proxy/namespaces/" + f.Namespace.Name + "/pods/" + pods[0].Name
+		expectations := map[string]string{
+			svcPrefix + ":portname1": "foo",
+			svcPrefix + ":portname2": "bar",
+			podPrefix + ":80":        "not accessible via service",
+			podPrefix + ":160":       "foo",
+			podPrefix + ":162":       "bar",
+			// TODO: below entries don't work, but I believe we should make them work.
+			// svcPrefix + ":80": "foo",
+			// svcPrefix + ":81": "bar",
+			// podPrefix + ":dest1": "foo",
+			// podPrefix + ":dest2": "bar",
+		}
+
+		errors := []string{}
+		for path, val := range expectations {
+			body, err := f.Client.Get().AbsPath(path).Do().Raw()
+			if err != nil {
+				errors = append(errors, fmt.Sprintf("path %v gave error %v", path, err))
+				continue
+			}
+			if e, a := val, string(body); e != a {
+				errors = append(errors, fmt.Sprintf("path %v: wanted %v, got %v", path, e, a))
+			}
+		}
+
+		if len(errors) != 0 {
+			Fail(strings.Join(errors, "\n"))
+		}
+	})
+}
+
+func pickNode(c *client.Client) (string, error) {
+	nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
+	if err != nil {
+		return "", err
+	}
+	if len(nodes.Items) == 0 {
+		return "", fmt.Errorf("no nodes exist, can't test node proxy")
+	}
+	return nodes.Items[0].Name, nil
+}

test/e2e/util.go

@@ -158,6 +158,19 @@ type RCConfig struct {
 	PollInterval  time.Duration
 	PodStatusFile *os.File
 	Replicas      int
+
+	// Env vars, set the same for every pod.
+	Env map[string]string
+
+	// Extra labels added to every pod.
+	Labels map[string]string
+
+	// Ports to declare in the container (map of name to containerPort).
+	Ports map[string]int
+
+	// Pointer to a list of pods; if non-nil, will be set to a list of pods
+	// created by this RC by RunRC.
+	CreatedPods *[]*api.Pod
 }
 
 func Logf(format string, a ...interface{}) {
@@ -841,6 +854,23 @@ func RunRC(config RCConfig) error {
 			},
 		},
 	}
+	if config.Env != nil {
+		for k, v := range config.Env {
+			c := &rc.Spec.Template.Spec.Containers[0]
+			c.Env = append(c.Env, api.EnvVar{Name: k, Value: v})
+		}
+	}
+	if config.Labels != nil {
+		for k, v := range config.Labels {
+			rc.Spec.Template.ObjectMeta.Labels[k] = v
+		}
+	}
+	if config.Ports != nil {
+		for k, v := range config.Ports {
+			c := &rc.Spec.Template.Spec.Containers[0]
+			c.Ports = append(c.Ports, api.ContainerPort{Name: k, ContainerPort: v})
+		}
+	}
 	_, err := config.Client.ReplicationControllers(config.Namespace).Create(rc)
 	if err != nil {
 		return fmt.Errorf("Error creating replication controller: %v", err)
@@ -866,6 +896,9 @@ func RunRC(config RCConfig) error {
 		inactive := 0
 		failedContainers := 0
 		pods := podStore.List()
+		if config.CreatedPods != nil {
+			*config.CreatedPods = pods
+		}
 		for _, p := range pods {
 			if p.Status.Phase == api.PodRunning {
 				running++