Merge pull request #10070 from lavalamp/e2eProxyFix

proxy e2e test improvements

contrib/for-tests/porter/pod.json

@@ -8,7 +8,7 @@
     "containers": [
       {
         "name": "porter",
-        "image": "gcr.io/google_containers/porter:91d46193649807d1340b46797774d8b2",
+        "image": "gcr.io/google_containers/porter:59ad46ed2c56ba50fa7f1dc176c07c37",
         "env": [
           {
             "name": "SERVE_PORT_80",

contrib/for-tests/porter/porter.go

@@ -33,7 +33,10 @@ const prefix = "SERVE_PORT_"
 
 func main() {
 	for _, vk := range os.Environ() {
-		parts := strings.Split(vk, "=")
+		// Put everything before the first = sign in parts[0], and
+		// everything else in parts[1] (even if there are multiple =
+		// characters).
+		parts := strings.SplitN(vk, "=", 2)
 		key := parts[0]
 		value := parts[1]
 		if strings.HasPrefix(key, prefix) {

docs/kubectl_proxy.md

@@ -41,7 +41,7 @@ $ kubectl proxy --api-prefix=/k8s-api
 
 ```
       --accept-hosts="^localhost$,^127\\.0\\.0\\.1$,^\\[::1\\]$": Regular expression for hosts that the proxy should accept.
-      --accept-paths="^/api/.*": Regular expression for paths that the proxy should accept.
+      --accept-paths="^/.*": Regular expression for paths that the proxy should accept.
       --api-prefix="/api/": Prefix to serve the proxied API under.
       --disable-filter=false: If true, disable request filtering in the proxy. This is dangerous, and can leave you vulnerable to XSRF attacks.  Use with caution.
   -h, --help=false: help for proxy
@@ -84,6 +84,6 @@ $ kubectl proxy --api-prefix=/k8s-api
 ### SEE ALSO
 * [kubectl](kubectl.md)	 - kubectl controls the Kubernetes cluster manager
 
-###### Auto generated by spf13/cobra at 2015-06-11 03:49:29.837564354 +0000 UTC
+###### Auto generated by spf13/cobra at 2015-06-23 19:00:28.69764897 +0000 UTC
 
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/kubectl_proxy.md?pixel)]()

docs/man/man1/kubectl-proxy.1

@@ -43,7 +43,7 @@ The above lets you 'curl localhost:8001/custom/api/v1/pods'
     Regular expression for hosts that the proxy should accept.
 
 .PP
-\fB\-\-accept\-paths\fP="^/api/.*"
+\fB\-\-accept\-paths\fP="^/.*"
     Regular expression for paths that the proxy should accept.
 
 .PP

hack/test-cmd.sh

@@ -25,11 +25,33 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 source "${KUBE_ROOT}/hack/lib/test.sh"
 
+# Stops the running kubectl proxy, if there is one.
+function stop-proxy()
+{
+  [[ -n "${PROXY_PID-}" ]] && kill "${PROXY_PID}" 1>&2 2>/dev/null
+  PROXY_PID=
+}
+
+# Starts "kubect proxy" to test the client proxy. You may pass options, e.g.
+# --api-prefix.
+function start-proxy()
+{
+  stop-proxy
+
+  kube::log::status "Starting kubectl proxy"
+  # the --www and --www-prefix are just to make something definitely show up for
+  # wait_for_url to see.
+  kubectl proxy -p ${PROXY_PORT} --www=. --www-prefix=/healthz "$@" 1>&2 &
+  PROXY_PID=$!
+  kube::util::wait_for_url "http://127.0.0.1:${PROXY_PORT}/healthz" "kubectl proxy $@"
+}
+
 function cleanup()
 {
-    [[ -n ${APISERVER_PID-} ]] && kill ${APISERVER_PID} 1>&2 2>/dev/null
+  [[ -n "${APISERVER_PID-}" ]] && kill "${APISERVER_PID}" 1>&2 2>/dev/null
-    [[ -n ${CTLRMGR_PID-} ]] && kill ${CTLRMGR_PID} 1>&2 2>/dev/null
+  [[ -n "${CTLRMGR_PID-}" ]] && kill "${CTLRMGR_PID}" 1>&2 2>/dev/null
-    [[ -n ${KUBELET_PID-} ]] && kill ${KUBELET_PID} 1>&2 2>/dev/null
+  [[ -n "${KUBELET_PID-}" ]] && kill "${KUBELET_PID}" 1>&2 2>/dev/null
+  stop-proxy
 
   kube::etcd::cleanup
   rm -rf "${KUBE_TEMP}"
@@ -37,6 +59,22 @@ function cleanup()
   kube::log::status "Clean up complete"
 }
 
+# Executes curl against the proxy. $1 is the path to use, $2 is the desired
+# return code. Prints a helpful message on failure.
+function check-curl-proxy-code()
+{
+  local status
+  local -r address=$1
+  local -r desired=$2
+  local -r full_address="${PROXY_HOST}:${PROXY_PORT}${address}"
+  status=$(curl -w "%{http_code}" --silent --output /dev/null "${full_address}")
+  if [ "${status}" == "${desired}" ]; then
+    return 0
+  fi
+  echo "For address ${full_address}, got ${status} but wanted ${desired}"
+  return 1
+}
+
 trap cleanup EXIT SIGINT
 
 kube::util::ensure-temp-dir
@@ -49,6 +87,8 @@ API_HOST=${API_HOST:-127.0.0.1}
 KUBELET_PORT=${KUBELET_PORT:-10250}
 KUBELET_HEALTHZ_PORT=${KUBELET_HEALTHZ_PORT:-10248}
 CTLRMGR_PORT=${CTLRMGR_PORT:-10252}
+PROXY_PORT=${PROXY_PORT:-8001}
+PROXY_HOST=127.0.0.1 # kubectl only serves on localhost.
 
 # Check kubectl
 kube::log::status "Running kubectl with no options"
@@ -146,6 +186,38 @@ for version in "${kube_api_versions[@]}"; do
   # Passing no arguments to create is an error
   ! kubectl create
 
+  #######################
+  # kubectl local proxy #
+  #######################
+
+  # Make sure the UI can be proxied
+  start-proxy --api-prefix=/
+  check-curl-proxy-code /ui 301
+  check-curl-proxy-code /metrics 200
+  if [[ -n "${version}" ]]; then
+    check-curl-proxy-code /api/${version}/namespaces 200
+  fi
+  stop-proxy
+
+  # Default proxy locks you into the /api path (legacy behavior)
+  start-proxy
+  check-curl-proxy-code /ui 404
+  check-curl-proxy-code /metrics 404
+  check-curl-proxy-code /api/ui 404
+  if [[ -n "${version}" ]]; then
+    check-curl-proxy-code /api/${version}/namespaces 200
+  fi
+  stop-proxy
+
+  # Custom paths let you see everything.
+  start-proxy --api-prefix=/custom
+  check-curl-proxy-code /custom/ui 301
+  check-curl-proxy-code /custom/metrics 200
+  if [[ -n "${version}" ]]; then
+    check-curl-proxy-code /custom/api/${version}/namespaces 200
+  fi
+  stop-proxy
+
   ###########################
   # POD creation / deletion #
   ###########################

pkg/apiserver/proxy.go

@@ -20,6 +20,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httputil"
@@ -55,6 +56,8 @@ type ProxyHandler struct {
 }
 
 func (r *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	proxyHandlerTraceID := rand.Int63()
+
 	var verb string
 	var apiResource string
 	var httpCode int
@@ -108,7 +111,7 @@ func (r *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	location, transport, err := redirector.ResourceLocation(ctx, id)
+	location, roundTripper, err := redirector.ResourceLocation(ctx, id)
 	if err != nil {
 		httplog.LogOf(req, w).Addf("Error getting ResourceLocation: %v", err)
 		status := errToAPIStatus(err)
@@ -123,8 +126,11 @@ func (r *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 	// If we have a custom dialer, and no pre-existing transport, initialize it to use the dialer.
-	if transport == nil && r.dial != nil {
+	if roundTripper == nil && r.dial != nil {
-		transport = &http.Transport{Dial: r.dial}
+		glog.V(5).Infof("[%x: %v] making a dial-only transport...", proxyHandlerTraceID, req.URL)
+		roundTripper = &http.Transport{Dial: r.dial}
+	} else if roundTripper != nil {
+		glog.V(5).Infof("[%x: %v] using transport %T...", proxyHandlerTraceID, req.URL, roundTripper)
 	}
 
 	// Default to http
@@ -158,7 +164,7 @@ func (r *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	// TODO convert this entire proxy to an UpgradeAwareProxy similar to
 	// https://github.com/openshift/origin/blob/master/pkg/util/httpproxy/upgradeawareproxy.go.
 	// That proxy needs to be modified to support multiple backends, not just 1.
-	if r.tryUpgrade(w, req, newReq, location, transport) {
+	if r.tryUpgrade(w, req, newReq, location, roundTripper) {
 		return
 	}
 
@@ -175,19 +181,33 @@ func (r *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	start := time.Now()
+	glog.V(4).Infof("[%x] Beginning proxy %s...", proxyHandlerTraceID, req.URL)
+	defer func() {
+		glog.V(4).Infof("[%x] Proxy %v finished %v.", proxyHandlerTraceID, req.URL, time.Now().Sub(start))
+	}()
+
 	proxy := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: location.Scheme, Host: location.Host})
-	if transport == nil {
+	alreadyRewriting := false
+	if roundTripper != nil {
+		_, alreadyRewriting = roundTripper.(*proxyutil.Transport)
+		glog.V(5).Infof("[%x] Not making a reriting transport for proxy %s...", proxyHandlerTraceID, req.URL)
+	}
+	if !alreadyRewriting {
+		glog.V(5).Infof("[%x] making a transport for proxy %s...", proxyHandlerTraceID, req.URL)
 		prepend := path.Join(r.prefix, resource, id)
 		if len(namespace) > 0 {
 			prepend = path.Join(r.prefix, "namespaces", namespace, resource, id)
 		}
-		transport = &proxyutil.Transport{
+		pTransport := &proxyutil.Transport{
 			Scheme:       req.URL.Scheme,
 			Host:         req.URL.Host,
 			PathPrepend:  prepend,
+			RoundTripper: roundTripper,
 		}
+		roundTripper = pTransport
 	}
-	proxy.Transport = transport
+	proxy.Transport = roundTripper
 	proxy.FlushInterval = 200 * time.Millisecond
 	proxy.ServeHTTP(w, newReq)
 }

pkg/client/request.go

@@ -804,7 +804,7 @@ func (r *Request) transformResponse(resp *http.Response, req *http.Request) Resu
 
 	return Result{
 		body:       body,
-		created: resp.StatusCode == http.StatusCreated,
+		statusCode: resp.StatusCode,
 		codec:      r.codec,
 	}
 }
@@ -880,8 +880,8 @@ func retryAfterSeconds(resp *http.Response) (int, bool) {
 // Result contains the result of calling Request.Do().
 type Result struct {
 	body       []byte
-	created bool
 	err        error
+	statusCode int
 
 	codec runtime.Codec
 }
@@ -899,6 +899,13 @@ func (r Result) Get() (runtime.Object, error) {
 	return r.codec.Decode(r.body)
 }
 
+// StatusCode returns the HTTP status code of the request. (Only valid if no
+// error was returned.)
+func (r Result) StatusCode(statusCode *int) Result {
+	*statusCode = r.statusCode
+	return r
+}
+
 // Into stores the result into obj, if possible.
 func (r Result) Into(obj runtime.Object) error {
 	if r.err != nil {
@@ -910,7 +917,7 @@ func (r Result) Into(obj runtime.Object) error {
 // WasCreated updates the provided bool pointer to whether the server returned
 // 201 created or a different response.
 func (r Result) WasCreated(wasCreated *bool) Result {
-	*wasCreated = r.created
+	*wasCreated = r.statusCode == http.StatusCreated
 	return r
 }
 

pkg/client/request_test.go

@@ -274,7 +274,7 @@ func TestTransformResponse(t *testing.T) {
 			test.Response.Body = ioutil.NopCloser(bytes.NewReader([]byte{}))
 		}
 		result := r.transformResponse(test.Response, &http.Request{})
-		response, created, err := result.body, result.created, result.err
+		response, created, err := result.body, result.statusCode == http.StatusCreated, result.err
 		hasErr := err != nil
 		if hasErr != test.Error {
 			t.Errorf("%d: unexpected error: %t %v", i, test.Error, err)

pkg/kubectl/proxy_server.go

@@ -31,7 +31,7 @@ import (
 
 const (
 	DefaultHostAcceptRE   = "^localhost$,^127\\.0\\.0\\.1$,^\\[::1\\]$"
-	DefaultPathAcceptRE   = "^/api/.*"
+	DefaultPathAcceptRE   = "^/.*"
 	DefaultPathRejectRE   = "^/api/.*/exec,^/api/.*/run"
 	DefaultMethodRejectRE = "POST,PUT,PATCH"
 )
@@ -75,6 +75,7 @@ func MakeRegexpArrayOrDie(str string) []*regexp.Regexp {
 func matchesRegexp(str string, regexps []*regexp.Regexp) bool {
 	for _, re := range regexps {
 		if re.MatchString(str) {
+			glog.V(6).Infof("%v matched %s", str, re)
 			return true
 		}
 	}
@@ -83,17 +84,28 @@ func matchesRegexp(str string, regexps []*regexp.Regexp) bool {
 
 func (f *FilterServer) accept(method, path, host string) bool {
 	if matchesRegexp(path, f.RejectPaths) {
+		glog.V(3).Infof("Filter rejecting %v %v %v", method, path, host)
 		return false
 	}
 	if matchesRegexp(method, f.RejectMethods) {
+		glog.V(3).Infof("Filter rejecting %v %v %v", method, path, host)
 		return false
 	}
 	if matchesRegexp(path, f.AcceptPaths) && matchesRegexp(host, f.AcceptHosts) {
+		glog.V(3).Infof("Filter accepting %v %v %v", method, path, host)
 		return true
 	}
+	glog.V(3).Infof("Filter rejecting %v %v %v", method, path, host)
 	return false
 }
 
+// Make a copy of f which passes requests along to the new delegate.
+func (f *FilterServer) HandlerFor(delegate http.Handler) *FilterServer {
+	f2 := *f
+	f2.delegate = delegate
+	return &f2
+}
+
 func (f *FilterServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	host, _, _ := net.SplitHostPort(req.Host)
 	if f.accept(req.Method, req.URL.Path, host) {
@@ -106,12 +118,12 @@ func (f *FilterServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 
 // ProxyServer is a http.Handler which proxies Kubernetes APIs to remote API server.
 type ProxyServer struct {
-	mux *http.ServeMux
+	handler http.Handler
-	httputil.ReverseProxy
 }
 
 // NewProxyServer creates and installs a new ProxyServer.
 // It automatically registers the created ProxyServer to http.DefaultServeMux.
+// 'filter', if non-nil, protects requests to the api only.
 func NewProxyServer(filebase string, apiProxyPrefix string, staticPrefix string, filter *FilterServer, cfg *client.Config) (*ProxyServer, error) {
 	host := cfg.Host
 	if !strings.HasSuffix(host, "/") {
@@ -121,46 +133,45 @@ func NewProxyServer(filebase string, apiProxyPrefix string, staticPrefix string,
 	if err != nil {
 		return nil, err
 	}
-	proxy := newProxyServer(target)
+	proxy := newProxy(target)
 	if proxy.Transport, err = client.TransportFor(cfg); err != nil {
 		return nil, err
 	}
-
+	proxyServer := http.Handler(proxy)
-	var server http.Handler
-	if strings.HasPrefix(apiProxyPrefix, "/api") {
-		server = proxy
-	} else {
-		server = http.StripPrefix(apiProxyPrefix, proxy)
-	}
 	if filter != nil {
-		filter.delegate = server
+		proxyServer = filter.HandlerFor(proxyServer)
-		server = filter
 	}
 
-	proxy.mux.Handle(apiProxyPrefix, server)
+	if !strings.HasPrefix(apiProxyPrefix, "/api") {
-	proxy.mux.Handle(staticPrefix, newFileHandler(staticPrefix, filebase))
+		proxyServer = stripLeaveSlash(apiProxyPrefix, proxyServer)
-	return proxy, nil
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle(apiProxyPrefix, proxyServer)
+	if filebase != "" {
+		// Require user to explicitly request this behavior rather than
+		// serving their working directory by default.
+		mux.Handle(staticPrefix, newFileHandler(staticPrefix, filebase))
+	}
+	return &ProxyServer{handler: mux}, nil
 }
 
 // Serve starts the server (http.DefaultServeMux) on given port, loops forever.
 func (s *ProxyServer) Serve(port int) error {
 	server := http.Server{
 		Addr:    fmt.Sprintf(":%d", port),
-		Handler: s.mux,
+		Handler: s.handler,
 	}
 	return server.ListenAndServe()
 }
 
-func newProxyServer(target *url.URL) *ProxyServer {
+func newProxy(target *url.URL) *httputil.ReverseProxy {
 	director := func(req *http.Request) {
 		req.URL.Scheme = target.Scheme
 		req.URL.Host = target.Host
 		req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path)
 	}
-	return &ProxyServer{
+	return &httputil.ReverseProxy{Director: director}
-		ReverseProxy: httputil.ReverseProxy{Director: director},
-		mux:          http.NewServeMux(),
-	}
 }
 
 func newFileHandler(prefix, base string) http.Handler {
@@ -178,3 +189,20 @@ func singleJoiningSlash(a, b string) string {
 	}
 	return a + b
 }
+
+// like http.StripPrefix, but always leaves an initial slash. (so that our
+// regexps will work.)
+func stripLeaveSlash(prefix string, h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		p := strings.TrimPrefix(req.URL.Path, prefix)
+		if len(p) >= len(req.URL.Path) {
+			http.NotFound(w, req)
+			return
+		}
+		if len(p) > 0 && p[:1] != "/" {
+			p = "/" + p
+		}
+		req.URL.Path = p
+		h.ServeHTTP(w, req)
+	})
+}

pkg/kubectl/proxy_server_test.go

@@ -98,10 +98,10 @@ func TestAccept(t *testing.T) {
 			acceptPaths:  DefaultPathAcceptRE,
 			rejectPaths:  DefaultPathRejectRE,
 			acceptHosts:  DefaultHostAcceptRE,
-			path:         "/foo/v1/pods",
+			path:         "/ui",
 			host:         "localhost",
 			method:       "GET",
-			expectAccept: false,
+			expectAccept: true,
 		},
 		{
 			acceptPaths:  DefaultPathAcceptRE,
@@ -230,7 +230,7 @@ func TestAPIRequests(t *testing.T) {
 
 	// httptest.NewServer should always generate a valid URL.
 	target, _ := url.Parse(ts.URL)
-	proxy := newProxyServer(target)
+	proxy := newProxy(target)
 
 	tests := []struct{ method, body string }{
 		{"GET", ""},
@@ -291,7 +291,7 @@ func TestPathHandling(t *testing.T) {
 			if err != nil {
 				t.Fatalf("%#v: %v", item, err)
 			}
-			pts := httptest.NewServer(p.mux)
+			pts := httptest.NewServer(p.handler)
 			defer pts.Close()
 
 			r, err := http.Get(pts.URL + item.reqPath)

pkg/util/proxy/transport.go

@@ -73,6 +73,8 @@ type Transport struct {
 	Scheme      string
 	Host        string
 	PathPrepend string
+
+	http.RoundTripper
 }
 
 // RoundTrip implements the http.RoundTripper interface
@@ -86,7 +88,11 @@ func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
 	req.Header.Set("X-Forwarded-Host", t.Host)
 	req.Header.Set("X-Forwarded-Proto", t.Scheme)
 
-	resp, err := http.DefaultTransport.RoundTrip(req)
+	rt := t.RoundTripper
+	if rt == nil {
+		rt = http.DefaultTransport
+	}
+	resp, err := rt.RoundTrip(req)
 
 	if err != nil {
 		message := fmt.Sprintf("Error: '%s'\nTrying to reach: '%v'", err.Error(), req.URL.String())

test/e2e/proxy.go

@@ -18,7 +18,9 @@ package e2e
 
 import (
 	"fmt"
+	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@@ -37,52 +39,23 @@ var _ = Describe("Proxy", func() {
 	}
 })
 
+const (
+	// Try all the proxy tests this many times (to catch even rare flakes).
+	proxyAttempts = 20
+	// Only print this many characters of the response (to keep the logs
+	// legible).
+	maxDisplayBodyLen = 100
+)
+
 func proxyContext(version string) {
 	f := NewFramework("proxy")
 	prefix := "/api/" + version
 
-	It("should proxy logs on node with explicit kubelet port", func() {
+	It("should proxy logs on node with explicit kubelet port", func() { nodeProxyTest(f, version, ":10250/logs/") })
-		node, err := pickNode(f.Client)
-		Expect(err).NotTo(HaveOccurred())
-		// AbsPath preserves the trailing '/'.
-		body, err := f.Client.Get().AbsPath(prefix + "/proxy/nodes/" + node + ":10250/logs/").Do().Raw()
-		if len(body) > 0 {
-			if len(body) > 100 {
-				body = body[:100]
-				body = append(body, '.', '.', '.')
-			}
-			Logf("Got: %s", body)
-		}
-		Expect(err).NotTo(HaveOccurred())
-	})
 
-	It("should proxy logs on node", func() {
+	It("should proxy logs on node", func() { nodeProxyTest(f, version, "/logs/") })
-		node, err := pickNode(f.Client)
-		Expect(err).NotTo(HaveOccurred())
-		body, err := f.Client.Get().AbsPath(prefix + "/proxy/nodes/" + node + "/logs/").Do().Raw()
-		if len(body) > 0 {
-			if len(body) > 100 {
-				body = body[:100]
-				body = append(body, '.', '.', '.')
-			}
-			Logf("Got: %s", body)
-		}
-		Expect(err).NotTo(HaveOccurred())
-	})
 
-	It("should proxy to cadvisor", func() {
+	It("should proxy to cadvisor", func() { nodeProxyTest(f, version, ":4194/containers/") })
-		node, err := pickNode(f.Client)
-		Expect(err).NotTo(HaveOccurred())
-		body, err := f.Client.Get().AbsPath(prefix + "/proxy/nodes/" + node + ":4194/containers/").Do().Raw()
-		if len(body) > 0 {
-			if len(body) > 100 {
-				body = body[:100]
-				body = append(body, '.', '.', '.')
-			}
-			Logf("Got: %s", body)
-		}
-		Expect(err).NotTo(HaveOccurred())
-	})
 
 	It("should proxy through a service and a pod", func() {
 		labels := map[string]string{"proxy-service-target": "true"}
@@ -118,13 +91,13 @@ func proxyContext(version string) {
 		pods := []*api.Pod{}
 		cfg := RCConfig{
 			Client:       f.Client,
-			Image:        "gcr.io/google_containers/porter:91d46193649807d1340b46797774d8b2",
+			Image:        "gcr.io/google_containers/porter:59ad46ed2c56ba50fa7f1dc176c07c37",
 			Name:         service.Name,
 			Namespace:    f.Namespace.Name,
 			Replicas:     1,
 			PollInterval: time.Second,
 			Env: map[string]string{
-				"SERVE_PORT_80":  "not accessible via service",
+				"SERVE_PORT_80":  `<a href="/rewriteme">test</a>`,
 				"SERVE_PORT_160": "foo",
 				"SERVE_PORT_162": "bar",
 			},
@@ -144,11 +117,11 @@ func proxyContext(version string) {
 		svcPrefix := prefix + "/proxy/namespaces/" + f.Namespace.Name + "/services/" + service.Name
 		podPrefix := prefix + "/proxy/namespaces/" + f.Namespace.Name + "/pods/" + pods[0].Name
 		expectations := map[string]string{
-			svcPrefix + ":portname1": "foo",
+			svcPrefix + ":portname1/": "foo",
-			svcPrefix + ":portname2": "bar",
+			svcPrefix + ":portname2/": "bar",
-			podPrefix + ":80":        "not accessible via service",
+			podPrefix + ":80/":        `<a href="` + podPrefix + `:80/rewriteme">test</a>`,
-			podPrefix + ":160":       "foo",
+			podPrefix + ":160/":       "foo",
-			podPrefix + ":162":       "bar",
+			podPrefix + ":162/":       "bar",
 			// TODO: below entries don't work, but I believe we should make them work.
 			// svcPrefix + ":80": "foo",
 			// svcPrefix + ":81": "bar",
@@ -156,17 +129,38 @@ func proxyContext(version string) {
 			// podPrefix + ":dest2": "bar",
 		}
 
+		wg := sync.WaitGroup{}
 		errors := []string{}
+		errLock := sync.Mutex{}
+		recordError := func(s string) {
+			errLock.Lock()
+			defer errLock.Unlock()
+			errors = append(errors, s)
+		}
+		for i := 0; i < proxyAttempts; i++ {
 			for path, val := range expectations {
-			body, err := f.Client.Get().AbsPath(path).Do().Raw()
+				wg.Add(1)
+				go func(i int, path, val string) {
+					defer wg.Done()
+					body, status, d, err := doProxy(f, path)
 					if err != nil {
-				errors = append(errors, fmt.Sprintf("path %v gave error %v", path, err))
+						recordError(fmt.Sprintf("%v: path %v gave error: %v", i, path, err))
-				continue
+						return
+					}
+					if status != http.StatusOK {
+						recordError(fmt.Sprintf("%v: path %v gave status: %v", i, path, status))
 					}
 					if e, a := val, string(body); e != a {
-				errors = append(errors, fmt.Sprintf("path %v: wanted %v, got %v", path, e, a))
+						recordError(fmt.Sprintf("%v: path %v: wanted %v, got %v", i, path, e, a))
+					}
+					if d > 15*time.Second {
+						recordError(fmt.Sprintf("%v: path %v took %v > 15s", i, path, d))
+					}
+				}(i, path, val)
+				time.Sleep(150 * time.Millisecond)
 			}
 		}
+		wg.Wait()
 
 		if len(errors) != 0 {
 			Fail(strings.Join(errors, "\n"))
@@ -174,6 +168,31 @@ func proxyContext(version string) {
 	})
 }
 
+func doProxy(f *Framework, path string) (body []byte, statusCode int, d time.Duration, err error) {
+	// About all of the proxy accesses in this file:
+	// * AbsPath is used because it preserves the trailing '/'.
+	// * Do().Raw() is used (instead of DoRaw()) because it will turn an
+	//   error from apiserver proxy into an actual error, and there is no
+	//   chance of the things we are talking to being confused for an error
+	//   that apiserver would have emitted.
+	start := time.Now()
+	body, err = f.Client.Get().AbsPath(path).Do().StatusCode(&statusCode).Raw()
+	d = time.Since(start)
+	if len(body) > 0 {
+		Logf("%v: %s (%v; %v)", path, truncate(body, maxDisplayBodyLen), statusCode, d)
+	}
+	return
+}
+
+func truncate(b []byte, maxLen int) []byte {
+	if len(b) <= maxLen-3 {
+		return b
+	}
+	b2 := append([]byte(nil), b[:maxLen-3]...)
+	b2 = append(b2, '.', '.', '.')
+	return b2
+}
+
 func pickNode(c *client.Client) (string, error) {
 	nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
 	if err != nil {
@@ -184,3 +203,15 @@ func pickNode(c *client.Client) (string, error) {
 	}
 	return nodes.Items[0].Name, nil
 }
+
+func nodeProxyTest(f *Framework, version, nodeDest string) {
+	prefix := "/api/" + version
+	node, err := pickNode(f.Client)
+	Expect(err).NotTo(HaveOccurred())
+	for i := 0; i < proxyAttempts; i++ {
+		_, status, d, err := doProxy(f, prefix+"/proxy/nodes/"+node+nodeDest)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(status).To(Equal(http.StatusOK))
+		Expect(d).To(BeNumerically("<", 15*time.Second))
+	}
+}