Merge pull request #10813 from jlowdermilk/kubectl-e2e

`kubectl proxy` supports picking random unused port, add e2e test

docs/man/man1/kubectl-proxy.1

@@ -60,7 +60,7 @@ The above lets you 'curl localhost:8001/custom/api/v1/pods'
 
 .PP
 \fB\-p\fP, \fB\-\-port\fP=8001
-    The port on which to run the proxy.
+    The port on which to run the proxy. Set to 0 to pick a random port.
 
 .PP
 \fB\-\-reject\-methods\fP="POST,PUT,PATCH"
@@ -185,6 +185,10 @@ The above lets you 'curl localhost:8001/custom/api/v1/pods'
 // Run a proxy to kubernetes apiserver on port 8011, serving static content from ./local/www/
 $ kubectl proxy \-\-port=8011 \-\-www=./local/www/
 
+// Run a proxy to kubernetes apiserver on an arbitrary local port.
+// The chosen port for the server will be output to stdout.
+$ kubectl proxy \-\-port=0
+
 // Run a proxy to kubernetes apiserver, changing the api prefix to k8s\-api
 // This makes e.g. the pods api available at localhost:8011/k8s\-api/v1/pods/
 $ kubectl proxy \-\-api\-prefix=/k8s\-api

docs/user-guide/kubectl/kubectl_proxy.md

@@ -65,6 +65,10 @@ kubectl proxy [--port=PORT] [--www=static-dir] [--www-prefix=prefix] [--api-pref
 // Run a proxy to kubernetes apiserver on port 8011, serving static content from ./local/www/
 $ kubectl proxy --port=8011 --www=./local/www/
 
+// Run a proxy to kubernetes apiserver on an arbitrary local port.
+// The chosen port for the server will be output to stdout.
+$ kubectl proxy --port=0
+
 // Run a proxy to kubernetes apiserver, changing the api prefix to k8s-api
 // This makes e.g. the pods api available at localhost:8011/k8s-api/v1/pods/
 $ kubectl proxy --api-prefix=/k8s-api
@@ -78,7 +82,7 @@ $ kubectl proxy --api-prefix=/k8s-api
       --api-prefix="/api/": Prefix to serve the proxied API under.
       --disable-filter=false: If true, disable request filtering in the proxy. This is dangerous, and can leave you vulnerable to XSRF attacks.  Use with caution.
   -h, --help=false: help for proxy
-  -p, --port=8001: The port on which to run the proxy.
+  -p, --port=8001: The port on which to run the proxy. Set to 0 to pick a random port.
       --reject-methods="POST,PUT,PATCH": Regular expression for HTTP methods that the proxy should reject.
       --reject-paths="^/api/.*/exec,^/api/.*/run": Regular expression for paths that the proxy should reject.
   -w, --www="": Also serve static files from the given directory under the specified prefix.

pkg/kubectl/cmd/proxy.go

@@ -31,6 +31,10 @@ const (
 	proxy_example = `// Run a proxy to kubernetes apiserver on port 8011, serving static content from ./local/www/
 $ kubectl proxy --port=8011 --www=./local/www/
 
+// Run a proxy to kubernetes apiserver on an arbitrary local port.
+// The chosen port for the server will be output to stdout.
+$ kubectl proxy --port=0
+
 // Run a proxy to kubernetes apiserver, changing the api prefix to k8s-api
 // This makes e.g. the pods api available at localhost:8011/k8s-api/v1/pods/
 $ kubectl proxy --api-prefix=/k8s-api`
@@ -69,14 +73,13 @@ The above lets you 'curl localhost:8001/custom/api/v1/pods'
 	cmd.Flags().String("reject-paths", kubectl.DefaultPathRejectRE, "Regular expression for paths that the proxy should reject.")
 	cmd.Flags().String("accept-hosts", kubectl.DefaultHostAcceptRE, "Regular expression for hosts that the proxy should accept.")
 	cmd.Flags().String("reject-methods", kubectl.DefaultMethodRejectRE, "Regular expression for HTTP methods that the proxy should reject.")
-	cmd.Flags().IntP("port", "p", 8001, "The port on which to run the proxy.")
+	cmd.Flags().IntP("port", "p", 8001, "The port on which to run the proxy. Set to 0 to pick a random port.")
 	cmd.Flags().Bool("disable-filter", false, "If true, disable request filtering in the proxy. This is dangerous, and can leave you vulnerable to XSRF attacks.  Use with caution.")
 	return cmd
 }
 
 func RunProxy(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command) error {
 	port := cmdutil.GetFlagInt(cmd, "port")
-	fmt.Fprintf(out, "Starting to serve on localhost:%d", port)
 
 	clientConfig, err := f.ClientConfig()
 	if err != nil {
@@ -102,11 +105,18 @@ func RunProxy(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command) error {
 		filter = nil
 	}
 
-	server, err := kubectl.NewProxyServer(cmdutil.GetFlagString(cmd, "www"), apiProxyPrefix, staticPrefix, filter, clientConfig)
+	server, err := kubectl.NewProxyServer(port, cmdutil.GetFlagString(cmd, "www"), apiProxyPrefix, staticPrefix, filter, clientConfig)
 	if err != nil {
 		return err
 	}
 
-	glog.Fatal(server.Serve(port))
+	// Separate listening from serving so we can report the bound port
+	// when it is chosen by os (port == 0)
+	l, err := server.Listen()
+	if err != nil {
+		glog.Fatal(err)
+	}
+	fmt.Fprintf(out, "Starting to serve on %s", l.Addr().String())
+	glog.Fatal(server.ServeOnListener(l))
 	return nil
 }

pkg/kubectl/proxy_server.go

@@ -139,12 +139,13 @@ func (f *FilterServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 // ProxyServer is a http.Handler which proxies Kubernetes APIs to remote API server.
 type ProxyServer struct {
 	handler http.Handler
+	port    int
 }
 
 // NewProxyServer creates and installs a new ProxyServer.
 // It automatically registers the created ProxyServer to http.DefaultServeMux.
 // 'filter', if non-nil, protects requests to the api only.
-func NewProxyServer(filebase string, apiProxyPrefix string, staticPrefix string, filter *FilterServer, cfg *client.Config) (*ProxyServer, error) {
+func NewProxyServer(port int, filebase string, apiProxyPrefix string, staticPrefix string, filter *FilterServer, cfg *client.Config) (*ProxyServer, error) {
 	host := cfg.Host
 	if !strings.HasSuffix(host, "/") {
 		host = host + "/"
@@ -173,16 +174,20 @@ func NewProxyServer(filebase string, apiProxyPrefix string, staticPrefix string,
 		// serving their working directory by default.
 		mux.Handle(staticPrefix, newFileHandler(staticPrefix, filebase))
 	}
-	return &ProxyServer{handler: mux}, nil
+	return &ProxyServer{handler: mux, port: port}, nil
 }
 
-// Serve starts the server (http.DefaultServeMux) on given port, loops forever.
-func (s *ProxyServer) Serve(port int) error {
+// Listen is a simple wrapper around net.Listen.
+func (s *ProxyServer) Listen() (net.Listener, error) {
+	return net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", s.port))
+}
+
+// Serve starts the server using given listener, loops forever.
+func (s *ProxyServer) ServeOnListener(l net.Listener) error {
 	server := http.Server{
-		Addr:    fmt.Sprintf(":%d", port),
 		Handler: s.handler,
 	}
-	return server.ListenAndServe()
+	return server.Serve(l)
 }
 
 func newProxy(target *url.URL) *httputil.ReverseProxy {

pkg/kubectl/proxy_server_test.go

@@ -287,7 +287,7 @@ func TestPathHandling(t *testing.T) {
 
 	for _, item := range table {
 		func() {
-			p, err := NewProxyServer("", item.prefix, "/not/used/for/this/test", nil, cc)
+			p, err := NewProxyServer(0, "", item.prefix, "/not/used/for/this/test", nil, cc)
 			if err != nil {
 				t.Fatalf("%#v: %v", item, err)
 			}

test/e2e/kubectl.go

@@ -22,8 +22,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -55,6 +57,7 @@ const (
 
 var (
 	portForwardRegexp = regexp.MustCompile("Forwarding from 127.0.0.1:([0-9]+) -> 80")
+	proxyRegexp       = regexp.MustCompile("Starting to serve on 127.0.0.1:([0-9]+)")
 )
 
 var _ = Describe("Kubectl client", func() {
@@ -165,11 +168,7 @@ var _ = Describe("Kubectl client", func() {
 		It("should support port-forward", func() {
 			By("forwarding the container port to a local port")
 			cmd := kubectlCmd("port-forward", fmt.Sprintf("--namespace=%v", ns), "-p", simplePodName, fmt.Sprintf(":%d", simplePodPort))
-			defer func() {
-				if err := cmd.Process.Kill(); err != nil {
-					Logf("ERROR failed to kill kubectl port-forward command! The process may leak")
-				}
-			}()
+			defer tryKill(cmd)
 			// This is somewhat ugly but is the only way to retrieve the port that was picked
 			// by the port-forward command. We don't want to hard code the port as we have no
 			// way of guaranteeing we can pick one that isn't in use, particularly on Jenkins.
@@ -195,6 +194,7 @@ var _ = Describe("Kubectl client", func() {
 			By("curling local port output")
 			localAddr := fmt.Sprintf("http://localhost:%s", match[1])
 			body, err := curl(localAddr)
+			Logf("got: %s", body)
 			if err != nil {
 				Failf("Failed http.Get of forwarded port (%s): %v", localAddr, err)
 			}
@@ -523,6 +523,27 @@ var _ = Describe("Kubectl client", func() {
 				Failf("Failed creating 1 pod with expected image %s. Number of pods = %v", image, len(pods))
 			}
 		})
+
+	})
+
+	Describe("Proxy server", func() {
+		// TODO: test proxy options (static, prefix, etc)
+		It("should support proxy with --port 0", func() {
+			By("starting the proxy server")
+			port, cmd, err := startProxyServer()
+			if cmd != nil {
+				defer tryKill(cmd)
+			}
+			if err != nil {
+				Failf("Failed to start proxy server: %v", err)
+			}
+			By("curling proxy /api/ output")
+			localAddr := fmt.Sprintf("http://localhost:%d/api/", port)
+			apiVersions, err := getAPIVersions(localAddr)
+			if len(apiVersions.Versions) < 1 {
+				Failf("Expected at least one supported apiversion, got %v", apiVersions)
+			}
+		})
 	})
 
 })
@@ -546,6 +567,42 @@ func checkOutput(output string, required [][]string) {
 	}
 }
 
+func getAPIVersions(apiEndpoint string) (*api.APIVersions, error) {
+	body, err := curl(apiEndpoint)
+	if err != nil {
+		return nil, fmt.Errorf("Failed http.Get of %s: %v", apiEndpoint, err)
+	}
+	var apiVersions api.APIVersions
+	if err := json.Unmarshal([]byte(body), &apiVersions); err != nil {
+		return nil, fmt.Errorf("Failed to parse /api output %s: %v", body, err)
+	}
+	return &apiVersions, nil
+}
+
+func startProxyServer() (int, *exec.Cmd, error) {
+	// Specifying port 0 indicates we want the os to pick a random port.
+	cmd := kubectlCmd("proxy", "-p", "0")
+	stdout, stderr, err := startCmdAndStreamOutput(cmd)
+	if err != nil {
+		return -1, nil, err
+	}
+	defer stdout.Close()
+	defer stderr.Close()
+	buf := make([]byte, 128)
+	var n int
+	if n, err = stdout.Read(buf); err != nil {
+		return -1, cmd, fmt.Errorf("Failed to read from kubectl proxy stdout: %v", err)
+	}
+	output := string(buf[:n])
+	match := proxyRegexp.FindStringSubmatch(output)
+	if len(match) == 2 {
+		if port, err := strconv.Atoi(match[1]); err == nil {
+			return port, cmd, nil
+		}
+	}
+	return -1, cmd, fmt.Errorf("Failed to parse port from proxy stdout: %s", output)
+}
+
 func curl(addr string) (string, error) {
 	resp, err := http.Get(addr)
 	if err != nil {

test/e2e/util.go

@@ -887,6 +887,13 @@ func startCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err e
 	return
 }
 
+// Rough equivalent of ctrl+c for cleaning up processes. Intended to be run in defer.
+func tryKill(cmd *exec.Cmd) {
+	if err := cmd.Process.Kill(); err != nil {
+		Logf("ERROR failed to kill command %v! The process may leak", cmd)
+	}
+}
+
 // testContainerOutputInNamespace runs the given pod in the given namespace and waits
 // for all of the containers in the podSpec to move into the 'Success' status.  It retrieves
 // the exact container log and searches for lines of expected output.