From f71a662dc6c77ffd7c13a0b6d14b6a7d013d19ae Mon Sep 17 00:00:00 2001 From: Jeff Lowdermilk Date: Mon, 6 Jul 2015 22:04:39 -0700 Subject: [PATCH 1/2] Make `kubectl proxy` support picking a random port --- docs/man/man1/kubectl-proxy.1 | 6 +++++- docs/user-guide/kubectl/kubectl_proxy.md | 6 +++++- pkg/kubectl/cmd/proxy.go | 18 ++++++++++++++---- pkg/kubectl/proxy_server.go | 17 +++++++++++------ pkg/kubectl/proxy_server_test.go | 2 +- 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/docs/man/man1/kubectl-proxy.1 b/docs/man/man1/kubectl-proxy.1 index 81a8bae67d4..3200c91c131 100644 --- a/docs/man/man1/kubectl-proxy.1 +++ b/docs/man/man1/kubectl-proxy.1 @@ -60,7 +60,7 @@ The above lets you 'curl localhost:8001/custom/api/v1/pods' .PP \fB\-p\fP, \fB\-\-port\fP=8001 - The port on which to run the proxy. + The port on which to run the proxy. Set to 0 to pick a random port. .PP \fB\-\-reject\-methods\fP="POST,PUT,PATCH" @@ -185,6 +185,10 @@ The above lets you 'curl localhost:8001/custom/api/v1/pods' // Run a proxy to kubernetes apiserver on port 8011, serving static content from ./local/www/ $ kubectl proxy \-\-port=8011 \-\-www=./local/www/ +// Run a proxy to kubernetes apiserver on an arbitrary local port. +// The chosen port for the server will be output to stdout. +$ kubectl proxy \-\-port=0 + // Run a proxy to kubernetes apiserver, changing the api prefix to k8s\-api // This makes e.g. the pods api available at localhost:8011/k8s\-api/v1/pods/ $ kubectl proxy \-\-api\-prefix=/k8s\-api diff --git a/docs/user-guide/kubectl/kubectl_proxy.md b/docs/user-guide/kubectl/kubectl_proxy.md index 2c5cb2eb9a9..60fcb0274c1 100644 --- a/docs/user-guide/kubectl/kubectl_proxy.md +++ b/docs/user-guide/kubectl/kubectl_proxy.md @@ -65,6 +65,10 @@ kubectl proxy [--port=PORT] [--www=static-dir] [--www-prefix=prefix] [--api-pref // Run a proxy to kubernetes apiserver on port 8011, serving static content from ./local/www/ $ kubectl proxy --port=8011 --www=./local/www/ +// Run a proxy to kubernetes apiserver on an arbitrary local port. +// The chosen port for the server will be output to stdout. +$ kubectl proxy --port=0 + // Run a proxy to kubernetes apiserver, changing the api prefix to k8s-api // This makes e.g. the pods api available at localhost:8011/k8s-api/v1/pods/ $ kubectl proxy --api-prefix=/k8s-api @@ -78,7 +82,7 @@ $ kubectl proxy --api-prefix=/k8s-api --api-prefix="/api/": Prefix to serve the proxied API under. --disable-filter=false: If true, disable request filtering in the proxy. This is dangerous, and can leave you vulnerable to XSRF attacks. Use with caution. -h, --help=false: help for proxy - -p, --port=8001: The port on which to run the proxy. + -p, --port=8001: The port on which to run the proxy. Set to 0 to pick a random port. --reject-methods="POST,PUT,PATCH": Regular expression for HTTP methods that the proxy should reject. --reject-paths="^/api/.*/exec,^/api/.*/run": Regular expression for paths that the proxy should reject. -w, --www="": Also serve static files from the given directory under the specified prefix. diff --git a/pkg/kubectl/cmd/proxy.go b/pkg/kubectl/cmd/proxy.go index bddb569a738..16ced295bc1 100644 --- a/pkg/kubectl/cmd/proxy.go +++ b/pkg/kubectl/cmd/proxy.go @@ -31,6 +31,10 @@ const ( proxy_example = `// Run a proxy to kubernetes apiserver on port 8011, serving static content from ./local/www/ $ kubectl proxy --port=8011 --www=./local/www/ +// Run a proxy to kubernetes apiserver on an arbitrary local port. +// The chosen port for the server will be output to stdout. +$ kubectl proxy --port=0 + // Run a proxy to kubernetes apiserver, changing the api prefix to k8s-api // This makes e.g. the pods api available at localhost:8011/k8s-api/v1/pods/ $ kubectl proxy --api-prefix=/k8s-api` @@ -69,14 +73,13 @@ The above lets you 'curl localhost:8001/custom/api/v1/pods' cmd.Flags().String("reject-paths", kubectl.DefaultPathRejectRE, "Regular expression for paths that the proxy should reject.") cmd.Flags().String("accept-hosts", kubectl.DefaultHostAcceptRE, "Regular expression for hosts that the proxy should accept.") cmd.Flags().String("reject-methods", kubectl.DefaultMethodRejectRE, "Regular expression for HTTP methods that the proxy should reject.") - cmd.Flags().IntP("port", "p", 8001, "The port on which to run the proxy.") + cmd.Flags().IntP("port", "p", 8001, "The port on which to run the proxy. Set to 0 to pick a random port.") cmd.Flags().Bool("disable-filter", false, "If true, disable request filtering in the proxy. This is dangerous, and can leave you vulnerable to XSRF attacks. Use with caution.") return cmd } func RunProxy(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command) error { port := cmdutil.GetFlagInt(cmd, "port") - fmt.Fprintf(out, "Starting to serve on localhost:%d", port) clientConfig, err := f.ClientConfig() if err != nil { @@ -102,11 +105,18 @@ func RunProxy(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command) error { filter = nil } - server, err := kubectl.NewProxyServer(cmdutil.GetFlagString(cmd, "www"), apiProxyPrefix, staticPrefix, filter, clientConfig) + server, err := kubectl.NewProxyServer(port, cmdutil.GetFlagString(cmd, "www"), apiProxyPrefix, staticPrefix, filter, clientConfig) if err != nil { return err } - glog.Fatal(server.Serve(port)) + // Separate listening from serving so we can report the bound port + // when it is chosen by os (port == 0) + l, err := server.Listen() + if err != nil { + glog.Fatal(err) + } + fmt.Fprintf(out, "Starting to serve on %s", l.Addr().String()) + glog.Fatal(server.ServeOnListener(l)) return nil } diff --git a/pkg/kubectl/proxy_server.go b/pkg/kubectl/proxy_server.go index 0d039325eed..9a468e38957 100644 --- a/pkg/kubectl/proxy_server.go +++ b/pkg/kubectl/proxy_server.go @@ -139,12 +139,13 @@ func (f *FilterServer) ServeHTTP(rw http.ResponseWriter, req *http.Request) { // ProxyServer is a http.Handler which proxies Kubernetes APIs to remote API server. type ProxyServer struct { handler http.Handler + port int } // NewProxyServer creates and installs a new ProxyServer. // It automatically registers the created ProxyServer to http.DefaultServeMux. // 'filter', if non-nil, protects requests to the api only. -func NewProxyServer(filebase string, apiProxyPrefix string, staticPrefix string, filter *FilterServer, cfg *client.Config) (*ProxyServer, error) { +func NewProxyServer(port int, filebase string, apiProxyPrefix string, staticPrefix string, filter *FilterServer, cfg *client.Config) (*ProxyServer, error) { host := cfg.Host if !strings.HasSuffix(host, "/") { host = host + "/" @@ -173,16 +174,20 @@ func NewProxyServer(filebase string, apiProxyPrefix string, staticPrefix string, // serving their working directory by default. mux.Handle(staticPrefix, newFileHandler(staticPrefix, filebase)) } - return &ProxyServer{handler: mux}, nil + return &ProxyServer{handler: mux, port: port}, nil } -// Serve starts the server (http.DefaultServeMux) on given port, loops forever. -func (s *ProxyServer) Serve(port int) error { +// Listen is a simple wrapper around net.Listen. +func (s *ProxyServer) Listen() (net.Listener, error) { + return net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", s.port)) +} + +// Serve starts the server using given listener, loops forever. +func (s *ProxyServer) ServeOnListener(l net.Listener) error { server := http.Server{ - Addr: fmt.Sprintf(":%d", port), Handler: s.handler, } - return server.ListenAndServe() + return server.Serve(l) } func newProxy(target *url.URL) *httputil.ReverseProxy { diff --git a/pkg/kubectl/proxy_server_test.go b/pkg/kubectl/proxy_server_test.go index 883c083562a..a943c528b7e 100644 --- a/pkg/kubectl/proxy_server_test.go +++ b/pkg/kubectl/proxy_server_test.go @@ -287,7 +287,7 @@ func TestPathHandling(t *testing.T) { for _, item := range table { func() { - p, err := NewProxyServer("", item.prefix, "/not/used/for/this/test", nil, cc) + p, err := NewProxyServer(0, "", item.prefix, "/not/used/for/this/test", nil, cc) if err != nil { t.Fatalf("%#v: %v", item, err) } From 69166f17ad73412e0ab54475c5e5318e1f709e7b Mon Sep 17 00:00:00 2001 From: Jeff Lowdermilk Date: Mon, 6 Jul 2015 22:09:22 -0700 Subject: [PATCH 2/2] e2e test for `kubectl proxy` --- test/e2e/kubectl.go | 67 +++++++++++++++++++++++++++++++++++++++++---- test/e2e/util.go | 7 +++++ 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index c309bcecb14..29f069ee6d1 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -22,8 +22,10 @@ import ( "fmt" "io/ioutil" "net/http" + "os/exec" "path/filepath" "regexp" + "strconv" "strings" "time" @@ -55,6 +57,7 @@ const ( var ( portForwardRegexp = regexp.MustCompile("Forwarding from 127.0.0.1:([0-9]+) -> 80") + proxyRegexp = regexp.MustCompile("Starting to serve on 127.0.0.1:([0-9]+)") ) var _ = Describe("Kubectl client", func() { @@ -165,11 +168,7 @@ var _ = Describe("Kubectl client", func() { It("should support port-forward", func() { By("forwarding the container port to a local port") cmd := kubectlCmd("port-forward", fmt.Sprintf("--namespace=%v", ns), "-p", simplePodName, fmt.Sprintf(":%d", simplePodPort)) - defer func() { - if err := cmd.Process.Kill(); err != nil { - Logf("ERROR failed to kill kubectl port-forward command! The process may leak") - } - }() + defer tryKill(cmd) // This is somewhat ugly but is the only way to retrieve the port that was picked // by the port-forward command. We don't want to hard code the port as we have no // way of guaranteeing we can pick one that isn't in use, particularly on Jenkins. @@ -195,6 +194,7 @@ var _ = Describe("Kubectl client", func() { By("curling local port output") localAddr := fmt.Sprintf("http://localhost:%s", match[1]) body, err := curl(localAddr) + Logf("got: %s", body) if err != nil { Failf("Failed http.Get of forwarded port (%s): %v", localAddr, err) } @@ -523,6 +523,27 @@ var _ = Describe("Kubectl client", func() { Failf("Failed creating 1 pod with expected image %s. Number of pods = %v", image, len(pods)) } }) + + }) + + Describe("Proxy server", func() { + // TODO: test proxy options (static, prefix, etc) + It("should support proxy with --port 0", func() { + By("starting the proxy server") + port, cmd, err := startProxyServer() + if cmd != nil { + defer tryKill(cmd) + } + if err != nil { + Failf("Failed to start proxy server: %v", err) + } + By("curling proxy /api/ output") + localAddr := fmt.Sprintf("http://localhost:%d/api/", port) + apiVersions, err := getAPIVersions(localAddr) + if len(apiVersions.Versions) < 1 { + Failf("Expected at least one supported apiversion, got %v", apiVersions) + } + }) }) }) @@ -546,6 +567,42 @@ func checkOutput(output string, required [][]string) { } } +func getAPIVersions(apiEndpoint string) (*api.APIVersions, error) { + body, err := curl(apiEndpoint) + if err != nil { + return nil, fmt.Errorf("Failed http.Get of %s: %v", apiEndpoint, err) + } + var apiVersions api.APIVersions + if err := json.Unmarshal([]byte(body), &apiVersions); err != nil { + return nil, fmt.Errorf("Failed to parse /api output %s: %v", body, err) + } + return &apiVersions, nil +} + +func startProxyServer() (int, *exec.Cmd, error) { + // Specifying port 0 indicates we want the os to pick a random port. + cmd := kubectlCmd("proxy", "-p", "0") + stdout, stderr, err := startCmdAndStreamOutput(cmd) + if err != nil { + return -1, nil, err + } + defer stdout.Close() + defer stderr.Close() + buf := make([]byte, 128) + var n int + if n, err = stdout.Read(buf); err != nil { + return -1, cmd, fmt.Errorf("Failed to read from kubectl proxy stdout: %v", err) + } + output := string(buf[:n]) + match := proxyRegexp.FindStringSubmatch(output) + if len(match) == 2 { + if port, err := strconv.Atoi(match[1]); err == nil { + return port, cmd, nil + } + } + return -1, cmd, fmt.Errorf("Failed to parse port from proxy stdout: %s", output) +} + func curl(addr string) (string, error) { resp, err := http.Get(addr) if err != nil { diff --git a/test/e2e/util.go b/test/e2e/util.go index ce0bbef1983..cb714030f85 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -887,6 +887,13 @@ func startCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err e return } +// Rough equivalent of ctrl+c for cleaning up processes. Intended to be run in defer. +func tryKill(cmd *exec.Cmd) { + if err := cmd.Process.Kill(); err != nil { + Logf("ERROR failed to kill command %v! The process may leak", cmd) + } +} + // testContainerOutputInNamespace runs the given pod in the given namespace and waits // for all of the containers in the podSpec to move into the 'Success' status. It retrieves // the exact container log and searches for lines of expected output.