From 53f9d42ed3c83cff5a33feae0fd8249ff20c2f86 Mon Sep 17 00:00:00 2001 From: Eric Tune Date: Wed, 5 Nov 2014 16:05:06 -0800 Subject: [PATCH] Add a third port which has HTTPS and auth(n,z) It is disabled by default. Document all the various and sundry (3) ports. --- cmd/apiserver/apiserver.go | 36 ++++++++++++++++++-- docs/accessing_the_api.md | 70 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 docs/accessing_the_api.md diff --git a/cmd/apiserver/apiserver.go b/cmd/apiserver/apiserver.go index 4aca7c1232b..cdca499d1eb 100644 --- a/cmd/apiserver/apiserver.go +++ b/cmd/apiserver/apiserver.go @@ -57,15 +57,18 @@ var ( "The port from which to serve read-only resources. If 0, don't serve on a "+ "read-only address. It is assumed that firewall rules are set up such that "+ "this port is not reachable from outside of the cluster.") + securePort = flag.Int("secure_port", 0, "The port from which to serve HTTPS with authentication and authorization. If 0, don't serve HTTPS ") + tlsCertFile = flag.String("tls_cert_file", "", "File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert).") + tlsPrivateKeyFile = flag.String("tls_private_key_file", "", "File containing x509 private key matching --tls_cert_file.") apiPrefix = flag.String("api_prefix", "/api", "The prefix for API requests on the server. Default '/api'.") storageVersion = flag.String("storage_version", "", "The version to store resources with. Defaults to server preferred") cloudProvider = flag.String("cloud_provider", "", "The provider for cloud services. Empty string for no provider.") cloudConfigFile = flag.String("cloud_config", "", "The path to the cloud provider configuration file. Empty string for no configuration file.") healthCheckMinions = flag.Bool("health_check_minions", true, "If true, health check minions and filter unhealthy ones. Default true.") eventTTL = flag.Duration("event_ttl", 48*time.Hour, "Amount of time to retain events. Default 2 days.") - tokenAuthFile = flag.String("token_auth_file", "", "If set, the file that will be used to secure the API server via token authentication.") - authorizationMode = flag.String("authorization_mode", "AlwaysAllow", "Selects how to do authorization. One of: "+strings.Join(apiserver.AuthorizationModeChoices, ",")) - authorizationPolicyFile = flag.String("authorization_policy_file", "", "File with authorization policy in csv format, used with --authorization_mode=ABAC.") + tokenAuthFile = flag.String("token_auth_file", "", "If set, the file that will be used to secure the secure port of the API server via token authentication.") + authorizationMode = flag.String("authorization_mode", "AlwaysAllow", "Selects how to do authorization on the secure port. One of: "+strings.Join(apiserver.AuthorizationModeChoices, ",")) + authorizationPolicyFile = flag.String("authorization_policy_file", "", "File with authorization policy in csv format, used with --authorization_mode=ABAC, on the secure port.") etcdServerList util.StringList etcdConfigFile = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers.") corsAllowedOriginList util.StringList @@ -172,10 +175,15 @@ func main() { } m := master.New(config) + // We serve on 3 ports. See docs/reaching_the_api.md roLocation := "" if *readOnlyPort != 0 { roLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(config.ReadOnlyPort)) } + secureLocation := "" + if *securePort != 0 { + secureLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(*securePort)) + } rwLocation := net.JoinHostPort(address.String(), strconv.Itoa(int(*port))) // See the flag commentary to understand our assumptions when opening the read-only and read-write ports. @@ -190,6 +198,7 @@ func main() { WriteTimeout: 5 * time.Minute, MaxHeaderBytes: 1 << 20, } + glog.Infof("Serving read-only insecurely on %s", roLocation) go func() { defer util.HandleCrash() for { @@ -201,6 +210,26 @@ func main() { }() } + if secureLocation != "" { + secureServer := &http.Server{ + Addr: secureLocation, + Handler: apiserver.RecoverPanics(m.Handler), + ReadTimeout: 5 * time.Minute, + WriteTimeout: 5 * time.Minute, + MaxHeaderBytes: 1 << 20, + } + glog.Infof("Serving securely on %s", secureLocation) + go func() { + defer util.HandleCrash() + for { + if err := secureServer.ListenAndServeTLS(*tlsCertFile, *tlsPrivateKeyFile); err != nil { + glog.Errorf("Unable to listen for secure (%v); will try again.", err) + } + time.Sleep(15 * time.Second) + } + }() + } + s := &http.Server{ Addr: rwLocation, Handler: apiserver.RecoverPanics(m.InsecureHandler), @@ -208,5 +237,6 @@ func main() { WriteTimeout: 5 * time.Minute, MaxHeaderBytes: 1 << 20, } + glog.Infof("Serving insecurely on %s", rwLocation) glog.Fatal(s.ListenAndServe()) } diff --git a/docs/accessing_the_api.md b/docs/accessing_the_api.md new file mode 100644 index 00000000000..c9124b004b7 --- /dev/null +++ b/docs/accessing_the_api.md @@ -0,0 +1,70 @@ +# Reaching the API + +## Ports and IPs Served On +The Kubernetes API is served by the Kubernetes APIServer process. Typically, +there is one of these running on a single kubernetes-master node. + +By default the Kubernetes APIserver serves +HTTP on 3 ports: + 1. Localhost Port + - serves HTTP + - default is port 8080, change with `-port` flag. + - defaults IP is localhost, change with `-address` flag. + - no authentication or authorization checks in HTTP + - protected by need to have host access + 2. ReadOnly Port + - default is port 7080, change with `-read_only_port` + - default IP is first non-localhost network interface, change with `-public_address_override` + - serves HTTP + - no authentication checks in HTTP + - only GET requests are allowed. + - requests are rate limited + 3. Secure Port + - default is port 6443, change with `-secure_port` + - default IP is first non-localhost network interface, change with `-public_address_override` + - serves HTTPS. Set cert with `-tls_cert_file` and key with `-tls_private_key_file`. + - uses token-file based [authentication](./authentication.md). + - uses policy-based [authorization](./authorization.md). + +## Proxies and Firewall rules + +Additionally, in typical configurations (i.e. GCE), there is a proxy (nginx) running +on the same machine as the apiserver process. The proxy serves HTTPS protected +by Basic Auth on port 443, and proxies to the apiserver on localhost:8080. +Typically, firewall rules will allow HTTPs access to port 443. + +The above are defaults and reflect how Kubernetes is deployed to GCE using +kube-up.sh. Other cloud providers may vary. + +## Use Cases vs IP:Ports + +There are three differently configured serving ports because there are a +variety of uses cases: + 1. Clients outside of a Kubernetes cluster, such as human running `kubectl` + on desktop machine. Currently, accesses the Localhost Port via a proxy (nginx) + running on the `kubernetes-master` machine. Proxy uses Basic Auth. + 2. Processes running in Containers on Kubernetes that need to do read from + the apiserver. Currently, these can use Readonly Port. + 3. Scheduler and Controller-manager processes, which need to do read-write + API operations. Currently, these have to run on the + operations on the apiserver. Currently, these have to run on the same + host as the apiserver and use the Localhost Port. + 4. Kubelets, which need to do read-write API operations and are necessarily + on different machines than the apiserver. Currently, kubelets do not + use the API. + +## Expected Changes. +The following changes to what is decribed above are planned: + - Kubelets will soon begin using the Secure Port to get their pods and + report events. Credentials will be distributed to kubelets at cluster + setup time initially. Policy will limit the actions kubelets can do. + - Scheduler and Controller-manager will use the Secure Port too. They + will then be able to run on different machines than the apiserver. + - A general mechanism will be provided for [giving credentials to + pods]( + https://github.com/GoogleCloudPlatform/kubernetes/issues/1907). + - The Readonly Port will no longer be needed and will be removed. + - Clients, like kubectl, will all support token-based auth, and the + Localhost will no longer be needed, and will not be the default. + However, the localhost port may continue to be an option for + installations that want to do their own auth proxy.