diff --git a/cmd/hyperkube/hyperkube.go b/cmd/hyperkube/hyperkube.go new file mode 100644 index 00000000000..fc00b551107 --- /dev/null +++ b/cmd/hyperkube/hyperkube.go @@ -0,0 +1,37 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// A binary that can morph into all of the other kubernetes binaries. You can +// also soft-link to it busybox style. +package main + +import ( + "os" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/hyperkube" + apiserver "github.com/GoogleCloudPlatform/kubernetes/pkg/master/server" +) + +func main() { + hk := hyperkube.HyperKube{ + Name: "hyperkube", + Long: "This is an all-in-one binary that can run any of the various Kubernetes servers.", + } + + hk.AddServer(apiserver.NewHyperkubeServer()) + + hk.RunToExit(os.Args) +} diff --git a/cmd/kube-apiserver/apiserver.go b/cmd/kube-apiserver/apiserver.go index a4366f15f61..3e10d4c0256 100644 --- a/cmd/kube-apiserver/apiserver.go +++ b/cmd/kube-apiserver/apiserver.go @@ -19,268 +19,22 @@ limitations under the License. package main import ( - "crypto/tls" - "net" - "net/http" - "strconv" - "strings" - "time" - - "github.com/GoogleCloudPlatform/kubernetes/pkg/admission" - "github.com/GoogleCloudPlatform/kubernetes/pkg/api" - "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" - "github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities" - "github.com/GoogleCloudPlatform/kubernetes/pkg/client" - "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider" - "github.com/GoogleCloudPlatform/kubernetes/pkg/master" - "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" + "github.com/GoogleCloudPlatform/kubernetes/pkg/master/server" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag" - "github.com/coreos/go-etcd/etcd" - "github.com/golang/glog" - flag "github.com/spf13/pflag" + "github.com/spf13/pflag" ) -var ( - // Note: the weird ""+ in below lines seems to be the only way to get gofmt to - // arrange these text blocks sensibly. Grrr. - port = flag.Int("port", 8080, ""+ - "The port to listen on. Default 8080. It is assumed that firewall rules are "+ - "set up such that this port is not reachable from outside of the cluster. It is "+ - "further assumed that port 443 on the cluster's public address is proxied to this "+ - "port. This is performed by nginx in the default setup.") - address = util.IP(net.ParseIP("127.0.0.1")) - publicAddressOverride = flag.String("public_address_override", "", ""+ - "Public serving address. Read only port will be opened on this address, "+ - "and it is assumed that port 443 at this address will be proxied/redirected "+ - "to '-address':'-port'. If blank, the address in the first listed interface "+ - "will be used.") - readOnlyPort = flag.Int("read_only_port", 7080, ""+ - "The port from which to serve read-only resources. If 0, don't serve on a "+ - "read-only address. It is assumed that firewall rules are set up such that "+ - "this port is not reachable from outside of the cluster.") - apiRate = flag.Float32("api_rate", 10.0, "API rate limit as QPS for the read only port") - apiBurst = flag.Int("api_burst", 200, "API burst amount for the read only port") - securePort = flag.Int("secure_port", 8443, "The port from which to serve HTTPS with authentication and authorization. If 0, don't serve HTTPS ") - tlsCertFile = flag.String("tls_cert_file", "", ""+ - "File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert). "+ - "If HTTPS serving is enabled, and --tls_cert_file and --tls_private_key_file are not provided, "+ - "a self-signed certificate and key are generated for the public address and saved to /var/run/kubernetes.") - tlsPrivateKeyFile = flag.String("tls_private_key_file", "", "File containing x509 private key matching --tls_cert_file.") - apiPrefix = flag.String("api_prefix", "/api", "The prefix for API requests on the server. Default '/api'.") - storageVersion = flag.String("storage_version", "", "The version to store resources with. Defaults to server preferred") - cloudProvider = flag.String("cloud_provider", "", "The provider for cloud services. Empty string for no provider.") - cloudConfigFile = flag.String("cloud_config", "", "The path to the cloud provider configuration file. Empty string for no configuration file.") - healthCheckMinions = flag.Bool("health_check_minions", true, "If true, health check minions and filter unhealthy ones. Default true.") - eventTTL = flag.Duration("event_ttl", 48*time.Hour, "Amount of time to retain events. Default 2 days.") - tokenAuthFile = flag.String("token_auth_file", "", "If set, the file that will be used to secure the secure port of the API server via token authentication.") - authorizationMode = flag.String("authorization_mode", "AlwaysAllow", "Selects how to do authorization on the secure port. One of: "+strings.Join(apiserver.AuthorizationModeChoices, ",")) - authorizationPolicyFile = flag.String("authorization_policy_file", "", "File with authorization policy in csv format, used with --authorization_mode=ABAC, on the secure port.") - admissionControl = flag.String("admission_control", "AlwaysAdmit", "Ordered list of plug-ins to do admission control of resources into cluster. Comma-delimited list of: "+strings.Join(admission.GetPlugins(), ", ")) - admissionControlConfigFile = flag.String("admission_control_config_file", "", "File with admission control configuration.") - etcdServerList util.StringList - etcdConfigFile = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers.") - corsAllowedOriginList util.StringList - allowPrivileged = flag.Bool("allow_privileged", false, "If true, allow privileged containers.") - portalNet util.IPNet // TODO: make this a list - enableLogsSupport = flag.Bool("enable_logs_support", true, "Enables server endpoint for log collection") - runtimeConfig util.ConfigurationMap - kubeletConfig = client.KubeletConfig{ - Port: 10250, - EnableHttps: false, - } - masterServiceNamespace = flag.String("master_service_namespace", api.NamespaceDefault, "The namespace from which the kubernetes master services should be injected into pods") -) - -func init() { - runtimeConfig = make(util.ConfigurationMap) - - flag.Var(&address, "address", "The IP address on to serve on (set to 0.0.0.0 for all interfaces)") - flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd_config") - flag.Var(&corsAllowedOriginList, "cors_allowed_origins", "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.") - flag.Var(&portalNet, "portal_net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.") - flag.Var(&runtimeConfig, "runtime_config", "A set of key=value pairs that describe runtime configuration that may be passed to the apiserver.") - client.BindKubeletClientConfigFlags(flag.CommandLine, &kubeletConfig) -} - -// TODO: Longer term we should read this from some config store, rather than a flag. -func verifyPortalFlags() { - if portalNet.IP == nil { - glog.Fatal("No -portal_net specified") - } -} - -func newEtcd(etcdConfigFile string, etcdServerList util.StringList) (helper tools.EtcdHelper, err error) { - var client tools.EtcdGetSet - if etcdConfigFile != "" { - client, err = etcd.NewClientFromFile(etcdConfigFile) - if err != nil { - return helper, err - } - } else { - client = etcd.NewClient(etcdServerList) - } - - return master.NewEtcdHelper(client, *storageVersion) -} - func main() { + s := server.NewAPIServer() + s.AddFlags(pflag.CommandLine) + util.InitFlags() util.InitLogs() defer util.FlushLogs() verflag.PrintAndExitIfRequested() - verifyPortalFlags() - if (*etcdConfigFile != "" && len(etcdServerList) != 0) || (*etcdConfigFile == "" && len(etcdServerList) == 0) { - glog.Fatalf("specify either -etcd_servers or -etcd_config") - } - - capabilities.Initialize(capabilities.Capabilities{ - AllowPrivileged: *allowPrivileged, - }) - - cloud := cloudprovider.InitCloudProvider(*cloudProvider, *cloudConfigFile) - - kubeletClient, err := client.NewKubeletClient(&kubeletConfig) - if err != nil { - glog.Fatalf("Failure to start kubelet client: %v", err) - } - - _, v1beta3 := runtimeConfig["api/v1beta3"] - - // TODO: expose same flags as client.BindClientConfigFlags but for a server - clientConfig := &client.Config{ - Host: net.JoinHostPort(address.String(), strconv.Itoa(int(*port))), - Version: *storageVersion, - } - client, err := client.New(clientConfig) - if err != nil { - glog.Fatalf("Invalid server address: %v", err) - } - - helper, err := newEtcd(*etcdConfigFile, etcdServerList) - if err != nil { - glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err) - } - - n := net.IPNet(portalNet) - - authenticator, err := apiserver.NewAuthenticatorFromTokenFile(*tokenAuthFile) - if err != nil { - glog.Fatalf("Invalid Authentication Config: %v", err) - } - - authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(*authorizationMode, *authorizationPolicyFile) - if err != nil { - glog.Fatalf("Invalid Authorization Config: %v", err) - } - - admissionControlPluginNames := strings.Split(*admissionControl, ",") - admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, *admissionControlConfigFile) - - config := &master.Config{ - Client: client, - Cloud: cloud, - EtcdHelper: helper, - EventTTL: *eventTTL, - KubeletClient: kubeletClient, - PortalNet: &n, - EnableLogsSupport: *enableLogsSupport, - EnableUISupport: true, - EnableSwaggerSupport: true, - APIPrefix: *apiPrefix, - CorsAllowedOriginList: corsAllowedOriginList, - ReadOnlyPort: *readOnlyPort, - ReadWritePort: *port, - PublicAddress: *publicAddressOverride, - Authenticator: authenticator, - Authorizer: authorizer, - AdmissionControl: admissionController, - EnableV1Beta3: v1beta3, - MasterServiceNamespace: *masterServiceNamespace, - } - m := master.New(config) - - // We serve on 3 ports. See docs/reaching_the_api.md - roLocation := "" - if *readOnlyPort != 0 { - roLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(config.ReadOnlyPort)) - } - secureLocation := "" - if *securePort != 0 { - secureLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(*securePort)) - } - rwLocation := net.JoinHostPort(address.String(), strconv.Itoa(int(*port))) - - // See the flag commentary to understand our assumptions when opening the read-only and read-write ports. - - if roLocation != "" { - // Default settings allow 10 read-only requests per second, allow up to 200 in a burst before enforcing. - rl := util.NewTokenBucketRateLimiter(*apiRate, *apiBurst) - readOnlyServer := &http.Server{ - Addr: roLocation, - Handler: apiserver.RecoverPanics(apiserver.ReadOnly(apiserver.RateLimit(rl, m.InsecureHandler))), - ReadTimeout: 5 * time.Minute, - WriteTimeout: 5 * time.Minute, - MaxHeaderBytes: 1 << 20, - } - glog.Infof("Serving read-only insecurely on %s", roLocation) - go func() { - defer util.HandleCrash() - for { - if err := readOnlyServer.ListenAndServe(); err != nil { - glog.Errorf("Unable to listen for read only traffic (%v); will try again.", err) - } - time.Sleep(15 * time.Second) - } - }() - } - - if secureLocation != "" { - secureServer := &http.Server{ - Addr: secureLocation, - Handler: apiserver.RecoverPanics(m.Handler), - ReadTimeout: 5 * time.Minute, - WriteTimeout: 5 * time.Minute, - MaxHeaderBytes: 1 << 20, - TLSConfig: &tls.Config{ - // Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability) - MinVersion: tls.VersionTLS10, - // Populate PeerCertificates in requests, but don't reject connections without certificates - // This allows certificates to be validated by authenticators, while still allowing other auth types - ClientAuth: tls.RequestClientCert, - }, - } - glog.Infof("Serving securely on %s", secureLocation) - go func() { - defer util.HandleCrash() - for { - if *tlsCertFile == "" && *tlsPrivateKeyFile == "" { - *tlsCertFile = "/var/run/kubernetes/apiserver.crt" - *tlsPrivateKeyFile = "/var/run/kubernetes/apiserver.key" - if err := util.GenerateSelfSignedCert(config.PublicAddress, *tlsCertFile, *tlsPrivateKeyFile); err != nil { - glog.Errorf("Unable to generate self signed cert: %v", err) - } else { - glog.Infof("Using self-signed cert (%s, %s)", *tlsCertFile, *tlsPrivateKeyFile) - } - } - if err := secureServer.ListenAndServeTLS(*tlsCertFile, *tlsPrivateKeyFile); err != nil { - glog.Errorf("Unable to listen for secure (%v); will try again.", err) - } - time.Sleep(15 * time.Second) - } - }() - } - - s := &http.Server{ - Addr: rwLocation, - Handler: apiserver.RecoverPanics(m.InsecureHandler), - ReadTimeout: 5 * time.Minute, - WriteTimeout: 5 * time.Minute, - MaxHeaderBytes: 1 << 20, - } - glog.Infof("Serving insecurely on %s", rwLocation) - glog.Fatal(s.ListenAndServe()) + s.Run(pflag.CommandLine.Args()) } diff --git a/docs/man/kube-apiserver.1.md b/docs/man/kube-apiserver.1.md index 4543d2720c5..2cfbeed4388 100644 --- a/docs/man/kube-apiserver.1.md +++ b/docs/man/kube-apiserver.1.md @@ -38,9 +38,6 @@ The the kube-apiserver several options. **--etcd_servers**=[] List of etcd servers to watch (http://ip:port), comma separated -**--health_check_minions**= - If true, health check minions and filter unhealthy ones. Default true. - **--log_backtrace_at=**:0 when logging hits line file:N, emit a stack trace diff --git a/pkg/hyperkube/hyperkube.go b/pkg/hyperkube/hyperkube.go index accfe7176be..895ed726a2b 100644 --- a/pkg/hyperkube/hyperkube.go +++ b/pkg/hyperkube/hyperkube.go @@ -26,6 +26,7 @@ import ( "path" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag" "github.com/spf13/pflag" ) @@ -128,6 +129,9 @@ func (hk *HyperKube) Run(args []string) error { hk.Usage() return err } + + verflag.PrintAndExitIfRequested() + args = baseFlags.Args() if len(args) > 0 && len(args[0]) > 0 { serverName = args[0] @@ -135,7 +139,7 @@ func (hk *HyperKube) Run(args []string) error { args = args[1:] } else { err = errors.New("No server specified") - hk.Println("Error", err) + hk.Printf("Error: %v\n\n", err) hk.Usage() return err } @@ -143,7 +147,7 @@ func (hk *HyperKube) Run(args []string) error { s, err := hk.FindServer(serverName) if err != nil { - hk.Println("Error:", err) + hk.Printf("Error: %v\n\n", err) hk.Usage() return err } @@ -152,13 +156,17 @@ func (hk *HyperKube) Run(args []string) error { err = s.Flags().Parse(args) if err != nil || hk.helpFlagVal { if err != nil { - hk.Println("Error:", err) + hk.Printf("Error: %v\n\n", err) } s.Usage() return err } + verflag.PrintAndExitIfRequested() + util.InitLogs() + defer util.FlushLogs() + err = s.Run(s, s.Flags().Args()) if err != nil { hk.Println("Error:", err) @@ -167,6 +175,15 @@ func (hk *HyperKube) Run(args []string) error { return err } +// RunToExit will run the hyperkube and then call os.Exit with an appropriate exit code. +func (hk *HyperKube) RunToExit(args []string) { + err := hk.Run(args) + if err != nil { + os.Exit(1) + } + os.Exit(0) +} + // Usage will write out a summary for all servers that this binary supports. func (hk *HyperKube) Usage() { tt := `{{if .Long}}{{.Long | trim | wrap ""}} diff --git a/pkg/hyperkube/server.go b/pkg/hyperkube/server.go index 706df1e1921..697211b5f19 100644 --- a/pkg/hyperkube/server.go +++ b/pkg/hyperkube/server.go @@ -38,7 +38,7 @@ type Server struct { hk *HyperKube } -// FullUsage returns the full usage string including all of the flags. +// Usage returns the full usage string including all of the flags. func (s *Server) Usage() error { tt := `{{if .Long}}{{.Long | trim | wrap ""}} {{end}}Usage: diff --git a/cmd/kube-apiserver/plugins.go b/pkg/master/server/plugins.go similarity index 96% rename from cmd/kube-apiserver/plugins.go rename to pkg/master/server/plugins.go index 9eadcf7c95b..472adbe9246 100644 --- a/cmd/kube-apiserver/plugins.go +++ b/pkg/master/server/plugins.go @@ -14,18 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package server // This file exists to force the desired plugin implementations to be linked. // This should probably be part of some configuration fed into the build for a // given binary target. import ( + // Cloud providers _ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/aws" _ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/gce" _ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/openstack" _ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/ovirt" _ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/vagrant" + // Admission policies _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit" _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/deny" _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/limitranger" diff --git a/pkg/master/server/server.go b/pkg/master/server/server.go new file mode 100644 index 00000000000..9c95dfd4d53 --- /dev/null +++ b/pkg/master/server/server.go @@ -0,0 +1,344 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package server does all of the work necessary to create a Kubernetes +// APIServer by binding together the API, master and APIServer infrastructure. +// It can be configured and called directly or via the hyperkube framework. +package server + +import ( + "crypto/tls" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/admission" + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" + "github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider" + "github.com/GoogleCloudPlatform/kubernetes/pkg/hyperkube" + "github.com/GoogleCloudPlatform/kubernetes/pkg/master" + "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + "github.com/coreos/go-etcd/etcd" + "github.com/golang/glog" + "github.com/spf13/pflag" +) + +// APIServer runs a kubernetes api server. +type APIServer struct { + Port int + Address util.IP + PublicAddressOverride string + ReadOnlyPort int + APIRate float32 + APIBurst int + SecurePort int + TLSCertFile string + TLSPrivateKeyFile string + APIPrefix string + StorageVersion string + CloudProvider string + CloudConfigFile string + EventTTL time.Duration + TokenAuthFile string + AuthorizationMode string + AuthorizationPolicyFile string + AdmissionControl string + AdmissionControlConfigFile string + EtcdServerList util.StringList + EtcdConfigFile string + CorsAllowedOriginList util.StringList + AllowPrivileged bool + PortalNet util.IPNet // TODO: make this a list + EnableLogsSupport bool + MasterServiceNamespace string + RuntimeConfig util.ConfigurationMap + KubeletConfig client.KubeletConfig +} + +// NewAPIServer creates a new APIServer object with default parameters +func NewAPIServer() *APIServer { + s := APIServer{ + Port: 8080, + Address: util.IP(net.ParseIP("127.0.0.1")), + ReadOnlyPort: 7080, + APIRate: 10.0, + APIBurst: 200, + SecurePort: 8443, + APIPrefix: "/api", + EventTTL: 48 * time.Hour, + AuthorizationMode: "AlwaysAllow", + AdmissionControl: "AlwaysAdmit", + EnableLogsSupport: true, + MasterServiceNamespace: api.NamespaceDefault, + + RuntimeConfig: make(util.ConfigurationMap), + KubeletConfig: client.KubeletConfig{ + Port: 10250, + EnableHttps: false, + }, + } + + return &s +} + +// NewHyperkubeServer creates a new hyperkube Server object that includes the +// description and flags. +func NewHyperkubeServer() *hyperkube.Server { + s := NewAPIServer() + + hks := hyperkube.Server{ + SimpleUsage: "apiserver", + Long: "The main API entrypoint and interface to the storage system. The API server is also the focal point for all authorization decisions.", + Run: func(_ *hyperkube.Server, args []string) error { + return s.Run(args) + }, + } + s.AddFlags(hks.Flags()) + return &hks +} + +// AddFlags adds flags for a specific APIServer to the specified FlagSet +func (s *APIServer) AddFlags(fs *pflag.FlagSet) { + // Note: the weird ""+ in below lines seems to be the only way to get gofmt to + // arrange these text blocks sensibly. Grrr. + fs.IntVar(&s.Port, "port", s.Port, ""+ + "The port to listen on. Default 8080. It is assumed that firewall rules are "+ + "set up such that this port is not reachable from outside of the cluster. It is "+ + "further assumed that port 443 on the cluster's public address is proxied to this "+ + "port. This is performed by nginx in the default setup.") + fs.Var(&s.Address, "address", "The IP address on to serve on (set to 0.0.0.0 for all interfaces)") + fs.StringVar(&s.PublicAddressOverride, "public_address_override", s.PublicAddressOverride, ""+ + "Public serving address. Read only port will be opened on this address, "+ + "and it is assumed that port 443 at this address will be proxied/redirected "+ + "to '-address':'-port'. If blank, the address in the first listed interface "+ + "will be used.") + fs.IntVar(&s.ReadOnlyPort, "read_only_port", s.ReadOnlyPort, ""+ + "The port from which to serve read-only resources. If 0, don't serve on a "+ + "read-only address. It is assumed that firewall rules are set up such that "+ + "this port is not reachable from outside of the cluster.") + fs.Float32Var(&s.APIRate, "api_rate", s.APIRate, "API rate limit as QPS for the read only port") + fs.IntVar(&s.APIBurst, "api_burst", s.APIBurst, "API burst amount for the read only port") + fs.IntVar(&s.SecurePort, "secure_port", s.SecurePort, + "The port from which to serve HTTPS with authentication and authorization. If 0, don't serve HTTPS ") + fs.StringVar(&s.TLSCertFile, "tls_cert_file", s.TLSCertFile, ""+ + "File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert). "+ + "If HTTPS serving is enabled, and --tls_cert_file and --tls_private_key_file are not provided, "+ + "a self-signed certificate and key are generated for the public address and saved to /var/run/kubernetes.") + fs.StringVar(&s.TLSPrivateKeyFile, "tls_private_key_file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls_cert_file.") + fs.StringVar(&s.APIPrefix, "api_prefix", s.APIPrefix, "The prefix for API requests on the server. Default '/api'.") + fs.StringVar(&s.StorageVersion, "storage_version", s.StorageVersion, "The version to store resources with. Defaults to server preferred") + fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.") + fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.") + fs.DurationVar(&s.EventTTL, "event_ttl", s.EventTTL, "Amount of time to retain events. Default 2 days.") + fs.StringVar(&s.TokenAuthFile, "token_auth_file", s.TokenAuthFile, "If set, the file that will be used to secure the secure port of the API server via token authentication.") + fs.StringVar(&s.AuthorizationMode, "authorization_mode", s.AuthorizationMode, "Selects how to do authorization on the secure port. One of: "+strings.Join(apiserver.AuthorizationModeChoices, ",")) + fs.StringVar(&s.AuthorizationPolicyFile, "authorization_policy_file", s.AuthorizationPolicyFile, "File with authorization policy in csv format, used with --authorization_mode=ABAC, on the secure port.") + fs.StringVar(&s.AdmissionControl, "admission_control", s.AdmissionControl, "Ordered list of plug-ins to do admission control of resources into cluster. Comma-delimited list of: "+strings.Join(admission.GetPlugins(), ", ")) + fs.StringVar(&s.AdmissionControlConfigFile, "admission_control_config_file", s.AdmissionControlConfigFile, "File with admission control configuration.") + fs.Var(&s.EtcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd_config") + fs.StringVar(&s.EtcdConfigFile, "etcd_config", s.EtcdConfigFile, "The config file for the etcd client. Mutually exclusive with -etcd_servers.") + fs.Var(&s.CorsAllowedOriginList, "cors_allowed_origins", "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.") + fs.BoolVar(&s.AllowPrivileged, "allow_privileged", s.AllowPrivileged, "If true, allow privileged containers.") + fs.Var(&s.PortalNet, "portal_net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.") + fs.StringVar(&s.MasterServiceNamespace, "master_service_namespace", s.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods") + fs.Var(&s.RuntimeConfig, "runtime_config", "A set of key=value pairs that describe runtime configuration that may be passed to the apiserver.") + client.BindKubeletClientConfigFlags(fs, &s.KubeletConfig) +} + +// TODO: Longer term we should read this from some config store, rather than a flag. +func (s *APIServer) verifyPortalFlags() { + if s.PortalNet.IP == nil { + glog.Fatal("No --portal_net specified") + } +} + +func newEtcd(etcdConfigFile string, etcdServerList util.StringList, storageVersion string) (helper tools.EtcdHelper, err error) { + var client tools.EtcdGetSet + if etcdConfigFile != "" { + client, err = etcd.NewClientFromFile(etcdConfigFile) + if err != nil { + return helper, err + } + } else { + client = etcd.NewClient(etcdServerList) + } + + return master.NewEtcdHelper(client, storageVersion) +} + +// Run runs the specified APIServer. This should never exit. +func (s *APIServer) Run(_ []string) error { + s.verifyPortalFlags() + + if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) { + glog.Fatalf("specify either --etcd_servers or --etcd_config") + } + + capabilities.Initialize(capabilities.Capabilities{ + AllowPrivileged: s.AllowPrivileged, + }) + + cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) + + kubeletClient, err := client.NewKubeletClient(&s.KubeletConfig) + if err != nil { + glog.Fatalf("Failure to start kubelet client: %v", err) + } + + _, v1beta3 := s.RuntimeConfig["api/v1beta3"] + + // TODO: expose same flags as client.BindClientConfigFlags but for a server + clientConfig := &client.Config{ + Host: net.JoinHostPort(s.Address.String(), strconv.Itoa(int(s.Port))), + Version: s.StorageVersion, + } + client, err := client.New(clientConfig) + if err != nil { + glog.Fatalf("Invalid server address: %v", err) + } + + helper, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, s.StorageVersion) + if err != nil { + glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err) + } + + n := net.IPNet(s.PortalNet) + + authenticator, err := apiserver.NewAuthenticatorFromTokenFile(s.TokenAuthFile) + if err != nil { + glog.Fatalf("Invalid Authentication Config: %v", err) + } + + authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(s.AuthorizationMode, s.AuthorizationPolicyFile) + if err != nil { + glog.Fatalf("Invalid Authorization Config: %v", err) + } + + admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") + admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile) + + config := &master.Config{ + Client: client, + Cloud: cloud, + EtcdHelper: helper, + EventTTL: s.EventTTL, + KubeletClient: kubeletClient, + PortalNet: &n, + EnableLogsSupport: s.EnableLogsSupport, + EnableUISupport: true, + EnableSwaggerSupport: true, + APIPrefix: s.APIPrefix, + CorsAllowedOriginList: s.CorsAllowedOriginList, + ReadOnlyPort: s.ReadOnlyPort, + ReadWritePort: s.Port, + PublicAddress: s.PublicAddressOverride, + Authenticator: authenticator, + Authorizer: authorizer, + AdmissionControl: admissionController, + EnableV1Beta3: v1beta3, + MasterServiceNamespace: s.MasterServiceNamespace, + } + m := master.New(config) + + // We serve on 3 ports. See docs/reaching_the_api.md + roLocation := "" + if s.ReadOnlyPort != 0 { + roLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(config.ReadOnlyPort)) + } + secureLocation := "" + if s.SecurePort != 0 { + secureLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(s.SecurePort)) + } + rwLocation := net.JoinHostPort(s.Address.String(), strconv.Itoa(int(s.Port))) + + // See the flag commentary to understand our assumptions when opening the read-only and read-write ports. + + if roLocation != "" { + // Default settings allow 1 read-only request per second, allow up to 20 in a burst before enforcing. + rl := util.NewTokenBucketRateLimiter(s.APIRate, s.APIBurst) + readOnlyServer := &http.Server{ + Addr: roLocation, + Handler: apiserver.RecoverPanics(apiserver.ReadOnly(apiserver.RateLimit(rl, m.InsecureHandler))), + ReadTimeout: 5 * time.Minute, + WriteTimeout: 5 * time.Minute, + MaxHeaderBytes: 1 << 20, + } + glog.Infof("Serving read-only insecurely on %s", roLocation) + go func() { + defer util.HandleCrash() + for { + if err := readOnlyServer.ListenAndServe(); err != nil { + glog.Errorf("Unable to listen for read only traffic (%v); will try again.", err) + } + time.Sleep(15 * time.Second) + } + }() + } + + if secureLocation != "" { + secureServer := &http.Server{ + Addr: secureLocation, + Handler: apiserver.RecoverPanics(m.Handler), + ReadTimeout: 5 * time.Minute, + WriteTimeout: 5 * time.Minute, + MaxHeaderBytes: 1 << 20, + TLSConfig: &tls.Config{ + // Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability) + MinVersion: tls.VersionTLS10, + // Populate PeerCertificates in requests, but don't reject connections without certificates + // This allows certificates to be validated by authenticators, while still allowing other auth types + ClientAuth: tls.RequestClientCert, + }, + } + glog.Infof("Serving securely on %s", secureLocation) + go func() { + defer util.HandleCrash() + for { + if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" { + s.TLSCertFile = "/var/run/kubernetes/apiserver.crt" + s.TLSPrivateKeyFile = "/var/run/kubernetes/apiserver.key" + if err := util.GenerateSelfSignedCert(config.PublicAddress, s.TLSCertFile, s.TLSPrivateKeyFile); err != nil { + glog.Errorf("Unable to generate self signed cert: %v", err) + } else { + glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile) + } + } + if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil { + glog.Errorf("Unable to listen for secure (%v); will try again.", err) + } + time.Sleep(15 * time.Second) + } + }() + } + + http := &http.Server{ + Addr: rwLocation, + Handler: apiserver.RecoverPanics(m.InsecureHandler), + ReadTimeout: 5 * time.Minute, + WriteTimeout: 5 * time.Minute, + MaxHeaderBytes: 1 << 20, + } + glog.Infof("Serving insecurely on %s", rwLocation) + glog.Fatal(http.ListenAndServe()) + return nil +}