diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index b46ca6d19e5..09a97ebb34d 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -35,90 +35,67 @@ import ( "github.com/spf13/pflag" ) -const ( - // Maximum duration before timing out read/write requests - // Set to a value larger than the timeouts in each watch server. - ReadWriteTimeout = time.Minute * 60 - // TODO: This can be tightened up. It still matches objects named watch or proxy. - defaultLongRunningRequestRE = "(/|^)((watch|proxy)(/|$)|(logs?|portforward|exec|attach)/?$)" -) - // APIServer runs a kubernetes api server. type APIServer struct { - InsecureBindAddress net.IP - InsecurePort int - BindAddress net.IP - AdvertiseAddress net.IP - SecurePort int - ExternalHost string - TLSCertFile string - TLSPrivateKeyFile string - CertDirectory string - APIPrefix string + *genericapiserver.ServerRunOptions APIGroupPrefix string - DeprecatedStorageVersion string - StorageVersions string - CloudProvider string - CloudConfigFile string - EventTTL time.Duration - BasicAuthFile string - ClientCAFile string - TokenAuthFile string - OIDCIssuerURL string - OIDCClientID string - OIDCCAFile string - OIDCUsernameClaim string - ServiceAccountKeyFile string - ServiceAccountLookup bool - KeystoneURL string - AuthorizationMode string - AuthorizationPolicyFile string + APIPrefix string AdmissionControl string AdmissionControlConfigFile string - EtcdServerList []string - EtcdServersOverrides []string - EtcdPathPrefix string - CorsAllowedOriginList []string + AdvertiseAddress net.IP AllowPrivileged bool - ServiceClusterIPRange net.IPNet // TODO: make this a list - ServiceNodePortRange util.PortRange + AuthorizationMode string + AuthorizationPolicyFile string + BasicAuthFile string + CloudConfigFile string + CloudProvider string + CorsAllowedOriginList []string + DeprecatedStorageVersion string EnableLogsSupport bool - MasterServiceNamespace string - MasterCount int - RuntimeConfig util.ConfigurationMap - KubeletConfig kubeletclient.KubeletClientConfig EnableProfiling bool EnableWatchCache bool - MaxRequestsInFlight int - MinRequestTimeout int - LongRunningRequestRE string - SSHUser string - SSHKeyfile string - MaxConnectionBytesPerSec int64 + EtcdPathPrefix string + EtcdServerList []string + EtcdServersOverrides []string + EventTTL time.Duration + ExternalHost string + KeystoneURL string + KubeletConfig kubeletclient.KubeletClientConfig KubernetesServiceNodePort int + MasterCount int + MasterServiceNamespace string + MaxConnectionBytesPerSec int64 + MinRequestTimeout int + OIDCCAFile string + OIDCClientID string + OIDCIssuerURL string + OIDCUsernameClaim string + RuntimeConfig util.ConfigurationMap + SSHKeyfile string + SSHUser string + ServiceAccountKeyFile string + ServiceAccountLookup bool + ServiceClusterIPRange net.IPNet // TODO: make this a list + ServiceNodePortRange util.PortRange + StorageVersions string + TokenAuthFile string } // NewAPIServer creates a new APIServer object with default parameters func NewAPIServer() *APIServer { s := APIServer{ - InsecurePort: 8080, - InsecureBindAddress: net.ParseIP("127.0.0.1"), - BindAddress: net.ParseIP("0.0.0.0"), - SecurePort: 6443, - APIPrefix: "/api", + ServerRunOptions: genericapiserver.NewServerRunOptions(), APIGroupPrefix: "/apis", - EventTTL: 1 * time.Hour, - AuthorizationMode: "AlwaysAllow", + APIPrefix: "/api", AdmissionControl: "AlwaysAdmit", - EtcdPathPrefix: genericapiserver.DefaultEtcdPathPrefix, + AuthorizationMode: "AlwaysAllow", EnableLogsSupport: true, - MasterServiceNamespace: api.NamespaceDefault, + EtcdPathPrefix: genericapiserver.DefaultEtcdPathPrefix, + EventTTL: 1 * time.Hour, MasterCount: 1, - CertDirectory: "/var/run/kubernetes", + MasterServiceNamespace: api.NamespaceDefault, + RuntimeConfig: make(util.ConfigurationMap), StorageVersions: latest.AllPreferredGroupVersions(), - LongRunningRequestRE: defaultLongRunningRequestRE, - - RuntimeConfig: make(util.ConfigurationMap), KubeletConfig: kubeletclient.KubeletClientConfig{ Port: ports.KubeletPort, EnableHttps: true, diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 4cde4ba83bb..8232b78ca40 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -23,15 +23,10 @@ import ( "crypto/tls" "fmt" "net" - "net/http" "os" - "path" - "regexp" "strconv" "strings" - "time" - systemd "github.com/coreos/go-systemd/daemon" "github.com/golang/glog" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -394,98 +389,7 @@ func Run(s *options.APIServer) error { Tunneler: tunneler, } m := master.New(config) - - // We serve on 2 ports. See docs/accessing_the_api.md - secureLocation := "" - if s.SecurePort != 0 { - secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort)) - } - insecureLocation := net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)) - - // See the flag commentary to understand our assumptions when opening the read-only and read-write ports. - - var sem chan bool - if s.MaxRequestsInFlight > 0 { - sem = make(chan bool, s.MaxRequestsInFlight) - } - - longRunningRE := regexp.MustCompile(s.LongRunningRequestRE) - longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) { - // TODO unify this with apiserver.MaxInFlightLimit - if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" { - return nil, "" - } - return time.After(time.Minute), "" - } - - if secureLocation != "" { - handler := apiserver.TimeoutHandler(m.Handler, longRunningTimeout) - secureServer := &http.Server{ - Addr: secureLocation, - Handler: apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)), - MaxHeaderBytes: 1 << 20, - TLSConfig: &tls.Config{ - // Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability) - MinVersion: tls.VersionTLS10, - }, - } - - if len(s.ClientCAFile) > 0 { - clientCAs, err := util.CertPoolFromFile(s.ClientCAFile) - if err != nil { - glog.Fatalf("Unable to load client CA file: %v", err) - } - // Populate PeerCertificates in requests, but don't reject connections without certificates - // This allows certificates to be validated by authenticators, while still allowing other auth types - secureServer.TLSConfig.ClientAuth = tls.RequestClientCert - // Specify allowed CAs for client certificates - secureServer.TLSConfig.ClientCAs = clientCAs - } - - glog.Infof("Serving securely on %s", secureLocation) - if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" { - s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt") - s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key") - // TODO (cjcullen): Is PublicAddress the right address to sign a cert with? - alternateIPs := []net.IP{config.ServiceReadWriteIP} - alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} - // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless - // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") - if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil { - glog.Errorf("Unable to generate self signed cert: %v", err) - } else { - glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile) - } - } - - go func() { - defer util.HandleCrash() - for { - // err == systemd.SdNotifyNoSocket when not running on a systemd system - if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { - glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) - } - if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil { - glog.Errorf("Unable to listen for secure (%v); will try again.", err) - } - time.Sleep(15 * time.Second) - } - }() - } - handler := apiserver.TimeoutHandler(m.InsecureHandler, longRunningTimeout) - http := &http.Server{ - Addr: insecureLocation, - Handler: apiserver.RecoverPanics(handler), - MaxHeaderBytes: 1 << 20, - } - if secureLocation == "" { - // err == systemd.SdNotifyNoSocket when not running on a systemd system - if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { - glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) - } - } - glog.Infof("Serving insecurely on %s", insecureLocation) - glog.Fatal(http.ListenAndServe()) + m.Run(s.ServerRunOptions) return nil } diff --git a/pkg/genericapiserver/genericapiserver.go b/pkg/genericapiserver/genericapiserver.go index da087ecd7e2..35c7dfda22c 100644 --- a/pkg/genericapiserver/genericapiserver.go +++ b/pkg/genericapiserver/genericapiserver.go @@ -22,6 +22,8 @@ import ( "net" "net/http" "net/http/pprof" + "path" + "regexp" "strconv" "strings" "time" @@ -43,6 +45,7 @@ import ( "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/sets" + systemd "github.com/coreos/go-systemd/daemon" "github.com/emicklei/go-restful" "github.com/emicklei/go-restful/swagger" "github.com/golang/glog" @@ -532,6 +535,98 @@ func (s *GenericAPIServer) InstallAPIGroups(groupsInfo []APIGroupInfo) error { return nil } +func (s *GenericAPIServer) Run(options *ServerRunOptions) { + // We serve on 2 ports. See docs/accessing_the_api.md + secureLocation := "" + if options.SecurePort != 0 { + secureLocation = net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)) + } + insecureLocation := net.JoinHostPort(options.InsecureBindAddress.String(), strconv.Itoa(options.InsecurePort)) + + var sem chan bool + if options.MaxRequestsInFlight > 0 { + sem = make(chan bool, options.MaxRequestsInFlight) + } + + longRunningRE := regexp.MustCompile(options.LongRunningRequestRE) + longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) { + // TODO unify this with apiserver.MaxInFlightLimit + if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" { + return nil, "" + } + return time.After(time.Minute), "" + } + + if secureLocation != "" { + handler := apiserver.TimeoutHandler(s.Handler, longRunningTimeout) + secureServer := &http.Server{ + Addr: secureLocation, + Handler: apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)), + MaxHeaderBytes: 1 << 20, + TLSConfig: &tls.Config{ + // Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability) + MinVersion: tls.VersionTLS10, + }, + } + + if len(options.ClientCAFile) > 0 { + clientCAs, err := util.CertPoolFromFile(options.ClientCAFile) + if err != nil { + glog.Fatalf("Unable to load client CA file: %v", err) + } + // Populate PeerCertificates in requests, but don't reject connections without certificates + // This allows certificates to be validated by authenticators, while still allowing other auth types + secureServer.TLSConfig.ClientAuth = tls.RequestClientCert + // Specify allowed CAs for client certificates + secureServer.TLSConfig.ClientCAs = clientCAs + } + + glog.Infof("Serving securely on %s", secureLocation) + if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" { + options.TLSCertFile = path.Join(options.CertDirectory, "apiserver.crt") + options.TLSPrivateKeyFile = path.Join(options.CertDirectory, "apiserver.key") + // TODO (cjcullen): Is ClusterIP the right address to sign a cert with? + alternateIPs := []net.IP{s.ServiceReadWriteIP} + alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} + // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless + // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") + if err := util.GenerateSelfSignedCert(s.ClusterIP.String(), options.TLSCertFile, options.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil { + glog.Errorf("Unable to generate self signed cert: %v", err) + } else { + glog.Infof("Using self-signed cert (%options, %options)", options.TLSCertFile, options.TLSPrivateKeyFile) + } + } + + go func() { + defer util.HandleCrash() + for { + // err == systemd.SdNotifyNoSocket when not running on a systemd system + if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { + glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) + } + if err := secureServer.ListenAndServeTLS(options.TLSCertFile, options.TLSPrivateKeyFile); err != nil { + glog.Errorf("Unable to listen for secure (%v); will try again.", err) + } + time.Sleep(15 * time.Second) + } + }() + } else { + // err == systemd.SdNotifyNoSocket when not running on a systemd system + if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { + glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) + } + } + + handler := apiserver.TimeoutHandler(s.InsecureHandler, longRunningTimeout) + http := &http.Server{ + Addr: insecureLocation, + Handler: apiserver.RecoverPanics(handler), + MaxHeaderBytes: 1 << 20, + } + glog.Infof("Serving insecurely on %s", insecureLocation) + glog.Fatal(http.ListenAndServe()) +} + func (s *GenericAPIServer) installAPIGroup(apiGroupInfo *APIGroupInfo) error { apiPrefix := s.APIGroupPrefix if apiGroupInfo.IsLegacyGroup { diff --git a/pkg/genericapiserver/server_run_options.go b/pkg/genericapiserver/server_run_options.go new file mode 100644 index 00000000000..2ebc3d77aa6 --- /dev/null +++ b/pkg/genericapiserver/server_run_options.go @@ -0,0 +1,51 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package genericapiserver + +import ( + "net" +) + +const ( + // TODO: This can be tightened up. It still matches objects named watch or proxy. + defaultLongRunningRequestRE = "(/|^)((watch|proxy)(/|$)|(logs?|portforward|exec|attach)/?$)" +) + +// ServerRunOptions contains the options while running a generic api server. +type ServerRunOptions struct { + BindAddress net.IP + CertDirectory string + ClientCAFile string + InsecureBindAddress net.IP + InsecurePort int + LongRunningRequestRE string + MaxRequestsInFlight int + SecurePort int + TLSCertFile string + TLSPrivateKeyFile string +} + +func NewServerRunOptions() *ServerRunOptions { + return &ServerRunOptions{ + BindAddress: net.ParseIP("0.0.0.0"), + CertDirectory: "/var/run/kubernetes", + InsecureBindAddress: net.ParseIP("127.0.0.1"), + InsecurePort: 8080, + LongRunningRequestRE: defaultLongRunningRequestRE, + SecurePort: 6443, + } +}