Extracting server run code to genericapiserver

cmd/kube-apiserver/app/options/options.go

@@ -35,90 +35,67 @@ import (
 	"github.com/spf13/pflag"
 )
 
-const (
-	// Maximum duration before timing out read/write requests
-	// Set to a value larger than the timeouts in each watch server.
-	ReadWriteTimeout = time.Minute * 60
-	// TODO: This can be tightened up. It still matches objects named watch or proxy.
-	defaultLongRunningRequestRE = "(/|^)((watch|proxy)(/|$)|(logs?|portforward|exec|attach)/?$)"
-)
-
 // APIServer runs a kubernetes api server.
 type APIServer struct {
-	InsecureBindAddress        net.IP
+	*genericapiserver.ServerRunOptions
-	InsecurePort               int
-	BindAddress                net.IP
-	AdvertiseAddress           net.IP
-	SecurePort                 int
-	ExternalHost               string
-	TLSCertFile                string
-	TLSPrivateKeyFile          string
-	CertDirectory              string
-	APIPrefix                  string
 	APIGroupPrefix             string
-	DeprecatedStorageVersion   string
+	APIPrefix                  string
-	StorageVersions            string
-	CloudProvider              string
-	CloudConfigFile            string
-	EventTTL                   time.Duration
-	BasicAuthFile              string
-	ClientCAFile               string
-	TokenAuthFile              string
-	OIDCIssuerURL              string
-	OIDCClientID               string
-	OIDCCAFile                 string
-	OIDCUsernameClaim          string
-	ServiceAccountKeyFile      string
-	ServiceAccountLookup       bool
-	KeystoneURL                string
-	AuthorizationMode          string
-	AuthorizationPolicyFile    string
 	AdmissionControl           string
 	AdmissionControlConfigFile string
-	EtcdServerList             []string
+	AdvertiseAddress           net.IP
-	EtcdServersOverrides       []string
-	EtcdPathPrefix             string
-	CorsAllowedOriginList      []string
 	AllowPrivileged            bool
-	ServiceClusterIPRange      net.IPNet // TODO: make this a list
+	AuthorizationMode          string
-	ServiceNodePortRange       util.PortRange
+	AuthorizationPolicyFile    string
+	BasicAuthFile              string
+	CloudConfigFile            string
+	CloudProvider              string
+	CorsAllowedOriginList      []string
+	DeprecatedStorageVersion   string
 	EnableLogsSupport          bool
-	MasterServiceNamespace     string
-	MasterCount                int
-	RuntimeConfig              util.ConfigurationMap
-	KubeletConfig              kubeletclient.KubeletClientConfig
 	EnableProfiling            bool
 	EnableWatchCache           bool
-	MaxRequestsInFlight        int
+	EtcdPathPrefix             string
-	MinRequestTimeout          int
+	EtcdServerList             []string
-	LongRunningRequestRE       string
+	EtcdServersOverrides       []string
-	SSHUser                    string
+	EventTTL                   time.Duration
-	SSHKeyfile                 string
+	ExternalHost               string
-	MaxConnectionBytesPerSec   int64
+	KeystoneURL                string
+	KubeletConfig              kubeletclient.KubeletClientConfig
 	KubernetesServiceNodePort  int
+	MasterCount                int
+	MasterServiceNamespace     string
+	MaxConnectionBytesPerSec   int64
+	MinRequestTimeout          int
+	OIDCCAFile                 string
+	OIDCClientID               string
+	OIDCIssuerURL              string
+	OIDCUsernameClaim          string
+	RuntimeConfig              util.ConfigurationMap
+	SSHKeyfile                 string
+	SSHUser                    string
+	ServiceAccountKeyFile      string
+	ServiceAccountLookup       bool
+	ServiceClusterIPRange      net.IPNet // TODO: make this a list
+	ServiceNodePortRange       util.PortRange
+	StorageVersions            string
+	TokenAuthFile              string
 }
 
 // NewAPIServer creates a new APIServer object with default parameters
 func NewAPIServer() *APIServer {
 	s := APIServer{
-		InsecurePort:           8080,
+		ServerRunOptions:       genericapiserver.NewServerRunOptions(),
-		InsecureBindAddress:    net.ParseIP("127.0.0.1"),
-		BindAddress:            net.ParseIP("0.0.0.0"),
-		SecurePort:             6443,
-		APIPrefix:              "/api",
 		APIGroupPrefix:         "/apis",
-		EventTTL:               1 * time.Hour,
+		APIPrefix:              "/api",
-		AuthorizationMode:      "AlwaysAllow",
 		AdmissionControl:       "AlwaysAdmit",
-		EtcdPathPrefix:         genericapiserver.DefaultEtcdPathPrefix,
+		AuthorizationMode:      "AlwaysAllow",
 		EnableLogsSupport:      true,
-		MasterServiceNamespace: api.NamespaceDefault,
+		EtcdPathPrefix:         genericapiserver.DefaultEtcdPathPrefix,
+		EventTTL:               1 * time.Hour,
 		MasterCount:            1,
-		CertDirectory:          "/var/run/kubernetes",
+		MasterServiceNamespace: api.NamespaceDefault,
-		StorageVersions:        latest.AllPreferredGroupVersions(),
-		LongRunningRequestRE:   defaultLongRunningRequestRE,
-
 		RuntimeConfig:          make(util.ConfigurationMap),
+		StorageVersions:        latest.AllPreferredGroupVersions(),
 		KubeletConfig: kubeletclient.KubeletClientConfig{
 			Port:        ports.KubeletPort,
 			EnableHttps: true,

cmd/kube-apiserver/app/server.go

@@ -23,15 +23,10 @@ import (
 	"crypto/tls"
 	"fmt"
 	"net"
-	"net/http"
 	"os"
-	"path"
-	"regexp"
 	"strconv"
 	"strings"
-	"time"
 
-	systemd "github.com/coreos/go-systemd/daemon"
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -394,98 +389,7 @@ func Run(s *options.APIServer) error {
 		Tunneler: tunneler,
 	}
 	m := master.New(config)
-
+	m.Run(s.ServerRunOptions)
-	// We serve on 2 ports.  See docs/accessing_the_api.md
-	secureLocation := ""
-	if s.SecurePort != 0 {
-		secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort))
-	}
-	insecureLocation := net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort))
-
-	// See the flag commentary to understand our assumptions when opening the read-only and read-write ports.
-
-	var sem chan bool
-	if s.MaxRequestsInFlight > 0 {
-		sem = make(chan bool, s.MaxRequestsInFlight)
-	}
-
-	longRunningRE := regexp.MustCompile(s.LongRunningRequestRE)
-	longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) {
-		// TODO unify this with apiserver.MaxInFlightLimit
-		if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" {
-			return nil, ""
-		}
-		return time.After(time.Minute), ""
-	}
-
-	if secureLocation != "" {
-		handler := apiserver.TimeoutHandler(m.Handler, longRunningTimeout)
-		secureServer := &http.Server{
-			Addr:           secureLocation,
-			Handler:        apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)),
-			MaxHeaderBytes: 1 << 20,
-			TLSConfig: &tls.Config{
-				// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
-				MinVersion: tls.VersionTLS10,
-			},
-		}
-
-		if len(s.ClientCAFile) > 0 {
-			clientCAs, err := util.CertPoolFromFile(s.ClientCAFile)
-			if err != nil {
-				glog.Fatalf("Unable to load client CA file: %v", err)
-			}
-			// Populate PeerCertificates in requests, but don't reject connections without certificates
-			// This allows certificates to be validated by authenticators, while still allowing other auth types
-			secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
-			// Specify allowed CAs for client certificates
-			secureServer.TLSConfig.ClientCAs = clientCAs
-		}
-
-		glog.Infof("Serving securely on %s", secureLocation)
-		if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
-			s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt")
-			s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key")
-			// TODO (cjcullen): Is PublicAddress the right address to sign a cert with?
-			alternateIPs := []net.IP{config.ServiceReadWriteIP}
-			alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}
-			// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
-			// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
-			if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil {
-				glog.Errorf("Unable to generate self signed cert: %v", err)
-			} else {
-				glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
-			}
-		}
-
-		go func() {
-			defer util.HandleCrash()
-			for {
-				// err == systemd.SdNotifyNoSocket when not running on a systemd system
-				if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
-					glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
-				}
-				if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
-					glog.Errorf("Unable to listen for secure (%v); will try again.", err)
-				}
-				time.Sleep(15 * time.Second)
-			}
-		}()
-	}
-	handler := apiserver.TimeoutHandler(m.InsecureHandler, longRunningTimeout)
-	http := &http.Server{
-		Addr:           insecureLocation,
-		Handler:        apiserver.RecoverPanics(handler),
-		MaxHeaderBytes: 1 << 20,
-	}
-	if secureLocation == "" {
-		// err == systemd.SdNotifyNoSocket when not running on a systemd system
-		if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
-			glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
-		}
-	}
-	glog.Infof("Serving insecurely on %s", insecureLocation)
-	glog.Fatal(http.ListenAndServe())
 	return nil
 }
 

pkg/genericapiserver/genericapiserver.go

@@ -22,6 +22,8 @@ import (
 	"net"
 	"net/http"
 	"net/http/pprof"
+	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -43,6 +45,7 @@ import (
 	"k8s.io/kubernetes/pkg/util"
 	"k8s.io/kubernetes/pkg/util/sets"
 
+	systemd "github.com/coreos/go-systemd/daemon"
 	"github.com/emicklei/go-restful"
 	"github.com/emicklei/go-restful/swagger"
 	"github.com/golang/glog"
@@ -532,6 +535,98 @@ func (s *GenericAPIServer) InstallAPIGroups(groupsInfo []APIGroupInfo) error {
 	return nil
 }
 
+func (s *GenericAPIServer) Run(options *ServerRunOptions) {
+	// We serve on 2 ports.  See docs/accessing_the_api.md
+	secureLocation := ""
+	if options.SecurePort != 0 {
+		secureLocation = net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort))
+	}
+	insecureLocation := net.JoinHostPort(options.InsecureBindAddress.String(), strconv.Itoa(options.InsecurePort))
+
+	var sem chan bool
+	if options.MaxRequestsInFlight > 0 {
+		sem = make(chan bool, options.MaxRequestsInFlight)
+	}
+
+	longRunningRE := regexp.MustCompile(options.LongRunningRequestRE)
+	longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) {
+		// TODO unify this with apiserver.MaxInFlightLimit
+		if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" {
+			return nil, ""
+		}
+		return time.After(time.Minute), ""
+	}
+
+	if secureLocation != "" {
+		handler := apiserver.TimeoutHandler(s.Handler, longRunningTimeout)
+		secureServer := &http.Server{
+			Addr:           secureLocation,
+			Handler:        apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)),
+			MaxHeaderBytes: 1 << 20,
+			TLSConfig: &tls.Config{
+				// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
+				MinVersion: tls.VersionTLS10,
+			},
+		}
+
+		if len(options.ClientCAFile) > 0 {
+			clientCAs, err := util.CertPoolFromFile(options.ClientCAFile)
+			if err != nil {
+				glog.Fatalf("Unable to load client CA file: %v", err)
+			}
+			// Populate PeerCertificates in requests, but don't reject connections without certificates
+			// This allows certificates to be validated by authenticators, while still allowing other auth types
+			secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
+			// Specify allowed CAs for client certificates
+			secureServer.TLSConfig.ClientCAs = clientCAs
+		}
+
+		glog.Infof("Serving securely on %s", secureLocation)
+		if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" {
+			options.TLSCertFile = path.Join(options.CertDirectory, "apiserver.crt")
+			options.TLSPrivateKeyFile = path.Join(options.CertDirectory, "apiserver.key")
+			// TODO (cjcullen): Is ClusterIP the right address to sign a cert with?
+			alternateIPs := []net.IP{s.ServiceReadWriteIP}
+			alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}
+			// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
+			// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
+			if err := util.GenerateSelfSignedCert(s.ClusterIP.String(), options.TLSCertFile, options.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil {
+				glog.Errorf("Unable to generate self signed cert: %v", err)
+			} else {
+				glog.Infof("Using self-signed cert (%options, %options)", options.TLSCertFile, options.TLSPrivateKeyFile)
+			}
+		}
+
+		go func() {
+			defer util.HandleCrash()
+			for {
+				// err == systemd.SdNotifyNoSocket when not running on a systemd system
+				if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
+					glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
+				}
+				if err := secureServer.ListenAndServeTLS(options.TLSCertFile, options.TLSPrivateKeyFile); err != nil {
+					glog.Errorf("Unable to listen for secure (%v); will try again.", err)
+				}
+				time.Sleep(15 * time.Second)
+			}
+		}()
+	} else {
+		// err == systemd.SdNotifyNoSocket when not running on a systemd system
+		if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
+			glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
+		}
+	}
+
+	handler := apiserver.TimeoutHandler(s.InsecureHandler, longRunningTimeout)
+	http := &http.Server{
+		Addr:           insecureLocation,
+		Handler:        apiserver.RecoverPanics(handler),
+		MaxHeaderBytes: 1 << 20,
+	}
+	glog.Infof("Serving insecurely on %s", insecureLocation)
+	glog.Fatal(http.ListenAndServe())
+}
+
 func (s *GenericAPIServer) installAPIGroup(apiGroupInfo *APIGroupInfo) error {
 	apiPrefix := s.APIGroupPrefix
 	if apiGroupInfo.IsLegacyGroup {

pkg/genericapiserver/server_run_options.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package genericapiserver
+
+import (
+	"net"
+)
+
+const (
+	// TODO: This can be tightened up. It still matches objects named watch or proxy.
+	defaultLongRunningRequestRE = "(/|^)((watch|proxy)(/|$)|(logs?|portforward|exec|attach)/?$)"
+)
+
+// ServerRunOptions contains the options while running a generic api server.
+type ServerRunOptions struct {
+	BindAddress          net.IP
+	CertDirectory        string
+	ClientCAFile         string
+	InsecureBindAddress  net.IP
+	InsecurePort         int
+	LongRunningRequestRE string
+	MaxRequestsInFlight  int
+	SecurePort           int
+	TLSCertFile          string
+	TLSPrivateKeyFile    string
+}
+
+func NewServerRunOptions() *ServerRunOptions {
+	return &ServerRunOptions{
+		BindAddress:          net.ParseIP("0.0.0.0"),
+		CertDirectory:        "/var/run/kubernetes",
+		InsecureBindAddress:  net.ParseIP("127.0.0.1"),
+		InsecurePort:         8080,
+		LongRunningRequestRE: defaultLongRunningRequestRE,
+		SecurePort:           6443,
+	}
+}