mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-28 22:17:14 +00:00
Merge pull request #3982 from jbeda/hyperkube-servers
Convert kube-apiserver to hyperkube
This commit is contained in:
Brian Grant
commit
881f2dca96
37
cmd/hyperkube/hyperkube.go
Normal file
37
cmd/hyperkube/hyperkube.go
Normal file
@ -0,0 +1,37 @@
|
||||
/*
|
||||
Copyright 2014 Google Inc. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// A binary that can morph into all of the other kubernetes binaries. You can
|
||||
// also soft-link to it busybox style.
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/hyperkube"
|
||||
apiserver "github.com/GoogleCloudPlatform/kubernetes/pkg/master/server"
|
||||
)
|
||||
|
||||
func main() {
|
||||
hk := hyperkube.HyperKube{
|
||||
Name: "hyperkube",
|
||||
Long: "This is an all-in-one binary that can run any of the various Kubernetes servers.",
|
||||
}
|
||||
|
||||
hk.AddServer(apiserver.NewHyperkubeServer())
|
||||
|
||||
hk.RunToExit(os.Args)
|
||||
}
|
||||
@ -19,268 +19,22 @@ limitations under the License.
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/admission"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/master/server"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag"
|
||||
|
||||
"github.com/coreos/go-etcd/etcd"
|
||||
"github.com/golang/glog"
|
||||
flag "github.com/spf13/pflag"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
var (
|
||||
// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
|
||||
// arrange these text blocks sensibly. Grrr.
|
||||
port = flag.Int("port", 8080, ""+
|
||||
"The port to listen on. Default 8080. It is assumed that firewall rules are "+
|
||||
"set up such that this port is not reachable from outside of the cluster. It is "+
|
||||
"further assumed that port 443 on the cluster's public address is proxied to this "+
|
||||
"port. This is performed by nginx in the default setup.")
|
||||
address = util.IP(net.ParseIP("127.0.0.1"))
|
||||
publicAddressOverride = flag.String("public_address_override", "", ""+
|
||||
"Public serving address. Read only port will be opened on this address, "+
|
||||
"and it is assumed that port 443 at this address will be proxied/redirected "+
|
||||
"to '-address':'-port'. If blank, the address in the first listed interface "+
|
||||
"will be used.")
|
||||
readOnlyPort = flag.Int("read_only_port", 7080, ""+
|
||||
"The port from which to serve read-only resources. If 0, don't serve on a "+
|
||||
"read-only address. It is assumed that firewall rules are set up such that "+
|
||||
"this port is not reachable from outside of the cluster.")
|
||||
apiRate = flag.Float32("api_rate", 10.0, "API rate limit as QPS for the read only port")
|
||||
apiBurst = flag.Int("api_burst", 200, "API burst amount for the read only port")
|
||||
securePort = flag.Int("secure_port", 8443, "The port from which to serve HTTPS with authentication and authorization. If 0, don't serve HTTPS ")
|
||||
tlsCertFile = flag.String("tls_cert_file", "", ""+
|
||||
"File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert). "+
|
||||
"If HTTPS serving is enabled, and --tls_cert_file and --tls_private_key_file are not provided, "+
|
||||
"a self-signed certificate and key are generated for the public address and saved to /var/run/kubernetes.")
|
||||
tlsPrivateKeyFile = flag.String("tls_private_key_file", "", "File containing x509 private key matching --tls_cert_file.")
|
||||
apiPrefix = flag.String("api_prefix", "/api", "The prefix for API requests on the server. Default '/api'.")
|
||||
storageVersion = flag.String("storage_version", "", "The version to store resources with. Defaults to server preferred")
|
||||
cloudProvider = flag.String("cloud_provider", "", "The provider for cloud services. Empty string for no provider.")
|
||||
cloudConfigFile = flag.String("cloud_config", "", "The path to the cloud provider configuration file. Empty string for no configuration file.")
|
||||
healthCheckMinions = flag.Bool("health_check_minions", true, "If true, health check minions and filter unhealthy ones. Default true.")
|
||||
eventTTL = flag.Duration("event_ttl", 48*time.Hour, "Amount of time to retain events. Default 2 days.")
|
||||
tokenAuthFile = flag.String("token_auth_file", "", "If set, the file that will be used to secure the secure port of the API server via token authentication.")
|
||||
authorizationMode = flag.String("authorization_mode", "AlwaysAllow", "Selects how to do authorization on the secure port. One of: "+strings.Join(apiserver.AuthorizationModeChoices, ","))
|
||||
authorizationPolicyFile = flag.String("authorization_policy_file", "", "File with authorization policy in csv format, used with --authorization_mode=ABAC, on the secure port.")
|
||||
admissionControl = flag.String("admission_control", "AlwaysAdmit", "Ordered list of plug-ins to do admission control of resources into cluster. Comma-delimited list of: "+strings.Join(admission.GetPlugins(), ", "))
|
||||
admissionControlConfigFile = flag.String("admission_control_config_file", "", "File with admission control configuration.")
|
||||
etcdServerList util.StringList
|
||||
etcdConfigFile = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers.")
|
||||
corsAllowedOriginList util.StringList
|
||||
allowPrivileged = flag.Bool("allow_privileged", false, "If true, allow privileged containers.")
|
||||
portalNet util.IPNet // TODO: make this a list
|
||||
enableLogsSupport = flag.Bool("enable_logs_support", true, "Enables server endpoint for log collection")
|
||||
runtimeConfig util.ConfigurationMap
|
||||
kubeletConfig = client.KubeletConfig{
|
||||
Port: 10250,
|
||||
EnableHttps: false,
|
||||
}
|
||||
masterServiceNamespace = flag.String("master_service_namespace", api.NamespaceDefault, "The namespace from which the kubernetes master services should be injected into pods")
|
||||
)
|
||||
|
||||
func init() {
|
||||
runtimeConfig = make(util.ConfigurationMap)
|
||||
|
||||
flag.Var(&address, "address", "The IP address on to serve on (set to 0.0.0.0 for all interfaces)")
|
||||
flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd_config")
|
||||
flag.Var(&corsAllowedOriginList, "cors_allowed_origins", "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.")
|
||||
flag.Var(&portalNet, "portal_net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.")
|
||||
flag.Var(&runtimeConfig, "runtime_config", "A set of key=value pairs that describe runtime configuration that may be passed to the apiserver.")
|
||||
client.BindKubeletClientConfigFlags(flag.CommandLine, &kubeletConfig)
|
||||
}
|
||||
|
||||
// TODO: Longer term we should read this from some config store, rather than a flag.
|
||||
func verifyPortalFlags() {
|
||||
if portalNet.IP == nil {
|
||||
glog.Fatal("No -portal_net specified")
|
||||
}
|
||||
}
|
||||
|
||||
func newEtcd(etcdConfigFile string, etcdServerList util.StringList) (helper tools.EtcdHelper, err error) {
|
||||
var client tools.EtcdGetSet
|
||||
if etcdConfigFile != "" {
|
||||
client, err = etcd.NewClientFromFile(etcdConfigFile)
|
||||
if err != nil {
|
||||
return helper, err
|
||||
}
|
||||
} else {
|
||||
client = etcd.NewClient(etcdServerList)
|
||||
}
|
||||
|
||||
return master.NewEtcdHelper(client, *storageVersion)
|
||||
}
|
||||
|
||||
func main() {
|
||||
s := server.NewAPIServer()
|
||||
s.AddFlags(pflag.CommandLine)
|
||||
|
||||
util.InitFlags()
|
||||
util.InitLogs()
|
||||
defer util.FlushLogs()
|
||||
|
||||
verflag.PrintAndExitIfRequested()
|
||||
verifyPortalFlags()
|
||||
|
||||
if (*etcdConfigFile != "" && len(etcdServerList) != 0) || (*etcdConfigFile == "" && len(etcdServerList) == 0) {
|
||||
glog.Fatalf("specify either -etcd_servers or -etcd_config")
|
||||
}
|
||||
|
||||
capabilities.Initialize(capabilities.Capabilities{
|
||||
AllowPrivileged: *allowPrivileged,
|
||||
})
|
||||
|
||||
cloud := cloudprovider.InitCloudProvider(*cloudProvider, *cloudConfigFile)
|
||||
|
||||
kubeletClient, err := client.NewKubeletClient(&kubeletConfig)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failure to start kubelet client: %v", err)
|
||||
}
|
||||
|
||||
_, v1beta3 := runtimeConfig["api/v1beta3"]
|
||||
|
||||
// TODO: expose same flags as client.BindClientConfigFlags but for a server
|
||||
clientConfig := &client.Config{
|
||||
Host: net.JoinHostPort(address.String(), strconv.Itoa(int(*port))),
|
||||
Version: *storageVersion,
|
||||
}
|
||||
client, err := client.New(clientConfig)
|
||||
if err != nil {
|
||||
glog.Fatalf("Invalid server address: %v", err)
|
||||
}
|
||||
|
||||
helper, err := newEtcd(*etcdConfigFile, etcdServerList)
|
||||
if err != nil {
|
||||
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
|
||||
}
|
||||
|
||||
n := net.IPNet(portalNet)
|
||||
|
||||
authenticator, err := apiserver.NewAuthenticatorFromTokenFile(*tokenAuthFile)
|
||||
if err != nil {
|
||||
glog.Fatalf("Invalid Authentication Config: %v", err)
|
||||
}
|
||||
|
||||
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(*authorizationMode, *authorizationPolicyFile)
|
||||
if err != nil {
|
||||
glog.Fatalf("Invalid Authorization Config: %v", err)
|
||||
}
|
||||
|
||||
admissionControlPluginNames := strings.Split(*admissionControl, ",")
|
||||
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, *admissionControlConfigFile)
|
||||
|
||||
config := &master.Config{
|
||||
Client: client,
|
||||
Cloud: cloud,
|
||||
EtcdHelper: helper,
|
||||
EventTTL: *eventTTL,
|
||||
KubeletClient: kubeletClient,
|
||||
PortalNet: &n,
|
||||
EnableLogsSupport: *enableLogsSupport,
|
||||
EnableUISupport: true,
|
||||
EnableSwaggerSupport: true,
|
||||
APIPrefix: *apiPrefix,
|
||||
CorsAllowedOriginList: corsAllowedOriginList,
|
||||
ReadOnlyPort: *readOnlyPort,
|
||||
ReadWritePort: *port,
|
||||
PublicAddress: *publicAddressOverride,
|
||||
Authenticator: authenticator,
|
||||
Authorizer: authorizer,
|
||||
AdmissionControl: admissionController,
|
||||
EnableV1Beta3: v1beta3,
|
||||
MasterServiceNamespace: *masterServiceNamespace,
|
||||
}
|
||||
m := master.New(config)
|
||||
|
||||
// We serve on 3 ports. See docs/reaching_the_api.md
|
||||
roLocation := ""
|
||||
if *readOnlyPort != 0 {
|
||||
roLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(config.ReadOnlyPort))
|
||||
}
|
||||
secureLocation := ""
|
||||
if *securePort != 0 {
|
||||
secureLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(*securePort))
|
||||
}
|
||||
rwLocation := net.JoinHostPort(address.String(), strconv.Itoa(int(*port)))
|
||||
|
||||
// See the flag commentary to understand our assumptions when opening the read-only and read-write ports.
|
||||
|
||||
if roLocation != "" {
|
||||
// Default settings allow 10 read-only requests per second, allow up to 200 in a burst before enforcing.
|
||||
rl := util.NewTokenBucketRateLimiter(*apiRate, *apiBurst)
|
||||
readOnlyServer := &http.Server{
|
||||
Addr: roLocation,
|
||||
Handler: apiserver.RecoverPanics(apiserver.ReadOnly(apiserver.RateLimit(rl, m.InsecureHandler))),
|
||||
ReadTimeout: 5 * time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
MaxHeaderBytes: 1 << 20,
|
||||
}
|
||||
glog.Infof("Serving read-only insecurely on %s", roLocation)
|
||||
go func() {
|
||||
defer util.HandleCrash()
|
||||
for {
|
||||
if err := readOnlyServer.ListenAndServe(); err != nil {
|
||||
glog.Errorf("Unable to listen for read only traffic (%v); will try again.", err)
|
||||
}
|
||||
time.Sleep(15 * time.Second)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if secureLocation != "" {
|
||||
secureServer := &http.Server{
|
||||
Addr: secureLocation,
|
||||
Handler: apiserver.RecoverPanics(m.Handler),
|
||||
ReadTimeout: 5 * time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
MaxHeaderBytes: 1 << 20,
|
||||
TLSConfig: &tls.Config{
|
||||
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
|
||||
MinVersion: tls.VersionTLS10,
|
||||
// Populate PeerCertificates in requests, but don't reject connections without certificates
|
||||
// This allows certificates to be validated by authenticators, while still allowing other auth types
|
||||
ClientAuth: tls.RequestClientCert,
|
||||
},
|
||||
}
|
||||
glog.Infof("Serving securely on %s", secureLocation)
|
||||
go func() {
|
||||
defer util.HandleCrash()
|
||||
for {
|
||||
if *tlsCertFile == "" && *tlsPrivateKeyFile == "" {
|
||||
*tlsCertFile = "/var/run/kubernetes/apiserver.crt"
|
||||
*tlsPrivateKeyFile = "/var/run/kubernetes/apiserver.key"
|
||||
if err := util.GenerateSelfSignedCert(config.PublicAddress, *tlsCertFile, *tlsPrivateKeyFile); err != nil {
|
||||
glog.Errorf("Unable to generate self signed cert: %v", err)
|
||||
} else {
|
||||
glog.Infof("Using self-signed cert (%s, %s)", *tlsCertFile, *tlsPrivateKeyFile)
|
||||
}
|
||||
}
|
||||
if err := secureServer.ListenAndServeTLS(*tlsCertFile, *tlsPrivateKeyFile); err != nil {
|
||||
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
|
||||
}
|
||||
time.Sleep(15 * time.Second)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
s := &http.Server{
|
||||
Addr: rwLocation,
|
||||
Handler: apiserver.RecoverPanics(m.InsecureHandler),
|
||||
ReadTimeout: 5 * time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
MaxHeaderBytes: 1 << 20,
|
||||
}
|
||||
glog.Infof("Serving insecurely on %s", rwLocation)
|
||||
glog.Fatal(s.ListenAndServe())
|
||||
s.Run(pflag.CommandLine.Args())
|
||||
}
|
||||
|
||||
@ -38,9 +38,6 @@ The the kube-apiserver several options.
|
||||
**--etcd_servers**=[]
|
||||
List of etcd servers to watch (http://ip:port), comma separated
|
||||
|
||||
**--health_check_minions**=
|
||||
If true, health check minions and filter unhealthy ones. Default true.
|
||||
|
||||
**--log_backtrace_at=**:0
|
||||
when logging hits line file:N, emit a stack trace
|
||||
|
||||
|
||||
@ -26,6 +26,7 @@ import (
|
||||
"path"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
@ -128,6 +129,9 @@ func (hk *HyperKube) Run(args []string) error {
|
||||
hk.Usage()
|
||||
return err
|
||||
}
|
||||
|
||||
verflag.PrintAndExitIfRequested()
|
||||
|
||||
args = baseFlags.Args()
|
||||
if len(args) > 0 && len(args[0]) > 0 {
|
||||
serverName = args[0]
|
||||
@ -135,7 +139,7 @@ func (hk *HyperKube) Run(args []string) error {
|
||||
args = args[1:]
|
||||
} else {
|
||||
err = errors.New("No server specified")
|
||||
hk.Println("Error", err)
|
||||
hk.Printf("Error: %v\n\n", err)
|
||||
hk.Usage()
|
||||
return err
|
||||
}
|
||||
@ -143,7 +147,7 @@ func (hk *HyperKube) Run(args []string) error {
|
||||
|
||||
s, err := hk.FindServer(serverName)
|
||||
if err != nil {
|
||||
hk.Println("Error:", err)
|
||||
hk.Printf("Error: %v\n\n", err)
|
||||
hk.Usage()
|
||||
return err
|
||||
}
|
||||
@ -152,13 +156,17 @@ func (hk *HyperKube) Run(args []string) error {
|
||||
err = s.Flags().Parse(args)
|
||||
if err != nil || hk.helpFlagVal {
|
||||
if err != nil {
|
||||
hk.Println("Error:", err)
|
||||
hk.Printf("Error: %v\n\n", err)
|
||||
}
|
||||
s.Usage()
|
||||
return err
|
||||
}
|
||||
|
||||
verflag.PrintAndExitIfRequested()
|
||||
|
||||
util.InitLogs()
|
||||
defer util.FlushLogs()
|
||||
|
||||
err = s.Run(s, s.Flags().Args())
|
||||
if err != nil {
|
||||
hk.Println("Error:", err)
|
||||
@ -167,6 +175,15 @@ func (hk *HyperKube) Run(args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// RunToExit will run the hyperkube and then call os.Exit with an appropriate exit code.
|
||||
func (hk *HyperKube) RunToExit(args []string) {
|
||||
err := hk.Run(args)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Usage will write out a summary for all servers that this binary supports.
|
||||
func (hk *HyperKube) Usage() {
|
||||
tt := `{{if .Long}}{{.Long | trim | wrap ""}}
|
||||
|
||||
@ -38,7 +38,7 @@ type Server struct {
|
||||
hk *HyperKube
|
||||
}
|
||||
|
||||
// FullUsage returns the full usage string including all of the flags.
|
||||
// Usage returns the full usage string including all of the flags.
|
||||
func (s *Server) Usage() error {
|
||||
tt := `{{if .Long}}{{.Long | trim | wrap ""}}
|
||||
{{end}}Usage:
|
||||
|
||||
@ -14,18 +14,20 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
package server
|
||||
|
||||
// This file exists to force the desired plugin implementations to be linked.
|
||||
// This should probably be part of some configuration fed into the build for a
|
||||
// given binary target.
|
||||
import (
|
||||
// Cloud providers
|
||||
_ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/aws"
|
||||
_ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/gce"
|
||||
_ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/openstack"
|
||||
_ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/ovirt"
|
||||
_ "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/vagrant"
|
||||
|
||||
// Admission policies
|
||||
_ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit"
|
||||
_ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/deny"
|
||||
_ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/limitranger"
|
||||
344
pkg/master/server/server.go
Normal file
344
pkg/master/server/server.go
Normal file
@ -0,0 +1,344 @@
|
||||
/*
|
||||
Copyright 2014 Google Inc. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package server does all of the work necessary to create a Kubernetes
|
||||
// APIServer by binding together the API, master and APIServer infrastructure.
|
||||
// It can be configured and called directly or via the hyperkube framework.
|
||||
package server
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/admission"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/hyperkube"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/master"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
|
||||
|
||||
"github.com/coreos/go-etcd/etcd"
|
||||
"github.com/golang/glog"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// APIServer runs a kubernetes api server.
|
||||
type APIServer struct {
|
||||
Port int
|
||||
Address util.IP
|
||||
PublicAddressOverride string
|
||||
ReadOnlyPort int
|
||||
APIRate float32
|
||||
APIBurst int
|
||||
SecurePort int
|
||||
TLSCertFile string
|
||||
TLSPrivateKeyFile string
|
||||
APIPrefix string
|
||||
StorageVersion string
|
||||
CloudProvider string
|
||||
CloudConfigFile string
|
||||
EventTTL time.Duration
|
||||
TokenAuthFile string
|
||||
AuthorizationMode string
|
||||
AuthorizationPolicyFile string
|
||||
AdmissionControl string
|
||||
AdmissionControlConfigFile string
|
||||
EtcdServerList util.StringList
|
||||
EtcdConfigFile string
|
||||
CorsAllowedOriginList util.StringList
|
||||
AllowPrivileged bool
|
||||
PortalNet util.IPNet // TODO: make this a list
|
||||
EnableLogsSupport bool
|
||||
MasterServiceNamespace string
|
||||
RuntimeConfig util.ConfigurationMap
|
||||
KubeletConfig client.KubeletConfig
|
||||
}
|
||||
|
||||
// NewAPIServer creates a new APIServer object with default parameters
|
||||
func NewAPIServer() *APIServer {
|
||||
s := APIServer{
|
||||
Port: 8080,
|
||||
Address: util.IP(net.ParseIP("127.0.0.1")),
|
||||
ReadOnlyPort: 7080,
|
||||
APIRate: 10.0,
|
||||
APIBurst: 200,
|
||||
SecurePort: 8443,
|
||||
APIPrefix: "/api",
|
||||
EventTTL: 48 * time.Hour,
|
||||
AuthorizationMode: "AlwaysAllow",
|
||||
AdmissionControl: "AlwaysAdmit",
|
||||
EnableLogsSupport: true,
|
||||
MasterServiceNamespace: api.NamespaceDefault,
|
||||
|
||||
RuntimeConfig: make(util.ConfigurationMap),
|
||||
KubeletConfig: client.KubeletConfig{
|
||||
Port: 10250,
|
||||
EnableHttps: false,
|
||||
},
|
||||
}
|
||||
|
||||
return &s
|
||||
}
|
||||
|
||||
// NewHyperkubeServer creates a new hyperkube Server object that includes the
|
||||
// description and flags.
|
||||
func NewHyperkubeServer() *hyperkube.Server {
|
||||
s := NewAPIServer()
|
||||
|
||||
hks := hyperkube.Server{
|
||||
SimpleUsage: "apiserver",
|
||||
Long: "The main API entrypoint and interface to the storage system. The API server is also the focal point for all authorization decisions.",
|
||||
Run: func(_ *hyperkube.Server, args []string) error {
|
||||
return s.Run(args)
|
||||
},
|
||||
}
|
||||
s.AddFlags(hks.Flags())
|
||||
return &hks
|
||||
}
|
||||
|
||||
// AddFlags adds flags for a specific APIServer to the specified FlagSet
|
||||
func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
|
||||
// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
|
||||
// arrange these text blocks sensibly. Grrr.
|
||||
fs.IntVar(&s.Port, "port", s.Port, ""+
|
||||
"The port to listen on. Default 8080. It is assumed that firewall rules are "+
|
||||
"set up such that this port is not reachable from outside of the cluster. It is "+
|
||||
"further assumed that port 443 on the cluster's public address is proxied to this "+
|
||||
"port. This is performed by nginx in the default setup.")
|
||||
fs.Var(&s.Address, "address", "The IP address on to serve on (set to 0.0.0.0 for all interfaces)")
|
||||
fs.StringVar(&s.PublicAddressOverride, "public_address_override", s.PublicAddressOverride, ""+
|
||||
"Public serving address. Read only port will be opened on this address, "+
|
||||
"and it is assumed that port 443 at this address will be proxied/redirected "+
|
||||
"to '-address':'-port'. If blank, the address in the first listed interface "+
|
||||
"will be used.")
|
||||
fs.IntVar(&s.ReadOnlyPort, "read_only_port", s.ReadOnlyPort, ""+
|
||||
"The port from which to serve read-only resources. If 0, don't serve on a "+
|
||||
"read-only address. It is assumed that firewall rules are set up such that "+
|
||||
"this port is not reachable from outside of the cluster.")
|
||||
fs.Float32Var(&s.APIRate, "api_rate", s.APIRate, "API rate limit as QPS for the read only port")
|
||||
fs.IntVar(&s.APIBurst, "api_burst", s.APIBurst, "API burst amount for the read only port")
|
||||
fs.IntVar(&s.SecurePort, "secure_port", s.SecurePort,
|
||||
"The port from which to serve HTTPS with authentication and authorization. If 0, don't serve HTTPS ")
|
||||
fs.StringVar(&s.TLSCertFile, "tls_cert_file", s.TLSCertFile, ""+
|
||||
"File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert). "+
|
||||
"If HTTPS serving is enabled, and --tls_cert_file and --tls_private_key_file are not provided, "+
|
||||
"a self-signed certificate and key are generated for the public address and saved to /var/run/kubernetes.")
|
||||
fs.StringVar(&s.TLSPrivateKeyFile, "tls_private_key_file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls_cert_file.")
|
||||
fs.StringVar(&s.APIPrefix, "api_prefix", s.APIPrefix, "The prefix for API requests on the server. Default '/api'.")
|
||||
fs.StringVar(&s.StorageVersion, "storage_version", s.StorageVersion, "The version to store resources with. Defaults to server preferred")
|
||||
fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services. Empty string for no provider.")
|
||||
fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
|
||||
fs.DurationVar(&s.EventTTL, "event_ttl", s.EventTTL, "Amount of time to retain events. Default 2 days.")
|
||||
fs.StringVar(&s.TokenAuthFile, "token_auth_file", s.TokenAuthFile, "If set, the file that will be used to secure the secure port of the API server via token authentication.")
|
||||
fs.StringVar(&s.AuthorizationMode, "authorization_mode", s.AuthorizationMode, "Selects how to do authorization on the secure port. One of: "+strings.Join(apiserver.AuthorizationModeChoices, ","))
|
||||
fs.StringVar(&s.AuthorizationPolicyFile, "authorization_policy_file", s.AuthorizationPolicyFile, "File with authorization policy in csv format, used with --authorization_mode=ABAC, on the secure port.")
|
||||
fs.StringVar(&s.AdmissionControl, "admission_control", s.AdmissionControl, "Ordered list of plug-ins to do admission control of resources into cluster. Comma-delimited list of: "+strings.Join(admission.GetPlugins(), ", "))
|
||||
fs.StringVar(&s.AdmissionControlConfigFile, "admission_control_config_file", s.AdmissionControlConfigFile, "File with admission control configuration.")
|
||||
fs.Var(&s.EtcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated. Mutually exclusive with -etcd_config")
|
||||
fs.StringVar(&s.EtcdConfigFile, "etcd_config", s.EtcdConfigFile, "The config file for the etcd client. Mutually exclusive with -etcd_servers.")
|
||||
fs.Var(&s.CorsAllowedOriginList, "cors_allowed_origins", "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. If this list is empty CORS will not be enabled.")
|
||||
fs.BoolVar(&s.AllowPrivileged, "allow_privileged", s.AllowPrivileged, "If true, allow privileged containers.")
|
||||
fs.Var(&s.PortalNet, "portal_net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.")
|
||||
fs.StringVar(&s.MasterServiceNamespace, "master_service_namespace", s.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods")
|
||||
fs.Var(&s.RuntimeConfig, "runtime_config", "A set of key=value pairs that describe runtime configuration that may be passed to the apiserver.")
|
||||
client.BindKubeletClientConfigFlags(fs, &s.KubeletConfig)
|
||||
}
|
||||
|
||||
// TODO: Longer term we should read this from some config store, rather than a flag.
|
||||
func (s *APIServer) verifyPortalFlags() {
|
||||
if s.PortalNet.IP == nil {
|
||||
glog.Fatal("No --portal_net specified")
|
||||
}
|
||||
}
|
||||
|
||||
func newEtcd(etcdConfigFile string, etcdServerList util.StringList, storageVersion string) (helper tools.EtcdHelper, err error) {
|
||||
var client tools.EtcdGetSet
|
||||
if etcdConfigFile != "" {
|
||||
client, err = etcd.NewClientFromFile(etcdConfigFile)
|
||||
if err != nil {
|
||||
return helper, err
|
||||
}
|
||||
} else {
|
||||
client = etcd.NewClient(etcdServerList)
|
||||
}
|
||||
|
||||
return master.NewEtcdHelper(client, storageVersion)
|
||||
}
|
||||
|
||||
// Run runs the specified APIServer. This should never exit.
|
||||
func (s *APIServer) Run(_ []string) error {
|
||||
s.verifyPortalFlags()
|
||||
|
||||
if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) {
|
||||
glog.Fatalf("specify either --etcd_servers or --etcd_config")
|
||||
}
|
||||
|
||||
capabilities.Initialize(capabilities.Capabilities{
|
||||
AllowPrivileged: s.AllowPrivileged,
|
||||
})
|
||||
|
||||
cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
|
||||
|
||||
kubeletClient, err := client.NewKubeletClient(&s.KubeletConfig)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failure to start kubelet client: %v", err)
|
||||
}
|
||||
|
||||
_, v1beta3 := s.RuntimeConfig["api/v1beta3"]
|
||||
|
||||
// TODO: expose same flags as client.BindClientConfigFlags but for a server
|
||||
clientConfig := &client.Config{
|
||||
Host: net.JoinHostPort(s.Address.String(), strconv.Itoa(int(s.Port))),
|
||||
Version: s.StorageVersion,
|
||||
}
|
||||
client, err := client.New(clientConfig)
|
||||
if err != nil {
|
||||
glog.Fatalf("Invalid server address: %v", err)
|
||||
}
|
||||
|
||||
helper, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, s.StorageVersion)
|
||||
if err != nil {
|
||||
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
|
||||
}
|
||||
|
||||
n := net.IPNet(s.PortalNet)
|
||||
|
||||
authenticator, err := apiserver.NewAuthenticatorFromTokenFile(s.TokenAuthFile)
|
||||
if err != nil {
|
||||
glog.Fatalf("Invalid Authentication Config: %v", err)
|
||||
}
|
||||
|
||||
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(s.AuthorizationMode, s.AuthorizationPolicyFile)
|
||||
if err != nil {
|
||||
glog.Fatalf("Invalid Authorization Config: %v", err)
|
||||
}
|
||||
|
||||
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
|
||||
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
|
||||
|
||||
config := &master.Config{
|
||||
Client: client,
|
||||
Cloud: cloud,
|
||||
EtcdHelper: helper,
|
||||
EventTTL: s.EventTTL,
|
||||
KubeletClient: kubeletClient,
|
||||
PortalNet: &n,
|
||||
EnableLogsSupport: s.EnableLogsSupport,
|
||||
EnableUISupport: true,
|
||||
EnableSwaggerSupport: true,
|
||||
APIPrefix: s.APIPrefix,
|
||||
CorsAllowedOriginList: s.CorsAllowedOriginList,
|
||||
ReadOnlyPort: s.ReadOnlyPort,
|
||||
ReadWritePort: s.Port,
|
||||
PublicAddress: s.PublicAddressOverride,
|
||||
Authenticator: authenticator,
|
||||
Authorizer: authorizer,
|
||||
AdmissionControl: admissionController,
|
||||
EnableV1Beta3: v1beta3,
|
||||
MasterServiceNamespace: s.MasterServiceNamespace,
|
||||
}
|
||||
m := master.New(config)
|
||||
|
||||
// We serve on 3 ports. See docs/reaching_the_api.md
|
||||
roLocation := ""
|
||||
if s.ReadOnlyPort != 0 {
|
||||
roLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(config.ReadOnlyPort))
|
||||
}
|
||||
secureLocation := ""
|
||||
if s.SecurePort != 0 {
|
||||
secureLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(s.SecurePort))
|
||||
}
|
||||
rwLocation := net.JoinHostPort(s.Address.String(), strconv.Itoa(int(s.Port)))
|
||||
|
||||
// See the flag commentary to understand our assumptions when opening the read-only and read-write ports.
|
||||
|
||||
if roLocation != "" {
|
||||
// Default settings allow 1 read-only request per second, allow up to 20 in a burst before enforcing.
|
||||
rl := util.NewTokenBucketRateLimiter(s.APIRate, s.APIBurst)
|
||||
readOnlyServer := &http.Server{
|
||||
Addr: roLocation,
|
||||
Handler: apiserver.RecoverPanics(apiserver.ReadOnly(apiserver.RateLimit(rl, m.InsecureHandler))),
|
||||
ReadTimeout: 5 * time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
MaxHeaderBytes: 1 << 20,
|
||||
}
|
||||
glog.Infof("Serving read-only insecurely on %s", roLocation)
|
||||
go func() {
|
||||
defer util.HandleCrash()
|
||||
for {
|
||||
if err := readOnlyServer.ListenAndServe(); err != nil {
|
||||
glog.Errorf("Unable to listen for read only traffic (%v); will try again.", err)
|
||||
}
|
||||
time.Sleep(15 * time.Second)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if secureLocation != "" {
|
||||
secureServer := &http.Server{
|
||||
Addr: secureLocation,
|
||||
Handler: apiserver.RecoverPanics(m.Handler),
|
||||
ReadTimeout: 5 * time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
MaxHeaderBytes: 1 << 20,
|
||||
TLSConfig: &tls.Config{
|
||||
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
|
||||
MinVersion: tls.VersionTLS10,
|
||||
// Populate PeerCertificates in requests, but don't reject connections without certificates
|
||||
// This allows certificates to be validated by authenticators, while still allowing other auth types
|
||||
ClientAuth: tls.RequestClientCert,
|
||||
},
|
||||
}
|
||||
glog.Infof("Serving securely on %s", secureLocation)
|
||||
go func() {
|
||||
defer util.HandleCrash()
|
||||
for {
|
||||
if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
|
||||
s.TLSCertFile = "/var/run/kubernetes/apiserver.crt"
|
||||
s.TLSPrivateKeyFile = "/var/run/kubernetes/apiserver.key"
|
||||
if err := util.GenerateSelfSignedCert(config.PublicAddress, s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
|
||||
glog.Errorf("Unable to generate self signed cert: %v", err)
|
||||
} else {
|
||||
glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
|
||||
}
|
||||
}
|
||||
if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
|
||||
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
|
||||
}
|
||||
time.Sleep(15 * time.Second)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
http := &http.Server{
|
||||
Addr: rwLocation,
|
||||
Handler: apiserver.RecoverPanics(m.InsecureHandler),
|
||||
ReadTimeout: 5 * time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
MaxHeaderBytes: 1 << 20,
|
||||
}
|
||||
glog.Infof("Serving insecurely on %s", rwLocation)
|
||||
glog.Fatal(http.ListenAndServe())
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user