genericapiserver: move MasterCount and service options into master

This commit is contained in:
Dr. Stefan Schimanski 2016-12-06 12:22:49 +01:00
parent 1eb9176455
commit 7267299c3c
10 changed files with 63 additions and 66 deletions

View File

@ -18,6 +18,7 @@ limitations under the License.
package options package options
import ( import (
"net"
"time" "time"
"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api"
@ -25,10 +26,14 @@ import (
genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options" genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
"k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/pkg/master/ports"
utilnet "k8s.io/kubernetes/pkg/util/net"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
// DefaultServiceNodePortRange is the default port range for NodePort services.
var DefaultServiceNodePortRange = utilnet.PortRange{Base: 30000, Size: 2768}
// ServerRunOptions runs a kubernetes api server. // ServerRunOptions runs a kubernetes api server.
type ServerRunOptions struct { type ServerRunOptions struct {
GenericServerRunOptions *genericoptions.ServerRunOptions GenericServerRunOptions *genericoptions.ServerRunOptions
@ -38,12 +43,16 @@ type ServerRunOptions struct {
Authentication *genericoptions.BuiltInAuthenticationOptions Authentication *genericoptions.BuiltInAuthenticationOptions
Authorization *genericoptions.BuiltInAuthorizationOptions Authorization *genericoptions.BuiltInAuthorizationOptions
AllowPrivileged bool AllowPrivileged bool
EventTTL time.Duration EventTTL time.Duration
KubeletConfig kubeletclient.KubeletClientConfig KubeletConfig kubeletclient.KubeletClientConfig
MaxConnectionBytesPerSec int64 KubernetesServiceNodePort int
SSHKeyfile string MasterCount int
SSHUser string MaxConnectionBytesPerSec int64
ServiceClusterIPRange net.IPNet // TODO: make this a list
ServiceNodePortRange utilnet.PortRange
SSHKeyfile string
SSHUser string
} }
// NewServerRunOptions creates a new ServerRunOptions object with default parameters // NewServerRunOptions creates a new ServerRunOptions object with default parameters
@ -56,7 +65,8 @@ func NewServerRunOptions() *ServerRunOptions {
Authentication: genericoptions.NewBuiltInAuthenticationOptions().WithAll(), Authentication: genericoptions.NewBuiltInAuthenticationOptions().WithAll(),
Authorization: genericoptions.NewBuiltInAuthorizationOptions(), Authorization: genericoptions.NewBuiltInAuthorizationOptions(),
EventTTL: 1 * time.Hour, EventTTL: 1 * time.Hour,
MasterCount: 1,
KubeletConfig: kubeletclient.KubeletClientConfig{ KubeletConfig: kubeletclient.KubeletClientConfig{
Port: ports.KubeletPort, Port: ports.KubeletPort,
PreferredAddressTypes: []string{ PreferredAddressTypes: []string{
@ -68,6 +78,7 @@ func NewServerRunOptions() *ServerRunOptions {
EnableHttps: true, EnableHttps: true,
HTTPTimeout: time.Duration(5) * time.Second, HTTPTimeout: time.Duration(5) * time.Second,
}, },
ServiceNodePortRange: DefaultServiceNodePortRange,
} }
return &s return &s
} }
@ -104,6 +115,30 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
"If non-zero, throttle each user connection to this number of bytes/sec. "+ "If non-zero, throttle each user connection to this number of bytes/sec. "+
"Currently only applies to long-running requests.") "Currently only applies to long-running requests.")
fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
"The number of apiservers running in the cluster.")
// See #14282 for details on how to test/try this option out.
// TODO: remove this comment once this option is tested in CI.
fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", s.KubernetesServiceNodePort, ""+
"If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be "+
"of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+
"service will be of type ClusterIP.")
fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+
"A CIDR notation IP range from which to assign service cluster IPs. This must not "+
"overlap with any IP ranges assigned to nodes for pods.")
fs.IPNetVar(&s.ServiceClusterIPRange, "portal-net", s.ServiceClusterIPRange,
"DEPRECATED: see --service-cluster-ip-range instead.")
fs.MarkDeprecated("portal-net", "see --service-cluster-ip-range instead")
fs.Var(&s.ServiceNodePortRange, "service-node-port-range", ""+
"A port range to reserve for services with NodePort visibility. "+
"Example: '30000-32767'. Inclusive at both ends of the range.")
fs.Var(&s.ServiceNodePortRange, "service-node-ports", "DEPRECATED: see --service-node-port-range instead")
fs.MarkDeprecated("service-node-ports", "see --service-node-port-range instead")
// Kubelet related flags: // Kubelet related flags:
fs.BoolVar(&s.KubeletConfig.EnableHttps, "kubelet-https", s.KubeletConfig.EnableHttps, fs.BoolVar(&s.KubeletConfig.EnableHttps, "kubelet-https", s.KubeletConfig.EnableHttps,
"Use https for kubelet connections.") "Use https for kubelet connections.")

View File

@ -14,18 +14,16 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package validation package options
import ( import (
"fmt" "fmt"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/genericapiserver/options"
utilerrors "k8s.io/kubernetes/pkg/util/errors" utilerrors "k8s.io/kubernetes/pkg/util/errors"
) )
// TODO: Longer term we should read this from some config store, rather than a flag. // TODO: Longer term we should read this from some config store, rather than a flag.
func verifyClusterIPFlags(options *options.ServerRunOptions) []error { func verifyClusterIPFlags(options *ServerRunOptions) []error {
errors := []error{} errors := []error{}
if options.ServiceClusterIPRange.IP == nil { if options.ServiceClusterIPRange.IP == nil {
errors = append(errors, fmt.Errorf("No --service-cluster-ip-range specified")) errors = append(errors, fmt.Errorf("No --service-cluster-ip-range specified"))
@ -37,7 +35,7 @@ func verifyClusterIPFlags(options *options.ServerRunOptions) []error {
return errors return errors
} }
func verifyServiceNodePort(options *options.ServerRunOptions) []error { func verifyServiceNodePort(options *ServerRunOptions) []error {
errors := []error{} errors := []error{}
if options.KubernetesServiceNodePort < 0 || options.KubernetesServiceNodePort > 65535 { if options.KubernetesServiceNodePort < 0 || options.KubernetesServiceNodePort > 65535 {
errors = append(errors, fmt.Errorf("--kubernetes-service-node-port %v must be between 0 and 65535, inclusive. If 0, the Kubernetes master service will be of type ClusterIP.", options.KubernetesServiceNodePort)) errors = append(errors, fmt.Errorf("--kubernetes-service-node-port %v must be between 0 and 65535, inclusive. If 0, the Kubernetes master service will be of type ClusterIP.", options.KubernetesServiceNodePort))
@ -49,7 +47,7 @@ func verifyServiceNodePort(options *options.ServerRunOptions) []error {
return errors return errors
} }
func ValidateRunOptions(options *options.ServerRunOptions) { func ValidateRunOptions(options *ServerRunOptions) error {
errors := []error{} errors := []error{}
if errs := verifyClusterIPFlags(options); len(errs) > 0 { if errs := verifyClusterIPFlags(options); len(errs) > 0 {
errors = append(errors, errs...) errors = append(errors, errs...)
@ -58,6 +56,7 @@ func ValidateRunOptions(options *options.ServerRunOptions) {
errors = append(errors, errs...) errors = append(errors, errs...)
} }
if err := utilerrors.NewAggregate(errors); err != nil { if err := utilerrors.NewAggregate(errors); err != nil {
glog.Fatalf("Validate server run options failed: %v", err) return fmt.Errorf("validate server run options failed: %v", err)
} }
return nil
} }

View File

@ -88,7 +88,7 @@ func Run(s *options.ServerRunOptions) error {
return err return err
} }
serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange) serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
if err != nil { if err != nil {
return fmt.Errorf("error determining service IP ranges: %v", err) return fmt.Errorf("error determining service IP ranges: %v", err)
} }
@ -97,8 +97,14 @@ func Run(s *options.ServerRunOptions) error {
return fmt.Errorf("error creating self-signed certificates: %v", err) return fmt.Errorf("error creating self-signed certificates: %v", err)
} }
// TODO(sttts): change signature of DefaultAndValidateRunOptions to aggregate errors
genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions) genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
// TODO(sttts): move all defaulting and validation above into cmd/kube-apiserver/app/options.DefaultAndValidateRunOptions()
if err != options.ValidateRunOptions(s) {
return err
}
genericConfig := genericapiserver.NewConfig(). // create the new config genericConfig := genericapiserver.NewConfig(). // create the new config
ApplyOptions(s.GenericServerRunOptions). // apply the options selected ApplyOptions(s.GenericServerRunOptions). // apply the options selected
ApplyInsecureServingOptions(s.InsecureServing) ApplyInsecureServingOptions(s.InsecureServing)
@ -313,10 +319,10 @@ func Run(s *options.ServerRunOptions) error {
APIServerServiceIP: apiServerServiceIP, APIServerServiceIP: apiServerServiceIP,
APIServerServicePort: 443, APIServerServicePort: 443,
ServiceNodePortRange: s.GenericServerRunOptions.ServiceNodePortRange, ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.GenericServerRunOptions.KubernetesServiceNodePort, KubernetesServiceNodePort: s.KubernetesServiceNodePort,
MasterCount: s.GenericServerRunOptions.MasterCount, MasterCount: s.MasterCount,
} }
if s.GenericServerRunOptions.EnableWatchCache { if s.GenericServerRunOptions.EnableWatchCache {

View File

@ -80,12 +80,10 @@ func NewServerRunOptions() *ServerRunOptions {
} }
func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error { func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error {
// Set ServiceClusterIPRange
_, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
serverOptions.GenericServerRunOptions.ServiceClusterIPRange = *serviceClusterIPRange
serverOptions.Etcd.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"} serverOptions.Etcd.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"}
genericvalidation.ValidateRunOptions(serverOptions.GenericServerRunOptions) // TODO(sttts): unify signature of DefaultAndValidateRunOptions with the others
genericapiserver.DefaultAndValidateRunOptions(serverOptions.GenericServerRunOptions)
if errs := serverOptions.Etcd.Validate(); len(errs) > 0 { if errs := serverOptions.Etcd.Validate(); len(errs) > 0 {
return utilerrors.NewAggregate(errs) return utilerrors.NewAggregate(errs)
} }

View File

@ -61,8 +61,7 @@ kube::log::status "Starting federation-apiserver"
--etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \ --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \
--advertise-address="10.10.10.10" \ --advertise-address="10.10.10.10" \
--cert-dir="${TMP_DIR}/certs" \ --cert-dir="${TMP_DIR}/certs" \
--token-auth-file=$TMP_DIR/tokenauth.csv \ --token-auth-file=$TMP_DIR/tokenauth.csv >/tmp/openapi-federation-api-server.log 2>&1 &
--service-cluster-ip-range="10.0.0.0/24" >/tmp/openapi-federation-api-server.log 2>&1 &
APISERVER_PID=$! APISERVER_PID=$!
kube::util::wait_for_url "${API_HOST}:${API_PORT}/" "apiserver: " kube::util::wait_for_url "${API_HOST}:${API_PORT}/" "apiserver: "

View File

@ -59,7 +59,6 @@ import (
openapicommon "k8s.io/kubernetes/pkg/genericapiserver/openapi/common" openapicommon "k8s.io/kubernetes/pkg/genericapiserver/openapi/common"
"k8s.io/kubernetes/pkg/genericapiserver/options" "k8s.io/kubernetes/pkg/genericapiserver/options"
"k8s.io/kubernetes/pkg/genericapiserver/routes" "k8s.io/kubernetes/pkg/genericapiserver/routes"
genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
"k8s.io/kubernetes/pkg/healthz" "k8s.io/kubernetes/pkg/healthz"
"k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/runtime"
certutil "k8s.io/kubernetes/pkg/util/cert" certutil "k8s.io/kubernetes/pkg/util/cert"
@ -631,8 +630,6 @@ func (s *GenericAPIServer) installAPI(c *Config) {
} }
func DefaultAndValidateRunOptions(options *options.ServerRunOptions) { func DefaultAndValidateRunOptions(options *options.ServerRunOptions) {
genericvalidation.ValidateRunOptions(options)
glog.Infof("Will report %v as public IP address.", options.AdvertiseAddress) glog.Infof("Will report %v as public IP address.", options.AdvertiseAddress)
// Set default value for ExternalAddress if not specified. // Set default value for ExternalAddress if not specified.

View File

@ -26,13 +26,10 @@ import (
"k8s.io/kubernetes/pkg/apimachinery/registered" "k8s.io/kubernetes/pkg/apimachinery/registered"
"k8s.io/kubernetes/pkg/runtime/schema" "k8s.io/kubernetes/pkg/runtime/schema"
"k8s.io/kubernetes/pkg/util/config" "k8s.io/kubernetes/pkg/util/config"
utilnet "k8s.io/kubernetes/pkg/util/net"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
var DefaultServiceNodePortRange = utilnet.PortRange{Base: 30000, Size: 2768}
// ServerRunOptions contains the options while running a generic api server. // ServerRunOptions contains the options while running a generic api server.
type ServerRunOptions struct { type ServerRunOptions struct {
AdmissionControl string AdmissionControl string
@ -54,14 +51,10 @@ type ServerRunOptions struct {
EnableSwaggerUI bool EnableSwaggerUI bool
EnableWatchCache bool EnableWatchCache bool
ExternalHost string ExternalHost string
KubernetesServiceNodePort int
MasterCount int
MaxRequestsInFlight int MaxRequestsInFlight int
MaxMutatingRequestsInFlight int MaxMutatingRequestsInFlight int
MinRequestTimeout int MinRequestTimeout int
RuntimeConfig config.ConfigurationMap RuntimeConfig config.ConfigurationMap
ServiceClusterIPRange net.IPNet // TODO: make this a list
ServiceNodePortRange utilnet.PortRange
StorageVersions string StorageVersions string
// The default values for StorageVersions. StorageVersions overrides // The default values for StorageVersions. StorageVersions overrides
// these; you can change this if you want to change the defaults (e.g., // these; you can change this if you want to change the defaults (e.g.,
@ -81,12 +74,10 @@ func NewServerRunOptions() *ServerRunOptions {
EnableProfiling: true, EnableProfiling: true,
EnableContentionProfiling: false, EnableContentionProfiling: false,
EnableWatchCache: true, EnableWatchCache: true,
MasterCount: 1,
MaxRequestsInFlight: 400, MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200, MaxMutatingRequestsInFlight: 200,
MinRequestTimeout: 1800, MinRequestTimeout: 1800,
RuntimeConfig: make(config.ConfigurationMap), RuntimeConfig: make(config.ConfigurationMap),
ServiceNodePortRange: DefaultServiceNodePortRange,
StorageVersions: registered.AllPreferredGroupVersions(), StorageVersions: registered.AllPreferredGroupVersions(),
} }
} }
@ -227,22 +218,12 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost, fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs).") "The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs).")
// See #14282 for details on how to test/try this option out.
// TODO: remove this comment once this option is tested in CI.
fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", s.KubernetesServiceNodePort, ""+
"If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be "+
"of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+
"service will be of type ClusterIP.")
// TODO: remove post-1.6 // TODO: remove post-1.6
fs.String("long-running-request-regexp", "", ""+ fs.String("long-running-request-regexp", "", ""+
"A regular expression matching long running requests which should "+ "A regular expression matching long running requests which should "+
"be excluded from maximum inflight request handling.") "be excluded from maximum inflight request handling.")
fs.MarkDeprecated("long-running-request-regexp", "regular expression matching of long-running requests is no longer supported") fs.MarkDeprecated("long-running-request-regexp", "regular expression matching of long-running requests is no longer supported")
fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
"The number of apiservers running in the cluster.")
deprecatedMasterServiceNamespace := api.NamespaceDefault deprecatedMasterServiceNamespace := api.NamespaceDefault
fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+ fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+
"DEPRECATED: the namespace from which the kubernetes master services should be injected into pods.") "DEPRECATED: the namespace from which the kubernetes master services should be injected into pods.")
@ -267,20 +248,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
"apis/<groupVersion>/<resource> can be used to turn on/off specific resources. api/all and "+ "apis/<groupVersion>/<resource> can be used to turn on/off specific resources. api/all and "+
"api/legacy are special keys to control all and legacy api versions respectively.") "api/legacy are special keys to control all and legacy api versions respectively.")
fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+
"A CIDR notation IP range from which to assign service cluster IPs. This must not "+
"overlap with any IP ranges assigned to nodes for pods.")
fs.IPNetVar(&s.ServiceClusterIPRange, "portal-net", s.ServiceClusterIPRange,
"DEPRECATED: see --service-cluster-ip-range instead.")
fs.MarkDeprecated("portal-net", "see --service-cluster-ip-range instead")
fs.Var(&s.ServiceNodePortRange, "service-node-port-range", ""+
"A port range to reserve for services with NodePort visibility. "+
"Example: '30000-32767'. Inclusive at both ends of the range.")
fs.Var(&s.ServiceNodePortRange, "service-node-ports", "DEPRECATED: see --service-node-port-range instead")
fs.MarkDeprecated("service-node-ports", "see --service-node-port-range instead")
deprecatedStorageVersion := "" deprecatedStorageVersion := ""
fs.StringVar(&deprecatedStorageVersion, "storage-version", deprecatedStorageVersion, fs.StringVar(&deprecatedStorageVersion, "storage-version", deprecatedStorageVersion,
"DEPRECATED: the version to store the legacy v1 resources with. Defaults to server preferred.") "DEPRECATED: the version to store the legacy v1 resources with. Defaults to server preferred.")

View File

@ -24,6 +24,7 @@ import (
"strconv" "strconv"
"time" "time"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api"
apiv1 "k8s.io/kubernetes/pkg/api/v1" apiv1 "k8s.io/kubernetes/pkg/api/v1"
appsapi "k8s.io/kubernetes/pkg/apis/apps/v1beta1" appsapi "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
@ -39,7 +40,6 @@ import (
corev1client "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1" corev1client "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion" coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
"k8s.io/kubernetes/pkg/genericapiserver" "k8s.io/kubernetes/pkg/genericapiserver"
"k8s.io/kubernetes/pkg/genericapiserver/options"
"k8s.io/kubernetes/pkg/healthz" "k8s.io/kubernetes/pkg/healthz"
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
"k8s.io/kubernetes/pkg/master/thirdparty" "k8s.io/kubernetes/pkg/master/thirdparty"
@ -116,7 +116,6 @@ type Config struct {
// Port names should align with ports defined in ExtraServicePorts // Port names should align with ports defined in ExtraServicePorts
ExtraEndpointPorts []api.EndpointPort ExtraEndpointPorts []api.EndpointPort
// If non-zero, the "kubernetes" services uses this port as NodePort. // If non-zero, the "kubernetes" services uses this port as NodePort.
// TODO(sttts): move into master
KubernetesServiceNodePort int KubernetesServiceNodePort int
// Number of masters running; all masters must be started with the // Number of masters running; all masters must be started with the

View File

@ -46,7 +46,7 @@ func (a *APIServer) Start() error {
if err != nil { if err != nil {
return err return err
} }
config.GenericServerRunOptions.ServiceClusterIPRange = *ipnet config.ServiceClusterIPRange = *ipnet
config.AllowPrivileged = true config.AllowPrivileged = true
errCh := make(chan error) errCh := make(chan error)
go func() { go func() {

View File

@ -20,7 +20,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net"
"net/http" "net/http"
"testing" "testing"
"time" "time"
@ -48,8 +47,6 @@ func TestRun(t *testing.T) {
s := options.NewServerRunOptions() s := options.NewServerRunOptions()
s.SecureServing.ServingOptions.BindPort = securePort s.SecureServing.ServingOptions.BindPort = securePort
s.InsecureServing.BindPort = insecurePort s.InsecureServing.BindPort = insecurePort
_, ipNet, _ := net.ParseCIDR("10.10.10.0/24")
s.GenericServerRunOptions.ServiceClusterIPRange = *ipNet
s.Etcd.StorageConfig.ServerList = []string{"http://localhost:2379"} s.Etcd.StorageConfig.ServerList = []string{"http://localhost:2379"}
go func() { go func() {
if err := app.Run(s); err != nil { if err := app.Run(s); err != nil {