From 1b3779baa0d7143b8bdf10b5eb3076b68e4064a7 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Fri, 2 Jun 2023 12:36:49 +0200 Subject: [PATCH] MOVE: cmd/kube-apiserver/app/options: split apart controlplane part --- cmd/kube-apiserver/app/options/completion.go | 121 ++---- cmd/kube-apiserver/app/options/options.go | 149 ++------ .../app/options/options_test.go | 360 +++++++++--------- cmd/kube-apiserver/app/options/validation.go | 59 +-- .../app/options/validation_test.go | 166 +------- pkg/controlplane/apiserver/options/options.go | 292 ++++++++++++++ .../apiserver/options/options_test.go | 291 ++++++++++++++ .../apiserver/options/validation.go | 90 +++++ .../apiserver/options/validation_test.go | 154 ++++++++ pkg/controlplane/services.go | 54 --- 10 files changed, 1080 insertions(+), 656 deletions(-) create mode 100644 pkg/controlplane/apiserver/options/options.go create mode 100644 pkg/controlplane/apiserver/options/options_test.go create mode 100644 pkg/controlplane/apiserver/options/validation.go create mode 100644 pkg/controlplane/apiserver/options/validation_test.go delete mode 100644 pkg/controlplane/services.go diff --git a/cmd/kube-apiserver/app/options/completion.go b/cmd/kube-apiserver/app/options/completion.go index 5cb9d589e19..00327414566 100644 --- a/cmd/kube-apiserver/app/options/completion.go +++ b/cmd/kube-apiserver/app/options/completion.go @@ -19,141 +19,78 @@ package options import ( "fmt" "net" - "os" "strings" - "time" - serveroptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/client-go/util/keyutil" + apiserveroptions "k8s.io/apiserver/pkg/server/options" _ "k8s.io/component-base/metrics/prometheus/workqueue" - "k8s.io/klog/v2" netutils "k8s.io/utils/net" - "k8s.io/kubernetes/pkg/controlplane" + controlplane "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/kubeapiserver" - kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" - "k8s.io/kubernetes/pkg/serviceaccount" + kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" ) // completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked. type completedOptions struct { - *ServerRunOptions + controlplane.CompletedOptions + CloudProvider *kubeoptions.CloudProviderOptions + + Extra } type CompletedOptions struct { - completedOptions + // Embed a private pointer that cannot be instantiated outside of this package. + *completedOptions } // Complete set default ServerRunOptions. // Should be called after kube-apiserver flags parsed. func Complete(opts *ServerRunOptions) (CompletedOptions, error) { - // set defaults - if err := opts.GenericServerRunOptions.DefaultAdvertiseAddress(opts.SecureServing.SecureServingOptions); err != nil { - return CompletedOptions{}, err + if opts == nil { + return CompletedOptions{completedOptions: &completedOptions{}}, nil } - // process s.ServiceClusterIPRange from list to Primary and Secondary + // process opts.ServiceClusterIPRange from list to Primary and Secondary // we process secondary only if provided by user apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(opts.ServiceClusterIPRanges) if err != nil { return CompletedOptions{}, err } - opts.PrimaryServiceClusterIPRange = primaryServiceIPRange - opts.SecondaryServiceClusterIPRange = secondaryServiceIPRange - opts.APIServerServiceIP = apiServerServiceIP - - if err := opts.SecureServing.MaybeDefaultWithSelfSignedCerts(opts.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil { - return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err) + controlplane, err := opts.Options.Complete([]string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}) + if err != nil { + return CompletedOptions{}, err } - if len(opts.GenericServerRunOptions.ExternalHost) == 0 { - if len(opts.GenericServerRunOptions.AdvertiseAddress) > 0 { - opts.GenericServerRunOptions.ExternalHost = opts.GenericServerRunOptions.AdvertiseAddress.String() - } else { - hostname, err := os.Hostname() - if err != nil { - return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err) - } - opts.GenericServerRunOptions.ExternalHost = hostname - } - klog.Infof("external host was not specified, using %v", opts.GenericServerRunOptions.ExternalHost) + completed := completedOptions{ + CompletedOptions: controlplane, + CloudProvider: opts.CloudProvider, + + Extra: opts.Extra, } - opts.Authentication.ApplyAuthorization(opts.Authorization) + completed.PrimaryServiceClusterIPRange = primaryServiceIPRange + completed.SecondaryServiceClusterIPRange = secondaryServiceIPRange + completed.APIServerServiceIP = apiServerServiceIP - // Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling - // TokenRequest functionality. This defaulting was convenient, but messed up - // a lot of people when they rotated their serving cert with no idea it was - // connected to their service account keys. We are taking this opportunity to - // remove this problematic defaulting. - if opts.ServiceAccountSigningKeyFile == "" { - // Default to the private server key for service account token signing - if len(opts.Authentication.ServiceAccounts.KeyFiles) == 0 && opts.SecureServing.ServerCert.CertKey.KeyFile != "" { - if kubeauthenticator.IsValidServiceAccountKeyFile(opts.SecureServing.ServerCert.CertKey.KeyFile) { - opts.Authentication.ServiceAccounts.KeyFiles = []string{opts.SecureServing.ServerCert.CertKey.KeyFile} - } else { - klog.Warning("No TLS key provided, service account token authentication disabled") - } - } - } - - if opts.ServiceAccountSigningKeyFile != "" && len(opts.Authentication.ServiceAccounts.Issuers) != 0 && opts.Authentication.ServiceAccounts.Issuers[0] != "" { - sk, err := keyutil.PrivateKeyFromFile(opts.ServiceAccountSigningKeyFile) - if err != nil { - return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) - } - if opts.Authentication.ServiceAccounts.MaxExpiration != 0 { - lowBound := time.Hour - upBound := time.Duration(1<<32) * time.Second - if opts.Authentication.ServiceAccounts.MaxExpiration < lowBound || - opts.Authentication.ServiceAccounts.MaxExpiration > upBound { - return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds") - } - if opts.Authentication.ServiceAccounts.ExtendExpiration { - if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second { - klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, opts.Authentication.ServiceAccounts.MaxExpiration) - } - if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second { - klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, opts.Authentication.ServiceAccounts.MaxExpiration) - } - } - } - - opts.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(opts.Authentication.ServiceAccounts.Issuers[0], sk) - if err != nil { - return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err) - } - opts.ServiceAccountTokenMaxExpiration = opts.Authentication.ServiceAccounts.MaxExpiration - } - - if opts.Etcd.EnableWatchCache { + if completed.Etcd != nil && completed.Etcd.EnableWatchCache { sizes := kubeapiserver.DefaultWatchCacheSizes() // Ensure that overrides parse correctly. - userSpecified, err := serveroptions.ParseWatchCacheSizes(opts.Etcd.WatchCacheSizes) + userSpecified, err := apiserveroptions.ParseWatchCacheSizes(completed.Etcd.WatchCacheSizes) if err != nil { return CompletedOptions{}, err } for resource, size := range userSpecified { sizes[resource] = size } - opts.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes) + completed.Etcd.WatchCacheSizes, err = apiserveroptions.WriteWatchCacheSizes(sizes) if err != nil { return CompletedOptions{}, err } } - for key, value := range opts.APIEnablement.RuntimeConfig { - if key == "v1" || strings.HasPrefix(key, "v1/") || - key == "api/v1" || strings.HasPrefix(key, "api/v1/") { - delete(opts.APIEnablement.RuntimeConfig, key) - opts.APIEnablement.RuntimeConfig["/v1"] = value - } - if key == "api/legacy" { - delete(opts.APIEnablement.RuntimeConfig, key) - } - } - - return CompletedOptions{completedOptions: completedOptions{ServerRunOptions: opts}}, nil + return CompletedOptions{ + completedOptions: &completed, + }, nil } func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) { diff --git a/cmd/kube-apiserver/app/options/options.go b/cmd/kube-apiserver/app/options/options.go index ae0cbdff003..df87d0aef77 100644 --- a/cmd/kube-apiserver/app/options/options.go +++ b/cmd/kube-apiserver/app/options/options.go @@ -23,45 +23,29 @@ import ( "time" utilnet "k8s.io/apimachinery/pkg/util/net" - genericoptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/storage/storagebackend" cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/logs" - "k8s.io/component-base/metrics" - logsapi "k8s.io/component-base/logs/api/v1" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/cluster/ports" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/reconcilers" _ "k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" - "k8s.io/kubernetes/pkg/serviceaccount" ) // ServerRunOptions runs a kubernetes api server. type ServerRunOptions struct { - GenericServerRunOptions *genericoptions.ServerRunOptions - Etcd *genericoptions.EtcdOptions - SecureServing *genericoptions.SecureServingOptionsWithLoopback - Audit *genericoptions.AuditOptions - Features *genericoptions.FeatureOptions - Admission *kubeoptions.AdmissionOptions - Authentication *kubeoptions.BuiltInAuthenticationOptions - Authorization *kubeoptions.BuiltInAuthorizationOptions - CloudProvider *kubeoptions.CloudProviderOptions - APIEnablement *genericoptions.APIEnablementOptions - EgressSelector *genericoptions.EgressSelectorOptions - Metrics *metrics.Options - Logs *logs.Options - Traces *genericoptions.TracingOptions + *controlplaneapiserver.Options // embedded to avoid noise in existing consumers + CloudProvider *kubeoptions.CloudProviderOptions + Extra +} + +type Extra struct { AllowPrivileged bool - EnableLogsHandler bool - EventTTL time.Duration KubeletConfig kubeletclient.KubeletClientConfig KubernetesServiceNodePort int - MaxConnectionBytesPerSec int64 // ServiceClusterIPRange is mapped to input provided by user ServiceClusterIPRanges string // PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results @@ -73,110 +57,53 @@ type ServerRunOptions struct { ServiceNodePortRange utilnet.PortRange - ProxyClientCertFile string - ProxyClientKeyFile string - - EnableAggregatorRouting bool - AggregatorRejectForwardingRedirects bool - - MasterCount int EndpointReconcilerType string - - ServiceAccountSigningKeyFile string - ServiceAccountIssuer serviceaccount.TokenGenerator - ServiceAccountTokenMaxExpiration time.Duration - - ShowHiddenMetricsForVersion string } // NewServerRunOptions creates a new ServerRunOptions object with default parameters func NewServerRunOptions() *ServerRunOptions { s := ServerRunOptions{ - GenericServerRunOptions: genericoptions.NewServerRunOptions(), - Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), - SecureServing: kubeoptions.NewSecureServingOptions(), - Audit: genericoptions.NewAuditOptions(), - Features: genericoptions.NewFeatureOptions(), - Admission: kubeoptions.NewAdmissionOptions(), - Authentication: kubeoptions.NewBuiltInAuthenticationOptions().WithAll(), - Authorization: kubeoptions.NewBuiltInAuthorizationOptions(), - CloudProvider: kubeoptions.NewCloudProviderOptions(), - APIEnablement: genericoptions.NewAPIEnablementOptions(), - EgressSelector: genericoptions.NewEgressSelectorOptions(), - Metrics: metrics.NewOptions(), - Logs: logs.NewOptions(), - Traces: genericoptions.NewTracingOptions(), + Options: controlplaneapiserver.NewOptions(), + CloudProvider: kubeoptions.NewCloudProviderOptions(), - EnableLogsHandler: true, - EventTTL: 1 * time.Hour, - MasterCount: 1, - EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), - KubeletConfig: kubeletclient.KubeletClientConfig{ - Port: ports.KubeletPort, - ReadOnlyPort: ports.KubeletReadOnlyPort, - PreferredAddressTypes: []string{ - // --override-hostname - string(api.NodeHostName), + Extra: Extra{ + EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), + KubeletConfig: kubeletclient.KubeletClientConfig{ + Port: ports.KubeletPort, + ReadOnlyPort: ports.KubeletReadOnlyPort, + PreferredAddressTypes: []string{ + // --override-hostname + string(api.NodeHostName), - // internal, preferring DNS if reported - string(api.NodeInternalDNS), - string(api.NodeInternalIP), + // internal, preferring DNS if reported + string(api.NodeInternalDNS), + string(api.NodeInternalIP), - // external, preferring DNS if reported - string(api.NodeExternalDNS), - string(api.NodeExternalIP), + // external, preferring DNS if reported + string(api.NodeExternalDNS), + string(api.NodeExternalIP), + }, + HTTPTimeout: time.Duration(5) * time.Second, }, - HTTPTimeout: time.Duration(5) * time.Second, + ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange, }, - ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange, - AggregatorRejectForwardingRedirects: true, } - // Overwrite the default for storage data format. - s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf" - return &s } // Flags returns flags for a specific APIServer by section name func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { - // Add the generic flags. - s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic")) - s.Etcd.AddFlags(fss.FlagSet("etcd")) - s.SecureServing.AddFlags(fss.FlagSet("secure serving")) - s.Audit.AddFlags(fss.FlagSet("auditing")) - s.Features.AddFlags(fss.FlagSet("features")) - s.Authentication.AddFlags(fss.FlagSet("authentication")) - s.Authorization.AddFlags(fss.FlagSet("authorization")) + s.Options.AddFlags(&fss) s.CloudProvider.AddFlags(fss.FlagSet("cloud provider")) - s.APIEnablement.AddFlags(fss.FlagSet("API enablement")) - s.EgressSelector.AddFlags(fss.FlagSet("egress selector")) - s.Admission.AddFlags(fss.FlagSet("admission")) - s.Metrics.AddFlags(fss.FlagSet("metrics")) - logsapi.AddFlags(s.Logs, fss.FlagSet("logs")) - s.Traces.AddFlags(fss.FlagSet("traces")) // Note: the weird ""+ in below lines seems to be the only way to get gofmt to // arrange these text blocks sensibly. Grrr. fs := fss.FlagSet("misc") - fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL, - "Amount of time to retain events.") fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged, "If true, allow privileged containers. [default=false]") - fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler, - "If true, install a /logs handler for the apiserver logs.") - fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19") - - fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+ - "If non-zero, throttle each user connection to this number of bytes/sec. "+ - "Currently only applies to long-running requests.") - - fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount, - "The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)") - fs.MarkDeprecated("apiserver-count", "apiserver-count is deprecated and will be removed in a future version.") - fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", s.EndpointReconcilerType, "Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+") master-count is deprecated, and will be removed in a future version.") @@ -219,27 +146,5 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { fs.StringVar(&s.KubeletConfig.TLSClientConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.TLSClientConfig.CAFile, "Path to a cert file for the certificate authority.") - fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+ - "Client certificate used to prove the identity of the aggregator or kube-apiserver "+ - "when it must call out during a request. This includes proxying requests to a user "+ - "api-server and calling out to webhook admission plugins. It is expected that this "+ - "cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+ - "That CA is published in the 'extension-apiserver-authentication' configmap in "+ - "the kube-system namespace. Components receiving calls from kube-aggregator should "+ - "use that CA to perform their half of the mutual TLS verification.") - fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+ - "Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+ - "when it must call out during a request. This includes proxying requests to a user "+ - "api-server and calling out to webhook admission plugins.") - - fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting, - "Turns on aggregator routing requests to endpoints IP rather than cluster IP.") - - fs.BoolVar(&s.AggregatorRejectForwardingRedirects, "aggregator-reject-forwarding-redirect", s.AggregatorRejectForwardingRedirects, - "Aggregator reject forwarding redirect response back to client.") - - fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+ - "Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.") - return fss } diff --git a/cmd/kube-apiserver/app/options/options_test.go b/cmd/kube-apiserver/app/options/options_test.go index 7ac44afb1db..2a27cc508e0 100644 --- a/cmd/kube-apiserver/app/options/options_test.go +++ b/cmd/kube-apiserver/app/options/options_test.go @@ -37,6 +37,7 @@ import ( "k8s.io/component-base/logs" "k8s.io/component-base/metrics" kapi "k8s.io/kubernetes/pkg/apis/core" + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options" "k8s.io/kubernetes/pkg/controlplane/reconcilers" kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" kubeletclient "k8s.io/kubernetes/pkg/kubelet/client" @@ -125,200 +126,205 @@ func TestAddFlags(t *testing.T) { // This is a snapshot of expected options parsed by args. expected := &ServerRunOptions{ - ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange, - ServiceClusterIPRanges: (&net.IPNet{IP: netutils.ParseIPSloppy("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(), - MasterCount: 5, - EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), - AllowPrivileged: false, - GenericServerRunOptions: &apiserveroptions.ServerRunOptions{ - AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"), - CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"}, - MaxRequestsInFlight: 400, - MaxMutatingRequestsInFlight: 200, - RequestTimeout: time.Duration(2) * time.Minute, - MinRequestTimeout: 1800, - JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), - MaxRequestBodyBytes: int64(3 * 1024 * 1024), - }, - Admission: &kubeoptions.AdmissionOptions{ - GenericAdmission: &apiserveroptions.AdmissionOptions{ - RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder, - DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins, - EnablePlugins: []string{"AlwaysDeny"}, - ConfigFile: "/admission-control-config", - Plugins: s.Admission.GenericAdmission.Plugins, - Decorators: s.Admission.GenericAdmission.Decorators, + Options: &controlplaneapiserver.Options{ + MasterCount: 5, + GenericServerRunOptions: &apiserveroptions.ServerRunOptions{ + AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"), + CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"}, + MaxRequestsInFlight: 400, + MaxMutatingRequestsInFlight: 200, + RequestTimeout: time.Duration(2) * time.Minute, + MinRequestTimeout: 1800, + JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), + MaxRequestBodyBytes: int64(3 * 1024 * 1024), }, - }, - Etcd: &apiserveroptions.EtcdOptions{ - StorageConfig: storagebackend.Config{ - Type: "etcd3", - Transport: storagebackend.TransportConfig{ - ServerList: nil, - KeyFile: "/var/run/kubernetes/etcd.key", - TrustedCAFile: "/var/run/kubernetes/etcdca.crt", - CertFile: "/var/run/kubernetes/etcdce.crt", - TracerProvider: oteltrace.NewNoopTracerProvider(), - }, - Paging: true, - Prefix: "/registry", - CompactionInterval: storagebackend.DefaultCompactInterval, - CountMetricPollPeriod: time.Minute, - DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval, - HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout, - ReadycheckTimeout: storagebackend.DefaultReadinessTimeout, - LeaseManagerConfig: etcd3.LeaseManagerConfig{ - ReuseDurationSeconds: 100, - MaxObjectCount: 1000, + Admission: &kubeoptions.AdmissionOptions{ + GenericAdmission: &apiserveroptions.AdmissionOptions{ + RecommendedPluginOrder: s.Options.Admission.GenericAdmission.RecommendedPluginOrder, + DefaultOffPlugins: s.Options.Admission.GenericAdmission.DefaultOffPlugins, + EnablePlugins: []string{"AlwaysDeny"}, + ConfigFile: "/admission-control-config", + Plugins: s.Options.Admission.GenericAdmission.Plugins, + Decorators: s.Options.Admission.GenericAdmission.Decorators, }, }, - DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", - DeleteCollectionWorkers: 1, - EnableGarbageCollection: true, - EnableWatchCache: true, - DefaultWatchCacheSize: 100, - }, - SecureServing: (&apiserveroptions.SecureServingOptions{ - BindAddress: netutils.ParseIPSloppy("192.168.10.20"), - BindPort: 6443, - ServerCert: apiserveroptions.GeneratableKeyCert{ - CertDirectory: "/var/run/kubernetes", - PairName: "apiserver", - }, - HTTP2MaxStreamsPerConnection: 42, - Required: true, - }).WithLoopback(), - EventTTL: 1 * time.Hour, - KubeletConfig: kubeletclient.KubeletClientConfig{ - Port: 10250, - ReadOnlyPort: 10255, - PreferredAddressTypes: []string{ - string(kapi.NodeHostName), - string(kapi.NodeInternalDNS), - string(kapi.NodeInternalIP), - string(kapi.NodeExternalDNS), - string(kapi.NodeExternalIP), - }, - HTTPTimeout: time.Duration(5) * time.Second, - TLSClientConfig: kubeletclient.KubeletTLSConfig{ - CertFile: "/var/run/kubernetes/ceserver.crt", - KeyFile: "/var/run/kubernetes/server.key", - CAFile: "/var/run/kubernetes/caserver.crt", - }, - }, - Audit: &apiserveroptions.AuditOptions{ - LogOptions: apiserveroptions.AuditLogOptions{ - Path: "/var/log", - MaxAge: 11, - MaxBackups: 12, - MaxSize: 13, - Format: "json", - BatchOptions: apiserveroptions.AuditBatchOptions{ - Mode: "blocking", - BatchConfig: auditbuffered.BatchConfig{ - BufferSize: 46, - MaxBatchSize: 47, - MaxBatchWait: 48 * time.Second, - ThrottleEnable: true, - ThrottleQPS: 49.5, - ThrottleBurst: 50, + Etcd: &apiserveroptions.EtcdOptions{ + StorageConfig: storagebackend.Config{ + Type: "etcd3", + Transport: storagebackend.TransportConfig{ + ServerList: nil, + KeyFile: "/var/run/kubernetes/etcd.key", + TrustedCAFile: "/var/run/kubernetes/etcdca.crt", + CertFile: "/var/run/kubernetes/etcdce.crt", + TracerProvider: oteltrace.NewNoopTracerProvider(), + }, + Paging: true, + Prefix: "/registry", + CompactionInterval: storagebackend.DefaultCompactInterval, + CountMetricPollPeriod: time.Minute, + DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval, + HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout, + ReadycheckTimeout: storagebackend.DefaultReadinessTimeout, + LeaseManagerConfig: etcd3.LeaseManagerConfig{ + ReuseDurationSeconds: 100, + MaxObjectCount: 1000, }, }, - TruncateOptions: apiserveroptions.AuditTruncateOptions{ - Enabled: true, - TruncateConfig: audittruncate.Config{ - MaxBatchSize: 45, - MaxEventSize: 44, - }, + DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", + DeleteCollectionWorkers: 1, + EnableGarbageCollection: true, + EnableWatchCache: true, + DefaultWatchCacheSize: 100, + }, + SecureServing: (&apiserveroptions.SecureServingOptions{ + BindAddress: netutils.ParseIPSloppy("192.168.10.20"), + BindPort: 6443, + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/var/run/kubernetes", + PairName: "apiserver", }, - GroupVersionString: "audit.k8s.io/v1", - }, - WebhookOptions: apiserveroptions.AuditWebhookOptions{ - ConfigFile: "/webhook-config", - BatchOptions: apiserveroptions.AuditBatchOptions{ - Mode: "blocking", - BatchConfig: auditbuffered.BatchConfig{ - BufferSize: 42, - MaxBatchSize: 43, - MaxBatchWait: 1 * time.Second, - ThrottleEnable: false, - ThrottleQPS: 43.5, - ThrottleBurst: 44, - AsyncDelegate: true, + HTTP2MaxStreamsPerConnection: 42, + Required: true, + }).WithLoopback(), + EventTTL: 1 * time.Hour, + Audit: &apiserveroptions.AuditOptions{ + LogOptions: apiserveroptions.AuditLogOptions{ + Path: "/var/log", + MaxAge: 11, + MaxBackups: 12, + MaxSize: 13, + Format: "json", + BatchOptions: apiserveroptions.AuditBatchOptions{ + Mode: "blocking", + BatchConfig: auditbuffered.BatchConfig{ + BufferSize: 46, + MaxBatchSize: 47, + MaxBatchWait: 48 * time.Second, + ThrottleEnable: true, + ThrottleQPS: 49.5, + ThrottleBurst: 50, + }, }, - }, - TruncateOptions: apiserveroptions.AuditTruncateOptions{ - Enabled: true, - TruncateConfig: audittruncate.Config{ - MaxBatchSize: 43, - MaxEventSize: 42, + TruncateOptions: apiserveroptions.AuditTruncateOptions{ + Enabled: true, + TruncateConfig: audittruncate.Config{ + MaxBatchSize: 45, + MaxEventSize: 44, + }, }, + GroupVersionString: "audit.k8s.io/v1", }, - InitialBackoff: 2 * time.Second, - GroupVersionString: "audit.k8s.io/v1", + WebhookOptions: apiserveroptions.AuditWebhookOptions{ + ConfigFile: "/webhook-config", + BatchOptions: apiserveroptions.AuditBatchOptions{ + Mode: "blocking", + BatchConfig: auditbuffered.BatchConfig{ + BufferSize: 42, + MaxBatchSize: 43, + MaxBatchWait: 1 * time.Second, + ThrottleEnable: false, + ThrottleQPS: 43.5, + ThrottleBurst: 44, + AsyncDelegate: true, + }, + }, + TruncateOptions: apiserveroptions.AuditTruncateOptions{ + Enabled: true, + TruncateConfig: audittruncate.Config{ + MaxBatchSize: 43, + MaxEventSize: 42, + }, + }, + InitialBackoff: 2 * time.Second, + GroupVersionString: "audit.k8s.io/v1", + }, + PolicyFile: "/policy", }, - PolicyFile: "/policy", + Features: &apiserveroptions.FeatureOptions{ + EnableProfiling: true, + EnableContentionProfiling: true, + }, + Authentication: &kubeoptions.BuiltInAuthenticationOptions{ + Anonymous: &kubeoptions.AnonymousAuthenticationOptions{ + Allow: false, + }, + ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{ + ClientCA: "/client-ca", + }, + WebHook: &kubeoptions.WebHookAuthenticationOptions{ + CacheTTL: 180000000000, + ConfigFile: "/token-webhook-config", + Version: "v1beta1", + RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), + }, + BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{}, + OIDC: &kubeoptions.OIDCAuthenticationOptions{ + UsernameClaim: "sub", + SigningAlgs: []string{"RS256"}, + }, + RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{}, + ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{ + Lookup: true, + ExtendExpiration: true, + }, + TokenFile: &kubeoptions.TokenFileAuthenticationOptions{}, + TokenSuccessCacheTTL: 10 * time.Second, + TokenFailureCacheTTL: 0, + }, + Authorization: &kubeoptions.BuiltInAuthorizationOptions{ + Modes: []string{"AlwaysDeny", "RBAC"}, + PolicyFile: "/policy", + WebhookConfigFile: "/webhook-config", + WebhookCacheAuthorizedTTL: 180000000000, + WebhookCacheUnauthorizedTTL: 60000000000, + WebhookVersion: "v1beta1", + WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), + }, + APIEnablement: &apiserveroptions.APIEnablementOptions{ + RuntimeConfig: cliflag.ConfigurationMap{}, + }, + EgressSelector: &apiserveroptions.EgressSelectorOptions{ + ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml", + }, + EnableLogsHandler: false, + EnableAggregatorRouting: true, + ProxyClientKeyFile: "/var/run/kubernetes/proxy.key", + ProxyClientCertFile: "/var/run/kubernetes/proxy.crt", + Metrics: &metrics.Options{}, + Logs: logs.NewOptions(), + Traces: &apiserveroptions.TracingOptions{ + ConfigFile: "/var/run/kubernetes/tracing_config.yaml", + }, + AggregatorRejectForwardingRedirects: true, }, - Features: &apiserveroptions.FeatureOptions{ - EnableProfiling: true, - EnableContentionProfiling: true, - }, - Authentication: &kubeoptions.BuiltInAuthenticationOptions{ - Anonymous: &kubeoptions.AnonymousAuthenticationOptions{ - Allow: false, + + Extra: Extra{ + ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange, + ServiceClusterIPRanges: (&net.IPNet{IP: netutils.ParseIPSloppy("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(), + EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), + AllowPrivileged: false, + KubeletConfig: kubeletclient.KubeletClientConfig{ + Port: 10250, + ReadOnlyPort: 10255, + PreferredAddressTypes: []string{ + string(kapi.NodeHostName), + string(kapi.NodeInternalDNS), + string(kapi.NodeInternalIP), + string(kapi.NodeExternalDNS), + string(kapi.NodeExternalIP), + }, + HTTPTimeout: time.Duration(5) * time.Second, + TLSClientConfig: kubeletclient.KubeletTLSConfig{ + CertFile: "/var/run/kubernetes/ceserver.crt", + KeyFile: "/var/run/kubernetes/server.key", + CAFile: "/var/run/kubernetes/caserver.crt", + }, }, - ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{ - ClientCA: "/client-ca", - }, - WebHook: &kubeoptions.WebHookAuthenticationOptions{ - CacheTTL: 180000000000, - ConfigFile: "/token-webhook-config", - Version: "v1beta1", - RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), - }, - BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{}, - OIDC: &kubeoptions.OIDCAuthenticationOptions{ - UsernameClaim: "sub", - SigningAlgs: []string{"RS256"}, - }, - RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{}, - ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{ - Lookup: true, - ExtendExpiration: true, - }, - TokenFile: &kubeoptions.TokenFileAuthenticationOptions{}, - TokenSuccessCacheTTL: 10 * time.Second, - TokenFailureCacheTTL: 0, - }, - Authorization: &kubeoptions.BuiltInAuthorizationOptions{ - Modes: []string{"AlwaysDeny", "RBAC"}, - PolicyFile: "/policy", - WebhookConfigFile: "/webhook-config", - WebhookCacheAuthorizedTTL: 180000000000, - WebhookCacheUnauthorizedTTL: 60000000000, - WebhookVersion: "v1beta1", - WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), }, CloudProvider: &kubeoptions.CloudProviderOptions{ CloudConfigFile: "/cloud-config", CloudProvider: "azure", }, - APIEnablement: &apiserveroptions.APIEnablementOptions{ - RuntimeConfig: cliflag.ConfigurationMap{}, - }, - EgressSelector: &apiserveroptions.EgressSelectorOptions{ - ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml", - }, - EnableLogsHandler: false, - EnableAggregatorRouting: true, - ProxyClientKeyFile: "/var/run/kubernetes/proxy.key", - ProxyClientCertFile: "/var/run/kubernetes/proxy.crt", - Metrics: &metrics.Options{}, - Logs: logs.NewOptions(), - Traces: &apiserveroptions.TracingOptions{ - ConfigFile: "/var/run/kubernetes/tracing_config.yaml", - }, - AggregatorRejectForwardingRedirects: true, } if !reflect.DeepEqual(expected, s) { diff --git a/cmd/kube-apiserver/app/options/validation.go b/cmd/kube-apiserver/app/options/validation.go index b766befa8ea..221efe7d245 100644 --- a/cmd/kube-apiserver/app/options/validation.go +++ b/cmd/kube-apiserver/app/options/validation.go @@ -22,11 +22,7 @@ import ( "net" "strings" - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - genericfeatures "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/features" netutils "k8s.io/utils/net" ) @@ -106,64 +102,13 @@ func validateServiceNodePort(options *ServerRunOptions) []error { return errs } -func validateTokenRequest(options *ServerRunOptions) []error { - var errs []error - - enableAttempted := options.ServiceAccountSigningKeyFile != "" || - (len(options.Authentication.ServiceAccounts.Issuers) != 0 && options.Authentication.ServiceAccounts.Issuers[0] != "") || - len(options.Authentication.APIAudiences) != 0 - - enableSucceeded := options.ServiceAccountIssuer != nil - - if !enableAttempted { - errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags")) - } - - if enableAttempted && !enableSucceeded { - errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together")) - } - - return errs -} - -func validateAPIPriorityAndFairness(options *ServerRunOptions) []error { - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && options.GenericServerRunOptions.EnablePriorityAndFairness { - // If none of the following runtime config options are specified, - // APF is assumed to be turned on. The internal APF controller uses - // v1beta3 so it should be enabled. - enabledAPIString := options.APIEnablement.RuntimeConfig.String() - testConfigs := []string{"flowcontrol.apiserver.k8s.io/v1beta3", "api/beta", "api/all"} // in the order of precedence - for _, testConfig := range testConfigs { - if strings.Contains(enabledAPIString, fmt.Sprintf("%s=false", testConfig)) { - return []error{fmt.Errorf("--runtime-config=%s=false conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true", testConfig)} - } - if strings.Contains(enabledAPIString, fmt.Sprintf("%s=true", testConfig)) { - return nil - } - } - } - - return nil -} - // Validate checks ServerRunOptions and return a slice of found errs. func (s *ServerRunOptions) Validate() []error { var errs []error - if s.MasterCount <= 0 { - errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) - } - errs = append(errs, s.Etcd.Validate()...) + + errs = append(errs, s.Options.Validate()...) errs = append(errs, validateClusterIPFlags(s)...) errs = append(errs, validateServiceNodePort(s)...) - errs = append(errs, validateAPIPriorityAndFairness(s)...) - errs = append(errs, s.SecureServing.Validate()...) - errs = append(errs, s.Authentication.Validate()...) - errs = append(errs, s.Authorization.Validate()...) - errs = append(errs, s.Audit.Validate()...) - errs = append(errs, s.Admission.Validate()...) - errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...) - errs = append(errs, validateTokenRequest(s)...) - errs = append(errs, s.Metrics.Validate()...) return errs } diff --git a/cmd/kube-apiserver/app/options/validation_test.go b/cmd/kube-apiserver/app/options/validation_test.go index 6cb8aa646d6..3b2281888ca 100644 --- a/cmd/kube-apiserver/app/options/validation_test.go +++ b/cmd/kube-apiserver/app/options/validation_test.go @@ -18,17 +18,12 @@ package options import ( "net" - "strings" "testing" utilnet "k8s.io/apimachinery/pkg/util/net" - kubeapiserveradmission "k8s.io/apiserver/pkg/admission" - genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" - basemetrics "k8s.io/component-base/metrics" "k8s.io/kubernetes/pkg/features" - kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" netutils "k8s.io/utils/net" ) @@ -53,9 +48,11 @@ func makeOptionsWithCIDRs(serviceCIDR string, secondaryServiceCIDR string) *Serv } } return &ServerRunOptions{ - ServiceClusterIPRanges: value, - PrimaryServiceClusterIPRange: primaryCIDR, - SecondaryServiceClusterIPRange: secondaryCIDR, + Extra: Extra{ + ServiceClusterIPRanges: value, + PrimaryServiceClusterIPRange: primaryCIDR, + SecondaryServiceClusterIPRange: secondaryCIDR, + }, } } @@ -203,9 +200,9 @@ func TestValidateServiceNodePort(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - err := validateServiceNodePort(tc.options) - if err != nil && !tc.expectErrors { - t.Errorf("expected no errors, error found %+v", err) + errs := validateServiceNodePort(tc.options) + if errs != nil && !tc.expectErrors { + t.Errorf("expected no errors, error found %+v", errs) } }) } @@ -217,8 +214,10 @@ func makeOptionsWithPort(kubernetesServiceNodePort int, base int, size int) *Ser Size: size, } return &ServerRunOptions{ - ServiceNodePortRange: portRange, - KubernetesServiceNodePort: kubernetesServiceNodePort, + Extra: Extra{ + ServiceNodePortRange: portRange, + KubernetesServiceNodePort: kubernetesServiceNodePort, + }, } } @@ -283,144 +282,3 @@ func TestValidateMaxCIDRRange(t *testing.T) { }) } } - -func TestValidateAPIPriorityAndFairness(t *testing.T) { - const conflict = "conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true" - tests := []struct { - runtimeConfig string - errShouldContain string - }{ - { - runtimeConfig: "api/all=false", - errShouldContain: conflict, - }, - { - runtimeConfig: "api/beta=false", - errShouldContain: conflict, - }, - { - runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta1=false", - errShouldContain: "", - }, - { - runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta2=false", - errShouldContain: "", - }, - { - runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta3=false", - errShouldContain: conflict, - }, - { - runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta3=true", - errShouldContain: "", - }, - } - - for _, test := range tests { - t.Run(test.runtimeConfig, func(t *testing.T) { - options := &ServerRunOptions{ - GenericServerRunOptions: &genericoptions.ServerRunOptions{ - EnablePriorityAndFairness: true, - }, - APIEnablement: genericoptions.NewAPIEnablementOptions(), - } - options.APIEnablement.RuntimeConfig.Set(test.runtimeConfig) - - var errMessageGot string - if errs := validateAPIPriorityAndFairness(options); len(errs) > 0 { - errMessageGot = errs[0].Error() - } - if !strings.Contains(errMessageGot, test.errShouldContain) { - t.Errorf("Expected error message to contain: %q, but got: %q", test.errShouldContain, errMessageGot) - } - }) - } -} - -func TestValidateServerRunOptions(t *testing.T) { - cidrOpts := makeOptionsWithCIDRs("10.0.0.0/16", "3000::/64") - nodePortOpts := makeOptionsWithPort(-1, 30065, 1) - - testCases := []struct { - name string - options *ServerRunOptions - expectErrors bool - }{ - { - name: "validate master count equal 0", - expectErrors: true, - options: &ServerRunOptions{ - MasterCount: 0, - GenericServerRunOptions: &genericoptions.ServerRunOptions{}, - Etcd: &genericoptions.EtcdOptions{}, - SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, - Audit: &genericoptions.AuditOptions{}, - Admission: &kubeoptions.AdmissionOptions{ - GenericAdmission: &genericoptions.AdmissionOptions{ - EnablePlugins: []string{"foo"}, - Plugins: kubeapiserveradmission.NewPlugins(), - }, - PluginNames: []string{"foo"}, - }, - Authentication: &kubeoptions.BuiltInAuthenticationOptions{ - APIAudiences: []string{"bar"}, - ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{ - Issuers: []string{"baz"}, - }, - }, - Authorization: &kubeoptions.BuiltInAuthorizationOptions{}, - APIEnablement: genericoptions.NewAPIEnablementOptions(), - Metrics: &basemetrics.Options{}, - ServiceClusterIPRanges: cidrOpts.ServiceClusterIPRanges, - PrimaryServiceClusterIPRange: cidrOpts.PrimaryServiceClusterIPRange, - SecondaryServiceClusterIPRange: cidrOpts.SecondaryServiceClusterIPRange, - ServiceNodePortRange: nodePortOpts.ServiceNodePortRange, - KubernetesServiceNodePort: nodePortOpts.KubernetesServiceNodePort, - ServiceAccountSigningKeyFile: "", - }, - }, - { - name: "validate token request enable not attempted", - expectErrors: true, - options: &ServerRunOptions{ - MasterCount: 1, - GenericServerRunOptions: &genericoptions.ServerRunOptions{}, - Etcd: &genericoptions.EtcdOptions{}, - SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, - Audit: &genericoptions.AuditOptions{}, - Admission: &kubeoptions.AdmissionOptions{ - GenericAdmission: &genericoptions.AdmissionOptions{ - EnablePlugins: []string{""}, - Plugins: kubeapiserveradmission.NewPlugins(), - }, - PluginNames: []string{""}, - }, - Authentication: &kubeoptions.BuiltInAuthenticationOptions{ - ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{}, - }, - Authorization: &kubeoptions.BuiltInAuthorizationOptions{}, - APIEnablement: genericoptions.NewAPIEnablementOptions(), - Metrics: &basemetrics.Options{}, - ServiceClusterIPRanges: cidrOpts.ServiceClusterIPRanges, - PrimaryServiceClusterIPRange: cidrOpts.PrimaryServiceClusterIPRange, - SecondaryServiceClusterIPRange: cidrOpts.SecondaryServiceClusterIPRange, - ServiceNodePortRange: nodePortOpts.ServiceNodePortRange, - KubernetesServiceNodePort: nodePortOpts.KubernetesServiceNodePort, - ServiceAccountSigningKeyFile: "", - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - errs := tc.options.Validate() - if len(errs) > 0 && !tc.expectErrors { - t.Errorf("expected no errors, errors found %+v", errs) - } - - if len(errs) == 0 && tc.expectErrors { - t.Errorf("expected errors, no errors found") - } - }) - } -} diff --git a/pkg/controlplane/apiserver/options/options.go b/pkg/controlplane/apiserver/options/options.go new file mode 100644 index 00000000000..f5b65b9279e --- /dev/null +++ b/pkg/controlplane/apiserver/options/options.go @@ -0,0 +1,292 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package options contains flags and options for initializing an apiserver +package options + +import ( + "fmt" + "net" + "os" + "strings" + "time" + + genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/client-go/util/keyutil" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/logs" + logsapi "k8s.io/component-base/logs/api/v1" + "k8s.io/component-base/metrics" + "k8s.io/klog/v2" + "k8s.io/utils/integer" + netutil "k8s.io/utils/net" + + _ "k8s.io/kubernetes/pkg/features" + kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" + kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" + "k8s.io/kubernetes/pkg/serviceaccount" +) + +type Options struct { + GenericServerRunOptions *genericoptions.ServerRunOptions + Etcd *genericoptions.EtcdOptions + SecureServing *genericoptions.SecureServingOptionsWithLoopback + Audit *genericoptions.AuditOptions + Features *genericoptions.FeatureOptions + Admission *kubeoptions.AdmissionOptions + Authentication *kubeoptions.BuiltInAuthenticationOptions + Authorization *kubeoptions.BuiltInAuthorizationOptions + APIEnablement *genericoptions.APIEnablementOptions + EgressSelector *genericoptions.EgressSelectorOptions + Metrics *metrics.Options + Logs *logs.Options + Traces *genericoptions.TracingOptions + + EnableLogsHandler bool + EventTTL time.Duration + MaxConnectionBytesPerSec int64 + + ProxyClientCertFile string + ProxyClientKeyFile string + + EnableAggregatorRouting bool + AggregatorRejectForwardingRedirects bool + + MasterCount int + + ServiceAccountSigningKeyFile string + ServiceAccountIssuer serviceaccount.TokenGenerator + ServiceAccountTokenMaxExpiration time.Duration + + ShowHiddenMetricsForVersion string +} + +// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked. +type completedOptions struct { + Options +} + +type CompletedOptions struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedOptions +} + +// NewOptions creates a new ServerRunOptions object with default parameters +func NewOptions() *Options { + s := Options{ + GenericServerRunOptions: genericoptions.NewServerRunOptions(), + Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), + SecureServing: kubeoptions.NewSecureServingOptions(), + Audit: genericoptions.NewAuditOptions(), + Features: genericoptions.NewFeatureOptions(), + Admission: kubeoptions.NewAdmissionOptions(), + Authentication: kubeoptions.NewBuiltInAuthenticationOptions().WithAll(), + Authorization: kubeoptions.NewBuiltInAuthorizationOptions(), + APIEnablement: genericoptions.NewAPIEnablementOptions(), + EgressSelector: genericoptions.NewEgressSelectorOptions(), + Metrics: metrics.NewOptions(), + Logs: logs.NewOptions(), + Traces: genericoptions.NewTracingOptions(), + + EnableLogsHandler: true, + EventTTL: 1 * time.Hour, + MasterCount: 1, + AggregatorRejectForwardingRedirects: true, + } + + // Overwrite the default for storage data format. + s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf" + + return &s +} + +func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) { + // Add the generic flags. + s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic")) + s.Etcd.AddFlags(fss.FlagSet("etcd")) + s.SecureServing.AddFlags(fss.FlagSet("secure serving")) + s.Audit.AddFlags(fss.FlagSet("auditing")) + s.Features.AddFlags(fss.FlagSet("features")) + s.Authentication.AddFlags(fss.FlagSet("authentication")) + s.Authorization.AddFlags(fss.FlagSet("authorization")) + s.APIEnablement.AddFlags(fss.FlagSet("API enablement")) + s.EgressSelector.AddFlags(fss.FlagSet("egress selector")) + s.Admission.AddFlags(fss.FlagSet("admission")) + s.Metrics.AddFlags(fss.FlagSet("metrics")) + logsapi.AddFlags(s.Logs, fss.FlagSet("logs")) + s.Traces.AddFlags(fss.FlagSet("traces")) + + // Note: the weird ""+ in below lines seems to be the only way to get gofmt to + // arrange these text blocks sensibly. Grrr. + fs := fss.FlagSet("misc") + fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL, + "Amount of time to retain events.") + + fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler, + "If true, install a /logs handler for the apiserver logs.") + fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19") + + fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+ + "If non-zero, throttle each user connection to this number of bytes/sec. "+ + "Currently only applies to long-running requests.") + + fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount, + "The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)") + fs.MarkDeprecated("apiserver-count", "apiserver-count is deprecated and will be removed in a future version.") + + fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+ + "Client certificate used to prove the identity of the aggregator or kube-apiserver "+ + "when it must call out during a request. This includes proxying requests to a user "+ + "api-server and calling out to webhook admission plugins. It is expected that this "+ + "cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+ + "That CA is published in the 'extension-apiserver-authentication' configmap in "+ + "the kube-system namespace. Components receiving calls from kube-aggregator should "+ + "use that CA to perform their half of the mutual TLS verification.") + fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+ + "Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+ + "when it must call out during a request. This includes proxying requests to a user "+ + "api-server and calling out to webhook admission plugins.") + + fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting, + "Turns on aggregator routing requests to endpoints IP rather than cluster IP.") + + fs.BoolVar(&s.AggregatorRejectForwardingRedirects, "aggregator-reject-forwarding-redirect", s.AggregatorRejectForwardingRedirects, + "Aggregator reject forwarding redirect response back to client.") + + fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+ + "Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.") +} + +func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) { + if o == nil { + return CompletedOptions{completedOptions: &completedOptions{}}, nil + } + + completed := completedOptions{ + Options: *o, + } + + // set defaults + if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil { + return CompletedOptions{}, err + } + + if err := completed.SecureServing.MaybeDefaultWithSelfSignedCerts(completed.GenericServerRunOptions.AdvertiseAddress.String(), alternateDNS, alternateIPs); err != nil { + return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err) + } + + if len(completed.GenericServerRunOptions.ExternalHost) == 0 { + if len(completed.GenericServerRunOptions.AdvertiseAddress) > 0 { + completed.GenericServerRunOptions.ExternalHost = completed.GenericServerRunOptions.AdvertiseAddress.String() + } else { + hostname, err := os.Hostname() + if err != nil { + return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err) + } + completed.GenericServerRunOptions.ExternalHost = hostname + } + klog.Infof("external host was not specified, using %v", completed.GenericServerRunOptions.ExternalHost) + } + + completed.Authentication.ApplyAuthorization(completed.Authorization) + + // Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling + // TokenRequest functionality. This defaulting was convenient, but messed up + // a lot of people when they rotated their serving cert with no idea it was + // connected to their service account keys. We are taking this opportunity to + // remove this problematic defaulting. + if completed.ServiceAccountSigningKeyFile == "" { + // Default to the private server key for service account token signing + if len(completed.Authentication.ServiceAccounts.KeyFiles) == 0 && completed.SecureServing.ServerCert.CertKey.KeyFile != "" { + if kubeauthenticator.IsValidServiceAccountKeyFile(completed.SecureServing.ServerCert.CertKey.KeyFile) { + completed.Authentication.ServiceAccounts.KeyFiles = []string{completed.SecureServing.ServerCert.CertKey.KeyFile} + } else { + klog.Warning("No TLS key provided, service account token authentication disabled") + } + } + } + + if completed.ServiceAccountSigningKeyFile != "" && len(completed.Authentication.ServiceAccounts.Issuers) != 0 && completed.Authentication.ServiceAccounts.Issuers[0] != "" { + sk, err := keyutil.PrivateKeyFromFile(completed.ServiceAccountSigningKeyFile) + if err != nil { + return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) + } + if completed.Authentication.ServiceAccounts.MaxExpiration != 0 { + lowBound := time.Hour + upBound := time.Duration(1<<32) * time.Second + if completed.Authentication.ServiceAccounts.MaxExpiration < lowBound || + completed.Authentication.ServiceAccounts.MaxExpiration > upBound { + return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds") + } + if completed.Authentication.ServiceAccounts.ExtendExpiration { + if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second { + klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, completed.Authentication.ServiceAccounts.MaxExpiration) + } + if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second { + klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, completed.Authentication.ServiceAccounts.MaxExpiration) + } + } + } + + completed.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(completed.Authentication.ServiceAccounts.Issuers[0], sk) + if err != nil { + return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err) + } + completed.ServiceAccountTokenMaxExpiration = completed.Authentication.ServiceAccounts.MaxExpiration + } + + for key, value := range completed.APIEnablement.RuntimeConfig { + if key == "v1" || strings.HasPrefix(key, "v1/") || + key == "api/v1" || strings.HasPrefix(key, "api/v1/") { + delete(completed.APIEnablement.RuntimeConfig, key) + completed.APIEnablement.RuntimeConfig["/v1"] = value + } + if key == "api/legacy" { + delete(completed.APIEnablement.RuntimeConfig, key) + } + } + + return CompletedOptions{ + completedOptions: &completed, + }, nil +} + +// ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and +// setting service ip range to the default value in kubeoptions.DefaultServiceIPCIDR +// for now until the default is removed per the deprecation timeline guidelines. +// Returns service ip range, api server service IP, and an error +func ServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) { + serviceClusterIPRange := passedServiceClusterIPRange + if passedServiceClusterIPRange.IP == nil { + klog.Warningf("No CIDR for service cluster IPs specified. Default value which was %s is deprecated and will be removed in future releases. Please specify it using --service-cluster-ip-range on kube-apiserver.", kubeoptions.DefaultServiceIPCIDR.String()) + serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR + } + + size := integer.Int64Min(netutil.RangeSize(&serviceClusterIPRange), 1<<16) + if size < 8 { + return net.IPNet{}, net.IP{}, fmt.Errorf("the service cluster IP range must be at least %d IP addresses", 8) + } + + // Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP. + apiServerServiceIP, err := netutil.GetIndexedIP(&serviceClusterIPRange, 1) + if err != nil { + return net.IPNet{}, net.IP{}, err + } + klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP) + + return serviceClusterIPRange, apiServerServiceIP, nil +} diff --git a/pkg/controlplane/apiserver/options/options_test.go b/pkg/controlplane/apiserver/options/options_test.go new file mode 100644 index 00000000000..b7754f65688 --- /dev/null +++ b/pkg/controlplane/apiserver/options/options_test.go @@ -0,0 +1,291 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "reflect" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/spf13/pflag" + oteltrace "go.opentelemetry.io/otel/trace" + + "k8s.io/apiserver/pkg/admission" + apiserveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/storage/etcd3" + "k8s.io/apiserver/pkg/storage/storagebackend" + auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered" + audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/logs" + "k8s.io/component-base/metrics" + netutils "k8s.io/utils/net" + + kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" +) + +func TestAddFlags(t *testing.T) { + fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError) + s := NewOptions() + var fss cliflag.NamedFlagSets + s.AddFlags(&fss) + for _, f := range fss.FlagSets { + fs.AddFlagSet(f) + } + + args := []string{ + "--enable-admission-plugins=AlwaysDeny", + "--admission-control-config-file=/admission-control-config", + "--advertise-address=192.168.10.10", + "--anonymous-auth=false", + "--apiserver-count=5", + "--audit-log-maxage=11", + "--audit-log-maxbackup=12", + "--audit-log-maxsize=13", + "--audit-log-path=/var/log", + "--audit-log-mode=blocking", + "--audit-log-batch-buffer-size=46", + "--audit-log-batch-max-size=47", + "--audit-log-batch-max-wait=48s", + "--audit-log-batch-throttle-enable=true", + "--audit-log-batch-throttle-qps=49.5", + "--audit-log-batch-throttle-burst=50", + "--audit-log-truncate-enabled=true", + "--audit-log-truncate-max-batch-size=45", + "--audit-log-truncate-max-event-size=44", + "--audit-log-version=audit.k8s.io/v1", + "--audit-policy-file=/policy", + "--audit-webhook-config-file=/webhook-config", + "--audit-webhook-mode=blocking", + "--audit-webhook-batch-buffer-size=42", + "--audit-webhook-batch-max-size=43", + "--audit-webhook-batch-max-wait=1s", + "--audit-webhook-batch-throttle-enable=false", + "--audit-webhook-batch-throttle-qps=43.5", + "--audit-webhook-batch-throttle-burst=44", + "--audit-webhook-truncate-enabled=true", + "--audit-webhook-truncate-max-batch-size=43", + "--audit-webhook-truncate-max-event-size=42", + "--audit-webhook-initial-backoff=2s", + "--audit-webhook-version=audit.k8s.io/v1", + "--authentication-token-webhook-cache-ttl=3m", + "--authentication-token-webhook-config-file=/token-webhook-config", + "--authorization-mode=AlwaysDeny,RBAC", + "--authorization-policy-file=/policy", + "--authorization-webhook-cache-authorized-ttl=3m", + "--authorization-webhook-cache-unauthorized-ttl=1m", + "--authorization-webhook-config-file=/webhook-config", + "--bind-address=192.168.10.20", + "--client-ca-file=/client-ca", + "--cors-allowed-origins=10.10.10.100,10.10.10.200", + "--contention-profiling=true", + "--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml", + "--enable-aggregator-routing=true", + "--enable-priority-and-fairness=false", + "--enable-logs-handler=false", + "--etcd-keyfile=/var/run/kubernetes/etcd.key", + "--etcd-certfile=/var/run/kubernetes/etcdce.crt", + "--etcd-cafile=/var/run/kubernetes/etcdca.crt", + "--http2-max-streams-per-connection=42", + "--tracing-config-file=/var/run/kubernetes/tracing_config.yaml", + "--proxy-client-cert-file=/var/run/kubernetes/proxy.crt", + "--proxy-client-key-file=/var/run/kubernetes/proxy.key", + "--request-timeout=2m", + "--storage-backend=etcd3", + "--lease-reuse-duration-seconds=100", + } + fs.Parse(args) + + // This is a snapshot of expected options parsed by args. + expected := &Options{ + MasterCount: 5, + GenericServerRunOptions: &apiserveroptions.ServerRunOptions{ + AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"), + CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"}, + MaxRequestsInFlight: 400, + MaxMutatingRequestsInFlight: 200, + RequestTimeout: time.Duration(2) * time.Minute, + MinRequestTimeout: 1800, + JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024), + MaxRequestBodyBytes: int64(3 * 1024 * 1024), + }, + Admission: &kubeoptions.AdmissionOptions{ + GenericAdmission: &apiserveroptions.AdmissionOptions{ + RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder, + DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins, + EnablePlugins: []string{"AlwaysDeny"}, + ConfigFile: "/admission-control-config", + Plugins: s.Admission.GenericAdmission.Plugins, + Decorators: s.Admission.GenericAdmission.Decorators, + }, + }, + Etcd: &apiserveroptions.EtcdOptions{ + StorageConfig: storagebackend.Config{ + Type: "etcd3", + Transport: storagebackend.TransportConfig{ + ServerList: nil, + KeyFile: "/var/run/kubernetes/etcd.key", + TrustedCAFile: "/var/run/kubernetes/etcdca.crt", + CertFile: "/var/run/kubernetes/etcdce.crt", + TracerProvider: oteltrace.NewNoopTracerProvider(), + }, + Paging: true, + Prefix: "/registry", + CompactionInterval: storagebackend.DefaultCompactInterval, + CountMetricPollPeriod: time.Minute, + DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval, + HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout, + ReadycheckTimeout: storagebackend.DefaultReadinessTimeout, + LeaseManagerConfig: etcd3.LeaseManagerConfig{ + ReuseDurationSeconds: 100, + MaxObjectCount: 1000, + }, + }, + DefaultStorageMediaType: "application/vnd.kubernetes.protobuf", + DeleteCollectionWorkers: 1, + EnableGarbageCollection: true, + EnableWatchCache: true, + DefaultWatchCacheSize: 100, + }, + SecureServing: (&apiserveroptions.SecureServingOptions{ + BindAddress: netutils.ParseIPSloppy("192.168.10.20"), + BindPort: 6443, + ServerCert: apiserveroptions.GeneratableKeyCert{ + CertDirectory: "/var/run/kubernetes", + PairName: "apiserver", + }, + HTTP2MaxStreamsPerConnection: 42, + Required: true, + }).WithLoopback(), + EventTTL: 1 * time.Hour, + Audit: &apiserveroptions.AuditOptions{ + LogOptions: apiserveroptions.AuditLogOptions{ + Path: "/var/log", + MaxAge: 11, + MaxBackups: 12, + MaxSize: 13, + Format: "json", + BatchOptions: apiserveroptions.AuditBatchOptions{ + Mode: "blocking", + BatchConfig: auditbuffered.BatchConfig{ + BufferSize: 46, + MaxBatchSize: 47, + MaxBatchWait: 48 * time.Second, + ThrottleEnable: true, + ThrottleQPS: 49.5, + ThrottleBurst: 50, + }, + }, + TruncateOptions: apiserveroptions.AuditTruncateOptions{ + Enabled: true, + TruncateConfig: audittruncate.Config{ + MaxBatchSize: 45, + MaxEventSize: 44, + }, + }, + GroupVersionString: "audit.k8s.io/v1", + }, + WebhookOptions: apiserveroptions.AuditWebhookOptions{ + ConfigFile: "/webhook-config", + BatchOptions: apiserveroptions.AuditBatchOptions{ + Mode: "blocking", + BatchConfig: auditbuffered.BatchConfig{ + BufferSize: 42, + MaxBatchSize: 43, + MaxBatchWait: 1 * time.Second, + ThrottleEnable: false, + ThrottleQPS: 43.5, + ThrottleBurst: 44, + AsyncDelegate: true, + }, + }, + TruncateOptions: apiserveroptions.AuditTruncateOptions{ + Enabled: true, + TruncateConfig: audittruncate.Config{ + MaxBatchSize: 43, + MaxEventSize: 42, + }, + }, + InitialBackoff: 2 * time.Second, + GroupVersionString: "audit.k8s.io/v1", + }, + PolicyFile: "/policy", + }, + Features: &apiserveroptions.FeatureOptions{ + EnableProfiling: true, + EnableContentionProfiling: true, + }, + Authentication: &kubeoptions.BuiltInAuthenticationOptions{ + Anonymous: &kubeoptions.AnonymousAuthenticationOptions{ + Allow: false, + }, + ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{ + ClientCA: "/client-ca", + }, + WebHook: &kubeoptions.WebHookAuthenticationOptions{ + CacheTTL: 180000000000, + ConfigFile: "/token-webhook-config", + Version: "v1beta1", + RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), + }, + BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{}, + OIDC: &kubeoptions.OIDCAuthenticationOptions{ + UsernameClaim: "sub", + SigningAlgs: []string{"RS256"}, + }, + RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{}, + ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{ + Lookup: true, + ExtendExpiration: true, + }, + TokenFile: &kubeoptions.TokenFileAuthenticationOptions{}, + TokenSuccessCacheTTL: 10 * time.Second, + TokenFailureCacheTTL: 0, + }, + Authorization: &kubeoptions.BuiltInAuthorizationOptions{ + Modes: []string{"AlwaysDeny", "RBAC"}, + PolicyFile: "/policy", + WebhookConfigFile: "/webhook-config", + WebhookCacheAuthorizedTTL: 180000000000, + WebhookCacheUnauthorizedTTL: 60000000000, + WebhookVersion: "v1beta1", + WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(), + }, + APIEnablement: &apiserveroptions.APIEnablementOptions{ + RuntimeConfig: cliflag.ConfigurationMap{}, + }, + EgressSelector: &apiserveroptions.EgressSelectorOptions{ + ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml", + }, + EnableLogsHandler: false, + EnableAggregatorRouting: true, + ProxyClientKeyFile: "/var/run/kubernetes/proxy.key", + ProxyClientCertFile: "/var/run/kubernetes/proxy.crt", + Metrics: &metrics.Options{}, + Logs: logs.NewOptions(), + Traces: &apiserveroptions.TracingOptions{ + ConfigFile: "/var/run/kubernetes/tracing_config.yaml", + }, + AggregatorRejectForwardingRedirects: true, + } + + if !reflect.DeepEqual(expected, s) { + t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{}))) + } +} diff --git a/pkg/controlplane/apiserver/options/validation.go b/pkg/controlplane/apiserver/options/validation.go new file mode 100644 index 00000000000..66f7543b0e2 --- /dev/null +++ b/pkg/controlplane/apiserver/options/validation.go @@ -0,0 +1,90 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "errors" + "fmt" + "strings" + + apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" + genericfeatures "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" + aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" + + "k8s.io/kubernetes/pkg/api/legacyscheme" +) + +func validateTokenRequest(options *Options) []error { + var errs []error + + enableAttempted := options.ServiceAccountSigningKeyFile != "" || + (len(options.Authentication.ServiceAccounts.Issuers) != 0 && options.Authentication.ServiceAccounts.Issuers[0] != "") || + len(options.Authentication.APIAudiences) != 0 + + enableSucceeded := options.ServiceAccountIssuer != nil + + if !enableAttempted { + errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags")) + } + + if enableAttempted && !enableSucceeded { + errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together")) + } + + return errs +} + +func validateAPIPriorityAndFairness(options *Options) []error { + if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && options.GenericServerRunOptions.EnablePriorityAndFairness { + // If none of the following runtime config options are specified, + // APF is assumed to be turned on. The internal APF controller uses + // v1beta3 so it should be enabled. + enabledAPIString := options.APIEnablement.RuntimeConfig.String() + testConfigs := []string{"flowcontrol.apiserver.k8s.io/v1beta3", "api/beta", "api/all"} // in the order of precedence + for _, testConfig := range testConfigs { + if strings.Contains(enabledAPIString, fmt.Sprintf("%s=false", testConfig)) { + return []error{fmt.Errorf("--runtime-config=%s=false conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true", testConfig)} + } + if strings.Contains(enabledAPIString, fmt.Sprintf("%s=true", testConfig)) { + return nil + } + } + } + + return nil +} + +// Validate checks Options and return a slice of found errs. +func (s *Options) Validate() []error { + var errs []error + if s.MasterCount <= 0 { + errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount)) + } + errs = append(errs, s.Etcd.Validate()...) + errs = append(errs, validateAPIPriorityAndFairness(s)...) + errs = append(errs, s.SecureServing.Validate()...) + errs = append(errs, s.Authentication.Validate()...) + errs = append(errs, s.Authorization.Validate()...) + errs = append(errs, s.Audit.Validate()...) + errs = append(errs, s.Admission.Validate()...) + errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...) + errs = append(errs, validateTokenRequest(s)...) + errs = append(errs, s.Metrics.Validate()...) + + return errs +} diff --git a/pkg/controlplane/apiserver/options/validation_test.go b/pkg/controlplane/apiserver/options/validation_test.go new file mode 100644 index 00000000000..d1df3423f93 --- /dev/null +++ b/pkg/controlplane/apiserver/options/validation_test.go @@ -0,0 +1,154 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "strings" + "testing" + + kubeapiserveradmission "k8s.io/apiserver/pkg/admission" + genericoptions "k8s.io/apiserver/pkg/server/options" + basemetrics "k8s.io/component-base/metrics" + + kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" +) + +func TestValidateAPIPriorityAndFairness(t *testing.T) { + const conflict = "conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true" + tests := []struct { + runtimeConfig string + errShouldContain string + }{ + { + runtimeConfig: "api/all=false", + errShouldContain: conflict, + }, + { + runtimeConfig: "api/beta=false", + errShouldContain: conflict, + }, + { + runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta1=false", + errShouldContain: "", + }, + { + runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta2=false", + errShouldContain: "", + }, + { + runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta3=false", + errShouldContain: conflict, + }, + { + runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta3=true", + errShouldContain: "", + }, + } + + for _, test := range tests { + t.Run(test.runtimeConfig, func(t *testing.T) { + options := &Options{ + GenericServerRunOptions: &genericoptions.ServerRunOptions{ + EnablePriorityAndFairness: true, + }, + APIEnablement: genericoptions.NewAPIEnablementOptions(), + } + options.APIEnablement.RuntimeConfig.Set(test.runtimeConfig) + + var errMessageGot string + if errs := validateAPIPriorityAndFairness(options); len(errs) > 0 { + errMessageGot = errs[0].Error() + } + if !strings.Contains(errMessageGot, test.errShouldContain) { + t.Errorf("Expected error message to contain: %q, but got: %q", test.errShouldContain, errMessageGot) + } + }) + } +} + +func TestValidateOptions(t *testing.T) { + testCases := []struct { + name string + options *Options + expectErrors bool + }{ + { + name: "validate master count equal 0", + expectErrors: true, + options: &Options{ + MasterCount: 0, + GenericServerRunOptions: &genericoptions.ServerRunOptions{}, + Etcd: &genericoptions.EtcdOptions{}, + SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, + Audit: &genericoptions.AuditOptions{}, + Admission: &kubeoptions.AdmissionOptions{ + GenericAdmission: &genericoptions.AdmissionOptions{ + EnablePlugins: []string{"foo"}, + Plugins: kubeapiserveradmission.NewPlugins(), + }, + PluginNames: []string{"foo"}, + }, + Authentication: &kubeoptions.BuiltInAuthenticationOptions{ + APIAudiences: []string{"bar"}, + ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{ + Issuers: []string{"baz"}, + }, + }, + APIEnablement: genericoptions.NewAPIEnablementOptions(), + Metrics: &basemetrics.Options{}, + ServiceAccountSigningKeyFile: "", + }, + }, + { + name: "validate token request enable not attempted", + expectErrors: true, + options: &Options{ + MasterCount: 1, + GenericServerRunOptions: &genericoptions.ServerRunOptions{}, + Etcd: &genericoptions.EtcdOptions{}, + SecureServing: &genericoptions.SecureServingOptionsWithLoopback{}, + Audit: &genericoptions.AuditOptions{}, + Admission: &kubeoptions.AdmissionOptions{ + GenericAdmission: &genericoptions.AdmissionOptions{ + EnablePlugins: []string{""}, + Plugins: kubeapiserveradmission.NewPlugins(), + }, + PluginNames: []string{""}, + }, + Authentication: &kubeoptions.BuiltInAuthenticationOptions{ + ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{}, + }, + APIEnablement: genericoptions.NewAPIEnablementOptions(), + Metrics: &basemetrics.Options{}, + ServiceAccountSigningKeyFile: "", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + errs := tc.options.Validate() + if len(errs) > 0 && !tc.expectErrors { + t.Errorf("expected no errors, errors found %+v", errs) + } + + if len(errs) == 0 && tc.expectErrors { + t.Errorf("expected errors, no errors found") + } + }) + } +} diff --git a/pkg/controlplane/services.go b/pkg/controlplane/services.go deleted file mode 100644 index ca895764b4d..00000000000 --- a/pkg/controlplane/services.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controlplane - -import ( - "fmt" - "net" - - "k8s.io/klog/v2" - "k8s.io/utils/integer" - utilnet "k8s.io/utils/net" - - kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" -) - -// ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and -// setting service ip range to the default value in kubeoptions.DefaultServiceIPCIDR -// for now until the default is removed per the deprecation timeline guidelines. -// Returns service ip range, api server service IP, and an error -func ServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) { - serviceClusterIPRange := passedServiceClusterIPRange - if passedServiceClusterIPRange.IP == nil { - klog.Warningf("No CIDR for service cluster IPs specified. Default value which was %s is deprecated and will be removed in future releases. Please specify it using --service-cluster-ip-range on kube-apiserver.", kubeoptions.DefaultServiceIPCIDR.String()) - serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR - } - - size := integer.Int64Min(utilnet.RangeSize(&serviceClusterIPRange), 1<<16) - if size < 8 { - return net.IPNet{}, net.IP{}, fmt.Errorf("the service cluster IP range must be at least %d IP addresses", 8) - } - - // Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP. - apiServerServiceIP, err := utilnet.GetIndexedIP(&serviceClusterIPRange, 1) - if err != nil { - return net.IPNet{}, net.IP{}, err - } - klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP) - - return serviceClusterIPRange, apiServerServiceIP, nil -}