github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-07 11:13:48 +00:00
Code Issues Projects Releases Wiki Activity

MOVE: cmd/kube-apiserver/app/options: split apart controlplane part

Browse Source
This commit is contained in:
Dr. Stefan Schimanski 2023-06-02 12:36:49 +02:00
parent 9f5a3f5e90
commit 1b3779baa0
No known key found for this signature in database
GPG Key ID: 4C68E0F19F95EC33
10 changed files with 1080 additions and 656 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
cmd/kube-apiserver/app/options/completion.go
View File

@ -19,141 +19,78 @@ package options
import (
"fmt"
"net"
"os"
"strings"
"time"
serveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/client-go/util/keyutil"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
_ "k8s.io/component-base/metrics/prometheus/workqueue"
"k8s.io/klog/v2"
netutils "k8s.io/utils/net"
"k8s.io/kubernetes/pkg/controlplane"
controlplane "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/kubeapiserver"
kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
"k8s.io/kubernetes/pkg/serviceaccount"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
)
// completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
type completedOptions struct {
*ServerRunOptions
controlplane.CompletedOptions
CloudProvider *kubeoptions.CloudProviderOptions
Extra
}
type CompletedOptions struct {
completedOptions
// Embed a private pointer that cannot be instantiated outside of this package.
*completedOptions
}
// Complete set default ServerRunOptions.
// Should be called after kube-apiserver flags parsed.
func Complete(opts *ServerRunOptions) (CompletedOptions, error) {
// set defaults
if err := opts.GenericServerRunOptions.DefaultAdvertiseAddress(opts.SecureServing.SecureServingOptions); err != nil {
return CompletedOptions{}, err
if opts == nil {
return CompletedOptions{completedOptions: &completedOptions{}}, nil
}
// process s.ServiceClusterIPRange from list to Primary and Secondary
// process opts.ServiceClusterIPRange from list to Primary and Secondary
// we process secondary only if provided by user
apiServerServiceIP, primaryServiceIPRange, secondaryServiceIPRange, err := getServiceIPAndRanges(opts.ServiceClusterIPRanges)
if err != nil {
return CompletedOptions{}, err
}
opts.PrimaryServiceClusterIPRange = primaryServiceIPRange
opts.SecondaryServiceClusterIPRange = secondaryServiceIPRange
opts.APIServerServiceIP = apiServerServiceIP
if err := opts.SecureServing.MaybeDefaultWithSelfSignedCerts(opts.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil {
return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err)
controlplane, err := opts.Options.Complete([]string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
if err != nil {
return CompletedOptions{}, err
}
if len(opts.GenericServerRunOptions.ExternalHost) == 0 {
if len(opts.GenericServerRunOptions.AdvertiseAddress) > 0 {
opts.GenericServerRunOptions.ExternalHost = opts.GenericServerRunOptions.AdvertiseAddress.String()
} else {
hostname, err := os.Hostname()
if err != nil {
return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err)
}
opts.GenericServerRunOptions.ExternalHost = hostname
}
klog.Infof("external host was not specified, using %v", opts.GenericServerRunOptions.ExternalHost)
completed := completedOptions{
CompletedOptions: controlplane,
CloudProvider: opts.CloudProvider,
Extra: opts.Extra,
}
opts.Authentication.ApplyAuthorization(opts.Authorization)
completed.PrimaryServiceClusterIPRange = primaryServiceIPRange
completed.SecondaryServiceClusterIPRange = secondaryServiceIPRange
completed.APIServerServiceIP = apiServerServiceIP
// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
// TokenRequest functionality. This defaulting was convenient, but messed up
// a lot of people when they rotated their serving cert with no idea it was
// connected to their service account keys. We are taking this opportunity to
// remove this problematic defaulting.
if opts.ServiceAccountSigningKeyFile == "" {
// Default to the private server key for service account token signing
if len(opts.Authentication.ServiceAccounts.KeyFiles) == 0 && opts.SecureServing.ServerCert.CertKey.KeyFile != "" {
if kubeauthenticator.IsValidServiceAccountKeyFile(opts.SecureServing.ServerCert.CertKey.KeyFile) {
opts.Authentication.ServiceAccounts.KeyFiles = []string{opts.SecureServing.ServerCert.CertKey.KeyFile}
} else {
klog.Warning("No TLS key provided, service account token authentication disabled")
}
}
}
if opts.ServiceAccountSigningKeyFile != "" && len(opts.Authentication.ServiceAccounts.Issuers) != 0 && opts.Authentication.ServiceAccounts.Issuers[0] != "" {
sk, err := keyutil.PrivateKeyFromFile(opts.ServiceAccountSigningKeyFile)
if err != nil {
return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
}
if opts.Authentication.ServiceAccounts.MaxExpiration != 0 {
lowBound := time.Hour
upBound := time.Duration(1<<32) * time.Second
if opts.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
opts.Authentication.ServiceAccounts.MaxExpiration > upBound {
return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
}
if opts.Authentication.ServiceAccounts.ExtendExpiration {
if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, opts.Authentication.ServiceAccounts.MaxExpiration)
}
if opts.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, opts.Authentication.ServiceAccounts.MaxExpiration)
}
}
}
opts.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(opts.Authentication.ServiceAccounts.Issuers[0], sk)
if err != nil {
return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err)
}
opts.ServiceAccountTokenMaxExpiration = opts.Authentication.ServiceAccounts.MaxExpiration
}
if opts.Etcd.EnableWatchCache {
if completed.Etcd != nil && completed.Etcd.EnableWatchCache {
sizes := kubeapiserver.DefaultWatchCacheSizes()
// Ensure that overrides parse correctly.
userSpecified, err := serveroptions.ParseWatchCacheSizes(opts.Etcd.WatchCacheSizes)
userSpecified, err := apiserveroptions.ParseWatchCacheSizes(completed.Etcd.WatchCacheSizes)
if err != nil {
return CompletedOptions{}, err
}
for resource, size := range userSpecified {
sizes[resource] = size
}
opts.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes)
completed.Etcd.WatchCacheSizes, err = apiserveroptions.WriteWatchCacheSizes(sizes)
if err != nil {
return CompletedOptions{}, err
}
}
for key, value := range opts.APIEnablement.RuntimeConfig {
if key == "v1" || strings.HasPrefix(key, "v1/") ||
key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
delete(opts.APIEnablement.RuntimeConfig, key)
opts.APIEnablement.RuntimeConfig["/v1"] = value
}
if key == "api/legacy" {
delete(opts.APIEnablement.RuntimeConfig, key)
}
}
return CompletedOptions{completedOptions: completedOptions{ServerRunOptions: opts}}, nil
return CompletedOptions{
completedOptions: &completed,
}, nil
}
func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, net.IPNet, error) {

149
cmd/kube-apiserver/app/options/options.go
View File

@ -23,45 +23,29 @@ import (
"time"
utilnet "k8s.io/apimachinery/pkg/util/net"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
logsapi "k8s.io/component-base/logs/api/v1"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/cluster/ports"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
_ "k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
"k8s.io/kubernetes/pkg/serviceaccount"
)
// ServerRunOptions runs a kubernetes api server.
type ServerRunOptions struct {
GenericServerRunOptions *genericoptions.ServerRunOptions
Etcd *genericoptions.EtcdOptions
SecureServing *genericoptions.SecureServingOptionsWithLoopback
Audit *genericoptions.AuditOptions
Features *genericoptions.FeatureOptions
Admission *kubeoptions.AdmissionOptions
Authentication *kubeoptions.BuiltInAuthenticationOptions
Authorization *kubeoptions.BuiltInAuthorizationOptions
CloudProvider *kubeoptions.CloudProviderOptions
APIEnablement *genericoptions.APIEnablementOptions
EgressSelector *genericoptions.EgressSelectorOptions
Metrics *metrics.Options
Logs *logs.Options
Traces *genericoptions.TracingOptions
*controlplaneapiserver.Options // embedded to avoid noise in existing consumers
CloudProvider *kubeoptions.CloudProviderOptions
Extra
}
type Extra struct {
AllowPrivileged bool
EnableLogsHandler bool
EventTTL time.Duration
KubeletConfig kubeletclient.KubeletClientConfig
KubernetesServiceNodePort int
MaxConnectionBytesPerSec int64
// ServiceClusterIPRange is mapped to input provided by user
ServiceClusterIPRanges string
// PrimaryServiceClusterIPRange and SecondaryServiceClusterIPRange are the results
@ -73,110 +57,53 @@ type ServerRunOptions struct {
ServiceNodePortRange utilnet.PortRange
ProxyClientCertFile string
ProxyClientKeyFile string
EnableAggregatorRouting bool
AggregatorRejectForwardingRedirects bool
MasterCount int
EndpointReconcilerType string
ServiceAccountSigningKeyFile string
ServiceAccountIssuer serviceaccount.TokenGenerator
ServiceAccountTokenMaxExpiration time.Duration
ShowHiddenMetricsForVersion string
}
// NewServerRunOptions creates a new ServerRunOptions object with default parameters
func NewServerRunOptions() *ServerRunOptions {
s := ServerRunOptions{
GenericServerRunOptions: genericoptions.NewServerRunOptions(),
Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
SecureServing: kubeoptions.NewSecureServingOptions(),
Audit: genericoptions.NewAuditOptions(),
Features: genericoptions.NewFeatureOptions(),
Admission: kubeoptions.NewAdmissionOptions(),
Authentication: kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
Authorization: kubeoptions.NewBuiltInAuthorizationOptions(),
CloudProvider: kubeoptions.NewCloudProviderOptions(),
APIEnablement: genericoptions.NewAPIEnablementOptions(),
EgressSelector: genericoptions.NewEgressSelectorOptions(),
Metrics: metrics.NewOptions(),
Logs: logs.NewOptions(),
Traces: genericoptions.NewTracingOptions(),
Options: controlplaneapiserver.NewOptions(),
CloudProvider: kubeoptions.NewCloudProviderOptions(),
EnableLogsHandler: true,
EventTTL: 1 * time.Hour,
MasterCount: 1,
EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
KubeletConfig: kubeletclient.KubeletClientConfig{
Port: ports.KubeletPort,
ReadOnlyPort: ports.KubeletReadOnlyPort,
PreferredAddressTypes: []string{
// --override-hostname
string(api.NodeHostName),
Extra: Extra{
EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
KubeletConfig: kubeletclient.KubeletClientConfig{
Port: ports.KubeletPort,
ReadOnlyPort: ports.KubeletReadOnlyPort,
PreferredAddressTypes: []string{
// --override-hostname
string(api.NodeHostName),
// internal, preferring DNS if reported
string(api.NodeInternalDNS),
string(api.NodeInternalIP),
// internal, preferring DNS if reported
string(api.NodeInternalDNS),
string(api.NodeInternalIP),
// external, preferring DNS if reported
string(api.NodeExternalDNS),
string(api.NodeExternalIP),
// external, preferring DNS if reported
string(api.NodeExternalDNS),
string(api.NodeExternalIP),
},
HTTPTimeout: time.Duration(5) * time.Second,
},
HTTPTimeout: time.Duration(5) * time.Second,
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
},
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
AggregatorRejectForwardingRedirects: true,
}
// Overwrite the default for storage data format.
s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
return &s
}
// Flags returns flags for a specific APIServer by section name
func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
// Add the generic flags.
s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic"))
s.Etcd.AddFlags(fss.FlagSet("etcd"))
s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
s.Audit.AddFlags(fss.FlagSet("auditing"))
s.Features.AddFlags(fss.FlagSet("features"))
s.Authentication.AddFlags(fss.FlagSet("authentication"))
s.Authorization.AddFlags(fss.FlagSet("authorization"))
s.Options.AddFlags(&fss)
s.CloudProvider.AddFlags(fss.FlagSet("cloud provider"))
s.APIEnablement.AddFlags(fss.FlagSet("API enablement"))
s.EgressSelector.AddFlags(fss.FlagSet("egress selector"))
s.Admission.AddFlags(fss.FlagSet("admission"))
s.Metrics.AddFlags(fss.FlagSet("metrics"))
logsapi.AddFlags(s.Logs, fss.FlagSet("logs"))
s.Traces.AddFlags(fss.FlagSet("traces"))
// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
// arrange these text blocks sensibly. Grrr.
fs := fss.FlagSet("misc")
fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL,
"Amount of time to retain events.")
fs.BoolVar(&s.AllowPrivileged, "allow-privileged", s.AllowPrivileged,
"If true, allow privileged containers. [default=false]")
fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler,
"If true, install a /logs handler for the apiserver logs.")
fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19")
fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+
"If non-zero, throttle each user connection to this number of bytes/sec. "+
"Currently only applies to long-running requests.")
fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
"The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)")
fs.MarkDeprecated("apiserver-count", "apiserver-count is deprecated and will be removed in a future version.")
fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", s.EndpointReconcilerType,
"Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+") master-count is deprecated, and will be removed in a future version.")
@ -219,27 +146,5 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
fs.StringVar(&s.KubeletConfig.TLSClientConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.TLSClientConfig.CAFile,
"Path to a cert file for the certificate authority.")
fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
"when it must call out during a request. This includes proxying requests to a user "+
"api-server and calling out to webhook admission plugins. It is expected that this "+
"cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+
"That CA is published in the 'extension-apiserver-authentication' configmap in "+
"the kube-system namespace. Components receiving calls from kube-aggregator should "+
"use that CA to perform their half of the mutual TLS verification.")
fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+
"Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+
"when it must call out during a request. This includes proxying requests to a user "+
"api-server and calling out to webhook admission plugins.")
fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting,
"Turns on aggregator routing requests to endpoints IP rather than cluster IP.")
fs.BoolVar(&s.AggregatorRejectForwardingRedirects, "aggregator-reject-forwarding-redirect", s.AggregatorRejectForwardingRedirects,
"Aggregator reject forwarding redirect response back to client.")
fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
return fss
}

360
cmd/kube-apiserver/app/options/options_test.go
View File

@ -37,6 +37,7 @@ import (
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
kapi "k8s.io/kubernetes/pkg/apis/core"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
"k8s.io/kubernetes/pkg/controlplane/reconcilers"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
@ -125,200 +126,205 @@ func TestAddFlags(t *testing.T) {
// This is a snapshot of expected options parsed by args.
expected := &ServerRunOptions{
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
ServiceClusterIPRanges: (&net.IPNet{IP: netutils.ParseIPSloppy("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(),
MasterCount: 5,
EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
AllowPrivileged: false,
GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"),
CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"},
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins,
EnablePlugins: []string{"AlwaysDeny"},
ConfigFile: "/admission-control-config",
Plugins: s.Admission.GenericAdmission.Plugins,
Decorators: s.Admission.GenericAdmission.Decorators,
Options: &controlplaneapiserver.Options{
MasterCount: 5,
GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"),
CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"},
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
},
},
Etcd: &apiserveroptions.EtcdOptions{
StorageConfig: storagebackend.Config{
Type: "etcd3",
Transport: storagebackend.TransportConfig{
ServerList: nil,
KeyFile: "/var/run/kubernetes/etcd.key",
TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
CertFile: "/var/run/kubernetes/etcdce.crt",
TracerProvider: oteltrace.NewNoopTracerProvider(),
},
Paging: true,
Prefix: "/registry",
CompactionInterval: storagebackend.DefaultCompactInterval,
CountMetricPollPeriod: time.Minute,
DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval,
HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout,
ReadycheckTimeout: storagebackend.DefaultReadinessTimeout,
LeaseManagerConfig: etcd3.LeaseManagerConfig{
ReuseDurationSeconds: 100,
MaxObjectCount: 1000,
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
RecommendedPluginOrder: s.Options.Admission.GenericAdmission.RecommendedPluginOrder,
DefaultOffPlugins: s.Options.Admission.GenericAdmission.DefaultOffPlugins,
EnablePlugins: []string{"AlwaysDeny"},
ConfigFile: "/admission-control-config",
Plugins: s.Options.Admission.GenericAdmission.Plugins,
Decorators: s.Options.Admission.GenericAdmission.Decorators,
},
},
DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
DeleteCollectionWorkers: 1,
EnableGarbageCollection: true,
EnableWatchCache: true,
DefaultWatchCacheSize: 100,
},
SecureServing: (&apiserveroptions.SecureServingOptions{
BindAddress: netutils.ParseIPSloppy("192.168.10.20"),
BindPort: 6443,
ServerCert: apiserveroptions.GeneratableKeyCert{
CertDirectory: "/var/run/kubernetes",
PairName: "apiserver",
},
HTTP2MaxStreamsPerConnection: 42,
Required: true,
}).WithLoopback(),
EventTTL: 1 * time.Hour,
KubeletConfig: kubeletclient.KubeletClientConfig{
Port: 10250,
ReadOnlyPort: 10255,
PreferredAddressTypes: []string{
string(kapi.NodeHostName),
string(kapi.NodeInternalDNS),
string(kapi.NodeInternalIP),
string(kapi.NodeExternalDNS),
string(kapi.NodeExternalIP),
},
HTTPTimeout: time.Duration(5) * time.Second,
TLSClientConfig: kubeletclient.KubeletTLSConfig{
CertFile: "/var/run/kubernetes/ceserver.crt",
KeyFile: "/var/run/kubernetes/server.key",
CAFile: "/var/run/kubernetes/caserver.crt",
},
},
Audit: &apiserveroptions.AuditOptions{
LogOptions: apiserveroptions.AuditLogOptions{
Path: "/var/log",
MaxAge: 11,
MaxBackups: 12,
MaxSize: 13,
Format: "json",
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 46,
MaxBatchSize: 47,
MaxBatchWait: 48 * time.Second,
ThrottleEnable: true,
ThrottleQPS: 49.5,
ThrottleBurst: 50,
Etcd: &apiserveroptions.EtcdOptions{
StorageConfig: storagebackend.Config{
Type: "etcd3",
Transport: storagebackend.TransportConfig{
ServerList: nil,
KeyFile: "/var/run/kubernetes/etcd.key",
TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
CertFile: "/var/run/kubernetes/etcdce.crt",
TracerProvider: oteltrace.NewNoopTracerProvider(),
},
Paging: true,
Prefix: "/registry",
CompactionInterval: storagebackend.DefaultCompactInterval,
CountMetricPollPeriod: time.Minute,
DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval,
HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout,
ReadycheckTimeout: storagebackend.DefaultReadinessTimeout,
LeaseManagerConfig: etcd3.LeaseManagerConfig{
ReuseDurationSeconds: 100,
MaxObjectCount: 1000,
},
},
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 45,
MaxEventSize: 44,
},
DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
DeleteCollectionWorkers: 1,
EnableGarbageCollection: true,
EnableWatchCache: true,
DefaultWatchCacheSize: 100,
},
SecureServing: (&apiserveroptions.SecureServingOptions{
BindAddress: netutils.ParseIPSloppy("192.168.10.20"),
BindPort: 6443,
ServerCert: apiserveroptions.GeneratableKeyCert{
CertDirectory: "/var/run/kubernetes",
PairName: "apiserver",
},
GroupVersionString: "audit.k8s.io/v1",
},
WebhookOptions: apiserveroptions.AuditWebhookOptions{
ConfigFile: "/webhook-config",
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 42,
MaxBatchSize: 43,
MaxBatchWait: 1 * time.Second,
ThrottleEnable: false,
ThrottleQPS: 43.5,
ThrottleBurst: 44,
AsyncDelegate: true,
HTTP2MaxStreamsPerConnection: 42,
Required: true,
}).WithLoopback(),
EventTTL: 1 * time.Hour,
Audit: &apiserveroptions.AuditOptions{
LogOptions: apiserveroptions.AuditLogOptions{
Path: "/var/log",
MaxAge: 11,
MaxBackups: 12,
MaxSize: 13,
Format: "json",
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 46,
MaxBatchSize: 47,
MaxBatchWait: 48 * time.Second,
ThrottleEnable: true,
ThrottleQPS: 49.5,
ThrottleBurst: 50,
},
},
},
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 43,
MaxEventSize: 42,
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 45,
MaxEventSize: 44,
},
},
GroupVersionString: "audit.k8s.io/v1",
},
InitialBackoff: 2 * time.Second,
GroupVersionString: "audit.k8s.io/v1",
WebhookOptions: apiserveroptions.AuditWebhookOptions{
ConfigFile: "/webhook-config",
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 42,
MaxBatchSize: 43,
MaxBatchWait: 1 * time.Second,
ThrottleEnable: false,
ThrottleQPS: 43.5,
ThrottleBurst: 44,
AsyncDelegate: true,
},
},
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 43,
MaxEventSize: 42,
},
},
InitialBackoff: 2 * time.Second,
GroupVersionString: "audit.k8s.io/v1",
},
PolicyFile: "/policy",
},
PolicyFile: "/policy",
Features: &apiserveroptions.FeatureOptions{
EnableProfiling: true,
EnableContentionProfiling: true,
},
Authentication: &kubeoptions.BuiltInAuthenticationOptions{
Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
Allow: false,
},
ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
ClientCA: "/client-ca",
},
WebHook: &kubeoptions.WebHookAuthenticationOptions{
CacheTTL: 180000000000,
ConfigFile: "/token-webhook-config",
Version: "v1beta1",
RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
},
BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
OIDC: &kubeoptions.OIDCAuthenticationOptions{
UsernameClaim: "sub",
SigningAlgs: []string{"RS256"},
},
RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
Lookup: true,
ExtendExpiration: true,
},
TokenFile: &kubeoptions.TokenFileAuthenticationOptions{},
TokenSuccessCacheTTL: 10 * time.Second,
TokenFailureCacheTTL: 0,
},
Authorization: &kubeoptions.BuiltInAuthorizationOptions{
Modes: []string{"AlwaysDeny", "RBAC"},
PolicyFile: "/policy",
WebhookConfigFile: "/webhook-config",
WebhookCacheAuthorizedTTL: 180000000000,
WebhookCacheUnauthorizedTTL: 60000000000,
WebhookVersion: "v1beta1",
WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
},
APIEnablement: &apiserveroptions.APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{},
},
EgressSelector: &apiserveroptions.EgressSelectorOptions{
ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
},
EnableLogsHandler: false,
EnableAggregatorRouting: true,
ProxyClientKeyFile: "/var/run/kubernetes/proxy.key",
ProxyClientCertFile: "/var/run/kubernetes/proxy.crt",
Metrics: &metrics.Options{},
Logs: logs.NewOptions(),
Traces: &apiserveroptions.TracingOptions{
ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
},
AggregatorRejectForwardingRedirects: true,
},
Features: &apiserveroptions.FeatureOptions{
EnableProfiling: true,
EnableContentionProfiling: true,
},
Authentication: &kubeoptions.BuiltInAuthenticationOptions{
Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
Allow: false,
Extra: Extra{
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
ServiceClusterIPRanges: (&net.IPNet{IP: netutils.ParseIPSloppy("192.168.128.0"), Mask: net.CIDRMask(17, 32)}).String(),
EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
AllowPrivileged: false,
KubeletConfig: kubeletclient.KubeletClientConfig{
Port: 10250,
ReadOnlyPort: 10255,
PreferredAddressTypes: []string{
string(kapi.NodeHostName),
string(kapi.NodeInternalDNS),
string(kapi.NodeInternalIP),
string(kapi.NodeExternalDNS),
string(kapi.NodeExternalIP),
},
HTTPTimeout: time.Duration(5) * time.Second,
TLSClientConfig: kubeletclient.KubeletTLSConfig{
CertFile: "/var/run/kubernetes/ceserver.crt",
KeyFile: "/var/run/kubernetes/server.key",
CAFile: "/var/run/kubernetes/caserver.crt",
},
},
ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
ClientCA: "/client-ca",
},
WebHook: &kubeoptions.WebHookAuthenticationOptions{
CacheTTL: 180000000000,
ConfigFile: "/token-webhook-config",
Version: "v1beta1",
RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
},
BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
OIDC: &kubeoptions.OIDCAuthenticationOptions{
UsernameClaim: "sub",
SigningAlgs: []string{"RS256"},
},
RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
Lookup: true,
ExtendExpiration: true,
},
TokenFile: &kubeoptions.TokenFileAuthenticationOptions{},
TokenSuccessCacheTTL: 10 * time.Second,
TokenFailureCacheTTL: 0,
},
Authorization: &kubeoptions.BuiltInAuthorizationOptions{
Modes: []string{"AlwaysDeny", "RBAC"},
PolicyFile: "/policy",
WebhookConfigFile: "/webhook-config",
WebhookCacheAuthorizedTTL: 180000000000,
WebhookCacheUnauthorizedTTL: 60000000000,
WebhookVersion: "v1beta1",
WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
},
CloudProvider: &kubeoptions.CloudProviderOptions{
CloudConfigFile: "/cloud-config",
CloudProvider: "azure",
},
APIEnablement: &apiserveroptions.APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{},
},
EgressSelector: &apiserveroptions.EgressSelectorOptions{
ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
},
EnableLogsHandler: false,
EnableAggregatorRouting: true,
ProxyClientKeyFile: "/var/run/kubernetes/proxy.key",
ProxyClientCertFile: "/var/run/kubernetes/proxy.crt",
Metrics: &metrics.Options{},
Logs: logs.NewOptions(),
Traces: &apiserveroptions.TracingOptions{
ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
},
AggregatorRejectForwardingRedirects: true,
}
if !reflect.DeepEqual(expected, s) {

59
cmd/kube-apiserver/app/options/validation.go
View File

@ -22,11 +22,7 @@ import (
"net"
"strings"
apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
genericfeatures "k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/features"
netutils "k8s.io/utils/net"
)
@ -106,64 +102,13 @@ func validateServiceNodePort(options *ServerRunOptions) []error {
return errs
}
func validateTokenRequest(options *ServerRunOptions) []error {
var errs []error
enableAttempted := options.ServiceAccountSigningKeyFile != "" ||
(len(options.Authentication.ServiceAccounts.Issuers) != 0 && options.Authentication.ServiceAccounts.Issuers[0] != "") ||
len(options.Authentication.APIAudiences) != 0
enableSucceeded := options.ServiceAccountIssuer != nil
if !enableAttempted {
errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags"))
}
if enableAttempted && !enableSucceeded {
errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together"))
}
return errs
}
func validateAPIPriorityAndFairness(options *ServerRunOptions) []error {
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && options.GenericServerRunOptions.EnablePriorityAndFairness {
// If none of the following runtime config options are specified,
// APF is assumed to be turned on. The internal APF controller uses
// v1beta3 so it should be enabled.
enabledAPIString := options.APIEnablement.RuntimeConfig.String()
testConfigs := []string{"flowcontrol.apiserver.k8s.io/v1beta3", "api/beta", "api/all"} // in the order of precedence
for _, testConfig := range testConfigs {
if strings.Contains(enabledAPIString, fmt.Sprintf("%s=false", testConfig)) {
return []error{fmt.Errorf("--runtime-config=%s=false conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true", testConfig)}
}
if strings.Contains(enabledAPIString, fmt.Sprintf("%s=true", testConfig)) {
return nil
}
}
}
return nil
}
// Validate checks ServerRunOptions and return a slice of found errs.
func (s *ServerRunOptions) Validate() []error {
var errs []error
if s.MasterCount <= 0 {
errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
}
errs = append(errs, s.Etcd.Validate()...)
errs = append(errs, s.Options.Validate()...)
errs = append(errs, validateClusterIPFlags(s)...)
errs = append(errs, validateServiceNodePort(s)...)
errs = append(errs, validateAPIPriorityAndFairness(s)...)
errs = append(errs, s.SecureServing.Validate()...)
errs = append(errs, s.Authentication.Validate()...)
errs = append(errs, s.Authorization.Validate()...)
errs = append(errs, s.Audit.Validate()...)
errs = append(errs, s.Admission.Validate()...)
errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...)
errs = append(errs, validateTokenRequest(s)...)
errs = append(errs, s.Metrics.Validate()...)
return errs
}

166
cmd/kube-apiserver/app/options/validation_test.go
View File

@ -18,17 +18,12 @@ package options
import (
"net"
"strings"
"testing"
utilnet "k8s.io/apimachinery/pkg/util/net"
kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
basemetrics "k8s.io/component-base/metrics"
"k8s.io/kubernetes/pkg/features"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
netutils "k8s.io/utils/net"
)
@ -53,9 +48,11 @@ func makeOptionsWithCIDRs(serviceCIDR string, secondaryServiceCIDR string) *Serv
}
}
return &ServerRunOptions{
ServiceClusterIPRanges: value,
PrimaryServiceClusterIPRange: primaryCIDR,
SecondaryServiceClusterIPRange: secondaryCIDR,
Extra: Extra{
ServiceClusterIPRanges: value,
PrimaryServiceClusterIPRange: primaryCIDR,
SecondaryServiceClusterIPRange: secondaryCIDR,
},
}
}
@ -203,9 +200,9 @@ func TestValidateServiceNodePort(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := validateServiceNodePort(tc.options)
if err != nil && !tc.expectErrors {
t.Errorf("expected no errors, error found %+v", err)
errs := validateServiceNodePort(tc.options)
if errs != nil && !tc.expectErrors {
t.Errorf("expected no errors, error found %+v", errs)
}
})
}
@ -217,8 +214,10 @@ func makeOptionsWithPort(kubernetesServiceNodePort int, base int, size int) *Ser
Size: size,
}
return &ServerRunOptions{
ServiceNodePortRange: portRange,
KubernetesServiceNodePort: kubernetesServiceNodePort,
Extra: Extra{
ServiceNodePortRange: portRange,
KubernetesServiceNodePort: kubernetesServiceNodePort,
},
}
}
@ -283,144 +282,3 @@ func TestValidateMaxCIDRRange(t *testing.T) {
})
}
}
func TestValidateAPIPriorityAndFairness(t *testing.T) {
const conflict = "conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true"
tests := []struct {
runtimeConfig string
errShouldContain string
}{
{
runtimeConfig: "api/all=false",
errShouldContain: conflict,
},
{
runtimeConfig: "api/beta=false",
errShouldContain: conflict,
},
{
runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta1=false",
errShouldContain: "",
},
{
runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta2=false",
errShouldContain: "",
},
{
runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta3=false",
errShouldContain: conflict,
},
{
runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta3=true",
errShouldContain: "",
},
}
for _, test := range tests {
t.Run(test.runtimeConfig, func(t *testing.T) {
options := &ServerRunOptions{
GenericServerRunOptions: &genericoptions.ServerRunOptions{
EnablePriorityAndFairness: true,
},
APIEnablement: genericoptions.NewAPIEnablementOptions(),
}
options.APIEnablement.RuntimeConfig.Set(test.runtimeConfig)
var errMessageGot string
if errs := validateAPIPriorityAndFairness(options); len(errs) > 0 {
errMessageGot = errs[0].Error()
}
if !strings.Contains(errMessageGot, test.errShouldContain) {
t.Errorf("Expected error message to contain: %q, but got: %q", test.errShouldContain, errMessageGot)
}
})
}
}
func TestValidateServerRunOptions(t *testing.T) {
cidrOpts := makeOptionsWithCIDRs("10.0.0.0/16", "3000::/64")
nodePortOpts := makeOptionsWithPort(-1, 30065, 1)
testCases := []struct {
name string
options *ServerRunOptions
expectErrors bool
}{
{
name: "validate master count equal 0",
expectErrors: true,
options: &ServerRunOptions{
MasterCount: 0,
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &genericoptions.AdmissionOptions{
EnablePlugins: []string{"foo"},
Plugins: kubeapiserveradmission.NewPlugins(),
},
PluginNames: []string{"foo"},
},
Authentication: &kubeoptions.BuiltInAuthenticationOptions{
APIAudiences: []string{"bar"},
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
Issuers: []string{"baz"},
},
},
Authorization: &kubeoptions.BuiltInAuthorizationOptions{},
APIEnablement: genericoptions.NewAPIEnablementOptions(),
Metrics: &basemetrics.Options{},
ServiceClusterIPRanges: cidrOpts.ServiceClusterIPRanges,
PrimaryServiceClusterIPRange: cidrOpts.PrimaryServiceClusterIPRange,
SecondaryServiceClusterIPRange: cidrOpts.SecondaryServiceClusterIPRange,
ServiceNodePortRange: nodePortOpts.ServiceNodePortRange,
KubernetesServiceNodePort: nodePortOpts.KubernetesServiceNodePort,
ServiceAccountSigningKeyFile: "",
},
},
{
name: "validate token request enable not attempted",
expectErrors: true,
options: &ServerRunOptions{
MasterCount: 1,
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &genericoptions.AdmissionOptions{
EnablePlugins: []string{""},
Plugins: kubeapiserveradmission.NewPlugins(),
},
PluginNames: []string{""},
},
Authentication: &kubeoptions.BuiltInAuthenticationOptions{
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{},
},
Authorization: &kubeoptions.BuiltInAuthorizationOptions{},
APIEnablement: genericoptions.NewAPIEnablementOptions(),
Metrics: &basemetrics.Options{},
ServiceClusterIPRanges: cidrOpts.ServiceClusterIPRanges,
PrimaryServiceClusterIPRange: cidrOpts.PrimaryServiceClusterIPRange,
SecondaryServiceClusterIPRange: cidrOpts.SecondaryServiceClusterIPRange,
ServiceNodePortRange: nodePortOpts.ServiceNodePortRange,
KubernetesServiceNodePort: nodePortOpts.KubernetesServiceNodePort,
ServiceAccountSigningKeyFile: "",
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
errs := tc.options.Validate()
if len(errs) > 0 && !tc.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)
}
if len(errs) == 0 && tc.expectErrors {
t.Errorf("expected errors, no errors found")
}
})
}
}

292
pkg/controlplane/apiserver/options/options.go Normal file
View File

@ -0,0 +1,292 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package options contains flags and options for initializing an apiserver
package options
import (
"fmt"
"net"
"os"
"strings"
"time"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/client-go/util/keyutil"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
logsapi "k8s.io/component-base/logs/api/v1"
"k8s.io/component-base/metrics"
"k8s.io/klog/v2"
"k8s.io/utils/integer"
netutil "k8s.io/utils/net"
_ "k8s.io/kubernetes/pkg/features"
kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
"k8s.io/kubernetes/pkg/serviceaccount"
)
type Options struct {
GenericServerRunOptions *genericoptions.ServerRunOptions
Etcd *genericoptions.EtcdOptions
SecureServing *genericoptions.SecureServingOptionsWithLoopback
Audit *genericoptions.AuditOptions
Features *genericoptions.FeatureOptions
Admission *kubeoptions.AdmissionOptions
Authentication *kubeoptions.BuiltInAuthenticationOptions
Authorization *kubeoptions.BuiltInAuthorizationOptions
APIEnablement *genericoptions.APIEnablementOptions
EgressSelector *genericoptions.EgressSelectorOptions
Metrics *metrics.Options
Logs *logs.Options
Traces *genericoptions.TracingOptions
EnableLogsHandler bool
EventTTL time.Duration
MaxConnectionBytesPerSec int64
ProxyClientCertFile string
ProxyClientKeyFile string
EnableAggregatorRouting bool
AggregatorRejectForwardingRedirects bool
MasterCount int
ServiceAccountSigningKeyFile string
ServiceAccountIssuer serviceaccount.TokenGenerator
ServiceAccountTokenMaxExpiration time.Duration
ShowHiddenMetricsForVersion string
}
// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
type completedOptions struct {
Options
}
type CompletedOptions struct {
// Embed a private pointer that cannot be instantiated outside of this package.
*completedOptions
}
// NewOptions creates a new ServerRunOptions object with default parameters
func NewOptions() *Options {
s := Options{
GenericServerRunOptions: genericoptions.NewServerRunOptions(),
Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)),
SecureServing: kubeoptions.NewSecureServingOptions(),
Audit: genericoptions.NewAuditOptions(),
Features: genericoptions.NewFeatureOptions(),
Admission: kubeoptions.NewAdmissionOptions(),
Authentication: kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
Authorization: kubeoptions.NewBuiltInAuthorizationOptions(),
APIEnablement: genericoptions.NewAPIEnablementOptions(),
EgressSelector: genericoptions.NewEgressSelectorOptions(),
Metrics: metrics.NewOptions(),
Logs: logs.NewOptions(),
Traces: genericoptions.NewTracingOptions(),
EnableLogsHandler: true,
EventTTL: 1 * time.Hour,
MasterCount: 1,
AggregatorRejectForwardingRedirects: true,
}
// Overwrite the default for storage data format.
s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
return &s
}
func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) {
// Add the generic flags.
s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic"))
s.Etcd.AddFlags(fss.FlagSet("etcd"))
s.SecureServing.AddFlags(fss.FlagSet("secure serving"))
s.Audit.AddFlags(fss.FlagSet("auditing"))
s.Features.AddFlags(fss.FlagSet("features"))
s.Authentication.AddFlags(fss.FlagSet("authentication"))
s.Authorization.AddFlags(fss.FlagSet("authorization"))
s.APIEnablement.AddFlags(fss.FlagSet("API enablement"))
s.EgressSelector.AddFlags(fss.FlagSet("egress selector"))
s.Admission.AddFlags(fss.FlagSet("admission"))
s.Metrics.AddFlags(fss.FlagSet("metrics"))
logsapi.AddFlags(s.Logs, fss.FlagSet("logs"))
s.Traces.AddFlags(fss.FlagSet("traces"))
// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
// arrange these text blocks sensibly. Grrr.
fs := fss.FlagSet("misc")
fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL,
"Amount of time to retain events.")
fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler,
"If true, install a /logs handler for the apiserver logs.")
fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19")
fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+
"If non-zero, throttle each user connection to this number of bytes/sec. "+
"Currently only applies to long-running requests.")
fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
"The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)")
fs.MarkDeprecated("apiserver-count", "apiserver-count is deprecated and will be removed in a future version.")
fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
"when it must call out during a request. This includes proxying requests to a user "+
"api-server and calling out to webhook admission plugins. It is expected that this "+
"cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+
"That CA is published in the 'extension-apiserver-authentication' configmap in "+
"the kube-system namespace. Components receiving calls from kube-aggregator should "+
"use that CA to perform their half of the mutual TLS verification.")
fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+
"Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+
"when it must call out during a request. This includes proxying requests to a user "+
"api-server and calling out to webhook admission plugins.")
fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting,
"Turns on aggregator routing requests to endpoints IP rather than cluster IP.")
fs.BoolVar(&s.AggregatorRejectForwardingRedirects, "aggregator-reject-forwarding-redirect", s.AggregatorRejectForwardingRedirects,
"Aggregator reject forwarding redirect response back to client.")
fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
}
func (o *Options) Complete(alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
if o == nil {
return CompletedOptions{completedOptions: &completedOptions{}}, nil
}
completed := completedOptions{
Options: *o,
}
// set defaults
if err := completed.GenericServerRunOptions.DefaultAdvertiseAddress(completed.SecureServing.SecureServingOptions); err != nil {
return CompletedOptions{}, err
}
if err := completed.SecureServing.MaybeDefaultWithSelfSignedCerts(completed.GenericServerRunOptions.AdvertiseAddress.String(), alternateDNS, alternateIPs); err != nil {
return CompletedOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err)
}
if len(completed.GenericServerRunOptions.ExternalHost) == 0 {
if len(completed.GenericServerRunOptions.AdvertiseAddress) > 0 {
completed.GenericServerRunOptions.ExternalHost = completed.GenericServerRunOptions.AdvertiseAddress.String()
} else {
hostname, err := os.Hostname()
if err != nil {
return CompletedOptions{}, fmt.Errorf("error finding host name: %v", err)
}
completed.GenericServerRunOptions.ExternalHost = hostname
}
klog.Infof("external host was not specified, using %v", completed.GenericServerRunOptions.ExternalHost)
}
completed.Authentication.ApplyAuthorization(completed.Authorization)
// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
// TokenRequest functionality. This defaulting was convenient, but messed up
// a lot of people when they rotated their serving cert with no idea it was
// connected to their service account keys. We are taking this opportunity to
// remove this problematic defaulting.
if completed.ServiceAccountSigningKeyFile == "" {
// Default to the private server key for service account token signing
if len(completed.Authentication.ServiceAccounts.KeyFiles) == 0 && completed.SecureServing.ServerCert.CertKey.KeyFile != "" {
if kubeauthenticator.IsValidServiceAccountKeyFile(completed.SecureServing.ServerCert.CertKey.KeyFile) {
completed.Authentication.ServiceAccounts.KeyFiles = []string{completed.SecureServing.ServerCert.CertKey.KeyFile}
} else {
klog.Warning("No TLS key provided, service account token authentication disabled")
}
}
}
if completed.ServiceAccountSigningKeyFile != "" && len(completed.Authentication.ServiceAccounts.Issuers) != 0 && completed.Authentication.ServiceAccounts.Issuers[0] != "" {
sk, err := keyutil.PrivateKeyFromFile(completed.ServiceAccountSigningKeyFile)
if err != nil {
return CompletedOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
}
if completed.Authentication.ServiceAccounts.MaxExpiration != 0 {
lowBound := time.Hour
upBound := time.Duration(1<<32) * time.Second
if completed.Authentication.ServiceAccounts.MaxExpiration < lowBound ||
completed.Authentication.ServiceAccounts.MaxExpiration > upBound {
return CompletedOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds")
}
if completed.Authentication.ServiceAccounts.ExtendExpiration {
if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second {
klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
}
if completed.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second {
klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, completed.Authentication.ServiceAccounts.MaxExpiration)
}
}
}
completed.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(completed.Authentication.ServiceAccounts.Issuers[0], sk)
if err != nil {
return CompletedOptions{}, fmt.Errorf("failed to build token generator: %v", err)
}
completed.ServiceAccountTokenMaxExpiration = completed.Authentication.ServiceAccounts.MaxExpiration
}
for key, value := range completed.APIEnablement.RuntimeConfig {
if key == "v1" || strings.HasPrefix(key, "v1/") ||
key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
delete(completed.APIEnablement.RuntimeConfig, key)
completed.APIEnablement.RuntimeConfig["/v1"] = value
}
if key == "api/legacy" {
delete(completed.APIEnablement.RuntimeConfig, key)
}
}
return CompletedOptions{
completedOptions: &completed,
}, nil
}
// ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and
// setting service ip range to the default value in kubeoptions.DefaultServiceIPCIDR
// for now until the default is removed per the deprecation timeline guidelines.
// Returns service ip range, api server service IP, and an error
func ServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) {
serviceClusterIPRange := passedServiceClusterIPRange
if passedServiceClusterIPRange.IP == nil {
klog.Warningf("No CIDR for service cluster IPs specified. Default value which was %s is deprecated and will be removed in future releases. Please specify it using --service-cluster-ip-range on kube-apiserver.", kubeoptions.DefaultServiceIPCIDR.String())
serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
}
size := integer.Int64Min(netutil.RangeSize(&serviceClusterIPRange), 1<<16)
if size < 8 {
return net.IPNet{}, net.IP{}, fmt.Errorf("the service cluster IP range must be at least %d IP addresses", 8)
}
// Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP.
apiServerServiceIP, err := netutil.GetIndexedIP(&serviceClusterIPRange, 1)
if err != nil {
return net.IPNet{}, net.IP{}, err
}
klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP)
return serviceClusterIPRange, apiServerServiceIP, nil
}

291
pkg/controlplane/apiserver/options/options_test.go Normal file
View File

@ -0,0 +1,291 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"reflect"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/spf13/pflag"
oteltrace "go.opentelemetry.io/otel/trace"
"k8s.io/apiserver/pkg/admission"
apiserveroptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/etcd3"
"k8s.io/apiserver/pkg/storage/storagebackend"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
"k8s.io/component-base/metrics"
netutils "k8s.io/utils/net"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
)
func TestAddFlags(t *testing.T) {
fs := pflag.NewFlagSet("addflagstest", pflag.PanicOnError)
s := NewOptions()
var fss cliflag.NamedFlagSets
s.AddFlags(&fss)
for _, f := range fss.FlagSets {
fs.AddFlagSet(f)
}
args := []string{
"--enable-admission-plugins=AlwaysDeny",
"--admission-control-config-file=/admission-control-config",
"--advertise-address=192.168.10.10",
"--anonymous-auth=false",
"--apiserver-count=5",
"--audit-log-maxage=11",
"--audit-log-maxbackup=12",
"--audit-log-maxsize=13",
"--audit-log-path=/var/log",
"--audit-log-mode=blocking",
"--audit-log-batch-buffer-size=46",
"--audit-log-batch-max-size=47",
"--audit-log-batch-max-wait=48s",
"--audit-log-batch-throttle-enable=true",
"--audit-log-batch-throttle-qps=49.5",
"--audit-log-batch-throttle-burst=50",
"--audit-log-truncate-enabled=true",
"--audit-log-truncate-max-batch-size=45",
"--audit-log-truncate-max-event-size=44",
"--audit-log-version=audit.k8s.io/v1",
"--audit-policy-file=/policy",
"--audit-webhook-config-file=/webhook-config",
"--audit-webhook-mode=blocking",
"--audit-webhook-batch-buffer-size=42",
"--audit-webhook-batch-max-size=43",
"--audit-webhook-batch-max-wait=1s",
"--audit-webhook-batch-throttle-enable=false",
"--audit-webhook-batch-throttle-qps=43.5",
"--audit-webhook-batch-throttle-burst=44",
"--audit-webhook-truncate-enabled=true",
"--audit-webhook-truncate-max-batch-size=43",
"--audit-webhook-truncate-max-event-size=42",
"--audit-webhook-initial-backoff=2s",
"--audit-webhook-version=audit.k8s.io/v1",
"--authentication-token-webhook-cache-ttl=3m",
"--authentication-token-webhook-config-file=/token-webhook-config",
"--authorization-mode=AlwaysDeny,RBAC",
"--authorization-policy-file=/policy",
"--authorization-webhook-cache-authorized-ttl=3m",
"--authorization-webhook-cache-unauthorized-ttl=1m",
"--authorization-webhook-config-file=/webhook-config",
"--bind-address=192.168.10.20",
"--client-ca-file=/client-ca",
"--cors-allowed-origins=10.10.10.100,10.10.10.200",
"--contention-profiling=true",
"--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml",
"--enable-aggregator-routing=true",
"--enable-priority-and-fairness=false",
"--enable-logs-handler=false",
"--etcd-keyfile=/var/run/kubernetes/etcd.key",
"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
"--etcd-cafile=/var/run/kubernetes/etcdca.crt",
"--http2-max-streams-per-connection=42",
"--tracing-config-file=/var/run/kubernetes/tracing_config.yaml",
"--proxy-client-cert-file=/var/run/kubernetes/proxy.crt",
"--proxy-client-key-file=/var/run/kubernetes/proxy.key",
"--request-timeout=2m",
"--storage-backend=etcd3",
"--lease-reuse-duration-seconds=100",
}
fs.Parse(args)
// This is a snapshot of expected options parsed by args.
expected := &Options{
MasterCount: 5,
GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
AdvertiseAddress: netutils.ParseIPSloppy("192.168.10.10"),
CorsAllowedOriginList: []string{"10.10.10.100", "10.10.10.200"},
MaxRequestsInFlight: 400,
MaxMutatingRequestsInFlight: 200,
RequestTimeout: time.Duration(2) * time.Minute,
MinRequestTimeout: 1800,
JSONPatchMaxCopyBytes: int64(3 * 1024 * 1024),
MaxRequestBodyBytes: int64(3 * 1024 * 1024),
},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &apiserveroptions.AdmissionOptions{
RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins,
EnablePlugins: []string{"AlwaysDeny"},
ConfigFile: "/admission-control-config",
Plugins: s.Admission.GenericAdmission.Plugins,
Decorators: s.Admission.GenericAdmission.Decorators,
},
},
Etcd: &apiserveroptions.EtcdOptions{
StorageConfig: storagebackend.Config{
Type: "etcd3",
Transport: storagebackend.TransportConfig{
ServerList: nil,
KeyFile: "/var/run/kubernetes/etcd.key",
TrustedCAFile: "/var/run/kubernetes/etcdca.crt",
CertFile: "/var/run/kubernetes/etcdce.crt",
TracerProvider: oteltrace.NewNoopTracerProvider(),
},
Paging: true,
Prefix: "/registry",
CompactionInterval: storagebackend.DefaultCompactInterval,
CountMetricPollPeriod: time.Minute,
DBMetricPollInterval: storagebackend.DefaultDBMetricPollInterval,
HealthcheckTimeout: storagebackend.DefaultHealthcheckTimeout,
ReadycheckTimeout: storagebackend.DefaultReadinessTimeout,
LeaseManagerConfig: etcd3.LeaseManagerConfig{
ReuseDurationSeconds: 100,
MaxObjectCount: 1000,
},
},
DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
DeleteCollectionWorkers: 1,
EnableGarbageCollection: true,
EnableWatchCache: true,
DefaultWatchCacheSize: 100,
},
SecureServing: (&apiserveroptions.SecureServingOptions{
BindAddress: netutils.ParseIPSloppy("192.168.10.20"),
BindPort: 6443,
ServerCert: apiserveroptions.GeneratableKeyCert{
CertDirectory: "/var/run/kubernetes",
PairName: "apiserver",
},
HTTP2MaxStreamsPerConnection: 42,
Required: true,
}).WithLoopback(),
EventTTL: 1 * time.Hour,
Audit: &apiserveroptions.AuditOptions{
LogOptions: apiserveroptions.AuditLogOptions{
Path: "/var/log",
MaxAge: 11,
MaxBackups: 12,
MaxSize: 13,
Format: "json",
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 46,
MaxBatchSize: 47,
MaxBatchWait: 48 * time.Second,
ThrottleEnable: true,
ThrottleQPS: 49.5,
ThrottleBurst: 50,
},
},
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 45,
MaxEventSize: 44,
},
},
GroupVersionString: "audit.k8s.io/v1",
},
WebhookOptions: apiserveroptions.AuditWebhookOptions{
ConfigFile: "/webhook-config",
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 42,
MaxBatchSize: 43,
MaxBatchWait: 1 * time.Second,
ThrottleEnable: false,
ThrottleQPS: 43.5,
ThrottleBurst: 44,
AsyncDelegate: true,
},
},
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 43,
MaxEventSize: 42,
},
},
InitialBackoff: 2 * time.Second,
GroupVersionString: "audit.k8s.io/v1",
},
PolicyFile: "/policy",
},
Features: &apiserveroptions.FeatureOptions{
EnableProfiling: true,
EnableContentionProfiling: true,
},
Authentication: &kubeoptions.BuiltInAuthenticationOptions{
Anonymous: &kubeoptions.AnonymousAuthenticationOptions{
Allow: false,
},
ClientCert: &apiserveroptions.ClientCertAuthenticationOptions{
ClientCA: "/client-ca",
},
WebHook: &kubeoptions.WebHookAuthenticationOptions{
CacheTTL: 180000000000,
ConfigFile: "/token-webhook-config",
Version: "v1beta1",
RetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
},
BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
OIDC: &kubeoptions.OIDCAuthenticationOptions{
UsernameClaim: "sub",
SigningAlgs: []string{"RS256"},
},
RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
Lookup: true,
ExtendExpiration: true,
},
TokenFile: &kubeoptions.TokenFileAuthenticationOptions{},
TokenSuccessCacheTTL: 10 * time.Second,
TokenFailureCacheTTL: 0,
},
Authorization: &kubeoptions.BuiltInAuthorizationOptions{
Modes: []string{"AlwaysDeny", "RBAC"},
PolicyFile: "/policy",
WebhookConfigFile: "/webhook-config",
WebhookCacheAuthorizedTTL: 180000000000,
WebhookCacheUnauthorizedTTL: 60000000000,
WebhookVersion: "v1beta1",
WebhookRetryBackoff: apiserveroptions.DefaultAuthWebhookRetryBackoff(),
},
APIEnablement: &apiserveroptions.APIEnablementOptions{
RuntimeConfig: cliflag.ConfigurationMap{},
},
EgressSelector: &apiserveroptions.EgressSelectorOptions{
ConfigFile: "/var/run/kubernetes/egress-selector/connectivity.yaml",
},
EnableLogsHandler: false,
EnableAggregatorRouting: true,
ProxyClientKeyFile: "/var/run/kubernetes/proxy.key",
ProxyClientCertFile: "/var/run/kubernetes/proxy.crt",
Metrics: &metrics.Options{},
Logs: logs.NewOptions(),
Traces: &apiserveroptions.TracingOptions{
ConfigFile: "/var/run/kubernetes/tracing_config.yaml",
},
AggregatorRejectForwardingRedirects: true,
}
if !reflect.DeepEqual(expected, s) {
t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", cmp.Diff(expected, s, cmpopts.IgnoreUnexported(admission.Plugins{})))
}
}

90
pkg/controlplane/apiserver/options/validation.go Normal file
View File

@ -0,0 +1,90 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"errors"
"fmt"
"strings"
apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
genericfeatures "k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
"k8s.io/kubernetes/pkg/api/legacyscheme"
)
func validateTokenRequest(options *Options) []error {
var errs []error
enableAttempted := options.ServiceAccountSigningKeyFile != "" ||
(len(options.Authentication.ServiceAccounts.Issuers) != 0 && options.Authentication.ServiceAccounts.Issuers[0] != "") ||
len(options.Authentication.APIAudiences) != 0
enableSucceeded := options.ServiceAccountIssuer != nil
if !enableAttempted {
errs = append(errs, errors.New("--service-account-signing-key-file and --service-account-issuer are required flags"))
}
if enableAttempted && !enableSucceeded {
errs = append(errs, errors.New("--service-account-signing-key-file, --service-account-issuer, and --api-audiences should be specified together"))
}
return errs
}
func validateAPIPriorityAndFairness(options *Options) []error {
if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && options.GenericServerRunOptions.EnablePriorityAndFairness {
// If none of the following runtime config options are specified,
// APF is assumed to be turned on. The internal APF controller uses
// v1beta3 so it should be enabled.
enabledAPIString := options.APIEnablement.RuntimeConfig.String()
testConfigs := []string{"flowcontrol.apiserver.k8s.io/v1beta3", "api/beta", "api/all"} // in the order of precedence
for _, testConfig := range testConfigs {
if strings.Contains(enabledAPIString, fmt.Sprintf("%s=false", testConfig)) {
return []error{fmt.Errorf("--runtime-config=%s=false conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true", testConfig)}
}
if strings.Contains(enabledAPIString, fmt.Sprintf("%s=true", testConfig)) {
return nil
}
}
}
return nil
}
// Validate checks Options and return a slice of found errs.
func (s *Options) Validate() []error {
var errs []error
if s.MasterCount <= 0 {
errs = append(errs, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
}
errs = append(errs, s.Etcd.Validate()...)
errs = append(errs, validateAPIPriorityAndFairness(s)...)
errs = append(errs, s.SecureServing.Validate()...)
errs = append(errs, s.Authentication.Validate()...)
errs = append(errs, s.Authorization.Validate()...)
errs = append(errs, s.Audit.Validate()...)
errs = append(errs, s.Admission.Validate()...)
errs = append(errs, s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme)...)
errs = append(errs, validateTokenRequest(s)...)
errs = append(errs, s.Metrics.Validate()...)
return errs
}

154
pkg/controlplane/apiserver/options/validation_test.go Normal file
View File

@ -0,0 +1,154 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"strings"
"testing"
kubeapiserveradmission "k8s.io/apiserver/pkg/admission"
genericoptions "k8s.io/apiserver/pkg/server/options"
basemetrics "k8s.io/component-base/metrics"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
)
func TestValidateAPIPriorityAndFairness(t *testing.T) {
const conflict = "conflicts with --enable-priority-and-fairness=true and --feature-gates=APIPriorityAndFairness=true"
tests := []struct {
runtimeConfig string
errShouldContain string
}{
{
runtimeConfig: "api/all=false",
errShouldContain: conflict,
},
{
runtimeConfig: "api/beta=false",
errShouldContain: conflict,
},
{
runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta1=false",
errShouldContain: "",
},
{
runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta2=false",
errShouldContain: "",
},
{
runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta3=false",
errShouldContain: conflict,
},
{
runtimeConfig: "flowcontrol.apiserver.k8s.io/v1beta3=true",
errShouldContain: "",
},
}
for _, test := range tests {
t.Run(test.runtimeConfig, func(t *testing.T) {
options := &Options{
GenericServerRunOptions: &genericoptions.ServerRunOptions{
EnablePriorityAndFairness: true,
},
APIEnablement: genericoptions.NewAPIEnablementOptions(),
}
options.APIEnablement.RuntimeConfig.Set(test.runtimeConfig)
var errMessageGot string
if errs := validateAPIPriorityAndFairness(options); len(errs) > 0 {
errMessageGot = errs[0].Error()
}
if !strings.Contains(errMessageGot, test.errShouldContain) {
t.Errorf("Expected error message to contain: %q, but got: %q", test.errShouldContain, errMessageGot)
}
})
}
}
func TestValidateOptions(t *testing.T) {
testCases := []struct {
name string
options *Options
expectErrors bool
}{
{
name: "validate master count equal 0",
expectErrors: true,
options: &Options{
MasterCount: 0,
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &genericoptions.AdmissionOptions{
EnablePlugins: []string{"foo"},
Plugins: kubeapiserveradmission.NewPlugins(),
},
PluginNames: []string{"foo"},
},
Authentication: &kubeoptions.BuiltInAuthenticationOptions{
APIAudiences: []string{"bar"},
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{
Issuers: []string{"baz"},
},
},
APIEnablement: genericoptions.NewAPIEnablementOptions(),
Metrics: &basemetrics.Options{},
ServiceAccountSigningKeyFile: "",
},
},
{
name: "validate token request enable not attempted",
expectErrors: true,
options: &Options{
MasterCount: 1,
GenericServerRunOptions: &genericoptions.ServerRunOptions{},
Etcd: &genericoptions.EtcdOptions{},
SecureServing: &genericoptions.SecureServingOptionsWithLoopback{},
Audit: &genericoptions.AuditOptions{},
Admission: &kubeoptions.AdmissionOptions{
GenericAdmission: &genericoptions.AdmissionOptions{
EnablePlugins: []string{""},
Plugins: kubeapiserveradmission.NewPlugins(),
},
PluginNames: []string{""},
},
Authentication: &kubeoptions.BuiltInAuthenticationOptions{
ServiceAccounts: &kubeoptions.ServiceAccountAuthenticationOptions{},
},
APIEnablement: genericoptions.NewAPIEnablementOptions(),
Metrics: &basemetrics.Options{},
ServiceAccountSigningKeyFile: "",
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
errs := tc.options.Validate()
if len(errs) > 0 && !tc.expectErrors {
t.Errorf("expected no errors, errors found %+v", errs)
}
if len(errs) == 0 && tc.expectErrors {
t.Errorf("expected errors, no errors found")
}
})
}
}

54
pkg/controlplane/services.go
View File

@ -1,54 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"net"
"k8s.io/klog/v2"
"k8s.io/utils/integer"
utilnet "k8s.io/utils/net"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
)
// ServiceIPRange checks if the serviceClusterIPRange flag is nil, raising a warning if so and
// setting service ip range to the default value in kubeoptions.DefaultServiceIPCIDR
// for now until the default is removed per the deprecation timeline guidelines.
// Returns service ip range, api server service IP, and an error
func ServiceIPRange(passedServiceClusterIPRange net.IPNet) (net.IPNet, net.IP, error) {
serviceClusterIPRange := passedServiceClusterIPRange
if passedServiceClusterIPRange.IP == nil {
klog.Warningf("No CIDR for service cluster IPs specified. Default value which was %s is deprecated and will be removed in future releases. Please specify it using --service-cluster-ip-range on kube-apiserver.", kubeoptions.DefaultServiceIPCIDR.String())
serviceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
}
size := integer.Int64Min(utilnet.RangeSize(&serviceClusterIPRange), 1<<16)
if size < 8 {
return net.IPNet{}, net.IP{}, fmt.Errorf("the service cluster IP range must be at least %d IP addresses", 8)
}
// Select the first valid IP from ServiceClusterIPRange to use as the GenericAPIServer service IP.
apiServerServiceIP, err := utilnet.GetIndexedIP(&serviceClusterIPRange, 1)
if err != nil {
return net.IPNet{}, net.IP{}, err
}
klog.V(4).Infof("Setting service IP to %q (read-write).", apiServerServiceIP)
return serviceClusterIPRange, apiServerServiceIP, nil
}