diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index deb3a6ad4da..543ed4d3ff4 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -57,6 +57,7 @@ import ( "k8s.io/klog/v2" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" + controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" "k8s.io/kubernetes/pkg/api/legacyscheme" @@ -223,9 +224,30 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( generatedopenapi.GetOpenAPIDefinitions, ) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, fmt.Errorf("failed to create generic config: %w", err) } + // generic controlplane admission initializers + controlPlaneAdmissionConfig := &controlplaneadmission.Config{ + ExternalInformers: versionedInformers, + LoopbackClientConfig: genericConfig.LoopbackClientConfig, + } + serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) + pluginInitializers, err := controlPlaneAdmissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err) + } + + // additional kube admission initializers + kubeAdmissionConfig := &kubeapiserveradmission.Config{ + CloudConfigFile: opts.CloudProvider.CloudConfigFile, + } + kubeInitializers, err := kubeAdmissionConfig.New() + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err) + } + pluginInitializers = append(pluginInitializers, kubeInitializers...) + capabilities.Setup(opts.AllowPrivileged, opts.MaxConnectionBytesPerSec) opts.Metrics.Apply() @@ -300,16 +322,6 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) ( } // setup admission - admissionConfig := &kubeapiserveradmission.Config{ - ExternalInformers: versionedInformers, - LoopbackClientConfig: genericConfig.LoopbackClientConfig, - CloudConfigFile: opts.CloudProvider.CloudConfigFile, - } - serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) - pluginInitializers, err := admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %v", err) - } clientgoExternalClient, err := clientset.NewForConfig(genericConfig.LoopbackClientConfig) if err != nil { return nil, nil, nil, fmt.Errorf("failed to create real client-go external client: %w", err) diff --git a/pkg/controlplane/apiserver/admission/config.go b/pkg/controlplane/apiserver/admission/config.go new file mode 100644 index 00000000000..1fa77f36934 --- /dev/null +++ b/pkg/controlplane/apiserver/admission/config.go @@ -0,0 +1,51 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "net/http" + + "go.opentelemetry.io/otel/trace" + + "k8s.io/apiserver/pkg/admission" + webhookinit "k8s.io/apiserver/pkg/admission/plugin/webhook/initializer" + egressselector "k8s.io/apiserver/pkg/server/egressselector" + "k8s.io/apiserver/pkg/util/webhook" + externalinformers "k8s.io/client-go/informers" + "k8s.io/client-go/rest" + "k8s.io/kubernetes/pkg/kubeapiserver/admission/exclusion" + quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" +) + +// Config holds the configuration needed to for initialize the admission plugins +type Config struct { + LoopbackClientConfig *rest.Config + ExternalInformers externalinformers.SharedInformerFactory +} + +// New sets up the plugins and admission start hooks needed for admission +func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselector.EgressSelector, serviceResolver webhook.ServiceResolver, tp trace.TracerProvider) ([]admission.PluginInitializer, error) { + webhookAuthResolverWrapper := webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, egressSelector, c.LoopbackClientConfig, tp) + webhookPluginInitializer := webhookinit.NewPluginInitializer(webhookAuthResolverWrapper, serviceResolver) + + kubePluginInitializer := NewPluginInitializer( + quotainstall.NewQuotaConfigurationForAdmission(), + exclusion.Excluded(), + ) + + return []admission.PluginInitializer{webhookPluginInitializer, kubePluginInitializer}, nil +} diff --git a/pkg/controlplane/apiserver/admission/initializer.go b/pkg/controlplane/apiserver/admission/initializer.go new file mode 100644 index 00000000000..202b6771d8f --- /dev/null +++ b/pkg/controlplane/apiserver/admission/initializer.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/admission/initializer" + quota "k8s.io/apiserver/pkg/quota/v1" +) + +// PluginInitializer is used for initialization of the generic controlplane admission plugins. +type PluginInitializer struct { + quotaConfiguration quota.Configuration + excludedAdmissionResources []schema.GroupResource +} + +var _ admission.PluginInitializer = &PluginInitializer{} + +// NewPluginInitializer constructs new instance of PluginInitializer +func NewPluginInitializer( + quotaConfiguration quota.Configuration, + excludedAdmissionResources []schema.GroupResource, +) *PluginInitializer { + return &PluginInitializer{ + quotaConfiguration: quotaConfiguration, + excludedAdmissionResources: excludedAdmissionResources, + } +} + +// Initialize checks the initialization interfaces implemented by each plugin +// and provide the appropriate initialization data +func (i *PluginInitializer) Initialize(plugin admission.Interface) { + if wants, ok := plugin.(initializer.WantsQuotaConfiguration); ok { + wants.SetQuotaConfiguration(i.quotaConfiguration) + } + + if wants, ok := plugin.(initializer.WantsExcludedAdmissionResources); ok { + wants.SetExcludedAdmissionResources(i.excludedAdmissionResources) + } +} diff --git a/pkg/controlplane/apiserver/admission/initializer_test.go b/pkg/controlplane/apiserver/admission/initializer_test.go new file mode 100644 index 00000000000..79dc618cd61 --- /dev/null +++ b/pkg/controlplane/apiserver/admission/initializer_test.go @@ -0,0 +1,65 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "context" + "testing" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + quota "k8s.io/apiserver/pkg/quota/v1" +) + +type doNothingAdmission struct{} + +func (doNothingAdmission) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error { + return nil +} +func (doNothingAdmission) Handles(o admission.Operation) bool { return false } +func (doNothingAdmission) Validate() error { return nil } + +type doNothingPluginInitialization struct{} + +func (doNothingPluginInitialization) ValidateInitialization() error { return nil } + +type doNothingQuotaConfiguration struct{} + +func (doNothingQuotaConfiguration) IgnoredResources() map[schema.GroupResource]struct{} { return nil } + +func (doNothingQuotaConfiguration) Evaluators() []quota.Evaluator { return nil } + +type WantsQuotaConfigurationAdmissionPlugin struct { + doNothingAdmission + doNothingPluginInitialization + config quota.Configuration +} + +func (p *WantsQuotaConfigurationAdmissionPlugin) SetQuotaConfiguration(config quota.Configuration) { + p.config = config +} + +func TestQuotaConfigurationAdmissionPlugin(t *testing.T) { + config := doNothingQuotaConfiguration{} + initializer := NewPluginInitializer(config, nil) + wantsQuotaConfigurationAdmission := &WantsQuotaConfigurationAdmissionPlugin{} + initializer.Initialize(wantsQuotaConfigurationAdmission) + + if wantsQuotaConfigurationAdmission.config == nil { + t.Errorf("Expected quota configuration to be initialized but found nil") + } +} diff --git a/pkg/kubeapiserver/admission/config.go b/pkg/kubeapiserver/admission/config.go index 249013611fa..20e2cf7c6a4 100644 --- a/pkg/kubeapiserver/admission/config.go +++ b/pkg/kubeapiserver/admission/config.go @@ -17,35 +17,20 @@ limitations under the License. package admission import ( - "net/http" "os" "k8s.io/klog/v2" - "go.opentelemetry.io/otel/trace" - "k8s.io/apiserver/pkg/admission" - webhookinit "k8s.io/apiserver/pkg/admission/plugin/webhook/initializer" - "k8s.io/apiserver/pkg/server/egressselector" - "k8s.io/apiserver/pkg/util/webhook" - externalinformers "k8s.io/client-go/informers" - "k8s.io/client-go/rest" - "k8s.io/kubernetes/pkg/kubeapiserver/admission/exclusion" - quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" ) // Config holds the configuration needed to for initialize the admission plugins type Config struct { - CloudConfigFile string - LoopbackClientConfig *rest.Config - ExternalInformers externalinformers.SharedInformerFactory + CloudConfigFile string } // New sets up the plugins and admission start hooks needed for admission -func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselector.EgressSelector, serviceResolver webhook.ServiceResolver, tp trace.TracerProvider) ([]admission.PluginInitializer, error) { - webhookAuthResolverWrapper := webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, egressSelector, c.LoopbackClientConfig, tp) - webhookPluginInitializer := webhookinit.NewPluginInitializer(webhookAuthResolverWrapper, serviceResolver) - +func (c *Config) New() ([]admission.PluginInitializer, error) { var cloudConfig []byte if c.CloudConfigFile != "" { var err error @@ -54,11 +39,6 @@ func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselec klog.Fatalf("Error reading from cloud configuration file %s: %#v", c.CloudConfigFile, err) } } - kubePluginInitializer := NewPluginInitializer( - cloudConfig, - quotainstall.NewQuotaConfigurationForAdmission(), - exclusion.Excluded(), - ) - return []admission.PluginInitializer{webhookPluginInitializer, kubePluginInitializer}, nil + return []admission.PluginInitializer{NewPluginInitializer(cloudConfig)}, nil } diff --git a/pkg/kubeapiserver/admission/initializer.go b/pkg/kubeapiserver/admission/initializer.go index aa295aecc29..32828508847 100644 --- a/pkg/kubeapiserver/admission/initializer.go +++ b/pkg/kubeapiserver/admission/initializer.go @@ -17,10 +17,7 @@ limitations under the License. package admission import ( - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/admission/initializer" - quota "k8s.io/apiserver/pkg/quota/v1" ) // TODO add a `WantsToRun` which takes a stopCh. Might make it generic. @@ -32,25 +29,15 @@ type WantsCloudConfig interface { // PluginInitializer is used for initialization of the Kubernetes specific admission plugins. type PluginInitializer struct { - cloudConfig []byte - quotaConfiguration quota.Configuration - excludedAdmissionResources []schema.GroupResource + cloudConfig []byte } var _ admission.PluginInitializer = &PluginInitializer{} // NewPluginInitializer constructs new instance of PluginInitializer -// TODO: switch these parameters to use the builder pattern or just make them -// all public, this construction method is pointless boilerplate. -func NewPluginInitializer( - cloudConfig []byte, - quotaConfiguration quota.Configuration, - excludedAdmissionResources []schema.GroupResource, -) *PluginInitializer { +func NewPluginInitializer(cloudConfig []byte) *PluginInitializer { return &PluginInitializer{ - cloudConfig: cloudConfig, - quotaConfiguration: quotaConfiguration, - excludedAdmissionResources: excludedAdmissionResources, + cloudConfig: cloudConfig, } } @@ -60,12 +47,4 @@ func (i *PluginInitializer) Initialize(plugin admission.Interface) { if wants, ok := plugin.(WantsCloudConfig); ok { wants.SetCloudConfig(i.cloudConfig) } - - if wants, ok := plugin.(initializer.WantsQuotaConfiguration); ok { - wants.SetQuotaConfiguration(i.quotaConfiguration) - } - - if wants, ok := plugin.(initializer.WantsExcludedAdmissionResources); ok { - wants.SetExcludedAdmissionResources(i.excludedAdmissionResources) - } } diff --git a/pkg/kubeapiserver/admission/initializer_test.go b/pkg/kubeapiserver/admission/initializer_test.go index 53d4eb1db01..cd291368477 100644 --- a/pkg/kubeapiserver/admission/initializer_test.go +++ b/pkg/kubeapiserver/admission/initializer_test.go @@ -20,9 +20,7 @@ import ( "context" "testing" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" - quota "k8s.io/apiserver/pkg/quota/v1" ) type doNothingAdmission struct{} @@ -33,10 +31,6 @@ func (doNothingAdmission) Admit(ctx context.Context, a admission.Attributes, o a func (doNothingAdmission) Handles(o admission.Operation) bool { return false } func (doNothingAdmission) Validate() error { return nil } -type doNothingPluginInitialization struct{} - -func (doNothingPluginInitialization) ValidateInitialization() error { return nil } - type WantsCloudConfigAdmissionPlugin struct { doNothingAdmission cloudConfig []byte @@ -48,7 +42,7 @@ func (p *WantsCloudConfigAdmissionPlugin) SetCloudConfig(cloudConfig []byte) { func TestCloudConfigAdmissionPlugin(t *testing.T) { cloudConfig := []byte("cloud-configuration") - initializer := NewPluginInitializer(cloudConfig, nil, nil) + initializer := NewPluginInitializer(cloudConfig) wantsCloudConfigAdmission := &WantsCloudConfigAdmissionPlugin{} initializer.Initialize(wantsCloudConfigAdmission) @@ -56,30 +50,3 @@ func TestCloudConfigAdmissionPlugin(t *testing.T) { t.Errorf("Expected cloud config to be initialized but found nil") } } - -type doNothingQuotaConfiguration struct{} - -func (doNothingQuotaConfiguration) IgnoredResources() map[schema.GroupResource]struct{} { return nil } - -func (doNothingQuotaConfiguration) Evaluators() []quota.Evaluator { return nil } - -type WantsQuotaConfigurationAdmissionPlugin struct { - doNothingAdmission - doNothingPluginInitialization - config quota.Configuration -} - -func (p *WantsQuotaConfigurationAdmissionPlugin) SetQuotaConfiguration(config quota.Configuration) { - p.config = config -} - -func TestQuotaConfigurationAdmissionPlugin(t *testing.T) { - config := doNothingQuotaConfiguration{} - initializer := NewPluginInitializer(nil, config, nil) - wantsQuotaConfigurationAdmission := &WantsQuotaConfigurationAdmissionPlugin{} - initializer.Initialize(wantsQuotaConfigurationAdmission) - - if wantsQuotaConfigurationAdmission.config == nil { - t.Errorf("Expected quota configuration to be initialized but found nil") - } -} diff --git a/plugin/pkg/admission/gc/gc_admission_test.go b/plugin/pkg/admission/gc/gc_admission_test.go index 4dc1e3b02e5..1127e538525 100644 --- a/plugin/pkg/admission/gc/gc_admission_test.go +++ b/plugin/pkg/admission/gc/gc_admission_test.go @@ -28,15 +28,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/admission" + apiserveradmission "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/admission/initializer" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" fakediscovery "k8s.io/client-go/discovery/fake" "k8s.io/client-go/restmapper" coretesting "k8s.io/client-go/testing" + api "k8s.io/kubernetes/pkg/apis/core" - kubeadmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" + controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission" ) type fakeAuthorizer struct{} @@ -111,7 +112,7 @@ func newGCPermissionsEnforcement() (*gcPermissionsEnforcement, error) { }, } gcAdmit := &gcPermissionsEnforcement{ - Handler: admission.NewHandler(admission.Create, admission.Update), + Handler: apiserveradmission.NewHandler(apiserveradmission.Create, apiserveradmission.Update), whiteList: whiteList, } @@ -138,9 +139,8 @@ func newGCPermissionsEnforcement() (*gcPermissionsEnforcement, error) { } restMapper := restmapper.NewDiscoveryRESTMapper(restMapperRes) genericPluginInitializer := initializer.New(nil, nil, nil, fakeAuthorizer{}, nil, nil, restMapper) - - pluginInitializer := kubeadmission.NewPluginInitializer(nil, nil, nil) - initializersChain := admission.PluginInitializers{} + pluginInitializer := controlplaneadmission.NewPluginInitializer(nil, nil) + initializersChain := apiserveradmission.PluginInitializers{} initializersChain = append(initializersChain, genericPluginInitializer) initializersChain = append(initializersChain, pluginInitializer) @@ -349,14 +349,14 @@ func TestGCAdmission(t *testing.T) { t.Error(err) } - operation := admission.Create + operation := apiserveradmission.Create var options runtime.Object = &metav1.CreateOptions{} if tc.oldObj != nil { - operation = admission.Update + operation = apiserveradmission.Update options = &metav1.UpdateOptions{} } user := &user.DefaultInfo{Name: tc.username} - attributes := admission.NewAttributesRecord(tc.newObj, tc.oldObj, schema.GroupVersionKind{}, metav1.NamespaceDefault, "foo", tc.resource, tc.subresource, operation, options, false, user) + attributes := apiserveradmission.NewAttributesRecord(tc.newObj, tc.oldObj, schema.GroupVersionKind{}, metav1.NamespaceDefault, "foo", tc.resource, tc.subresource, operation, options, false, user) err = gcAdmit.Validate(context.TODO(), attributes, nil) if !tc.checkError(err) { @@ -668,14 +668,14 @@ func TestBlockOwnerDeletionAdmission(t *testing.T) { gcAdmit.restMapper = tc.restMapperOverride } - operation := admission.Create + operation := apiserveradmission.Create var options runtime.Object = &metav1.CreateOptions{} if tc.oldObj != nil { - operation = admission.Update + operation = apiserveradmission.Update options = &metav1.UpdateOptions{} } user := &user.DefaultInfo{Name: tc.username} - attributes := admission.NewAttributesRecord(tc.newObj, tc.oldObj, schema.GroupVersionKind{}, metav1.NamespaceDefault, "foo", tc.resource, tc.subresource, operation, options, false, user) + attributes := apiserveradmission.NewAttributesRecord(tc.newObj, tc.oldObj, schema.GroupVersionKind{}, metav1.NamespaceDefault, "foo", tc.resource, tc.subresource, operation, options, false, user) err = gcAdmit.Validate(context.TODO(), attributes, nil) if !tc.checkError(err) { diff --git a/plugin/pkg/admission/resourcequota/admission_test.go b/plugin/pkg/admission/resourcequota/admission_test.go index 03763d3d7c5..2f49cfdb8b2 100644 --- a/plugin/pkg/admission/resourcequota/admission_test.go +++ b/plugin/pkg/admission/resourcequota/admission_test.go @@ -37,7 +37,7 @@ import ( testcore "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" api "k8s.io/kubernetes/pkg/apis/core" - kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" + controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission" "k8s.io/kubernetes/pkg/quota/v1/install" ) @@ -115,7 +115,7 @@ func createHandlerWithConfig(kubeClient kubernetes.Interface, informerFactory in initializers := admission.PluginInitializers{ genericadmissioninitializer.New(kubeClient, nil, informerFactory, nil, nil, stopCh, nil), - kubeapiserveradmission.NewPluginInitializer(nil, quotaConfiguration, nil), + controlplaneadmission.NewPluginInitializer(quotaConfiguration, nil), } initializers.Initialize(handler)