Merge pull request #124621 from sttts/sttts-controlplane-admission-initializers

kube-apiserver: split admission initializers into generic and non-generic

cmd/kube-apiserver/app/server.go

@@ -57,6 +57,7 @@ import (
 	"k8s.io/klog/v2"
 	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
 	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+	controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission"
 
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
@@ -223,9 +224,30 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 		generatedopenapi.GetOpenAPIDefinitions,
 	)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, nil, fmt.Errorf("failed to create generic config: %w", err)
 	}
 
+	// generic controlplane admission initializers
+	controlPlaneAdmissionConfig := &controlplaneadmission.Config{
+		ExternalInformers:    versionedInformers,
+		LoopbackClientConfig: genericConfig.LoopbackClientConfig,
+	}
+	serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
+	pluginInitializers, err := controlPlaneAdmissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err)
+	}
+
+	// additional kube admission initializers
+	kubeAdmissionConfig := &kubeapiserveradmission.Config{
+		CloudConfigFile: opts.CloudProvider.CloudConfigFile,
+	}
+	kubeInitializers, err := kubeAdmissionConfig.New()
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err)
+	}
+	pluginInitializers = append(pluginInitializers, kubeInitializers...)
+
 	capabilities.Setup(opts.AllowPrivileged, opts.MaxConnectionBytesPerSec)
 
 	opts.Metrics.Apply()
@@ -300,16 +322,6 @@ func CreateKubeAPIServerConfig(opts options.CompletedOptions) (
 	}
 
 	// setup admission
-	admissionConfig := &kubeapiserveradmission.Config{
-		ExternalInformers:    versionedInformers,
-		LoopbackClientConfig: genericConfig.LoopbackClientConfig,
-		CloudConfigFile:      opts.CloudProvider.CloudConfigFile,
-	}
-	serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
-	pluginInitializers, err := admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %v", err)
-	}
 	clientgoExternalClient, err := clientset.NewForConfig(genericConfig.LoopbackClientConfig)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("failed to create real client-go external client: %w", err)

pkg/controlplane/apiserver/admission/config.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package admission
+
+import (
+	"net/http"
+
+	"go.opentelemetry.io/otel/trace"
+
+	"k8s.io/apiserver/pkg/admission"
+	webhookinit "k8s.io/apiserver/pkg/admission/plugin/webhook/initializer"
+	egressselector "k8s.io/apiserver/pkg/server/egressselector"
+	"k8s.io/apiserver/pkg/util/webhook"
+	externalinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/rest"
+	"k8s.io/kubernetes/pkg/kubeapiserver/admission/exclusion"
+	quotainstall "k8s.io/kubernetes/pkg/quota/v1/install"
+)
+
+// Config holds the configuration needed to for initialize the admission plugins
+type Config struct {
+	LoopbackClientConfig *rest.Config
+	ExternalInformers    externalinformers.SharedInformerFactory
+}
+
+// New sets up the plugins and admission start hooks needed for admission
+func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselector.EgressSelector, serviceResolver webhook.ServiceResolver, tp trace.TracerProvider) ([]admission.PluginInitializer, error) {
+	webhookAuthResolverWrapper := webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, egressSelector, c.LoopbackClientConfig, tp)
+	webhookPluginInitializer := webhookinit.NewPluginInitializer(webhookAuthResolverWrapper, serviceResolver)
+
+	kubePluginInitializer := NewPluginInitializer(
+		quotainstall.NewQuotaConfigurationForAdmission(),
+		exclusion.Excluded(),
+	)
+
+	return []admission.PluginInitializer{webhookPluginInitializer, kubePluginInitializer}, nil
+}

pkg/controlplane/apiserver/admission/initializer.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package admission
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/admission/initializer"
+	quota "k8s.io/apiserver/pkg/quota/v1"
+)
+
+// PluginInitializer is used for initialization of the generic controlplane admission plugins.
+type PluginInitializer struct {
+	quotaConfiguration         quota.Configuration
+	excludedAdmissionResources []schema.GroupResource
+}
+
+var _ admission.PluginInitializer = &PluginInitializer{}
+
+// NewPluginInitializer constructs new instance of PluginInitializer
+func NewPluginInitializer(
+	quotaConfiguration quota.Configuration,
+	excludedAdmissionResources []schema.GroupResource,
+) *PluginInitializer {
+	return &PluginInitializer{
+		quotaConfiguration:         quotaConfiguration,
+		excludedAdmissionResources: excludedAdmissionResources,
+	}
+}
+
+// Initialize checks the initialization interfaces implemented by each plugin
+// and provide the appropriate initialization data
+func (i *PluginInitializer) Initialize(plugin admission.Interface) {
+	if wants, ok := plugin.(initializer.WantsQuotaConfiguration); ok {
+		wants.SetQuotaConfiguration(i.quotaConfiguration)
+	}
+
+	if wants, ok := plugin.(initializer.WantsExcludedAdmissionResources); ok {
+		wants.SetExcludedAdmissionResources(i.excludedAdmissionResources)
+	}
+}

pkg/controlplane/apiserver/admission/initializer_test.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package admission
+
+import (
+	"context"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/admission"
+	quota "k8s.io/apiserver/pkg/quota/v1"
+)
+
+type doNothingAdmission struct{}
+
+func (doNothingAdmission) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error {
+	return nil
+}
+func (doNothingAdmission) Handles(o admission.Operation) bool { return false }
+func (doNothingAdmission) Validate() error                    { return nil }
+
+type doNothingPluginInitialization struct{}
+
+func (doNothingPluginInitialization) ValidateInitialization() error { return nil }
+
+type doNothingQuotaConfiguration struct{}
+
+func (doNothingQuotaConfiguration) IgnoredResources() map[schema.GroupResource]struct{} { return nil }
+
+func (doNothingQuotaConfiguration) Evaluators() []quota.Evaluator { return nil }
+
+type WantsQuotaConfigurationAdmissionPlugin struct {
+	doNothingAdmission
+	doNothingPluginInitialization
+	config quota.Configuration
+}
+
+func (p *WantsQuotaConfigurationAdmissionPlugin) SetQuotaConfiguration(config quota.Configuration) {
+	p.config = config
+}
+
+func TestQuotaConfigurationAdmissionPlugin(t *testing.T) {
+	config := doNothingQuotaConfiguration{}
+	initializer := NewPluginInitializer(config, nil)
+	wantsQuotaConfigurationAdmission := &WantsQuotaConfigurationAdmissionPlugin{}
+	initializer.Initialize(wantsQuotaConfigurationAdmission)
+
+	if wantsQuotaConfigurationAdmission.config == nil {
+		t.Errorf("Expected quota configuration to be initialized but found nil")
+	}
+}

pkg/kubeapiserver/admission/config.go

@@ -17,35 +17,20 @@ limitations under the License.
 package admission
 
 import (
-	"net/http"
 	"os"
 
 	"k8s.io/klog/v2"
 
-	"go.opentelemetry.io/otel/trace"
-
 	"k8s.io/apiserver/pkg/admission"
-	webhookinit "k8s.io/apiserver/pkg/admission/plugin/webhook/initializer"
-	"k8s.io/apiserver/pkg/server/egressselector"
-	"k8s.io/apiserver/pkg/util/webhook"
-	externalinformers "k8s.io/client-go/informers"
-	"k8s.io/client-go/rest"
-	"k8s.io/kubernetes/pkg/kubeapiserver/admission/exclusion"
-	quotainstall "k8s.io/kubernetes/pkg/quota/v1/install"
 )
 
 // Config holds the configuration needed to for initialize the admission plugins
 type Config struct {
 	CloudConfigFile string
-	LoopbackClientConfig *rest.Config
-	ExternalInformers    externalinformers.SharedInformerFactory
 }
 
 // New sets up the plugins and admission start hooks needed for admission
-func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselector.EgressSelector, serviceResolver webhook.ServiceResolver, tp trace.TracerProvider) ([]admission.PluginInitializer, error) {
+func (c *Config) New() ([]admission.PluginInitializer, error) {
-	webhookAuthResolverWrapper := webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, egressSelector, c.LoopbackClientConfig, tp)
-	webhookPluginInitializer := webhookinit.NewPluginInitializer(webhookAuthResolverWrapper, serviceResolver)
-
 	var cloudConfig []byte
 	if c.CloudConfigFile != "" {
 		var err error
@@ -54,11 +39,6 @@ func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselec
 			klog.Fatalf("Error reading from cloud configuration file %s: %#v", c.CloudConfigFile, err)
 		}
 	}
-	kubePluginInitializer := NewPluginInitializer(
-		cloudConfig,
-		quotainstall.NewQuotaConfigurationForAdmission(),
-		exclusion.Excluded(),
-	)
 
-	return []admission.PluginInitializer{webhookPluginInitializer, kubePluginInitializer}, nil
+	return []admission.PluginInitializer{NewPluginInitializer(cloudConfig)}, nil
 }

pkg/kubeapiserver/admission/initializer.go

@@ -17,10 +17,7 @@ limitations under the License.
 package admission
 
 import (
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
-	"k8s.io/apiserver/pkg/admission/initializer"
-	quota "k8s.io/apiserver/pkg/quota/v1"
 )
 
 // TODO add a `WantsToRun` which takes a stopCh.  Might make it generic.
@@ -33,24 +30,14 @@ type WantsCloudConfig interface {
 // PluginInitializer is used for initialization of the Kubernetes specific admission plugins.
 type PluginInitializer struct {
 	cloudConfig []byte
-	quotaConfiguration         quota.Configuration
-	excludedAdmissionResources []schema.GroupResource
 }
 
 var _ admission.PluginInitializer = &PluginInitializer{}
 
 // NewPluginInitializer constructs new instance of PluginInitializer
-// TODO: switch these parameters to use the builder pattern or just make them
+func NewPluginInitializer(cloudConfig []byte) *PluginInitializer {
-// all public, this construction method is pointless boilerplate.
-func NewPluginInitializer(
-	cloudConfig []byte,
-	quotaConfiguration quota.Configuration,
-	excludedAdmissionResources []schema.GroupResource,
-) *PluginInitializer {
 	return &PluginInitializer{
 		cloudConfig: cloudConfig,
-		quotaConfiguration:         quotaConfiguration,
-		excludedAdmissionResources: excludedAdmissionResources,
 	}
 }
 
@@ -60,12 +47,4 @@ func (i *PluginInitializer) Initialize(plugin admission.Interface) {
 	if wants, ok := plugin.(WantsCloudConfig); ok {
 		wants.SetCloudConfig(i.cloudConfig)
 	}
-
-	if wants, ok := plugin.(initializer.WantsQuotaConfiguration); ok {
-		wants.SetQuotaConfiguration(i.quotaConfiguration)
-	}
-
-	if wants, ok := plugin.(initializer.WantsExcludedAdmissionResources); ok {
-		wants.SetExcludedAdmissionResources(i.excludedAdmissionResources)
-	}
 }

pkg/kubeapiserver/admission/initializer_test.go

@@ -20,9 +20,7 @@ import (
 	"context"
 	"testing"
 
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
-	quota "k8s.io/apiserver/pkg/quota/v1"
 )
 
 type doNothingAdmission struct{}
@@ -33,10 +31,6 @@ func (doNothingAdmission) Admit(ctx context.Context, a admission.Attributes, o a
 func (doNothingAdmission) Handles(o admission.Operation) bool { return false }
 func (doNothingAdmission) Validate() error                    { return nil }
 
-type doNothingPluginInitialization struct{}
-
-func (doNothingPluginInitialization) ValidateInitialization() error { return nil }
-
 type WantsCloudConfigAdmissionPlugin struct {
 	doNothingAdmission
 	cloudConfig []byte
@@ -48,7 +42,7 @@ func (p *WantsCloudConfigAdmissionPlugin) SetCloudConfig(cloudConfig []byte) {
 
 func TestCloudConfigAdmissionPlugin(t *testing.T) {
 	cloudConfig := []byte("cloud-configuration")
-	initializer := NewPluginInitializer(cloudConfig, nil, nil)
+	initializer := NewPluginInitializer(cloudConfig)
 	wantsCloudConfigAdmission := &WantsCloudConfigAdmissionPlugin{}
 	initializer.Initialize(wantsCloudConfigAdmission)
 
@@ -56,30 +50,3 @@ func TestCloudConfigAdmissionPlugin(t *testing.T) {
 		t.Errorf("Expected cloud config to be initialized but found nil")
 	}
 }
-
-type doNothingQuotaConfiguration struct{}
-
-func (doNothingQuotaConfiguration) IgnoredResources() map[schema.GroupResource]struct{} { return nil }
-
-func (doNothingQuotaConfiguration) Evaluators() []quota.Evaluator { return nil }
-
-type WantsQuotaConfigurationAdmissionPlugin struct {
-	doNothingAdmission
-	doNothingPluginInitialization
-	config quota.Configuration
-}
-
-func (p *WantsQuotaConfigurationAdmissionPlugin) SetQuotaConfiguration(config quota.Configuration) {
-	p.config = config
-}
-
-func TestQuotaConfigurationAdmissionPlugin(t *testing.T) {
-	config := doNothingQuotaConfiguration{}
-	initializer := NewPluginInitializer(nil, config, nil)
-	wantsQuotaConfigurationAdmission := &WantsQuotaConfigurationAdmissionPlugin{}
-	initializer.Initialize(wantsQuotaConfigurationAdmission)
-
-	if wantsQuotaConfigurationAdmission.config == nil {
-		t.Errorf("Expected quota configuration to be initialized but found nil")
-	}
-}

plugin/pkg/admission/gc/gc_admission_test.go

@@ -28,15 +28,16 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/admission"
+	apiserveradmission "k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/admission/initializer"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	fakediscovery "k8s.io/client-go/discovery/fake"
 	"k8s.io/client-go/restmapper"
 	coretesting "k8s.io/client-go/testing"
+
 	api "k8s.io/kubernetes/pkg/apis/core"
-	kubeadmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
+	controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission"
 )
 
 type fakeAuthorizer struct{}
@@ -111,7 +112,7 @@ func newGCPermissionsEnforcement() (*gcPermissionsEnforcement, error) {
 		},
 	}
 	gcAdmit := &gcPermissionsEnforcement{
-		Handler:   admission.NewHandler(admission.Create, admission.Update),
+		Handler:   apiserveradmission.NewHandler(apiserveradmission.Create, apiserveradmission.Update),
 		whiteList: whiteList,
 	}
 
@@ -138,9 +139,8 @@ func newGCPermissionsEnforcement() (*gcPermissionsEnforcement, error) {
 	}
 	restMapper := restmapper.NewDiscoveryRESTMapper(restMapperRes)
 	genericPluginInitializer := initializer.New(nil, nil, nil, fakeAuthorizer{}, nil, nil, restMapper)
-
+	pluginInitializer := controlplaneadmission.NewPluginInitializer(nil, nil)
-	pluginInitializer := kubeadmission.NewPluginInitializer(nil, nil, nil)
+	initializersChain := apiserveradmission.PluginInitializers{}
-	initializersChain := admission.PluginInitializers{}
 	initializersChain = append(initializersChain, genericPluginInitializer)
 	initializersChain = append(initializersChain, pluginInitializer)
 
@@ -349,14 +349,14 @@ func TestGCAdmission(t *testing.T) {
 				t.Error(err)
 			}
 
-			operation := admission.Create
+			operation := apiserveradmission.Create
 			var options runtime.Object = &metav1.CreateOptions{}
 			if tc.oldObj != nil {
-				operation = admission.Update
+				operation = apiserveradmission.Update
 				options = &metav1.UpdateOptions{}
 			}
 			user := &user.DefaultInfo{Name: tc.username}
-			attributes := admission.NewAttributesRecord(tc.newObj, tc.oldObj, schema.GroupVersionKind{}, metav1.NamespaceDefault, "foo", tc.resource, tc.subresource, operation, options, false, user)
+			attributes := apiserveradmission.NewAttributesRecord(tc.newObj, tc.oldObj, schema.GroupVersionKind{}, metav1.NamespaceDefault, "foo", tc.resource, tc.subresource, operation, options, false, user)
 
 			err = gcAdmit.Validate(context.TODO(), attributes, nil)
 			if !tc.checkError(err) {
@@ -668,14 +668,14 @@ func TestBlockOwnerDeletionAdmission(t *testing.T) {
 				gcAdmit.restMapper = tc.restMapperOverride
 			}
 
-			operation := admission.Create
+			operation := apiserveradmission.Create
 			var options runtime.Object = &metav1.CreateOptions{}
 			if tc.oldObj != nil {
-				operation = admission.Update
+				operation = apiserveradmission.Update
 				options = &metav1.UpdateOptions{}
 			}
 			user := &user.DefaultInfo{Name: tc.username}
-			attributes := admission.NewAttributesRecord(tc.newObj, tc.oldObj, schema.GroupVersionKind{}, metav1.NamespaceDefault, "foo", tc.resource, tc.subresource, operation, options, false, user)
+			attributes := apiserveradmission.NewAttributesRecord(tc.newObj, tc.oldObj, schema.GroupVersionKind{}, metav1.NamespaceDefault, "foo", tc.resource, tc.subresource, operation, options, false, user)
 
 			err = gcAdmit.Validate(context.TODO(), attributes, nil)
 			if !tc.checkError(err) {

plugin/pkg/admission/resourcequota/admission_test.go

@@ -37,7 +37,7 @@ import (
 	testcore "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/cache"
 	api "k8s.io/kubernetes/pkg/apis/core"
-	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
+	controlplaneadmission "k8s.io/kubernetes/pkg/controlplane/apiserver/admission"
 	"k8s.io/kubernetes/pkg/quota/v1/install"
 )
 
@@ -115,7 +115,7 @@ func createHandlerWithConfig(kubeClient kubernetes.Interface, informerFactory in
 
 	initializers := admission.PluginInitializers{
 		genericadmissioninitializer.New(kubeClient, nil, informerFactory, nil, nil, stopCh, nil),
-		kubeapiserveradmission.NewPluginInitializer(nil, quotaConfiguration, nil),
+		controlplaneadmission.NewPluginInitializer(quotaConfiguration, nil),
 	}
 	initializers.Initialize(handler)