Merge pull request #61404 from hzxuzhonghu/fix-webhook-crd-bug

Automatic merge from submit-queue (batch tested with PRs 61404, 61025). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

bug fix: create/update CRD panic when mutating webhook configured

apiextentions apiserver use its own scheme to create a admission chain before creating the apiextensions apiserver config.
Fixes #61355

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

cmd/kube-apiserver/app/aggregator.go

@@ -32,6 +32,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/admission"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/healthz"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
@@ -49,11 +50,27 @@ import (
 	"k8s.io/kubernetes/pkg/master/controller/crdregistration"
 )
 
-func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, commandOptions *options.ServerRunOptions, externalInformers kubeexternalinformers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, proxyTransport *http.Transport) (*aggregatorapiserver.Config, error) {
+func createAggregatorConfig(
+	kubeAPIServerConfig genericapiserver.Config,
+	commandOptions *options.ServerRunOptions,
+	externalInformers kubeexternalinformers.SharedInformerFactory,
+	serviceResolver aggregatorapiserver.ServiceResolver,
+	proxyTransport *http.Transport,
+	pluginInitializers []admission.PluginInitializer,
+) (*aggregatorapiserver.Config, error) {
 	// make a shallow copy to let us twiddle a few things
 	// most of the config actually remains the same.  We only need to mess with a couple items related to the particulars of the aggregator
 	genericConfig := kubeAPIServerConfig
 
+	// override genericConfig.AdmissionControl with kube-aggregator's scheme,
+	// because aggregator apiserver should use its own scheme to convert its own resources.
+	commandOptions.Admission.ApplyTo(
+		&genericConfig,
+		externalInformers,
+		genericConfig.LoopbackClientConfig,
+		aggregatorscheme.Scheme,
+		pluginInitializers...)
+
 	// the aggregator doesn't wire these up.  It just delegates them to the kubeapiserver
 	genericConfig.EnableSwaggerUI = false
 	genericConfig.SwaggerConfig = nil

cmd/kube-apiserver/app/apiextensions.go

@@ -23,17 +23,32 @@ import (
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
 	apiextensionscmd "k8s.io/apiextensions-apiserver/pkg/cmd/server"
+	"k8s.io/apiserver/pkg/admission"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
 	kubeexternalinformers "k8s.io/client-go/informers"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 )
 
-func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, externalInformers kubeexternalinformers.SharedInformerFactory, commandOptions *options.ServerRunOptions) (*apiextensionsapiserver.Config, error) {
+func createAPIExtensionsConfig(
+	kubeAPIServerConfig genericapiserver.Config,
+	externalInformers kubeexternalinformers.SharedInformerFactory,
+	pluginInitializers []admission.PluginInitializer,
+	commandOptions *options.ServerRunOptions,
+) (*apiextensionsapiserver.Config, error) {
 	// make a shallow copy to let us twiddle a few things
 	// most of the config actually remains the same.  We only need to mess with a couple items related to the particulars of the apiextensions
 	genericConfig := kubeAPIServerConfig
 
+	// override genericConfig.AdmissionControl with apiextensions' scheme,
+	// because apiextentions apiserver should use its own scheme to convert resources.
+	commandOptions.Admission.ApplyTo(
+		&genericConfig,
+		externalInformers,
+		genericConfig.LoopbackClientConfig,
+		apiextensionsapiserver.Scheme,
+		pluginInitializers...)
+
 	// copy the etcd options so we don't mutate originals.
 	etcdOptions := *commandOptions.Etcd
 	etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion)

cmd/kube-apiserver/app/server.go

@@ -148,13 +148,13 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc
 		return nil, err
 	}
 
-	kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := CreateKubeAPIServerConfig(runOptions, nodeTunneler, proxyTransport)
+	kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(runOptions, nodeTunneler, proxyTransport)
 	if err != nil {
 		return nil, err
 	}
 
 	// If additional API servers are added, they should be gated.
-	apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, versionedInformers, runOptions)
+	apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, versionedInformers, pluginInitializer, runOptions)
 	if err != nil {
 		return nil, err
 	}
@@ -189,7 +189,7 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc
 	apiExtensionsServer.GenericAPIServer.PrepareRun()
 
 	// aggregator comes last in the chain
-	aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, runOptions, versionedInformers, serviceResolver, proxyTransport)
+	aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, runOptions, versionedInformers, serviceResolver, proxyTransport, pluginInitializer)
 	if err != nil {
 		return nil, err
 	}
@@ -269,25 +269,40 @@ func CreateNodeDialer(s *options.ServerRunOptions) (tunneler.Tunneler, *http.Tra
 }
 
 // CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them
-func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunneler.Tunneler, proxyTransport *http.Transport) (*master.Config, informers.SharedInformerFactory, clientgoinformers.SharedInformerFactory, *kubeserver.InsecureServingInfo, aggregatorapiserver.ServiceResolver, error) {
+func CreateKubeAPIServerConfig(
+	s *options.ServerRunOptions,
+	nodeTunneler tunneler.Tunneler,
+	proxyTransport *http.Transport,
+) (
+	config *master.Config,
+	sharedInformers informers.SharedInformerFactory,
+	versionedInformers clientgoinformers.SharedInformerFactory,
+	insecureServingInfo *kubeserver.InsecureServingInfo,
+	serviceResolver aggregatorapiserver.ServiceResolver,
+	pluginInitializers []admission.PluginInitializer,
+	lastErr error,
+) {
 	// set defaults in the options before trying to create the generic config
-	if err := defaultOptions(s); err != nil {
+	if lastErr = defaultOptions(s); lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
 
 	// validate options
 	if errs := s.Validate(); len(errs) != 0 {
-		return nil, nil, nil, nil, nil, utilerrors.NewAggregate(errs)
+		lastErr = utilerrors.NewAggregate(errs)
+		return
 	}
 
-	genericConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := BuildGenericConfig(s, proxyTransport)
+	var genericConfig *genericapiserver.Config
-	if err != nil {
+	genericConfig, sharedInformers, versionedInformers, insecureServingInfo, serviceResolver, pluginInitializers, lastErr = BuildGenericConfig(s, proxyTransport)
-		return nil, nil, nil, nil, nil, err
+	if lastErr != nil {
+		return
 	}
 
 	if _, port, err := net.SplitHostPort(s.Etcd.StorageConfig.ServerList[0]); err == nil && port != "0" && len(port) != 0 {
 		if err := utilwait.PollImmediate(etcdRetryInterval, etcdRetryLimit*etcdRetryInterval, preflight.EtcdConnection{ServerList: s.Etcd.StorageConfig.ServerList}.CheckEtcdServers); err != nil {
-			return nil, nil, nil, nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err)
+			lastErr = fmt.Errorf("error waiting for etcd connection: %v", err)
+			return
 		}
 	}
 
@@ -302,23 +317,23 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
 		PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
 	})
 
-	serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
+	serviceIPRange, apiServerServiceIP, lastErr := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
-	if err != nil {
+	if lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
 
-	storageFactory, err := BuildStorageFactory(s, genericConfig.MergedResourceConfig)
+	storageFactory, lastErr := BuildStorageFactory(s, genericConfig.MergedResourceConfig)
-	if err != nil {
+	if lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
 
-	clientCA, err := readCAorNil(s.Authentication.ClientCert.ClientCA)
+	clientCA, lastErr := readCAorNil(s.Authentication.ClientCert.ClientCA)
-	if err != nil {
+	if lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
-	requestHeaderProxyCA, err := readCAorNil(s.Authentication.RequestHeader.ClientCAFile)
+	requestHeaderProxyCA, lastErr := readCAorNil(s.Authentication.RequestHeader.ClientCAFile)
-	if err != nil {
+	if lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
 
 	var issuer serviceaccount.TokenGenerator
@@ -327,23 +342,26 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
 		s.Authentication.ServiceAccounts.Issuer != "" ||
 		len(s.Authentication.ServiceAccounts.APIAudiences) > 0 {
 		if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
-			return nil, nil, nil, nil, nil, fmt.Errorf("the TokenRequest feature is not enabled but --service-account-signing-key-file, --service-account-issuer and/or --service-account-api-audiences flags were passed")
+			lastErr = fmt.Errorf("the TokenRequest feature is not enabled but --service-account-signing-key-file, --service-account-issuer and/or --service-account-api-audiences flags were passed")
+			return
 		}
 		if s.ServiceAccountSigningKeyFile == "" ||
 			s.Authentication.ServiceAccounts.Issuer == "" ||
 			len(s.Authentication.ServiceAccounts.APIAudiences) == 0 ||
 			len(s.Authentication.ServiceAccounts.KeyFiles) == 0 {
-			return nil, nil, nil, nil, nil, fmt.Errorf("service-account-signing-key-file, service-account-issuer, service-account-api-audiences and service-account-key-file should be specified together")
+			lastErr = fmt.Errorf("service-account-signing-key-file, service-account-issuer, service-account-api-audiences and service-account-key-file should be specified together")
+			return
 		}
 		sk, err := certutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile)
 		if err != nil {
-			return nil, nil, nil, nil, nil, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
+			lastErr = fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
+			return
 		}
 		issuer = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuer, sk)
 		apiAudiences = s.Authentication.ServiceAccounts.APIAudiences
 	}
 
-	config := &master.Config{
+	config = &master.Config{
 		GenericConfig: genericConfig,
 		ExtraConfig: master.ExtraConfig{
 			ClientCARegistrationHook: master.ClientCARegistrationHook{
@@ -384,34 +402,44 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
 		config.ExtraConfig.KubeletClientConfig.Dial = nodeTunneler.Dial
 	}
 
-	return config, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, nil
+	return
 }
 
 // BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it
-func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transport) (*genericapiserver.Config, informers.SharedInformerFactory, clientgoinformers.SharedInformerFactory, *kubeserver.InsecureServingInfo, aggregatorapiserver.ServiceResolver, error) {
+func BuildGenericConfig(
-	genericConfig := genericapiserver.NewConfig(legacyscheme.Codecs)
+	s *options.ServerRunOptions,
-	if err := s.GenericServerRunOptions.ApplyTo(genericConfig); err != nil {
+	proxyTransport *http.Transport,
-		return nil, nil, nil, nil, nil, err
+) (
+	genericConfig *genericapiserver.Config,
+	sharedInformers informers.SharedInformerFactory,
+	versionedInformers clientgoinformers.SharedInformerFactory,
+	insecureServingInfo *kubeserver.InsecureServingInfo,
+	serviceResolver aggregatorapiserver.ServiceResolver,
+	pluginInitializers []admission.PluginInitializer,
+	lastErr error,
+) {
+	genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
+	if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {
+		return
 	}
 
-	insecureServingOptions, err := s.InsecureServing.ApplyTo(genericConfig)
+	if insecureServingInfo, lastErr = s.InsecureServing.ApplyTo(genericConfig); lastErr != nil {
-	if err != nil {
+		return
-		return nil, nil, nil, nil, nil, err
 	}
-	if err := s.SecureServing.ApplyTo(genericConfig); err != nil {
+	if lastErr = s.SecureServing.ApplyTo(genericConfig); lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
-	if err := s.Authentication.ApplyTo(genericConfig); err != nil {
+	if lastErr = s.Authentication.ApplyTo(genericConfig); lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
-	if err := s.Audit.ApplyTo(genericConfig); err != nil {
+	if lastErr = s.Audit.ApplyTo(genericConfig); lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
-	if err := s.Features.ApplyTo(genericConfig); err != nil {
+	if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
-	if err := s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Registry); err != nil {
+	if lastErr = s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Registry); lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
 
 	genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, legacyscheme.Scheme)
@@ -426,12 +454,12 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 	kubeVersion := version.Get()
 	genericConfig.Version = &kubeVersion
 
-	storageFactory, err := BuildStorageFactory(s, genericConfig.MergedResourceConfig)
+	storageFactory, lastErr := BuildStorageFactory(s, genericConfig.MergedResourceConfig)
-	if err != nil {
+	if lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
-	if err := s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); err != nil {
+	if lastErr = s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil {
-		return nil, nil, nil, nil, nil, err
+		return
 	}
 
 	// Use protobufs for self-communication.
@@ -444,7 +472,8 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 	if err != nil {
 		kubeAPIVersions := os.Getenv("KUBE_API_VERSIONS")
 		if len(kubeAPIVersions) == 0 {
-			return nil, nil, nil, nil, nil, fmt.Errorf("failed to create clientset: %v", err)
+			lastErr = fmt.Errorf("failed to create clientset: %v", err)
+			return
 		}
 
 		// KUBE_API_VERSIONS is used in test-update-storage-objects.sh, disabling a number of API
@@ -456,14 +485,14 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 	}
 
 	kubeClientConfig := genericConfig.LoopbackClientConfig
-	sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)
+	sharedInformers = informers.NewSharedInformerFactory(client, 10*time.Minute)
 	clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig)
 	if err != nil {
-		return nil, nil, nil, nil, nil, fmt.Errorf("failed to create real external clientset: %v", err)
+		lastErr = fmt.Errorf("failed to create real external clientset: %v", err)
+		return
 	}
-	versionedInformers := clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
+	versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
 
-	var serviceResolver aggregatorapiserver.ServiceResolver
 	if s.EnableAggregatorRouting {
 		serviceResolver = aggregatorapiserver.NewEndpointServiceResolver(
 			versionedInformers.Core().V1().Services().Lister(),
@@ -477,12 +506,14 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 
 	genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, clientgoExternalClient, sharedInformers)
 	if err != nil {
-		return nil, nil, nil, nil, nil, fmt.Errorf("invalid authentication config: %v", err)
+		lastErr = fmt.Errorf("invalid authentication config: %v", err)
+		return
 	}
 
 	genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, sharedInformers, versionedInformers)
 	if err != nil {
-		return nil, nil, nil, nil, nil, fmt.Errorf("invalid authorization config: %v", err)
+		lastErr = fmt.Errorf("invalid authorization config: %v", err)
+		return
 	}
 	if !sets.NewString(s.Authorization.Modes...).Has(modes.ModeRBAC) {
 		genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName)
@@ -511,7 +542,7 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 			},
 		}
 	}
-	pluginInitializers, err := BuildAdmissionPluginInitializers(
+	pluginInitializers, err = BuildAdmissionPluginInitializers(
 		s,
 		client,
 		sharedInformers,
@@ -519,7 +550,8 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 		webhookAuthResolverWrapper,
 	)
 	if err != nil {
-		return nil, nil, nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %v", err)
+		lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err)
+		return
 	}
 
 	err = s.Admission.ApplyTo(
@@ -529,14 +561,20 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 		legacyscheme.Scheme,
 		pluginInitializers...)
 	if err != nil {
-		return nil, nil, nil, nil, nil, fmt.Errorf("failed to initialize admission: %v", err)
+		lastErr = fmt.Errorf("failed to initialize admission: %v", err)
 	}
 
-	return genericConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, nil
+	return
 }
 
 // BuildAdmissionPluginInitializers constructs the admission plugin initializer
-func BuildAdmissionPluginInitializers(s *options.ServerRunOptions, client internalclientset.Interface, sharedInformers informers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, webhookAuthWrapper webhookconfig.AuthenticationInfoResolverWrapper) ([]admission.PluginInitializer, error) {
+func BuildAdmissionPluginInitializers(
+	s *options.ServerRunOptions,
+	client internalclientset.Interface,
+	sharedInformers informers.SharedInformerFactory,
+	serviceResolver aggregatorapiserver.ServiceResolver,
+	webhookAuthWrapper webhookconfig.AuthenticationInfoResolverWrapper,
+) ([]admission.PluginInitializer, error) {
 	var cloudConfig []byte
 
 	if s.CloudProvider.CloudConfigFile != "" {

test/e2e/apimachinery/webhook.go

@@ -27,6 +27,7 @@ import (
 	extensions "k8s.io/api/extensions/v1beta1"
 	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -161,6 +162,13 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
 		testMutatingCRDWebhook(f, testcrd.Crd, testcrd.DynamicClient)
 	})
 
+	It("Should deny crd creation", func() {
+		crdWebhookCleanup := registerValidatingWebhookForCRD(f, context)
+		defer crdWebhookCleanup()
+
+		testCRDDenyWebhook(f)
+	})
+
 	// TODO: add more e2e tests for mutating webhooks
 	// 1. mutating webhook that mutates pod
 	// 2. mutating webhook that sends empty patch
@@ -1121,3 +1129,92 @@ func testMutatingCRDWebhook(f *framework.Framework, crd *apiextensionsv1beta1.Cu
 		framework.Failf("\nexpected %#v\n, got %#v\n", expectedCRData, mutatedCR.Object["data"])
 	}
 }
+
+func registerValidatingWebhookForCRD(f *framework.Framework, context *certContext) func() {
+	client := f.ClientSet
+	By("Registering the crd webhook via the AdmissionRegistration API")
+
+	namespace := f.Namespace.Name
+	configName := webhookConfigName
+	_, err := client.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Create(&v1beta1.ValidatingWebhookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: configName,
+		},
+		Webhooks: []v1beta1.Webhook{
+			{
+				Name: "deny-crd.k8s.io",
+				Rules: []v1beta1.RuleWithOperations{{
+					Operations: []v1beta1.OperationType{v1beta1.Create},
+					Rule: v1beta1.Rule{
+						APIGroups:   []string{"apiextensions.k8s.io"},
+						APIVersions: []string{"*"},
+						Resources:   []string{"customresourcedefinitions"},
+					},
+				}},
+				ClientConfig: v1beta1.WebhookClientConfig{
+					Service: &v1beta1.ServiceReference{
+						Namespace: namespace,
+						Name:      serviceName,
+						Path:      strPtr("/always-deny"),
+					},
+					CABundle: context.signingCert,
+				},
+			},
+		},
+	})
+	framework.ExpectNoError(err, "registering crd webhook config %s with namespace %s", configName, namespace)
+
+	// The webhook configuration is honored in 10s.
+	time.Sleep(10 * time.Second)
+	return func() {
+		client.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Delete(configName, nil)
+	}
+}
+
+func testCRDDenyWebhook(f *framework.Framework) {
+	By("Creating a custom resource definition that should be denied by the webhook")
+	name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny")
+	kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny")
+	group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName)
+	apiVersion := "v1"
+	testcrd := &framework.TestCrd{
+		Name:       name,
+		Kind:       kind,
+		ApiGroup:   group,
+		ApiVersion: apiVersion,
+	}
+
+	// Creating a custom resource definition for use by assorted tests.
+	config, err := framework.LoadConfig()
+	if err != nil {
+		framework.Failf("failed to load config: %v", err)
+		return
+	}
+	apiExtensionClient, err := crdclientset.NewForConfig(config)
+	if err != nil {
+		framework.Failf("failed to initialize apiExtensionClient: %v", err)
+		return
+	}
+	crd := &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{Name: testcrd.GetMetaName()},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   testcrd.ApiGroup,
+			Version: testcrd.ApiVersion,
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural:   testcrd.GetPluralName(),
+				Singular: testcrd.Name,
+				Kind:     testcrd.Kind,
+				ListKind: testcrd.GetListName(),
+			},
+			Scope: apiextensionsv1beta1.NamespaceScoped,
+		},
+	}
+
+	// create CRD
+	_, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
+	Expect(err).NotTo(BeNil())
+	expectedErrMsg := "this webhook denies all requests"
+	if !strings.Contains(err.Error(), expectedErrMsg) {
+		framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error())
+	}
+}

test/integration/etcd/etcd_storage_path_test.go

@@ -740,7 +740,7 @@ func startRealMasterOrDie(t *testing.T, certDir string) (*allClient, clientv3.KV
 		if err != nil {
 			t.Fatal(err)
 		}
-		kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport)
+		kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport)
 		if err != nil {
 			t.Fatal(err)
 		}

test/integration/examples/apiserver_test.go

@@ -115,7 +115,7 @@ func TestAggregatedAPIServer(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport)
+		kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport)
 		if err != nil {
 			t.Fatal(err)
 		}

test/integration/tls/ciphers_test.go

@@ -66,7 +66,7 @@ func runBasicSecureAPIServer(t *testing.T, ciphers []string) (uint32, error) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport)
+		kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport)
 		if err != nil {
 			t.Fatal(err)
 		}