From 4c26831147dd6fa00ea813db433f5f8bd432d184 Mon Sep 17 00:00:00 2001 From: hzxuzhonghu Date: Tue, 20 Mar 2018 20:13:48 +0800 Subject: [PATCH 1/2] apiserver's webhook admission use its own scheme --- cmd/kube-apiserver/app/aggregator.go | 19 +- cmd/kube-apiserver/app/apiextensions.go | 17 +- cmd/kube-apiserver/app/server.go | 162 +++++++++++------- .../etcd/etcd_storage_path_test.go | 2 +- test/integration/examples/apiserver_test.go | 2 +- test/integration/tls/ciphers_test.go | 2 +- 6 files changed, 137 insertions(+), 67 deletions(-) diff --git a/cmd/kube-apiserver/app/aggregator.go b/cmd/kube-apiserver/app/aggregator.go index 6cc58a24815..9a1f5ad22a6 100644 --- a/cmd/kube-apiserver/app/aggregator.go +++ b/cmd/kube-apiserver/app/aggregator.go @@ -32,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/admission" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/healthz" genericoptions "k8s.io/apiserver/pkg/server/options" @@ -49,11 +50,27 @@ import ( "k8s.io/kubernetes/pkg/master/controller/crdregistration" ) -func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, commandOptions *options.ServerRunOptions, externalInformers kubeexternalinformers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, proxyTransport *http.Transport) (*aggregatorapiserver.Config, error) { +func createAggregatorConfig( + kubeAPIServerConfig genericapiserver.Config, + commandOptions *options.ServerRunOptions, + externalInformers kubeexternalinformers.SharedInformerFactory, + serviceResolver aggregatorapiserver.ServiceResolver, + proxyTransport *http.Transport, + pluginInitializers []admission.PluginInitializer, +) (*aggregatorapiserver.Config, error) { // make a shallow copy to let us twiddle a few things // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the aggregator genericConfig := kubeAPIServerConfig + // override genericConfig.AdmissionControl with kube-aggregator's scheme, + // because aggregator apiserver should use its own scheme to convert its own resources. + commandOptions.Admission.ApplyTo( + &genericConfig, + externalInformers, + genericConfig.LoopbackClientConfig, + aggregatorscheme.Scheme, + pluginInitializers...) + // the aggregator doesn't wire these up. It just delegates them to the kubeapiserver genericConfig.EnableSwaggerUI = false genericConfig.SwaggerConfig = nil diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go index 58399bbcd15..95edfe3d4e8 100644 --- a/cmd/kube-apiserver/app/apiextensions.go +++ b/cmd/kube-apiserver/app/apiextensions.go @@ -23,17 +23,32 @@ import ( "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" apiextensionscmd "k8s.io/apiextensions-apiserver/pkg/cmd/server" + "k8s.io/apiserver/pkg/admission" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" kubeexternalinformers "k8s.io/client-go/informers" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) -func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, externalInformers kubeexternalinformers.SharedInformerFactory, commandOptions *options.ServerRunOptions) (*apiextensionsapiserver.Config, error) { +func createAPIExtensionsConfig( + kubeAPIServerConfig genericapiserver.Config, + externalInformers kubeexternalinformers.SharedInformerFactory, + pluginInitializers []admission.PluginInitializer, + commandOptions *options.ServerRunOptions, +) (*apiextensionsapiserver.Config, error) { // make a shallow copy to let us twiddle a few things // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the apiextensions genericConfig := kubeAPIServerConfig + // override genericConfig.AdmissionControl with apiextensions' scheme, + // because apiextentions apiserver should use its own scheme to convert resources. + commandOptions.Admission.ApplyTo( + &genericConfig, + externalInformers, + genericConfig.LoopbackClientConfig, + apiextensionsapiserver.Scheme, + pluginInitializers...) + // copy the etcd options so we don't mutate originals. etcdOptions := *commandOptions.Etcd etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion) diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 8dbc5ac5a6d..f8e0c660abc 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -148,13 +148,13 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc return nil, err } - kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := CreateKubeAPIServerConfig(runOptions, nodeTunneler, proxyTransport) + kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(runOptions, nodeTunneler, proxyTransport) if err != nil { return nil, err } // If additional API servers are added, they should be gated. - apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, versionedInformers, runOptions) + apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, versionedInformers, pluginInitializer, runOptions) if err != nil { return nil, err } @@ -189,7 +189,7 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc apiExtensionsServer.GenericAPIServer.PrepareRun() // aggregator comes last in the chain - aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, runOptions, versionedInformers, serviceResolver, proxyTransport) + aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, runOptions, versionedInformers, serviceResolver, proxyTransport, pluginInitializer) if err != nil { return nil, err } @@ -269,25 +269,40 @@ func CreateNodeDialer(s *options.ServerRunOptions) (tunneler.Tunneler, *http.Tra } // CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them -func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunneler.Tunneler, proxyTransport *http.Transport) (*master.Config, informers.SharedInformerFactory, clientgoinformers.SharedInformerFactory, *kubeserver.InsecureServingInfo, aggregatorapiserver.ServiceResolver, error) { +func CreateKubeAPIServerConfig( + s *options.ServerRunOptions, + nodeTunneler tunneler.Tunneler, + proxyTransport *http.Transport, +) ( + config *master.Config, + sharedInformers informers.SharedInformerFactory, + versionedInformers clientgoinformers.SharedInformerFactory, + insecureServingInfo *kubeserver.InsecureServingInfo, + serviceResolver aggregatorapiserver.ServiceResolver, + pluginInitializers []admission.PluginInitializer, + lastErr error, +) { // set defaults in the options before trying to create the generic config - if err := defaultOptions(s); err != nil { - return nil, nil, nil, nil, nil, err + if lastErr = defaultOptions(s); lastErr != nil { + return } // validate options if errs := s.Validate(); len(errs) != 0 { - return nil, nil, nil, nil, nil, utilerrors.NewAggregate(errs) + lastErr = utilerrors.NewAggregate(errs) + return } - genericConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := BuildGenericConfig(s, proxyTransport) - if err != nil { - return nil, nil, nil, nil, nil, err + var genericConfig *genericapiserver.Config + genericConfig, sharedInformers, versionedInformers, insecureServingInfo, serviceResolver, pluginInitializers, lastErr = BuildGenericConfig(s, proxyTransport) + if lastErr != nil { + return } if _, port, err := net.SplitHostPort(s.Etcd.StorageConfig.ServerList[0]); err == nil && port != "0" && len(port) != 0 { if err := utilwait.PollImmediate(etcdRetryInterval, etcdRetryLimit*etcdRetryInterval, preflight.EtcdConnection{ServerList: s.Etcd.StorageConfig.ServerList}.CheckEtcdServers); err != nil { - return nil, nil, nil, nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err) + lastErr = fmt.Errorf("error waiting for etcd connection: %v", err) + return } } @@ -302,23 +317,23 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec, }) - serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange) - if err != nil { - return nil, nil, nil, nil, nil, err + serviceIPRange, apiServerServiceIP, lastErr := master.DefaultServiceIPRange(s.ServiceClusterIPRange) + if lastErr != nil { + return } - storageFactory, err := BuildStorageFactory(s, genericConfig.MergedResourceConfig) - if err != nil { - return nil, nil, nil, nil, nil, err + storageFactory, lastErr := BuildStorageFactory(s, genericConfig.MergedResourceConfig) + if lastErr != nil { + return } - clientCA, err := readCAorNil(s.Authentication.ClientCert.ClientCA) - if err != nil { - return nil, nil, nil, nil, nil, err + clientCA, lastErr := readCAorNil(s.Authentication.ClientCert.ClientCA) + if lastErr != nil { + return } - requestHeaderProxyCA, err := readCAorNil(s.Authentication.RequestHeader.ClientCAFile) - if err != nil { - return nil, nil, nil, nil, nil, err + requestHeaderProxyCA, lastErr := readCAorNil(s.Authentication.RequestHeader.ClientCAFile) + if lastErr != nil { + return } var issuer serviceaccount.TokenGenerator @@ -327,23 +342,26 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele s.Authentication.ServiceAccounts.Issuer != "" || len(s.Authentication.ServiceAccounts.APIAudiences) > 0 { if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) { - return nil, nil, nil, nil, nil, fmt.Errorf("the TokenRequest feature is not enabled but --service-account-signing-key-file, --service-account-issuer and/or --service-account-api-audiences flags were passed") + lastErr = fmt.Errorf("the TokenRequest feature is not enabled but --service-account-signing-key-file, --service-account-issuer and/or --service-account-api-audiences flags were passed") + return } if s.ServiceAccountSigningKeyFile == "" || s.Authentication.ServiceAccounts.Issuer == "" || len(s.Authentication.ServiceAccounts.APIAudiences) == 0 || len(s.Authentication.ServiceAccounts.KeyFiles) == 0 { - return nil, nil, nil, nil, nil, fmt.Errorf("service-account-signing-key-file, service-account-issuer, service-account-api-audiences and service-account-key-file should be specified together") + lastErr = fmt.Errorf("service-account-signing-key-file, service-account-issuer, service-account-api-audiences and service-account-key-file should be specified together") + return } sk, err := certutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile) if err != nil { - return nil, nil, nil, nil, nil, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) + lastErr = fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) + return } issuer = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuer, sk) apiAudiences = s.Authentication.ServiceAccounts.APIAudiences } - config := &master.Config{ + config = &master.Config{ GenericConfig: genericConfig, ExtraConfig: master.ExtraConfig{ ClientCARegistrationHook: master.ClientCARegistrationHook{ @@ -385,34 +403,44 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele config.ExtraConfig.KubeletClientConfig.Dial = nodeTunneler.Dial } - return config, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, nil + return } // BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it -func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transport) (*genericapiserver.Config, informers.SharedInformerFactory, clientgoinformers.SharedInformerFactory, *kubeserver.InsecureServingInfo, aggregatorapiserver.ServiceResolver, error) { - genericConfig := genericapiserver.NewConfig(legacyscheme.Codecs) - if err := s.GenericServerRunOptions.ApplyTo(genericConfig); err != nil { - return nil, nil, nil, nil, nil, err +func BuildGenericConfig( + s *options.ServerRunOptions, + proxyTransport *http.Transport, +) ( + genericConfig *genericapiserver.Config, + sharedInformers informers.SharedInformerFactory, + versionedInformers clientgoinformers.SharedInformerFactory, + insecureServingInfo *kubeserver.InsecureServingInfo, + serviceResolver aggregatorapiserver.ServiceResolver, + pluginInitializers []admission.PluginInitializer, + lastErr error, +) { + genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs) + if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil { + return } - insecureServingOptions, err := s.InsecureServing.ApplyTo(genericConfig) - if err != nil { - return nil, nil, nil, nil, nil, err + if insecureServingInfo, lastErr = s.InsecureServing.ApplyTo(genericConfig); lastErr != nil { + return } - if err := s.SecureServing.ApplyTo(genericConfig); err != nil { - return nil, nil, nil, nil, nil, err + if lastErr = s.SecureServing.ApplyTo(genericConfig); lastErr != nil { + return } - if err := s.Authentication.ApplyTo(genericConfig); err != nil { - return nil, nil, nil, nil, nil, err + if lastErr = s.Authentication.ApplyTo(genericConfig); lastErr != nil { + return } - if err := s.Audit.ApplyTo(genericConfig); err != nil { - return nil, nil, nil, nil, nil, err + if lastErr = s.Audit.ApplyTo(genericConfig); lastErr != nil { + return } - if err := s.Features.ApplyTo(genericConfig); err != nil { - return nil, nil, nil, nil, nil, err + if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil { + return } - if err := s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Registry); err != nil { - return nil, nil, nil, nil, nil, err + if lastErr = s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Registry); lastErr != nil { + return } genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, legacyscheme.Scheme) @@ -427,12 +455,12 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp kubeVersion := version.Get() genericConfig.Version = &kubeVersion - storageFactory, err := BuildStorageFactory(s, genericConfig.MergedResourceConfig) - if err != nil { - return nil, nil, nil, nil, nil, err + storageFactory, lastErr := BuildStorageFactory(s, genericConfig.MergedResourceConfig) + if lastErr != nil { + return } - if err := s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); err != nil { - return nil, nil, nil, nil, nil, err + if lastErr = s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil { + return } // Use protobufs for self-communication. @@ -445,7 +473,8 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp if err != nil { kubeAPIVersions := os.Getenv("KUBE_API_VERSIONS") if len(kubeAPIVersions) == 0 { - return nil, nil, nil, nil, nil, fmt.Errorf("failed to create clientset: %v", err) + lastErr = fmt.Errorf("failed to create clientset: %v", err) + return } // KUBE_API_VERSIONS is used in test-update-storage-objects.sh, disabling a number of API @@ -457,14 +486,14 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp } kubeClientConfig := genericConfig.LoopbackClientConfig - sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute) + sharedInformers = informers.NewSharedInformerFactory(client, 10*time.Minute) clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig) if err != nil { - return nil, nil, nil, nil, nil, fmt.Errorf("failed to create real external clientset: %v", err) + lastErr = fmt.Errorf("failed to create real external clientset: %v", err) + return } - versionedInformers := clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) + versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) - var serviceResolver aggregatorapiserver.ServiceResolver if s.EnableAggregatorRouting { serviceResolver = aggregatorapiserver.NewEndpointServiceResolver( versionedInformers.Core().V1().Services().Lister(), @@ -478,12 +507,14 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, clientgoExternalClient, sharedInformers) if err != nil { - return nil, nil, nil, nil, nil, fmt.Errorf("invalid authentication config: %v", err) + lastErr = fmt.Errorf("invalid authentication config: %v", err) + return } genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, sharedInformers, versionedInformers) if err != nil { - return nil, nil, nil, nil, nil, fmt.Errorf("invalid authorization config: %v", err) + lastErr = fmt.Errorf("invalid authorization config: %v", err) + return } if !sets.NewString(s.Authorization.Modes...).Has(modes.ModeRBAC) { genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName) @@ -512,7 +543,7 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp }, } } - pluginInitializers, err := BuildAdmissionPluginInitializers( + pluginInitializers, err = BuildAdmissionPluginInitializers( s, client, sharedInformers, @@ -520,7 +551,8 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp webhookAuthResolverWrapper, ) if err != nil { - return nil, nil, nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %v", err) + lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err) + return } err = s.Admission.ApplyTo( @@ -530,14 +562,20 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp legacyscheme.Scheme, pluginInitializers...) if err != nil { - return nil, nil, nil, nil, nil, fmt.Errorf("failed to initialize admission: %v", err) + lastErr = fmt.Errorf("failed to initialize admission: %v", err) } - return genericConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, nil + return } // BuildAdmissionPluginInitializers constructs the admission plugin initializer -func BuildAdmissionPluginInitializers(s *options.ServerRunOptions, client internalclientset.Interface, sharedInformers informers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, webhookAuthWrapper webhookconfig.AuthenticationInfoResolverWrapper) ([]admission.PluginInitializer, error) { +func BuildAdmissionPluginInitializers( + s *options.ServerRunOptions, + client internalclientset.Interface, + sharedInformers informers.SharedInformerFactory, + serviceResolver aggregatorapiserver.ServiceResolver, + webhookAuthWrapper webhookconfig.AuthenticationInfoResolverWrapper, +) ([]admission.PluginInitializer, error) { var cloudConfig []byte if s.CloudProvider.CloudConfigFile != "" { diff --git a/test/integration/etcd/etcd_storage_path_test.go b/test/integration/etcd/etcd_storage_path_test.go index 3c87d257098..396ace28a0e 100644 --- a/test/integration/etcd/etcd_storage_path_test.go +++ b/test/integration/etcd/etcd_storage_path_test.go @@ -740,7 +740,7 @@ func startRealMasterOrDie(t *testing.T, certDir string) (*allClient, clientv3.KV if err != nil { t.Fatal(err) } - kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) + kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) if err != nil { t.Fatal(err) } diff --git a/test/integration/examples/apiserver_test.go b/test/integration/examples/apiserver_test.go index 496e799f6e4..dc9d7847897 100644 --- a/test/integration/examples/apiserver_test.go +++ b/test/integration/examples/apiserver_test.go @@ -115,7 +115,7 @@ func TestAggregatedAPIServer(t *testing.T) { if err != nil { t.Fatal(err) } - kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) + kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) if err != nil { t.Fatal(err) } diff --git a/test/integration/tls/ciphers_test.go b/test/integration/tls/ciphers_test.go index 8d4ab5fea0f..45925719ca4 100644 --- a/test/integration/tls/ciphers_test.go +++ b/test/integration/tls/ciphers_test.go @@ -66,7 +66,7 @@ func runBasicSecureAPIServer(t *testing.T, ciphers []string) (uint32, error) { if err != nil { t.Fatal(err) } - kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) + kubeAPIServerConfig, sharedInformers, versionedInformers, _, _, _, err := app.CreateKubeAPIServerConfig(kubeAPIServerOptions, tunneler, proxyTransport) if err != nil { t.Fatal(err) } From 932a07299012c72e95655c101514f99fa4157409 Mon Sep 17 00:00:00 2001 From: hzxuzhonghu Date: Thu, 22 Mar 2018 20:33:06 +0800 Subject: [PATCH 2/2] add e2e case for crd webhook --- test/e2e/apimachinery/webhook.go | 97 ++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index a5a9a12daf9..e3ce39e7ea8 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -27,6 +27,7 @@ import ( extensions "k8s.io/api/extensions/v1beta1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -161,6 +162,13 @@ var _ = SIGDescribe("AdmissionWebhook", func() { testMutatingCRDWebhook(f, testcrd.Crd, testcrd.DynamicClient) }) + It("Should deny crd creation", func() { + crdWebhookCleanup := registerValidatingWebhookForCRD(f, context) + defer crdWebhookCleanup() + + testCRDDenyWebhook(f) + }) + // TODO: add more e2e tests for mutating webhooks // 1. mutating webhook that mutates pod // 2. mutating webhook that sends empty patch @@ -1121,3 +1129,92 @@ func testMutatingCRDWebhook(f *framework.Framework, crd *apiextensionsv1beta1.Cu framework.Failf("\nexpected %#v\n, got %#v\n", expectedCRData, mutatedCR.Object["data"]) } } + +func registerValidatingWebhookForCRD(f *framework.Framework, context *certContext) func() { + client := f.ClientSet + By("Registering the crd webhook via the AdmissionRegistration API") + + namespace := f.Namespace.Name + configName := webhookConfigName + _, err := client.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Create(&v1beta1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: configName, + }, + Webhooks: []v1beta1.Webhook{ + { + Name: "deny-crd.k8s.io", + Rules: []v1beta1.RuleWithOperations{{ + Operations: []v1beta1.OperationType{v1beta1.Create}, + Rule: v1beta1.Rule{ + APIGroups: []string{"apiextensions.k8s.io"}, + APIVersions: []string{"*"}, + Resources: []string{"customresourcedefinitions"}, + }, + }}, + ClientConfig: v1beta1.WebhookClientConfig{ + Service: &v1beta1.ServiceReference{ + Namespace: namespace, + Name: serviceName, + Path: strPtr("/always-deny"), + }, + CABundle: context.signingCert, + }, + }, + }, + }) + framework.ExpectNoError(err, "registering crd webhook config %s with namespace %s", configName, namespace) + + // The webhook configuration is honored in 10s. + time.Sleep(10 * time.Second) + return func() { + client.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Delete(configName, nil) + } +} + +func testCRDDenyWebhook(f *framework.Framework) { + By("Creating a custom resource definition that should be denied by the webhook") + name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny") + kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny") + group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) + apiVersion := "v1" + testcrd := &framework.TestCrd{ + Name: name, + Kind: kind, + ApiGroup: group, + ApiVersion: apiVersion, + } + + // Creating a custom resource definition for use by assorted tests. + config, err := framework.LoadConfig() + if err != nil { + framework.Failf("failed to load config: %v", err) + return + } + apiExtensionClient, err := crdclientset.NewForConfig(config) + if err != nil { + framework.Failf("failed to initialize apiExtensionClient: %v", err) + return + } + crd := &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: testcrd.GetMetaName()}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: testcrd.ApiGroup, + Version: testcrd.ApiVersion, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: testcrd.GetPluralName(), + Singular: testcrd.Name, + Kind: testcrd.Kind, + ListKind: testcrd.GetListName(), + }, + Scope: apiextensionsv1beta1.NamespaceScoped, + }, + } + + // create CRD + _, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) + Expect(err).NotTo(BeNil()) + expectedErrMsg := "this webhook denies all requests" + if !strings.Contains(err.Error(), expectedErrMsg) { + framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) + } +}