diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index c4ce92cf48f..426ac4f3a92 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -299,7 +299,7 @@ if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then fi # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely. -ADMISSION_CONTROL="${KUBE_ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota}" +ADMISSION_CONTROL="${KUBE_ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota,GenericAdmissionWebhook}" # Optional: if set to true kube-up will automatically check for existing resources and clean them up. KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false} diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index c3a476d92c4..c90196d92f0 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -419,7 +419,7 @@ function start_apiserver { fi # Admission Controllers to invoke prior to persisting objects in cluster - ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount${security_admission},DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,GenericAdmissionWebhook + ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount${security_admission},DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota # This is the default dir and filename where the apiserver will generate a self-signed cert # which should be able to be used as the CA to verify itself diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go index 2636c58df70..8408e89b946 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go @@ -298,5 +298,6 @@ func (a *GenericAdmissionWebhook) hookClient(h *v1alpha1.ExternalAdmissionHook) cfg.TLSClientConfig.ServerName = serverName cfg.TLSClientConfig.CAData = h.ClientConfig.CABundle cfg.ContentConfig.NegotiatedSerializer = a.negotiatedSerializer + cfg.ContentConfig.ContentType = runtime.ContentTypeJSON return rest.UnversionedRESTClientFor(cfg) } diff --git a/test/e2e/apimachinery/aggregator.go b/test/e2e/apimachinery/aggregator.go index 5a3ee164ce8..319ba02e30d 100644 --- a/test/e2e/apimachinery/aggregator.go +++ b/test/e2e/apimachinery/aggregator.go @@ -18,12 +18,9 @@ package apimachinery import ( "crypto/rand" - "crypto/x509" "encoding/json" "fmt" - "io/ioutil" "math/big" - "os" "strings" "time" @@ -38,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/discovery" - "k8s.io/client-go/util/cert" apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" rbacapi "k8s.io/kubernetes/pkg/apis/rbac" utilversion "k8s.io/kubernetes/pkg/util/version" @@ -48,12 +44,6 @@ import ( . "github.com/onsi/ginkgo" ) -type aggregatorContext struct { - apiserverCert []byte - apiserverKey []byte - apiserverSigningCert []byte -} - var serverAggregatorVersion = utilversion.MustParseSemantic("v1.7.0") var _ = SIGDescribe("Aggregator", func() { @@ -88,62 +78,6 @@ func cleanTest(f *framework.Framework) { _ = client.RbacV1beta1().ClusterRoleBindings().Delete("wardler:"+namespace+":anonymous", nil) } -func setupSampleAPIServerCert(namespaceName, serviceName string) *aggregatorContext { - aggregatorCertDir, err := ioutil.TempDir("", "test-e2e-aggregator") - if err != nil { - framework.Failf("Failed to create a temp dir for cert generation %v", err) - } - defer os.RemoveAll(aggregatorCertDir) - apiserverSigningKey, err := cert.NewPrivateKey() - if err != nil { - framework.Failf("Failed to create CA private key for apiserver %v", err) - } - apiserverSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-sampleapiserver-ca"}, apiserverSigningKey) - if err != nil { - framework.Failf("Failed to create CA cert for apiserver %v", err) - } - apiserverCACertFile, err := ioutil.TempFile(aggregatorCertDir, "apiserver-ca.crt") - if err != nil { - framework.Failf("Failed to create a temp file for ca cert generation %v", err) - } - if err := ioutil.WriteFile(apiserverCACertFile.Name(), cert.EncodeCertPEM(apiserverSigningCert), 0644); err != nil { - framework.Failf("Failed to write CA cert for apiserver %v", err) - } - apiserverKey, err := cert.NewPrivateKey() - if err != nil { - framework.Failf("Failed to create private key for apiserver %v", err) - } - apiserverCert, err := cert.NewSignedCert( - cert.Config{ - CommonName: serviceName + "." + namespaceName + ".svc", - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - }, - apiserverKey, apiserverSigningCert, apiserverSigningKey, - ) - if err != nil { - framework.Failf("Failed to create cert for apiserver %v", err) - } - apiserverCertFile, err := ioutil.TempFile(aggregatorCertDir, "apiserver.crt") - if err != nil { - framework.Failf("Failed to create a temp file for cert generation %v", err) - } - apiserverKeyFile, err := ioutil.TempFile(aggregatorCertDir, "apiserver.key") - if err != nil { - framework.Failf("Failed to create a temp file for key generation %v", err) - } - if err := ioutil.WriteFile(apiserverCertFile.Name(), cert.EncodeCertPEM(apiserverCert), 0600); err != nil { - framework.Failf("Failed to write cert file for apiserver %v", err) - } - if err := ioutil.WriteFile(apiserverKeyFile.Name(), cert.EncodePrivateKeyPEM(apiserverKey), 0644); err != nil { - framework.Failf("Failed to write key file for apiserver %v", err) - } - return &aggregatorContext{ - apiserverCert: cert.EncodeCertPEM(apiserverCert), - apiserverKey: cert.EncodePrivateKeyPEM(apiserverKey), - apiserverSigningCert: cert.EncodeCertPEM(apiserverSigningCert), - } -} - // A basic test if the sample-apiserver code from 1.7 and compiled against 1.7 // will work on the current Aggregator/API-Server. func TestSampleAPIServer(f *framework.Framework, image string) { @@ -154,7 +88,7 @@ func TestSampleAPIServer(f *framework.Framework, image string) { aggrclient := f.AggregatorClient namespace := f.Namespace.Name - context := setupSampleAPIServerCert(namespace, "sample-api") + context := setupServerCert(namespace, "sample-api") if framework.ProviderIs("gke") { // kubectl create clusterrolebinding user-cluster-admin-binding --clusterrole=cluster-admin --user=user@domain.com authenticated := rbacv1beta1.Subject{Kind: rbacv1beta1.GroupKind, Name: user.AllAuthenticated} @@ -172,8 +106,8 @@ func TestSampleAPIServer(f *framework.Framework, image string) { }, Type: v1.SecretTypeOpaque, Data: map[string][]byte{ - "tls.crt": context.apiserverCert, - "tls.key": context.apiserverKey, + "tls.crt": context.cert, + "tls.key": context.key, }, } _, err := client.CoreV1().Secrets(namespace).Create(secret) @@ -349,7 +283,7 @@ func TestSampleAPIServer(f *framework.Framework, image string) { }, Group: "wardle.k8s.io", Version: "v1alpha1", - CABundle: context.apiserverSigningCert, + CABundle: context.signingCert, GroupPriorityMinimum: 2000, VersionPriority: 200, }, diff --git a/test/e2e/apimachinery/certs.go b/test/e2e/apimachinery/certs.go new file mode 100644 index 00000000000..26c8e06e4d9 --- /dev/null +++ b/test/e2e/apimachinery/certs.go @@ -0,0 +1,90 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + "crypto/x509" + "io/ioutil" + "os" + + "k8s.io/client-go/util/cert" + "k8s.io/kubernetes/test/e2e/framework" +) + +type certContext struct { + cert []byte + key []byte + signingCert []byte +} + +// Setup the server cert. For example, user apiservers and admission webhooks +// can use the cert to prove their identify to the kube-apiserver +func setupServerCert(namespaceName, serviceName string) *certContext { + certDir, err := ioutil.TempDir("", "test-e2e-server-cert") + if err != nil { + framework.Failf("Failed to create a temp dir for cert generation %v", err) + } + defer os.RemoveAll(certDir) + signingKey, err := cert.NewPrivateKey() + if err != nil { + framework.Failf("Failed to create CA private key %v", err) + } + signingCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-server-cert-ca"}, signingKey) + if err != nil { + framework.Failf("Failed to create CA cert for apiserver %v", err) + } + caCertFile, err := ioutil.TempFile(certDir, "ca.crt") + if err != nil { + framework.Failf("Failed to create a temp file for ca cert generation %v", err) + } + if err := ioutil.WriteFile(caCertFile.Name(), cert.EncodeCertPEM(signingCert), 0644); err != nil { + framework.Failf("Failed to write CA cert %v", err) + } + key, err := cert.NewPrivateKey() + if err != nil { + framework.Failf("Failed to create private key for %v", err) + } + signedCert, err := cert.NewSignedCert( + cert.Config{ + CommonName: serviceName + "." + namespaceName + ".svc", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }, + key, signingCert, signingKey, + ) + if err != nil { + framework.Failf("Failed to create cert%v", err) + } + certFile, err := ioutil.TempFile(certDir, "server.crt") + if err != nil { + framework.Failf("Failed to create a temp file for cert generation %v", err) + } + keyFile, err := ioutil.TempFile(certDir, "server.key") + if err != nil { + framework.Failf("Failed to create a temp file for key generation %v", err) + } + if err = ioutil.WriteFile(certFile.Name(), cert.EncodeCertPEM(signedCert), 0600); err != nil { + framework.Failf("Failed to write cert file %v", err) + } + if err = ioutil.WriteFile(keyFile.Name(), cert.EncodePrivateKeyPEM(key), 0644); err != nil { + framework.Failf("Failed to write key file %v", err) + } + return &certContext{ + cert: cert.EncodeCertPEM(signedCert), + key: cert.EncodePrivateKeyPEM(key), + signingCert: cert.EncodeCertPEM(signingCert), + } +} diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go new file mode 100644 index 00000000000..3f17210c0c5 --- /dev/null +++ b/test/e2e/apimachinery/webhook.go @@ -0,0 +1,294 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + "strings" + "time" + + "k8s.io/api/admissionregistration/v1alpha1" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + rbacv1beta1 "k8s.io/api/rbac/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + utilversion "k8s.io/kubernetes/pkg/util/version" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + _ "github.com/stretchr/testify/assert" +) + +const ( + secretName = "sample-webhook-secret" + deploymentName = "sample-webhook-deployment" + serviceName = "e2e-test-webhook" + roleBindingName = "webhook-auth-reader" + webhookConfigName = "e2e-test-webhook-config" +) + +var serverWebhookVersion = utilversion.MustParseSemantic("v1.8.0") + +var _ = SIGDescribe("AdmissionWebhook", func() { + f := framework.NewDefaultFramework("webhook") + framework.AddCleanupAction(func() { + cleanWebhookTest(f) + }) + + It("Should be able to deny pod creation", func() { + // Make sure the relevant provider supports admission webhook + framework.SkipUnlessServerVersionGTE(serverWebhookVersion, f.ClientSet.Discovery()) + framework.SkipUnlessProviderIs("gce", "gke") + + _, err := f.ClientSet.AdmissionregistrationV1alpha1().ExternalAdmissionHookConfigurations().List(metav1.ListOptions{}) + if errors.IsNotFound(err) { + framework.Skipf("dynamic configuration of webhooks requires the alpha admissionregistration.k8s.io group to be enabled") + } + + By("Setting up server cert") + namespaceName := f.Namespace.Name + context := setupServerCert(namespaceName, serviceName) + createAuthReaderRoleBinding(f, namespaceName) + // Note that in 1.9 we will have backwards incompatible change to + // admission webhooks, so the image will be updated to 1.9 sometime in + // the development 1.9 cycle. + deployWebhookAndService(f, "gcr.io/kubernetes-e2e-test-images/k8s-sample-admission-webhook-amd64:1.8v1", context) + registerWebhook(f, context) + testWebhook(f) + }) +}) + +func createAuthReaderRoleBinding(f *framework.Framework, namespace string) { + By("Create role binding to let webhook read extension-apiserver-authentication") + client := f.ClientSet + // Create the role binding to allow the webhook read the extension-apiserver-authentication configmap + _, err := client.RbacV1beta1().RoleBindings("kube-system").Create(&rbacv1beta1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingName, + Annotations: map[string]string{ + rbacv1beta1.AutoUpdateAnnotationKey: "true", + }, + }, + RoleRef: rbacv1beta1.RoleRef{ + APIGroup: "", + Kind: "Role", + Name: "extension-apiserver-authentication-reader", + }, + // Webhook uses the default service account. + Subjects: []rbacv1beta1.Subject{ + { + Kind: "ServiceAccount", + Name: "default", + Namespace: namespace, + }, + }, + }) + framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace) +} + +func deployWebhookAndService(f *framework.Framework, image string, context *certContext) { + By("Deploying the webhook pod") + + client := f.ClientSet + + // Creating the secret that contains the webhook's cert. + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + }, + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + "tls.crt": context.cert, + "tls.key": context.key, + }, + } + namespace := f.Namespace.Name + _, err := client.CoreV1().Secrets(namespace).Create(secret) + framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace) + + // Create the deployment of the webhook + podLabels := map[string]string{"app": "sample-webhook", "webhook": "true"} + replicas := int32(1) + zero := int64(0) + mounts := []v1.VolumeMount{ + { + Name: "webhook-certs", + ReadOnly: true, + MountPath: "/webhook.local.config/certificates", + }, + } + volumes := []v1.Volume{ + { + Name: "webhook-certs", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{SecretName: secretName}, + }, + }, + } + containers := []v1.Container{ + { + Name: "sample-webhook", + VolumeMounts: mounts, + Args: []string{ + "--tls-cert-file=/webhook.local.config/certificates/tls.crt", + "--tls-private-key-file=/webhook.local.config/certificates/tls.key", + "--alsologtostderr", + "-v=4", + "2>&1", + }, + Image: image, + }, + } + d := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + }, + Spec: extensions.DeploymentSpec{ + Replicas: &replicas, + Strategy: extensions.DeploymentStrategy{ + Type: extensions.RollingUpdateDeploymentStrategyType, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + Spec: v1.PodSpec{ + TerminationGracePeriodSeconds: &zero, + Containers: containers, + Volumes: volumes, + }, + }, + }, + } + deployment, err := client.ExtensionsV1beta1().Deployments(namespace).Create(d) + framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, namespace) + By("Wait for the deployment to be ready") + err = framework.WaitForDeploymentRevisionAndImage(client, namespace, deploymentName, "1", image) + framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace) + err = framework.WaitForDeploymentComplete(client, deployment) + framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentName, namespace) + + By("Deploying the webhook service") + + serviceLabels := map[string]string{"webhook": "true"} + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: serviceName, + Labels: map[string]string{"test": "webhook"}, + }, + Spec: v1.ServiceSpec{ + Selector: serviceLabels, + Ports: []v1.ServicePort{ + { + Protocol: "TCP", + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + _, err = client.CoreV1().Services(namespace).Create(service) + framework.ExpectNoError(err, "creating service %s in namespace %s", serviceName, namespace) + + By("Verifying the service has paired with the endpoint") + err = framework.WaitForServiceEndpointsNum(client, namespace, serviceName, 1, 1*time.Second, 30*time.Second) + framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceName, 1) +} + +func registerWebhook(f *framework.Framework, context *certContext) { + client := f.ClientSet + By("Registering the webhook via the AdmissionRegistration API") + + namespace := f.Namespace.Name + _, err := client.AdmissionregistrationV1alpha1().ExternalAdmissionHookConfigurations().Create(&v1alpha1.ExternalAdmissionHookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: webhookConfigName, + }, + ExternalAdmissionHooks: []v1alpha1.ExternalAdmissionHook{ + { + Name: "e2e-test-webhook.k8s.io", + Rules: []v1alpha1.RuleWithOperations{{ + Operations: []v1alpha1.OperationType{v1alpha1.Create}, + Rule: v1alpha1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }}, + ClientConfig: v1alpha1.AdmissionHookClientConfig{ + Service: v1alpha1.ServiceReference{ + Namespace: namespace, + Name: serviceName, + }, + CABundle: context.signingCert, + }, + }, + }, + }) + framework.ExpectNoError(err, "registering webhook config %s with namespace %s", webhookConfigName, namespace) + + // The webhook configuration is honored in 1s. + time.Sleep(2 * time.Second) +} + +func testWebhook(f *framework.Framework) { + By("create a pod that should be denied by the webhook") + client := f.ClientSet + // Creating the pod, the request should be rejected + pod := nonCompliantPod(f) + _, err := client.CoreV1().Pods(f.Namespace.Name).Create(pod) + Expect(err).NotTo(BeNil()) + expectedErrMsg := "the pod contains unwanted container name" + if !strings.Contains(err.Error(), expectedErrMsg) { + framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) + } + // TODO: Test if webhook can detect pod with non-compliant metadata. + // Currently metadata is lost because webhook uses the external version of + // the objects, and the apiserver sends the internal objects. +} + +func nonCompliantPod(f *framework.Framework) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "disallowed-pod", + Labels: map[string]string{ + "webhook-e2e-test": "disallow", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "webhook-disallow", + Image: framework.GetPauseImageName(f.ClientSet), + }, + }, + }, + } +} + +func cleanWebhookTest(f *framework.Framework) { + client := f.ClientSet + _ = client.AdmissionregistrationV1alpha1().ExternalAdmissionHookConfigurations().Delete(webhookConfigName, nil) + namespaceName := f.Namespace.Name + _ = client.CoreV1().Services(namespaceName).Delete(serviceName, nil) + _ = client.ExtensionsV1beta1().Deployments(namespaceName).Delete(deploymentName, nil) + _ = client.CoreV1().Secrets(namespaceName).Delete(secretName, nil) + _ = client.RbacV1beta1().RoleBindings("kube-system").Delete(roleBindingName, nil) +}