Add a e2e test for the admission webhook

cluster/gce/config-test.sh

@@ -299,7 +299,7 @@ if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then
 fi
 
 # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely.
-ADMISSION_CONTROL="${KUBE_ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota}"
+ADMISSION_CONTROL="${KUBE_ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota,GenericAdmissionWebhook}"
 
 # Optional: if set to true kube-up will automatically check for existing resources and clean them up.
 KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false}

hack/local-up-cluster.sh

@@ -419,7 +419,7 @@ function start_apiserver {
     fi
 
     # Admission Controllers to invoke prior to persisting objects in cluster
-    ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount${security_admission},DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,GenericAdmissionWebhook
+    ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount${security_admission},DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota
     # This is the default dir and filename where the apiserver will generate a self-signed cert
     # which should be able to be used as the CA to verify itself
 

staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go

@@ -298,5 +298,6 @@ func (a *GenericAdmissionWebhook) hookClient(h *v1alpha1.ExternalAdmissionHook)
 	cfg.TLSClientConfig.ServerName = serverName
 	cfg.TLSClientConfig.CAData = h.ClientConfig.CABundle
 	cfg.ContentConfig.NegotiatedSerializer = a.negotiatedSerializer
+	cfg.ContentConfig.ContentType = runtime.ContentTypeJSON
 	return rest.UnversionedRESTClientFor(cfg)
 }

test/e2e/apimachinery/aggregator.go

@@ -18,12 +18,9 @@ package apimachinery
 
 import (
 	"crypto/rand"
-	"crypto/x509"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"math/big"
-	"os"
 	"strings"
 	"time"
 
@@ -38,7 +35,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/client-go/discovery"
-	"k8s.io/client-go/util/cert"
 	apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
 	rbacapi "k8s.io/kubernetes/pkg/apis/rbac"
 	utilversion "k8s.io/kubernetes/pkg/util/version"
@@ -48,12 +44,6 @@ import (
 	. "github.com/onsi/ginkgo"
 )
 
-type aggregatorContext struct {
-	apiserverCert        []byte
-	apiserverKey         []byte
-	apiserverSigningCert []byte
-}
-
 var serverAggregatorVersion = utilversion.MustParseSemantic("v1.7.0")
 
 var _ = SIGDescribe("Aggregator", func() {
@@ -88,62 +78,6 @@ func cleanTest(f *framework.Framework) {
 	_ = client.RbacV1beta1().ClusterRoleBindings().Delete("wardler:"+namespace+":anonymous", nil)
 }
 
-func setupSampleAPIServerCert(namespaceName, serviceName string) *aggregatorContext {
-	aggregatorCertDir, err := ioutil.TempDir("", "test-e2e-aggregator")
-	if err != nil {
-		framework.Failf("Failed to create a temp dir for cert generation %v", err)
-	}
-	defer os.RemoveAll(aggregatorCertDir)
-	apiserverSigningKey, err := cert.NewPrivateKey()
-	if err != nil {
-		framework.Failf("Failed to create CA private key for apiserver %v", err)
-	}
-	apiserverSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-sampleapiserver-ca"}, apiserverSigningKey)
-	if err != nil {
-		framework.Failf("Failed to create CA cert for apiserver %v", err)
-	}
-	apiserverCACertFile, err := ioutil.TempFile(aggregatorCertDir, "apiserver-ca.crt")
-	if err != nil {
-		framework.Failf("Failed to create a temp file for ca cert generation %v", err)
-	}
-	if err := ioutil.WriteFile(apiserverCACertFile.Name(), cert.EncodeCertPEM(apiserverSigningCert), 0644); err != nil {
-		framework.Failf("Failed to write CA cert for apiserver %v", err)
-	}
-	apiserverKey, err := cert.NewPrivateKey()
-	if err != nil {
-		framework.Failf("Failed to create private key for apiserver %v", err)
-	}
-	apiserverCert, err := cert.NewSignedCert(
-		cert.Config{
-			CommonName: serviceName + "." + namespaceName + ".svc",
-			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		},
-		apiserverKey, apiserverSigningCert, apiserverSigningKey,
-	)
-	if err != nil {
-		framework.Failf("Failed to create cert for apiserver %v", err)
-	}
-	apiserverCertFile, err := ioutil.TempFile(aggregatorCertDir, "apiserver.crt")
-	if err != nil {
-		framework.Failf("Failed to create a temp file for cert generation %v", err)
-	}
-	apiserverKeyFile, err := ioutil.TempFile(aggregatorCertDir, "apiserver.key")
-	if err != nil {
-		framework.Failf("Failed to create a temp file for key generation %v", err)
-	}
-	if err := ioutil.WriteFile(apiserverCertFile.Name(), cert.EncodeCertPEM(apiserverCert), 0600); err != nil {
-		framework.Failf("Failed to write cert file for apiserver %v", err)
-	}
-	if err := ioutil.WriteFile(apiserverKeyFile.Name(), cert.EncodePrivateKeyPEM(apiserverKey), 0644); err != nil {
-		framework.Failf("Failed to write key file for apiserver %v", err)
-	}
-	return &aggregatorContext{
-		apiserverCert:        cert.EncodeCertPEM(apiserverCert),
-		apiserverKey:         cert.EncodePrivateKeyPEM(apiserverKey),
-		apiserverSigningCert: cert.EncodeCertPEM(apiserverSigningCert),
-	}
-}
-
 // A basic test if the sample-apiserver code from 1.7 and compiled against 1.7
 // will work on the current Aggregator/API-Server.
 func TestSampleAPIServer(f *framework.Framework, image string) {
@@ -154,7 +88,7 @@ func TestSampleAPIServer(f *framework.Framework, image string) {
 	aggrclient := f.AggregatorClient
 
 	namespace := f.Namespace.Name
-	context := setupSampleAPIServerCert(namespace, "sample-api")
+	context := setupServerCert(namespace, "sample-api")
 	if framework.ProviderIs("gke") {
 		// kubectl create clusterrolebinding user-cluster-admin-binding --clusterrole=cluster-admin --user=user@domain.com
 		authenticated := rbacv1beta1.Subject{Kind: rbacv1beta1.GroupKind, Name: user.AllAuthenticated}
@@ -172,8 +106,8 @@ func TestSampleAPIServer(f *framework.Framework, image string) {
 		},
 		Type: v1.SecretTypeOpaque,
 		Data: map[string][]byte{
-			"tls.crt": context.apiserverCert,
+			"tls.crt": context.cert,
-			"tls.key": context.apiserverKey,
+			"tls.key": context.key,
 		},
 	}
 	_, err := client.CoreV1().Secrets(namespace).Create(secret)
@@ -349,7 +283,7 @@ func TestSampleAPIServer(f *framework.Framework, image string) {
 			},
 			Group:                "wardle.k8s.io",
 			Version:              "v1alpha1",
-			CABundle:             context.apiserverSigningCert,
+			CABundle:             context.signingCert,
 			GroupPriorityMinimum: 2000,
 			VersionPriority:      200,
 		},

test/e2e/apimachinery/certs.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apimachinery
+
+import (
+	"crypto/x509"
+	"io/ioutil"
+	"os"
+
+	"k8s.io/client-go/util/cert"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+type certContext struct {
+	cert        []byte
+	key         []byte
+	signingCert []byte
+}
+
+// Setup the server cert. For example, user apiservers and admission webhooks
+// can use the cert to prove their identify to the kube-apiserver
+func setupServerCert(namespaceName, serviceName string) *certContext {
+	certDir, err := ioutil.TempDir("", "test-e2e-server-cert")
+	if err != nil {
+		framework.Failf("Failed to create a temp dir for cert generation %v", err)
+	}
+	defer os.RemoveAll(certDir)
+	signingKey, err := cert.NewPrivateKey()
+	if err != nil {
+		framework.Failf("Failed to create CA private key %v", err)
+	}
+	signingCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-server-cert-ca"}, signingKey)
+	if err != nil {
+		framework.Failf("Failed to create CA cert for apiserver %v", err)
+	}
+	caCertFile, err := ioutil.TempFile(certDir, "ca.crt")
+	if err != nil {
+		framework.Failf("Failed to create a temp file for ca cert generation %v", err)
+	}
+	if err := ioutil.WriteFile(caCertFile.Name(), cert.EncodeCertPEM(signingCert), 0644); err != nil {
+		framework.Failf("Failed to write CA cert %v", err)
+	}
+	key, err := cert.NewPrivateKey()
+	if err != nil {
+		framework.Failf("Failed to create private key for %v", err)
+	}
+	signedCert, err := cert.NewSignedCert(
+		cert.Config{
+			CommonName: serviceName + "." + namespaceName + ".svc",
+			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		},
+		key, signingCert, signingKey,
+	)
+	if err != nil {
+		framework.Failf("Failed to create cert%v", err)
+	}
+	certFile, err := ioutil.TempFile(certDir, "server.crt")
+	if err != nil {
+		framework.Failf("Failed to create a temp file for cert generation %v", err)
+	}
+	keyFile, err := ioutil.TempFile(certDir, "server.key")
+	if err != nil {
+		framework.Failf("Failed to create a temp file for key generation %v", err)
+	}
+	if err = ioutil.WriteFile(certFile.Name(), cert.EncodeCertPEM(signedCert), 0600); err != nil {
+		framework.Failf("Failed to write cert file %v", err)
+	}
+	if err = ioutil.WriteFile(keyFile.Name(), cert.EncodePrivateKeyPEM(key), 0644); err != nil {
+		framework.Failf("Failed to write key file %v", err)
+	}
+	return &certContext{
+		cert:        cert.EncodeCertPEM(signedCert),
+		key:         cert.EncodePrivateKeyPEM(key),
+		signingCert: cert.EncodeCertPEM(signingCert),
+	}
+}

test/e2e/apimachinery/webhook.go

@@ -0,0 +1,294 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apimachinery
+
+import (
+	"strings"
+	"time"
+
+	"k8s.io/api/admissionregistration/v1alpha1"
+	"k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
+	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	utilversion "k8s.io/kubernetes/pkg/util/version"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	_ "github.com/stretchr/testify/assert"
+)
+
+const (
+	secretName        = "sample-webhook-secret"
+	deploymentName    = "sample-webhook-deployment"
+	serviceName       = "e2e-test-webhook"
+	roleBindingName   = "webhook-auth-reader"
+	webhookConfigName = "e2e-test-webhook-config"
+)
+
+var serverWebhookVersion = utilversion.MustParseSemantic("v1.8.0")
+
+var _ = SIGDescribe("AdmissionWebhook", func() {
+	f := framework.NewDefaultFramework("webhook")
+	framework.AddCleanupAction(func() {
+		cleanWebhookTest(f)
+	})
+
+	It("Should be able to deny pod creation", func() {
+		// Make sure the relevant provider supports admission webhook
+		framework.SkipUnlessServerVersionGTE(serverWebhookVersion, f.ClientSet.Discovery())
+		framework.SkipUnlessProviderIs("gce", "gke")
+
+		_, err := f.ClientSet.AdmissionregistrationV1alpha1().ExternalAdmissionHookConfigurations().List(metav1.ListOptions{})
+		if errors.IsNotFound(err) {
+			framework.Skipf("dynamic configuration of webhooks requires the alpha admissionregistration.k8s.io group to be enabled")
+		}
+
+		By("Setting up server cert")
+		namespaceName := f.Namespace.Name
+		context := setupServerCert(namespaceName, serviceName)
+		createAuthReaderRoleBinding(f, namespaceName)
+		// Note that in 1.9 we will have backwards incompatible change to
+		// admission webhooks, so the image will be updated to 1.9 sometime in
+		// the development 1.9 cycle.
+		deployWebhookAndService(f, "gcr.io/kubernetes-e2e-test-images/k8s-sample-admission-webhook-amd64:1.8v1", context)
+		registerWebhook(f, context)
+		testWebhook(f)
+	})
+})
+
+func createAuthReaderRoleBinding(f *framework.Framework, namespace string) {
+	By("Create role binding to let webhook read extension-apiserver-authentication")
+	client := f.ClientSet
+	// Create the role binding to allow the webhook read the extension-apiserver-authentication configmap
+	_, err := client.RbacV1beta1().RoleBindings("kube-system").Create(&rbacv1beta1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: roleBindingName,
+			Annotations: map[string]string{
+				rbacv1beta1.AutoUpdateAnnotationKey: "true",
+			},
+		},
+		RoleRef: rbacv1beta1.RoleRef{
+			APIGroup: "",
+			Kind:     "Role",
+			Name:     "extension-apiserver-authentication-reader",
+		},
+		// Webhook uses the default service account.
+		Subjects: []rbacv1beta1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      "default",
+				Namespace: namespace,
+			},
+		},
+	})
+	framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace)
+}
+
+func deployWebhookAndService(f *framework.Framework, image string, context *certContext) {
+	By("Deploying the webhook pod")
+
+	client := f.ClientSet
+
+	// Creating the secret that contains the webhook's cert.
+	secret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: secretName,
+		},
+		Type: v1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"tls.crt": context.cert,
+			"tls.key": context.key,
+		},
+	}
+	namespace := f.Namespace.Name
+	_, err := client.CoreV1().Secrets(namespace).Create(secret)
+	framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace)
+
+	// Create the deployment of the webhook
+	podLabels := map[string]string{"app": "sample-webhook", "webhook": "true"}
+	replicas := int32(1)
+	zero := int64(0)
+	mounts := []v1.VolumeMount{
+		{
+			Name:      "webhook-certs",
+			ReadOnly:  true,
+			MountPath: "/webhook.local.config/certificates",
+		},
+	}
+	volumes := []v1.Volume{
+		{
+			Name: "webhook-certs",
+			VolumeSource: v1.VolumeSource{
+				Secret: &v1.SecretVolumeSource{SecretName: secretName},
+			},
+		},
+	}
+	containers := []v1.Container{
+		{
+			Name:         "sample-webhook",
+			VolumeMounts: mounts,
+			Args: []string{
+				"--tls-cert-file=/webhook.local.config/certificates/tls.crt",
+				"--tls-private-key-file=/webhook.local.config/certificates/tls.key",
+				"--alsologtostderr",
+				"-v=4",
+				"2>&1",
+			},
+			Image: image,
+		},
+	}
+	d := &extensions.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: deploymentName,
+		},
+		Spec: extensions.DeploymentSpec{
+			Replicas: &replicas,
+			Strategy: extensions.DeploymentStrategy{
+				Type: extensions.RollingUpdateDeploymentStrategyType,
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: podLabels,
+				},
+				Spec: v1.PodSpec{
+					TerminationGracePeriodSeconds: &zero,
+					Containers:                    containers,
+					Volumes:                       volumes,
+				},
+			},
+		},
+	}
+	deployment, err := client.ExtensionsV1beta1().Deployments(namespace).Create(d)
+	framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, namespace)
+	By("Wait for the deployment to be ready")
+	err = framework.WaitForDeploymentRevisionAndImage(client, namespace, deploymentName, "1", image)
+	framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace)
+	err = framework.WaitForDeploymentComplete(client, deployment)
+	framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentName, namespace)
+
+	By("Deploying the webhook service")
+
+	serviceLabels := map[string]string{"webhook": "true"}
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      serviceName,
+			Labels:    map[string]string{"test": "webhook"},
+		},
+		Spec: v1.ServiceSpec{
+			Selector: serviceLabels,
+			Ports: []v1.ServicePort{
+				{
+					Protocol:   "TCP",
+					Port:       443,
+					TargetPort: intstr.FromInt(443),
+				},
+			},
+		},
+	}
+	_, err = client.CoreV1().Services(namespace).Create(service)
+	framework.ExpectNoError(err, "creating service %s in namespace %s", serviceName, namespace)
+
+	By("Verifying the service has paired with the endpoint")
+	err = framework.WaitForServiceEndpointsNum(client, namespace, serviceName, 1, 1*time.Second, 30*time.Second)
+	framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceName, 1)
+}
+
+func registerWebhook(f *framework.Framework, context *certContext) {
+	client := f.ClientSet
+	By("Registering the webhook via the AdmissionRegistration API")
+
+	namespace := f.Namespace.Name
+	_, err := client.AdmissionregistrationV1alpha1().ExternalAdmissionHookConfigurations().Create(&v1alpha1.ExternalAdmissionHookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: webhookConfigName,
+		},
+		ExternalAdmissionHooks: []v1alpha1.ExternalAdmissionHook{
+			{
+				Name: "e2e-test-webhook.k8s.io",
+				Rules: []v1alpha1.RuleWithOperations{{
+					Operations: []v1alpha1.OperationType{v1alpha1.Create},
+					Rule: v1alpha1.Rule{
+						APIGroups:   []string{""},
+						APIVersions: []string{"v1"},
+						Resources:   []string{"pods"},
+					},
+				}},
+				ClientConfig: v1alpha1.AdmissionHookClientConfig{
+					Service: v1alpha1.ServiceReference{
+						Namespace: namespace,
+						Name:      serviceName,
+					},
+					CABundle: context.signingCert,
+				},
+			},
+		},
+	})
+	framework.ExpectNoError(err, "registering webhook config %s with namespace %s", webhookConfigName, namespace)
+
+	// The webhook configuration is honored in 1s.
+	time.Sleep(2 * time.Second)
+}
+
+func testWebhook(f *framework.Framework) {
+	By("create a pod that should be denied by the webhook")
+	client := f.ClientSet
+	// Creating the pod, the request should be rejected
+	pod := nonCompliantPod(f)
+	_, err := client.CoreV1().Pods(f.Namespace.Name).Create(pod)
+	Expect(err).NotTo(BeNil())
+	expectedErrMsg := "the pod contains unwanted container name"
+	if !strings.Contains(err.Error(), expectedErrMsg) {
+		framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error())
+	}
+	// TODO: Test if webhook can detect pod with non-compliant metadata.
+	// Currently metadata is lost because webhook uses the external version of
+	// the objects, and the apiserver sends the internal objects.
+}
+
+func nonCompliantPod(f *framework.Framework) *v1.Pod {
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "disallowed-pod",
+			Labels: map[string]string{
+				"webhook-e2e-test": "disallow",
+			},
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "webhook-disallow",
+					Image: framework.GetPauseImageName(f.ClientSet),
+				},
+			},
+		},
+	}
+}
+
+func cleanWebhookTest(f *framework.Framework) {
+	client := f.ClientSet
+	_ = client.AdmissionregistrationV1alpha1().ExternalAdmissionHookConfigurations().Delete(webhookConfigName, nil)
+	namespaceName := f.Namespace.Name
+	_ = client.CoreV1().Services(namespaceName).Delete(serviceName, nil)
+	_ = client.ExtensionsV1beta1().Deployments(namespaceName).Delete(deploymentName, nil)
+	_ = client.CoreV1().Secrets(namespaceName).Delete(secretName, nil)
+	_ = client.RbacV1beta1().RoleBindings("kube-system").Delete(roleBindingName, nil)
+}