diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index c4ce92cf48f..426ac4f3a92 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -299,7 +299,7 @@ if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then fi # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely. -ADMISSION_CONTROL="${KUBE_ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota}" +ADMISSION_CONTROL="${KUBE_ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,Priority,ResourceQuota,GenericAdmissionWebhook}" # Optional: if set to true kube-up will automatically check for existing resources and clean them up. KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false} diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index c3a476d92c4..c90196d92f0 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -419,7 +419,7 @@ function start_apiserver { fi # Admission Controllers to invoke prior to persisting objects in cluster - ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount${security_admission},DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,GenericAdmissionWebhook + ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount${security_admission},DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota # This is the default dir and filename where the apiserver will generate a self-signed cert # which should be able to be used as the CA to verify itself diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go index 2636c58df70..8408e89b946 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go @@ -298,5 +298,6 @@ func (a *GenericAdmissionWebhook) hookClient(h *v1alpha1.ExternalAdmissionHook) cfg.TLSClientConfig.ServerName = serverName cfg.TLSClientConfig.CAData = h.ClientConfig.CABundle cfg.ContentConfig.NegotiatedSerializer = a.negotiatedSerializer + cfg.ContentConfig.ContentType = runtime.ContentTypeJSON return rest.UnversionedRESTClientFor(cfg) } diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index f2b4521cd3a..fb03124778e 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -9,6 +9,7 @@ go_library( name = "go_default_library", srcs = [ "aggregator.go", + "certs.go", "chunking.go", "custom_resource_definition.go", "etcd_failure.go", @@ -18,6 +19,7 @@ go_library( "initializers.go", "namespace.go", "table_conversion.go", + "webhook.go", ], importpath = "k8s.io/kubernetes/test/e2e/apimachinery", deps = [ @@ -33,6 +35,7 @@ go_library( "//test/utils/image:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library", "//vendor/k8s.io/api/batch/v1:go_default_library", "//vendor/k8s.io/api/batch/v1beta1:go_default_library", diff --git a/test/e2e/apimachinery/aggregator.go b/test/e2e/apimachinery/aggregator.go index 5a3ee164ce8..319ba02e30d 100644 --- a/test/e2e/apimachinery/aggregator.go +++ b/test/e2e/apimachinery/aggregator.go @@ -18,12 +18,9 @@ package apimachinery import ( "crypto/rand" - "crypto/x509" "encoding/json" "fmt" - "io/ioutil" "math/big" - "os" "strings" "time" @@ -38,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/discovery" - "k8s.io/client-go/util/cert" apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1" rbacapi "k8s.io/kubernetes/pkg/apis/rbac" utilversion "k8s.io/kubernetes/pkg/util/version" @@ -48,12 +44,6 @@ import ( . "github.com/onsi/ginkgo" ) -type aggregatorContext struct { - apiserverCert []byte - apiserverKey []byte - apiserverSigningCert []byte -} - var serverAggregatorVersion = utilversion.MustParseSemantic("v1.7.0") var _ = SIGDescribe("Aggregator", func() { @@ -88,62 +78,6 @@ func cleanTest(f *framework.Framework) { _ = client.RbacV1beta1().ClusterRoleBindings().Delete("wardler:"+namespace+":anonymous", nil) } -func setupSampleAPIServerCert(namespaceName, serviceName string) *aggregatorContext { - aggregatorCertDir, err := ioutil.TempDir("", "test-e2e-aggregator") - if err != nil { - framework.Failf("Failed to create a temp dir for cert generation %v", err) - } - defer os.RemoveAll(aggregatorCertDir) - apiserverSigningKey, err := cert.NewPrivateKey() - if err != nil { - framework.Failf("Failed to create CA private key for apiserver %v", err) - } - apiserverSigningCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-sampleapiserver-ca"}, apiserverSigningKey) - if err != nil { - framework.Failf("Failed to create CA cert for apiserver %v", err) - } - apiserverCACertFile, err := ioutil.TempFile(aggregatorCertDir, "apiserver-ca.crt") - if err != nil { - framework.Failf("Failed to create a temp file for ca cert generation %v", err) - } - if err := ioutil.WriteFile(apiserverCACertFile.Name(), cert.EncodeCertPEM(apiserverSigningCert), 0644); err != nil { - framework.Failf("Failed to write CA cert for apiserver %v", err) - } - apiserverKey, err := cert.NewPrivateKey() - if err != nil { - framework.Failf("Failed to create private key for apiserver %v", err) - } - apiserverCert, err := cert.NewSignedCert( - cert.Config{ - CommonName: serviceName + "." + namespaceName + ".svc", - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - }, - apiserverKey, apiserverSigningCert, apiserverSigningKey, - ) - if err != nil { - framework.Failf("Failed to create cert for apiserver %v", err) - } - apiserverCertFile, err := ioutil.TempFile(aggregatorCertDir, "apiserver.crt") - if err != nil { - framework.Failf("Failed to create a temp file for cert generation %v", err) - } - apiserverKeyFile, err := ioutil.TempFile(aggregatorCertDir, "apiserver.key") - if err != nil { - framework.Failf("Failed to create a temp file for key generation %v", err) - } - if err := ioutil.WriteFile(apiserverCertFile.Name(), cert.EncodeCertPEM(apiserverCert), 0600); err != nil { - framework.Failf("Failed to write cert file for apiserver %v", err) - } - if err := ioutil.WriteFile(apiserverKeyFile.Name(), cert.EncodePrivateKeyPEM(apiserverKey), 0644); err != nil { - framework.Failf("Failed to write key file for apiserver %v", err) - } - return &aggregatorContext{ - apiserverCert: cert.EncodeCertPEM(apiserverCert), - apiserverKey: cert.EncodePrivateKeyPEM(apiserverKey), - apiserverSigningCert: cert.EncodeCertPEM(apiserverSigningCert), - } -} - // A basic test if the sample-apiserver code from 1.7 and compiled against 1.7 // will work on the current Aggregator/API-Server. func TestSampleAPIServer(f *framework.Framework, image string) { @@ -154,7 +88,7 @@ func TestSampleAPIServer(f *framework.Framework, image string) { aggrclient := f.AggregatorClient namespace := f.Namespace.Name - context := setupSampleAPIServerCert(namespace, "sample-api") + context := setupServerCert(namespace, "sample-api") if framework.ProviderIs("gke") { // kubectl create clusterrolebinding user-cluster-admin-binding --clusterrole=cluster-admin --user=user@domain.com authenticated := rbacv1beta1.Subject{Kind: rbacv1beta1.GroupKind, Name: user.AllAuthenticated} @@ -172,8 +106,8 @@ func TestSampleAPIServer(f *framework.Framework, image string) { }, Type: v1.SecretTypeOpaque, Data: map[string][]byte{ - "tls.crt": context.apiserverCert, - "tls.key": context.apiserverKey, + "tls.crt": context.cert, + "tls.key": context.key, }, } _, err := client.CoreV1().Secrets(namespace).Create(secret) @@ -349,7 +283,7 @@ func TestSampleAPIServer(f *framework.Framework, image string) { }, Group: "wardle.k8s.io", Version: "v1alpha1", - CABundle: context.apiserverSigningCert, + CABundle: context.signingCert, GroupPriorityMinimum: 2000, VersionPriority: 200, }, diff --git a/test/e2e/apimachinery/certs.go b/test/e2e/apimachinery/certs.go new file mode 100644 index 00000000000..26c8e06e4d9 --- /dev/null +++ b/test/e2e/apimachinery/certs.go @@ -0,0 +1,90 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + "crypto/x509" + "io/ioutil" + "os" + + "k8s.io/client-go/util/cert" + "k8s.io/kubernetes/test/e2e/framework" +) + +type certContext struct { + cert []byte + key []byte + signingCert []byte +} + +// Setup the server cert. For example, user apiservers and admission webhooks +// can use the cert to prove their identify to the kube-apiserver +func setupServerCert(namespaceName, serviceName string) *certContext { + certDir, err := ioutil.TempDir("", "test-e2e-server-cert") + if err != nil { + framework.Failf("Failed to create a temp dir for cert generation %v", err) + } + defer os.RemoveAll(certDir) + signingKey, err := cert.NewPrivateKey() + if err != nil { + framework.Failf("Failed to create CA private key %v", err) + } + signingCert, err := cert.NewSelfSignedCACert(cert.Config{CommonName: "e2e-server-cert-ca"}, signingKey) + if err != nil { + framework.Failf("Failed to create CA cert for apiserver %v", err) + } + caCertFile, err := ioutil.TempFile(certDir, "ca.crt") + if err != nil { + framework.Failf("Failed to create a temp file for ca cert generation %v", err) + } + if err := ioutil.WriteFile(caCertFile.Name(), cert.EncodeCertPEM(signingCert), 0644); err != nil { + framework.Failf("Failed to write CA cert %v", err) + } + key, err := cert.NewPrivateKey() + if err != nil { + framework.Failf("Failed to create private key for %v", err) + } + signedCert, err := cert.NewSignedCert( + cert.Config{ + CommonName: serviceName + "." + namespaceName + ".svc", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }, + key, signingCert, signingKey, + ) + if err != nil { + framework.Failf("Failed to create cert%v", err) + } + certFile, err := ioutil.TempFile(certDir, "server.crt") + if err != nil { + framework.Failf("Failed to create a temp file for cert generation %v", err) + } + keyFile, err := ioutil.TempFile(certDir, "server.key") + if err != nil { + framework.Failf("Failed to create a temp file for key generation %v", err) + } + if err = ioutil.WriteFile(certFile.Name(), cert.EncodeCertPEM(signedCert), 0600); err != nil { + framework.Failf("Failed to write cert file %v", err) + } + if err = ioutil.WriteFile(keyFile.Name(), cert.EncodePrivateKeyPEM(key), 0644); err != nil { + framework.Failf("Failed to write key file %v", err) + } + return &certContext{ + cert: cert.EncodeCertPEM(signedCert), + key: cert.EncodePrivateKeyPEM(key), + signingCert: cert.EncodeCertPEM(signingCert), + } +} diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go new file mode 100644 index 00000000000..3f17210c0c5 --- /dev/null +++ b/test/e2e/apimachinery/webhook.go @@ -0,0 +1,294 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + "strings" + "time" + + "k8s.io/api/admissionregistration/v1alpha1" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + rbacv1beta1 "k8s.io/api/rbac/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + utilversion "k8s.io/kubernetes/pkg/util/version" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + _ "github.com/stretchr/testify/assert" +) + +const ( + secretName = "sample-webhook-secret" + deploymentName = "sample-webhook-deployment" + serviceName = "e2e-test-webhook" + roleBindingName = "webhook-auth-reader" + webhookConfigName = "e2e-test-webhook-config" +) + +var serverWebhookVersion = utilversion.MustParseSemantic("v1.8.0") + +var _ = SIGDescribe("AdmissionWebhook", func() { + f := framework.NewDefaultFramework("webhook") + framework.AddCleanupAction(func() { + cleanWebhookTest(f) + }) + + It("Should be able to deny pod creation", func() { + // Make sure the relevant provider supports admission webhook + framework.SkipUnlessServerVersionGTE(serverWebhookVersion, f.ClientSet.Discovery()) + framework.SkipUnlessProviderIs("gce", "gke") + + _, err := f.ClientSet.AdmissionregistrationV1alpha1().ExternalAdmissionHookConfigurations().List(metav1.ListOptions{}) + if errors.IsNotFound(err) { + framework.Skipf("dynamic configuration of webhooks requires the alpha admissionregistration.k8s.io group to be enabled") + } + + By("Setting up server cert") + namespaceName := f.Namespace.Name + context := setupServerCert(namespaceName, serviceName) + createAuthReaderRoleBinding(f, namespaceName) + // Note that in 1.9 we will have backwards incompatible change to + // admission webhooks, so the image will be updated to 1.9 sometime in + // the development 1.9 cycle. + deployWebhookAndService(f, "gcr.io/kubernetes-e2e-test-images/k8s-sample-admission-webhook-amd64:1.8v1", context) + registerWebhook(f, context) + testWebhook(f) + }) +}) + +func createAuthReaderRoleBinding(f *framework.Framework, namespace string) { + By("Create role binding to let webhook read extension-apiserver-authentication") + client := f.ClientSet + // Create the role binding to allow the webhook read the extension-apiserver-authentication configmap + _, err := client.RbacV1beta1().RoleBindings("kube-system").Create(&rbacv1beta1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingName, + Annotations: map[string]string{ + rbacv1beta1.AutoUpdateAnnotationKey: "true", + }, + }, + RoleRef: rbacv1beta1.RoleRef{ + APIGroup: "", + Kind: "Role", + Name: "extension-apiserver-authentication-reader", + }, + // Webhook uses the default service account. + Subjects: []rbacv1beta1.Subject{ + { + Kind: "ServiceAccount", + Name: "default", + Namespace: namespace, + }, + }, + }) + framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace) +} + +func deployWebhookAndService(f *framework.Framework, image string, context *certContext) { + By("Deploying the webhook pod") + + client := f.ClientSet + + // Creating the secret that contains the webhook's cert. + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + }, + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + "tls.crt": context.cert, + "tls.key": context.key, + }, + } + namespace := f.Namespace.Name + _, err := client.CoreV1().Secrets(namespace).Create(secret) + framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace) + + // Create the deployment of the webhook + podLabels := map[string]string{"app": "sample-webhook", "webhook": "true"} + replicas := int32(1) + zero := int64(0) + mounts := []v1.VolumeMount{ + { + Name: "webhook-certs", + ReadOnly: true, + MountPath: "/webhook.local.config/certificates", + }, + } + volumes := []v1.Volume{ + { + Name: "webhook-certs", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{SecretName: secretName}, + }, + }, + } + containers := []v1.Container{ + { + Name: "sample-webhook", + VolumeMounts: mounts, + Args: []string{ + "--tls-cert-file=/webhook.local.config/certificates/tls.crt", + "--tls-private-key-file=/webhook.local.config/certificates/tls.key", + "--alsologtostderr", + "-v=4", + "2>&1", + }, + Image: image, + }, + } + d := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + }, + Spec: extensions.DeploymentSpec{ + Replicas: &replicas, + Strategy: extensions.DeploymentStrategy{ + Type: extensions.RollingUpdateDeploymentStrategyType, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + Spec: v1.PodSpec{ + TerminationGracePeriodSeconds: &zero, + Containers: containers, + Volumes: volumes, + }, + }, + }, + } + deployment, err := client.ExtensionsV1beta1().Deployments(namespace).Create(d) + framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, namespace) + By("Wait for the deployment to be ready") + err = framework.WaitForDeploymentRevisionAndImage(client, namespace, deploymentName, "1", image) + framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace) + err = framework.WaitForDeploymentComplete(client, deployment) + framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentName, namespace) + + By("Deploying the webhook service") + + serviceLabels := map[string]string{"webhook": "true"} + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: serviceName, + Labels: map[string]string{"test": "webhook"}, + }, + Spec: v1.ServiceSpec{ + Selector: serviceLabels, + Ports: []v1.ServicePort{ + { + Protocol: "TCP", + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + _, err = client.CoreV1().Services(namespace).Create(service) + framework.ExpectNoError(err, "creating service %s in namespace %s", serviceName, namespace) + + By("Verifying the service has paired with the endpoint") + err = framework.WaitForServiceEndpointsNum(client, namespace, serviceName, 1, 1*time.Second, 30*time.Second) + framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceName, 1) +} + +func registerWebhook(f *framework.Framework, context *certContext) { + client := f.ClientSet + By("Registering the webhook via the AdmissionRegistration API") + + namespace := f.Namespace.Name + _, err := client.AdmissionregistrationV1alpha1().ExternalAdmissionHookConfigurations().Create(&v1alpha1.ExternalAdmissionHookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: webhookConfigName, + }, + ExternalAdmissionHooks: []v1alpha1.ExternalAdmissionHook{ + { + Name: "e2e-test-webhook.k8s.io", + Rules: []v1alpha1.RuleWithOperations{{ + Operations: []v1alpha1.OperationType{v1alpha1.Create}, + Rule: v1alpha1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }}, + ClientConfig: v1alpha1.AdmissionHookClientConfig{ + Service: v1alpha1.ServiceReference{ + Namespace: namespace, + Name: serviceName, + }, + CABundle: context.signingCert, + }, + }, + }, + }) + framework.ExpectNoError(err, "registering webhook config %s with namespace %s", webhookConfigName, namespace) + + // The webhook configuration is honored in 1s. + time.Sleep(2 * time.Second) +} + +func testWebhook(f *framework.Framework) { + By("create a pod that should be denied by the webhook") + client := f.ClientSet + // Creating the pod, the request should be rejected + pod := nonCompliantPod(f) + _, err := client.CoreV1().Pods(f.Namespace.Name).Create(pod) + Expect(err).NotTo(BeNil()) + expectedErrMsg := "the pod contains unwanted container name" + if !strings.Contains(err.Error(), expectedErrMsg) { + framework.Failf("expect error contains %q, got %q", expectedErrMsg, err.Error()) + } + // TODO: Test if webhook can detect pod with non-compliant metadata. + // Currently metadata is lost because webhook uses the external version of + // the objects, and the apiserver sends the internal objects. +} + +func nonCompliantPod(f *framework.Framework) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "disallowed-pod", + Labels: map[string]string{ + "webhook-e2e-test": "disallow", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "webhook-disallow", + Image: framework.GetPauseImageName(f.ClientSet), + }, + }, + }, + } +} + +func cleanWebhookTest(f *framework.Framework) { + client := f.ClientSet + _ = client.AdmissionregistrationV1alpha1().ExternalAdmissionHookConfigurations().Delete(webhookConfigName, nil) + namespaceName := f.Namespace.Name + _ = client.CoreV1().Services(namespaceName).Delete(serviceName, nil) + _ = client.ExtensionsV1beta1().Deployments(namespaceName).Delete(deploymentName, nil) + _ = client.CoreV1().Secrets(namespaceName).Delete(secretName, nil) + _ = client.RbacV1beta1().RoleBindings("kube-system").Delete(roleBindingName, nil) +} diff --git a/test/images/BUILD b/test/images/BUILD index 73e9ce74a63..ba74aece486 100644 --- a/test/images/BUILD +++ b/test/images/BUILD @@ -31,6 +31,7 @@ filegroup( "//test/images/resource-consumer:all-srcs", "//test/images/serve-hostname:all-srcs", "//test/images/test-webserver:all-srcs", + "//test/images/webhook:all-srcs", ], tags = ["automanaged"], ) diff --git a/test/images/webhook/BUILD b/test/images/webhook/BUILD new file mode 100644 index 00000000000..00067234a85 --- /dev/null +++ b/test/images/webhook/BUILD @@ -0,0 +1,40 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "config.go", + "main.go", + ], + importpath = "k8s.io/kubernetes/test/images/webhook", + visibility = ["//visibility:private"], + deps = [ + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/api/admission/v1alpha1:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/rest:go_default_library", + ], +) + +go_binary( + name = "webhook", + importpath = "k8s.io/kubernetes/test/images/webhook", + library = ":go_default_library", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/images/webhook/Dockerfile b/test/images/webhook/Dockerfile new file mode 100644 index 00000000000..2a43f424b21 --- /dev/null +++ b/test/images/webhook/Dockerfile @@ -0,0 +1,18 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:latest + +ADD webhook /webhook +ENTRYPOINT ["/webhook"] diff --git a/test/images/webhook/Makefile b/test/images/webhook/Makefile new file mode 100644 index 00000000000..7f706cbaf13 --- /dev/null +++ b/test/images/webhook/Makefile @@ -0,0 +1,19 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +build: + CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webhook . + docker build --no-cache -t gcr.io/kubernetes-e2e-test-images/k8s-sample-admission-webhook-amd64:1.8v1 . +push: + gcloud docker --push gcr.io/kubernetes-e2e-test-images/k8s-sample-admission-webhook-amd64:1.8v1 . diff --git a/test/images/webhook/README.md b/test/images/webhook/README.md new file mode 100644 index 00000000000..14895d90997 --- /dev/null +++ b/test/images/webhook/README.md @@ -0,0 +1,51 @@ +# Kubernetes External Admission Webhook Example + +The example shows how to build and deploy an external webhook that only admits +pods creation and update if the container images have the "grc.io" prefix. + +## Prerequisites + +Please use a Kubernetes release at least as new as v1.8.0 or v1.9.0-alpha.1, +because the generated server cert/key only works with Kubernetes release that +contains this [change](https://github.com/kubernetes/kubernetes/pull/50476). +Please checkout the `pre-v1.8` tag for an example that works with older +clusters. + +Please enable the admission webhook feature +([doc](https://kubernetes.io/docs/admin/extensible-admission-controllers/#enable-external-admission-webhooks)). + +## Build the code + +```bash +make build +``` + +## Deploy the code + +```bash +make deploy-only +``` + +The Makefile assumes your cluster is created by the +[hack/local-up-cluster.sh](https://github.com/kubernetes/kubernetes/blob/master/hack/local-up-cluster.sh). +Please modify the Makefile accordingly if your cluster is created differently. + +## Explanation on the CAs/Certs/Keys + +The apiserver initiates a tls connection with the webhook, so the apiserver is +the tls client, and the webhook is the tls server. + +The webhook proves its identity by the `serverCert` in the certs.go. The server +cert is signed by the CA in certs.go. To let the apiserver trust the `caCert`, +the webhook registers itself with the apiserver via the +`admissionregistration/v1alpha1/externalAdmissionHook` API, with +`clientConfig.caBundle=caCert`. + +For maximum protection, this example webhook requires and verifies the client +(i.e., the apiserver in this case) cert. The cert presented by the apiserver is +signed by a client CA, whose cert is stored in the configmap +`extension-apiserver-authentication` in the `kube-system` namespace. See the +`getAPIServerCert` function for more information. Usually you don't need to +worry about setting up this CA cert. It's taken care of when the cluster is +created. You can disable the client cert verification by setting the +`tls.Config.ClientAuth` to `tls.NoClientCert` in `config.go`. diff --git a/test/images/webhook/config.go b/test/images/webhook/config.go new file mode 100644 index 00000000000..c3f736eaa06 --- /dev/null +++ b/test/images/webhook/config.go @@ -0,0 +1,51 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/golang/glog" +) + +// Get a clientset with in-cluster config. +func getClient() *kubernetes.Clientset { + config, err := rest.InClusterConfig() + if err != nil { + glog.Fatal(err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + glog.Fatal(err) + } + return clientset +} + +func configTLS(config Config, clientset *kubernetes.Clientset) *tls.Config { + sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) + if err != nil { + glog.Fatal(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{sCert}, + // TODO: uses mutual tls after we agree on what cert the apiserver should use. + // ClientAuth: tls.RequireAndVerifyClientCert, + } +} diff --git a/test/images/webhook/main.go b/test/images/webhook/main.go new file mode 100644 index 00000000000..f6e6500e5f3 --- /dev/null +++ b/test/images/webhook/main.go @@ -0,0 +1,130 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "flag" + "io/ioutil" + "net/http" + "strings" + + "github.com/golang/glog" + "k8s.io/api/admission/v1alpha1" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Config contains the server (the webhook) cert and key. +type Config struct { + CertFile string + KeyFile string +} + +func (c *Config) addFlags() { + flag.StringVar(&c.CertFile, "tls-cert-file", c.CertFile, ""+ + "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ + "after server cert).") + flag.StringVar(&c.KeyFile, "tls-private-key-file", c.KeyFile, ""+ + "File containing the default x509 private key matching --tls-cert-file.") +} + +// only allow pods to pull images from specific registry. +func admit(data []byte) *v1alpha1.AdmissionReviewStatus { + ar := v1alpha1.AdmissionReview{} + if err := json.Unmarshal(data, &ar); err != nil { + glog.Error(err) + return nil + } + podResource := metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} + if ar.Spec.Resource != podResource { + glog.Errorf("expect resource to be %s", podResource) + return nil + } + + raw := ar.Spec.Object.Raw + pod := v1.Pod{} + if err := json.Unmarshal(raw, &pod); err != nil { + glog.Error(err) + return nil + } + reviewStatus := v1alpha1.AdmissionReviewStatus{} + reviewStatus.Allowed = true + // Note: the apiserver encodes the api.Pod. Decoding it as a v1.Pod will + // lose the metadata. So the following check on labels will not work + // until we let the apiserver encodes the versioned object. + for k, v := range pod.Labels { + if k == "webhook-e2e-test" && v == "webhook-disallow" { + reviewStatus.Allowed = false + reviewStatus.Result = &metav1.Status{ + Reason: "the pod contains unwanted label", + } + } + } + for _, container := range pod.Spec.Containers { + if strings.Contains(container.Name, "webhook-disallow") { + reviewStatus.Allowed = false + reviewStatus.Result = &metav1.Status{ + Message: "the pod contains unwanted container name", + } + } + } + return &reviewStatus +} + +func serve(w http.ResponseWriter, r *http.Request) { + var body []byte + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } + } + + // verify the content type is accurate + contentType := r.Header.Get("Content-Type") + if contentType != "application/json" { + glog.Errorf("contentType=%s, expect application/json", contentType) + return + } + + reviewStatus := admit(body) + ar := v1alpha1.AdmissionReview{ + Status: *reviewStatus, + } + + resp, err := json.Marshal(ar) + if err != nil { + glog.Error(err) + } + if _, err := w.Write(resp); err != nil { + glog.Error(err) + } +} + +func main() { + var config Config + config.addFlags() + flag.Parse() + + http.HandleFunc("/", serve) + clientset := getClient() + server := &http.Server{ + Addr: ":443", + TLSConfig: configTLS(config, clientset), + } + server.ListenAndServeTLS("", "") +} diff --git a/test/images/webhook/webhook b/test/images/webhook/webhook new file mode 100755 index 00000000000..22a214bcf83 Binary files /dev/null and b/test/images/webhook/webhook differ