From 234fbfa63a0785678f213e3d9742e124082a7f6b Mon Sep 17 00:00:00 2001 From: jennybuckley Date: Tue, 23 Apr 2019 12:46:58 -0700 Subject: [PATCH 1/2] Use semantic equality in client ca post start hook --- pkg/master/BUILD | 1 + pkg/master/client_ca_hook.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/master/BUILD b/pkg/master/BUILD index c46d692a45b..ea530b5af0a 100644 --- a/pkg/master/BUILD +++ b/pkg/master/BUILD @@ -104,6 +104,7 @@ go_library( "//staging/src/k8s.io/api/storage/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1alpha1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", diff --git a/pkg/master/client_ca_hook.go b/pkg/master/client_ca_hook.go index e8a52dd9951..07de81d9568 100644 --- a/pkg/master/client_ca_hook.go +++ b/pkg/master/client_ca_hook.go @@ -19,10 +19,10 @@ package master import ( "encoding/json" "fmt" - "reflect" "time" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -132,7 +132,7 @@ func writeConfigMap(client corev1client.ConfigMapsGetter, name string, data map[ return err } - if !reflect.DeepEqual(existing.Data, data) { + if !apiequality.Semantic.DeepEqual(existing.Data, data) { existing.Data = data _, err = client.ConfigMaps(metav1.NamespaceSystem).Update(existing) } From d10ee7c87497aa0c19387034eb1e40a79924611e Mon Sep 17 00:00:00 2001 From: jennybuckley Date: Tue, 23 Apr 2019 12:49:28 -0700 Subject: [PATCH 2/2] Add integration test for restarting kube-apiserver with a broken webhook --- .../apiserver/admissionwebhook/BUILD | 3 + .../admissionwebhook/broken_webhook_test.go | 183 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 test/integration/apiserver/admissionwebhook/broken_webhook_test.go diff --git a/test/integration/apiserver/admissionwebhook/BUILD b/test/integration/apiserver/admissionwebhook/BUILD index 226908f4068..ba46651ed2e 100644 --- a/test/integration/apiserver/admissionwebhook/BUILD +++ b/test/integration/apiserver/admissionwebhook/BUILD @@ -4,6 +4,7 @@ go_test( name = "go_default_test", srcs = [ "admission_test.go", + "broken_webhook_test.go", "main_test.go", ], tags = [ @@ -12,8 +13,10 @@ go_test( ], deps = [ "//cmd/kube-apiserver/app/options:go_default_library", + "//cmd/kube-apiserver/app/testing:go_default_library", "//staging/src/k8s.io/api/admission/v1beta1:go_default_library", "//staging/src/k8s.io/api/admissionregistration/v1beta1:go_default_library", + "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/apps/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", diff --git a/test/integration/apiserver/admissionwebhook/broken_webhook_test.go b/test/integration/apiserver/admissionwebhook/broken_webhook_test.go new file mode 100644 index 00000000000..5b24dd40a5d --- /dev/null +++ b/test/integration/apiserver/admissionwebhook/broken_webhook_test.go @@ -0,0 +1,183 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admissionwebhook + +import ( + "fmt" + "testing" + "time" + + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" + "k8s.io/kubernetes/test/integration/framework" +) + +var ( + brokenWebhookName = "integration-broken-webhook-test-webhook-config" + deploymentNamePrefix = "integration-broken-webhook-test-deployment" +) + +func TestBrokenWebhook(t *testing.T) { + var tearDownFn kubeapiservertesting.TearDownFunc + defer func() { + if tearDownFn != nil { + tearDownFn() + } + }() + + etcdConfig := framework.SharedEtcd() + server := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, etcdConfig) + tearDownFn = server.TearDownFn + + client, err := kubernetes.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + t.Logf("Creating Deployment to ensure apiserver is functional") + _, err = client.AppsV1().Deployments("default").Create(exampleDeployment(generateDeploymentName(0))) + if err != nil { + t.Fatalf("Failed to create deployment: %v", err) + } + + t.Logf("Creating Broken Webhook that will block all operations on all objects") + _, err = client.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Create(brokenWebhookConfig(brokenWebhookName)) + if err != nil { + t.Fatalf("Failed to register broken webhook: %v", err) + } + + // There is no guarantee on how long it takes the apiserver to honor the configuration and there is + // no API to determine if the configuration is being honored, so we will just wait 10s, which is long enough + // in most cases. + time.Sleep(10 * time.Second) + + // test whether the webhook blocks requests + t.Logf("Attempt to create Deployment which should fail due to the webhook") + _, err = client.AppsV1().Deployments("default").Create(exampleDeployment(generateDeploymentName(1))) + if err == nil { + t.Fatalf("Expected the broken webhook to cause creating a deployment to fail, but it succeeded.") + } + + t.Logf("Restarting apiserver") + tearDownFn = nil + server.TearDownFn() + server = kubeapiservertesting.StartTestServerOrDie(t, nil, nil, etcdConfig) + tearDownFn = server.TearDownFn + + client, err = kubernetes.NewForConfig(server.ClientConfig) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // test whether the webhook still blocks requests after restarting + t.Logf("Attempt again to create Deployment which should fail due to the webhook") + _, err = client.AppsV1().Deployments("default").Create(exampleDeployment(generateDeploymentName(2))) + if err == nil { + t.Fatalf("Expected the broken webhook to cause creating a deployment to fail, but it succeeded.") + } + + t.Logf("Deleting the broken webhook to fix the cluster") + err = client.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Delete(brokenWebhookName, nil) + if err != nil { + t.Fatalf("Failed to delete broken webhook: %v", err) + } + + // The webhook deletion is honored in 10s. + time.Sleep(10 * time.Second) + + // test if the deleted webhook no longer blocks requests + t.Logf("Creating Deployment to ensure webhook is deleted") + _, err = client.AppsV1().Deployments("default").Create(exampleDeployment(generateDeploymentName(3))) + if err != nil { + t.Fatalf("Failed to create deployment: %v", err) + } +} + +func generateDeploymentName(suffix int) string { + return fmt.Sprintf("%v-%v", deploymentNamePrefix, suffix) +} + +func exampleDeployment(name string) *appsv1.Deployment { + var replicas int32 = 1 + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: name, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"foo": "bar"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"foo": "bar"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "foo", + }, + }, + }, + }, + }, + } +} + +func brokenWebhookConfig(name string) *admissionregistrationv1beta1.ValidatingWebhookConfiguration { + var path string + var failurePolicy admissionregistrationv1beta1.FailurePolicyType = admissionregistrationv1beta1.Fail + return &admissionregistrationv1beta1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Webhooks: []admissionregistrationv1beta1.Webhook{ + { + Name: "broken-webhook.k8s.io", + Rules: []admissionregistrationv1beta1.RuleWithOperations{{ + Operations: []admissionregistrationv1beta1.OperationType{admissionregistrationv1beta1.OperationAll}, + Rule: admissionregistrationv1beta1.Rule{ + APIGroups: []string{"*"}, + APIVersions: []string{"*"}, + Resources: []string{"*/*"}, + }, + }}, + // This client config references a non existent service + // so it should always fail. + ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{ + Service: &admissionregistrationv1beta1.ServiceReference{ + Namespace: "default", + Name: "invalid-webhook-service", + Path: &path, + }, + CABundle: nil, + }, + FailurePolicy: &failurePolicy, + }, + }, + } +}