From 26a6cdda86f054c7b3dcc2176f90952684950af6 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Thu, 30 May 2019 15:14:33 -0400 Subject: [PATCH] Set integration tests to use distinct namespaces TestWatchBasedManager was racing with the default namespace creation. To fix that flake and to ensure integration tests using a shared etcd don't accidentally overlap in the future, move the three main tests using the default namespace to separate namespaces, and have TestWatchBasedManager create that namespace before it runs. Make StartTestServer wait for default namespace creation, which will reduce other flakes until future changes completely remove use of default namespace. From a failed integration run: watch_manager_test.go:66: namespaces "default" not found watch_manager_test.go:66: namespaces "default" not found watch_manager_test.go:66: namespaces "default" not found --- cmd/kube-apiserver/app/testing/BUILD | 2 + cmd/kube-apiserver/app/testing/testserver.go | 24 ++++++++ test/integration/apiserver/apiserver_test.go | 56 ++++++++----------- .../integration/kubelet/watch_manager_test.go | 12 ++-- test/integration/master/crd_test.go | 20 ++++++- 5 files changed, 75 insertions(+), 39 deletions(-) diff --git a/cmd/kube-apiserver/app/testing/BUILD b/cmd/kube-apiserver/app/testing/BUILD index 3c3c380d024..758f8fcb0f8 100644 --- a/cmd/kube-apiserver/app/testing/BUILD +++ b/cmd/kube-apiserver/app/testing/BUILD @@ -13,6 +13,8 @@ go_library( deps = [ "//cmd/kube-apiserver/app:go_default_library", "//cmd/kube-apiserver/app/options:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", diff --git a/cmd/kube-apiserver/app/testing/testserver.go b/cmd/kube-apiserver/app/testing/testserver.go index cfa15352ae4..9fa1c82516e 100644 --- a/cmd/kube-apiserver/app/testing/testserver.go +++ b/cmd/kube-apiserver/app/testing/testserver.go @@ -27,6 +27,8 @@ import ( pflag "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -158,6 +160,8 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) } + + // wait until healthz endpoint returns ok err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { select { case err := <-errCh: @@ -177,6 +181,26 @@ func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, custo return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err) } + // wait until default namespace is created + err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) { + select { + case err := <-errCh: + return false, err + default: + } + + if _, err := client.CoreV1().Namespaces().Get("default", metav1.GetOptions{}); err != nil { + if !errors.IsNotFound(err) { + t.Logf("Unable to get default namespace: %v", err) + } + return false, nil + } + return true, nil + }) + if err != nil { + return result, fmt.Errorf("failed to wait for default namespace to be created: %v", err) + } + // from here the caller must call tearDown result.ClientConfig = server.LoopbackClientConfig result.ClientConfig.QPS = 1000 diff --git a/test/integration/apiserver/apiserver_test.go b/test/integration/apiserver/apiserver_test.go index fbaf98b8a19..212209b9303 100644 --- a/test/integration/apiserver/apiserver_test.go +++ b/test/integration/apiserver/apiserver_test.go @@ -404,6 +404,7 @@ func TestNameInFieldSelector(t *testing.T) { } func TestAPICRDProtobuf(t *testing.T) { + testNamespace := "test-api-crd-protobuf" tearDown, config, _, err := fixtures.StartDefaultServer(t) if err != nil { t.Fatal(err) @@ -442,7 +443,7 @@ func TestAPICRDProtobuf(t *testing.T) { t.Fatal(err) } crdGVR := schema.GroupVersionResource{Group: fooCRD.Spec.Group, Version: fooCRD.Spec.Version, Resource: "foos"} - crclient := dynamicClient.Resource(crdGVR).Namespace("default") + crclient := dynamicClient.Resource(crdGVR).Namespace(testNamespace) testcases := []struct { name string @@ -554,6 +555,7 @@ func TestAPICRDProtobuf(t *testing.T) { } func TestTransform(t *testing.T) { + testNamespace := "test-transform" tearDown, config, _, err := fixtures.StartDefaultServer(t) if err != nil { t.Fatal(err) @@ -592,7 +594,7 @@ func TestTransform(t *testing.T) { t.Fatal(err) } crdGVR := schema.GroupVersionResource{Group: fooCRD.Spec.Group, Version: fooCRD.Spec.Version, Resource: "foos"} - crclient := dynamicClient.Resource(crdGVR).Namespace("default") + crclient := dynamicClient.Resource(crdGVR).Namespace(testNamespace) testcases := []struct { name string @@ -673,12 +675,11 @@ func TestTransform(t *testing.T) { name: "v1beta1 verify columns on services", accept: "application/json;as=Table;g=meta.k8s.io;v=v1beta1", object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - svc, err := clientset.CoreV1().Services(ns).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-1"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) + svc, err := clientset.CoreV1().Services(testNamespace).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-1"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) if err != nil { t.Fatalf("unable to create service: %v", err) } - if _, err := clientset.CoreV1().Services(ns).Patch(svc.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().Services(testNamespace).Patch(svc.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update service: %v", err) } return svc, "", "services" @@ -692,12 +693,11 @@ func TestTransform(t *testing.T) { accept: "application/json;as=Table;g=meta.k8s.io;v=v1beta1", includeObject: metav1.IncludeNone, object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - obj, err := clientset.CoreV1().Services(ns).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-2"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) + obj, err := clientset.CoreV1().Services(testNamespace).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-2"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) if err != nil { t.Fatalf("unable to create object: %v", err) } - if _, err := clientset.CoreV1().Services(ns).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().Services(testNamespace).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update object: %v", err) } return obj, "", "services" @@ -711,12 +711,11 @@ func TestTransform(t *testing.T) { accept: "application/json;as=Table;g=meta.k8s.io;v=v1beta1", includeObject: metav1.IncludeObject, object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - obj, err := clientset.CoreV1().Services(ns).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-3"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) + obj, err := clientset.CoreV1().Services(testNamespace).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-3"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) if err != nil { t.Fatalf("unable to create object: %v", err) } - if _, err := clientset.CoreV1().Services(ns).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().Services(testNamespace).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update object: %v", err) } return obj, "", "services" @@ -736,12 +735,11 @@ func TestTransform(t *testing.T) { name: "v1beta1 verify partial metadata object on config maps", accept: "application/json;as=PartialObjectMetadata;g=meta.k8s.io;v=v1beta1", object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - obj, err := clientset.CoreV1().ConfigMaps(ns).Create(&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test-1", Annotations: map[string]string{"test": "0"}}}) + obj, err := clientset.CoreV1().ConfigMaps(testNamespace).Create(&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test-1", Annotations: map[string]string{"test": "0"}}}) if err != nil { t.Fatalf("unable to create object: %v", err) } - if _, err := clientset.CoreV1().ConfigMaps(ns).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().ConfigMaps(testNamespace).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update object: %v", err) } return obj, "", "configmaps" @@ -754,12 +752,11 @@ func TestTransform(t *testing.T) { name: "v1beta1 verify partial metadata object on config maps in protobuf", accept: "application/vnd.kubernetes.protobuf;as=PartialObjectMetadata;g=meta.k8s.io;v=v1beta1", object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - obj, err := clientset.CoreV1().ConfigMaps(ns).Create(&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test-2", Annotations: map[string]string{"test": "0"}}}) + obj, err := clientset.CoreV1().ConfigMaps(testNamespace).Create(&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test-2", Annotations: map[string]string{"test": "0"}}}) if err != nil { t.Fatalf("unable to create object: %v", err) } - if _, err := clientset.CoreV1().ConfigMaps(ns).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().ConfigMaps(testNamespace).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update object: %v", err) } return obj, "", "configmaps" @@ -934,12 +931,11 @@ func TestTransform(t *testing.T) { name: "v1 verify columns on services", accept: "application/json;as=Table;g=meta.k8s.io;v=v1", object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - svc, err := clientset.CoreV1().Services(ns).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-5"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) + svc, err := clientset.CoreV1().Services(testNamespace).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-5"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) if err != nil { t.Fatalf("unable to create service: %v", err) } - if _, err := clientset.CoreV1().Services(ns).Patch(svc.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().Services(testNamespace).Patch(svc.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update service: %v", err) } return svc, "", "services" @@ -953,12 +949,11 @@ func TestTransform(t *testing.T) { accept: "application/json;as=Table;g=meta.k8s.io;v=v1", includeObject: metav1.IncludeNone, object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - obj, err := clientset.CoreV1().Services(ns).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-6"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) + obj, err := clientset.CoreV1().Services(testNamespace).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-6"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) if err != nil { t.Fatalf("unable to create object: %v", err) } - if _, err := clientset.CoreV1().Services(ns).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().Services(testNamespace).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update object: %v", err) } return obj, "", "services" @@ -972,12 +967,11 @@ func TestTransform(t *testing.T) { accept: "application/json;as=Table;g=meta.k8s.io;v=v1", includeObject: metav1.IncludeObject, object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - obj, err := clientset.CoreV1().Services(ns).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-7"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) + obj, err := clientset.CoreV1().Services(testNamespace).Create(&v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "test-7"}, Spec: v1.ServiceSpec{Ports: []v1.ServicePort{{Port: 1000}}}}) if err != nil { t.Fatalf("unable to create object: %v", err) } - if _, err := clientset.CoreV1().Services(ns).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().Services(testNamespace).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update object: %v", err) } return obj, "", "services" @@ -997,12 +991,11 @@ func TestTransform(t *testing.T) { name: "v1 verify partial metadata object on config maps", accept: "application/json;as=PartialObjectMetadata;g=meta.k8s.io;v=v1", object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - obj, err := clientset.CoreV1().ConfigMaps(ns).Create(&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test-3", Annotations: map[string]string{"test": "0"}}}) + obj, err := clientset.CoreV1().ConfigMaps(testNamespace).Create(&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test-3", Annotations: map[string]string{"test": "0"}}}) if err != nil { t.Fatalf("unable to create object: %v", err) } - if _, err := clientset.CoreV1().ConfigMaps(ns).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().ConfigMaps(testNamespace).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update object: %v", err) } return obj, "", "configmaps" @@ -1015,12 +1008,11 @@ func TestTransform(t *testing.T) { name: "v1 verify partial metadata object on config maps in protobuf", accept: "application/vnd.kubernetes.protobuf;as=PartialObjectMetadata;g=meta.k8s.io;v=v1", object: func(t *testing.T) (metav1.Object, string, string) { - ns := "default" - obj, err := clientset.CoreV1().ConfigMaps(ns).Create(&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test-4", Annotations: map[string]string{"test": "0"}}}) + obj, err := clientset.CoreV1().ConfigMaps(testNamespace).Create(&v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "test-4", Annotations: map[string]string{"test": "0"}}}) if err != nil { t.Fatalf("unable to create object: %v", err) } - if _, err := clientset.CoreV1().ConfigMaps(ns).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { + if _, err := clientset.CoreV1().ConfigMaps(testNamespace).Patch(obj.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"test":"1"}}}`)); err != nil { t.Fatalf("unable to update object: %v", err) } return obj, "", "configmaps" diff --git a/test/integration/kubelet/watch_manager_test.go b/test/integration/kubelet/watch_manager_test.go index 226b1baeb1e..add0a2a1688 100644 --- a/test/integration/kubelet/watch_manager_test.go +++ b/test/integration/kubelet/watch_manager_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -35,6 +35,7 @@ import ( ) func TestWatchBasedManager(t *testing.T) { + testNamespace := "test-watch-based-manager" server := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) defer server.TearDownFn() @@ -43,6 +44,9 @@ func TestWatchBasedManager(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } + if _, err := client.CoreV1().Namespaces().Create((&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}})); err != nil { + t.Fatal(err) + } listObj := func(namespace string, options metav1.ListOptions) (runtime.Object, error) { return client.CoreV1().Secrets(namespace).List(options) @@ -62,7 +66,7 @@ func TestWatchBasedManager(t *testing.T) { defer wg.Done() for j := 0; j < 100; j++ { name := fmt.Sprintf("s%d", i*100+j) - if _, err := client.CoreV1().Secrets("default").Create(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}); err != nil { + if _, err := client.CoreV1().Secrets(testNamespace).Create(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}}); err != nil { t.Fatal(err) } } @@ -81,9 +85,9 @@ func TestWatchBasedManager(t *testing.T) { for j := 0; j < 100; j++ { name := fmt.Sprintf("s%d", i*100+j) start := time.Now() - store.AddReference("default", name) + store.AddReference(testNamespace, name) err := wait.PollImmediate(10*time.Millisecond, 10*time.Second, func() (bool, error) { - obj, err := store.Get("default", name) + obj, err := store.Get(testNamespace, name) if err != nil { t.Logf("failed on %s, retrying: %v", name, err) return false, nil diff --git a/test/integration/master/crd_test.go b/test/integration/master/crd_test.go index 75db01ac1f7..df0e5fb02d5 100644 --- a/test/integration/master/crd_test.go +++ b/test/integration/master/crd_test.go @@ -24,6 +24,7 @@ import ( "github.com/go-openapi/spec" + v1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -42,10 +43,14 @@ func TestCRDShadowGroup(t *testing.T) { result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) defer result.TearDownFn() + testNamespace := "test-crd-shadow-group" kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) if err != nil { t.Fatalf("Unexpected error: %v", err) } + if _, err := kubeclient.CoreV1().Namespaces().Create((&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}})); err != nil { + t.Fatal(err) + } apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig) if err != nil { @@ -53,8 +58,8 @@ func TestCRDShadowGroup(t *testing.T) { } t.Logf("Creating a NetworkPolicy") - nwPolicy, err := kubeclient.NetworkingV1().NetworkPolicies("default").Create(&networkingv1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: metav1.NamespaceDefault}, + nwPolicy, err := kubeclient.NetworkingV1().NetworkPolicies(testNamespace).Create(&networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{Name: "abc", Namespace: testNamespace}, Spec: networkingv1.NetworkPolicySpec{ PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}, Ingress: []networkingv1.NetworkPolicyIngressRule{}, @@ -100,6 +105,15 @@ func TestCRD(t *testing.T) { result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) defer result.TearDownFn() + testNamespace := "test-crd" + kubeclient, err := kubernetes.NewForConfig(result.ClientConfig) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if _, err := kubeclient.CoreV1().Namespaces().Create((&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testNamespace}})); err != nil { + t.Fatal(err) + } + apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig) if err != nil { t.Fatalf("Unexpected error: %v", err) @@ -128,7 +142,7 @@ func TestCRD(t *testing.T) { t.Fatalf("Unexpected error: %v", err) } fooResource := schema.GroupVersionResource{Group: "cr.bar.com", Version: "v1", Resource: "foos"} - _, err = dynamicClient.Resource(fooResource).Namespace("default").List(metav1.ListOptions{}) + _, err = dynamicClient.Resource(fooResource).Namespace(testNamespace).List(metav1.ListOptions{}) if err != nil { t.Errorf("Failed to list foos.cr.bar.com instances: %v", err) }