From b8137809039e293b947e3231aaa7f51f4381a878 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 8 May 2019 11:19:15 +0200 Subject: [PATCH] apiextensions: add default integration tests --- .../integration/conversion/conversion_test.go | 141 ++++++++++- .../test/integration/defaulting_test.go | 229 ++++++++++++++++++ 2 files changed, 366 insertions(+), 4 deletions(-) create mode 100644 staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go index e0c77cb8b06..be28d93609b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/conversion/conversion_test.go @@ -59,14 +59,18 @@ func checks(checkers ...Checker) []Checker { } func TestWebhookConverter(t *testing.T) { - testWebhookConverter(t, false) + testWebhookConverter(t, false, false) } func TestWebhookConverterWithPruning(t *testing.T) { - testWebhookConverter(t, true) + testWebhookConverter(t, true, false) } -func testWebhookConverter(t *testing.T, pruning bool) { +func TestWebhookConverterWithDefaulting(t *testing.T) { + testWebhookConverter(t, true, true) +} + +func testWebhookConverter(t *testing.T, pruning, defaulting bool) { tests := []struct { group string handler http.Handler @@ -80,7 +84,7 @@ func testWebhookConverter(t *testing.T, pruning bool) { { group: "nontrivial-converter", handler: NewObjectConverterWebhookHandler(t, nontrivialConverter), - checks: checks(validateStorageVersion, validateServed, validateMixedStorageVersions("v1alpha1", "v1beta1", "v1beta2"), validateNonTrivialConverted, validateNonTrivialConvertedList, validateStoragePruning), + checks: checks(validateStorageVersion, validateServed, validateMixedStorageVersions("v1alpha1", "v1beta1", "v1beta2"), validateNonTrivialConverted, validateNonTrivialConvertedList, validateStoragePruning, validateDefaulting), }, { group: "metadata-mutating-converter", @@ -110,7 +114,12 @@ func testWebhookConverter(t *testing.T, pruning bool) { etcd3watcher.TestOnlySetFatalOnDecodeError(false) defer etcd3watcher.TestOnlySetFatalOnDecodeError(true) + // enable necessary features defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceWebhookConversion, true)() + if defaulting { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, apiextensionsfeatures.CustomResourceDefaulting, true)() + } + tearDown, config, options, err := fixtures.StartDefaultServer(t) if err != nil { t.Fatal(err) @@ -132,6 +141,12 @@ func testWebhookConverter(t *testing.T, pruning bool) { crd := multiVersionFixture.DeepCopy() crd.Spec.PreserveUnknownFields = pointer.BoolPtr(!pruning) + if !defaulting { + for i := range crd.Spec.Versions { + delete(crd.Spec.Versions[i].Schema.OpenAPIV3Schema.Properties, "defaults") + } + } + RESTOptionsGetter := serveroptions.NewCRDRESTOptionsGetter(*options.RecommendedOptions.Etcd) restOptions, err := RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) if err != nil { @@ -520,6 +535,91 @@ func validateUIDMutation(t *testing.T, ctc *conversionTestContext) { } } +func validateDefaulting(t *testing.T, ctc *conversionTestContext) { + if _, defaulting := ctc.crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["defaults"]; !defaulting { + return + } + + ns := ctc.namespace + storageVersion := "v1beta1" + + for _, createVersion := range ctc.crd.Spec.Versions { + t.Run(fmt.Sprintf("getting objects created as %s", createVersion.Name), func(t *testing.T) { + name := "defaulting-" + createVersion.Name + client := ctc.versionedClient(ns, createVersion.Name) + + fixture := newConversionMultiVersionFixture(ns, name, createVersion.Name) + if err := unstructured.SetNestedField(fixture.Object, map[string]interface{}{}, "defaults"); err != nil { + t.Fatal(err) + } + created, err := client.Create(fixture, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + + // check that defaulting happens + // - in the request version when doing no-op conversion when deserializing + // - when reading back from storage in the storage version + // only the first is persisted. + defaults, found, err := unstructured.NestedMap(created.Object, "defaults") + if err != nil { + t.Fatal(err) + } else if !found { + t.Fatalf("expected .defaults to exist") + } + expectedLen := 1 + if !createVersion.Storage { + expectedLen++ + } + if len(defaults) != expectedLen { + t.Fatalf("after %s create expected .defaults to have %d values, but got: %v", createVersion.Name, expectedLen, defaults) + } + if _, found := defaults[createVersion.Name].(bool); !found { + t.Errorf("after %s create expected .defaults[%s] to be true, but .defaults is: %v", createVersion.Name, createVersion.Name, defaults) + } + if _, found := defaults[storageVersion].(bool); !found { + t.Errorf("after %s create expected .defaults[%s] to be true because it is the storage version, but .defaults is: %v", createVersion.Name, storageVersion, defaults) + } + + // verify that only the request version default is persisted + persisted, err := ctc.etcdObjectReader.GetStoredCustomResource(ns, name) + if err != nil { + t.Fatal(err) + } + if _, found, err := unstructured.NestedBool(persisted.Object, "defaults", storageVersion); err != nil { + t.Fatal(err) + } else if createVersion.Name != storageVersion && found { + t.Errorf("after %s create .defaults[storage version %s] not to be persisted, but got in etcd: %v", createVersion.Name, storageVersion, defaults) + } + + // check that when reading any other version, we do not default that version, but only the (non-persisted) storage version default + for _, v := range ctc.crd.Spec.Versions { + if v.Name == createVersion.Name { + // create version is persisted anyway, nothing to verify + continue + } + + got, err := ctc.versionedClient(ns, v.Name).Get(created.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, found, err := unstructured.NestedBool(got.Object, "defaults", v.Name); err != nil { + t.Fatal(err) + } else if v.Name != storageVersion && found { + t.Errorf("after %s GET expected .defaults[%s] not to be true because only storage version %s is defaulted on read, but .defaults is: %v", v.Name, v.Name, storageVersion, defaults) + } + + if _, found, err := unstructured.NestedBool(got.Object, "defaults", storageVersion); err != nil { + t.Fatal(err) + } else if !found { + t.Errorf("after non-create, non-storage %s GET expected .defaults[storage version %s] to be true, but .defaults is: %v", v.Name, storageVersion, defaults) + } + } + }) + } +} + func expectConversionFailureMessage(id, message string) func(t *testing.T, ctc *conversionTestContext) { return func(t *testing.T, ctc *conversionTestContext) { ns := ctc.namespace @@ -918,6 +1018,14 @@ var multiVersionFixture = &apiextensionsv1beta1.CustomResourceDefinition{ "num2": {Type: "integer"}, }, }, + "defaults": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "v1alpha1": {Type: "boolean"}, + "v1beta1": {Type: "boolean", Default: jsonPtr(true)}, + "v1beta2": {Type: "boolean"}, + }, + }, }, }, }, @@ -944,6 +1052,14 @@ var multiVersionFixture = &apiextensionsv1beta1.CustomResourceDefinition{ "num2": {Type: "integer"}, }, }, + "defaults": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "v1alpha1": {Type: "boolean", Default: jsonPtr(true)}, + "v1beta1": {Type: "boolean"}, + "v1beta2": {Type: "boolean"}, + }, + }, }, }, }, @@ -970,6 +1086,14 @@ var multiVersionFixture = &apiextensionsv1beta1.CustomResourceDefinition{ "num2": {Type: "integer"}, }, }, + "defaults": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "v1alpha1": {Type: "boolean"}, + "v1beta1": {Type: "boolean"}, + "v1beta2": {Type: "boolean", Default: jsonPtr(true)}, + }, + }, }, }, }, @@ -1089,3 +1213,12 @@ func closeOnCall(h http.Handler) (chan struct{}, http.Handler) { h.ServeHTTP(w, r) }) } + +func jsonPtr(x interface{}) *apiextensionsv1beta1.JSON { + bs, err := json.Marshal(x) + if err != nil { + panic(err) + } + ret := apiextensionsv1beta1.JSON{Raw: bs} + return &ret +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go new file mode 100644 index 00000000000..b9c5749ab0c --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/defaulting_test.go @@ -0,0 +1,229 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "strings" + "testing" + "time" + + "sigs.k8s.io/yaml" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" + utilfeaturetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/utils/pointer" + + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apiextensions-apiserver/pkg/features" + "k8s.io/apiextensions-apiserver/test/integration/fixtures" +) + +var defaultingFixture = &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.tests.apiextensions.k8s.io"}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "tests.apiextensions.k8s.io", + Version: "v1beta1", + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "foos", + Singular: "foo", + Kind: "Foo", + ListKind: "FooList", + }, + Scope: apiextensionsv1beta1.ClusterScoped, + PreserveUnknownFields: pointer.BoolPtr(false), + Subresources: &apiextensionsv1beta1.CustomResourceSubresources{ + Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{}, + }, + }, +} + +const defaultingFooSchema = ` +type: object +properties: + spec: + type: object + properties: + a: + type: string + default: "A" + b: + type: string + default: "B" + status: + type: object + properties: + a: + type: string + default: "A" + b: + type: string + default: "B" +` + +func TestCustomResourceDefaulting(t *testing.T) { + defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CustomResourceDefaulting, true)() + + tearDownFn, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) + if err != nil { + t.Fatal(err) + } + defer tearDownFn() + + crd := defaultingFixture.DeepCopy() + crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{} + if err := yaml.Unmarshal([]byte(defaultingFooSchema), &crd.Spec.Validation.OpenAPIV3Schema); err != nil { + t.Fatal(err) + } + + crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + + mustExist := func(obj map[string]interface{}, pths [][]string) { + for _, pth := range pths { + if _, found, _ := unstructured.NestedFieldNoCopy(obj, pth...); !found { + t.Errorf("Expected '%s' field exist", strings.Join(pth, ".")) + } + } + } + mustNotExist := func(obj map[string]interface{}, pths [][]string) { + for _, pth := range pths { + if fld, found, _ := unstructured.NestedFieldNoCopy(obj, pth...); found { + t.Errorf("Expected '%s' field to not exist, but it does: %v", strings.Join(pth, "."), fld) + } + } + } + updateCRD := func(update func(*apiextensionsv1beta1.CustomResourceDefinition)) { + var err error + for retry := 0; retry < 10; retry++ { + var obj *apiextensionsv1beta1.CustomResourceDefinition + obj, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + update(obj) + obj, err = apiExtensionClient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(obj) + if err != nil && apierrors.IsConflict(err) { + continue + } else if err != nil { + t.Fatal(err) + } + crd = obj + break + } + if err != nil { + t.Fatal(err) + } + } + addDefault := func(key string, value interface{}) { + updateCRD(func(obj *apiextensionsv1beta1.CustomResourceDefinition) { + for _, root := range []string{"spec", "status"} { + obj.Spec.Validation.OpenAPIV3Schema.Properties[root].Properties[key] = apiextensionsv1beta1.JSONSchemaProps{ + Type: "string", + Default: jsonPtr(value), + } + } + }) + } + removeDefault := func(key string) { + updateCRD(func(obj *apiextensionsv1beta1.CustomResourceDefinition) { + for _, root := range []string{"spec", "status"} { + props := obj.Spec.Validation.OpenAPIV3Schema.Properties[root].Properties[key] + props.Default = nil + obj.Spec.Validation.OpenAPIV3Schema.Properties[root].Properties[key] = props + } + }) + } + + t.Logf("Creating CR and expecting defaulted fields in spec, but status does not exist at all") + fooClient := dynamicClient.Resource(schema.GroupVersionResource{crd.Spec.Group, crd.Spec.Version, crd.Spec.Names.Plural}) + foo := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(fooInstance), &foo.Object); err != nil { + t.Fatal(err) + } + unstructured.SetNestedField(foo.Object, "a", "spec", "a") + unstructured.SetNestedField(foo.Object, "b", "status", "b") + foo, err = fooClient.Create(foo, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Unable to create CR: %v", err) + } + t.Logf("CR created: %#v", foo.UnstructuredContent()) + mustExist(foo.Object, [][]string{{"spec", "a"}, {"spec", "b"}}) + mustNotExist(foo.Object, [][]string{{"status"}}) + + t.Logf("Updating status and expecting 'a' and 'b' to show up.") + unstructured.SetNestedField(foo.Object, map[string]interface{}{}, "status") + if foo, err = fooClient.UpdateStatus(foo, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + mustExist(foo.Object, [][]string{{"spec", "a"}, {"spec", "b"}, {"status", "a"}, {"status", "b"}}) + + t.Logf("Add 'c' default and wait until GET sees it in both status and spec") + addDefault("c", "C") + if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + obj, err := fooClient.Get(foo.GetName(), metav1.GetOptions{}) + if err != nil { + return false, err + } + _, found, _ := unstructured.NestedString(obj.Object, "spec", "c") + foo = obj + return found, nil + }); err != nil { + t.Fatal(err) + } + mustExist(foo.Object, [][]string{{"spec", "a"}, {"spec", "b"}, {"spec", "c"}, {"status", "a"}, {"status", "b"}, {"status", "c"}}) + + t.Logf("Updating status, expecting 'c' to be set in spec and status") + if foo, err = fooClient.UpdateStatus(foo, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + mustExist(foo.Object, [][]string{{"spec", "a"}, {"spec", "b"}, {"spec", "c"}, {"status", "a"}, {"status", "b"}, {"status", "c"}}) + + t.Logf("Removing 'a', 'b' and `c` properties. Expecting that 'c' goes away in spec, but not in status. 'a' and 'b' were peristed.") + removeDefault("a") + removeDefault("b") + removeDefault("c") + if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) { + obj, err := fooClient.Get(foo.GetName(), metav1.GetOptions{}) + if err != nil { + return false, err + } + _, found, _ := unstructured.NestedString(obj.Object, "spec", "c") + foo = obj + return !found, nil + }); err != nil { + t.Fatal(err) + } + mustExist(foo.Object, [][]string{{"spec", "a"}, {"spec", "b"}, {"status", "a"}, {"status", "b"}, {"status", "c"}}) + mustNotExist(foo.Object, [][]string{{"spec", "c"}}) +} + +func jsonPtr(x interface{}) *apiextensionsv1beta1.JSON { + bs, err := json.Marshal(x) + if err != nil { + panic(err) + } + ret := apiextensionsv1beta1.JSON{Raw: bs} + return &ret +}