From 8235e389fb180dfc49cb25b930dd169554a9c44c Mon Sep 17 00:00:00 2001 From: Mehdy Bohlool Date: Tue, 14 Aug 2018 12:56:10 -0700 Subject: [PATCH 1/4] Example webhook implementation (used in E2E test) --- test/images/crd-conversion-webhook/BASEIMAGE | 4 + test/images/crd-conversion-webhook/Dockerfile | 18 ++ test/images/crd-conversion-webhook/Makefile | 25 +++ test/images/crd-conversion-webhook/README.md | 11 ++ test/images/crd-conversion-webhook/VERSION | 1 + test/images/crd-conversion-webhook/config.go | 51 +++++ .../converter/converter_test.go | 97 ++++++++++ .../converter/example_converter.go | 79 ++++++++ .../converter/framework.go | 178 ++++++++++++++++++ test/images/crd-conversion-webhook/main.go | 52 +++++ 10 files changed, 516 insertions(+) create mode 100644 test/images/crd-conversion-webhook/BASEIMAGE create mode 100644 test/images/crd-conversion-webhook/Dockerfile create mode 100644 test/images/crd-conversion-webhook/Makefile create mode 100644 test/images/crd-conversion-webhook/README.md create mode 100644 test/images/crd-conversion-webhook/VERSION create mode 100644 test/images/crd-conversion-webhook/config.go create mode 100644 test/images/crd-conversion-webhook/converter/converter_test.go create mode 100644 test/images/crd-conversion-webhook/converter/example_converter.go create mode 100644 test/images/crd-conversion-webhook/converter/framework.go create mode 100644 test/images/crd-conversion-webhook/main.go diff --git a/test/images/crd-conversion-webhook/BASEIMAGE b/test/images/crd-conversion-webhook/BASEIMAGE new file mode 100644 index 00000000000..114844f395e --- /dev/null +++ b/test/images/crd-conversion-webhook/BASEIMAGE @@ -0,0 +1,4 @@ +amd64=alpine:3.6 +arm=arm32v6/alpine:3.6 +arm64=arm64v8/alpine:3.6 +ppc64le=ppc64le/alpine:3.6 diff --git a/test/images/crd-conversion-webhook/Dockerfile b/test/images/crd-conversion-webhook/Dockerfile new file mode 100644 index 00000000000..1743be6bbdb --- /dev/null +++ b/test/images/crd-conversion-webhook/Dockerfile @@ -0,0 +1,18 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +ADD crd_conversion_webhook /crd_conversion_webhook +ENTRYPOINT ["/crd_conversion_webhook"] diff --git a/test/images/crd-conversion-webhook/Makefile b/test/images/crd-conversion-webhook/Makefile new file mode 100644 index 00000000000..b0decfb3086 --- /dev/null +++ b/test/images/crd-conversion-webhook/Makefile @@ -0,0 +1,25 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SRCS=crd_conversion_webhook +ARCH ?= amd64 +TARGET ?= $(CURDIR) +GOLANG_VERSION ?= latest +SRC_DIR = $(notdir $(shell pwd)) +export + +bin: + ../image-util.sh bin $(SRCS) + +.PHONY: bin diff --git a/test/images/crd-conversion-webhook/README.md b/test/images/crd-conversion-webhook/README.md new file mode 100644 index 00000000000..b04d34fde49 --- /dev/null +++ b/test/images/crd-conversion-webhook/README.md @@ -0,0 +1,11 @@ +# Kubernetes External Admission Webhook Test Image + +The image tests CustomResourceConversionWebhook. After deploying it to kubernetes cluster, +administrator needs to create a CustomResourceConversion.Webhook +in kubernetes cluster to use remote webhook for conversions. + +## Build the code + +```bash +make build +``` diff --git a/test/images/crd-conversion-webhook/VERSION b/test/images/crd-conversion-webhook/VERSION new file mode 100644 index 00000000000..d24646986ce --- /dev/null +++ b/test/images/crd-conversion-webhook/VERSION @@ -0,0 +1 @@ +1.13rev2 diff --git a/test/images/crd-conversion-webhook/config.go b/test/images/crd-conversion-webhook/config.go new file mode 100644 index 00000000000..b410efeb153 --- /dev/null +++ b/test/images/crd-conversion-webhook/config.go @@ -0,0 +1,51 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "crypto/tls" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/golang/glog" +) + +// Get a clientset with in-cluster config. +func getClient() *kubernetes.Clientset { + config, err := rest.InClusterConfig() + if err != nil { + glog.Fatal(err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + glog.Fatal(err) + } + return clientset +} + +func configTLS(config Config, clientset *kubernetes.Clientset) *tls.Config { + sCert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile) + if err != nil { + glog.Fatal(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{sCert}, + // TODO: uses mutual tls after we agree on what cert the apiserver should use. + // ClientAuth: tls.RequireAndVerifyClientCert, + } +} diff --git a/test/images/crd-conversion-webhook/converter/converter_test.go b/test/images/crd-conversion-webhook/converter/converter_test.go new file mode 100644 index 00000000000..e8e12391e2b --- /dev/null +++ b/test/images/crd-conversion-webhook/converter/converter_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package converter + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" +) + +func TestConverter(t *testing.T) { + sampleObj := `kind: ConversionReview +apiVersion: apiextensions.k8s.io/v1beta1 +request: + uid: 0000-0000-0000-0000 + desiredAPIVersion: stable.example.com/v2 + objects: + - apiVersion: stable.example.com/v1 + kind: CronTab + metadata: + name: my-new-cron-object + spec: + cronSpec: "* * * * */5" + image: my-awesome-cron-image + hostPort: "localhost:7070" +` + // First try json, it should fail as the data is taml + response := httptest.NewRecorder() + request, err := http.NewRequest("POST", "/convert", strings.NewReader(sampleObj)) + if err != nil { + t.Fatal(err) + } + request.Header.Add("Content-Type", "application/json") + ServeExampleConvert(response, request) + convertReview := v1beta1.ConversionReview{} + scheme := runtime.NewScheme() + jsonSerializer := json.NewSerializer(json.DefaultMetaFactory, scheme, scheme, false) + if _, _, err := jsonSerializer.Decode(response.Body.Bytes(), nil, &convertReview); err != nil { + t.Fatal(err) + } + if convertReview.Response.Result.Status != v1.StatusFailure { + t.Fatalf("expected the operation to fail when yaml is provided with json header") + } else if !strings.Contains(convertReview.Response.Result.Message, "json parse error") { + t.Fatalf("expected to fail on json parser, but it failed with: %v", convertReview.Response.Result.Message) + } + + // Now try yaml, and it should successfully convert + response = httptest.NewRecorder() + request, err = http.NewRequest("POST", "/convert", strings.NewReader(sampleObj)) + if err != nil { + t.Fatal(err) + } + request.Header.Add("Content-Type", "application/yaml") + ServeExampleConvert(response, request) + convertReview = v1beta1.ConversionReview{} + yamlSerializer := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme, scheme) + if _, _, err := yamlSerializer.Decode(response.Body.Bytes(), nil, &convertReview); err != nil { + t.Fatalf("cannot decode data: \n %v\n Error: %v", response.Body, err) + } + if convertReview.Response.Result.Status != v1.StatusSuccess { + t.Fatalf("cr conversion failed: %v", convertReview.Response) + } + convertedObj := unstructured.Unstructured{} + if _, _, err := yamlSerializer.Decode(convertReview.Response.ConvertedObjects[0].Raw, nil, &convertedObj); err != nil { + t.Fatal(err) + } + if e, a := "stable.example.com/v2", convertedObj.GetAPIVersion(); e != a { + t.Errorf("expected= %v, actual= %v", e, a) + } + if e, a := "localhost", convertedObj.Object["host"]; e != a { + t.Errorf("expected= %v, actual= %v", e, a) + } + if e, a := "7070", convertedObj.Object["port"]; e != a { + t.Errorf("expected= %v, actual= %v", e, a) + } +} diff --git a/test/images/crd-conversion-webhook/converter/example_converter.go b/test/images/crd-conversion-webhook/converter/example_converter.go new file mode 100644 index 00000000000..9dbb4d817e4 --- /dev/null +++ b/test/images/crd-conversion-webhook/converter/example_converter.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package converter + +import ( + "fmt" + "strings" + + "github.com/golang/glog" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func convertExampleCRD(Object *unstructured.Unstructured, toVersion string) (*unstructured.Unstructured, metav1.Status) { + glog.V(2).Info("converting crd") + + convertedObject := Object.DeepCopy() + fromVersion := Object.GetAPIVersion() + + if toVersion == fromVersion { + return nil, statusErrorWithMessage("conversion from a version to itself should not call the webhook: %s", toVersion) + } + + switch Object.GetAPIVersion() { + case "stable.example.com/v1": + switch toVersion { + case "stable.example.com/v2": + hostPort, ok := convertedObject.Object["hostPort"] + if ok { + delete(convertedObject.Object, "hostPort") + parts := strings.Split(hostPort.(string), ":") + if len(parts) != 2 { + return nil, statusErrorWithMessage("invalid hostPort value `%v`", hostPort) + } + convertedObject.Object["host"] = parts[0] + convertedObject.Object["port"] = parts[1] + } + default: + return nil, statusErrorWithMessage("unexpected conversion version %q", toVersion) + } + case "stable.example.com/v2": + switch toVersion { + case "stable.example.com/v1": + host, hasHost := convertedObject.Object["host"] + port, hasPort := convertedObject.Object["port"] + if hasHost || hasPort { + if !hasHost { + host = "" + } + if !hasPort { + port = "" + } + convertedObject.Object["hostPort"] = fmt.Sprintf("%s:%s", host, port) + delete(convertedObject.Object, "host") + delete(convertedObject.Object, "port") + } + default: + return nil, statusErrorWithMessage("unexpected conversion version %q", toVersion) + } + default: + return nil, statusErrorWithMessage("unexpected conversion version %q", fromVersion) + } + return convertedObject, statusSucceed() +} diff --git a/test/images/crd-conversion-webhook/converter/framework.go b/test/images/crd-conversion-webhook/converter/framework.go new file mode 100644 index 00000000000..e1a612dd8e0 --- /dev/null +++ b/test/images/crd-conversion-webhook/converter/framework.go @@ -0,0 +1,178 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package converter + +import ( + "bitbucket.org/ww/goautoneg" + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/golang/glog" + + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" +) + +// convertFunc is the user defined function for any conversion. The code in this file is a +// template that can be use for any CR conversion given this function. +type convertFunc func(Object *unstructured.Unstructured, version string) (*unstructured.Unstructured, metav1.Status) + +// conversionResponseFailureWithMessagef is a helper function to create an AdmissionResponse +// with a formatted embedded error message. +func conversionResponseFailureWithMessagef(msg string, params ...interface{}) *v1beta1.ConversionResponse { + return &v1beta1.ConversionResponse{ + Result: metav1.Status{ + Message: fmt.Sprintf(msg, params...), + Status: metav1.StatusFailure, + }, + } + +} + +func statusErrorWithMessage(msg string, params ...interface{}) metav1.Status { + return metav1.Status{ + Message: fmt.Sprintf(msg, params...), + Status: metav1.StatusFailure, + } +} + +func statusSucceed() metav1.Status { + return metav1.Status{ + Status: metav1.StatusSuccess, + } +} + +// doConversion converts the requested object given the conversion function and returns a conversion response. +// failures will be reported as Reason in the conversion response. +func doConversion(convertRequest *v1beta1.ConversionRequest, convert convertFunc) *v1beta1.ConversionResponse { + var convertedObjects []runtime.RawExtension + for _, obj := range convertRequest.Objects { + cr := unstructured.Unstructured{} + if err := cr.UnmarshalJSON(obj.Raw); err != nil { + glog.Error(err) + return conversionResponseFailureWithMessagef("failed to unmarshall object (%v) with error: %v", string(obj.Raw), err) + } + convertedCR, status := convert(&cr, convertRequest.DesiredAPIVersion) + if status.Status != metav1.StatusSuccess { + glog.Error(status.String()) + return &v1beta1.ConversionResponse{ + Result: status, + } + } + convertedCR.SetAPIVersion(convertRequest.DesiredAPIVersion) + convertedObjects = append(convertedObjects, runtime.RawExtension{Object: convertedCR}) + } + return &v1beta1.ConversionResponse{ + ConvertedObjects: convertedObjects, + Result: statusSucceed(), + } +} + +func serve(w http.ResponseWriter, r *http.Request, convert convertFunc) { + var body []byte + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } + } + + contentType := r.Header.Get("Content-Type") + serializer := getInputSerializer(contentType) + if serializer == nil { + msg := fmt.Sprintf("invalid Content-Type header `%s`", contentType) + glog.Errorf(msg) + http.Error(w, msg, http.StatusBadRequest) + return + } + + glog.V(2).Infof("handling request: %v", body) + convertReview := v1beta1.ConversionReview{} + if _, _, err := serializer.Decode(body, nil, &convertReview); err != nil { + glog.Error(err) + convertReview.Response = conversionResponseFailureWithMessagef("failed to deserialize body (%v) with error %v", string(body), err) + } else { + convertReview.Response = doConversion(convertReview.Request, convert) + convertReview.Response.UID = convertReview.Request.UID + } + glog.V(2).Info(fmt.Sprintf("sending response: %v", convertReview.Response)) + + // reset the request, it is not needed in a response. + convertReview.Request = &v1beta1.ConversionRequest{} + + accept := r.Header.Get("Accept") + outSerializer := getOutputSerializer(accept) + if outSerializer == nil { + msg := fmt.Sprintf("invalid accept header `%s`", accept) + glog.Errorf(msg) + http.Error(w, msg, http.StatusBadRequest) + return + } + err := outSerializer.Encode(&convertReview, w) + if err != nil { + glog.Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +// ServeExampleConvert servers endpoint for the example converter defined as convertExampleCRD function. +func ServeExampleConvert(w http.ResponseWriter, r *http.Request) { + serve(w, r, convertExampleCRD) +} + +type mediaType struct { + Type, SubType string +} + +var scheme = runtime.NewScheme() +var serializers = map[mediaType]runtime.Serializer{ + {"application", "json"}: json.NewSerializer(json.DefaultMetaFactory, scheme, scheme, false), + {"application", "yaml"}: json.NewYAMLSerializer(json.DefaultMetaFactory, scheme, scheme), +} + +func getInputSerializer(contentType string) runtime.Serializer { + parts := strings.SplitN(contentType, "/", 2) + if len(parts) != 2 { + return nil + } + return serializers[mediaType{parts[0], parts[1]}] +} + +func getOutputSerializer(accept string) runtime.Serializer { + if len(accept) == 0 { + return serializers[mediaType{"application", "json"}] + } + + clauses := goautoneg.ParseAccept(accept) + for _, clause := range clauses { + for k, v := range serializers { + switch { + case clause.Type == k.Type && clause.SubType == k.SubType, + clause.Type == k.Type && clause.SubType == "*", + clause.Type == "*" && clause.SubType == "*": + return v + } + } + } + + return nil +} diff --git a/test/images/crd-conversion-webhook/main.go b/test/images/crd-conversion-webhook/main.go new file mode 100644 index 00000000000..6c80074f13c --- /dev/null +++ b/test/images/crd-conversion-webhook/main.go @@ -0,0 +1,52 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "net/http" + + "k8s.io/kubernetes/test/images/crd-conversion-webhook/converter" +) + +// Config contains the server (the webhook) cert and key. +type Config struct { + CertFile string + KeyFile string +} + +func (c *Config) addFlags() { + flag.StringVar(&c.CertFile, "tls-cert-file", c.CertFile, ""+ + "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ + "after server cert).") + flag.StringVar(&c.KeyFile, "tls-private-key-file", c.KeyFile, ""+ + "File containing the default x509 private key matching --tls-cert-file.") +} + +func main() { + var config Config + config.addFlags() + flag.Parse() + + http.HandleFunc("/crdconvert", converter.ServeExampleConvert) + clientset := getClient() + server := &http.Server{ + Addr: ":443", + TLSConfig: configTLS(config, clientset), + } + server.ListenAndServeTLS("", "") +} From ea54a0c50420df701d15c6404f1b09606cdea355 Mon Sep 17 00:00:00 2001 From: Mehdy Bohlool Date: Wed, 15 Aug 2018 16:16:44 -0700 Subject: [PATCH 2/4] E2E Test --- .../apimachinery/crd_conversion_webhook.go | 396 ++++++++++++++++++ test/e2e/apimachinery/webhook.go | 30 +- test/e2e/framework/crd_util.go | 65 ++- test/utils/image/manifest.go | 1 + 4 files changed, 466 insertions(+), 26 deletions(-) create mode 100644 test/e2e/apimachinery/crd_conversion_webhook.go diff --git a/test/e2e/apimachinery/crd_conversion_webhook.go b/test/e2e/apimachinery/crd_conversion_webhook.go new file mode 100644 index 00000000000..f58e94da3b5 --- /dev/null +++ b/test/e2e/apimachinery/crd_conversion_webhook.go @@ -0,0 +1,396 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apimachinery + +import ( + "k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver/test/integration" + "time" + + apps "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/intstr" + utilversion "k8s.io/apimachinery/pkg/util/version" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + _ "github.com/stretchr/testify/assert" +) + +const ( + secretCRDName = "sample-custom-resource-conversion-webhook-secret" + deploymentCRDName = "sample-crd-conversion-webhook-deployment" + serviceCRDName = "e2e-test-crd-conversion-webhook" + roleBindingCRDName = "crd-conversion-webhook-auth-reader" +) + +var serverCRDConversionWebhookVersion = utilversion.MustParseSemantic("v1.13.0-alpha") + +var apiVersions = []v1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + { + Name: "v2", + Served: true, + Storage: false, + }, +} + +var alternativeApiVersions = []v1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: false, + }, + { + Name: "v2", + Served: true, + Storage: true, + }, +} + +var _ = SIGDescribe("CustomResourceConversionWebhook [Feature:CustomResourceWebhookConversion]", func() { + var context *certContext + f := framework.NewDefaultFramework("crd-webhook") + + var client clientset.Interface + var namespaceName string + + BeforeEach(func() { + client = f.ClientSet + namespaceName = f.Namespace.Name + + // Make sure the relevant provider supports conversion webhook + framework.SkipUnlessServerVersionGTE(serverCRDConversionWebhookVersion, f.ClientSet.Discovery()) + + By("Setting up server cert") + context = setupServerCert(f.Namespace.Name, serviceCRDName) + createAuthReaderRoleBindingForCRDConversion(f, f.Namespace.Name) + + deployCustomResourceWebhookAndService(f, imageutils.GetE2EImage(imageutils.CRDConversionWebhook), context) + }) + + AfterEach(func() { + cleanCRDWebhookTest(client, namespaceName) + }) + + It("Should be able to convert from CR v1 to CR v2", func() { + testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions, + &v1beta1.WebhookClientConfig{ + CABundle: context.signingCert, + Service: &v1beta1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceCRDName, + Path: strPtr("/crdconvert"), + }}) + if err != nil { + return + } + defer testcrd.CleanUp() + testCustomResourceConversionWebhook(f, testcrd.Crd, testcrd.DynamicClients) + }) + + It("Should be able to convert a non homogeneous list of CRs", func() { + testcrd, err := framework.CreateMultiVersionTestCRD(f, "stable.example.com", apiVersions, + &v1beta1.WebhookClientConfig{ + CABundle: context.signingCert, + Service: &v1beta1.ServiceReference{ + Namespace: f.Namespace.Name, + Name: serviceCRDName, + Path: strPtr("/crdconvert"), + }}) + if err != nil { + return + } + defer testcrd.CleanUp() + testCRListConversion(f, testcrd) + }) +}) + +func cleanCRDWebhookTest(client clientset.Interface, namespaceName string) { + _ = client.CoreV1().Services(namespaceName).Delete(serviceCRDName, nil) + _ = client.AppsV1().Deployments(namespaceName).Delete(deploymentCRDName, nil) + _ = client.CoreV1().Secrets(namespaceName).Delete(secretCRDName, nil) + _ = client.RbacV1().RoleBindings("kube-system").Delete(roleBindingCRDName, nil) +} + +func createAuthReaderRoleBindingForCRDConversion(f *framework.Framework, namespace string) { + By("Create role binding to let cr conversion webhook read extension-apiserver-authentication") + client := f.ClientSet + // Create the role binding to allow the webhook read the extension-apiserver-authentication configmap + _, err := client.RbacV1().RoleBindings("kube-system").Create(&rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBindingCRDName, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "", + Kind: "Role", + Name: "extension-apiserver-authentication-reader", + }, + // Webhook uses the default service account. + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "default", + Namespace: namespace, + }, + }, + }) + if err != nil && errors.IsAlreadyExists(err) { + framework.Logf("role binding %s already exists", roleBindingCRDName) + } else { + framework.ExpectNoError(err, "creating role binding %s:webhook to access configMap", namespace) + } +} + +func deployCustomResourceWebhookAndService(f *framework.Framework, image string, context *certContext) { + By("Deploying the custom resource conversion webhook pod") + client := f.ClientSet + + // Creating the secret that contains the webhook's cert. + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretCRDName, + }, + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + "tls.crt": context.cert, + "tls.key": context.key, + }, + } + namespace := f.Namespace.Name + _, err := client.CoreV1().Secrets(namespace).Create(secret) + framework.ExpectNoError(err, "creating secret %q in namespace %q", secretName, namespace) + + // Create the deployment of the webhook + podLabels := map[string]string{"app": "sample-crd-conversion-webhook", "crd-webhook": "true"} + replicas := int32(1) + zero := int64(0) + mounts := []v1.VolumeMount{ + { + Name: "crd-conversion-webhook-certs", + ReadOnly: true, + MountPath: "/webhook.local.config/certificates", + }, + } + volumes := []v1.Volume{ + { + Name: "crd-conversion-webhook-certs", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{SecretName: secretCRDName}, + }, + }, + } + containers := []v1.Container{ + { + Name: "sample-crd-conversion-webhook", + VolumeMounts: mounts, + Args: []string{ + "--tls-cert-file=/webhook.local.config/certificates/tls.crt", + "--tls-private-key-file=/webhook.local.config/certificates/tls.key", + "--alsologtostderr", + "-v=4", + "2>&1", + }, + Image: image, + }, + } + d := &apps.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentCRDName, + Labels: podLabels, + }, + Spec: apps.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: podLabels, + }, + Strategy: apps.DeploymentStrategy{ + Type: apps.RollingUpdateDeploymentStrategyType, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + Spec: v1.PodSpec{ + TerminationGracePeriodSeconds: &zero, + Containers: containers, + Volumes: volumes, + }, + }, + }, + } + deployment, err := client.AppsV1().Deployments(namespace).Create(d) + framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentCRDName, namespace) + By("Wait for the deployment to be ready") + err = framework.WaitForDeploymentRevisionAndImage(client, namespace, deploymentCRDName, "1", image) + framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, namespace) + err = framework.WaitForDeploymentComplete(client, deployment) + framework.ExpectNoError(err, "waiting for the deployment status valid", image, deploymentCRDName, namespace) + + By("Deploying the webhook service") + + serviceLabels := map[string]string{"crd-webhook": "true"} + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: serviceCRDName, + Labels: map[string]string{"test": "crd-webhook"}, + }, + Spec: v1.ServiceSpec{ + Selector: serviceLabels, + Ports: []v1.ServicePort{ + { + Protocol: "TCP", + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + }, + } + _, err = client.CoreV1().Services(namespace).Create(service) + framework.ExpectNoError(err, "creating service %s in namespace %s", serviceCRDName, namespace) + + By("Verifying the service has paired with the endpoint") + err = framework.WaitForServiceEndpointsNum(client, namespace, serviceCRDName, 1, 1*time.Second, 30*time.Second) + framework.ExpectNoError(err, "waiting for service %s/%s have %d endpoint", namespace, serviceCRDName, 1) +} + +func verifyV1Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) { + Expect(obj.GetAPIVersion()).To(BeEquivalentTo(crd.Spec.Group + "/v1")) + hostPort, exists := obj.Object["hostPort"] + Expect(exists).To(BeTrue()) + Expect(hostPort).To(BeEquivalentTo("localhost:8080")) + _, hostExists := obj.Object["host"] + Expect(hostExists).To(BeFalse()) + _, portExists := obj.Object["port"] + Expect(portExists).To(BeFalse()) +} + +func verifyV2Object(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, obj *unstructured.Unstructured) { + Expect(obj.GetAPIVersion()).To(BeEquivalentTo(crd.Spec.Group + "/v2")) + _, hostPortExists := obj.Object["hostPort"] + Expect(hostPortExists).To(BeFalse()) + host, hostExists := obj.Object["host"] + Expect(hostExists).To(BeTrue()) + Expect(host).To(BeEquivalentTo("localhost")) + port, portExists := obj.Object["port"] + Expect(portExists).To(BeTrue()) + Expect(port).To(BeEquivalentTo("8080")) +} + +func testCustomResourceConversionWebhook(f *framework.Framework, crd *v1beta1.CustomResourceDefinition, customResourceClients map[string]dynamic.ResourceInterface) { + name := "cr-instance-1" + By("Creating a v1 custom resource") + crInstance := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": crd.Spec.Names.Kind, + "apiVersion": crd.Spec.Group + "/v1", + "metadata": map[string]interface{}{ + "name": name, + "namespace": f.Namespace.Name, + }, + "hostPort": "localhost:8080", + }, + } + _, err := customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{}) + Expect(err).To(BeNil()) + By("v2 custom resource should be converted") + v2crd, err := customResourceClients["v2"].Get(name, metav1.GetOptions{}) + verifyV2Object(f, crd, v2crd) +} + +func testCRListConversion(f *framework.Framework, testCrd *framework.TestCrd) { + crd := testCrd.Crd + customResourceClients := testCrd.DynamicClients + name1 := "cr-instance-1" + name2 := "cr-instance-2" + By("Creating a v1 custom resource") + crInstance := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": crd.Spec.Names.Kind, + "apiVersion": crd.Spec.Group + "/v1", + "metadata": map[string]interface{}{ + "name": name1, + "namespace": f.Namespace.Name, + }, + "hostPort": "localhost:8080", + }, + } + _, err := customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{}) + Expect(err).To(BeNil()) + + // Now cr-instance-1 is stored as v1. lets change storage version + crd, err = integration.UpdateCustomResourceDefinitionWithRetry(testCrd.ApiExtensionClient, crd.Name, func(c *v1beta1.CustomResourceDefinition) { + c.Spec.Versions = alternativeApiVersions + }) + Expect(err).To(BeNil()) + By("Create a v2 custom resource") + crInstance = &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": crd.Spec.Names.Kind, + "apiVersion": crd.Spec.Group + "/v1", + "metadata": map[string]interface{}{ + "name": name2, + "namespace": f.Namespace.Name, + }, + "hostPort": "localhost:8080", + }, + } + + // After changing a CRD, the resources for versions will be re-created that can be result in + // cancelled connection (e.g. "grpc connection closed" or "context canceled"). + // Just retrying fixes that. + for i := 0; i < 5; i++ { + _, err = customResourceClients["v1"].Create(crInstance, metav1.CreateOptions{}) + if err == nil { + break + } + } + Expect(err).To(BeNil()) + + // Now that we have a v1 and v2 object, both list operation in v1 and v2 should work as expected. + + By("List CRs in v1") + list, err := customResourceClients["v1"].List(metav1.ListOptions{}) + Expect(err).To(BeNil()) + Expect(len(list.Items)).To(BeIdenticalTo(2)) + Expect((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) || + (list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)).To(BeTrue()) + verifyV1Object(f, crd, &list.Items[0]) + verifyV1Object(f, crd, &list.Items[1]) + + By("List CRs in v2") + list, err = customResourceClients["v2"].List(metav1.ListOptions{}) + Expect(err).To(BeNil()) + Expect(len(list.Items)).To(BeIdenticalTo(2)) + Expect((list.Items[0].GetName() == name1 && list.Items[1].GetName() == name2) || + (list.Items[0].GetName() == name2 && list.Items[1].GetName() == name1)).To(BeTrue()) + verifyV2Object(f, crd, &list.Items[0]) + verifyV2Object(f, crd, &list.Items[1]) +} diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index 72c5ea67c09..e2686dccce5 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -136,7 +136,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { defer testcrd.CleanUp() webhookCleanup := registerWebhookForCustomResource(f, context, testcrd) defer webhookCleanup() - testCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient) + testCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient()) }) It("Should unconditionally reject operations on fail closed webhook", func() { @@ -173,7 +173,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { defer testcrd.CleanUp() webhookCleanup := registerMutatingWebhookForCustomResource(f, context, testcrd) defer webhookCleanup() - testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClient) + testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.GetV1DynamicClient()) }) It("Should deny crd creation", func() { @@ -1157,7 +1157,7 @@ func registerWebhookForCustomResource(f *framework.Framework, context *certConte Operations: []v1beta1.OperationType{v1beta1.Create}, Rule: v1beta1.Rule{ APIGroups: []string{testcrd.ApiGroup}, - APIVersions: []string{testcrd.ApiVersion}, + APIVersions: testcrd.GetAPIVersions(), Resources: []string{testcrd.GetPluralName()}, }, }}, @@ -1198,7 +1198,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c Operations: []v1beta1.OperationType{v1beta1.Create}, Rule: v1beta1.Rule{ APIGroups: []string{testcrd.ApiGroup}, - APIVersions: []string{testcrd.ApiVersion}, + APIVersions: testcrd.GetAPIVersions(), Resources: []string{testcrd.GetPluralName()}, }, }}, @@ -1217,7 +1217,7 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, context *c Operations: []v1beta1.OperationType{v1beta1.Create}, Rule: v1beta1.Rule{ APIGroups: []string{testcrd.ApiGroup}, - APIVersions: []string{testcrd.ApiVersion}, + APIVersions: testcrd.GetAPIVersions(), Resources: []string{testcrd.GetPluralName()}, }, }}, @@ -1343,12 +1343,18 @@ func testCRDDenyWebhook(f *framework.Framework) { name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, "deny") kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, "deny") group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) - apiVersion := "v1" + apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + } testcrd := &framework.TestCrd{ - Name: name, - Kind: kind, - ApiGroup: group, - ApiVersion: apiVersion, + Name: name, + Kind: kind, + ApiGroup: group, + Versions: apiVersions, } // Creating a custom resource definition for use by assorted tests. @@ -1370,8 +1376,8 @@ func testCRDDenyWebhook(f *framework.Framework) { }, }, Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: testcrd.ApiGroup, - Version: testcrd.ApiVersion, + Group: testcrd.ApiGroup, + Versions: testcrd.Versions, Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ Plural: testcrd.GetPluralName(), Singular: testcrd.Name, diff --git a/test/e2e/framework/crd_util.go b/test/e2e/framework/crd_util.go index 0421158d4aa..dc48d188d76 100644 --- a/test/e2e/framework/crd_util.go +++ b/test/e2e/framework/crd_util.go @@ -35,25 +35,23 @@ type TestCrd struct { Name string Kind string ApiGroup string - ApiVersion string + Versions []apiextensionsv1beta1.CustomResourceDefinitionVersion ApiExtensionClient *crdclientset.Clientset Crd *apiextensionsv1beta1.CustomResourceDefinition - DynamicClient dynamic.ResourceInterface + DynamicClients map[string]dynamic.ResourceInterface CleanUp CleanCrdFn } // CreateTestCRD creates a new CRD specifically for the calling test. -func CreateTestCRD(f *Framework) (*TestCrd, error) { +func CreateMultiVersionTestCRD(f *Framework, group string, apiVersions []apiextensionsv1beta1.CustomResourceDefinitionVersion, conversionWebhook *apiextensionsv1beta1.WebhookClientConfig) (*TestCrd, error) { suffix := randomSuffix() name := fmt.Sprintf("e2e-test-%s-%s-crd", f.BaseName, suffix) kind := fmt.Sprintf("E2e-test-%s-%s-crd", f.BaseName, suffix) - group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) - apiVersion := "v1" testcrd := &TestCrd{ - Name: name, - Kind: kind, - ApiGroup: group, - ApiVersion: apiVersion, + Name: name, + Kind: kind, + ApiGroup: group, + Versions: apiVersions, } // Creating a custom resource definition for use by assorted tests. @@ -75,6 +73,13 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) { crd := newCRDForTest(testcrd) + if conversionWebhook != nil { + crd.Spec.Conversion = &apiextensionsv1beta1.CustomResourceConversion{ + Strategy: "Webhook", + WebhookClientConfig: conversionWebhook, + } + } + //create CRD and waits for the resource to be recognized and available. crd, err = fixtures.CreateNewCustomResourceDefinitionWatchUnsafe(crd, apiExtensionClient) if err != nil { @@ -82,12 +87,17 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) { return nil, err } - gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: crd.Spec.Version, Resource: crd.Spec.Names.Plural} - resourceClient := dynamicClient.Resource(gvr).Namespace(f.Namespace.Name) + resourceClients := map[string]dynamic.ResourceInterface{} + for _, v := range crd.Spec.Versions { + if v.Served { + gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: v.Name, Resource: crd.Spec.Names.Plural} + resourceClients[v.Name] = dynamicClient.Resource(gvr).Namespace(f.Namespace.Name) + } + } testcrd.ApiExtensionClient = apiExtensionClient testcrd.Crd = crd - testcrd.DynamicClient = resourceClient + testcrd.DynamicClients = resourceClients testcrd.CleanUp = func() error { err := fixtures.DeleteCustomResourceDefinition(crd, apiExtensionClient) if err != nil { @@ -98,13 +108,26 @@ func CreateTestCRD(f *Framework) (*TestCrd, error) { return testcrd, nil } +// CreateTestCRD creates a new CRD specifically for the calling test. +func CreateTestCRD(f *Framework) (*TestCrd, error) { + group := fmt.Sprintf("%s-crd-test.k8s.io", f.BaseName) + apiVersions := []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + }, + } + return CreateMultiVersionTestCRD(f, group, apiVersions, nil) +} + // newCRDForTest generates a CRD definition for the test func newCRDForTest(testcrd *TestCrd) *apiextensionsv1beta1.CustomResourceDefinition { return &apiextensionsv1beta1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: testcrd.GetMetaName()}, Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: testcrd.ApiGroup, - Version: testcrd.ApiVersion, + Group: testcrd.ApiGroup, + Versions: testcrd.Versions, Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ Plural: testcrd.GetPluralName(), Singular: testcrd.Name, @@ -130,3 +153,17 @@ func (c *TestCrd) GetPluralName() string { func (c *TestCrd) GetListName() string { return c.Name + "List" } + +func (c *TestCrd) GetAPIVersions() []string { + ret := []string{} + for _, v := range c.Versions { + if v.Served { + ret = append(ret, v.Name) + } + } + return ret +} + +func (c *TestCrd) GetV1DynamicClient() dynamic.ResourceInterface { + return c.DynamicClients["v1"] +} diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 4e83ca47a0e..2e5015bd3a7 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -92,6 +92,7 @@ var ( // Preconfigured image configs var ( + CRDConversionWebhook = Config{e2eRegistry, "crd-conversion-webhook", "1.13rev2"} AdmissionWebhook = Config{e2eRegistry, "webhook", "1.13v1"} APIServer = Config{e2eRegistry, "sample-apiserver", "1.10"} AppArmorLoader = Config{e2eRegistry, "apparmor-loader", "1.0"} From e2ca575d0f40d94578c7c0babce543ab5199d2d0 Mon Sep 17 00:00:00 2001 From: Mehdy Bohlool Date: Fri, 9 Nov 2018 14:55:06 -0800 Subject: [PATCH 3/4] CRD Conversion --- cmd/kube-apiserver/app/apiextensions.go | 5 + cmd/kube-apiserver/app/server.go | 3 +- .../pkg/apiserver/apiserver.go | 13 +- .../pkg/apiserver/conversion/converter.go | 67 +++- .../pkg/apiserver/conversion/nop_converter.go | 6 +- .../apiserver/conversion/webhook_converter.go | 350 ++++++++++++++++++ .../pkg/apiserver/customresource_handler.go | 19 +- .../apiserver/customresource_handler_test.go | 14 +- .../pkg/cmd/server/options/options.go | 14 + .../test/integration/helpers.go | 4 +- .../test/integration/validation_test.go | 2 +- .../test/integration/versioning_test.go | 2 +- .../plugin/webhook/generic/webhook.go | 1 + .../apimachinery/crd_conversion_webhook.go | 2 +- 14 files changed, 471 insertions(+), 31 deletions(-) create mode 100644 staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/webhook_converter.go diff --git a/cmd/kube-apiserver/app/apiextensions.go b/cmd/kube-apiserver/app/apiextensions.go index ee5f743daad..d179644a250 100644 --- a/cmd/kube-apiserver/app/apiextensions.go +++ b/cmd/kube-apiserver/app/apiextensions.go @@ -28,6 +28,7 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/webhook" kubeexternalinformers "k8s.io/client-go/informers" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" ) @@ -38,6 +39,8 @@ func createAPIExtensionsConfig( pluginInitializers []admission.PluginInitializer, commandOptions *options.ServerRunOptions, masterCount int, + serviceResolver webhook.ServiceResolver, + authResolverWrapper webhook.AuthenticationInfoResolverWrapper, ) (*apiextensionsapiserver.Config, error) { // make a shallow copy to let us twiddle a few things // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the apiextensions @@ -74,6 +77,8 @@ func createAPIExtensionsConfig( ExtraConfig: apiextensionsapiserver.ExtraConfig{ CRDRESTOptionsGetter: apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions), MasterCount: masterCount, + AuthResolverWrapper: authResolverWrapper, + ServiceResolver: serviceResolver, }, } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index 0d21910d6b0..0da41a9b1b7 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -165,7 +165,8 @@ func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan } // If additional API servers are added, they should be gated. - apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount) + apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, kubeAPIServerConfig.ExtraConfig.VersionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount, + serviceResolver, webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, kubeAPIServerConfig.GenericConfig.LoopbackClientConfig)) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go index f1fc89ba96c..b6830d762c8 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go @@ -31,6 +31,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" + "k8s.io/apiserver/pkg/util/webhook" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" @@ -78,6 +79,11 @@ type ExtraConfig struct { // MasterCount is used to detect whether cluster is HA, and if it is // the CRD Establishing will be hold by 5 seconds. MasterCount int + + // ServiceResolver is used in CR webhook converters to resolve webhook's service names + ServiceResolver webhook.ServiceResolver + // AuthResolverWrapper is used in CR webhook converters + AuthResolverWrapper webhook.AuthenticationInfoResolverWrapper } type Config struct { @@ -167,7 +173,7 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) delegate: delegateHandler, } establishingController := establish.NewEstablishingController(s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions(), crdClient.Apiextensions()) - crdHandler := NewCustomResourceDefinitionHandler( + crdHandler, err := NewCustomResourceDefinitionHandler( versionDiscoveryHandler, groupDiscoveryHandler, s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions(), @@ -175,8 +181,13 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) c.ExtraConfig.CRDRESTOptionsGetter, c.GenericConfig.AdmissionControl, establishingController, + c.ExtraConfig.ServiceResolver, + c.ExtraConfig.AuthResolverWrapper, c.ExtraConfig.MasterCount, ) + if err != nil { + return nil, err + } s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", crdHandler) s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", crdHandler) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go index a66f82969dc..9345e9bd27b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/converter.go @@ -20,39 +20,74 @@ import ( "fmt" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/apiserver/pkg/util/webhook" ) -// NewCRDConverter returns a new CRD converter based on the conversion settings in crd object. -func NewCRDConverter(crd *apiextensions.CustomResourceDefinition) (safe, unsafe runtime.ObjectConvertor) { +// CRConverterFactory is the factory for all CR converters. +type CRConverterFactory struct { + // webhookConverterFactory is the factory for webhook converters. + // This field should not be used if CustomResourceWebhookConversion feature is disabled. + webhookConverterFactory *webhookConverterFactory +} + +// NewCRConverterFactory creates a new CRConverterFactory +func NewCRConverterFactory(serviceResolver webhook.ServiceResolver, authResolverWrapper webhook.AuthenticationInfoResolverWrapper) (*CRConverterFactory, error) { + converterFactory := &CRConverterFactory{} + if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) { + webhookConverterFactory, err := newWebhookConverterFactory(serviceResolver, authResolverWrapper) + if err != nil { + return nil, err + } + converterFactory.webhookConverterFactory = webhookConverterFactory + } + return converterFactory, nil +} + +// NewConverter returns a new CR converter based on the conversion settings in crd object. +func (m *CRConverterFactory) NewConverter(crd *apiextensions.CustomResourceDefinition) (safe, unsafe runtime.ObjectConvertor, err error) { validVersions := map[schema.GroupVersion]bool{} for _, version := range crd.Spec.Versions { validVersions[schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}] = true } - // The only converter right now is nopConverter. More converters will be returned based on the - // CRD object when they introduced. - unsafe = &crdConverter{ - clusterScoped: crd.Spec.Scope == apiextensions.ClusterScoped, - delegate: &nopConverter{ - validVersions: validVersions, - }, + switch crd.Spec.Conversion.Strategy { + case apiextensions.NoneConverter: + unsafe = &crConverter{ + clusterScoped: crd.Spec.Scope == apiextensions.ClusterScoped, + delegate: &nopConverter{ + validVersions: validVersions, + }, + } + return &safeConverterWrapper{unsafe}, unsafe, nil + case apiextensions.WebhookConverter: + if !utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourceWebhookConversion) { + return nil, nil, fmt.Errorf("webhook conversion is disabled on this cluster") + } + unsafe, err := m.webhookConverterFactory.NewWebhookConverter(validVersions, crd) + if err != nil { + return nil, nil, err + } + return &safeConverterWrapper{unsafe}, unsafe, nil } - return &safeConverterWrapper{unsafe}, unsafe + + return nil, nil, fmt.Errorf("unknown conversion strategy %q for CRD %s", crd.Spec.Conversion.Strategy, crd.Name) } -var _ runtime.ObjectConvertor = &crdConverter{} +var _ runtime.ObjectConvertor = &crConverter{} -// crdConverter extends the delegate with generic CRD conversion behaviour. The delegate will implement the +// crConverter extends the delegate with generic CR conversion behaviour. The delegate will implement the // user defined conversion strategy given in the CustomResourceDefinition. -type crdConverter struct { +type crConverter struct { delegate runtime.ObjectConvertor clusterScoped bool } -func (c *crdConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, value string) (string, string, error) { +func (c *crConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, value string) (string, string, error) { // We currently only support metadata.namespace and metadata.name. switch { case label == "metadata.name": @@ -64,12 +99,12 @@ func (c *crdConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, val } } -func (c *crdConverter) Convert(in, out, context interface{}) error { +func (c *crConverter) Convert(in, out, context interface{}) error { return c.delegate.Convert(in, out, context) } // ConvertToVersion converts in object to the given gvk in place and returns the same `in` object. -func (c *crdConverter) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) { +func (c *crConverter) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) { // Run the converter on the list items instead of list itself if list, ok := in.(*unstructured.UnstructuredList); ok { for i := range list.Items { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go index 7fae8137596..8791238c5d0 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/nop_converter.go @@ -49,11 +49,11 @@ func (c *nopConverter) Convert(in, out, context interface{}) error { outGVK := unstructOut.GroupVersionKind() if !c.validVersions[outGVK.GroupVersion()] { - return fmt.Errorf("request to convert CRD from an invalid group/version: %s", outGVK.String()) + return fmt.Errorf("request to convert CR from an invalid group/version: %s", outGVK.String()) } inGVK := unstructIn.GroupVersionKind() if !c.validVersions[inGVK.GroupVersion()] { - return fmt.Errorf("request to convert CRD to an invalid group/version: %s", inGVK.String()) + return fmt.Errorf("request to convert CR to an invalid group/version: %s", inGVK.String()) } unstructOut.SetUnstructuredContent(unstructIn.UnstructuredContent()) @@ -72,7 +72,7 @@ func (c *nopConverter) ConvertToVersion(in runtime.Object, target runtime.GroupV return nil, fmt.Errorf("%v is unstructured and is not suitable for converting to %q", kind, target) } if !c.validVersions[gvk.GroupVersion()] { - return nil, fmt.Errorf("request to convert CRD to an invalid group/version: %s", gvk.String()) + return nil, fmt.Errorf("request to convert CR to an invalid group/version: %s", gvk.String()) } in.GetObjectKind().SetGroupVersionKind(gvk) return in, nil diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/webhook_converter.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/webhook_converter.go new file mode 100644 index 00000000000..3cc295ea187 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/webhook_converter.go @@ -0,0 +1,350 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "context" + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apiserver/pkg/util/webhook" + "k8s.io/client-go/rest" + + internal "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" +) + +type webhookConverterFactory struct { + clientManager webhook.ClientManager +} + +func newWebhookConverterFactory(serviceResolver webhook.ServiceResolver, authResolverWrapper webhook.AuthenticationInfoResolverWrapper) (*webhookConverterFactory, error) { + clientManager, err := webhook.NewClientManager(v1beta1.SchemeGroupVersion, v1beta1.AddToScheme) + if err != nil { + return nil, err + } + authInfoResolver, err := webhook.NewDefaultAuthenticationInfoResolver("") + if err != nil { + return nil, err + } + // Set defaults which may be overridden later. + clientManager.SetAuthenticationInfoResolver(authInfoResolver) + clientManager.SetAuthenticationInfoResolverWrapper(authResolverWrapper) + clientManager.SetServiceResolver(serviceResolver) + return &webhookConverterFactory{clientManager}, nil +} + +// webhookConverter is a converter that calls an external webhook to do the CR conversion. +type webhookConverter struct { + validVersions map[schema.GroupVersion]bool + clientManager webhook.ClientManager + restClient *rest.RESTClient + name string + nopConverter nopConverter +} + +func webhookClientConfigForCRD(crd *internal.CustomResourceDefinition) *webhook.ClientConfig { + apiConfig := crd.Spec.Conversion.WebhookClientConfig + ret := webhook.ClientConfig{ + Name: fmt.Sprintf("conversion_webhook_for_%s", crd.Name), + CABundle: apiConfig.CABundle, + } + if apiConfig.URL != nil { + ret.URL = *apiConfig.URL + } + if apiConfig.Service != nil { + ret.Service = &webhook.ClientConfigService{ + Name: apiConfig.Service.Name, + Namespace: apiConfig.Service.Namespace, + } + if apiConfig.Service.Path != nil { + ret.Service.Path = *apiConfig.Service.Path + } + } + return &ret +} + +var _ runtime.ObjectConvertor = &webhookConverter{} + +func (f *webhookConverterFactory) NewWebhookConverter(validVersions map[schema.GroupVersion]bool, crd *internal.CustomResourceDefinition) (*webhookConverter, error) { + restClient, err := f.clientManager.HookClient(*webhookClientConfigForCRD(crd)) + if err != nil { + return nil, err + } + return &webhookConverter{ + clientManager: f.clientManager, + validVersions: validVersions, + restClient: restClient, + name: crd.Name, + nopConverter: nopConverter{validVersions: validVersions}, + }, nil +} + +func (webhookConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, label, value string) (string, string, error) { + return "", "", errors.New("unstructured cannot convert field labels") +} + +func (c *webhookConverter) Convert(in, out, context interface{}) error { + unstructIn, ok := in.(*unstructured.Unstructured) + if !ok { + return fmt.Errorf("input type %T in not valid for unstructured conversion", in) + } + + unstructOut, ok := out.(*unstructured.Unstructured) + if !ok { + return fmt.Errorf("output type %T in not valid for unstructured conversion", out) + } + + outGVK := unstructOut.GroupVersionKind() + if !c.validVersions[outGVK.GroupVersion()] { + return fmt.Errorf("request to convert CR from an invalid group/version: %s", outGVK.String()) + } + inGVK := unstructIn.GroupVersionKind() + if !c.validVersions[inGVK.GroupVersion()] { + return fmt.Errorf("request to convert CR to an invalid group/version: %s", inGVK.String()) + } + + converted, err := c.ConvertToVersion(unstructIn, outGVK.GroupVersion()) + if err != nil { + return err + } + unstructuredConverted, ok := converted.(runtime.Unstructured) + if !ok { + // this should not happened + return fmt.Errorf("CR conversion failed") + } + unstructOut.SetUnstructuredContent(unstructuredConverted.UnstructuredContent()) + return nil +} + +func createConversionReview(obj runtime.Object, apiVersion string) *v1beta1.ConversionReview { + listObj, isList := obj.(*unstructured.UnstructuredList) + var objects []runtime.RawExtension + if isList { + for i := 0; i < len(listObj.Items); i++ { + // Only sent item for conversion, if the apiVersion is different + if listObj.Items[i].GetAPIVersion() != apiVersion { + objects = append(objects, runtime.RawExtension{Object: &listObj.Items[i]}) + } + } + } else { + if obj.GetObjectKind().GroupVersionKind().GroupVersion().String() != apiVersion { + objects = []runtime.RawExtension{{Object: obj}} + } + } + return &v1beta1.ConversionReview{ + Request: &v1beta1.ConversionRequest{ + Objects: objects, + DesiredAPIVersion: apiVersion, + UID: uuid.NewUUID(), + }, + Response: &v1beta1.ConversionResponse{}, + } +} + +func getRawExtensionObject(rx runtime.RawExtension) (runtime.Object, error) { + if rx.Object != nil { + return rx.Object, nil + } + u := unstructured.Unstructured{} + err := u.UnmarshalJSON(rx.Raw) + if err != nil { + return nil, err + } + return &u, nil +} + +// getTargetGroupVersion returns group/version which should be used to convert in objects to. +// String version of the return item is APIVersion. +func getTargetGroupVersion(in runtime.Object, target runtime.GroupVersioner) (schema.GroupVersion, error) { + fromGVK := in.GetObjectKind().GroupVersionKind() + toGVK, ok := target.KindForGroupVersionKinds([]schema.GroupVersionKind{fromGVK}) + if !ok { + // TODO: should this be a typed error? + return schema.GroupVersion{}, fmt.Errorf("%v is unstructured and is not suitable for converting to %q", fromGVK.String(), target) + } + return toGVK.GroupVersion(), nil +} + +func (c *webhookConverter) ConvertToVersion(in runtime.Object, target runtime.GroupVersioner) (runtime.Object, error) { + // In general, the webhook should not do any defaulting or validation. A special case of that is an empty object + // conversion that must result an empty object and practically is the same as nopConverter. + // A smoke test in API machinery calls the converter on empty objects. As this case happens consistently + // it special cased here not to call webhook converter. The test initiated here: + // https://github.com/kubernetes/kubernetes/blob/dbb448bbdcb9e440eee57024ffa5f1698956a054/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher.go#L201 + if isEmptyUnstructuredObject(in) { + return c.nopConverter.ConvertToVersion(in, target) + } + + toGV, err := getTargetGroupVersion(in, target) + if err != nil { + return nil, err + } + if !c.validVersions[toGV] { + return nil, fmt.Errorf("request to convert CR to an invalid group/version: %s", toGV.String()) + } + fromGV := in.GetObjectKind().GroupVersionKind().GroupVersion() + if !c.validVersions[fromGV] { + return nil, fmt.Errorf("request to convert CR from an invalid group/version: %s", fromGV.String()) + } + listObj, isList := in.(*unstructured.UnstructuredList) + if isList { + for i, item := range listObj.Items { + fromGV := item.GroupVersionKind().GroupVersion() + if !c.validVersions[fromGV] { + return nil, fmt.Errorf("input list has invalid group/version `%v` at `%v` index", fromGV, i) + } + } + } + + request := createConversionReview(in, toGV.String()) + if len(request.Request.Objects) == 0 { + if !isList { + return in, nil + } + out := listObj.DeepCopy() + out.SetAPIVersion(toGV.String()) + return out, nil + } + response := &v1beta1.ConversionReview{} + // TODO: Figure out if adding one second timeout make sense here. + ctx := context.TODO() + r := c.restClient.Post().Context(ctx).Body(request).Do() + if err := r.Into(response); err != nil { + // TODO: Return a webhook specific error to be able to convert it to meta.Status + return nil, fmt.Errorf("calling to conversion webhook failed for %s: %v", c.name, err) + } + + if response.Response == nil { + // TODO: Return a webhook specific error to be able to convert it to meta.Status + return nil, fmt.Errorf("conversion webhook response was absent for %s", c.name) + } + + if response.Response.Result.Status != v1.StatusSuccess { + // TODO return status message as error + return nil, fmt.Errorf("conversion request failed for %v, Response: %v", in.GetObjectKind(), response) + } + + if len(response.Response.ConvertedObjects) != len(request.Request.Objects) { + return nil, fmt.Errorf("expected %v converted objects, got %v", len(request.Request.Objects), len(response.Response.ConvertedObjects)) + } + + if isList { + convertedList := listObj.DeepCopy() + // Collection of items sent for conversion is different than list items + // because only items that needed conversion has been sent. + convertedIndex := 0 + for i := 0; i < len(listObj.Items); i++ { + if listObj.Items[i].GetAPIVersion() == toGV.String() { + // This item has not been sent for conversion, skip it. + continue + } + converted, err := getRawExtensionObject(response.Response.ConvertedObjects[convertedIndex]) + convertedIndex++ + original := listObj.Items[i] + if err != nil { + return nil, fmt.Errorf("invalid converted object at index %v: %v", convertedIndex, err) + } + if e, a := toGV, converted.GetObjectKind().GroupVersionKind().GroupVersion(); e != a { + return nil, fmt.Errorf("invalid converted object at index %v: invalid groupVersion, e=%v, a=%v", convertedIndex, e, a) + } + if e, a := original.GetObjectKind().GroupVersionKind().Kind, converted.GetObjectKind().GroupVersionKind().Kind; e != a { + return nil, fmt.Errorf("invalid converted object at index %v: invalid kind, e=%v, a=%v", convertedIndex, e, a) + } + unstructConverted, ok := converted.(*unstructured.Unstructured) + if !ok { + // this should not happened + return nil, fmt.Errorf("CR conversion failed") + } + if err := validateConvertedObject(&listObj.Items[i], unstructConverted); err != nil { + return nil, fmt.Errorf("invalid converted object at index %v: %v", convertedIndex, err) + } + convertedList.Items[i] = *unstructConverted + } + convertedList.SetAPIVersion(toGV.String()) + return convertedList, nil + } + + if len(response.Response.ConvertedObjects) != 1 { + // This should not happened + return nil, fmt.Errorf("CR conversion failed") + } + converted, err := getRawExtensionObject(response.Response.ConvertedObjects[0]) + if err != nil { + return nil, err + } + if e, a := toGV, converted.GetObjectKind().GroupVersionKind().GroupVersion(); e != a { + return nil, fmt.Errorf("invalid converted object: invalid groupVersion, e=%v, a=%v", e, a) + } + if e, a := in.GetObjectKind().GroupVersionKind().Kind, converted.GetObjectKind().GroupVersionKind().Kind; e != a { + return nil, fmt.Errorf("invalid converted object: invalid kind, e=%v, a=%v", e, a) + } + unstructConverted, ok := converted.(*unstructured.Unstructured) + if !ok { + // this should not happened + return nil, fmt.Errorf("CR conversion failed") + } + unstructIn, ok := in.(*unstructured.Unstructured) + if !ok { + // this should not happened + return nil, fmt.Errorf("CR conversion failed") + } + if err := validateConvertedObject(unstructIn, unstructConverted); err != nil { + return nil, fmt.Errorf("invalid converted object: %v", err) + } + return converted, nil +} + +func validateConvertedObject(unstructIn, unstructOut *unstructured.Unstructured) error { + if e, a := unstructIn.GetKind(), unstructOut.GetKind(); e != a { + return fmt.Errorf("must have the same kind: %v != %v", e, a) + } + if e, a := unstructIn.GetName(), unstructOut.GetName(); e != a { + return fmt.Errorf("must have the same name: %v != %v", e, a) + } + if e, a := unstructIn.GetNamespace(), unstructOut.GetNamespace(); e != a { + return fmt.Errorf("must have the same namespace: %v != %v", e, a) + } + if e, a := unstructIn.GetUID(), unstructOut.GetUID(); e != a { + return fmt.Errorf("must have the same UID: %v != %v", e, a) + } + return nil +} + +// isEmptyUnstructuredObject returns true if in is an empty unstructured object, i.e. an unstructured object that does +// not have any field except apiVersion and kind. +func isEmptyUnstructuredObject(in runtime.Object) bool { + u, ok := in.(*unstructured.Unstructured) + if !ok { + return false + } + if len(u.Object) != 2 { + return false + } + if _, ok := u.Object["kind"]; !ok { + return false + } + if _, ok := u.Object["apiVersion"]; !ok { + return false + } + return true +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 3903d9fcb43..b4be7b11578 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -67,6 +67,7 @@ import ( apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apiextensions-apiserver/pkg/registry/customresource" "k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor" + "k8s.io/apiserver/pkg/util/webhook" ) // crdHandler serves the `/apis` endpoint. @@ -93,6 +94,8 @@ type crdHandler struct { // MasterCount is used to implement sleep to improve // CRD establishing process for HA clusters. masterCount int + + converterFactory *conversion.CRConverterFactory } // crdInfo stores enough information to serve the storage for the custom resource @@ -129,7 +132,9 @@ func NewCustomResourceDefinitionHandler( restOptionsGetter generic.RESTOptionsGetter, admission admission.Interface, establishingController *establish.EstablishingController, - masterCount int) *crdHandler { + serviceResolver webhook.ServiceResolver, + authResolverWrapper webhook.AuthenticationInfoResolverWrapper, + masterCount int) (*crdHandler, error) { ret := &crdHandler{ versionDiscoveryHandler: versionDiscoveryHandler, groupDiscoveryHandler: groupDiscoveryHandler, @@ -147,10 +152,15 @@ func NewCustomResourceDefinitionHandler( ret.removeDeadStorage() }, }) + crConverterFactory, err := conversion.NewCRConverterFactory(serviceResolver, authResolverWrapper) + if err != nil { + return nil, err + } + ret.converterFactory = crConverterFactory ret.customStorage.Store(crdStorageMap{}) - return ret + return ret, nil } func (r *crdHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -433,7 +443,10 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource scaleScopes := map[string]handlers.RequestScope{} for _, v := range crd.Spec.Versions { - safeConverter, unsafeConverter := conversion.NewCRDConverter(crd) + safeConverter, unsafeConverter, err := r.converterFactory.NewConverter(crd) + if err != nil { + return nil, err + } // In addition to Unstructured objects (Custom Resources), we also may sometimes need to // decode unversioned Options objects, so we delegate to parameterScheme for such types. parameterScheme := runtime.NewScheme() diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go index fa7c84e6631..62ed7cca25d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler_test.go @@ -79,14 +79,24 @@ func TestConvertFieldLabel(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - crd := apiextensions.CustomResourceDefinition{} + crd := apiextensions.CustomResourceDefinition{ + Spec: apiextensions.CustomResourceDefinitionSpec{ + Conversion: &apiextensions.CustomResourceConversion{ + Strategy: "None", + }, + }, + } if test.clusterScoped { crd.Spec.Scope = apiextensions.ClusterScoped } else { crd.Spec.Scope = apiextensions.NamespaceScoped } - _, c := conversion.NewCRDConverter(&crd) + f, err := conversion.NewCRConverterFactory(nil, nil) + if err != nil { + t.Fatal(err) + } + _, c, err := f.NewConverter(&crd) label, value, err := c.ConvertFieldLabel(schema.GroupVersionKind{}, test.label, "value") if e, a := test.expectError, err != nil; e != a { diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go index 995a3155555..1f564d1cfb3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/options.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "net" + "net/url" "github.com/spf13/pflag" @@ -30,6 +31,9 @@ import ( genericregistry "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/apiserver/pkg/util/proxy" + "k8s.io/apiserver/pkg/util/webhook" + "k8s.io/client-go/listers/core/v1" ) const defaultEtcdPathPrefix = "/registry/apiextensions.kubernetes.io" @@ -94,6 +98,8 @@ func (o CustomResourceDefinitionsServerOptions) Config() (*apiserver.Config, err GenericConfig: serverConfig, ExtraConfig: apiserver.ExtraConfig{ CRDRESTOptionsGetter: NewCRDRESTOptionsGetter(*o.RecommendedOptions.Etcd), + ServiceResolver: &serviceResolver{serverConfig.SharedInformerFactory.Core().V1().Services().Lister()}, + AuthResolverWrapper: webhook.NewDefaultAuthenticationInfoResolverWrapper(nil, serverConfig.LoopbackClientConfig), }, } return config, nil @@ -114,3 +120,11 @@ func NewCRDRESTOptionsGetter(etcdOptions genericoptions.EtcdOptions) genericregi return ret } + +type serviceResolver struct { + services v1.ServiceLister +} + +func (r *serviceResolver) ResolveEndpoint(namespace, name string) (*url.URL, error) { + return proxy.ResolveCluster(r.services, namespace, name) +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go index 20d63c10007..7a7d6611e86 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/helpers.go @@ -76,8 +76,8 @@ func newNamespacedCustomResourceClient(ns string, client dynamic.Interface, crd return newNamespacedCustomResourceVersionedClient(ns, client, crd, crd.Spec.Versions[0].Name) } -// updateCustomResourceDefinitionWithRetry updates a CRD, retrying up to 5 times on version conflict errors. -func updateCustomResourceDefinitionWithRetry(client clientset.Interface, name string, update func(*apiextensionsv1beta1.CustomResourceDefinition)) (*apiextensionsv1beta1.CustomResourceDefinition, error) { +// UpdateCustomResourceDefinitionWithRetry updates a CRD, retrying up to 5 times on version conflict errors. +func UpdateCustomResourceDefinitionWithRetry(client clientset.Interface, name string, update func(*apiextensionsv1beta1.CustomResourceDefinition)) (*apiextensionsv1beta1.CustomResourceDefinition, error) { for i := 0; i < 5; i++ { crd, err := client.ApiextensionsV1beta1().CustomResourceDefinitions().Get(name, metav1.GetOptions{}) if err != nil { diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go index 3c3322fcd01..997183c519f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/validation_test.go @@ -445,7 +445,7 @@ func TestCRValidationOnCRDUpdate(t *testing.T) { } // update the CRD to a less stricter schema - _, err = updateCustomResourceDefinitionWithRetry(apiExtensionClient, "noxus.mygroup.example.com", func(crd *apiextensionsv1beta1.CustomResourceDefinition) { + _, err = UpdateCustomResourceDefinitionWithRetry(apiExtensionClient, "noxus.mygroup.example.com", func(crd *apiextensionsv1beta1.CustomResourceDefinition) { validationSchema, err := getSchemaForVersion(crd, v.Name) if err != nil { t.Fatal(err) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go index 7a813ab49cf..b56d3950e7d 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/versioning_test.go @@ -64,7 +64,7 @@ func TestInternalVersionIsHandlerVersion(t *testing.T) { // update validation via update because the cache priming in CreateNewCustomResourceDefinition will fail otherwise t.Logf("Updating CRD to validate apiVersion") - noxuDefinition, err = updateCustomResourceDefinitionWithRetry(apiExtensionClient, noxuDefinition.Name, func(crd *apiextensionsv1beta1.CustomResourceDefinition) { + noxuDefinition, err = UpdateCustomResourceDefinitionWithRetry(apiExtensionClient, noxuDefinition.Name, func(crd *apiextensionsv1beta1.CustomResourceDefinition) { crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{ OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ diff --git a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go index 408187fd1f7..13b898bca9b 100644 --- a/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go +++ b/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go @@ -164,6 +164,7 @@ func (a *Webhook) Dispatch(attr admission.Attributes) error { return admission.NewForbidden(attr, fmt.Errorf("not yet ready to handle request")) } hooks := a.hookSource.Webhooks() + // TODO: Figure out if adding one second timeout make sense here. ctx := context.TODO() var relevantHooks []*v1beta1.Webhook diff --git a/test/e2e/apimachinery/crd_conversion_webhook.go b/test/e2e/apimachinery/crd_conversion_webhook.go index f58e94da3b5..57614cc8856 100644 --- a/test/e2e/apimachinery/crd_conversion_webhook.go +++ b/test/e2e/apimachinery/crd_conversion_webhook.go @@ -17,13 +17,13 @@ limitations under the License. package apimachinery import ( - "k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver/test/integration" "time" apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apiextensions-apiserver/test/integration" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" From d51d0164c5cb1657c98c5823078c38ae5fdfda80 Mon Sep 17 00:00:00 2001 From: Mehdy Bohlool Date: Thu, 1 Nov 2018 15:34:57 -0700 Subject: [PATCH 4/4] Update generated files --- .../Godeps/Godeps.json | 12 +++++ .../pkg/apiserver/BUILD | 1 + .../pkg/apiserver/conversion/BUILD | 8 ++++ .../pkg/cmd/server/options/BUILD | 3 ++ test/e2e/apimachinery/BUILD | 3 ++ test/images/BUILD | 1 + test/images/crd-conversion-webhook/BUILD | 40 ++++++++++++++++ .../crd-conversion-webhook/converter/BUILD | 47 +++++++++++++++++++ 8 files changed, 115 insertions(+) create mode 100644 test/images/crd-conversion-webhook/BUILD create mode 100644 test/images/crd-conversion-webhook/converter/BUILD diff --git a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json index 89f7422679e..5730ff31adc 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json +++ b/staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json @@ -2342,6 +2342,10 @@ "ImportPath": "k8s.io/apimachinery/pkg/util/sets", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apimachinery/pkg/util/uuid", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apimachinery/pkg/util/validation", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2450,6 +2454,10 @@ "ImportPath": "k8s.io/apiserver/pkg/util/logs", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/apiserver/pkg/util/proxy", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/apiserver/pkg/util/webhook", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" @@ -2474,6 +2482,10 @@ "ImportPath": "k8s.io/client-go/kubernetes/scheme", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, + { + "ImportPath": "k8s.io/client-go/listers/core/v1", + "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, { "ImportPath": "k8s.io/client-go/rest", "Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD index 5ad15a9b68c..9aa35c30d46 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD @@ -67,6 +67,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/server/storage:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", "//staging/src/k8s.io/client-go/scale:go_default_library", "//staging/src/k8s.io/client-go/scale/scheme/autoscalingv1:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD index 93a4b1364d1..560e9388341 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion/BUILD @@ -5,15 +5,23 @@ go_library( srcs = [ "converter.go", "nop_converter.go", + "webhook_converter.go", ], importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion", importpath = "k8s.io/apiextensions-apiserver/pkg/apiserver/conversion", visibility = ["//visibility:public"], deps = [ "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/BUILD index 48e77981168..08a69b3436c 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/cmd/server/options/BUILD @@ -14,6 +14,9 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server:go_default_library", "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/proxy:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/webhook:go_default_library", + "//staging/src/k8s.io/client-go/listers/core/v1:go_default_library", "//vendor/github.com/spf13/pflag:go_default_library", ], ) diff --git a/test/e2e/apimachinery/BUILD b/test/e2e/apimachinery/BUILD index bc57dc1698b..ea3fbc2d673 100644 --- a/test/e2e/apimachinery/BUILD +++ b/test/e2e/apimachinery/BUILD @@ -11,6 +11,7 @@ go_library( "aggregator.go", "certs.go", "chunking.go", + "crd_conversion_webhook.go", "crd_watch.go", "custom_resource_definition.go", "etcd_failure.go", @@ -36,9 +37,11 @@ go_library( "//staging/src/k8s.io/api/batch/v1beta1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/extensions/v1beta1:go_default_library", + "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/test/integration:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/test/images/BUILD b/test/images/BUILD index a797bb178a7..a7f93f6cf3c 100644 --- a/test/images/BUILD +++ b/test/images/BUILD @@ -12,6 +12,7 @@ filegroup( srcs = [ ":package-srcs", "//test/images/apparmor-loader:all-srcs", + "//test/images/crd-conversion-webhook:all-srcs", "//test/images/echoserver:all-srcs", "//test/images/entrypoint-tester:all-srcs", "//test/images/fakegitserver:all-srcs", diff --git a/test/images/crd-conversion-webhook/BUILD b/test/images/crd-conversion-webhook/BUILD new file mode 100644 index 00000000000..dff07ed4d97 --- /dev/null +++ b/test/images/crd-conversion-webhook/BUILD @@ -0,0 +1,40 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "config.go", + "main.go", + ], + importpath = "k8s.io/kubernetes/test/images/crd-conversion-webhook", + visibility = ["//visibility:private"], + deps = [ + "//staging/src/k8s.io/client-go/kubernetes:go_default_library", + "//staging/src/k8s.io/client-go/rest:go_default_library", + "//test/images/crd-conversion-webhook/converter:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + ], +) + +go_binary( + name = "crd-conversion-webhook", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//test/images/crd-conversion-webhook/converter:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/images/crd-conversion-webhook/converter/BUILD b/test/images/crd-conversion-webhook/converter/BUILD new file mode 100644 index 00000000000..de5b0dbb947 --- /dev/null +++ b/test/images/crd-conversion-webhook/converter/BUILD @@ -0,0 +1,47 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "example_converter.go", + "framework.go", + ], + importpath = "k8s.io/kubernetes/test/images/crd-conversion-webhook/converter", + visibility = ["//visibility:public"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", + "//vendor/bitbucket.org/ww/goautoneg:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["converter_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +)