From 7484892009d12ce974b3f36291d8f17ce4ab6973 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 2 May 2019 12:02:59 +0200 Subject: [PATCH 1/8] apiextensions: add preserveUnknownFields API --- .../pkg/apis/apiextensions/fuzzer/fuzzer.go | 4 + .../pkg/apis/apiextensions/types.go | 6 + .../apis/apiextensions/v1beta1/defaults.go | 3 + .../apiextensions/v1beta1/generated.pb.go | 400 ++++++++++-------- .../apiextensions/v1beta1/generated.proto | 6 + .../pkg/apis/apiextensions/v1beta1/types.go | 6 + .../v1beta1/zz_generated.conversion.go | 2 + .../v1beta1/zz_generated.deepcopy.go | 5 + .../apiextensions/zz_generated.deepcopy.go | 5 + 9 files changed, 255 insertions(+), 182 deletions(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go index 3344ce2be25..51ab78021ed 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go @@ -25,6 +25,7 @@ import ( "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/utils/pointer" ) var swaggerMetadataDescriptions = metav1.ObjectMeta{}.SwaggerDoc() @@ -69,6 +70,9 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} { if obj.Conversion.Strategy == apiextensions.WebhookConverter && len(obj.Conversion.ConversionReviewVersions) == 0 { obj.Conversion.ConversionReviewVersions = []string{"v1beta1"} } + if obj.PreserveUnknownFields == nil { + obj.PreserveUnknownFields = pointer.BoolPtr(true) + } }, func(obj *apiextensions.CustomResourceDefinition, c fuzz.Continue) { c.FuzzNoCustom(obj) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go index ce6aae71182..03344e2d025 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go @@ -73,6 +73,12 @@ type CustomResourceDefinitionSpec struct { // `conversion` defines conversion settings for the CRD. Conversion *CustomResourceConversion + + // preserveUnknownFields disables pruning of object fields which are not + // specified in the OpenAPI schema. apiVersion, kind, metadata and known + // fields inside metadata are always preserved. + // Defaults to true in v1beta and will default to false in v1. + PreserveUnknownFields *bool } // CustomResourceConversion describes how to convert different versions of a CR. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go index ff9e28a112d..f87b06fa398 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go @@ -72,6 +72,9 @@ func SetDefaults_CustomResourceDefinitionSpec(obj *CustomResourceDefinitionSpec) if obj.Conversion.Strategy == WebhookConverter && len(obj.Conversion.ConversionReviewVersions) == 0 { obj.Conversion.ConversionReviewVersions = []string{SchemeGroupVersion.Version} } + if obj.PreserveUnknownFields == nil { + obj.PreserveUnknownFields = utilpointer.BoolPtr(true) + } } // SetDefaults_ServiceReference sets defaults for Webhook's ServiceReference diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.pb.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.pb.go index 0be4a2ea828..31736626375 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.pb.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.pb.go @@ -709,6 +709,16 @@ func (m *CustomResourceDefinitionSpec) MarshalTo(dAtA []byte) (int, error) { } i += n13 } + if m.PreserveUnknownFields != nil { + dAtA[i] = 0x50 + i++ + if *m.PreserveUnknownFields { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -1855,6 +1865,9 @@ func (m *CustomResourceDefinitionSpec) Size() (n int) { l = m.Conversion.Size() n += 1 + l + sovGenerated(uint64(l)) } + if m.PreserveUnknownFields != nil { + n += 2 + } return n } @@ -2339,6 +2352,7 @@ func (this *CustomResourceDefinitionSpec) String() string { `Versions:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.Versions), "CustomResourceDefinitionVersion", "CustomResourceDefinitionVersion", 1), `&`, ``, 1) + `,`, `AdditionalPrinterColumns:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.AdditionalPrinterColumns), "CustomResourceColumnDefinition", "CustomResourceColumnDefinition", 1), `&`, ``, 1) + `,`, `Conversion:` + strings.Replace(fmt.Sprintf("%v", this.Conversion), "CustomResourceConversion", "CustomResourceConversion", 1) + `,`, + `PreserveUnknownFields:` + valueToStringGenerated(this.PreserveUnknownFields) + `,`, `}`, }, "") return s @@ -4316,6 +4330,27 @@ func (m *CustomResourceDefinitionSpec) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PreserveUnknownFields", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + b := bool(v != 0) + m.PreserveUnknownFields = &b default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -7466,186 +7501,187 @@ func init() { } var fileDescriptorGenerated = []byte{ - // 2884 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x73, 0x1c, 0x47, - 0x15, 0xf7, 0xec, 0x6a, 0xa5, 0x55, 0x4b, 0xb2, 0xa4, 0xb6, 0xad, 0x8c, 0x15, 0x67, 0x57, 0xde, - 0x90, 0x20, 0x82, 0xbd, 0x4a, 0x4c, 0x42, 0x42, 0xaa, 0x38, 0x68, 0x25, 0x25, 0xa5, 0xc4, 0xfa, - 0xa0, 0xd7, 0x4e, 0x0c, 0xf9, 0x6c, 0xcd, 0xf4, 0xae, 0xc6, 0x9a, 0x2f, 0x4f, 0xcf, 0xac, 0xa4, - 0x0a, 0x50, 0x7c, 0x54, 0x0a, 0x8a, 0x02, 0x42, 0x91, 0x5c, 0x28, 0xe0, 0x10, 0x28, 0x2e, 0x1c, - 0xe0, 0x00, 0x37, 0xf8, 0x03, 0x72, 0x4c, 0x71, 0xca, 0x81, 0xda, 0xc2, 0x9b, 0x33, 0x37, 0xaa, - 0xa8, 0xd2, 0x89, 0xea, 0x8f, 0xe9, 0x99, 0x9d, 0xdd, 0xb5, 0x5d, 0xf1, 0x6e, 0xcc, 0x4d, 0xf3, - 0xbe, 0x7e, 0xaf, 0x5f, 0xbf, 0x7e, 0xfd, 0xfa, 0xad, 0x40, 0xe3, 0xe0, 0x39, 0x5a, 0xb5, 0xbc, - 0x95, 0x83, 0x68, 0x8f, 0x04, 0x2e, 0x09, 0x09, 0x5d, 0x69, 0x11, 0xd7, 0xf4, 0x82, 0x15, 0xc9, - 0xc0, 0xbe, 0x45, 0x8e, 0x42, 0xe2, 0x52, 0xcb, 0x73, 0xe9, 0x65, 0xec, 0x5b, 0x94, 0x04, 0x2d, - 0x12, 0xac, 0xf8, 0x07, 0x4d, 0xc6, 0xa3, 0xdd, 0x02, 0x2b, 0xad, 0xa7, 0xf6, 0x48, 0x88, 0x9f, - 0x5a, 0x69, 0x12, 0x97, 0x04, 0x38, 0x24, 0x66, 0xd5, 0x0f, 0xbc, 0xd0, 0x83, 0x5f, 0x17, 0xe6, - 0xaa, 0x5d, 0xd2, 0x6f, 0x29, 0x73, 0x55, 0xff, 0xa0, 0xc9, 0x78, 0xb4, 0x5b, 0xa0, 0x2a, 0xcd, - 0x2d, 0x5e, 0x6e, 0x5a, 0xe1, 0x7e, 0xb4, 0x57, 0x35, 0x3c, 0x67, 0xa5, 0xe9, 0x35, 0xbd, 0x15, - 0x6e, 0x75, 0x2f, 0x6a, 0xf0, 0x2f, 0xfe, 0xc1, 0xff, 0x12, 0x68, 0x8b, 0x4f, 0x27, 0xce, 0x3b, - 0xd8, 0xd8, 0xb7, 0x5c, 0x12, 0x1c, 0x27, 0x1e, 0x3b, 0x24, 0xc4, 0x2b, 0xad, 0x1e, 0x1f, 0x17, - 0x57, 0x06, 0x69, 0x05, 0x91, 0x1b, 0x5a, 0x0e, 0xe9, 0x51, 0xf8, 0xea, 0xdd, 0x14, 0xa8, 0xb1, - 0x4f, 0x1c, 0x9c, 0xd5, 0xab, 0x9c, 0x68, 0x60, 0x7e, 0xcd, 0x73, 0x5b, 0x24, 0x60, 0xab, 0x44, - 0xe4, 0x56, 0x44, 0x68, 0x08, 0x6b, 0x20, 0x1f, 0x59, 0xa6, 0xae, 0x2d, 0x69, 0xcb, 0x93, 0xb5, - 0x27, 0x3f, 0x6a, 0x97, 0x4f, 0x75, 0xda, 0xe5, 0xfc, 0xf5, 0xcd, 0xf5, 0x93, 0x76, 0xf9, 0xe2, - 0x20, 0xa4, 0xf0, 0xd8, 0x27, 0xb4, 0x7a, 0x7d, 0x73, 0x1d, 0x31, 0x65, 0xf8, 0x22, 0x98, 0x37, - 0x09, 0xb5, 0x02, 0x62, 0xae, 0xee, 0x6e, 0xbe, 0x22, 0xec, 0xeb, 0x39, 0x6e, 0xf1, 0xbc, 0xb4, - 0x38, 0xbf, 0x9e, 0x15, 0x40, 0xbd, 0x3a, 0xf0, 0x06, 0x98, 0xf0, 0xf6, 0x6e, 0x12, 0x23, 0xa4, - 0x7a, 0x7e, 0x29, 0xbf, 0x3c, 0x75, 0xe5, 0x72, 0x35, 0xd9, 0x41, 0xe5, 0x02, 0xdf, 0x36, 0xb9, - 0xd8, 0x2a, 0xc2, 0x87, 0x1b, 0xf1, 0xce, 0xd5, 0x66, 0x25, 0xda, 0xc4, 0x8e, 0xb0, 0x82, 0x62, - 0x73, 0x95, 0xdf, 0xe7, 0x00, 0x4c, 0x2f, 0x9e, 0xfa, 0x9e, 0x4b, 0xc9, 0x50, 0x56, 0x4f, 0xc1, - 0x9c, 0xc1, 0x2d, 0x87, 0xc4, 0x94, 0xb8, 0x7a, 0xee, 0xb3, 0x78, 0xaf, 0x4b, 0xfc, 0xb9, 0xb5, - 0x8c, 0x39, 0xd4, 0x03, 0x00, 0xaf, 0x81, 0xf1, 0x80, 0xd0, 0xc8, 0x0e, 0xf5, 0xfc, 0x92, 0xb6, - 0x3c, 0x75, 0xe5, 0xd2, 0x40, 0x28, 0x9e, 0xdf, 0x2c, 0xf9, 0xaa, 0xad, 0xa7, 0xaa, 0xf5, 0x10, - 0x87, 0x11, 0xad, 0x9d, 0x96, 0x48, 0xe3, 0x88, 0xdb, 0x40, 0xd2, 0x56, 0xe5, 0xc7, 0x39, 0x30, - 0x97, 0x8e, 0x52, 0xcb, 0x22, 0x87, 0xf0, 0x10, 0x4c, 0x04, 0x22, 0x59, 0x78, 0x9c, 0xa6, 0xae, - 0xec, 0x56, 0xef, 0xeb, 0x58, 0x55, 0x7b, 0x92, 0xb0, 0x36, 0xc5, 0xf6, 0x4c, 0x7e, 0xa0, 0x18, - 0x0d, 0xbe, 0x03, 0x8a, 0x81, 0xdc, 0x28, 0x9e, 0x4d, 0x53, 0x57, 0xbe, 0x31, 0x44, 0x64, 0x61, - 0xb8, 0x36, 0xdd, 0x69, 0x97, 0x8b, 0xf1, 0x17, 0x52, 0x80, 0x95, 0xf7, 0x73, 0xa0, 0xb4, 0x16, - 0xd1, 0xd0, 0x73, 0x10, 0xa1, 0x5e, 0x14, 0x18, 0x64, 0xcd, 0xb3, 0x23, 0xc7, 0x5d, 0x27, 0x0d, - 0xcb, 0xb5, 0x42, 0x96, 0xad, 0x4b, 0x60, 0xcc, 0xc5, 0x0e, 0x91, 0xd9, 0x33, 0x2d, 0x63, 0x3a, - 0xb6, 0x8d, 0x1d, 0x82, 0x38, 0x87, 0x49, 0xb0, 0x64, 0x91, 0x67, 0x41, 0x49, 0x5c, 0x3b, 0xf6, - 0x09, 0xe2, 0x1c, 0xf8, 0x38, 0x18, 0x6f, 0x78, 0x81, 0x83, 0xc5, 0x3e, 0x4e, 0x26, 0x3b, 0xf3, - 0x02, 0xa7, 0x22, 0xc9, 0x85, 0xcf, 0x80, 0x29, 0x93, 0x50, 0x23, 0xb0, 0x7c, 0x06, 0xad, 0x8f, - 0x71, 0xe1, 0x33, 0x52, 0x78, 0x6a, 0x3d, 0x61, 0xa1, 0xb4, 0x1c, 0xbc, 0x04, 0x8a, 0x7e, 0x60, - 0x79, 0x81, 0x15, 0x1e, 0xeb, 0x85, 0x25, 0x6d, 0xb9, 0x50, 0x9b, 0x93, 0x3a, 0xc5, 0x5d, 0x49, - 0x47, 0x4a, 0x02, 0x2e, 0x81, 0xe2, 0x4b, 0xf5, 0x9d, 0xed, 0x5d, 0x1c, 0xee, 0xeb, 0xe3, 0x1c, - 0x61, 0x8c, 0x49, 0xa3, 0xe2, 0x4d, 0x49, 0xad, 0xfc, 0x33, 0x07, 0xf4, 0x6c, 0x54, 0xe2, 0x90, - 0xc2, 0x17, 0x40, 0x91, 0x86, 0xac, 0xe2, 0x34, 0x8f, 0x65, 0x4c, 0x9e, 0x88, 0xc1, 0xea, 0x92, - 0x7e, 0xd2, 0x2e, 0x2f, 0x24, 0x1a, 0x31, 0x95, 0xc7, 0x43, 0xe9, 0xc2, 0xdf, 0x6a, 0xe0, 0xcc, - 0x21, 0xd9, 0xdb, 0xf7, 0xbc, 0x83, 0x35, 0xdb, 0x22, 0x6e, 0xb8, 0xe6, 0xb9, 0x0d, 0xab, 0x29, - 0x73, 0x00, 0xdd, 0x67, 0x0e, 0xbc, 0xda, 0x6b, 0xb9, 0xf6, 0x50, 0xa7, 0x5d, 0x3e, 0xd3, 0x87, - 0x81, 0xfa, 0xf9, 0x01, 0x6f, 0x00, 0xdd, 0xc8, 0x1c, 0x12, 0x59, 0xc0, 0x44, 0xd9, 0x9a, 0xac, - 0x5d, 0xe8, 0xb4, 0xcb, 0xfa, 0xda, 0x00, 0x19, 0x34, 0x50, 0xbb, 0xf2, 0xc3, 0x7c, 0x36, 0xbc, - 0xa9, 0x74, 0x7b, 0x1b, 0x14, 0xd9, 0x31, 0x36, 0x71, 0x88, 0xe5, 0x41, 0x7c, 0xf2, 0xde, 0x0e, - 0xbd, 0xa8, 0x19, 0x5b, 0x24, 0xc4, 0x35, 0x28, 0x37, 0x04, 0x24, 0x34, 0xa4, 0xac, 0xc2, 0xef, - 0x80, 0x31, 0xea, 0x13, 0x43, 0x06, 0xfa, 0xb5, 0xfb, 0x3d, 0x6c, 0x03, 0x16, 0x52, 0xf7, 0x89, - 0x91, 0x9c, 0x05, 0xf6, 0x85, 0x38, 0x2c, 0x7c, 0x57, 0x03, 0xe3, 0x94, 0x17, 0x28, 0x59, 0xd4, - 0xde, 0x18, 0x95, 0x07, 0x99, 0x2a, 0x28, 0xbe, 0x91, 0x04, 0xaf, 0xfc, 0x27, 0x07, 0x2e, 0x0e, - 0x52, 0x5d, 0xf3, 0x5c, 0x53, 0x6c, 0xc7, 0xa6, 0x3c, 0xdb, 0x22, 0xd3, 0x9f, 0x49, 0x9f, 0xed, - 0x93, 0x76, 0xf9, 0xb1, 0xbb, 0x1a, 0x48, 0x15, 0x81, 0xaf, 0xa9, 0x75, 0x8b, 0x42, 0x71, 0xb1, - 0xdb, 0xb1, 0x93, 0x76, 0x79, 0x56, 0xa9, 0x75, 0xfb, 0x0a, 0x5b, 0x00, 0xda, 0x98, 0x86, 0xd7, - 0x02, 0xec, 0x52, 0x61, 0xd6, 0x72, 0x88, 0x0c, 0xdf, 0x13, 0xf7, 0x96, 0x1e, 0x4c, 0xa3, 0xb6, - 0x28, 0x21, 0xe1, 0xd5, 0x1e, 0x6b, 0xa8, 0x0f, 0x02, 0xab, 0x5b, 0x01, 0xc1, 0x54, 0x95, 0xa2, - 0xd4, 0x8d, 0xc2, 0xa8, 0x48, 0x72, 0xe1, 0x97, 0xc0, 0x84, 0x43, 0x28, 0xc5, 0x4d, 0xc2, 0xeb, - 0xcf, 0x64, 0x72, 0x45, 0x6f, 0x09, 0x32, 0x8a, 0xf9, 0xac, 0x3f, 0xb9, 0x30, 0x28, 0x6a, 0x57, - 0x2d, 0x1a, 0xc2, 0xd7, 0x7b, 0x0e, 0x40, 0xf5, 0xde, 0x56, 0xc8, 0xb4, 0x79, 0xfa, 0xab, 0xe2, - 0x17, 0x53, 0x52, 0xc9, 0xff, 0x6d, 0x50, 0xb0, 0x42, 0xe2, 0xc4, 0x77, 0xf7, 0xab, 0x23, 0xca, - 0xbd, 0xda, 0x8c, 0xf4, 0xa1, 0xb0, 0xc9, 0xd0, 0x90, 0x00, 0xad, 0xfc, 0x21, 0x07, 0x1e, 0x19, - 0xa4, 0xc2, 0x2e, 0x14, 0xca, 0x22, 0xee, 0xdb, 0x51, 0x80, 0x6d, 0x99, 0x71, 0x2a, 0xe2, 0xbb, - 0x9c, 0x8a, 0x24, 0x97, 0x95, 0x7c, 0x6a, 0xb9, 0xcd, 0xc8, 0xc6, 0x81, 0x4c, 0x27, 0xb5, 0xea, - 0xba, 0xa4, 0x23, 0x25, 0x01, 0xab, 0x00, 0xd0, 0x7d, 0x2f, 0x08, 0x39, 0x86, 0xac, 0x5e, 0xa7, - 0x59, 0x81, 0xa8, 0x2b, 0x2a, 0x4a, 0x49, 0xb0, 0x1b, 0xed, 0xc0, 0x72, 0x4d, 0xb9, 0xeb, 0xea, - 0x14, 0xbf, 0x6c, 0xb9, 0x26, 0xe2, 0x1c, 0x86, 0x6f, 0x5b, 0x34, 0x64, 0x14, 0xb9, 0xe5, 0x5d, - 0x51, 0xe7, 0x92, 0x4a, 0x82, 0xe1, 0x1b, 0xac, 0xea, 0x7b, 0x81, 0x45, 0xa8, 0x3e, 0x9e, 0xe0, - 0xaf, 0x29, 0x2a, 0x4a, 0x49, 0x54, 0x7e, 0x5d, 0x1c, 0x9c, 0x24, 0xac, 0x94, 0xc0, 0x47, 0x41, - 0xa1, 0x19, 0x78, 0x91, 0x2f, 0xa3, 0xa4, 0xa2, 0xfd, 0x22, 0x23, 0x22, 0xc1, 0x63, 0x59, 0xd9, - 0xea, 0x6a, 0x53, 0x55, 0x56, 0xc6, 0xcd, 0x69, 0xcc, 0x87, 0xdf, 0xd7, 0x40, 0xc1, 0x95, 0xc1, - 0x61, 0x29, 0xf7, 0xfa, 0x88, 0xf2, 0x82, 0x87, 0x37, 0x71, 0x57, 0x44, 0x5e, 0x20, 0xc3, 0xa7, - 0x41, 0x81, 0x1a, 0x9e, 0x4f, 0x64, 0xd4, 0x4b, 0xb1, 0x50, 0x9d, 0x11, 0x4f, 0xda, 0xe5, 0x99, - 0xd8, 0x1c, 0x27, 0x20, 0x21, 0x0c, 0x7f, 0xa4, 0x01, 0xd0, 0xc2, 0xb6, 0x65, 0x62, 0xde, 0x32, - 0x14, 0xb8, 0xfb, 0xc3, 0x4d, 0xeb, 0x57, 0x94, 0x79, 0xb1, 0x69, 0xc9, 0x37, 0x4a, 0x41, 0xc3, - 0xf7, 0x34, 0x30, 0x4d, 0xa3, 0xbd, 0x40, 0x6a, 0x51, 0xde, 0x5c, 0x4c, 0x5d, 0xf9, 0xe6, 0x50, - 0x7d, 0xa9, 0xa7, 0x00, 0x6a, 0x73, 0x9d, 0x76, 0x79, 0x3a, 0x4d, 0x41, 0x5d, 0x0e, 0xc0, 0x9f, - 0x6a, 0xa0, 0xd8, 0x8a, 0xef, 0xec, 0x09, 0x7e, 0xe0, 0xdf, 0x1c, 0xd1, 0xc6, 0xca, 0x8c, 0x4a, - 0x4e, 0x81, 0xea, 0x03, 0x94, 0x07, 0xf0, 0x6f, 0x1a, 0xd0, 0xb1, 0x29, 0x0a, 0x3c, 0xb6, 0x77, - 0x03, 0xcb, 0x0d, 0x49, 0x20, 0xfa, 0x4d, 0xaa, 0x17, 0xb9, 0x7b, 0xc3, 0xbd, 0x0b, 0xb3, 0xbd, - 0x6c, 0x6d, 0x49, 0x7a, 0xa7, 0xaf, 0x0e, 0x70, 0x03, 0x0d, 0x74, 0x90, 0x27, 0x5a, 0xd2, 0xd2, - 0xe8, 0x93, 0x23, 0x48, 0xb4, 0xa4, 0x97, 0x92, 0xd5, 0x21, 0xe9, 0xa0, 0x52, 0xd0, 0x95, 0xf7, - 0xf2, 0xd9, 0xa6, 0x3d, 0x7b, 0xe9, 0xc3, 0x0f, 0x84, 0xb3, 0x62, 0x29, 0x54, 0xd7, 0x78, 0x70, - 0xdf, 0x1e, 0xd1, 0xde, 0xab, 0x5b, 0x3b, 0x69, 0xbc, 0x14, 0x89, 0xa2, 0x94, 0x1f, 0xf0, 0x57, - 0x1a, 0x98, 0xc1, 0x86, 0x41, 0xfc, 0x90, 0x98, 0xa2, 0x16, 0xe7, 0x3e, 0x87, 0x72, 0x73, 0x4e, - 0x7a, 0x35, 0xb3, 0x9a, 0x86, 0x46, 0xdd, 0x9e, 0xc0, 0xe7, 0xc1, 0x69, 0x1a, 0x7a, 0x01, 0x31, - 0x33, 0x5d, 0x2e, 0xec, 0xb4, 0xcb, 0xa7, 0xeb, 0x5d, 0x1c, 0x94, 0x91, 0xac, 0x7c, 0x3a, 0x06, - 0xca, 0x77, 0x39, 0x19, 0xf7, 0xf0, 0x8e, 0x7a, 0x1c, 0x8c, 0xf3, 0xe5, 0x9a, 0x3c, 0x2a, 0xc5, - 0x54, 0xe7, 0xc6, 0xa9, 0x48, 0x72, 0x59, 0x5d, 0x67, 0xf8, 0xac, 0xdb, 0xc8, 0x73, 0x41, 0x55, - 0xd7, 0xeb, 0x82, 0x8c, 0x62, 0x3e, 0x7c, 0x07, 0x8c, 0x8b, 0x39, 0x09, 0x2f, 0xaa, 0x23, 0x2c, - 0x8c, 0x80, 0xfb, 0xc9, 0xa1, 0x90, 0x84, 0xec, 0x2d, 0x88, 0x85, 0x07, 0x5d, 0x10, 0xef, 0x58, - 0x81, 0xc6, 0xff, 0xcf, 0x2b, 0x50, 0xe5, 0xbf, 0x5a, 0xf6, 0xdc, 0xa7, 0x96, 0x5a, 0x37, 0xb0, - 0x4d, 0xe0, 0x3a, 0x98, 0x63, 0x8f, 0x0c, 0x44, 0x7c, 0xdb, 0x32, 0x30, 0xe5, 0x6f, 0x5c, 0x91, - 0x70, 0x6a, 0xec, 0x52, 0xcf, 0xf0, 0x51, 0x8f, 0x06, 0x7c, 0x09, 0x40, 0xd1, 0x78, 0x77, 0xd9, - 0x11, 0x3d, 0x84, 0x6a, 0xa1, 0xeb, 0x3d, 0x12, 0xa8, 0x8f, 0x16, 0x5c, 0x03, 0xf3, 0x36, 0xde, - 0x23, 0x76, 0x9d, 0xd8, 0xc4, 0x08, 0xbd, 0x80, 0x9b, 0x12, 0x53, 0x80, 0x73, 0x9d, 0x76, 0x79, - 0xfe, 0x6a, 0x96, 0x89, 0x7a, 0xe5, 0x2b, 0x17, 0xb3, 0xc7, 0x2b, 0xbd, 0x70, 0xf1, 0x9c, 0xf9, - 0x30, 0x07, 0x16, 0x07, 0x67, 0x06, 0xfc, 0x41, 0xf2, 0xea, 0x12, 0x4d, 0xf5, 0x9b, 0xa3, 0xca, - 0x42, 0xf9, 0xec, 0x02, 0xbd, 0x4f, 0x2e, 0xf8, 0x5d, 0xd6, 0xe1, 0x60, 0x3b, 0x9e, 0xf3, 0xbc, - 0x31, 0x32, 0x17, 0x18, 0x48, 0x6d, 0x52, 0x34, 0x4f, 0xd8, 0xe6, 0xbd, 0x12, 0xb6, 0x49, 0xe5, - 0x8f, 0x5a, 0xf6, 0xe1, 0x9d, 0x9c, 0x60, 0xf8, 0x33, 0x0d, 0xcc, 0x7a, 0x3e, 0x71, 0x57, 0x77, - 0x37, 0x5f, 0xf9, 0x8a, 0x38, 0xc9, 0x32, 0x54, 0xdb, 0xf7, 0xe9, 0xe7, 0x4b, 0xf5, 0x9d, 0x6d, - 0x61, 0x70, 0x37, 0xf0, 0x7c, 0x5a, 0x3b, 0xd3, 0x69, 0x97, 0x67, 0x77, 0xba, 0xa1, 0x50, 0x16, - 0xbb, 0xe2, 0x80, 0x73, 0x1b, 0x47, 0x21, 0x09, 0x5c, 0x6c, 0xaf, 0x7b, 0x46, 0xe4, 0x10, 0x37, - 0x14, 0x8e, 0x66, 0x86, 0x44, 0xda, 0x3d, 0x0e, 0x89, 0x1e, 0x01, 0xf9, 0x28, 0xb0, 0x65, 0x16, - 0x4f, 0xa9, 0x21, 0x28, 0xba, 0x8a, 0x18, 0xbd, 0x72, 0x11, 0x8c, 0x31, 0x3f, 0xe1, 0x79, 0x90, - 0x0f, 0xf0, 0x21, 0xb7, 0x3a, 0x5d, 0x9b, 0x60, 0x22, 0x08, 0x1f, 0x22, 0x46, 0xab, 0xfc, 0xbb, - 0x04, 0x66, 0x33, 0x6b, 0x81, 0x8b, 0x20, 0xa7, 0x26, 0xab, 0x40, 0x1a, 0xcd, 0x6d, 0xae, 0xa3, - 0x9c, 0x65, 0xc2, 0x67, 0x55, 0xf1, 0x15, 0xa0, 0x65, 0x55, 0xcf, 0x39, 0x95, 0xb5, 0xb4, 0x89, - 0x39, 0xe6, 0x48, 0x5c, 0x38, 0x99, 0x0f, 0xa4, 0x21, 0x4f, 0x89, 0xf0, 0x81, 0x34, 0x10, 0xa3, - 0x7d, 0xd6, 0x09, 0x59, 0x3c, 0xa2, 0x2b, 0xdc, 0xc3, 0x88, 0x6e, 0xfc, 0x8e, 0x23, 0xba, 0x47, - 0x41, 0x21, 0xb4, 0x42, 0x9b, 0xe8, 0x13, 0xdd, 0x2f, 0x8f, 0x6b, 0x8c, 0x88, 0x04, 0x0f, 0xde, - 0x04, 0x13, 0x26, 0x69, 0xe0, 0xc8, 0x0e, 0xf5, 0x22, 0x4f, 0xa1, 0xb5, 0x21, 0xa4, 0x90, 0x98, - 0x9f, 0xae, 0x0b, 0xbb, 0x28, 0x06, 0x80, 0x8f, 0x81, 0x09, 0x07, 0x1f, 0x59, 0x4e, 0xe4, 0xf0, - 0x9e, 0x4c, 0x13, 0x62, 0x5b, 0x82, 0x84, 0x62, 0x1e, 0xab, 0x8c, 0xe4, 0xc8, 0xb0, 0x23, 0x6a, - 0xb5, 0x88, 0x64, 0xea, 0x80, 0xdf, 0x9e, 0xaa, 0x32, 0x6e, 0x64, 0xf8, 0xa8, 0x47, 0x83, 0x83, - 0x59, 0x2e, 0x57, 0x9e, 0x4a, 0x81, 0x09, 0x12, 0x8a, 0x79, 0xdd, 0x60, 0x52, 0x7e, 0x7a, 0x10, - 0x98, 0x54, 0xee, 0xd1, 0x80, 0x5f, 0x06, 0x93, 0x0e, 0x3e, 0xba, 0x4a, 0xdc, 0x66, 0xb8, 0xaf, - 0xcf, 0x2c, 0x69, 0xcb, 0xf9, 0xda, 0x4c, 0xa7, 0x5d, 0x9e, 0xdc, 0x8a, 0x89, 0x28, 0xe1, 0x73, - 0x61, 0xcb, 0x95, 0xc2, 0xa7, 0x53, 0xc2, 0x31, 0x11, 0x25, 0x7c, 0xd6, 0x41, 0xf8, 0x38, 0x64, - 0x87, 0x4b, 0x9f, 0xed, 0x7e, 0x19, 0xee, 0x0a, 0x32, 0x8a, 0xf9, 0x70, 0x19, 0x14, 0x1d, 0x7c, - 0xc4, 0x5f, 0xf1, 0xfa, 0x1c, 0x37, 0xcb, 0x67, 0xc9, 0x5b, 0x92, 0x86, 0x14, 0x97, 0x4b, 0x5a, - 0xae, 0x90, 0x9c, 0x4f, 0x49, 0x4a, 0x1a, 0x52, 0x5c, 0x96, 0xc4, 0x91, 0x6b, 0xdd, 0x8a, 0x88, - 0x10, 0x86, 0x3c, 0x32, 0x2a, 0x89, 0xaf, 0x27, 0x2c, 0x94, 0x96, 0x63, 0xaf, 0x68, 0x27, 0xb2, - 0x43, 0xcb, 0xb7, 0xc9, 0x4e, 0x43, 0x3f, 0xc3, 0xe3, 0xcf, 0xfb, 0xe4, 0x2d, 0x45, 0x45, 0x29, - 0x09, 0x48, 0xc0, 0x18, 0x71, 0x23, 0x47, 0x3f, 0xcb, 0x2f, 0xf6, 0xa1, 0xa4, 0xa0, 0x3a, 0x39, - 0x1b, 0x6e, 0xe4, 0x20, 0x6e, 0x1e, 0x3e, 0x0b, 0x66, 0x1c, 0x7c, 0xc4, 0xca, 0x01, 0x09, 0x42, - 0xf6, 0xbe, 0x3f, 0xc7, 0x17, 0x3f, 0xcf, 0x3a, 0xce, 0xad, 0x34, 0x03, 0x75, 0xcb, 0x71, 0x45, - 0xcb, 0x4d, 0x29, 0x2e, 0xa4, 0x14, 0xd3, 0x0c, 0xd4, 0x2d, 0xc7, 0x22, 0x1d, 0x90, 0x5b, 0x91, - 0x15, 0x10, 0x53, 0x7f, 0x88, 0x37, 0xa9, 0x72, 0xbe, 0x2f, 0x68, 0x48, 0x71, 0x61, 0x2b, 0x1e, - 0xf7, 0xe8, 0xfc, 0x18, 0x5e, 0x1f, 0x6e, 0x25, 0xdf, 0x09, 0x56, 0x83, 0x00, 0x1f, 0x8b, 0x9b, - 0x26, 0x3d, 0xe8, 0x81, 0x14, 0x14, 0xb0, 0x6d, 0xef, 0x34, 0xf4, 0xf3, 0x3c, 0xf6, 0xc3, 0xbe, - 0x41, 0x54, 0xd5, 0x59, 0x65, 0x20, 0x48, 0x60, 0x31, 0x50, 0xcf, 0x65, 0xa9, 0xb1, 0x38, 0x5a, - 0xd0, 0x1d, 0x06, 0x82, 0x04, 0x16, 0x5f, 0xa9, 0x7b, 0xbc, 0xd3, 0xd0, 0x1f, 0x1e, 0xf1, 0x4a, - 0x19, 0x08, 0x12, 0x58, 0xd0, 0x02, 0x79, 0xd7, 0x0b, 0xf5, 0x0b, 0x23, 0xb9, 0x9e, 0xf9, 0x85, - 0xb3, 0xed, 0x85, 0x88, 0x61, 0xc0, 0x5f, 0x6a, 0x00, 0xf8, 0x49, 0x8a, 0x3e, 0x32, 0x94, 0x29, - 0x42, 0x06, 0xb2, 0x9a, 0xe4, 0xf6, 0x86, 0x1b, 0x06, 0xc7, 0xc9, 0x3b, 0x32, 0x75, 0x06, 0x52, - 0x5e, 0xc0, 0xdf, 0x69, 0xe0, 0x6c, 0xba, 0x4d, 0x56, 0xee, 0x95, 0x78, 0x44, 0xae, 0x0d, 0x3b, - 0xcd, 0x6b, 0x9e, 0x67, 0xd7, 0xf4, 0x4e, 0xbb, 0x7c, 0x76, 0xb5, 0x0f, 0x2a, 0xea, 0xeb, 0x0b, - 0xfc, 0x93, 0x06, 0xe6, 0x65, 0x15, 0x4d, 0x79, 0x58, 0xe6, 0x01, 0x24, 0xc3, 0x0e, 0x60, 0x16, - 0x47, 0xc4, 0x51, 0xfd, 0x2e, 0xdd, 0xc3, 0x47, 0xbd, 0xae, 0xc1, 0xbf, 0x6a, 0x60, 0xda, 0x24, - 0x3e, 0x71, 0x4d, 0xe2, 0x1a, 0xcc, 0xd7, 0xa5, 0xa1, 0x8c, 0x0d, 0xb2, 0xbe, 0xae, 0xa7, 0x20, - 0x84, 0x9b, 0x55, 0xe9, 0xe6, 0x74, 0x9a, 0x75, 0xd2, 0x2e, 0x2f, 0x24, 0xaa, 0x69, 0x0e, 0xea, - 0xf2, 0x12, 0xbe, 0xaf, 0x81, 0xd9, 0x64, 0x03, 0xc4, 0x95, 0x72, 0x71, 0x84, 0x79, 0xc0, 0xdb, - 0xd7, 0xd5, 0x6e, 0x40, 0x94, 0xf5, 0x00, 0xfe, 0x59, 0x63, 0x9d, 0x5a, 0xfc, 0xee, 0xa3, 0x7a, - 0x85, 0xc7, 0xf2, 0xad, 0xa1, 0xc7, 0x52, 0x21, 0x88, 0x50, 0x5e, 0x4a, 0x5a, 0x41, 0xc5, 0x39, - 0x69, 0x97, 0xcf, 0xa5, 0x23, 0xa9, 0x18, 0x28, 0xed, 0x21, 0xfc, 0x89, 0x06, 0xa6, 0x49, 0xd2, - 0x71, 0x53, 0xfd, 0xd1, 0xa1, 0x04, 0xb1, 0x6f, 0x13, 0x2f, 0x5e, 0xea, 0x29, 0x16, 0x45, 0x5d, - 0xd8, 0xac, 0x83, 0x24, 0x47, 0xd8, 0xf1, 0x6d, 0xa2, 0x7f, 0x61, 0xc8, 0x1d, 0xe4, 0x86, 0xb0, - 0x8b, 0x62, 0x00, 0x78, 0x09, 0x14, 0xdd, 0xc8, 0xb6, 0xf1, 0x9e, 0x4d, 0xf4, 0xc7, 0x78, 0x2f, - 0xa2, 0xa6, 0x98, 0xdb, 0x92, 0x8e, 0x94, 0x04, 0x6c, 0x80, 0xa5, 0xa3, 0x97, 0xd5, 0x7f, 0xf4, - 0xec, 0x06, 0x84, 0xe3, 0x5f, 0x77, 0x0f, 0x5c, 0xef, 0xd0, 0x7d, 0xc1, 0x22, 0xb6, 0x49, 0xf5, - 0xc7, 0xb9, 0x95, 0xc5, 0x4e, 0xbb, 0xbc, 0x70, 0xa3, 0xaf, 0x04, 0xba, 0xab, 0x0d, 0xf8, 0x1a, - 0x78, 0x38, 0x25, 0xb3, 0xe1, 0xec, 0x11, 0xd3, 0x24, 0x66, 0xfc, 0x70, 0xd3, 0xbf, 0xc8, 0x21, - 0xd4, 0x01, 0xbf, 0x91, 0x15, 0x40, 0x77, 0xd2, 0x86, 0x57, 0xc1, 0x42, 0x8a, 0xbd, 0xe9, 0x86, - 0x3b, 0x41, 0x3d, 0x0c, 0x2c, 0xb7, 0xa9, 0x2f, 0x73, 0xbb, 0x67, 0xe3, 0x13, 0x79, 0x23, 0xc5, - 0x43, 0x03, 0x74, 0x16, 0xd9, 0xd3, 0x31, 0x53, 0x7a, 0xe0, 0x1c, 0xc8, 0x1f, 0x10, 0xf9, 0x0b, - 0x39, 0x62, 0x7f, 0x42, 0x13, 0x14, 0x5a, 0xd8, 0x8e, 0xe2, 0xd7, 0xef, 0x90, 0xaf, 0x2d, 0x24, - 0x8c, 0x3f, 0x9f, 0x7b, 0x4e, 0x5b, 0xfc, 0x40, 0x03, 0x0b, 0xfd, 0x2b, 0xe2, 0x03, 0x75, 0xeb, - 0x37, 0x1a, 0x98, 0xef, 0x29, 0x7e, 0x7d, 0x3c, 0xba, 0xd5, 0xed, 0xd1, 0x6b, 0xc3, 0xae, 0x62, - 0x62, 0xd7, 0x78, 0xeb, 0x96, 0x76, 0xef, 0xe7, 0x1a, 0x98, 0xcb, 0xd6, 0x93, 0x07, 0x19, 0xaf, - 0xca, 0x07, 0x39, 0xb0, 0xd0, 0xbf, 0xe3, 0x84, 0x81, 0x7a, 0x5a, 0x8f, 0x66, 0x44, 0xd1, 0x6f, - 0x9c, 0xf9, 0xae, 0x06, 0xa6, 0x6e, 0x2a, 0xb9, 0xf8, 0x17, 0xd4, 0xa1, 0x0f, 0x47, 0xe2, 0x02, - 0x9e, 0x30, 0x28, 0x4a, 0xe3, 0x56, 0xfe, 0xa2, 0x81, 0x73, 0x7d, 0x6f, 0x26, 0xf6, 0x86, 0xc7, - 0xb6, 0xed, 0x1d, 0x8a, 0x19, 0x57, 0x6a, 0x80, 0xbc, 0xca, 0xa9, 0x48, 0x72, 0x53, 0xd1, 0xcb, - 0x7d, 0x5e, 0xd1, 0xab, 0xfc, 0x5d, 0x03, 0x17, 0xee, 0x94, 0x89, 0x0f, 0x64, 0x4b, 0x97, 0x41, - 0x51, 0x76, 0x95, 0xc7, 0x7c, 0x3b, 0xe5, 0x43, 0x4a, 0x16, 0x0d, 0xfe, 0x4f, 0x43, 0xe2, 0xaf, - 0xca, 0x87, 0x1a, 0x98, 0xab, 0x93, 0xa0, 0x65, 0x19, 0x04, 0x91, 0x06, 0x09, 0x88, 0x6b, 0x10, - 0xb8, 0x02, 0x26, 0xf9, 0x4f, 0x97, 0x3e, 0x36, 0xe2, 0xb9, 0xfe, 0xbc, 0x0c, 0xf9, 0xe4, 0x76, - 0xcc, 0x40, 0x89, 0x8c, 0xfa, 0x0d, 0x20, 0x37, 0xf0, 0x37, 0x80, 0x0b, 0x60, 0xcc, 0x4f, 0x26, - 0xa4, 0x45, 0xc6, 0xe5, 0x43, 0x51, 0x4e, 0xe5, 0x5c, 0x2f, 0x08, 0xf9, 0xd8, 0xa7, 0x20, 0xb9, - 0x5e, 0x10, 0x22, 0x4e, 0xad, 0xfc, 0x43, 0x03, 0xfd, 0xfe, 0xbd, 0x07, 0xb6, 0xc0, 0x04, 0x15, - 0xae, 0xcb, 0xd0, 0xee, 0xdc, 0x67, 0x68, 0xb3, 0x81, 0x10, 0xf7, 0x6a, 0x4c, 0x8d, 0xc1, 0x58, - 0x74, 0x0d, 0x5c, 0x8b, 0x5c, 0x53, 0x4e, 0x3c, 0xa7, 0x45, 0x74, 0xd7, 0x56, 0x05, 0x0d, 0x29, - 0x2e, 0x3c, 0x2f, 0x66, 0x73, 0xa9, 0x81, 0x57, 0x3c, 0x97, 0xab, 0x5d, 0xfe, 0xe8, 0x76, 0xe9, - 0xd4, 0xc7, 0xb7, 0x4b, 0xa7, 0x3e, 0xb9, 0x5d, 0x3a, 0xf5, 0xbd, 0x4e, 0x49, 0xfb, 0xa8, 0x53, - 0xd2, 0x3e, 0xee, 0x94, 0xb4, 0x4f, 0x3a, 0x25, 0xed, 0x5f, 0x9d, 0x92, 0xf6, 0x8b, 0x4f, 0x4b, - 0xa7, 0xbe, 0x35, 0x21, 0x5d, 0xfb, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe1, 0x05, 0x6b, 0x7f, - 0x74, 0x2b, 0x00, 0x00, + // 2908 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x73, 0x23, 0x47, + 0x15, 0xdf, 0x91, 0x2c, 0x5b, 0x6e, 0xdb, 0x6b, 0xbb, 0x77, 0xed, 0xcc, 0x3a, 0x1b, 0xc9, 0xab, + 0x90, 0x60, 0xc2, 0xae, 0x9c, 0x2c, 0x09, 0x09, 0xa9, 0xe2, 0x60, 0xd9, 0x4e, 0xca, 0xc9, 0xda, + 0x32, 0xad, 0xdd, 0x64, 0x21, 0x9f, 0x6d, 0x4d, 0x4b, 0x9e, 0xf5, 0x7c, 0xed, 0xf4, 0x8c, 0x6c, + 0x57, 0x80, 0xe2, 0xa3, 0x52, 0x50, 0x14, 0x10, 0x8a, 0xe4, 0x42, 0x15, 0x1c, 0x02, 0xc5, 0x85, + 0x03, 0x1c, 0xe0, 0x06, 0x7f, 0x40, 0x8e, 0x29, 0x4e, 0x39, 0x50, 0x2a, 0x56, 0xb9, 0xc2, 0x8d, + 0x2a, 0xaa, 0x7c, 0xa2, 0xfa, 0x63, 0x7a, 0x46, 0x23, 0x69, 0xd7, 0x95, 0x95, 0xb2, 0xdc, 0xac, + 0xf7, 0xf5, 0x7b, 0xfd, 0xfa, 0xf5, 0xeb, 0xd7, 0x6f, 0x0c, 0x1a, 0x07, 0xcf, 0xd1, 0xb2, 0xe9, + 0xae, 0x1e, 0x84, 0x7b, 0xc4, 0x77, 0x48, 0x40, 0xe8, 0x6a, 0x8b, 0x38, 0x86, 0xeb, 0xaf, 0x4a, + 0x06, 0xf6, 0x4c, 0x72, 0x14, 0x10, 0x87, 0x9a, 0xae, 0x43, 0xaf, 0x60, 0xcf, 0xa4, 0xc4, 0x6f, + 0x11, 0x7f, 0xd5, 0x3b, 0x68, 0x32, 0x1e, 0xed, 0x16, 0x58, 0x6d, 0x3d, 0xb5, 0x47, 0x02, 0xfc, + 0xd4, 0x6a, 0x93, 0x38, 0xc4, 0xc7, 0x01, 0x31, 0xca, 0x9e, 0xef, 0x06, 0x2e, 0xfc, 0xba, 0x30, + 0x57, 0xee, 0x92, 0x7e, 0x4b, 0x99, 0x2b, 0x7b, 0x07, 0x4d, 0xc6, 0xa3, 0xdd, 0x02, 0x65, 0x69, + 0x6e, 0xe9, 0x4a, 0xd3, 0x0c, 0xf6, 0xc3, 0xbd, 0x72, 0xdd, 0xb5, 0x57, 0x9b, 0x6e, 0xd3, 0x5d, + 0xe5, 0x56, 0xf7, 0xc2, 0x06, 0xff, 0xc5, 0x7f, 0xf0, 0xbf, 0x04, 0xda, 0xd2, 0xd3, 0xb1, 0xf3, + 0x36, 0xae, 0xef, 0x9b, 0x0e, 0xf1, 0x8f, 0x63, 0x8f, 0x6d, 0x12, 0xe0, 0xd5, 0x56, 0x8f, 0x8f, + 0x4b, 0xab, 0x83, 0xb4, 0xfc, 0xd0, 0x09, 0x4c, 0x9b, 0xf4, 0x28, 0x7c, 0xf5, 0x5e, 0x0a, 0xb4, + 0xbe, 0x4f, 0x6c, 0x9c, 0xd6, 0x2b, 0x9d, 0x68, 0x60, 0x7e, 0xdd, 0x75, 0x5a, 0xc4, 0x67, 0xab, + 0x44, 0xe4, 0x76, 0x48, 0x68, 0x00, 0x2b, 0x20, 0x1b, 0x9a, 0x86, 0xae, 0x2d, 0x6b, 0x2b, 0x93, + 0x95, 0x27, 0x3f, 0x6a, 0x17, 0xcf, 0x74, 0xda, 0xc5, 0xec, 0x8d, 0xad, 0x8d, 0x93, 0x76, 0xf1, + 0xd2, 0x20, 0xa4, 0xe0, 0xd8, 0x23, 0xb4, 0x7c, 0x63, 0x6b, 0x03, 0x31, 0x65, 0xf8, 0x22, 0x98, + 0x37, 0x08, 0x35, 0x7d, 0x62, 0xac, 0xed, 0x6e, 0xbd, 0x22, 0xec, 0xeb, 0x19, 0x6e, 0xf1, 0x82, + 0xb4, 0x38, 0xbf, 0x91, 0x16, 0x40, 0xbd, 0x3a, 0xf0, 0x26, 0x98, 0x70, 0xf7, 0x6e, 0x91, 0x7a, + 0x40, 0xf5, 0xec, 0x72, 0x76, 0x65, 0xea, 0xea, 0x95, 0x72, 0xbc, 0x83, 0xca, 0x05, 0xbe, 0x6d, + 0x72, 0xb1, 0x65, 0x84, 0x0f, 0x37, 0xa3, 0x9d, 0xab, 0xcc, 0x4a, 0xb4, 0x89, 0xaa, 0xb0, 0x82, + 0x22, 0x73, 0xa5, 0xdf, 0x65, 0x00, 0x4c, 0x2e, 0x9e, 0x7a, 0xae, 0x43, 0xc9, 0x50, 0x56, 0x4f, + 0xc1, 0x5c, 0x9d, 0x5b, 0x0e, 0x88, 0x21, 0x71, 0xf5, 0xcc, 0x67, 0xf1, 0x5e, 0x97, 0xf8, 0x73, + 0xeb, 0x29, 0x73, 0xa8, 0x07, 0x00, 0x5e, 0x07, 0xe3, 0x3e, 0xa1, 0xa1, 0x15, 0xe8, 0xd9, 0x65, + 0x6d, 0x65, 0xea, 0xea, 0xe5, 0x81, 0x50, 0x3c, 0xbf, 0x59, 0xf2, 0x95, 0x5b, 0x4f, 0x95, 0x6b, + 0x01, 0x0e, 0x42, 0x5a, 0x39, 0x2b, 0x91, 0xc6, 0x11, 0xb7, 0x81, 0xa4, 0xad, 0xd2, 0x8f, 0x33, + 0x60, 0x2e, 0x19, 0xa5, 0x96, 0x49, 0x0e, 0xe1, 0x21, 0x98, 0xf0, 0x45, 0xb2, 0xf0, 0x38, 0x4d, + 0x5d, 0xdd, 0x2d, 0xdf, 0xd7, 0xb1, 0x2a, 0xf7, 0x24, 0x61, 0x65, 0x8a, 0xed, 0x99, 0xfc, 0x81, + 0x22, 0x34, 0xf8, 0x0e, 0xc8, 0xfb, 0x72, 0xa3, 0x78, 0x36, 0x4d, 0x5d, 0xfd, 0xc6, 0x10, 0x91, + 0x85, 0xe1, 0xca, 0x74, 0xa7, 0x5d, 0xcc, 0x47, 0xbf, 0x90, 0x02, 0x2c, 0xbd, 0x9f, 0x01, 0x85, + 0xf5, 0x90, 0x06, 0xae, 0x8d, 0x08, 0x75, 0x43, 0xbf, 0x4e, 0xd6, 0x5d, 0x2b, 0xb4, 0x9d, 0x0d, + 0xd2, 0x30, 0x1d, 0x33, 0x60, 0xd9, 0xba, 0x0c, 0xc6, 0x1c, 0x6c, 0x13, 0x99, 0x3d, 0xd3, 0x32, + 0xa6, 0x63, 0x3b, 0xd8, 0x26, 0x88, 0x73, 0x98, 0x04, 0x4b, 0x16, 0x79, 0x16, 0x94, 0xc4, 0xf5, + 0x63, 0x8f, 0x20, 0xce, 0x81, 0x8f, 0x83, 0xf1, 0x86, 0xeb, 0xdb, 0x58, 0xec, 0xe3, 0x64, 0xbc, + 0x33, 0x2f, 0x70, 0x2a, 0x92, 0x5c, 0xf8, 0x0c, 0x98, 0x32, 0x08, 0xad, 0xfb, 0xa6, 0xc7, 0xa0, + 0xf5, 0x31, 0x2e, 0x7c, 0x4e, 0x0a, 0x4f, 0x6d, 0xc4, 0x2c, 0x94, 0x94, 0x83, 0x97, 0x41, 0xde, + 0xf3, 0x4d, 0xd7, 0x37, 0x83, 0x63, 0x3d, 0xb7, 0xac, 0xad, 0xe4, 0x2a, 0x73, 0x52, 0x27, 0xbf, + 0x2b, 0xe9, 0x48, 0x49, 0xc0, 0x65, 0x90, 0x7f, 0xa9, 0x56, 0xdd, 0xd9, 0xc5, 0xc1, 0xbe, 0x3e, + 0xce, 0x11, 0xc6, 0x98, 0x34, 0xca, 0xdf, 0x92, 0xd4, 0xd2, 0x3f, 0x32, 0x40, 0x4f, 0x47, 0x25, + 0x0a, 0x29, 0x7c, 0x01, 0xe4, 0x69, 0xc0, 0x2a, 0x4e, 0xf3, 0x58, 0xc6, 0xe4, 0x89, 0x08, 0xac, + 0x26, 0xe9, 0x27, 0xed, 0xe2, 0x62, 0xac, 0x11, 0x51, 0x79, 0x3c, 0x94, 0x2e, 0xfc, 0x8d, 0x06, + 0xce, 0x1d, 0x92, 0xbd, 0x7d, 0xd7, 0x3d, 0x58, 0xb7, 0x4c, 0xe2, 0x04, 0xeb, 0xae, 0xd3, 0x30, + 0x9b, 0x32, 0x07, 0xd0, 0x7d, 0xe6, 0xc0, 0xab, 0xbd, 0x96, 0x2b, 0x0f, 0x75, 0xda, 0xc5, 0x73, + 0x7d, 0x18, 0xa8, 0x9f, 0x1f, 0xf0, 0x26, 0xd0, 0xeb, 0xa9, 0x43, 0x22, 0x0b, 0x98, 0x28, 0x5b, + 0x93, 0x95, 0x8b, 0x9d, 0x76, 0x51, 0x5f, 0x1f, 0x20, 0x83, 0x06, 0x6a, 0x97, 0x7e, 0x98, 0x4d, + 0x87, 0x37, 0x91, 0x6e, 0x6f, 0x83, 0x3c, 0x3b, 0xc6, 0x06, 0x0e, 0xb0, 0x3c, 0x88, 0x4f, 0x9e, + 0xee, 0xd0, 0x8b, 0x9a, 0xb1, 0x4d, 0x02, 0x5c, 0x81, 0x72, 0x43, 0x40, 0x4c, 0x43, 0xca, 0x2a, + 0xfc, 0x0e, 0x18, 0xa3, 0x1e, 0xa9, 0xcb, 0x40, 0xbf, 0x76, 0xbf, 0x87, 0x6d, 0xc0, 0x42, 0x6a, + 0x1e, 0xa9, 0xc7, 0x67, 0x81, 0xfd, 0x42, 0x1c, 0x16, 0xbe, 0xab, 0x81, 0x71, 0xca, 0x0b, 0x94, + 0x2c, 0x6a, 0x6f, 0x8c, 0xca, 0x83, 0x54, 0x15, 0x14, 0xbf, 0x91, 0x04, 0x2f, 0xfd, 0x27, 0x03, + 0x2e, 0x0d, 0x52, 0x5d, 0x77, 0x1d, 0x43, 0x6c, 0xc7, 0x96, 0x3c, 0xdb, 0x22, 0xd3, 0x9f, 0x49, + 0x9e, 0xed, 0x93, 0x76, 0xf1, 0xb1, 0x7b, 0x1a, 0x48, 0x14, 0x81, 0xaf, 0xa9, 0x75, 0x8b, 0x42, + 0x71, 0xa9, 0xdb, 0xb1, 0x93, 0x76, 0x71, 0x56, 0xa9, 0x75, 0xfb, 0x0a, 0x5b, 0x00, 0x5a, 0x98, + 0x06, 0xd7, 0x7d, 0xec, 0x50, 0x61, 0xd6, 0xb4, 0x89, 0x0c, 0xdf, 0x13, 0xa7, 0x4b, 0x0f, 0xa6, + 0x51, 0x59, 0x92, 0x90, 0xf0, 0x5a, 0x8f, 0x35, 0xd4, 0x07, 0x81, 0xd5, 0x2d, 0x9f, 0x60, 0xaa, + 0x4a, 0x51, 0xe2, 0x46, 0x61, 0x54, 0x24, 0xb9, 0xf0, 0x4b, 0x60, 0xc2, 0x26, 0x94, 0xe2, 0x26, + 0xe1, 0xf5, 0x67, 0x32, 0xbe, 0xa2, 0xb7, 0x05, 0x19, 0x45, 0x7c, 0xd6, 0x9f, 0x5c, 0x1c, 0x14, + 0xb5, 0x6b, 0x26, 0x0d, 0xe0, 0xeb, 0x3d, 0x07, 0xa0, 0x7c, 0xba, 0x15, 0x32, 0x6d, 0x9e, 0xfe, + 0xaa, 0xf8, 0x45, 0x94, 0x44, 0xf2, 0x7f, 0x1b, 0xe4, 0xcc, 0x80, 0xd8, 0xd1, 0xdd, 0xfd, 0xea, + 0x88, 0x72, 0xaf, 0x32, 0x23, 0x7d, 0xc8, 0x6d, 0x31, 0x34, 0x24, 0x40, 0x4b, 0xbf, 0xcf, 0x80, + 0x47, 0x06, 0xa9, 0xb0, 0x0b, 0x85, 0xb2, 0x88, 0x7b, 0x56, 0xe8, 0x63, 0x4b, 0x66, 0x9c, 0x8a, + 0xf8, 0x2e, 0xa7, 0x22, 0xc9, 0x65, 0x25, 0x9f, 0x9a, 0x4e, 0x33, 0xb4, 0xb0, 0x2f, 0xd3, 0x49, + 0xad, 0xba, 0x26, 0xe9, 0x48, 0x49, 0xc0, 0x32, 0x00, 0x74, 0xdf, 0xf5, 0x03, 0x8e, 0x21, 0xab, + 0xd7, 0x59, 0x56, 0x20, 0x6a, 0x8a, 0x8a, 0x12, 0x12, 0xec, 0x46, 0x3b, 0x30, 0x1d, 0x43, 0xee, + 0xba, 0x3a, 0xc5, 0x2f, 0x9b, 0x8e, 0x81, 0x38, 0x87, 0xe1, 0x5b, 0x26, 0x0d, 0x18, 0x45, 0x6e, + 0x79, 0x57, 0xd4, 0xb9, 0xa4, 0x92, 0x60, 0xf8, 0x75, 0x56, 0xf5, 0x5d, 0xdf, 0x24, 0x54, 0x1f, + 0x8f, 0xf1, 0xd7, 0x15, 0x15, 0x25, 0x24, 0x4a, 0xff, 0xca, 0x0f, 0x4e, 0x12, 0x56, 0x4a, 0xe0, + 0xa3, 0x20, 0xd7, 0xf4, 0xdd, 0xd0, 0x93, 0x51, 0x52, 0xd1, 0x7e, 0x91, 0x11, 0x91, 0xe0, 0xb1, + 0xac, 0x6c, 0x75, 0xb5, 0xa9, 0x2a, 0x2b, 0xa3, 0xe6, 0x34, 0xe2, 0xc3, 0xef, 0x6b, 0x20, 0xe7, + 0xc8, 0xe0, 0xb0, 0x94, 0x7b, 0x7d, 0x44, 0x79, 0xc1, 0xc3, 0x1b, 0xbb, 0x2b, 0x22, 0x2f, 0x90, + 0xe1, 0xd3, 0x20, 0x47, 0xeb, 0xae, 0x47, 0x64, 0xd4, 0x0b, 0x91, 0x50, 0x8d, 0x11, 0x4f, 0xda, + 0xc5, 0x99, 0xc8, 0x1c, 0x27, 0x20, 0x21, 0x0c, 0x7f, 0xa4, 0x01, 0xd0, 0xc2, 0x96, 0x69, 0x60, + 0xde, 0x32, 0xe4, 0xb8, 0xfb, 0xc3, 0x4d, 0xeb, 0x57, 0x94, 0x79, 0xb1, 0x69, 0xf1, 0x6f, 0x94, + 0x80, 0x86, 0xef, 0x69, 0x60, 0x9a, 0x86, 0x7b, 0xbe, 0xd4, 0xa2, 0xbc, 0xb9, 0x98, 0xba, 0xfa, + 0xcd, 0xa1, 0xfa, 0x52, 0x4b, 0x00, 0x54, 0xe6, 0x3a, 0xed, 0xe2, 0x74, 0x92, 0x82, 0xba, 0x1c, + 0x80, 0x3f, 0xd5, 0x40, 0xbe, 0x15, 0xdd, 0xd9, 0x13, 0xfc, 0xc0, 0xbf, 0x39, 0xa2, 0x8d, 0x95, + 0x19, 0x15, 0x9f, 0x02, 0xd5, 0x07, 0x28, 0x0f, 0xe0, 0x5f, 0x35, 0xa0, 0x63, 0x43, 0x14, 0x78, + 0x6c, 0xed, 0xfa, 0xa6, 0x13, 0x10, 0x5f, 0xf4, 0x9b, 0x54, 0xcf, 0x73, 0xf7, 0x86, 0x7b, 0x17, + 0xa6, 0x7b, 0xd9, 0xca, 0xb2, 0xf4, 0x4e, 0x5f, 0x1b, 0xe0, 0x06, 0x1a, 0xe8, 0x20, 0x4f, 0xb4, + 0xb8, 0xa5, 0xd1, 0x27, 0x47, 0x90, 0x68, 0x71, 0x2f, 0x25, 0xab, 0x43, 0xdc, 0x41, 0x25, 0xa0, + 0x61, 0x15, 0x2c, 0x78, 0x3e, 0xe1, 0x00, 0x37, 0x9c, 0x03, 0xc7, 0x3d, 0x74, 0x5e, 0x30, 0x89, + 0x65, 0x50, 0x1d, 0x2c, 0x6b, 0x2b, 0xf9, 0xca, 0x85, 0x4e, 0xbb, 0xb8, 0xb0, 0xdb, 0x4f, 0x00, + 0xf5, 0xd7, 0x2b, 0xbd, 0x97, 0x4d, 0xbf, 0x02, 0xd2, 0x5d, 0x04, 0xfc, 0x40, 0xac, 0x5e, 0xc4, + 0x86, 0xea, 0x1a, 0xdf, 0xad, 0xb7, 0x47, 0x94, 0x4c, 0xaa, 0x0d, 0x88, 0x3b, 0x39, 0x45, 0xa2, + 0x28, 0xe1, 0x07, 0xfc, 0x95, 0x06, 0x66, 0x70, 0xbd, 0x4e, 0xbc, 0x80, 0x18, 0xa2, 0xb8, 0x67, + 0x3e, 0x87, 0xfa, 0xb5, 0x20, 0xbd, 0x9a, 0x59, 0x4b, 0x42, 0xa3, 0x6e, 0x4f, 0xe0, 0xf3, 0xe0, + 0x2c, 0x0d, 0x5c, 0x9f, 0x18, 0xa9, 0xb6, 0x19, 0x76, 0xda, 0xc5, 0xb3, 0xb5, 0x2e, 0x0e, 0x4a, + 0x49, 0x96, 0x3e, 0x1d, 0x03, 0xc5, 0x7b, 0x1c, 0xb5, 0x53, 0x3c, 0xcc, 0x1e, 0x07, 0xe3, 0x7c, + 0xb9, 0x06, 0x8f, 0x4a, 0x3e, 0xd1, 0x0a, 0x72, 0x2a, 0x92, 0x5c, 0x76, 0x51, 0x30, 0x7c, 0xd6, + 0xbe, 0x64, 0xb9, 0xa0, 0xba, 0x28, 0x6a, 0x82, 0x8c, 0x22, 0x3e, 0x7c, 0x07, 0x8c, 0x8b, 0xc1, + 0x0b, 0xaf, 0xd2, 0x23, 0xac, 0xb4, 0x80, 0xfb, 0xc9, 0xa1, 0x90, 0x84, 0xec, 0xad, 0xb0, 0xb9, + 0x07, 0x5d, 0x61, 0xef, 0x5a, 0xd2, 0xc6, 0xff, 0xcf, 0x4b, 0x5a, 0xe9, 0xbf, 0x5a, 0xfa, 0xdc, + 0x27, 0x96, 0x5a, 0xab, 0x63, 0x8b, 0xc0, 0x0d, 0x30, 0xc7, 0x5e, 0x2d, 0x88, 0x78, 0x96, 0x59, + 0xc7, 0x94, 0x3f, 0x9a, 0x45, 0xc2, 0xa9, 0x39, 0x4e, 0x2d, 0xc5, 0x47, 0x3d, 0x1a, 0xf0, 0x25, + 0x00, 0x45, 0x27, 0xdf, 0x65, 0x47, 0x34, 0x25, 0xaa, 0x27, 0xaf, 0xf5, 0x48, 0xa0, 0x3e, 0x5a, + 0x70, 0x1d, 0xcc, 0x5b, 0x78, 0x8f, 0x58, 0x35, 0x62, 0x91, 0x7a, 0xe0, 0xfa, 0xdc, 0x94, 0x18, + 0x2b, 0x2c, 0x74, 0xda, 0xc5, 0xf9, 0x6b, 0x69, 0x26, 0xea, 0x95, 0x2f, 0x5d, 0x4a, 0x1f, 0xaf, + 0xe4, 0xc2, 0xc5, 0xfb, 0xe8, 0xc3, 0x0c, 0x58, 0x1a, 0x9c, 0x19, 0xf0, 0x07, 0xf1, 0x33, 0x4e, + 0x74, 0xe9, 0x6f, 0x8e, 0x2a, 0x0b, 0xe5, 0x3b, 0x0e, 0xf4, 0xbe, 0xe1, 0xe0, 0x77, 0x59, 0xcb, + 0x84, 0xad, 0x68, 0x70, 0xf4, 0xc6, 0xc8, 0x5c, 0x60, 0x20, 0x95, 0x49, 0xd1, 0x8d, 0x61, 0x8b, + 0x37, 0x5f, 0xd8, 0x22, 0xa5, 0x3f, 0x68, 0xe9, 0x97, 0x7c, 0x7c, 0x82, 0xe1, 0xcf, 0x34, 0x30, + 0xeb, 0x7a, 0xc4, 0x59, 0xdb, 0xdd, 0x7a, 0xe5, 0x2b, 0xe2, 0x24, 0xcb, 0x50, 0xed, 0xdc, 0xa7, + 0x9f, 0x2f, 0xd5, 0xaa, 0x3b, 0xc2, 0xe0, 0xae, 0xef, 0x7a, 0xb4, 0x72, 0xae, 0xd3, 0x2e, 0xce, + 0x56, 0xbb, 0xa1, 0x50, 0x1a, 0xbb, 0x64, 0x83, 0x85, 0xcd, 0xa3, 0x80, 0xf8, 0x0e, 0xb6, 0x36, + 0xdc, 0x7a, 0x68, 0x13, 0x27, 0x10, 0x8e, 0xa6, 0xa6, 0x4e, 0xda, 0x29, 0xa7, 0x4e, 0x8f, 0x80, + 0x6c, 0xe8, 0x5b, 0x32, 0x8b, 0xa7, 0xd4, 0x54, 0x15, 0x5d, 0x43, 0x8c, 0x5e, 0xba, 0x04, 0xc6, + 0x98, 0x9f, 0xf0, 0x02, 0xc8, 0xfa, 0xf8, 0x90, 0x5b, 0x9d, 0xae, 0x4c, 0x30, 0x11, 0x84, 0x0f, + 0x11, 0xa3, 0x95, 0xfe, 0x5d, 0x00, 0xb3, 0xa9, 0xb5, 0xc0, 0x25, 0x90, 0x51, 0xa3, 0x5a, 0x20, + 0x8d, 0x66, 0xb6, 0x36, 0x50, 0xc6, 0x34, 0xe0, 0xb3, 0xaa, 0xf8, 0x0a, 0xd0, 0xa2, 0xaa, 0xe7, + 0x9c, 0xca, 0x7a, 0xe4, 0xd8, 0x1c, 0x73, 0x24, 0x2a, 0x9c, 0xcc, 0x07, 0xd2, 0x90, 0xa7, 0x44, + 0xf8, 0x40, 0x1a, 0x88, 0xd1, 0x3e, 0xeb, 0xc8, 0x2d, 0x9a, 0xf9, 0xe5, 0x4e, 0x31, 0xf3, 0x1b, + 0xbf, 0xeb, 0xcc, 0xef, 0x51, 0x90, 0x0b, 0xcc, 0xc0, 0x22, 0xfa, 0x44, 0xf7, 0x53, 0xe6, 0x3a, + 0x23, 0x22, 0xc1, 0x83, 0xb7, 0xc0, 0x84, 0x41, 0x1a, 0x38, 0xb4, 0x02, 0x3d, 0xcf, 0x53, 0x68, + 0x7d, 0x08, 0x29, 0x24, 0x06, 0xb2, 0x1b, 0xc2, 0x2e, 0x8a, 0x00, 0xe0, 0x63, 0x60, 0xc2, 0xc6, + 0x47, 0xa6, 0x1d, 0xda, 0xbc, 0xc9, 0xd3, 0x84, 0xd8, 0xb6, 0x20, 0xa1, 0x88, 0xc7, 0x2a, 0x23, + 0x39, 0xaa, 0x5b, 0x21, 0x35, 0x5b, 0x44, 0x32, 0x65, 0x03, 0xa6, 0x2a, 0xe3, 0x66, 0x8a, 0x8f, + 0x7a, 0x34, 0x38, 0x98, 0xe9, 0x70, 0xe5, 0xa9, 0x04, 0x98, 0x20, 0xa1, 0x88, 0xd7, 0x0d, 0x26, + 0xe5, 0xa7, 0x07, 0x81, 0x49, 0xe5, 0x1e, 0x0d, 0xf8, 0x65, 0x30, 0x69, 0xe3, 0xa3, 0x6b, 0xc4, + 0x69, 0x06, 0xfb, 0xfa, 0xcc, 0xb2, 0xb6, 0x92, 0xad, 0xcc, 0x74, 0xda, 0xc5, 0xc9, 0xed, 0x88, + 0x88, 0x62, 0x3e, 0x17, 0x36, 0x1d, 0x29, 0x7c, 0x36, 0x21, 0x1c, 0x11, 0x51, 0xcc, 0x67, 0x1d, + 0x84, 0x87, 0x03, 0x76, 0xb8, 0xf4, 0xd9, 0xee, 0xa7, 0xe6, 0xae, 0x20, 0xa3, 0x88, 0x0f, 0x57, + 0x40, 0xde, 0xc6, 0x47, 0x7c, 0x2c, 0xa0, 0xcf, 0x71, 0xb3, 0x7c, 0x38, 0xbd, 0x2d, 0x69, 0x48, + 0x71, 0xb9, 0xa4, 0xe9, 0x08, 0xc9, 0xf9, 0x84, 0xa4, 0xa4, 0x21, 0xc5, 0x65, 0x49, 0x1c, 0x3a, + 0xe6, 0xed, 0x90, 0x08, 0x61, 0xc8, 0x23, 0xa3, 0x92, 0xf8, 0x46, 0xcc, 0x42, 0x49, 0x39, 0xf6, + 0x2c, 0xb7, 0x43, 0x2b, 0x30, 0x3d, 0x8b, 0x54, 0x1b, 0xfa, 0x39, 0x1e, 0x7f, 0xde, 0x78, 0x6f, + 0x2b, 0x2a, 0x4a, 0x48, 0x40, 0x02, 0xc6, 0x88, 0x13, 0xda, 0xfa, 0x79, 0x7e, 0xb1, 0x0f, 0x25, + 0x05, 0xd5, 0xc9, 0xd9, 0x74, 0x42, 0x1b, 0x71, 0xf3, 0xf0, 0x59, 0x30, 0x63, 0xe3, 0x23, 0x56, + 0x0e, 0x88, 0x1f, 0x98, 0x84, 0xea, 0x0b, 0x7c, 0xf1, 0xf3, 0xac, 0xe3, 0xdc, 0x4e, 0x32, 0x50, + 0xb7, 0x1c, 0x57, 0x34, 0x9d, 0x84, 0xe2, 0x62, 0x42, 0x31, 0xc9, 0x40, 0xdd, 0x72, 0x2c, 0xd2, + 0x3e, 0xb9, 0x1d, 0x9a, 0x3e, 0x31, 0xf4, 0x87, 0x78, 0x93, 0x2a, 0x3f, 0x18, 0x08, 0x1a, 0x52, + 0x5c, 0xd8, 0x8a, 0xe6, 0x47, 0x3a, 0x3f, 0x86, 0x37, 0x86, 0x5b, 0xc9, 0xab, 0xfe, 0x9a, 0xef, + 0xe3, 0x63, 0x71, 0xd3, 0x24, 0x27, 0x47, 0x90, 0x82, 0x1c, 0xb6, 0xac, 0x6a, 0x43, 0xbf, 0xc0, + 0x63, 0x3f, 0xec, 0x1b, 0x44, 0x55, 0x9d, 0x35, 0x06, 0x82, 0x04, 0x16, 0x03, 0x75, 0x1d, 0x96, + 0x1a, 0x4b, 0xa3, 0x05, 0xad, 0x32, 0x10, 0x24, 0xb0, 0xf8, 0x4a, 0x9d, 0xe3, 0x6a, 0x43, 0x7f, + 0x78, 0xc4, 0x2b, 0x65, 0x20, 0x48, 0x60, 0x41, 0x13, 0x64, 0x1d, 0x37, 0xd0, 0x2f, 0x8e, 0xe4, + 0x7a, 0xe6, 0x17, 0xce, 0x8e, 0x1b, 0x20, 0x86, 0x01, 0x7f, 0xa9, 0x01, 0xe0, 0xc5, 0x29, 0xfa, + 0xc8, 0x50, 0xc6, 0x12, 0x29, 0xc8, 0x72, 0x9c, 0xdb, 0x9b, 0x4e, 0xe0, 0x1f, 0xc7, 0xef, 0xc8, + 0xc4, 0x19, 0x48, 0x78, 0x01, 0x7f, 0xab, 0x81, 0xf3, 0xc9, 0x36, 0x59, 0xb9, 0x57, 0xe0, 0x11, + 0xb9, 0x3e, 0xec, 0x34, 0xaf, 0xb8, 0xae, 0x55, 0xd1, 0x3b, 0xed, 0xe2, 0xf9, 0xb5, 0x3e, 0xa8, + 0xa8, 0xaf, 0x2f, 0xf0, 0x8f, 0x1a, 0x98, 0x97, 0x55, 0x34, 0xe1, 0x61, 0x91, 0x07, 0x90, 0x0c, + 0x3b, 0x80, 0x69, 0x1c, 0x11, 0x47, 0xf5, 0xa1, 0xbb, 0x87, 0x8f, 0x7a, 0x5d, 0x83, 0x7f, 0xd1, + 0xc0, 0xb4, 0x41, 0x3c, 0xe2, 0x18, 0xc4, 0xa9, 0x33, 0x5f, 0x97, 0x87, 0x32, 0x36, 0x48, 0xfb, + 0xba, 0x91, 0x80, 0x10, 0x6e, 0x96, 0xa5, 0x9b, 0xd3, 0x49, 0xd6, 0x49, 0xbb, 0xb8, 0x18, 0xab, + 0x26, 0x39, 0xa8, 0xcb, 0x4b, 0xf8, 0xbe, 0x06, 0x66, 0xe3, 0x0d, 0x10, 0x57, 0xca, 0xa5, 0x11, + 0xe6, 0x01, 0x6f, 0x5f, 0xd7, 0xba, 0x01, 0x51, 0xda, 0x03, 0xf8, 0x27, 0x8d, 0x75, 0x6a, 0xd1, + 0xbb, 0x8f, 0xea, 0x25, 0x1e, 0xcb, 0xb7, 0x86, 0x1e, 0x4b, 0x85, 0x20, 0x42, 0x79, 0x39, 0x6e, + 0x05, 0x15, 0xe7, 0xa4, 0x5d, 0x5c, 0x48, 0x46, 0x52, 0x31, 0x50, 0xd2, 0x43, 0xf8, 0x13, 0x0d, + 0x4c, 0x93, 0xb8, 0xe3, 0xa6, 0xfa, 0xa3, 0x43, 0x09, 0x62, 0xdf, 0x26, 0x5e, 0xbc, 0xd4, 0x13, + 0x2c, 0x8a, 0xba, 0xb0, 0x59, 0x07, 0x49, 0x8e, 0xb0, 0xed, 0x59, 0x44, 0xff, 0xc2, 0x90, 0x3b, + 0xc8, 0x4d, 0x61, 0x17, 0x45, 0x00, 0xf0, 0x32, 0xc8, 0x3b, 0xa1, 0x65, 0xe1, 0x3d, 0x8b, 0xe8, + 0x8f, 0xf1, 0x5e, 0x44, 0x8d, 0x45, 0x77, 0x24, 0x1d, 0x29, 0x09, 0xd8, 0x00, 0xcb, 0x47, 0x2f, + 0xab, 0x7f, 0x11, 0xea, 0x3b, 0xb8, 0xd3, 0x1f, 0xe7, 0x56, 0x96, 0x3a, 0xed, 0xe2, 0xe2, 0xcd, + 0xfe, 0xa3, 0xbd, 0x7b, 0xda, 0x80, 0xaf, 0x81, 0x87, 0x13, 0x32, 0x9b, 0xf6, 0x1e, 0x31, 0x0c, + 0x62, 0x44, 0x0f, 0x37, 0xfd, 0x8b, 0x62, 0x78, 0x18, 0x1d, 0xf0, 0x9b, 0x69, 0x01, 0x74, 0x37, + 0x6d, 0x78, 0x0d, 0x2c, 0x26, 0xd8, 0x5b, 0x4e, 0x50, 0xf5, 0x6b, 0x81, 0x6f, 0x3a, 0x4d, 0x7d, + 0x85, 0xdb, 0x3d, 0x1f, 0x9d, 0xc8, 0x9b, 0x09, 0x1e, 0x1a, 0xa0, 0xb3, 0xc4, 0x9e, 0x8e, 0xa9, + 0xd2, 0x03, 0xe7, 0x40, 0xf6, 0x80, 0xc8, 0x4f, 0xee, 0x88, 0xfd, 0x09, 0x0d, 0x90, 0x6b, 0x61, + 0x2b, 0x8c, 0x5e, 0xbf, 0x43, 0xbe, 0xb6, 0x90, 0x30, 0xfe, 0x7c, 0xe6, 0x39, 0x6d, 0xe9, 0x03, + 0x0d, 0x2c, 0xf6, 0xaf, 0x88, 0x0f, 0xd4, 0xad, 0x5f, 0x6b, 0x60, 0xbe, 0xa7, 0xf8, 0xf5, 0xf1, + 0xe8, 0x76, 0xb7, 0x47, 0xaf, 0x0d, 0xbb, 0x8a, 0x89, 0x5d, 0xe3, 0xad, 0x5b, 0xd2, 0xbd, 0x9f, + 0x6b, 0x60, 0x2e, 0x5d, 0x4f, 0x1e, 0x64, 0xbc, 0x4a, 0x1f, 0x64, 0xc0, 0x62, 0xff, 0x8e, 0x13, + 0xfa, 0xea, 0x69, 0x3d, 0x9a, 0x11, 0x45, 0xbf, 0x71, 0xe6, 0xbb, 0x1a, 0x98, 0xba, 0xa5, 0xe4, + 0xa2, 0x4f, 0xb2, 0x43, 0x1f, 0x8e, 0x44, 0x05, 0x3c, 0x66, 0x50, 0x94, 0xc4, 0x2d, 0xfd, 0x59, + 0x03, 0x0b, 0x7d, 0x6f, 0x26, 0xf6, 0x86, 0xc7, 0x96, 0xe5, 0x1e, 0x8a, 0x19, 0x57, 0x62, 0x80, + 0xbc, 0xc6, 0xa9, 0x48, 0x72, 0x13, 0xd1, 0xcb, 0x7c, 0x5e, 0xd1, 0x2b, 0xfd, 0x4d, 0x03, 0x17, + 0xef, 0x96, 0x89, 0x0f, 0x64, 0x4b, 0x57, 0x40, 0x5e, 0x76, 0x95, 0xc7, 0x7c, 0x3b, 0xe5, 0x43, + 0x4a, 0x16, 0x0d, 0xfe, 0x5f, 0x48, 0xe2, 0xaf, 0xd2, 0x87, 0x1a, 0x98, 0xab, 0x11, 0xbf, 0x65, + 0xd6, 0x09, 0x22, 0x0d, 0xe2, 0x13, 0xa7, 0x4e, 0xe0, 0x2a, 0x98, 0xe4, 0xdf, 0x42, 0x3d, 0x5c, + 0x8f, 0xe6, 0xfa, 0xf3, 0x32, 0xe4, 0x93, 0x3b, 0x11, 0x03, 0xc5, 0x32, 0xea, 0x1b, 0x40, 0x66, + 0xe0, 0x37, 0x80, 0x8b, 0x60, 0xcc, 0x8b, 0x27, 0xa4, 0x79, 0xc6, 0xe5, 0x43, 0x51, 0x4e, 0xe5, + 0x5c, 0xd7, 0x0f, 0xf8, 0xd8, 0x27, 0x27, 0xb9, 0xae, 0x1f, 0x20, 0x4e, 0x2d, 0xfd, 0x5d, 0x03, + 0xfd, 0xfe, 0x5f, 0x08, 0xb6, 0xc0, 0x04, 0x15, 0xae, 0xcb, 0xd0, 0x56, 0xef, 0x33, 0xb4, 0xe9, + 0x40, 0x88, 0x7b, 0x35, 0xa2, 0x46, 0x60, 0x2c, 0xba, 0x75, 0x5c, 0x09, 0x1d, 0x43, 0x4e, 0x3c, + 0xa7, 0x45, 0x74, 0xd7, 0xd7, 0x04, 0x0d, 0x29, 0x2e, 0xbc, 0x20, 0x66, 0x73, 0x89, 0x81, 0x57, + 0x34, 0x97, 0xab, 0x5c, 0xf9, 0xe8, 0x4e, 0xe1, 0xcc, 0xc7, 0x77, 0x0a, 0x67, 0x3e, 0xb9, 0x53, + 0x38, 0xf3, 0xbd, 0x4e, 0x41, 0xfb, 0xa8, 0x53, 0xd0, 0x3e, 0xee, 0x14, 0xb4, 0x4f, 0x3a, 0x05, + 0xed, 0x9f, 0x9d, 0x82, 0xf6, 0x8b, 0x4f, 0x0b, 0x67, 0xbe, 0x35, 0x21, 0x5d, 0xfb, 0x5f, 0x00, + 0x00, 0x00, 0xff, 0xff, 0x10, 0x75, 0x48, 0x06, 0xc5, 0x2b, 0x00, 0x00, } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto index 5dc1aac4db1..09a713fef7b 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto @@ -247,6 +247,12 @@ message CustomResourceDefinitionSpec { // `conversion` defines conversion settings for the CRD. // +optional optional CustomResourceConversion conversion = 9; + + // preserverUnknownFields disables pruning of object fields which are not + // specified in the OpenAPI schema. apiVersion, kind, metadata and known + // fields inside metadata are always excluded from pruning. + // Defaults to true in v1beta and to will default to false in v1. + optional bool preserveUnknownFields = 10; } // CustomResourceDefinitionStatus indicates the state of the CustomResourceDefinition diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go index 0ca6673bf4a..32d6ee7e8e1 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/types.go @@ -78,6 +78,12 @@ type CustomResourceDefinitionSpec struct { // `conversion` defines conversion settings for the CRD. // +optional Conversion *CustomResourceConversion `json:"conversion,omitempty" protobuf:"bytes,9,opt,name=conversion"` + + // preserveUnknownFields disables pruning of object fields which are not + // specified in the OpenAPI schema. apiVersion, kind, metadata and known + // fields inside metadata are always preserved. + // Defaults to true in v1beta and will default to false in v1. + PreserveUnknownFields *bool `json:"preserveUnknownFields,omitempty" protobuf:"varint,10,opt,name=preserveUnknownFields"` } // CustomResourceConversion describes how to convert different versions of a CR. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go index 13ce5e288e6..90be856c7f9 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.conversion.go @@ -504,6 +504,7 @@ func autoConvert_v1beta1_CustomResourceDefinitionSpec_To_apiextensions_CustomRes } else { out.Conversion = nil } + out.PreserveUnknownFields = (*bool)(unsafe.Pointer(in.PreserveUnknownFields)) return nil } @@ -550,6 +551,7 @@ func autoConvert_apiextensions_CustomResourceDefinitionSpec_To_v1beta1_CustomRes } else { out.Conversion = nil } + out.PreserveUnknownFields = (*bool)(unsafe.Pointer(in.PreserveUnknownFields)) return nil } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go index bc7eaa3309d..96f176f4612 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/zz_generated.deepcopy.go @@ -283,6 +283,11 @@ func (in *CustomResourceDefinitionSpec) DeepCopyInto(out *CustomResourceDefiniti *out = new(CustomResourceConversion) (*in).DeepCopyInto(*out) } + if in.PreserveUnknownFields != nil { + in, out := &in.PreserveUnknownFields, &out.PreserveUnknownFields + *out = new(bool) + **out = **in + } return } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go index 667e556ac95..845b6ef030a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/zz_generated.deepcopy.go @@ -201,6 +201,11 @@ func (in *CustomResourceDefinitionSpec) DeepCopyInto(out *CustomResourceDefiniti *out = new(CustomResourceConversion) (*in).DeepCopyInto(*out) } + if in.PreserveUnknownFields != nil { + in, out := &in.PreserveUnknownFields, &out.PreserveUnknownFields + *out = new(bool) + **out = **in + } return } From 32d05973f580711d4d03d743c20595f1c7bb922b Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 2 May 2019 12:03:24 +0200 Subject: [PATCH 2/8] apiextensions: add structural schema validation if preserveUnknownFields=false --- .../apiextensions/validation/validation.go | 36 +- .../validation/validation_test.go | 522 +++++++++++++++++- 2 files changed, 526 insertions(+), 32 deletions(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go index 2950ffaae27..59cbedaac83 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation.go @@ -21,6 +21,7 @@ import ( "reflect" "strings" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" apiequality "k8s.io/apimachinery/pkg/api/equality" genericvalidation "k8s.io/apimachinery/pkg/api/validation" "k8s.io/apimachinery/pkg/util/sets" @@ -102,9 +103,9 @@ func ValidateUpdateCustomResourceDefinitionStatus(obj, oldObj *apiextensions.Cus } // ValidateCustomResourceDefinitionVersion statically validates. -func ValidateCustomResourceDefinitionVersion(version *apiextensions.CustomResourceDefinitionVersion, fldPath *field.Path, statusEnabled bool) field.ErrorList { +func ValidateCustomResourceDefinitionVersion(version *apiextensions.CustomResourceDefinitionVersion, fldPath *field.Path, mustBeStructural, statusEnabled bool) field.ErrorList { allErrs := field.ErrorList{} - allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(version.Schema, statusEnabled, fldPath.Child("schema"))...) + allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(version.Schema, mustBeStructural, statusEnabled, fldPath.Child("schema"))...) allErrs = append(allErrs, ValidateCustomResourceDefinitionSubresources(version.Subresources, fldPath.Child("subresources"))...) for i := range version.AdditionalPrinterColumns { allErrs = append(allErrs, ValidateCustomResourceColumnDefinition(&version.AdditionalPrinterColumns[i], fldPath.Child("additionalPrinterColumns").Index(i))...) @@ -130,6 +131,20 @@ func validateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi allErrs = append(allErrs, validateEnumStrings(fldPath.Child("scope"), string(spec.Scope), []string{string(apiextensions.ClusterScoped), string(apiextensions.NamespaceScoped)}, true)...) + mustBeStructural := false + if spec.PreserveUnknownFields == nil || *spec.PreserveUnknownFields == false { + mustBeStructural = true + // check that either a global schema or versioned schemas are set + if spec.Validation == nil || spec.Validation.OpenAPIV3Schema == nil { + for i, v := range spec.Versions { + schemaPath := fldPath.Child("versions").Index(i).Child("schema", "openAPIV3Schema") + if v.Served && (v.Schema == nil || v.Schema.OpenAPIV3Schema == nil) { + allErrs = append(allErrs, field.Required(schemaPath, "because otherwise all fields are pruned")) + } + } + } + } + storageFlagCount := 0 versionsMap := map[string]bool{} uniqueNames := true @@ -146,7 +161,7 @@ func validateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi allErrs = append(allErrs, field.Invalid(fldPath.Child("versions").Index(i).Child("name"), spec.Versions[i].Name, strings.Join(errs, ","))) } subresources := getSubresourcesForVersion(spec, version.Name) - allErrs = append(allErrs, ValidateCustomResourceDefinitionVersion(&version, fldPath.Child("versions").Index(i), hasStatusEnabled(subresources))...) + allErrs = append(allErrs, ValidateCustomResourceDefinitionVersion(&version, fldPath.Child("versions").Index(i), mustBeStructural, hasStatusEnabled(subresources))...) } // The top-level and per-version fields are mutual exclusive @@ -201,7 +216,7 @@ func validateCustomResourceDefinitionSpec(spec *apiextensions.CustomResourceDefi } allErrs = append(allErrs, ValidateCustomResourceDefinitionNames(&spec.Names, fldPath.Child("names"))...) - allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(spec.Validation, hasAnyStatusEnabled(spec), fldPath.Child("validation"))...) + allErrs = append(allErrs, ValidateCustomResourceDefinitionValidation(spec.Validation, mustBeStructural, hasAnyStatusEnabled(spec), fldPath.Child("validation"))...) allErrs = append(allErrs, ValidateCustomResourceDefinitionSubresources(spec.Subresources, fldPath.Child("subresources"))...) for i := range spec.AdditionalPrinterColumns { @@ -531,7 +546,7 @@ type specStandardValidator interface { } // ValidateCustomResourceDefinitionValidation statically validates -func ValidateCustomResourceDefinitionValidation(customResourceValidation *apiextensions.CustomResourceValidation, statusSubresourceEnabled bool, fldPath *field.Path) field.ErrorList { +func ValidateCustomResourceDefinitionValidation(customResourceValidation *apiextensions.CustomResourceValidation, mustBeStructural, statusSubresourceEnabled bool, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if customResourceValidation == nil { @@ -573,6 +588,17 @@ func ValidateCustomResourceDefinitionValidation(customResourceValidation *apiext openAPIV3Schema := &specStandardValidatorV3{} allErrs = append(allErrs, ValidateCustomResourceDefinitionOpenAPISchema(schema, fldPath.Child("openAPIV3Schema"), openAPIV3Schema)...) + + if mustBeStructural { + if ss, err := structuralschema.NewStructural(schema); err != nil { + // if the generic schema validation did its job, we should never get an error here. Hence, we hide it if there are validation errors already. + if len(allErrs) == 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("openAPIV3Schema"), "", err.Error())) + } + } else { + allErrs = append(allErrs, structuralschema.ValidateStructural(ss, fldPath.Child("openAPIV3Schema"))...) + } + } } // if validation passed otherwise, make sure we can actually construct a schema validator from this custom resource validation. diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go index babd1c309bc..6cf96dcc575 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/validation_test.go @@ -103,6 +103,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { }, }, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -147,6 +148,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { }, }, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -191,6 +193,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { }, }, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -231,6 +234,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { URL: strPtr(""), }, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -272,6 +276,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { URL: strPtr("https://example.com/webhook"), }, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -310,6 +315,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Strategy: apiextensions.ConversionStrategyType("None"), ConversionReviewVersions: []string{"v1beta1"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -351,6 +357,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { }, ConversionReviewVersions: []string{"invalid-version"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -392,6 +399,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { }, ConversionReviewVersions: []string{"0v"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -434,6 +442,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { }, ConversionReviewVersions: []string{"invalid-version", "v1beta1"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -473,6 +482,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { }, ConversionReviewVersions: []string{"v1beta1", "v1beta1"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -510,6 +520,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Conversion: &apiextensions.CustomResourceConversion{ Strategy: apiextensions.ConversionStrategyType("Webhook"), }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -547,6 +558,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Conversion: &apiextensions.CustomResourceConversion{ Strategy: apiextensions.ConversionStrategyType("non_existing_conversion"), }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -584,6 +596,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Conversion: &apiextensions.CustomResourceConversion{ Strategy: apiextensions.ConversionStrategyType("None"), }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -621,6 +634,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Conversion: &apiextensions.CustomResourceConversion{ Strategy: apiextensions.ConversionStrategyType("None"), }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -659,6 +673,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Conversion: &apiextensions.CustomResourceConversion{ Strategy: apiextensions.ConversionStrategyType("None"), }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -691,6 +706,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Conversion: &apiextensions.CustomResourceConversion{ Strategy: apiextensions.ConversionStrategyType("None"), }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{}, @@ -709,6 +725,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Names: apiextensions.CustomResourceDefinitionNames{ Plural: "plural", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, }, errors: []validationMatch{ @@ -752,6 +769,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Kind: "value()*a", ListKind: "value()*a", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -798,6 +816,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Kind: "matching", ListKind: "matching", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -843,6 +862,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { AdditionalProperties: &apiextensions.JSONSchemaPropsOrBool{Allows: false}, }, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -880,6 +900,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { }, }, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -923,6 +944,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Kind: "Plural", ListKind: "PluralList", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -962,6 +984,7 @@ func TestValidateCustomResourceDefinition(t *testing.T) { Kind: "Plural", ListKind: "PluralList", }, + PreserveUnknownFields: pointer.BoolPtr(false), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -971,36 +994,206 @@ func TestValidateCustomResourceDefinition(t *testing.T) { invalid("spec", "validation", "openAPIV3Schema", "x-kubernetes-preserve-unknown-fields"), }, }, + { + name: "preserveUnknownFields with unstructural global schema", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validUnstructuralValidationSchema, + }, + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + }, + { + Name: "version2", + Served: true, + Storage: false, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + required("spec", "validation", "openAPIV3Schema", "properties[spec]", "type"), + required("spec", "validation", "openAPIV3Schema", "properties[status]", "type"), + required("spec", "validation", "openAPIV3Schema", "items", "type"), + }, + }, + { + name: "preserveUnknownFields with unstructural schema in one version", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validValidationSchema, + }, + }, + { + Name: "version2", + Served: true, + Storage: false, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validUnstructuralValidationSchema, + }, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + required("spec", "versions[1]", "schema", "openAPIV3Schema", "properties[spec]", "type"), + required("spec", "versions[1]", "schema", "openAPIV3Schema", "properties[status]", "type"), + required("spec", "versions[1]", "schema", "openAPIV3Schema", "items", "type"), + }, + }, + { + name: "preserveUnknownFields with no schema in one version", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validValidationSchema, + }, + }, + { + Name: "version2", + Served: true, + Storage: false, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: nil, + }, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + required("spec", "versions[1]", "schema", "openAPIV3Schema"), + }, + }, + { + name: "preserveUnknownFields with no schema at all", + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "plural.group.com"}, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + Schema: nil, + }, + { + Name: "version2", + Served: true, + Storage: false, + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: nil, + }, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + required("spec", "versions[0]", "schema", "openAPIV3Schema"), + required("spec", "versions[1]", "schema", "openAPIV3Schema"), + }, + }, } for _, tc := range tests { - // duplicate defaulting behaviour - if tc.resource.Spec.Conversion != nil && tc.resource.Spec.Conversion.Strategy == apiextensions.WebhookConverter && len(tc.resource.Spec.Conversion.ConversionReviewVersions) == 0 { - tc.resource.Spec.Conversion.ConversionReviewVersions = []string{"v1beta1"} - } - errs := ValidateCustomResourceDefinition(tc.resource) - seenErrs := make([]bool, len(errs)) + t.Run(tc.name, func(t *testing.T) { + // duplicate defaulting behaviour + if tc.resource.Spec.Conversion != nil && tc.resource.Spec.Conversion.Strategy == apiextensions.WebhookConverter && len(tc.resource.Spec.Conversion.ConversionReviewVersions) == 0 { + tc.resource.Spec.Conversion.ConversionReviewVersions = []string{"v1beta1"} + } + errs := ValidateCustomResourceDefinition(tc.resource) + seenErrs := make([]bool, len(errs)) - for _, expectedError := range tc.errors { - found := false - for i, err := range errs { - if expectedError.matches(err) && !seenErrs[i] { - found = true - seenErrs[i] = true - break + for _, expectedError := range tc.errors { + found := false + for i, err := range errs { + if expectedError.matches(err) && !seenErrs[i] { + found = true + seenErrs[i] = true + break + } + } + + if !found { + t.Errorf("expected %v at %v, got %v", expectedError.errorType, expectedError.path.String(), errs) } } - if !found { - t.Errorf("%s: expected %v at %v, got %v", tc.name, expectedError.errorType, expectedError.path.String(), errs) + for i, seen := range seenErrs { + if !seen { + t.Errorf("unexpected error: %v", errs[i]) + } } - } - - for i, seen := range seenErrs { - if !seen { - t.Errorf("%s: unexpected error: %v", tc.name, errs[i]) - } - } + }) } } @@ -1043,6 +1236,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { }, ConversionReviewVersions: []string{"invalid-version"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -1078,6 +1272,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { }, ConversionReviewVersions: []string{"invalid-version_0, invalid-version"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -1117,6 +1312,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { }, ConversionReviewVersions: []string{"invalid-version"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -1152,6 +1348,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { }, ConversionReviewVersions: []string{"v1beta1", "invalid-version"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -1193,6 +1390,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { }, ConversionReviewVersions: []string{"invalid-version"}, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -1227,6 +1425,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { URL: strPtr("https://example.com/webhook"), }, }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -1260,6 +1459,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind", ListKind: "listkind", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1292,6 +1492,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind", ListKind: "listkind", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1329,6 +1530,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind", ListKind: "listkind", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1364,6 +1566,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind", ListKind: "listkind", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1406,6 +1609,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind", ListKind: "listkind", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1442,6 +1646,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind", ListKind: "listkind", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1481,6 +1686,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind", ListKind: "listkind", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1516,6 +1722,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind2", ListKind: "listkind2", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1556,6 +1763,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind", ListKind: "listkind", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1591,6 +1799,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "kind2", ListKind: "listkind2", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ AcceptedNames: apiextensions.CustomResourceDefinitionNames{ @@ -1639,6 +1848,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "Plural", ListKind: "PluralList", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -1680,6 +1890,7 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { Kind: "Plural", ListKind: "PluralList", }, + PreserveUnknownFields: pointer.BoolPtr(true), }, Status: apiextensions.CustomResourceDefinitionStatus{ StoredVersions: []string{"version"}, @@ -1690,6 +1901,208 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { forbidden("spec", "subresources"), }, }, + { + name: "switch off preserveUnknownFields with structural schema before and after", + old: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "plural.group.com", + ResourceVersion: "42", + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validValidationSchema, + }, + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(true), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "plural.group.com", + ResourceVersion: "42", + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validUnstructuralValidationSchema, + }, + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{ + required("spec", "validation", "openAPIV3Schema", "properties[spec]", "type"), + required("spec", "validation", "openAPIV3Schema", "properties[status]", "type"), + required("spec", "validation", "openAPIV3Schema", "items", "type"), + }, + }, + { + name: "switch off preserveUnknownFields without structural schema before, but with after", + old: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "plural.group.com", + ResourceVersion: "42", + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validUnstructuralValidationSchema, + }, + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(true), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "plural.group.com", + ResourceVersion: "42", + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validValidationSchema, + }, + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{}, + }, + { + name: "switch on preserveUnknownFields without structural schema", + old: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "plural.group.com", + ResourceVersion: "42", + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validValidationSchema, + }, + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(false), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + resource: &apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "plural.group.com", + ResourceVersion: "42", + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "group.com", + Version: "version", + Validation: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: validUnstructuralValidationSchema, + }, + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "version", + Served: true, + Storage: true, + }, + }, + Scope: apiextensions.NamespaceScoped, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "plural", + Singular: "singular", + Kind: "Plural", + ListKind: "PluralList", + }, + PreserveUnknownFields: pointer.BoolPtr(true), + }, + Status: apiextensions.CustomResourceDefinitionStatus{ + StoredVersions: []string{"version"}, + }, + }, + errors: []validationMatch{}, + }, } for _, tc := range tests { @@ -1721,10 +2134,11 @@ func TestValidateCustomResourceDefinitionUpdate(t *testing.T) { func TestValidateCustomResourceDefinitionValidation(t *testing.T) { tests := []struct { - name string - input apiextensions.CustomResourceValidation - statusEnabled bool - wantError bool + name string + input apiextensions.CustomResourceValidation + mustBeStructural bool + statusEnabled bool + wantError bool }{ { name: "empty", @@ -1844,10 +2258,28 @@ func TestValidateCustomResourceDefinitionValidation(t *testing.T) { }, wantError: false, }, + { + name: "must be structural, but isn't", + input: apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{}, + }, + mustBeStructural: true, + wantError: true, + }, + { + name: "must be structural", + input: apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + }, + }, + mustBeStructural: true, + wantError: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := ValidateCustomResourceDefinitionValidation(&tt.input, tt.statusEnabled, field.NewPath("spec", "validation")) + got := ValidateCustomResourceDefinitionValidation(&tt.input, tt.mustBeStructural, tt.statusEnabled, field.NewPath("spec", "validation")) if !tt.wantError && len(got) > 0 { t.Errorf("Expected no error, but got: %v", got) } else if tt.wantError && len(got) == 0 { @@ -1860,6 +2292,42 @@ func TestValidateCustomResourceDefinitionValidation(t *testing.T) { var example = apiextensions.JSON(`"This is an example"`) var validValidationSchema = &apiextensions.JSONSchemaProps{ + Description: "This is a description", + Type: "object", + Format: "date-time", + Title: "This is a title", + Maximum: float64Ptr(10), + ExclusiveMaximum: true, + Minimum: float64Ptr(5), + ExclusiveMinimum: true, + MaxLength: int64Ptr(10), + MinLength: int64Ptr(5), + Pattern: "^[a-z]$", + MaxItems: int64Ptr(10), + MinItems: int64Ptr(5), + MultipleOf: float64Ptr(3), + Required: []string{"spec", "status"}, + Properties: map[string]apiextensions.JSONSchemaProps{ + "spec": { + Type: "object", + Items: &apiextensions.JSONSchemaPropsOrArray{ + Schema: &apiextensions.JSONSchemaProps{ + Description: "This is a schema nested under Items", + Type: "string", + }, + }, + }, + "status": { + Type: "object", + }, + }, + ExternalDocs: &apiextensions.ExternalDocumentation{ + Description: "This is an external documentation description", + }, + Example: &example, +} + +var validUnstructuralValidationSchema = &apiextensions.JSONSchemaProps{ Description: "This is a description", Type: "object", Format: "date-time", From 52577aa90837e64f744213e2a07b270f0ca2262a Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 2 May 2019 12:04:54 +0200 Subject: [PATCH 3/8] apiextensions: low-level pruning algorithms --- .../pkg/apiserver/schema/pruning/algorithm.go | 91 ++++++ .../schema/pruning/algorithm_test.go | 298 ++++++++++++++++++ 2 files changed, 389 insertions(+) create mode 100644 staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/algorithm.go create mode 100644 staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/algorithm_test.go diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/algorithm.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/algorithm.go new file mode 100644 index 00000000000..c0c1591c798 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/algorithm.go @@ -0,0 +1,91 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pruning + +import ( + "fmt" + + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" +) + +// Prune removes object fields in obj which are not specified in s. +func Prune(obj map[string]interface{}, s *structuralschema.Structural) { + prune(obj, s) +} + +func prune(x interface{}, s *structuralschema.Structural) { + if s != nil && s.XPreserveUnknownFields { + skipPrune(x, s) + return + } + + switch x := x.(type) { + case map[string]interface{}: + if s == nil { + for k := range x { + delete(x, k) + } + return + } + for k, v := range x { + prop, ok := s.Properties[k] + if ok { + prune(v, &prop) + } else if s.AdditionalProperties != nil { + prune(v, s.AdditionalProperties.Structural) + } else { + delete(x, k) + } + fmt.Printf("deleting %q => %#v\n", k, x) + } + case []interface{}: + if s == nil { + for _, v := range x { + prune(v, nil) + } + return + } + for _, v := range x { + prune(v, s.Items) + } + default: + // scalars, do nothing + } +} + +func skipPrune(x interface{}, s *structuralschema.Structural) { + if s == nil { + return + } + + switch x := x.(type) { + case map[string]interface{}: + for k, v := range x { + if prop, ok := s.Properties[k]; ok { + prune(v, &prop) + } else { + skipPrune(v, nil) + } + } + case []interface{}: + for _, v := range x { + skipPrune(v, s.Items) + } + default: + // scalars, do nothing + } +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/algorithm_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/algorithm_test.go new file mode 100644 index 00000000000..94e6de774cb --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/algorithm_test.go @@ -0,0 +1,298 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pruning + +import ( + "bytes" + "reflect" + "testing" + + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/json" +) + +func TestPrune(t *testing.T) { + tests := []struct { + name string + json string + schema *structuralschema.Structural + expected string + }{ + {"empty", "null", nil, "null"}, + {"scalar", "4", &structuralschema.Structural{}, "4"}, + {"scalar array", "[1,2]", &structuralschema.Structural{ + Items: &structuralschema.Structural{}, + }, "[1,2]"}, + {"object array", `[{"a":1},{"b":1},{"a":1,"b":2,"c":3}]`, &structuralschema.Structural{ + Items: &structuralschema.Structural{ + Properties: map[string]structuralschema.Structural{ + "a": {}, + "c": {}, + }, + }, + }, `[{"a":1},{},{"a":1,"c":3}]`}, + {"object array with nil schema", `[{"a":1},{"b":1},{"a":1,"b":2,"c":3}]`, nil, `[{},{},{}]`}, + {"object array object", `{"array":[{"a":1},{"b":1},{"a":1,"b":2,"c":3}],"unspecified":{"a":1},"specified":{"a":1,"b":2,"c":3}}`, &structuralschema.Structural{ + Properties: map[string]structuralschema.Structural{ + "array": { + Items: &structuralschema.Structural{ + Properties: map[string]structuralschema.Structural{ + "a": {}, + "c": {}, + }, + }, + }, + "specified": { + Properties: map[string]structuralschema.Structural{ + "a": {}, + "c": {}, + }, + }, + }, + }, `{"array":[{"a":1},{},{"a":1,"c":3}],"specified":{"a":1,"c":3}}`}, + {"nested x-kubernetes-preserve-unknown-fields", ` +{ + "unspecified":"bar", + "alpha": "abc", + "beta": 42.0, + "unspecifiedObject": {"unspecified": "bar"}, + "pruning": { + "unspecified": "bar", + "unspecifiedObject": {"unspecified": "bar"}, + "pruning": {"unspecified": "bar"}, + "preserving": {"unspecified": "bar"} + }, + "preserving": { + "unspecified": "bar", + "unspecifiedObject": {"unspecified": "bar"}, + "pruning": {"unspecified": "bar"}, + "preserving": {"unspecified": "bar"} + } +} +`, &structuralschema.Structural{ + Generic: structuralschema.Generic{Type: "object"}, + Extensions: structuralschema.Extensions{XPreserveUnknownFields: true}, + Properties: map[string]structuralschema.Structural{ + "alpha": {Generic: structuralschema.Generic{Type: "string"}}, + "beta": {Generic: structuralschema.Generic{Type: "number"}}, + "pruning": { + Generic: structuralschema.Generic{Type: "object"}, + Properties: map[string]structuralschema.Structural{ + "preserving": { + Generic: structuralschema.Generic{Type: "object"}, + Extensions: structuralschema.Extensions{XPreserveUnknownFields: true}, + }, + "pruning": { + Generic: structuralschema.Generic{Type: "object"}, + }, + }, + }, + "preserving": { + Generic: structuralschema.Generic{Type: "object"}, + Extensions: structuralschema.Extensions{XPreserveUnknownFields: true}, + Properties: map[string]structuralschema.Structural{ + "preserving": { + Generic: structuralschema.Generic{Type: "object"}, + Extensions: structuralschema.Extensions{XPreserveUnknownFields: true}, + }, + "pruning": { + Generic: structuralschema.Generic{Type: "object"}, + }, + }, + }, + }, + }, ` +{ + "unspecified":"bar", + "alpha": "abc", + "beta": 42.0, + "unspecifiedObject": {"unspecified": "bar"}, + "pruning": { + "pruning": {}, + "preserving": {"unspecified": "bar"} + }, + "preserving": { + "unspecified": "bar", + "unspecifiedObject": {"unspecified": "bar"}, + "pruning": {}, + "preserving": {"unspecified": "bar"} + } +} +`}, + {"additionalProperties with schema", `{"a":1,"b":1,"c":{"a":1,"b":2,"c":{"a":1}}}`, &structuralschema.Structural{ + Properties: map[string]structuralschema.Structural{ + "a": {}, + "c": { + Generic: structuralschema.Generic{ + AdditionalProperties: &structuralschema.StructuralOrBool{ + Structural: &structuralschema.Structural{ + Generic: structuralschema.Generic{ + Type: "integer", + }, + }, + }, + }, + }, + }, + }, `{"a":1,"c":{"a":1,"b":2,"c":{}}}`}, + {"additionalProperties with bool", `{"a":1,"b":1,"c":{"a":1,"b":2,"c":{"a":1}}}`, &structuralschema.Structural{ + Properties: map[string]structuralschema.Structural{ + "a": {}, + "c": { + Generic: structuralschema.Generic{ + AdditionalProperties: &structuralschema.StructuralOrBool{ + Bool: false, + }, + }, + }, + }, + }, `{"a":1,"c":{"a":1,"b":2,"c":{}}}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var in interface{} + if err := json.Unmarshal([]byte(tt.json), &in); err != nil { + t.Fatal(err) + } + + var expected interface{} + if err := json.Unmarshal([]byte(tt.expected), &expected); err != nil { + t.Fatal(err) + } + + prune(in, tt.schema) + if !reflect.DeepEqual(in, expected) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + err := enc.Encode(in) + if err != nil { + t.Fatalf("unexpected result mashalling error: %v", err) + } + t.Errorf("expected: %s\ngot: %s", tt.expected, buf.String()) + } + }) + } +} + +const smallInstance = ` +{ + "unspecified":"bar", + "alpha": "abc", + "beta": 42.0, + "unspecifiedObject": {"unspecified": "bar"}, + "pruning": { + "pruning": {}, + "preserving": {"unspecified": "bar"} + }, + "preserving": { + "unspecified": "bar", + "unspecifiedObject": {"unspecified": "bar"}, + "pruning": {}, + "preserving": {"unspecified": "bar"} + } +} +` + +func BenchmarkPrune(b *testing.B) { + b.StopTimer() + b.ReportAllocs() + + schema := &structuralschema.Structural{ + Generic: structuralschema.Generic{Type: "object"}, + Extensions: structuralschema.Extensions{XPreserveUnknownFields: true}, + Properties: map[string]structuralschema.Structural{ + "alpha": {Generic: structuralschema.Generic{Type: "string"}}, + "beta": {Generic: structuralschema.Generic{Type: "number"}}, + "pruning": { + Generic: structuralschema.Generic{Type: "object"}, + Properties: map[string]structuralschema.Structural{ + "preserving": { + Generic: structuralschema.Generic{Type: "object"}, + Extensions: structuralschema.Extensions{XPreserveUnknownFields: true}, + }, + "pruning": { + Generic: structuralschema.Generic{Type: "object"}, + }, + }, + }, + "preserving": { + Generic: structuralschema.Generic{Type: "object"}, + Extensions: structuralschema.Extensions{XPreserveUnknownFields: true}, + Properties: map[string]structuralschema.Structural{ + "preserving": { + Generic: structuralschema.Generic{Type: "object"}, + Extensions: structuralschema.Extensions{XPreserveUnknownFields: true}, + }, + "pruning": { + Generic: structuralschema.Generic{Type: "object"}, + }, + }, + }, + }, + } + + var obj map[string]interface{} + err := json.Unmarshal([]byte(smallInstance), &obj) + if err != nil { + b.Fatal(err) + } + + instances := make([]map[string]interface{}, 0, b.N) + for i := 0; i < b.N; i++ { + instances = append(instances, runtime.DeepCopyJSON(obj)) + } + + b.StartTimer() + for i := 0; i < b.N; i++ { + Prune(instances[i], schema) + } +} + +func BenchmarkDeepCopy(b *testing.B) { + b.StopTimer() + b.ReportAllocs() + + var obj map[string]interface{} + err := json.Unmarshal([]byte(smallInstance), &obj) + if err != nil { + b.Fatal(err) + } + + instances := make([]map[string]interface{}, 0, b.N) + + b.StartTimer() + for i := 0; i < b.N; i++ { + instances = append(instances, runtime.DeepCopyJSON(obj)) + } +} + +func BenchmarkUnmarshal(b *testing.B) { + b.StopTimer() + b.ReportAllocs() + + instances := make([]map[string]interface{}, b.N) + + b.StartTimer() + for i := 0; i < b.N; i++ { + err := json.Unmarshal([]byte(smallInstance), &instances[i]) + if err != nil { + b.Fatal(err) + } + } +} From 70ee02725f1c70a3309e10b93d3ee02696747486 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 2 May 2019 12:03:42 +0200 Subject: [PATCH 4/8] apiextensions: wire pruning into handler --- .../pkg/apiserver/customresource_handler.go | 69 ++++++++++++++----- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 0cfbe187295..748c35145ca 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -30,6 +30,8 @@ import ( "github.com/go-openapi/validate" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" "k8s.io/apiextensions-apiserver/pkg/apiserver/conversion" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + structuralpruning "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" informers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion" listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion" @@ -39,6 +41,7 @@ import ( apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features" "k8s.io/apiextensions-apiserver/pkg/registry/customresource" "k8s.io/apiextensions-apiserver/pkg/registry/customresource/tableconvertor" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -520,6 +523,20 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource return nil, err } + // Check for nil because we dereference this throughout the handler code. + // Note: we always default this to non-nil. But we should guard these dereferences any way. + if crd.Spec.PreserveUnknownFields == nil { + return nil, fmt.Errorf("unexpected nil spec.preserveUnknownFields in the CustomResourceDefinition") + } + + var structuralSchema *structuralschema.Structural + if validationSchema != nil { + structuralSchema, err = structuralschema.NewStructural(validationSchema.OpenAPIV3Schema) + if *crd.Spec.PreserveUnknownFields == false && err != nil { + return nil, err // validation should avoid this + } + } + var statusSpec *apiextensions.CustomResourceSubresourceStatus var statusValidator *validate.SchemaValidator subresources, err := apiextensions.GetSubresourcesForVersion(crd, v.Name) @@ -570,10 +587,12 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource scaleSpec, ), crdConversionRESTOptionsGetter{ - RESTOptionsGetter: r.restOptionsGetter, - converter: safeConverter, - decoderVersion: schema.GroupVersion{Group: crd.Spec.Group, Version: v.Name}, - encoderVersion: schema.GroupVersion{Group: crd.Spec.Group, Version: storageVersion}, + RESTOptionsGetter: r.restOptionsGetter, + converter: safeConverter, + decoderVersion: schema.GroupVersion{Group: crd.Spec.Group, Version: v.Name}, + encoderVersion: schema.GroupVersion{Group: crd.Spec.Group, Version: storageVersion}, + structuralSchema: structuralSchema, + preserveUnknownFields: *crd.Spec.PreserveUnknownFields, }, crd.Status.AcceptedNames.Categories, table, @@ -595,7 +614,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource ClusterScoped: clusterScoped, SelfLinkPathPrefix: selfLinkPrefix, }, - Serializer: unstructuredNegotiatedSerializer{typer: typer, creator: creator, converter: safeConverter}, + Serializer: unstructuredNegotiatedSerializer{typer: typer, creator: creator, converter: safeConverter, structuralSchema: structuralSchema, preserveUnknownFields: *crd.Spec.PreserveUnknownFields}, ParameterCodec: parameterCodec, Creater: creator, @@ -646,6 +665,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource // shallow copy statusScope := *requestScopes[v.Name] statusScope.Subresource = "status" + statusScope.Serializer = unstructuredNegotiatedSerializer{typer: typer, creator: creator, converter: safeConverter, structuralSchema: structuralSchema, preserveUnknownFields: *crd.Spec.PreserveUnknownFields} statusScope.Namer = handlers.ContextBasedNaming{ SelfLinker: meta.NewAccessor(), ClusterScoped: clusterScoped, @@ -680,6 +700,9 @@ type unstructuredNegotiatedSerializer struct { typer runtime.ObjectTyper creator runtime.ObjectCreater converter runtime.ObjectConvertor + + structuralSchema *structuralschema.Structural + preserveUnknownFields bool } func (s unstructuredNegotiatedSerializer) SupportedMediaTypes() []runtime.SerializerInfo { @@ -712,7 +735,7 @@ func (s unstructuredNegotiatedSerializer) EncoderForVersion(encoder runtime.Enco } func (s unstructuredNegotiatedSerializer) DecoderToVersion(decoder runtime.Decoder, gv runtime.GroupVersioner) runtime.Decoder { - d := schemaCoercingDecoder{delegate: decoder, validator: unstructuredSchemaCoercer{}} + d := schemaCoercingDecoder{delegate: decoder, validator: unstructuredSchemaCoercer{structuralSchema: s.structuralSchema, preserveUnknownFields: s.preserveUnknownFields}} return versioning.NewDefaultingCodecForScheme(Scheme, nil, d, nil, gv) } @@ -801,19 +824,23 @@ func (in crdStorageMap) clone() crdStorageMap { // provided custom converter and custom encoder and decoder version. type crdConversionRESTOptionsGetter struct { generic.RESTOptionsGetter - converter runtime.ObjectConvertor - encoderVersion schema.GroupVersion - decoderVersion schema.GroupVersion + converter runtime.ObjectConvertor + encoderVersion schema.GroupVersion + decoderVersion schema.GroupVersion + structuralSchema *structuralschema.Structural + preserveUnknownFields bool } func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) { ret, err := t.RESTOptionsGetter.GetRESTOptions(resource) if err == nil { d := schemaCoercingDecoder{delegate: ret.StorageConfig.Codec, validator: unstructuredSchemaCoercer{ - // drop invalid fields while decoding old CRs (before we had any ObjectMeta validation) - dropInvalidMetadata: true, + // drop invalid fields while decoding old CRs (before we haven't had any ObjectMeta validation) + dropInvalidMetadata: true, + structuralSchema: t.structuralSchema, + preserveUnknownFields: t.preserveUnknownFields, }} - c := schemaCoercingConverter{delegate: t.converter, validator: unstructuredSchemaCoercer{}} + c := schemaCoercingConverter{delegate: t.converter, validator: unstructuredSchemaCoercer{structuralSchema: t.structuralSchema, preserveUnknownFields: t.preserveUnknownFields}} ret.StorageConfig.Codec = versioning.NewCodec( ret.StorageConfig.Codec, d, @@ -894,13 +921,15 @@ func (v schemaCoercingConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, return v.delegate.ConvertFieldLabel(gvk, label, value) } -// unstructuredSchemaCoercer does the validation for Unstructured that json.Unmarshal -// does for native types. This includes: -// - validating and pruning ObjectMeta (here with optional error instead of pruning) -// - TODO: application of an OpenAPI validator (against the whole object or a top-level field of it). -// - TODO: optionally application of post-validation algorithms like defaulting and/or OpenAPI based pruning. +// unstructuredSchemaCoercer adds to unstructured unmarshalling what json.Unmarshal does +// in addition for native types when decoding into Golang structs: +// +// - validating and pruning ObjectMeta +// - generic pruning of unknown fields following a structural schema. type unstructuredSchemaCoercer struct { - dropInvalidMetadata bool + dropInvalidMetadata bool + structuralSchema *structuralschema.Structural + preserveUnknownFields bool } func (v *unstructuredSchemaCoercer) apply(u *unstructured.Unstructured) error { @@ -918,6 +947,10 @@ func (v *unstructuredSchemaCoercer) apply(u *unstructured.Unstructured) error { return err } + if !v.preserveUnknownFields { + structuralpruning.Prune(u.Object, v.structuralSchema) + } + // restore meta fields, starting clean if foundKind { u.SetKind(kind) From 3f3ed79484adf050fc7654a887a151be3a14022b Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 13 May 2019 17:08:28 +0200 Subject: [PATCH 5/8] apiextensions: only prune correct GroupKind --- .../pkg/apiserver/customresource_handler.go | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 748c35145ca..c9088ad4962 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -592,6 +592,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource decoderVersion: schema.GroupVersion{Group: crd.Spec.Group, Version: v.Name}, encoderVersion: schema.GroupVersion{Group: crd.Spec.Group, Version: storageVersion}, structuralSchema: structuralSchema, + structuralSchemaGK: kind.GroupKind(), preserveUnknownFields: *crd.Spec.PreserveUnknownFields, }, crd.Status.AcceptedNames.Categories, @@ -614,7 +615,14 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource ClusterScoped: clusterScoped, SelfLinkPathPrefix: selfLinkPrefix, }, - Serializer: unstructuredNegotiatedSerializer{typer: typer, creator: creator, converter: safeConverter, structuralSchema: structuralSchema, preserveUnknownFields: *crd.Spec.PreserveUnknownFields}, + Serializer: unstructuredNegotiatedSerializer{ + typer: typer, + creator: creator, + converter: safeConverter, + structuralSchema: structuralSchema, + structuralSchemaGK: kind.GroupKind(), + preserveUnknownFields: *crd.Spec.PreserveUnknownFields, + }, ParameterCodec: parameterCodec, Creater: creator, @@ -665,7 +673,13 @@ func (r *crdHandler) getOrCreateServingInfoFor(crd *apiextensions.CustomResource // shallow copy statusScope := *requestScopes[v.Name] statusScope.Subresource = "status" - statusScope.Serializer = unstructuredNegotiatedSerializer{typer: typer, creator: creator, converter: safeConverter, structuralSchema: structuralSchema, preserveUnknownFields: *crd.Spec.PreserveUnknownFields} + statusScope.Serializer = unstructuredNegotiatedSerializer{ + typer: typer, creator: creator, + converter: safeConverter, + structuralSchema: structuralSchema, + structuralSchemaGK: kind.GroupKind(), + preserveUnknownFields: *crd.Spec.PreserveUnknownFields, + } statusScope.Namer = handlers.ContextBasedNaming{ SelfLinker: meta.NewAccessor(), ClusterScoped: clusterScoped, @@ -702,6 +716,7 @@ type unstructuredNegotiatedSerializer struct { converter runtime.ObjectConvertor structuralSchema *structuralschema.Structural + structuralSchemaGK schema.GroupKind preserveUnknownFields bool } @@ -735,7 +750,7 @@ func (s unstructuredNegotiatedSerializer) EncoderForVersion(encoder runtime.Enco } func (s unstructuredNegotiatedSerializer) DecoderToVersion(decoder runtime.Decoder, gv runtime.GroupVersioner) runtime.Decoder { - d := schemaCoercingDecoder{delegate: decoder, validator: unstructuredSchemaCoercer{structuralSchema: s.structuralSchema, preserveUnknownFields: s.preserveUnknownFields}} + d := schemaCoercingDecoder{delegate: decoder, validator: unstructuredSchemaCoercer{structuralSchema: s.structuralSchema, structuralSchemaGK: s.structuralSchemaGK, preserveUnknownFields: s.preserveUnknownFields}} return versioning.NewDefaultingCodecForScheme(Scheme, nil, d, nil, gv) } @@ -828,6 +843,7 @@ type crdConversionRESTOptionsGetter struct { encoderVersion schema.GroupVersion decoderVersion schema.GroupVersion structuralSchema *structuralschema.Structural + structuralSchemaGK schema.GroupKind preserveUnknownFields bool } @@ -838,9 +854,14 @@ func (t crdConversionRESTOptionsGetter) GetRESTOptions(resource schema.GroupReso // drop invalid fields while decoding old CRs (before we haven't had any ObjectMeta validation) dropInvalidMetadata: true, structuralSchema: t.structuralSchema, + structuralSchemaGK: t.structuralSchemaGK, + preserveUnknownFields: t.preserveUnknownFields, + }} + c := schemaCoercingConverter{delegate: t.converter, validator: unstructuredSchemaCoercer{ + structuralSchema: t.structuralSchema, + structuralSchemaGK: t.structuralSchemaGK, preserveUnknownFields: t.preserveUnknownFields, }} - c := schemaCoercingConverter{delegate: t.converter, validator: unstructuredSchemaCoercer{structuralSchema: t.structuralSchema, preserveUnknownFields: t.preserveUnknownFields}} ret.StorageConfig.Codec = versioning.NewCodec( ret.StorageConfig.Codec, d, @@ -927,8 +948,10 @@ func (v schemaCoercingConverter) ConvertFieldLabel(gvk schema.GroupVersionKind, // - validating and pruning ObjectMeta // - generic pruning of unknown fields following a structural schema. type unstructuredSchemaCoercer struct { - dropInvalidMetadata bool + dropInvalidMetadata bool + structuralSchema *structuralschema.Structural + structuralSchemaGK schema.GroupKind preserveUnknownFields bool } @@ -947,7 +970,12 @@ func (v *unstructuredSchemaCoercer) apply(u *unstructured.Unstructured) error { return err } - if !v.preserveUnknownFields { + // compare group and kind because also other object like DeleteCollection options pass through here + gv, err := schema.ParseGroupVersion(apiVersion) + if err != nil { + return err + } + if !v.preserveUnknownFields && gv.Group == v.structuralSchemaGK.Group && kind == v.structuralSchemaGK.Kind { structuralpruning.Prune(u.Object, v.structuralSchema) } From 77bfddacfdb3eb7335dfff1762d3917427592595 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 2 May 2019 12:04:39 +0200 Subject: [PATCH 6/8] apiextensions: add pruning integration tests --- .../test/integration/pruning_test.go | 459 ++++++++++++++++++ test/e2e/apimachinery/webhook.go | 38 +- test/integration/etcd/data.go | 31 ++ 3 files changed, 522 insertions(+), 6 deletions(-) create mode 100644 staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go b/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go new file mode 100644 index 00000000000..5ed8591ec03 --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/pruning_test.go @@ -0,0 +1,459 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "path" + "strings" + "testing" + + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/pkg/transport" + "sigs.k8s.io/yaml" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/client-go/dynamic" + "k8s.io/utils/pointer" + + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apiextensions-apiserver/test/integration/fixtures" +) + +var pruningFixture = &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "foos.tests.apiextensions.k8s.io"}, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "tests.apiextensions.k8s.io", + Version: "v1beta1", + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "foos", + Singular: "foo", + Kind: "Foo", + ListKind: "FooList", + }, + Scope: apiextensionsv1beta1.ClusterScoped, + PreserveUnknownFields: pointer.BoolPtr(false), + Subresources: &apiextensionsv1beta1.CustomResourceSubresources{ + Status: &apiextensionsv1beta1.CustomResourceSubresourceStatus{}, + }, + }, +} + +const ( + fooSchema = ` +type: object +properties: + alpha: + type: string + beta: + type: number +` + + fooStatusSchema = ` +type: object +properties: + status: + type: object + properties: + alpha: + type: string + beta: + type: number +` + + fooSchemaPreservingUnknownFields = ` +type: object +properties: + alpha: + type: string + beta: + type: number + preserving: + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + preserving: + type: object + x-kubernetes-preserve-unknown-fields: true + pruning: + type: object + pruning: + type: object + properties: + preserving: + type: object + x-kubernetes-preserve-unknown-fields: true + pruning: + type: object +x-kubernetes-preserve-unknown-fields: true +` + + fooInstance = ` +kind: Foo +apiVersion: tests.apiextensions.k8s.io/v1beta1 +metadata: + name: foo +` +) + +func TestPruningCreate(t *testing.T) { + tearDownFn, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) + if err != nil { + t.Fatal(err) + } + defer tearDownFn() + + crd := pruningFixture.DeepCopy() + crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{} + if err := yaml.Unmarshal([]byte(fooSchema), &crd.Spec.Validation.OpenAPIV3Schema); err != nil { + t.Fatal(err) + } + + crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + + t.Logf("Creating CR and expect 'unspecified' fields to be pruned") + fooClient := dynamicClient.Resource(schema.GroupVersionResource{crd.Spec.Group, crd.Spec.Version, crd.Spec.Names.Plural}) + foo := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(fooInstance), &foo.Object); err != nil { + t.Fatal(err) + } + unstructured.SetNestedField(foo.Object, "bar", "unspecified") + unstructured.SetNestedField(foo.Object, "abc", "alpha") + unstructured.SetNestedField(foo.Object, float64(42.0), "beta") + unstructured.SetNestedField(foo.Object, "bar", "metadata", "unspecified") + unstructured.SetNestedField(foo.Object, "bar", "metadata", "labels", "foo") + foo, err = fooClient.Create(foo, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Unable to create CR: %v", err) + } + t.Logf("CR created: %#v", foo.UnstructuredContent()) + + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "unspecified"); found { + t.Errorf("Expected 'unspecified' field to be pruned, but it was not") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "alpha"); !found { + t.Errorf("Expected specified 'alpha' field to stay, but it was pruned") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "beta"); !found { + t.Errorf("Expected specified 'beta' field to stay, but it was pruned") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "metadata", "unspecified"); found { + t.Errorf("Expected 'metadata.unspecified' field to be pruned, but it was not") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "metadata", "labels", "foo"); !found { + t.Errorf("Expected specified 'metadata.labels[foo]' field to stay, but it was pruned") + } +} + +func TestPruningStatus(t *testing.T) { + tearDownFn, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) + if err != nil { + t.Fatal(err) + } + defer tearDownFn() + + crd := pruningFixture.DeepCopy() + crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{} + if err := yaml.Unmarshal([]byte(fooStatusSchema), &crd.Spec.Validation.OpenAPIV3Schema); err != nil { + t.Fatal(err) + } + + crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + + t.Logf("Creating CR and expect 'unspecified' fields to be pruned") + fooClient := dynamicClient.Resource(schema.GroupVersionResource{crd.Spec.Group, crd.Spec.Version, crd.Spec.Names.Plural}) + foo := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(fooInstance), &foo.Object); err != nil { + t.Fatal(err) + } + foo, err = fooClient.Create(foo, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Unable to create CR: %v", err) + } + t.Logf("CR created: %#v", foo.UnstructuredContent()) + + unstructured.SetNestedField(foo.Object, "bar", "status", "unspecified") + unstructured.SetNestedField(foo.Object, "abc", "status", "alpha") + unstructured.SetNestedField(foo.Object, float64(42.0), "status", "beta") + unstructured.SetNestedField(foo.Object, "bar", "metadata", "unspecified") + + foo, err = fooClient.UpdateStatus(foo, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Unable to update status: %v", err) + } + + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "unspecified"); found { + t.Errorf("Expected 'status.unspecified' field to be pruned, but it was not") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "status", "alpha"); !found { + t.Errorf("Expected specified 'status.alpha' field to stay, but it was pruned") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "status", "beta"); !found { + t.Errorf("Expected specified 'status.beta' field to stay, but it was pruned") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "metadata", "unspecified"); found { + t.Errorf("Expected 'metadata.unspecified' field to be pruned, but it was not") + } +} + +func TestPruningFromStorage(t *testing.T) { + tearDown, config, options, err := fixtures.StartDefaultServer(t) + if err != nil { + t.Fatal(err) + } + defer tearDown() + + apiExtensionClient, err := clientset.NewForConfig(config) + if err != nil { + t.Fatal(err) + } + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + t.Fatal(err) + } + + serverConfig, err := options.Config() + if err != nil { + t.Fatal(err) + } + + crd := pruningFixture.DeepCopy() + crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{} + if err := yaml.Unmarshal([]byte(fooSchema), &crd.Spec.Validation.OpenAPIV3Schema); err != nil { + t.Fatal(err) + } + + crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + + restOptions, err := serverConfig.GenericConfig.RESTOptionsGetter.GetRESTOptions(schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}) + if err != nil { + t.Fatal(err) + } + tlsInfo := transport.TLSInfo{ + CertFile: restOptions.StorageConfig.Transport.CertFile, + KeyFile: restOptions.StorageConfig.Transport.KeyFile, + CAFile: restOptions.StorageConfig.Transport.CAFile, + } + tlsConfig, err := tlsInfo.ClientConfig() + if err != nil { + t.Fatal(err) + } + etcdConfig := clientv3.Config{ + Endpoints: restOptions.StorageConfig.Transport.ServerList, + TLS: tlsConfig, + } + etcdclient, err := clientv3.New(etcdConfig) + if err != nil { + t.Fatal(err) + } + + t.Logf("Creating object with unknown field manually in etcd") + + original := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(fooInstance), &original.Object); err != nil { + t.Fatal(err) + } + unstructured.SetNestedField(original.Object, "bar", "unspecified") + unstructured.SetNestedField(original.Object, "abc", "alpha") + unstructured.SetNestedField(original.Object, float64(42), "beta") + unstructured.SetNestedField(original.Object, "bar", "metadata", "labels", "foo") + + // Note: we don't add metadata.unspecified as in the other tests. ObjectMeta pruning is independent of the generic pruning + // and we do not guarantee that we prune ObjectMeta on read from etcd. + + ctx := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault) + key := path.Join("/", restOptions.StorageConfig.Prefix, crd.Spec.Group, "foos/foo") + val, _ := json.Marshal(original.UnstructuredContent()) + if _, err := etcdclient.Put(ctx, key, string(val)); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + t.Logf("Checking that CustomResource is pruned from unknown fields") + fooClient := dynamicClient.Resource(schema.GroupVersionResource{crd.Spec.Group, crd.Spec.Version, crd.Spec.Names.Plural}) + foo, err := fooClient.Get("foo", metav1.GetOptions{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "unspecified"); found { + t.Errorf("Expected 'unspecified' field to be pruned, but it was not") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "alpha"); !found { + t.Errorf("Expected specified 'alpha' field to stay, but it was pruned") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "beta"); !found { + t.Errorf("Expected specified 'beta' field to stay, but it was pruned") + } + + // Note: we don't check metadata.foo as in the other tests. ObjectMeta pruning is independent of the generic pruning + // and we do not guarantee that we prune ObjectMeta on read from etcd. + + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "metadata", "labels", "foo"); !found { + t.Errorf("Expected specified 'metadata.labels[foo]' field to stay, but it was pruned") + } +} + +func TestPruningPatch(t *testing.T) { + tearDownFn, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) + if err != nil { + t.Fatal(err) + } + defer tearDownFn() + + crd := pruningFixture.DeepCopy() + crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{} + if err := yaml.Unmarshal([]byte(fooSchema), &crd.Spec.Validation.OpenAPIV3Schema); err != nil { + t.Fatal(err) + } + crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + + fooClient := dynamicClient.Resource(schema.GroupVersionResource{crd.Spec.Group, crd.Spec.Version, crd.Spec.Names.Plural}) + foo := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(fooInstance), &foo.Object); err != nil { + t.Fatal(err) + } + foo, err = fooClient.Create(foo, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Unable to create CR: %v", err) + } + t.Logf("CR created: %#v", foo.UnstructuredContent()) + + // a patch with a change + patch := []byte(`{"alpha": "abc", "beta": 42.0, "unspecified": "bar", "metadata": {"unspecified": "bar", "labels":{"foo":"bar"}}}`) + if foo, err = fooClient.Patch("foo", types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "unspecified"); found { + t.Errorf("Expected 'unspecified' field to be pruned, but it was not") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "alpha"); !found { + t.Errorf("Expected specified 'alpha' field to stay, but it was pruned") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "beta"); !found { + t.Errorf("Expected specified 'beta' field to stay, but it was pruned") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "metadata", "unspecified"); found { + t.Errorf("Expected 'metadata.unspecified' field to be pruned, but it was not") + } + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, "metadata", "labels", "foo"); !found { + t.Errorf("Expected specified 'metadata.labels[foo]' field to stay, but it was pruned") + } +} + +func TestPruningCreatePreservingUnknownFields(t *testing.T) { + tearDownFn, apiExtensionClient, dynamicClient, err := fixtures.StartDefaultServerWithClients(t) + if err != nil { + t.Fatal(err) + } + defer tearDownFn() + + crd := pruningFixture.DeepCopy() + crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{} + if err := yaml.Unmarshal([]byte(fooSchemaPreservingUnknownFields), &crd.Spec.Validation.OpenAPIV3Schema); err != nil { + t.Fatal(err) + } + + crd, err = fixtures.CreateNewCustomResourceDefinition(crd, apiExtensionClient, dynamicClient) + if err != nil { + t.Fatal(err) + } + + t.Logf("Creating CR and expect 'unspecified' field to be pruned") + fooClient := dynamicClient.Resource(schema.GroupVersionResource{crd.Spec.Group, crd.Spec.Version, crd.Spec.Names.Plural}) + foo := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(fooInstance), &foo.Object); err != nil { + t.Fatal(err) + } + unstructured.SetNestedField(foo.Object, "bar", "unspecified") + unstructured.SetNestedField(foo.Object, "abc", "alpha") + unstructured.SetNestedField(foo.Object, float64(42.0), "beta") + unstructured.SetNestedField(foo.Object, "bar", "metadata", "unspecified") + unstructured.SetNestedField(foo.Object, "bar", "metadata", "labels", "foo") + unstructured.SetNestedField(foo.Object, map[string]interface{}{ + "unspecified": "bar", + "unspecifiedObject": map[string]interface{}{"unspecified": "bar"}, + "pruning": map[string]interface{}{"unspecified": "bar"}, + "preserving": map[string]interface{}{"unspecified": "bar"}, + }, "pruning") + unstructured.SetNestedField(foo.Object, map[string]interface{}{ + "unspecified": "bar", + "unspecifiedObject": map[string]interface{}{"unspecified": "bar"}, + "pruning": map[string]interface{}{"unspecified": "bar"}, + "preserving": map[string]interface{}{"unspecified": "bar"}, + }, "preserving") + + foo, err = fooClient.Create(foo, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Unable to create CR: %v", err) + } + t.Logf("CR created: %#v", foo.UnstructuredContent()) + + for _, pth := range [][]string{ + {"unspecified"}, + {"alpha"}, + {"beta"}, + {"metadata", "labels", "foo"}, + + {"pruning", "pruning"}, + {"pruning", "preserving"}, + {"pruning", "preserving", "unspecified"}, + + {"preserving", "unspecified"}, + {"preserving", "unspecifiedObject"}, + {"preserving", "unspecifiedObject", "unspecified"}, + {"preserving", "pruning"}, + {"preserving", "preserving"}, + {"preserving", "preserving", "unspecified"}, + } { + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, pth...); !found { + t.Errorf("Expected '%s' field to stay, but it was pruned", strings.Join(pth, ".")) + } + } + for _, pth := range [][]string{ + {"metadata", "unspecified"}, + + {"pruning", "unspecified"}, + {"pruning", "unspecifiedObject"}, + {"pruning", "unspecifiedObject", "unspecified"}, + {"pruning", "pruning", "unspecified"}, + + {"preserving", "pruning", "unspecified"}, + } { + if _, found, _ := unstructured.NestedFieldNoCopy(foo.Object, pth...); found { + t.Errorf("Expected '%s' field to be pruned, but it was not", strings.Join(pth, ".")) + } + } +} diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index d63d239e121..b795a48ccaf 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -181,7 +181,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() { defer testcrd.CleanUp() webhookCleanup := registerMutatingWebhookForCustomResource(f, context, testcrd) defer webhookCleanup() - testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClients["v1"]) + testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClients["v1"], false) }) ginkgo.It("Should deny crd creation", func() { @@ -202,6 +202,30 @@ var _ = SIGDescribe("AdmissionWebhook", func() { testMultiVersionCustomResourceWebhook(f, testcrd) }) + ginkgo.It("Should mutate custom resource with pruning", func() { + const prune = true + testcrd, err := createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f, func(crd *apiextensionsv1beta1.CustomResourceDefinition) { + crd.Spec.PreserveUnknownFields = pointer.BoolPtr(false) + crd.Spec.Validation = &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "mutation-start": {Type: "string"}, + "mutation-stage-1": {Type: "string"}, + // mutation-stage-2 is intentionally missing such that it is pruned + }, + }, + } + }) + if err != nil { + return + } + defer testcrd.CleanUp() + webhookCleanup := registerMutatingWebhookForCustomResource(f, context, testcrd) + defer webhookCleanup() + testMutatingCustomResourceWebhook(f, testcrd.Crd, testcrd.DynamicClients["v1"], prune) + }) + ginkgo.It("Should deny crd creation", func() { crdWebhookCleanup := registerValidatingWebhookForCRD(f, context) defer crdWebhookCleanup() @@ -1329,7 +1353,7 @@ func testCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1beta1 } } -func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1beta1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface) { +func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextensionsv1beta1.CustomResourceDefinition, customResourceClient dynamic.ResourceInterface, prune bool) { ginkgo.By("Creating a custom resource that should be mutated by the webhook") crName := "cr-instance-1" cr := &unstructured.Unstructured{ @@ -1350,7 +1374,9 @@ func testMutatingCustomResourceWebhook(f *framework.Framework, crd *apiextension expectedCRData := map[string]interface{}{ "mutation-start": "yes", "mutation-stage-1": "yes", - "mutation-stage-2": "yes", + } + if !prune { + expectedCRData["mutation-stage-2"] = "yes" } if !reflect.DeepEqual(expectedCRData, mutatedCR.Object["data"]) { framework.Failf("\nexpected %#v\n, got %#v\n", expectedCRData, mutatedCR.Object["data"]) @@ -1571,9 +1597,9 @@ func testSlowWebhookTimeoutNoError(f *framework.Framework) { // createAdmissionWebhookMultiVersionTestCRDWithV1Storage creates a new CRD specifically // for the admissin webhook calling test. -func createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f *framework.Framework) (*crd.TestCrd, error) { +func createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f *framework.Framework, opts ...crd.Option) (*crd.TestCrd, error) { group := fmt.Sprintf("%s-multiversion-crd-test.k8s.io", f.BaseName) - return crd.CreateMultiVersionTestCRD(f, group, func(crd *apiextensionsv1beta1.CustomResourceDefinition) { + return crd.CreateMultiVersionTestCRD(f, group, append([]crd.Option{func(crd *apiextensionsv1beta1.CustomResourceDefinition) { crd.Spec.Versions = []apiextensionsv1beta1.CustomResourceDefinitionVersion{ { Name: "v1", @@ -1586,7 +1612,7 @@ func createAdmissionWebhookMultiVersionTestCRDWithV1Storage(f *framework.Framewo Storage: false, }, } - }) + }}, opts...)...) } // servedAPIVersions returns the API versions served by the CRD. diff --git a/test/integration/etcd/data.go b/test/integration/etcd/data.go index db7dc8ecf64..2f96aceaf3e 100644 --- a/test/integration/etcd/data.go +++ b/test/integration/etcd/data.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/features" + "k8s.io/utils/pointer" ) // GetEtcdStorageData returns etcd data for all persisted objects. @@ -485,6 +486,10 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes ExpectedEtcdPath: "/registry/awesome.bears.com/pandas/cr4panda", ExpectedGVK: gvkP("awesome.bears.com", "v1", "Panda"), }, + gvr("random.numbers.com", "v1", "integers"): { + Stub: `{"kind": "Integer", "apiVersion": "random.numbers.com/v1", "metadata": {"name": "fortytwo"}, "value": 42, "garbage": "oiujnasdf"}`, // requires TypeMeta due to CRD scheme's UnstructuredObjectTyper + ExpectedEtcdPath: "/registry/random.numbers.com/integers/fortytwo", + }, // -- // k8s.io/kubernetes/pkg/apis/auditregistration/v1alpha1 @@ -580,6 +585,32 @@ func GetCustomResourceDefinitionData() []*apiextensionsv1beta1.CustomResourceDef }, }, }, + // cluster scoped with legacy version field and pruning. + { + ObjectMeta: metav1.ObjectMeta{ + Name: "integers.random.numbers.com", + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: "random.numbers.com", + Version: "v1", + Scope: apiextensionsv1beta1.ClusterScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: "integers", + Kind: "Integer", + }, + Validation: &apiextensionsv1beta1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "value": { + Type: "number", + }, + }, + }, + }, + PreserveUnknownFields: pointer.BoolPtr(false), + }, + }, // cluster scoped with versions field { ObjectMeta: metav1.ObjectMeta{ From c6712455bd7bc23d69e4b903bbacb2518e173f44 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 13 May 2019 20:41:25 +0200 Subject: [PATCH 7/8] apiextensions: add pruning e2e & integration tests for admission webhooks --- test/e2e/apimachinery/webhook.go | 11 +++-- .../admissionwebhook/admission_test.go | 43 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/test/e2e/apimachinery/webhook.go b/test/e2e/apimachinery/webhook.go index b795a48ccaf..fa05c07cd01 100644 --- a/test/e2e/apimachinery/webhook.go +++ b/test/e2e/apimachinery/webhook.go @@ -210,9 +210,14 @@ var _ = SIGDescribe("AdmissionWebhook", func() { OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{ Type: "object", Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ - "mutation-start": {Type: "string"}, - "mutation-stage-1": {Type: "string"}, - // mutation-stage-2 is intentionally missing such that it is pruned + "data": { + Type: "object", + Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{ + "mutation-start": {Type: "string"}, + "mutation-stage-1": {Type: "string"}, + // mutation-stage-2 is intentionally missing such that it is pruned + }, + }, }, }, } diff --git a/test/integration/apiserver/admissionwebhook/admission_test.go b/test/integration/apiserver/admissionwebhook/admission_test.go index 3672eb3874f..4dd0b91d4cd 100644 --- a/test/integration/apiserver/admissionwebhook/admission_test.go +++ b/test/integration/apiserver/admissionwebhook/admission_test.go @@ -113,6 +113,9 @@ var ( gvr("", "v1", "nodes/proxy"): {"*": testSubresourceProxy}, gvr("", "v1", "pods/proxy"): {"*": testSubresourceProxy}, gvr("", "v1", "services/proxy"): {"*": testSubresourceProxy}, + + gvr("random.numbers.com", "v1", "integers"): {"create": testPruningRandomNumbers}, + gvr("custom.fancy.com", "v2", "pants"): {"create": testNoPruningCustomFancy}, } // admissionExemptResources lists objects which are exempt from admission validation/mutation, @@ -931,6 +934,46 @@ func testSubresourceProxy(c *testContext) { } } +func testPruningRandomNumbers(c *testContext) { + testResourceCreate(c) + + cr2pant, err := c.client.Resource(c.gvr).Get("fortytwo", metav1.GetOptions{}) + if err != nil { + c.t.Error(err) + return + } + + foo, found, err := unstructured.NestedString(cr2pant.Object, "foo") + if err != nil { + c.t.Error(err) + return + } + if found { + c.t.Errorf("expected .foo to be pruned, but got: %s", foo) + } +} + +func testNoPruningCustomFancy(c *testContext) { + testResourceCreate(c) + + cr2pant, err := c.client.Resource(c.gvr).Get("cr2pant", metav1.GetOptions{}) + if err != nil { + c.t.Error(err) + return + } + + foo, _, err := unstructured.NestedString(cr2pant.Object, "foo") + if err != nil { + c.t.Error(err) + return + } + + // check that no pruning took place + if expected, got := "test", foo; expected != got { + c.t.Errorf("expected /foo to be %q, got: %q", expected, got) + } +} + // // utility methods // From d10f8c1ad3e00bd7c690e833f79b342159e06819 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 13 May 2019 14:00:23 +0200 Subject: [PATCH 8/8] Update generated files --- api/openapi-spec/swagger.json | 4 +++ .../pkg/apis/apiextensions/fuzzer/BUILD | 1 + .../apiextensions/v1beta1/generated.proto | 6 ++-- .../pkg/apis/apiextensions/validation/BUILD | 1 + .../pkg/apiserver/BUILD | 2 ++ .../pkg/apiserver/schema/BUILD | 5 ++- .../pkg/apiserver/schema/pruning/BUILD | 35 +++++++++++++++++++ .../test/integration/BUILD | 2 ++ test/integration/etcd/BUILD | 1 + vendor/modules.txt | 1 + 10 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/BUILD diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index e2185d97442..33d6eaf1d7d 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -16500,6 +16500,10 @@ "$ref": "#/definitions/io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1beta1.CustomResourceDefinitionNames", "description": "Names are the names used to describe this custom resource" }, + "preserveUnknownFields": { + "description": "preserveUnknownFields disables pruning of object fields which are not specified in the OpenAPI schema. apiVersion, kind, metadata and known fields inside metadata are always preserved. Defaults to true in v1beta and will default to false in v1.", + "type": "boolean" + }, "scope": { "description": "Scope indicates whether this resource is cluster or namespace scoped. Default is namespaced", "type": "string" diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/BUILD index 4bd40a500b7..f70d642333a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/BUILD @@ -15,6 +15,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto index 09a713fef7b..166ad8bd135 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/generated.proto @@ -248,10 +248,10 @@ message CustomResourceDefinitionSpec { // +optional optional CustomResourceConversion conversion = 9; - // preserverUnknownFields disables pruning of object fields which are not + // preserveUnknownFields disables pruning of object fields which are not // specified in the OpenAPI schema. apiVersion, kind, metadata and known - // fields inside metadata are always excluded from pruning. - // Defaults to true in v1beta and to will default to false in v1. + // fields inside metadata are always preserved. + // Defaults to true in v1beta and will default to false in v1. optional bool preserveUnknownFields = 10; } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD index 319042d44d5..1cb403092cb 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation/BUILD @@ -14,6 +14,7 @@ go_library( deps = [ "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD index 60686c95c91..1c2dd779f40 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD @@ -23,6 +23,8 @@ go_library( "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/conversion:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset:go_default_library", diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/BUILD index 34ecd395b82..426c8bd6fb1 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/BUILD @@ -31,7 +31,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/BUILD b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/BUILD new file mode 100644 index 00000000000..4e01a72badb --- /dev/null +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning/BUILD @@ -0,0 +1,35 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["algorithm.go"], + importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning", + importpath = "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning", + visibility = ["//visibility:public"], + deps = ["//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema:go_default_library"], +) + +go_test( + name = "go_default_test", + srcs = ["algorithm_test.go"], + embed = [":go_default_library"], + deps = [ + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD index eebb1c7511b..8830d55815e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD +++ b/staging/src/k8s.io/apiextensions-apiserver/test/integration/BUILD @@ -14,6 +14,7 @@ go_test( "change_test.go", "finalization_test.go", "objectmeta_test.go", + "pruning_test.go", "registration_test.go", "subresources_test.go", "table_test.go", @@ -54,6 +55,7 @@ go_test( "//vendor/github.com/coreos/etcd/pkg/transport:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/test/integration/etcd/BUILD b/test/integration/etcd/BUILD index ecf1142563a..a76bd60cbd5 100644 --- a/test/integration/etcd/BUILD +++ b/test/integration/etcd/BUILD @@ -76,5 +76,6 @@ go_library( "//test/integration/framework:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library", "//vendor/github.com/coreos/etcd/clientv3/concurrency:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ], ) diff --git a/vendor/modules.txt b/vendor/modules.txt index e26509bca1b..4fbd11c8fc7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1073,6 +1073,7 @@ k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation k8s.io/apiextensions-apiserver/pkg/apiserver k8s.io/apiextensions-apiserver/pkg/apiserver/conversion k8s.io/apiextensions-apiserver/pkg/apiserver/schema +k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning k8s.io/apiextensions-apiserver/pkg/apiserver/validation k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme