diff --git a/staging/src/k8s.io/apiserver/pkg/cel/openapi/extensions.go b/staging/src/k8s.io/apiserver/pkg/cel/openapi/extensions.go new file mode 100644 index 00000000000..e7817cfe230 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/cel/openapi/extensions.go @@ -0,0 +1,81 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +var intOrStringFormat = intstr.IntOrString{}.OpenAPISchemaFormat() + +func isExtension(schema *spec.Schema, key string) bool { + v, ok := schema.Extensions.GetBool(key) + return v && ok +} + +func isXIntOrString(schema *spec.Schema) bool { + return schema.Format == intOrStringFormat || isExtension(schema, extIntOrString) +} + +func isXEmbeddedResource(schema *spec.Schema) bool { + return isExtension(schema, extEmbeddedResource) +} + +func isXPreserveUnknownFields(schema *spec.Schema) bool { + return isExtension(schema, extPreserveUnknownFields) +} + +func getXListType(schema *spec.Schema) string { + s, _ := schema.Extensions.GetString(extListType) + return s +} + +func getXListMapKeys(schema *spec.Schema) []string { + items, ok := schema.Extensions[extListMapKeys] + if !ok { + return nil + } + // items may be any of + // - a slice of string + // - a slice of interface{}, a.k.a any, but item's real type is string + // there is no direct conversion, so do that manually + switch items.(type) { + case []string: + return items.([]string) + case []any: + a := items.([]any) + result := make([]string, 0, len(a)) + for _, item := range a { + // item must be a string + s, ok := item.(string) + if !ok { + return nil + } + result = append(result, s) + } + return result + } + // no further attempt of handling unexpected type + return nil +} + +const extIntOrString = "x-kubernetes-int-or-string" +const extEmbeddedResource = "x-kubernetes-embedded-resource" +const extPreserveUnknownFields = "x-kubernetes-preserve-unknown-fields" +const extListType = "x-kubernetes-list-type" +const extListMapKeys = "x-kubernetes-list-map-keys" diff --git a/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist.go b/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist.go new file mode 100644 index 00000000000..d018fc8a10d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist.go @@ -0,0 +1,179 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "fmt" + "strings" + + "k8s.io/kube-openapi/pkg/validation/spec" +) + +// mapList provides a "lookup by key" operation for lists (arrays) with x-kubernetes-list-type=map. +type mapList interface { + // get returns the first element having given key, for all + // x-kubernetes-list-map-keys, to the provided object. If the provided object isn't itself a valid mapList element, + // get returns nil. + get(interface{}) interface{} +} + +type keyStrategy interface { + // CompositeKeyFor returns a composite key for the provided object, if possible, and a + // boolean that indicates whether or not a key could be generated for the provided object. + CompositeKeyFor(map[string]interface{}) (interface{}, bool) +} + +// singleKeyStrategy is a cheaper strategy for associative lists that have exactly one key. +type singleKeyStrategy struct { + key string +} + +// CompositeKeyFor directly returns the value of the single key to +// use as a composite key. +func (ks *singleKeyStrategy) CompositeKeyFor(obj map[string]interface{}) (interface{}, bool) { + v, ok := obj[ks.key] + if !ok { + return nil, false + } + + switch v.(type) { + case bool, float64, int64, string: + return v, true + default: + return nil, false // non-scalar + } +} + +// multiKeyStrategy computes a composite key of all key values. +type multiKeyStrategy struct { + sts *spec.Schema +} + +// CompositeKeyFor returns a composite key computed from the values of all +// keys. +func (ks *multiKeyStrategy) CompositeKeyFor(obj map[string]interface{}) (interface{}, bool) { + const keyDelimiter = "\x00" // 0 byte should never appear in the composite key except as delimiter + + var delimited strings.Builder + for _, key := range getXListMapKeys(ks.sts) { + v, ok := obj[key] + if !ok { + return nil, false + } + + switch v.(type) { + case bool: + fmt.Fprintf(&delimited, keyDelimiter+"%t", v) + case float64: + fmt.Fprintf(&delimited, keyDelimiter+"%f", v) + case int64: + fmt.Fprintf(&delimited, keyDelimiter+"%d", v) + case string: + fmt.Fprintf(&delimited, keyDelimiter+"%q", v) + default: + return nil, false // values must be scalars + } + } + return delimited.String(), true +} + +// emptyMapList is a mapList containing no elements. +type emptyMapList struct{} + +func (emptyMapList) get(interface{}) interface{} { + return nil +} + +type mapListImpl struct { + sts *spec.Schema + ks keyStrategy + // keyedItems contains all lazily keyed map items + keyedItems map[interface{}]interface{} + // unkeyedItems contains all map items that have not yet been keyed + unkeyedItems []interface{} +} + +func (a *mapListImpl) get(obj interface{}) interface{} { + mobj, ok := obj.(map[string]interface{}) + if !ok { + return nil + } + + key, ok := a.ks.CompositeKeyFor(mobj) + if !ok { + return nil + } + if match, ok := a.keyedItems[key]; ok { + return match + } + // keep keying items until we either find a match or run out of unkeyed items + for len(a.unkeyedItems) > 0 { + // dequeue an unkeyed item + item := a.unkeyedItems[0] + a.unkeyedItems = a.unkeyedItems[1:] + + // key the item + mitem, ok := item.(map[string]interface{}) + if !ok { + continue + } + itemKey, ok := a.ks.CompositeKeyFor(mitem) + if !ok { + continue + } + if _, exists := a.keyedItems[itemKey]; !exists { + a.keyedItems[itemKey] = mitem + } + + // if it matches, short-circuit + if itemKey == key { + return mitem + } + } + + return nil +} + +func makeKeyStrategy(sts *spec.Schema) keyStrategy { + listMapKeys := getXListMapKeys(sts) + if len(listMapKeys) == 1 { + key := listMapKeys[0] + return &singleKeyStrategy{ + key: key, + } + } + + return &multiKeyStrategy{ + sts: sts, + } +} + +// makeMapList returns a queryable interface over the provided x-kubernetes-list-type=map +// keyedItems. If the provided schema is _not_ an array with x-kubernetes-list-type=map, returns an +// empty mapList. +func makeMapList(sts *spec.Schema, items []interface{}) (rv mapList) { + if !sts.Type.Contains("array") || getXListType(sts) != "map" || len(getXListMapKeys(sts)) == 0 { + return emptyMapList{} + } + ks := makeKeyStrategy(sts) + return &mapListImpl{ + sts: sts, + ks: ks, + keyedItems: map[interface{}]interface{}{}, + unkeyedItems: items, + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist_test.go b/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist_test.go new file mode 100644 index 00000000000..134a0f2cdbb --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist_test.go @@ -0,0 +1,309 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "reflect" + "testing" + + "k8s.io/kube-openapi/pkg/validation/spec" +) + +func TestMapList(t *testing.T) { + for _, tc := range []struct { + name string + sts *spec.Schema + items []interface{} + warmUpQueries []interface{} + query interface{} + expected interface{} + }{ + { + name: "default list type", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + }}, + query: map[string]interface{}{}, + expected: nil, + }, + { + name: "non list type", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"map"}, + }}, + query: map[string]interface{}{}, + expected: nil, + }, + { + name: "non-map list type", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + }, + VendorExtensible: spec.VendorExtensible{Extensions: map[string]interface{}{ + extListType: listTypeSet, + }}}, + query: map[string]interface{}{}, + expected: nil, + }, + { + name: "no keys", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + }}, + query: map[string]interface{}{}, + expected: nil, + }, + { + name: "single key", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + }, + VendorExtensible: spec.VendorExtensible{Extensions: map[string]interface{}{ + extListType: listTypeMap, + extListMapKeys: []any{"k"}, + }}}, + items: []interface{}{ + map[string]interface{}{ + "k": "a", + "v1": "a", + }, + map[string]interface{}{ + "k": "b", + "v1": "b", + }, + }, + query: map[string]interface{}{ + "k": "b", + "v1": "B", + }, + expected: map[string]interface{}{ + "k": "b", + "v1": "b", + }, + }, + { + name: "single key ignoring non-map query", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + }}, + items: []interface{}{ + map[string]interface{}{ + "k": "a", + "v1": "a", + }, + }, + query: 42, + expected: nil, + }, + { + name: "single key ignoring unkeyable query", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + }, + VendorExtensible: spec.VendorExtensible{Extensions: map[string]interface{}{ + extListType: listTypeMap, + extListMapKeys: []any{"k"}, + }}}, + items: []interface{}{ + map[string]interface{}{ + "k": "a", + "v1": "a", + }, + }, + query: map[string]interface{}{ + "k": map[string]interface{}{ + "keys": "must", + "be": "scalars", + }, + "v1": "A", + }, + expected: nil, + }, + { + name: "ignores item of invalid type", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + }, + VendorExtensible: spec.VendorExtensible{Extensions: map[string]interface{}{ + extListType: listTypeMap, + extListMapKeys: []any{"k"}, + }}}, + items: []interface{}{ + map[string]interface{}{ + "k": "a", + "v1": "a", + }, + 5, + }, + query: map[string]interface{}{ + "k": "a", + "v1": "A", + }, + expected: map[string]interface{}{ + "k": "a", + "v1": "a", + }, + }, + { + name: "keep first entry when duplicated keys are encountered", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + }, + VendorExtensible: spec.VendorExtensible{Extensions: map[string]interface{}{ + extListType: listTypeMap, + extListMapKeys: []any{"k"}, + }}}, + items: []interface{}{ + map[string]interface{}{ + "k": "a", + "v1": "a", + }, + map[string]interface{}{ + "k": "a", + "v1": "b", + }, + }, + query: map[string]interface{}{ + "k": "a", + "v1": "A", + }, + expected: map[string]interface{}{ + "k": "a", + "v1": "a", + }, + }, + { + name: "keep first entry when duplicated multi-keys are encountered", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + }, + VendorExtensible: spec.VendorExtensible{Extensions: map[string]interface{}{ + extListType: listTypeMap, + extListMapKeys: []any{"k1", "k2"}, + }}}, + items: []interface{}{ + map[string]interface{}{ + "k1": "a", + "k2": "b", + "v1": "a", + }, + map[string]interface{}{ + "k1": "a", + "k2": "b", + "v1": "b", + }, + map[string]interface{}{ + "k1": "x", + "k2": "y", + "v1": "z", + }, + }, + warmUpQueries: []interface{}{ + map[string]interface{}{ + "k1": "x", + "k2": "y", + }, + }, + query: map[string]interface{}{ + "k1": "a", + "k2": "b", + }, + expected: map[string]interface{}{ + "k1": "a", + "k2": "b", + "v1": "a", + }, + }, + { + name: "multiple keys with defaults ignores item with nil value for key", + sts: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Properties: map[string]spec.Schema{ + "kb": {SchemaProps: spec.SchemaProps{ + Default: true, + }}, + "kf": {SchemaProps: spec.SchemaProps{ + Default: 2.0, + }}, + "ki": {SchemaProps: spec.SchemaProps{ + Default: int64(64), + }}, + "ks": { + SchemaProps: spec.SchemaProps{ + Default: "hello", + }}, + }, + }, + VendorExtensible: spec.VendorExtensible{Extensions: map[string]interface{}{ + extListType: listTypeMap, + extListMapKeys: []any{"kb", "kf", "ki", "ks"}, + }}}, + items: []interface{}{ + map[string]interface{}{ + "kb": nil, + "kf": float64(2.0), + "ki": int64(42), + "ks": "hello", + "v1": "a", + }, + map[string]interface{}{ + "kb": false, + "kf": float64(2.0), + "ki": int64(42), + "ks": "hello", + "v1": "b", + }, + }, + query: map[string]interface{}{ + "kb": false, + "kf": float64(2.0), + "ki": int64(42), + "ks": "hello", + "v1": "B", + }, + expected: map[string]interface{}{ + "kb": false, + "kf": float64(2.0), + "ki": int64(42), + "ks": "hello", + "v1": "b", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + mapList := makeMapList(tc.sts, tc.items) + for _, warmUp := range tc.warmUpQueries { + mapList.get(warmUp) + } + actual := mapList.get(tc.query) + if !reflect.DeepEqual(tc.expected, actual) { + t.Errorf("got: %v, expected %v", actual, tc.expected) + } + }) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/cel/openapi/schemas.go b/staging/src/k8s.io/apiserver/pkg/cel/openapi/schemas.go new file mode 100644 index 00000000000..28a3663af23 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/cel/openapi/schemas.go @@ -0,0 +1,262 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "time" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/types" + + apiservercel "k8s.io/apiserver/pkg/cel" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +const maxRequestSizeBytes = apiservercel.DefaultMaxRequestSizeBytes + +// SchemaDeclType converts the structural schema to a CEL declaration, or returns nil if the +// structural schema should not be exposed in CEL expressions. +// Set isResourceRoot to true for the root of a custom resource or embedded resource. +// +// Schemas with XPreserveUnknownFields not exposed unless they are objects. Array and "maps" schemas +// are not exposed if their items or additionalProperties schemas are not exposed. Object Properties are not exposed +// if their schema is not exposed. +// +// The CEL declaration for objects with XPreserveUnknownFields does not expose unknown fields. +func SchemaDeclType(s *spec.Schema, isResourceRoot bool) *apiservercel.DeclType { + if s == nil { + return nil + } + if isXIntOrString(s) { + // schemas using XIntOrString are not required to have a type. + + // intOrStringType represents the x-kubernetes-int-or-string union type in CEL expressions. + // In CEL, the type is represented as dynamic value, which can be thought of as a union type of all types. + // All type checking for XIntOrString is deferred to runtime, so all access to values of this type must + // be guarded with a type check, e.g.: + // + // To require that the string representation be a percentage: + // `type(intOrStringField) == string && intOrStringField.matches(r'(\d+(\.\d+)?%)')` + // To validate requirements on both the int and string representation: + // `type(intOrStringField) == int ? intOrStringField < 5 : double(intOrStringField.replace('%', '')) < 0.5 + // + dyn := apiservercel.NewSimpleTypeWithMinSize("dyn", cel.DynType, nil, 1) // smallest value for a serialized x-kubernetes-int-or-string is 0 + // handle x-kubernetes-int-or-string by returning the max length/min serialized size of the largest possible string + dyn.MaxElements = maxRequestSizeBytes - 2 + return dyn + } + + // We ignore XPreserveUnknownFields since we don't support validation rules on + // data that we don't have schema information for. + + if isResourceRoot { + // 'apiVersion', 'kind', 'metadata.name' and 'metadata.generateName' are always accessible to validator rules + // at the root of resources, even if not specified in the schema. + // This includes the root of a custom resource and the root of XEmbeddedResource objects. + s = WithTypeAndObjectMeta(s) + } + + // If the schema is not an "int-or-string", type must present. + if len(s.Type) == 0 { + return nil + } + + switch s.Type[0] { + case "array": + if s.Items != nil { + itemsType := SchemaDeclType(s.Items.Schema, isXEmbeddedResource(s.Items.Schema)) + if itemsType == nil { + return nil + } + var maxItems int64 + if s.MaxItems != nil { + maxItems = zeroIfNegative(*s.MaxItems) + } else { + maxItems = estimateMaxArrayItemsFromMinSize(itemsType.MinSerializedSize) + } + return apiservercel.NewListType(itemsType, maxItems) + } + return nil + case "object": + if s.AdditionalProperties != nil && s.AdditionalProperties.Schema != nil { + propsType := SchemaDeclType(s.AdditionalProperties.Schema, isXEmbeddedResource(s.AdditionalProperties.Schema)) + if propsType != nil { + var maxProperties int64 + if s.MaxProperties != nil { + maxProperties = zeroIfNegative(*s.MaxProperties) + } else { + maxProperties = estimateMaxAdditionalPropertiesFromMinSize(propsType.MinSerializedSize) + } + return apiservercel.NewMapType(apiservercel.StringType, propsType, maxProperties) + } + return nil + } + fields := make(map[string]*apiservercel.DeclField, len(s.Properties)) + + required := map[string]bool{} + if s.Required != nil { + for _, f := range s.Required { + required[f] = true + } + } + // an object will always be serialized at least as {}, so account for that + minSerializedSize := int64(2) + for name, prop := range s.Properties { + var enumValues []interface{} + if prop.Enum != nil { + for _, e := range prop.Enum { + enumValues = append(enumValues, e) + } + } + if fieldType := SchemaDeclType(&prop, isXEmbeddedResource(&prop)); fieldType != nil { + if propName, ok := apiservercel.Escape(name); ok { + fields[propName] = apiservercel.NewDeclField(propName, fieldType, required[name], enumValues, prop.Default) + } + // the min serialized size for an object is 2 (for {}) plus the min size of all its required + // properties + // only include required properties without a default value; default values are filled in + // server-side + if required[name] && prop.Default == nil { + minSerializedSize += int64(len(name)) + fieldType.MinSerializedSize + 4 + } + } + } + objType := apiservercel.NewObjectType("object", fields) + objType.MinSerializedSize = minSerializedSize + return objType + case "string": + switch s.Format { + case "byte": + byteWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("bytes", cel.BytesType, types.Bytes([]byte{}), apiservercel.MinStringSize) + if s.MaxLength != nil { + byteWithMaxLength.MaxElements = zeroIfNegative(*s.MaxLength) + } else { + byteWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) + } + return byteWithMaxLength + case "duration": + durationWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("duration", cel.DurationType, types.Duration{Duration: time.Duration(0)}, int64(apiservercel.MinDurationSizeJSON)) + durationWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) + return durationWithMaxLength + case "date": + timestampWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("timestamp", cel.TimestampType, types.Timestamp{Time: time.Time{}}, int64(apiservercel.JSONDateSize)) + timestampWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) + return timestampWithMaxLength + case "date-time": + timestampWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("timestamp", cel.TimestampType, types.Timestamp{Time: time.Time{}}, int64(apiservercel.MinDatetimeSizeJSON)) + timestampWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) + return timestampWithMaxLength + } + + strWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("string", cel.StringType, types.String(""), apiservercel.MinStringSize) + if s.MaxLength != nil { + // multiply the user-provided max length by 4 in the case of an otherwise-untyped string + // we do this because the OpenAPIv3 spec indicates that maxLength is specified in runes/code points, + // but we need to reason about length for things like request size, so we use bytes in this code (and an individual + // unicode code point can be up to 4 bytes long) + strWithMaxLength.MaxElements = zeroIfNegative(*s.MaxLength) * 4 + } else { + strWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) + } + return strWithMaxLength + case "boolean": + return apiservercel.BoolType + case "number": + return apiservercel.DoubleType + case "integer": + return apiservercel.IntType + } + return nil +} + +func zeroIfNegative(v int64) int64 { + if v < 0 { + return 0 + } + return v +} + +// WithTypeAndObjectMeta ensures the kind, apiVersion and +// metadata.name and metadata.generateName properties are specified, making a shallow copy of the provided schema if needed. +func WithTypeAndObjectMeta(s *spec.Schema) *spec.Schema { + if s.Properties != nil && + s.Properties["kind"].Type.Contains("string") && + s.Properties["apiVersion"].Type.Contains("string") && + s.Properties["metadata"].Type.Contains("object") && + s.Properties["metadata"].Properties != nil && + s.Properties["metadata"].Properties["name"].Type.Contains("string") && + s.Properties["metadata"].Properties["generateName"].Type.Contains("string") { + return s + } + result := *s + props := make(map[string]spec.Schema, len(s.Properties)) + for k, prop := range s.Properties { + props[k] = prop + } + stringType := spec.StringProperty() + props["kind"] = *stringType + props["apiVersion"] = *stringType + props["metadata"] = spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": *stringType, + "generateName": *stringType, + }, + }, + } + result.Properties = props + + return &result +} + +// estimateMaxStringLengthPerRequest estimates the maximum string length (in characters) +// of a string compatible with the format requirements in the provided schema. +// must only be called on schemas of type "string" or x-kubernetes-int-or-string: true +func estimateMaxStringLengthPerRequest(s *spec.Schema) int64 { + if isXIntOrString(s) { + return maxRequestSizeBytes - 2 + } + switch s.Format { + case "duration": + return apiservercel.MaxDurationSizeJSON + case "date": + return apiservercel.JSONDateSize + case "date-time": + return apiservercel.MaxDatetimeSizeJSON + default: + // subtract 2 to account for "" + return maxRequestSizeBytes - 2 + } +} + +// estimateMaxArrayItemsPerRequest estimates the maximum number of array items with +// the provided minimum serialized size that can fit into a single request. +func estimateMaxArrayItemsFromMinSize(minSize int64) int64 { + // subtract 2 to account for [ and ] + return (maxRequestSizeBytes - 2) / (minSize + 1) +} + +// estimateMaxAdditionalPropertiesPerRequest estimates the maximum number of additional properties +// with the provided minimum serialized size that can fit into a single request. +func estimateMaxAdditionalPropertiesFromMinSize(minSize int64) int64 { + // 2 bytes for key + "" + colon + comma + smallest possible value, realistically the actual keys + // will all vary in length + keyValuePairSize := minSize + 6 + // subtract 2 to account for { and } + return (maxRequestSizeBytes - 2) / keyValuePairSize +} diff --git a/staging/src/k8s.io/apiserver/pkg/cel/openapi/schemas_test.go b/staging/src/k8s.io/apiserver/pkg/cel/openapi/schemas_test.go new file mode 100644 index 00000000000..c48aea73867 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/cel/openapi/schemas_test.go @@ -0,0 +1,463 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "reflect" + "testing" + + "github.com/google/cel-go/common/types" + + "google.golang.org/protobuf/proto" + + apiservercel "k8s.io/apiserver/pkg/cel" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +func TestSchemaDeclType(t *testing.T) { + ts := testSchema() + cust := SchemaDeclType(ts, false) + if cust.TypeName() != "object" { + t.Errorf("incorrect type name, got %v, wanted object", cust.TypeName()) + } + if len(cust.Fields) != 4 { + t.Errorf("incorrect number of fields, got %d, wanted 4", len(cust.Fields)) + } + for _, f := range cust.Fields { + prop, found := ts.Properties[f.Name] + if !found { + t.Errorf("type field not found in schema, field: %s", f.Name) + } + fdv := f.DefaultValue() + if prop.Default != nil { + pdv := types.DefaultTypeAdapter.NativeToValue(prop.Default) + if !reflect.DeepEqual(fdv, pdv) { + t.Errorf("field and schema do not agree on default value for field: %s, field value: %v, schema default: %v", f.Name, fdv, pdv) + } + } + if (len(prop.Enum) == 0) && len(f.EnumValues()) != 0 { + t.Errorf("field had more enum values than the property. field: %s", f.Name) + } + + fevs := f.EnumValues() + for _, fev := range fevs { + found := false + for _, pev := range prop.Enum { + celpev := types.DefaultTypeAdapter.NativeToValue(pev) + if reflect.DeepEqual(fev, celpev) { + found = true + break + } + } + if !found { + t.Errorf( + "could not find field enum value in property definition. field: %s, enum: %v", + f.Name, fev) + } + } + + } + for _, name := range ts.Required { + df, found := cust.FindField(name) + if !found { + t.Errorf("custom type missing required field. field=%s", name) + } + if !df.Required { + t.Errorf("field marked as required in schema, but optional in type. field=%s", df.Name) + } + } + +} + +func TestSchemaDeclTypes(t *testing.T) { + ts := testSchema() + cust := SchemaDeclType(ts, true).MaybeAssignTypeName("CustomObject") + typeMap := apiservercel.FieldTypeMap("CustomObject", cust) + nested, _ := cust.FindField("nested") + metadata, _ := cust.FindField("metadata") + expectedObjTypeMap := map[string]*apiservercel.DeclType{ + "CustomObject": cust, + "CustomObject.nested": nested.Type, + "CustomObject.metadata": metadata.Type, + } + objTypeMap := map[string]*apiservercel.DeclType{} + for name, t := range typeMap { + if t.IsObject() { + objTypeMap[name] = t + } + } + if len(objTypeMap) != len(expectedObjTypeMap) { + t.Errorf("got different type set. got=%v, wanted=%v", objTypeMap, expectedObjTypeMap) + } + for exp, expType := range expectedObjTypeMap { + actType, found := objTypeMap[exp] + if !found { + t.Errorf("missing type in rule types: %s", exp) + continue + } + expT, err := expType.ExprType() + if err != nil { + t.Errorf("fail to get cel type: %s", err) + } + actT, err := actType.ExprType() + if err != nil { + t.Errorf("fail to get cel type: %s", err) + } + if !proto.Equal(expT, actT) { + t.Errorf("incompatible CEL types. got=%v, wanted=%v", expT, actT) + } + } +} + +func testSchema() *spec.Schema { + // Manual construction of a schema with the following definition: + // + // schema: + // type: object + // metadata: + // custom_type: "CustomObject" + // required: + // - name + // - value + // properties: + // name: + // type: string + // nested: + // type: object + // properties: + // subname: + // type: string + // flags: + // type: object + // additionalProperties: + // type: boolean + // dates: + // type: array + // items: + // type: string + // format: date-time + // metadata: + // type: object + // additionalProperties: + // type: object + // properties: + // key: + // type: string + // values: + // type: array + // items: string + // value: + // type: integer + // format: int64 + // default: 1 + // enum: [1,2,3] + ts := &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": *spec.StringProperty(), + "value": {SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Default: int64(1), + Format: "int64", + Enum: []any{1, 2, 3}, + }}, + "nested": {SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "subname": *spec.StringProperty(), + "flags": {SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Schema: spec.BooleanProperty(), + }, + }}, + "dates": {SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "date-time", + }}}}}, + }, + }, + }, + "metadata": {SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": *spec.StringProperty(), + "value": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + }}}, + }, + }, + }, + }}, + }}} + return ts +} + +func arraySchema(arrayType, format string, maxItems *int64) *spec.Schema { + return &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{arrayType}, + Format: format, + }}}, + MaxItems: maxItems, + }, + } +} + +func maxPtr(max int64) *int64 { + return &max +} + +func TestEstimateMaxLengthJSON(t *testing.T) { + type maxLengthTest struct { + Name string + InputSchema *spec.Schema + ExpectedMaxElements int64 + } + tests := []maxLengthTest{ + { + Name: "booleanArray", + InputSchema: arraySchema("boolean", "", nil), + // expected JSON is [true,true,...], so our length should be (maxRequestSizeBytes - 2) / 5 + ExpectedMaxElements: 629145, + }, + { + Name: "durationArray", + InputSchema: arraySchema("string", "duration", nil), + // expected JSON is ["0","0",...] so our length should be (maxRequestSizeBytes - 2) / 4 + ExpectedMaxElements: 786431, + }, + { + Name: "datetimeArray", + InputSchema: arraySchema("string", "date-time", nil), + // expected JSON is ["2000-01-01T01:01:01","2000-01-01T01:01:01",...] so our length should be (maxRequestSizeBytes - 2) / 22 + ExpectedMaxElements: 142987, + }, + { + Name: "dateArray", + InputSchema: arraySchema("string", "date", nil), + // expected JSON is ["2000-01-01","2000-01-02",...] so our length should be (maxRequestSizeBytes - 2) / 13 + ExpectedMaxElements: 241978, + }, + { + Name: "numberArray", + InputSchema: arraySchema("integer", "", nil), + // expected JSON is [0,0,...] so our length should be (maxRequestSizeBytes - 2) / 2 + ExpectedMaxElements: 1572863, + }, + { + Name: "stringArray", + InputSchema: arraySchema("string", "", nil), + // expected JSON is ["","",...] so our length should be (maxRequestSizeBytes - 2) / 3 + ExpectedMaxElements: 1048575, + }, + { + Name: "stringMap", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + }}, + }, + }}, + // expected JSON is {"":"","":"",...} so our length should be (3000000 - 2) / 6 + ExpectedMaxElements: 393215, + }, + { + Name: "objectOptionalPropertyArray", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "required": *spec.StringProperty(), + "optional": *spec.StringProperty(), + }, + Required: []string{"required"}, + }}}, + }}, + // expected JSON is [{"required":"",},{"required":"",},...] so our length should be (maxRequestSizeBytes - 2) / 17 + ExpectedMaxElements: 185042, + }, + { + Name: "arrayWithLength", + InputSchema: arraySchema("integer", "int64", maxPtr(10)), + // manually set by MaxItems + ExpectedMaxElements: 10, + }, + { + Name: "stringWithLength", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + MaxLength: maxPtr(20), + }}, + // manually set by MaxLength, but we expect a 4x multiplier compared to the original input + // since OpenAPIv3 maxLength uses code points, but DeclType works with bytes + ExpectedMaxElements: 80, + }, + { + Name: "mapWithLength", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Schema: spec.StringProperty(), + }, + Format: "string", + MaxProperties: maxPtr(15), + }}, + // manually set by MaxProperties + ExpectedMaxElements: 15, + }, + { + Name: "durationMaxSize", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "duration", + }}, + // should be exactly equal to maxDurationSizeJSON + ExpectedMaxElements: apiservercel.MaxDurationSizeJSON, + }, + { + Name: "dateSize", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "date", + }}, + // should be exactly equal to dateSizeJSON + ExpectedMaxElements: apiservercel.JSONDateSize, + }, + { + Name: "maxdatetimeSize", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "date-time", + }}, + // should be exactly equal to maxDatetimeSizeJSON + ExpectedMaxElements: apiservercel.MaxDatetimeSizeJSON, + }, + { + Name: "maxintOrStringSize", + InputSchema: &spec.Schema{ + VendorExtensible: spec.VendorExtensible{Extensions: map[string]interface{}{ + extIntOrString: true, + }}}, + // should be exactly equal to maxRequestSizeBytes - 2 (to allow for quotes in the case of a string) + ExpectedMaxElements: apiservercel.DefaultMaxRequestSizeBytes - 2, + }, + { + Name: "objectDefaultFieldArray", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "field": {SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Default: "default", + }, + }}, + Required: []string{"field"}, + }}}, + }, + }, + // expected JSON is [{},{},...] so our length should be (maxRequestSizeBytes - 2) / 3 + ExpectedMaxElements: 1048575, + }, + { + Name: "byteStringSize", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "byte", + }}, + // expected JSON is "" so our length should be (maxRequestSizeBytes - 2) + ExpectedMaxElements: 3145726, + }, + { + Name: "byteStringSetMaxLength", + InputSchema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "byte", + MaxLength: maxPtr(20), + }}, + // note that unlike regular strings we don't have to take unicode into account, + // so we expect the max length to be exactly equal to the user-supplied one + ExpectedMaxElements: 20, + }, + } + for _, testCase := range tests { + t.Run(testCase.Name, func(t *testing.T) { + decl := SchemaDeclType(testCase.InputSchema, false) + if decl.MaxElements != testCase.ExpectedMaxElements { + t.Errorf("wrong maxElements (got %d, expected %d)", decl.MaxElements, testCase.ExpectedMaxElements) + } + }) + } +} + +func genNestedSchema(depth int) *spec.Schema { + var generator func(d int) spec.Schema + generator = func(d int) spec.Schema { + nodeTemplate := &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{}, + }} + if d == 1 { + return *nodeTemplate + } else { + mapType := generator(d - 1) + nodeTemplate.AdditionalProperties.Schema = &mapType + return *nodeTemplate + } + } + schema := generator(depth) + return &schema +} + +func BenchmarkDeeplyNestedSchemaDeclType(b *testing.B) { + benchmarkSchema := genNestedSchema(10) + b.ResetTimer() + for i := 0; i < b.N; i++ { + SchemaDeclType(benchmarkSchema, false) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/cel/openapi/values.go b/staging/src/k8s.io/apiserver/pkg/cel/openapi/values.go new file mode 100644 index 00000000000..b435f98c64e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/cel/openapi/values.go @@ -0,0 +1,702 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "fmt" + "reflect" + "sync" + "time" + + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/common/types/traits" + + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apiserver/pkg/cel" + "k8s.io/kube-openapi/pkg/validation/spec" + "k8s.io/kube-openapi/pkg/validation/strfmt" +) + +// UnstructuredToVal converts a Kubernetes unstructured data element to a CEL Val. +// The root schema of custom resource schema is expected contain type meta and object meta schemas. +// If Embedded resources do not contain type meta and object meta schemas, they will be added automatically. +func UnstructuredToVal(unstructured interface{}, schema *spec.Schema) ref.Val { + if unstructured == nil { + if schema.Nullable { + return types.NullValue + } + return types.NewErr("invalid data, got null for schema with nullable=false") + } + if isXIntOrString(schema) { + switch v := unstructured.(type) { + case string: + return types.String(v) + case int: + return types.Int(v) + case int32: + return types.Int(v) + case int64: + return types.Int(v) + } + return types.NewErr("invalid data, expected XIntOrString value to be either a string or integer") + } + if schema.Type.Contains("object") { + m, ok := unstructured.(map[string]interface{}) + if !ok { + return types.NewErr("invalid data, expected a map for the provided schema with type=object") + } + if isXEmbeddedResource(schema) || schema.Properties != nil { + if isXEmbeddedResource(schema) { + schema = WithTypeAndObjectMeta(schema) + } + return &unstructuredMap{ + value: m, + schema: schema, + propSchema: func(key string) (*spec.Schema, bool) { + if schema, ok := schema.Properties[key]; ok { + return &schema, true + } + return nil, false + }, + } + } + if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil { + return &unstructuredMap{ + value: m, + schema: schema, + propSchema: func(key string) (*spec.Schema, bool) { + return schema.AdditionalProperties.Schema, true + }, + } + } + // A object with x-kubernetes-preserve-unknown-fields but no properties or additionalProperties is treated + // as an empty object. + if isXPreserveUnknownFields(schema) { + return &unstructuredMap{ + value: m, + schema: schema, + propSchema: func(key string) (*spec.Schema, bool) { + return nil, false + }, + } + } + return types.NewErr("invalid object type, expected either Properties or AdditionalProperties with Allows=true and non-empty Schema") + } + + if schema.Type.Contains("array") { + l, ok := unstructured.([]interface{}) + if !ok { + return types.NewErr("invalid data, expected an array for the provided schema with type=array") + } + if schema.Items == nil { + return types.NewErr("invalid array type, expected Items with a non-empty Schema") + } + typedList := unstructuredList{elements: l, itemsSchema: schema.Items.Schema} + listType := getXListType(schema) + if listType != "" { + switch listType { + case "map": + mapKeys := getXListMapKeys(schema) + return &unstructuredMapList{unstructuredList: typedList, escapedKeyProps: escapeKeyProps(mapKeys)} + case "set": + return &unstructuredSetList{unstructuredList: typedList} + case "atomic": + return &typedList + default: + return types.NewErr("invalid x-kubernetes-list-type, expected 'map', 'set' or 'atomic' but got %s", listType) + } + } + return &typedList + } + + if schema.Type.Contains("string") { + str, ok := unstructured.(string) + if !ok { + return types.NewErr("invalid data, expected string, got %T", unstructured) + } + switch schema.Format { + case "duration": + d, err := strfmt.ParseDuration(str) + if err != nil { + return types.NewErr("Invalid duration %s: %v", str, err) + } + return types.Duration{Duration: d} + case "date": + d, err := time.Parse(strfmt.RFC3339FullDate, str) // strfmt uses this format for OpenAPIv3 value validation + if err != nil { + return types.NewErr("Invalid date formatted string %s: %v", str, err) + } + return types.Timestamp{Time: d} + case "date-time": + d, err := strfmt.ParseDateTime(str) + if err != nil { + return types.NewErr("Invalid date-time formatted string %s: %v", str, err) + } + return types.Timestamp{Time: time.Time(d)} + case "byte": + base64 := strfmt.Base64{} + err := base64.UnmarshalText([]byte(str)) + if err != nil { + return types.NewErr("Invalid byte formatted string %s: %v", str, err) + } + return types.Bytes(base64) + } + + return types.String(str) + } + if schema.Type.Contains("number") { + switch v := unstructured.(type) { + // float representations of whole numbers (e.g. 1.0, 0.0) can convert to int representations (e.g. 1, 0) in yaml + // to json translation, and then get parsed as int64s + case int: + return types.Double(v) + case int32: + return types.Double(v) + case int64: + return types.Double(v) + + case float32: + return types.Double(v) + case float64: + return types.Double(v) + default: + return types.NewErr("invalid data, expected float, got %T", unstructured) + } + } + if schema.Type.Contains("integer") { + switch v := unstructured.(type) { + case int: + return types.Int(v) + case int32: + return types.Int(v) + case int64: + return types.Int(v) + default: + return types.NewErr("invalid data, expected int, got %T", unstructured) + } + } + if schema.Type.Contains("boolean") { + b, ok := unstructured.(bool) + if !ok { + return types.NewErr("invalid data, expected bool, got %T", unstructured) + } + return types.Bool(b) + } + + if isXPreserveUnknownFields(schema) { + return &unknownPreserved{u: unstructured} + } + + return types.NewErr("invalid type, expected object, array, number, integer, boolean or string, or no type with x-kubernetes-int-or-string or x-kubernetes-preserve-unknown-fields is true, got %s", schema.Type) +} + +// unknownPreserved represents unknown data preserved in custom resources via x-kubernetes-preserve-unknown-fields. +// It preserves the data at runtime without assuming it is of any particular type and supports only equality checking. +// unknownPreserved should be used only for values are not directly accessible in CEL expressions, i.e. for data +// where there is no corresponding CEL type declaration. +type unknownPreserved struct { + u interface{} +} + +func (t *unknownPreserved) ConvertToNative(refType reflect.Type) (interface{}, error) { + return nil, fmt.Errorf("type conversion to '%s' not supported for values preserved by x-kubernetes-preserve-unknown-fields", refType) +} + +func (t *unknownPreserved) ConvertToType(typeValue ref.Type) ref.Val { + return types.NewErr("type conversion to '%s' not supported for values preserved by x-kubernetes-preserve-unknown-fields", typeValue.TypeName()) +} + +func (t *unknownPreserved) Equal(other ref.Val) ref.Val { + return types.Bool(equality.Semantic.DeepEqual(t.u, other.Value())) +} + +func (t *unknownPreserved) Type() ref.Type { + return types.UnknownType +} + +func (t *unknownPreserved) Value() interface{} { + return t.u // used by Equal checks +} + +// unstructuredMapList represents an unstructured data instance of an OpenAPI array with x-kubernetes-list-type=map. +type unstructuredMapList struct { + unstructuredList + escapedKeyProps []string + + sync.Once // for for lazy load of mapOfList since it is only needed if Equals is called + mapOfList map[interface{}]interface{} +} + +func (t *unstructuredMapList) getMap() map[interface{}]interface{} { + t.Do(func() { + t.mapOfList = make(map[interface{}]interface{}, len(t.elements)) + for _, e := range t.elements { + t.mapOfList[t.toMapKey(e)] = e + } + }) + return t.mapOfList +} + +// toMapKey returns a valid golang map key for the given element of the map list. +// element must be a valid map list entry where all map key props are scalar types (which are comparable in go +// and valid for use in a golang map key). +func (t *unstructuredMapList) toMapKey(element interface{}) interface{} { + eObj, ok := element.(map[string]interface{}) + if !ok { + return types.NewErr("unexpected data format for element of array with x-kubernetes-list-type=map: %T", element) + } + // Arrays are comparable in go and may be used as map keys, but maps and slices are not. + // So we can special case small numbers of key props as arrays and fall back to serialization + // for larger numbers of key props + if len(t.escapedKeyProps) == 1 { + return eObj[t.escapedKeyProps[0]] + } + if len(t.escapedKeyProps) == 2 { + return [2]interface{}{eObj[t.escapedKeyProps[0]], eObj[t.escapedKeyProps[1]]} + } + if len(t.escapedKeyProps) == 3 { + return [3]interface{}{eObj[t.escapedKeyProps[0]], eObj[t.escapedKeyProps[1]], eObj[t.escapedKeyProps[2]]} + } + + key := make([]interface{}, len(t.escapedKeyProps)) + for i, kf := range t.escapedKeyProps { + key[i] = eObj[kf] + } + return fmt.Sprintf("%v", key) +} + +// Equal on a map list ignores list element order. +func (t *unstructuredMapList) Equal(other ref.Val) ref.Val { + oMapList, ok := other.(traits.Lister) + if !ok { + return types.MaybeNoSuchOverloadErr(other) + } + sz := types.Int(len(t.elements)) + if sz != oMapList.Size() { + return types.False + } + tMap := t.getMap() + for it := oMapList.Iterator(); it.HasNext() == types.True; { + v := it.Next() + k := t.toMapKey(v.Value()) + tVal, ok := tMap[k] + if !ok { + return types.False + } + eq := UnstructuredToVal(tVal, t.itemsSchema).Equal(v) + if eq != types.True { + return eq // either false or error + } + } + return types.True +} + +// Add for a map list `X + Y` performs a merge where the array positions of all keys in `X` are preserved but the values +// are overwritten by values in `Y` when the key sets of `X` and `Y` intersect. Elements in `Y` with +// non-intersecting keys are appended, retaining their partial order. +func (t *unstructuredMapList) Add(other ref.Val) ref.Val { + oMapList, ok := other.(traits.Lister) + if !ok { + return types.MaybeNoSuchOverloadErr(other) + } + elements := make([]interface{}, len(t.elements)) + keyToIdx := map[interface{}]int{} + for i, e := range t.elements { + k := t.toMapKey(e) + keyToIdx[k] = i + elements[i] = e + } + for it := oMapList.Iterator(); it.HasNext() == types.True; { + v := it.Next().Value() + k := t.toMapKey(v) + if overwritePosition, ok := keyToIdx[k]; ok { + elements[overwritePosition] = v + } else { + elements = append(elements, v) + } + } + return &unstructuredMapList{ + unstructuredList: unstructuredList{elements: elements, itemsSchema: t.itemsSchema}, + escapedKeyProps: t.escapedKeyProps, + } +} + +// escapeKeyProps returns identifiers with Escape applied to each. +// Identifiers that cannot be escaped are left as-is. They are inaccessible to CEL programs but are +// are still needed internally to perform equality checks. +func escapeKeyProps(idents []string) []string { + result := make([]string, len(idents)) + for i, prop := range idents { + if escaped, ok := cel.Escape(prop); ok { + result[i] = escaped + } else { + result[i] = prop + } + } + return result +} + +// unstructuredSetList represents an unstructured data instance of an OpenAPI array with x-kubernetes-list-type=set. +type unstructuredSetList struct { + unstructuredList + escapedKeyProps []string + + sync.Once // for for lazy load of setOfList since it is only needed if Equals is called + set map[interface{}]struct{} +} + +func (t *unstructuredSetList) getSet() map[interface{}]struct{} { + // sets are only allowed to contain scalar elements, which are comparable in go, and can safely be used as + // golang map keys + t.Do(func() { + t.set = make(map[interface{}]struct{}, len(t.elements)) + for _, e := range t.elements { + t.set[e] = struct{}{} + } + }) + return t.set +} + +// Equal on a map list ignores list element order. +func (t *unstructuredSetList) Equal(other ref.Val) ref.Val { + oSetList, ok := other.(traits.Lister) + if !ok { + return types.MaybeNoSuchOverloadErr(other) + } + sz := types.Int(len(t.elements)) + if sz != oSetList.Size() { + return types.False + } + tSet := t.getSet() + for it := oSetList.Iterator(); it.HasNext() == types.True; { + next := it.Next().Value() + _, ok := tSet[next] + if !ok { + return types.False + } + } + return types.True +} + +// Add for a set list `X + Y` performs a union where the array positions of all elements in `X` are preserved and +// non-intersecting elements in `Y` are appended, retaining their partial order. +func (t *unstructuredSetList) Add(other ref.Val) ref.Val { + oSetList, ok := other.(traits.Lister) + if !ok { + return types.MaybeNoSuchOverloadErr(other) + } + elements := t.elements + set := t.getSet() + for it := oSetList.Iterator(); it.HasNext() == types.True; { + next := it.Next().Value() + if _, ok := set[next]; !ok { + set[next] = struct{}{} + elements = append(elements, next) + } + } + return &unstructuredSetList{ + unstructuredList: unstructuredList{elements: elements, itemsSchema: t.itemsSchema}, + escapedKeyProps: t.escapedKeyProps, + } +} + +// unstructuredList represents an unstructured data instance of an OpenAPI array with x-kubernetes-list-type=atomic (the default). +type unstructuredList struct { + elements []interface{} + itemsSchema *spec.Schema +} + +var _ = traits.Lister(&unstructuredList{}) + +func (t *unstructuredList) ConvertToNative(typeDesc reflect.Type) (interface{}, error) { + switch typeDesc.Kind() { + case reflect.Slice: + return t.elements, nil + } + return nil, fmt.Errorf("type conversion error from '%s' to '%s'", t.Type(), typeDesc) +} + +func (t *unstructuredList) ConvertToType(typeValue ref.Type) ref.Val { + switch typeValue { + case types.ListType: + return t + case types.TypeType: + return types.ListType + } + return types.NewErr("type conversion error from '%s' to '%s'", t.Type(), typeValue.TypeName()) +} + +func (t *unstructuredList) Equal(other ref.Val) ref.Val { + oList, ok := other.(traits.Lister) + if !ok { + return types.MaybeNoSuchOverloadErr(other) + } + sz := types.Int(len(t.elements)) + if sz != oList.Size() { + return types.False + } + for i := types.Int(0); i < sz; i++ { + eq := t.Get(i).Equal(oList.Get(i)) + if eq != types.True { + return eq // either false or error + } + } + return types.True +} + +func (t *unstructuredList) Type() ref.Type { + return types.ListType +} + +func (t *unstructuredList) Value() interface{} { + return t.elements +} + +func (t *unstructuredList) Add(other ref.Val) ref.Val { + oList, ok := other.(traits.Lister) + if !ok { + return types.MaybeNoSuchOverloadErr(other) + } + elements := t.elements + for it := oList.Iterator(); it.HasNext() == types.True; { + next := it.Next().Value() + elements = append(elements, next) + } + + return &unstructuredList{elements: elements, itemsSchema: t.itemsSchema} +} + +func (t *unstructuredList) Contains(val ref.Val) ref.Val { + if types.IsUnknownOrError(val) { + return val + } + var err ref.Val + sz := len(t.elements) + for i := 0; i < sz; i++ { + elem := UnstructuredToVal(t.elements[i], t.itemsSchema) + cmp := elem.Equal(val) + b, ok := cmp.(types.Bool) + if !ok && err == nil { + err = types.MaybeNoSuchOverloadErr(cmp) + } + if b == types.True { + return types.True + } + } + if err != nil { + return err + } + return types.False +} + +func (t *unstructuredList) Get(idx ref.Val) ref.Val { + iv, isInt := idx.(types.Int) + if !isInt { + return types.ValOrErr(idx, "unsupported index: %v", idx) + } + i := int(iv) + if i < 0 || i >= len(t.elements) { + return types.NewErr("index out of bounds: %v", idx) + } + return UnstructuredToVal(t.elements[i], t.itemsSchema) +} + +func (t *unstructuredList) Iterator() traits.Iterator { + items := make([]ref.Val, len(t.elements)) + for i, item := range t.elements { + itemCopy := item + items[i] = UnstructuredToVal(itemCopy, t.itemsSchema) + } + return &listIterator{unstructuredList: t, items: items} +} + +type listIterator struct { + *unstructuredList + items []ref.Val + idx int +} + +func (it *listIterator) HasNext() ref.Val { + return types.Bool(it.idx < len(it.items)) +} + +func (it *listIterator) Next() ref.Val { + item := it.items[it.idx] + it.idx++ + return item +} + +func (t *unstructuredList) Size() ref.Val { + return types.Int(len(t.elements)) +} + +// unstructuredMap represented an unstructured data instance of an OpenAPI object. +type unstructuredMap struct { + value map[string]interface{} + schema *spec.Schema + // propSchema finds the schema to use for a particular map key. + propSchema func(key string) (*spec.Schema, bool) +} + +var _ = traits.Mapper(&unstructuredMap{}) + +func (t *unstructuredMap) ConvertToNative(typeDesc reflect.Type) (interface{}, error) { + switch typeDesc.Kind() { + case reflect.Map: + return t.value, nil + } + return nil, fmt.Errorf("type conversion error from '%s' to '%s'", t.Type(), typeDesc) +} + +func (t *unstructuredMap) ConvertToType(typeValue ref.Type) ref.Val { + switch typeValue { + case types.MapType: + return t + case types.TypeType: + return types.MapType + } + return types.NewErr("type conversion error from '%s' to '%s'", t.Type(), typeValue.TypeName()) +} + +func (t *unstructuredMap) Equal(other ref.Val) ref.Val { + oMap, isMap := other.(traits.Mapper) + if !isMap { + return types.MaybeNoSuchOverloadErr(other) + } + if t.Size() != oMap.Size() { + return types.False + } + for key, value := range t.value { + if propSchema, ok := t.propSchema(key); ok { + ov, found := oMap.Find(types.String(key)) + if !found { + return types.False + } + v := UnstructuredToVal(value, propSchema) + vEq := v.Equal(ov) + if vEq != types.True { + return vEq // either false or error + } + } else { + // Must be an object with properties. + // Since we've encountered an unknown field, fallback to unstructured equality checking. + ouMap, ok := other.(*unstructuredMap) + if !ok { + // The compiler ensures equality is against the same type of object, so this should be unreachable + return types.MaybeNoSuchOverloadErr(other) + } + if oValue, ok := ouMap.value[key]; ok { + if !equality.Semantic.DeepEqual(value, oValue) { + return types.False + } + } + } + } + return types.True +} + +func (t *unstructuredMap) Type() ref.Type { + return types.MapType +} + +func (t *unstructuredMap) Value() interface{} { + return t.value +} + +func (t *unstructuredMap) Contains(key ref.Val) ref.Val { + v, found := t.Find(key) + if v != nil && types.IsUnknownOrError(v) { + return v + } + + return types.Bool(found) +} + +func (t *unstructuredMap) Get(key ref.Val) ref.Val { + v, found := t.Find(key) + if found { + return v + } + return types.ValOrErr(key, "no such key: %v", key) +} + +func (t *unstructuredMap) Iterator() traits.Iterator { + isObject := t.schema.Properties != nil + keys := make([]ref.Val, len(t.value)) + i := 0 + for k := range t.value { + if _, ok := t.propSchema(k); ok { + mapKey := k + if isObject { + if escaped, ok := cel.Escape(k); ok { + mapKey = escaped + } + } + keys[i] = types.String(mapKey) + i++ + } + } + return &mapIterator{unstructuredMap: t, keys: keys} +} + +type mapIterator struct { + *unstructuredMap + keys []ref.Val + idx int +} + +func (it *mapIterator) HasNext() ref.Val { + return types.Bool(it.idx < len(it.keys)) +} + +func (it *mapIterator) Next() ref.Val { + key := it.keys[it.idx] + it.idx++ + return key +} + +func (t *unstructuredMap) Size() ref.Val { + return types.Int(len(t.value)) +} + +func (t *unstructuredMap) Find(key ref.Val) (ref.Val, bool) { + isObject := t.schema.Properties != nil + keyStr, ok := key.(types.String) + if !ok { + return types.MaybeNoSuchOverloadErr(key), true + } + k := keyStr.Value().(string) + if isObject { + k, ok = cel.Unescape(k) + if !ok { + return nil, false + } + } + if v, ok := t.value[k]; ok { + // If this is an object with properties, not an object with additionalProperties, + // then null valued nullable fields are treated the same as absent optional fields. + if isObject && v == nil { + return nil, false + } + if propSchema, ok := t.propSchema(k); ok { + return UnstructuredToVal(v, propSchema), true + } + } + + return nil, false +} diff --git a/staging/src/k8s.io/apiserver/pkg/cel/openapi/values_test.go b/staging/src/k8s.io/apiserver/pkg/cel/openapi/values_test.go new file mode 100644 index 00000000000..0607327f8bb --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/cel/openapi/values_test.go @@ -0,0 +1,660 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "reflect" + "testing" + + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/common/types/traits" + + "k8s.io/kube-openapi/pkg/validation/spec" +) + +var ( + listTypeSet = "set" + listTypeMap = "map" + stringSchema = spec.StringProperty() + intSchema = spec.Int64Property() + + mapListElementSchema = &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "key": *stringSchema, + "val": *intSchema, + }, + }} + mapListSchema = &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{Schema: mapListElementSchema}, + }, + VendorExtensible: spec.VendorExtensible{ + Extensions: map[string]interface{}{ + extListType: listTypeMap, + extListMapKeys: []any{"key"}, + }}, + } + multiKeyMapListSchema = &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "key1": *stringSchema, + "key2": *stringSchema, + "val": *intSchema, + }, + }}}}, + VendorExtensible: spec.VendorExtensible{ + Extensions: map[string]interface{}{ + extListType: listTypeMap, + extListMapKeys: []any{"key1", "key2"}, + }}, + } + setListSchema = &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{Schema: stringSchema}}, + VendorExtensible: spec.VendorExtensible{ + Extensions: map[string]interface{}{ + extListType: listTypeSet, + }}, + } + atomicListSchema = &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{Schema: stringSchema}, + }} + objectSchema = &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "field1": *stringSchema, + "field2": *stringSchema, + }, + }} + mapSchema = &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{Schema: stringSchema}, + }} +) + +func TestEquality(t *testing.T) { + cases := []struct { + name string + lhs ref.Val + rhs ref.Val + equal bool + }{ + { + name: "map lists are equal regardless of order", + lhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key": "a", + "val": 1, + }, + map[string]interface{}{ + "key": "b", + "val": 2, + }, + }, mapListSchema), + rhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key": "b", + "val": 2, + }, + map[string]interface{}{ + "key": "a", + "val": 1, + }, + }, mapListSchema), + equal: true, + }, + { + name: "map lists are not equal if contents differs", + lhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key": "a", + "val": 1, + }, + map[string]interface{}{ + "key": "b", + "val": 2, + }, + }, mapListSchema), + rhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key": "a", + "val": 1, + }, + map[string]interface{}{ + "key": "b", + "val": 3, + }, + }, mapListSchema), + equal: false, + }, + { + name: "map lists are not equal if length differs", + lhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key": "a", + "val": 1, + }, + map[string]interface{}{ + "key": "b", + "val": 2, + }, + }, mapListSchema), + rhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key": "a", + "val": 1, + }, + map[string]interface{}{ + "key": "b", + "val": 2, + }, + map[string]interface{}{ + "key": "c", + "val": 3, + }, + }, mapListSchema), + equal: false, + }, + { + name: "multi-key map lists are equal regardless of order", + lhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key1": "a1", + "key2": "a2", + "val": 1, + }, + map[string]interface{}{ + "key1": "b1", + "key2": "b2", + "val": 2, + }, + }, multiKeyMapListSchema), + rhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key1": "b1", + "key2": "b2", + "val": 2, + }, + map[string]interface{}{ + "key1": "a1", + "key2": "a2", + "val": 1, + }, + }, multiKeyMapListSchema), + equal: true, + }, + { + name: "multi-key map lists with different contents are not equal", + lhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key1": "a1", + "key2": "a2", + "val": 1, + }, + map[string]interface{}{ + "key1": "b1", + "key2": "b2", + "val": 2, + }, + }, multiKeyMapListSchema), + rhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key1": "a1", + "key2": "a2", + "val": 1, + }, + map[string]interface{}{ + "key1": "b1", + "key2": "b2", + "val": 3, + }, + }, multiKeyMapListSchema), + equal: false, + }, + { + name: "multi-key map lists with different keys are not equal", + lhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key1": "a1", + "key2": "a2", + "val": 1, + }, + map[string]interface{}{ + "key1": "b1", + "key2": "b2", + "val": 2, + }, + }, multiKeyMapListSchema), + rhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key1": "a1", + "key2": "a2", + "val": 1, + }, + map[string]interface{}{ + "key1": "c1", + "key2": "c2", + "val": 3, + }, + }, multiKeyMapListSchema), + equal: false, + }, + { + name: "multi-key map lists with different lengths are not equal", + lhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key1": "a1", + "key2": "a2", + "val": 1, + }, + }, multiKeyMapListSchema), + rhs: UnstructuredToVal([]interface{}{ + map[string]interface{}{ + "key1": "a1", + "key2": "a2", + "val": 1, + }, + map[string]interface{}{ + "key1": "b1", + "key2": "b2", + "val": 3, + }, + }, multiKeyMapListSchema), + equal: false, + }, + { + name: "set lists are equal regardless of order", + lhs: UnstructuredToVal([]interface{}{"a", "b"}, setListSchema), + rhs: UnstructuredToVal([]interface{}{"b", "a"}, setListSchema), + equal: true, + }, + { + name: "set lists are not equal if contents differ", + lhs: UnstructuredToVal([]interface{}{"a", "b"}, setListSchema), + rhs: UnstructuredToVal([]interface{}{"a", "c"}, setListSchema), + equal: false, + }, + { + name: "set lists are not equal if lengths differ", + lhs: UnstructuredToVal([]interface{}{"a", "b"}, setListSchema), + rhs: UnstructuredToVal([]interface{}{"a", "b", "c"}, setListSchema), + equal: false, + }, + { + name: "identical atomic lists are equal", + lhs: UnstructuredToVal([]interface{}{"a", "b"}, atomicListSchema), + rhs: UnstructuredToVal([]interface{}{"a", "b"}, atomicListSchema), + equal: true, + }, + { + name: "atomic lists are not equal if order differs", + lhs: UnstructuredToVal([]interface{}{"a", "b"}, atomicListSchema), + rhs: UnstructuredToVal([]interface{}{"b", "a"}, atomicListSchema), + equal: false, + }, + { + name: "atomic lists are not equal if contents differ", + lhs: UnstructuredToVal([]interface{}{"a", "b"}, atomicListSchema), + rhs: UnstructuredToVal([]interface{}{"a", "c"}, atomicListSchema), + equal: false, + }, + { + name: "atomic lists are not equal if lengths differ", + lhs: UnstructuredToVal([]interface{}{"a", "b"}, atomicListSchema), + rhs: UnstructuredToVal([]interface{}{"a", "b", "c"}, atomicListSchema), + equal: false, + }, + { + name: "identical objects are equal", + lhs: UnstructuredToVal(map[string]interface{}{"field1": "a", "field2": "b"}, objectSchema), + rhs: UnstructuredToVal(map[string]interface{}{"field1": "a", "field2": "b"}, objectSchema), + equal: true, + }, + { + name: "objects are equal regardless of field order", + lhs: UnstructuredToVal(map[string]interface{}{"field1": "a", "field2": "b"}, objectSchema), + rhs: UnstructuredToVal(map[string]interface{}{"field2": "b", "field1": "a"}, objectSchema), + equal: true, + }, + { + name: "objects are not equal if contents differs", + lhs: UnstructuredToVal(map[string]interface{}{"field1": "a", "field2": "b"}, objectSchema), + rhs: UnstructuredToVal(map[string]interface{}{"field1": "a", "field2": "c"}, objectSchema), + equal: false, + }, + { + name: "objects are not equal if length differs", + lhs: UnstructuredToVal(map[string]interface{}{"field1": "a", "field2": "b"}, objectSchema), + rhs: UnstructuredToVal(map[string]interface{}{"field1": "a"}, objectSchema), + equal: false, + }, + { + name: "identical maps are equal", + lhs: UnstructuredToVal(map[string]interface{}{"key1": "a", "key2": "b"}, mapSchema), + rhs: UnstructuredToVal(map[string]interface{}{"key1": "a", "key2": "b"}, mapSchema), + equal: true, + }, + { + name: "maps are equal regardless of field order", + lhs: UnstructuredToVal(map[string]interface{}{"key1": "a", "key2": "b"}, mapSchema), + rhs: UnstructuredToVal(map[string]interface{}{"key2": "b", "key1": "a"}, mapSchema), + equal: true, + }, + { + name: "maps are not equal if contents differs", + lhs: UnstructuredToVal(map[string]interface{}{"key1": "a", "key2": "b"}, mapSchema), + rhs: UnstructuredToVal(map[string]interface{}{"key1": "a", "key2": "c"}, mapSchema), + equal: false, + }, + { + name: "maps are not equal if length differs", + lhs: UnstructuredToVal(map[string]interface{}{"key1": "a", "key2": "b"}, mapSchema), + rhs: UnstructuredToVal(map[string]interface{}{"key1": "a", "key2": "b", "key3": "c"}, mapSchema), + equal: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Compare types with schema against themselves + if tc.lhs.Equal(tc.rhs) != types.Bool(tc.equal) { + t.Errorf("expected Equals to return %v", tc.equal) + } + if tc.rhs.Equal(tc.lhs) != types.Bool(tc.equal) { + t.Errorf("expected Equals to return %v", tc.equal) + } + + // Compare types with schema against native types. This is slightly different than how + // CEL performs equality against data literals, but is a good sanity check. + if tc.lhs.Equal(types.DefaultTypeAdapter.NativeToValue(tc.rhs.Value())) != types.Bool(tc.equal) { + t.Errorf("expected unstructuredVal.Equals() to return %v", tc.equal) + } + if tc.rhs.Equal(types.DefaultTypeAdapter.NativeToValue(tc.lhs.Value())) != types.Bool(tc.equal) { + t.Errorf("expected unstructuredVal.Equals() to return %v", tc.equal) + } + }) + } +} + +func TestLister(t *testing.T) { + cases := []struct { + name string + unstructured []interface{} + schema *spec.Schema + itemSchema *spec.Schema + size int64 + notContains []ref.Val + addition []interface{} + expectAdded []interface{} + }{ + { + name: "map list", + unstructured: []interface{}{ + map[string]interface{}{ + "key": "a", + "val": 1, + }, + map[string]interface{}{ + "key": "b", + "val": 2, + }, + }, + schema: mapListSchema, + itemSchema: mapListElementSchema, + size: 2, + notContains: []ref.Val{ + UnstructuredToVal(map[string]interface{}{ + "key": "a", + "val": 2, + }, mapListElementSchema), + UnstructuredToVal(map[string]interface{}{ + "key": "c", + "val": 1, + }, mapListElementSchema), + }, + addition: []interface{}{ + map[string]interface{}{ + "key": "b", + "val": 3, + }, + map[string]interface{}{ + "key": "c", + "val": 4, + }, + }, + expectAdded: []interface{}{ + map[string]interface{}{ + "key": "a", + "val": 1, + }, + map[string]interface{}{ + "key": "b", + "val": 3, + }, + map[string]interface{}{ + "key": "c", + "val": 4, + }, + }, + }, + { + name: "set list", + unstructured: []interface{}{"a", "b"}, + schema: setListSchema, + itemSchema: stringSchema, + size: 2, + notContains: []ref.Val{UnstructuredToVal("c", stringSchema)}, + addition: []interface{}{"b", "c"}, + expectAdded: []interface{}{"a", "b", "c"}, + }, + { + name: "atomic list", + unstructured: []interface{}{"a", "b"}, + schema: atomicListSchema, + itemSchema: stringSchema, + size: 2, + notContains: []ref.Val{UnstructuredToVal("c", stringSchema)}, + addition: []interface{}{"b", "c"}, + expectAdded: []interface{}{"a", "b", "b", "c"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + lister := UnstructuredToVal(tc.unstructured, tc.schema).(traits.Lister) + if lister.Size().Value() != tc.size { + t.Errorf("Expected Size to return %d but got %d", tc.size, lister.Size().Value()) + } + iter := lister.Iterator() + for i := 0; i < int(tc.size); i++ { + get := lister.Get(types.Int(i)).Value() + if !reflect.DeepEqual(get, tc.unstructured[i]) { + t.Errorf("Expected Get to return %v for index %d but got %v", tc.unstructured[i], i, get) + } + if iter.HasNext() != types.True { + t.Error("Expected HasNext to return true") + } + next := iter.Next().Value() + if !reflect.DeepEqual(next, tc.unstructured[i]) { + t.Errorf("Expected Next to return %v for index %d but got %v", tc.unstructured[i], i, next) + } + } + if iter.HasNext() != types.False { + t.Error("Expected HasNext to return false") + } + for _, contains := range tc.unstructured { + if lister.Contains(UnstructuredToVal(contains, tc.itemSchema)) != types.True { + t.Errorf("Expected Contains to return true for %v", contains) + } + } + for _, notContains := range tc.notContains { + if lister.Contains(notContains) != types.False { + t.Errorf("Expected Contains to return false for %v", notContains) + } + } + + addition := UnstructuredToVal(tc.addition, tc.schema).(traits.Lister) + added := lister.Add(addition).Value() + if !reflect.DeepEqual(added, tc.expectAdded) { + t.Errorf("Expected Add to return %v but got %v", tc.expectAdded, added) + } + }) + } +} + +func TestMapper(t *testing.T) { + cases := []struct { + name string + unstructured map[string]interface{} + schema *spec.Schema + propertySchema func(key string) (*spec.Schema, bool) + size int64 + notContains []ref.Val + }{ + { + name: "object", + unstructured: map[string]interface{}{ + "field1": "a", + "field2": "b", + }, + schema: objectSchema, + propertySchema: func(key string) (*spec.Schema, bool) { + if s, ok := objectSchema.Properties[key]; ok { + return &s, true + } + return nil, false + }, + size: 2, + notContains: []ref.Val{ + UnstructuredToVal("field3", stringSchema), + }, + }, + { + name: "map", + unstructured: map[string]interface{}{ + "key1": "a", + "key2": "b", + }, + schema: mapSchema, + propertySchema: func(key string) (*spec.Schema, bool) { return mapSchema.AdditionalProperties.Schema, true }, + size: 2, + notContains: []ref.Val{ + UnstructuredToVal("key3", stringSchema), + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mapper := UnstructuredToVal(tc.unstructured, tc.schema).(traits.Mapper) + if mapper.Size().Value() != tc.size { + t.Errorf("Expected Size to return %d but got %d", tc.size, mapper.Size().Value()) + } + iter := mapper.Iterator() + iterResults := map[interface{}]struct{}{} + keys := map[interface{}]struct{}{} + for k := range tc.unstructured { + keys[k] = struct{}{} + get := mapper.Get(types.String(k)).Value() + if !reflect.DeepEqual(get, tc.unstructured[k]) { + t.Errorf("Expected Get to return %v for key %s but got %v", tc.unstructured[k], k, get) + } + if iter.HasNext() != types.True { + t.Error("Expected HasNext to return true") + } + iterResults[iter.Next().Value()] = struct{}{} + } + if !reflect.DeepEqual(iterResults, keys) { + t.Errorf("Expected accumulation of iterator.Next calls to be %v but got %v", keys, iterResults) + } + if iter.HasNext() != types.False { + t.Error("Expected HasNext to return false") + } + for contains := range tc.unstructured { + if mapper.Contains(UnstructuredToVal(contains, stringSchema)) != types.True { + t.Errorf("Expected Contains to return true for %v", contains) + } + } + for _, notContains := range tc.notContains { + if mapper.Contains(notContains) != types.False { + t.Errorf("Expected Contains to return false for %v", notContains) + } + } + }) + } +} + +func BenchmarkUnstructuredToVal(b *testing.B) { + u := []interface{}{ + map[string]interface{}{ + "key": "a", + "val": 1, + }, + map[string]interface{}{ + "key": "b", + "val": 2, + }, + map[string]interface{}{ + "key": "@b", + "val": 2, + }, + } + + b.ReportAllocs() + b.ResetTimer() + + for n := 0; n < b.N; n++ { + if val := UnstructuredToVal(u, mapListSchema); val == nil { + b.Fatal(val) + } + } +} + +func BenchmarkUnstructuredToValWithEscape(b *testing.B) { + u := []interface{}{ + map[string]interface{}{ + "key": "a.1", + "val": "__i.1", + }, + map[string]interface{}{ + "key": "b.1", + "val": 2, + }, + } + + b.ReportAllocs() + b.ResetTimer() + + for n := 0; n < b.N; n++ { + if val := UnstructuredToVal(u, mapListSchema); val == nil { + b.Fatal(val) + } + } +}