diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go index 5ead877b803..0b76b41291a 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go @@ -27,7 +27,6 @@ import ( apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" - celmodel "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/model" apiservercel "k8s.io/apiserver/pkg/cel" "k8s.io/apiserver/pkg/cel/library" "k8s.io/apiserver/pkg/cel/metrics" @@ -53,6 +52,10 @@ const ( // checkFrequency configures the number of iterations within a comprehension to evaluate // before checking whether the function evaluation has been interrupted checkFrequency = 100 + + // maxRequestSizeBytes is the maximum size of a request to the API server + // TODO(DangerOnTheRanger): wire in MaxRequestBodyBytes from apiserver/pkg/server/options/server_run_options.go to make this configurable + maxRequestSizeBytes = apiservercel.DefaultMaxRequestSizeBytes ) // CompilationResult represents the cel compilation result for one rule @@ -149,7 +152,7 @@ func Compile(s *schema.Structural, declType *apiservercel.DeclType, perCallLimit estimator := newCostEstimator(root) // compResults is the return value which saves a list of compilation results in the same order as x-kubernetes-validations rules. compResults := make([]CompilationResult, len(celRules)) - maxCardinality := celmodel.MaxCardinality(root.MinSerializedSize) + maxCardinality := maxCardinality(root.MinSerializedSize) for i, rule := range celRules { compResults[i] = compileRule(rule, env, perCallLimit, estimator, maxCardinality) } @@ -262,3 +265,14 @@ func (c *sizeEstimator) EstimateSize(element checker.AstNode) *checker.SizeEstim func (c *sizeEstimator) EstimateCallCost(function, overloadID string, target *checker.AstNode, args []checker.AstNode) *checker.CallEstimate { return nil } + +// maxCardinality returns the maximum number of times data conforming to the minimum size given could possibly exist in +// an object serialized to JSON. For cases where a schema is contained under map or array schemas of unbounded +// size, this can be used as an estimate as the worst case number of times data matching the schema could be repeated. +// Note that this only assumes a single comma between data elements, so if the schema is contained under only maps, +// this estimates a higher cardinality that would be possible. DeclType.MinSerializedSize is meant to be passed to +// this function. +func maxCardinality(minSize int64) uint64 { + sz := minSize + 1 // assume at least one comma between elements + return uint64(maxRequestSizeBytes / sz) +} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/maplist.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/maplist.go index 7e26ad956fc..99d9f505914 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/maplist.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/maplist.go @@ -17,162 +17,13 @@ limitations under the License. package cel import ( - "fmt" - "strings" - "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + celopenapi "k8s.io/apiserver/pkg/cel/openapi" ) -// mapList provides a "lookup by key" operation for lists (arrays) with x-kubernetes-list-type=map. -type mapList interface { - // get returns the first element having given key, for all - // x-kubernetes-list-map-keys, to the provided object. If the provided object isn't itself a valid mapList element, - // get returns nil. - get(interface{}) interface{} -} - -type keyStrategy interface { - // CompositeKeyFor returns a composite key for the provided object, if possible, and a - // boolean that indicates whether or not a key could be generated for the provided object. - CompositeKeyFor(map[string]interface{}) (interface{}, bool) -} - -// singleKeyStrategy is a cheaper strategy for associative lists that have exactly one key. -type singleKeyStrategy struct { - key string -} - -// CompositeKeyFor directly returns the value of the single key to -// use as a composite key. -func (ks *singleKeyStrategy) CompositeKeyFor(obj map[string]interface{}) (interface{}, bool) { - v, ok := obj[ks.key] - if !ok { - return nil, false - } - - switch v.(type) { - case bool, float64, int64, string: - return v, true - default: - return nil, false // non-scalar - } -} - -// multiKeyStrategy computes a composite key of all key values. -type multiKeyStrategy struct { - sts *schema.Structural -} - -// CompositeKeyFor returns a composite key computed from the values of all -// keys. -func (ks *multiKeyStrategy) CompositeKeyFor(obj map[string]interface{}) (interface{}, bool) { - const keyDelimiter = "\x00" // 0 byte should never appear in the composite key except as delimiter - - var delimited strings.Builder - for _, key := range ks.sts.XListMapKeys { - v, ok := obj[key] - if !ok { - return nil, false - } - - switch v.(type) { - case bool: - fmt.Fprintf(&delimited, keyDelimiter+"%t", v) - case float64: - fmt.Fprintf(&delimited, keyDelimiter+"%f", v) - case int64: - fmt.Fprintf(&delimited, keyDelimiter+"%d", v) - case string: - fmt.Fprintf(&delimited, keyDelimiter+"%q", v) - default: - return nil, false // values must be scalars - } - } - return delimited.String(), true -} - -// emptyMapList is a mapList containing no elements. -type emptyMapList struct{} - -func (emptyMapList) get(interface{}) interface{} { - return nil -} - -type mapListImpl struct { - sts *schema.Structural - ks keyStrategy - // keyedItems contains all lazily keyed map items - keyedItems map[interface{}]interface{} - // unkeyedItems contains all map items that have not yet been keyed - unkeyedItems []interface{} -} - -func (a *mapListImpl) get(obj interface{}) interface{} { - mobj, ok := obj.(map[string]interface{}) - if !ok { - return nil - } - - key, ok := a.ks.CompositeKeyFor(mobj) - if !ok { - return nil - } - if match, ok := a.keyedItems[key]; ok { - return match - } - // keep keying items until we either find a match or run out of unkeyed items - for len(a.unkeyedItems) > 0 { - // dequeue an unkeyed item - item := a.unkeyedItems[0] - a.unkeyedItems = a.unkeyedItems[1:] - - // key the item - mitem, ok := item.(map[string]interface{}) - if !ok { - continue - } - itemKey, ok := a.ks.CompositeKeyFor(mitem) - if !ok { - continue - } - if _, exists := a.keyedItems[itemKey]; !exists { - a.keyedItems[itemKey] = mitem - } - - // if it matches, short-circuit - if itemKey == key { - return mitem - } - } - - return nil -} - -func makeKeyStrategy(sts *schema.Structural) keyStrategy { - if len(sts.XListMapKeys) == 1 { - key := sts.XListMapKeys[0] - return &singleKeyStrategy{ - key: key, - } - } - - return &multiKeyStrategy{ - sts: sts, - } -} - // makeMapList returns a queryable interface over the provided x-kubernetes-list-type=map // keyedItems. If the provided schema is _not_ an array with x-kubernetes-list-type=map, returns an // empty mapList. -func makeMapList(sts *schema.Structural, items []interface{}) (rv mapList) { - if sts.Type != "array" || sts.XListType == nil || *sts.XListType != "map" || len(sts.XListMapKeys) == 0 || len(items) == 0 { - return emptyMapList{} - } - ks := makeKeyStrategy(sts) - return &mapListImpl{ - sts: sts, - ks: ks, - keyedItems: map[interface{}]interface{}{}, - unkeyedItems: items, - } +func makeMapList(sts *schema.Structural, items []interface{}) (rv celopenapi.MapList) { + return celopenapi.MakeMapList(sts.ToKubeOpenAPI(), items) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/maplist_test.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/maplist_test.go index 97c2b80f76e..85ad0ae1f34 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/maplist_test.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/maplist_test.go @@ -323,9 +323,9 @@ func TestMapList(t *testing.T) { t.Run(tc.name, func(t *testing.T) { mapList := makeMapList(&tc.sts, tc.items) for _, warmUp := range tc.warmUpQueries { - mapList.get(warmUp) + mapList.Get(warmUp) } - actual := mapList.get(tc.query) + actual := mapList.Get(tc.query) if !reflect.DeepEqual(tc.expected, actual) { t.Errorf("got: %v, expected %v", actual, tc.expected) } diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/model/schemas.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/model/schemas.go index b9b47caea50..1bd8713026f 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/model/schemas.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/model/schemas.go @@ -17,19 +17,12 @@ limitations under the License. package model import ( - "time" - - "github.com/google/cel-go/cel" - "github.com/google/cel-go/common/types" - apiservercel "k8s.io/apiserver/pkg/cel" + celopenapi "k8s.io/apiserver/pkg/cel/openapi" "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" ) -// TODO(DangerOnTheRanger): wire in MaxRequestBodyBytes from apiserver/pkg/server/options/server_run_options.go to make this configurable -const maxRequestSizeBytes = apiservercel.DefaultMaxRequestSizeBytes - // SchemaDeclType converts the structural schema to a CEL declaration, or returns nil if the // structural schema should not be exposed in CEL expressions. // Set isResourceRoot to true for the root of a custom resource or embedded resource. @@ -40,152 +33,7 @@ const maxRequestSizeBytes = apiservercel.DefaultMaxRequestSizeBytes // // The CEL declaration for objects with XPreserveUnknownFields does not expose unknown fields. func SchemaDeclType(s *schema.Structural, isResourceRoot bool) *apiservercel.DeclType { - if s == nil { - return nil - } - if s.XIntOrString { - // schemas using XIntOrString are not required to have a type. - - // intOrStringType represents the x-kubernetes-int-or-string union type in CEL expressions. - // In CEL, the type is represented as dynamic value, which can be thought of as a union type of all types. - // All type checking for XIntOrString is deferred to runtime, so all access to values of this type must - // be guarded with a type check, e.g.: - // - // To require that the string representation be a percentage: - // `type(intOrStringField) == string && intOrStringField.matches(r'(\d+(\.\d+)?%)')` - // To validate requirements on both the int and string representation: - // `type(intOrStringField) == int ? intOrStringField < 5 : double(intOrStringField.replace('%', '')) < 0.5 - // - dyn := apiservercel.NewSimpleTypeWithMinSize("dyn", cel.DynType, nil, 1) // smallest value for a serialized x-kubernetes-int-or-string is 0 - // handle x-kubernetes-int-or-string by returning the max length/min serialized size of the largest possible string - dyn.MaxElements = maxRequestSizeBytes - 2 - return dyn - } - - // We ignore XPreserveUnknownFields since we don't support validation rules on - // data that we don't have schema information for. - - if isResourceRoot { - // 'apiVersion', 'kind', 'metadata.name' and 'metadata.generateName' are always accessible to validator rules - // at the root of resources, even if not specified in the schema. - // This includes the root of a custom resource and the root of XEmbeddedResource objects. - s = WithTypeAndObjectMeta(s) - } - - switch s.Type { - case "array": - if s.Items != nil { - itemsType := SchemaDeclType(s.Items, s.Items.XEmbeddedResource) - if itemsType == nil { - return nil - } - var maxItems int64 - if s.ValueValidation != nil && s.ValueValidation.MaxItems != nil { - maxItems = zeroIfNegative(*s.ValueValidation.MaxItems) - } else { - maxItems = estimateMaxArrayItemsFromMinSize(itemsType.MinSerializedSize) - } - return apiservercel.NewListType(itemsType, maxItems) - } - return nil - case "object": - if s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil { - propsType := SchemaDeclType(s.AdditionalProperties.Structural, s.AdditionalProperties.Structural.XEmbeddedResource) - if propsType != nil { - var maxProperties int64 - if s.ValueValidation != nil && s.ValueValidation.MaxProperties != nil { - maxProperties = zeroIfNegative(*s.ValueValidation.MaxProperties) - } else { - maxProperties = estimateMaxAdditionalPropertiesFromMinSize(propsType.MinSerializedSize) - } - return apiservercel.NewMapType(apiservercel.StringType, propsType, maxProperties) - } - return nil - } - fields := make(map[string]*apiservercel.DeclField, len(s.Properties)) - - required := map[string]bool{} - if s.ValueValidation != nil { - for _, f := range s.ValueValidation.Required { - required[f] = true - } - } - // an object will always be serialized at least as {}, so account for that - minSerializedSize := int64(2) - for name, prop := range s.Properties { - var enumValues []interface{} - if prop.ValueValidation != nil { - for _, e := range prop.ValueValidation.Enum { - enumValues = append(enumValues, e.Object) - } - } - if fieldType := SchemaDeclType(&prop, prop.XEmbeddedResource); fieldType != nil { - if propName, ok := apiservercel.Escape(name); ok { - fields[propName] = apiservercel.NewDeclField(propName, fieldType, required[name], enumValues, prop.Default.Object) - } - // the min serialized size for an object is 2 (for {}) plus the min size of all its required - // properties - // only include required properties without a default value; default values are filled in - // server-side - if required[name] && prop.Default.Object == nil { - minSerializedSize += int64(len(name)) + fieldType.MinSerializedSize + 4 - } - } - } - objType := apiservercel.NewObjectType("object", fields) - objType.MinSerializedSize = minSerializedSize - return objType - case "string": - if s.ValueValidation != nil { - switch s.ValueValidation.Format { - case "byte": - byteWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("bytes", cel.BytesType, types.Bytes([]byte{}), apiservercel.MinStringSize) - if s.ValueValidation.MaxLength != nil { - byteWithMaxLength.MaxElements = zeroIfNegative(*s.ValueValidation.MaxLength) - } else { - byteWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) - } - return byteWithMaxLength - case "duration": - durationWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("duration", cel.DurationType, types.Duration{Duration: time.Duration(0)}, int64(apiservercel.MinDurationSizeJSON)) - durationWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) - return durationWithMaxLength - case "date": - timestampWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("timestamp", cel.TimestampType, types.Timestamp{Time: time.Time{}}, int64(apiservercel.JSONDateSize)) - timestampWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) - return timestampWithMaxLength - case "date-time": - timestampWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("timestamp", cel.TimestampType, types.Timestamp{Time: time.Time{}}, int64(apiservercel.MinDatetimeSizeJSON)) - timestampWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) - return timestampWithMaxLength - } - } - strWithMaxLength := apiservercel.NewSimpleTypeWithMinSize("string", cel.StringType, types.String(""), apiservercel.MinStringSize) - if s.ValueValidation != nil && s.ValueValidation.MaxLength != nil { - // multiply the user-provided max length by 4 in the case of an otherwise-untyped string - // we do this because the OpenAPIv3 spec indicates that maxLength is specified in runes/code points, - // but we need to reason about length for things like request size, so we use bytes in this code (and an individual - // unicode code point can be up to 4 bytes long) - strWithMaxLength.MaxElements = zeroIfNegative(*s.ValueValidation.MaxLength) * 4 - } else { - strWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) - } - return strWithMaxLength - case "boolean": - return apiservercel.BoolType - case "number": - return apiservercel.DoubleType - case "integer": - return apiservercel.IntType - } - return nil -} - -func zeroIfNegative(v int64) int64 { - if v < 0 { - return 0 - } - return v + return celopenapi.SchemaDeclType(s.ToKubeOpenAPI(), isResourceRoot) } // WithTypeAndObjectMeta ensures the kind, apiVersion and @@ -223,52 +71,3 @@ func WithTypeAndObjectMeta(s *schema.Structural) *schema.Structural { return result } - -// MaxCardinality returns the maximum number of times data conforming to the minimum size given could possibly exist in -// an object serialized to JSON. For cases where a schema is contained under map or array schemas of unbounded -// size, this can be used as an estimate as the worst case number of times data matching the schema could be repeated. -// Note that this only assumes a single comma between data elements, so if the schema is contained under only maps, -// this estimates a higher cardinality that would be possible. DeclType.MinSerializedSize is meant to be passed to -// this function. -func MaxCardinality(minSize int64) uint64 { - sz := minSize + 1 // assume at least one comma between elements - return uint64(maxRequestSizeBytes / sz) -} - -// estimateMaxStringLengthPerRequest estimates the maximum string length (in characters) -// of a string compatible with the format requirements in the provided schema. -// must only be called on schemas of type "string" or x-kubernetes-int-or-string: true -func estimateMaxStringLengthPerRequest(s *schema.Structural) int64 { - if s.ValueValidation == nil || s.XIntOrString { - // subtract 2 to account for "" - return maxRequestSizeBytes - 2 - } - switch s.ValueValidation.Format { - case "duration": - return apiservercel.MaxDurationSizeJSON - case "date": - return apiservercel.JSONDateSize - case "date-time": - return apiservercel.MaxDatetimeSizeJSON - default: - // subtract 2 to account for "" - return maxRequestSizeBytes - 2 - } -} - -// estimateMaxArrayItemsPerRequest estimates the maximum number of array items with -// the provided minimum serialized size that can fit into a single request. -func estimateMaxArrayItemsFromMinSize(minSize int64) int64 { - // subtract 2 to account for [ and ] - return (maxRequestSizeBytes - 2) / (minSize + 1) -} - -// estimateMaxAdditionalPropertiesPerRequest estimates the maximum number of additional properties -// with the provided minimum serialized size that can fit into a single request. -func estimateMaxAdditionalPropertiesFromMinSize(minSize int64) int64 { - // 2 bytes for key + "" + colon + comma + smallest possible value, realistically the actual keys - // will all vary in length - keyValuePairSize := minSize + 6 - // subtract 2 to account for { and } - return (maxRequestSizeBytes - 2) / keyValuePairSize -} diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation.go index d3626dc3a60..2a039b6b58e 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/validation.go @@ -366,7 +366,7 @@ func (s *Validator) validateArray(ctx context.Context, fldPath *field.Path, sts correlatableOldItems := makeMapList(sts, oldObj) for i := range obj { var err field.ErrorList - err, remainingBudget = s.Items.Validate(ctx, fldPath.Index(i), sts.Items, obj[i], correlatableOldItems.get(obj[i]), remainingBudget) + err, remainingBudget = s.Items.Validate(ctx, fldPath.Index(i), sts.Items, obj[i], correlatableOldItems.Get(obj[i]), remainingBudget) errs = append(errs, err...) if remainingBudget < 0 { return errs, remainingBudget diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/values.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/values.go index c7e159f5527..8bd4bb8e5b3 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/values.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/values.go @@ -17,688 +17,16 @@ limitations under the License. package cel import ( - "fmt" - "reflect" - "sync" - "time" - - "github.com/google/cel-go/common/types" "github.com/google/cel-go/common/types/ref" - "github.com/google/cel-go/common/types/traits" + + celopenapi "k8s.io/apiserver/pkg/cel/openapi" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" - "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/model" - "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apiserver/pkg/cel" - "k8s.io/kube-openapi/pkg/validation/strfmt" ) // UnstructuredToVal converts a Kubernetes unstructured data element to a CEL Val. // The root schema of custom resource schema is expected contain type meta and object meta schemas. // If Embedded resources do not contain type meta and object meta schemas, they will be added automatically. func UnstructuredToVal(unstructured interface{}, schema *structuralschema.Structural) ref.Val { - if unstructured == nil { - if schema.Nullable { - return types.NullValue - } - return types.NewErr("invalid data, got null for schema with nullable=false") - } - if schema.XIntOrString { - switch v := unstructured.(type) { - case string: - return types.String(v) - case int: - return types.Int(v) - case int32: - return types.Int(v) - case int64: - return types.Int(v) - } - return types.NewErr("invalid data, expected XIntOrString value to be either a string or integer") - } - if schema.Type == "object" { - m, ok := unstructured.(map[string]interface{}) - if !ok { - return types.NewErr("invalid data, expected a map for the provided schema with type=object") - } - if schema.XEmbeddedResource || schema.Properties != nil { - if schema.XEmbeddedResource { - schema = model.WithTypeAndObjectMeta(schema) - } - return &unstructuredMap{ - value: m, - schema: schema, - propSchema: func(key string) (*structuralschema.Structural, bool) { - if schema, ok := schema.Properties[key]; ok { - return &schema, true - } - return nil, false - }, - } - } - if schema.AdditionalProperties != nil && schema.AdditionalProperties.Structural != nil { - return &unstructuredMap{ - value: m, - schema: schema, - propSchema: func(key string) (*structuralschema.Structural, bool) { - return schema.AdditionalProperties.Structural, true - }, - } - } - // A object with x-kubernetes-preserve-unknown-fields but no properties or additionalProperties is treated - // as an empty object. - if schema.XPreserveUnknownFields { - return &unstructuredMap{ - value: m, - schema: schema, - propSchema: func(key string) (*structuralschema.Structural, bool) { - return nil, false - }, - } - } - return types.NewErr("invalid object type, expected either Properties or AdditionalProperties with Allows=true and non-empty Schema") - } - - if schema.Type == "array" { - l, ok := unstructured.([]interface{}) - if !ok { - return types.NewErr("invalid data, expected an array for the provided schema with type=array") - } - if schema.Items == nil { - return types.NewErr("invalid array type, expected Items with a non-empty Schema") - } - typedList := unstructuredList{elements: l, itemsSchema: schema.Items} - listType := schema.XListType - if listType != nil { - switch *listType { - case "map": - mapKeys := schema.Extensions.XListMapKeys - return &unstructuredMapList{unstructuredList: typedList, escapedKeyProps: escapeKeyProps(mapKeys)} - case "set": - return &unstructuredSetList{unstructuredList: typedList} - case "atomic": - return &typedList - default: - return types.NewErr("invalid x-kubernetes-list-type, expected 'map', 'set' or 'atomic' but got %s", *listType) - } - } - return &typedList - } - - if schema.Type == "string" { - str, ok := unstructured.(string) - if !ok { - return types.NewErr("invalid data, expected string, got %T", unstructured) - } - if schema.ValueValidation != nil { - switch schema.ValueValidation.Format { - case "duration": - d, err := strfmt.ParseDuration(str) - if err != nil { - return types.NewErr("Invalid duration %s: %v", str, err) - } - return types.Duration{Duration: d} - case "date": - d, err := time.Parse(strfmt.RFC3339FullDate, str) // strfmt uses this format for OpenAPIv3 value validation - if err != nil { - return types.NewErr("Invalid date formatted string %s: %v", str, err) - } - return types.Timestamp{Time: d} - case "date-time": - d, err := strfmt.ParseDateTime(str) - if err != nil { - return types.NewErr("Invalid date-time formatted string %s: %v", str, err) - } - return types.Timestamp{Time: time.Time(d)} - case "byte": - base64 := strfmt.Base64{} - err := base64.UnmarshalText([]byte(str)) - if err != nil { - return types.NewErr("Invalid byte formatted string %s: %v", str, err) - } - return types.Bytes(base64) - } - } - return types.String(str) - } - if schema.Type == "number" { - switch v := unstructured.(type) { - // float representations of whole numbers (e.g. 1.0, 0.0) can convert to int representations (e.g. 1, 0) in yaml - // to json translation, and then get parsed as int64s - case int: - return types.Double(v) - case int32: - return types.Double(v) - case int64: - return types.Double(v) - - case float32: - return types.Double(v) - case float64: - return types.Double(v) - default: - return types.NewErr("invalid data, expected float, got %T", unstructured) - } - } - if schema.Type == "integer" { - switch v := unstructured.(type) { - case int: - return types.Int(v) - case int32: - return types.Int(v) - case int64: - return types.Int(v) - default: - return types.NewErr("invalid data, expected int, got %T", unstructured) - } - } - if schema.Type == "boolean" { - b, ok := unstructured.(bool) - if !ok { - return types.NewErr("invalid data, expected bool, got %T", unstructured) - } - return types.Bool(b) - } - - if schema.XPreserveUnknownFields { - return &unknownPreserved{u: unstructured} - } - - return types.NewErr("invalid type, expected object, array, number, integer, boolean or string, or no type with x-kubernetes-int-or-string or x-kubernetes-preserve-unknown-fields is true, got %s", schema.Type) -} - -// unknownPreserved represents unknown data preserved in custom resources via x-kubernetes-preserve-unknown-fields. -// It preserves the data at runtime without assuming it is of any particular type and supports only equality checking. -// unknownPreserved should be used only for values are not directly accessible in CEL expressions, i.e. for data -// where there is no corresponding CEL type declaration. -type unknownPreserved struct { - u interface{} -} - -func (t *unknownPreserved) ConvertToNative(refType reflect.Type) (interface{}, error) { - return nil, fmt.Errorf("type conversion to '%s' not supported for values preserved by x-kubernetes-preserve-unknown-fields", refType) -} - -func (t *unknownPreserved) ConvertToType(typeValue ref.Type) ref.Val { - return types.NewErr("type conversion to '%s' not supported for values preserved by x-kubernetes-preserve-unknown-fields", typeValue.TypeName()) -} - -func (t *unknownPreserved) Equal(other ref.Val) ref.Val { - return types.Bool(equality.Semantic.DeepEqual(t.u, other.Value())) -} - -func (t *unknownPreserved) Type() ref.Type { - return types.UnknownType -} - -func (t *unknownPreserved) Value() interface{} { - return t.u // used by Equal checks -} - -// unstructuredMapList represents an unstructured data instance of an OpenAPI array with x-kubernetes-list-type=map. -type unstructuredMapList struct { - unstructuredList - escapedKeyProps []string - - sync.Once // for for lazy load of mapOfList since it is only needed if Equals is called - mapOfList map[interface{}]interface{} -} - -func (t *unstructuredMapList) getMap() map[interface{}]interface{} { - t.Do(func() { - t.mapOfList = make(map[interface{}]interface{}, len(t.elements)) - for _, e := range t.elements { - t.mapOfList[t.toMapKey(e)] = e - } - }) - return t.mapOfList -} - -// toMapKey returns a valid golang map key for the given element of the map list. -// element must be a valid map list entry where all map key props are scalar types (which are comparable in go -// and valid for use in a golang map key). -func (t *unstructuredMapList) toMapKey(element interface{}) interface{} { - eObj, ok := element.(map[string]interface{}) - if !ok { - return types.NewErr("unexpected data format for element of array with x-kubernetes-list-type=map: %T", element) - } - // Arrays are comparable in go and may be used as map keys, but maps and slices are not. - // So we can special case small numbers of key props as arrays and fall back to serialization - // for larger numbers of key props - if len(t.escapedKeyProps) == 1 { - return eObj[t.escapedKeyProps[0]] - } - if len(t.escapedKeyProps) == 2 { - return [2]interface{}{eObj[t.escapedKeyProps[0]], eObj[t.escapedKeyProps[1]]} - } - if len(t.escapedKeyProps) == 3 { - return [3]interface{}{eObj[t.escapedKeyProps[0]], eObj[t.escapedKeyProps[1]], eObj[t.escapedKeyProps[2]]} - } - - key := make([]interface{}, len(t.escapedKeyProps)) - for i, kf := range t.escapedKeyProps { - key[i] = eObj[kf] - } - return fmt.Sprintf("%v", key) -} - -// Equal on a map list ignores list element order. -func (t *unstructuredMapList) Equal(other ref.Val) ref.Val { - oMapList, ok := other.(traits.Lister) - if !ok { - return types.MaybeNoSuchOverloadErr(other) - } - sz := types.Int(len(t.elements)) - if sz != oMapList.Size() { - return types.False - } - tMap := t.getMap() - for it := oMapList.Iterator(); it.HasNext() == types.True; { - v := it.Next() - k := t.toMapKey(v.Value()) - tVal, ok := tMap[k] - if !ok { - return types.False - } - eq := UnstructuredToVal(tVal, t.itemsSchema).Equal(v) - if eq != types.True { - return eq // either false or error - } - } - return types.True -} - -// Add for a map list `X + Y` performs a merge where the array positions of all keys in `X` are preserved but the values -// are overwritten by values in `Y` when the key sets of `X` and `Y` intersect. Elements in `Y` with -// non-intersecting keys are appended, retaining their partial order. -func (t *unstructuredMapList) Add(other ref.Val) ref.Val { - oMapList, ok := other.(traits.Lister) - if !ok { - return types.MaybeNoSuchOverloadErr(other) - } - elements := make([]interface{}, len(t.elements)) - keyToIdx := map[interface{}]int{} - for i, e := range t.elements { - k := t.toMapKey(e) - keyToIdx[k] = i - elements[i] = e - } - for it := oMapList.Iterator(); it.HasNext() == types.True; { - v := it.Next().Value() - k := t.toMapKey(v) - if overwritePosition, ok := keyToIdx[k]; ok { - elements[overwritePosition] = v - } else { - elements = append(elements, v) - } - } - return &unstructuredMapList{ - unstructuredList: unstructuredList{elements: elements, itemsSchema: t.itemsSchema}, - escapedKeyProps: t.escapedKeyProps, - } -} - -// escapeKeyProps returns identifiers with Escape applied to each. -// Identifiers that cannot be escaped are left as-is. They are inaccessible to CEL programs but are -// are still needed internally to perform equality checks. -func escapeKeyProps(idents []string) []string { - result := make([]string, len(idents)) - for i, prop := range idents { - if escaped, ok := cel.Escape(prop); ok { - result[i] = escaped - } else { - result[i] = prop - } - } - return result -} - -// unstructuredSetList represents an unstructured data instance of an OpenAPI array with x-kubernetes-list-type=set. -type unstructuredSetList struct { - unstructuredList - escapedKeyProps []string - - sync.Once // for for lazy load of setOfList since it is only needed if Equals is called - set map[interface{}]struct{} -} - -func (t *unstructuredSetList) getSet() map[interface{}]struct{} { - // sets are only allowed to contain scalar elements, which are comparable in go, and can safely be used as - // golang map keys - t.Do(func() { - t.set = make(map[interface{}]struct{}, len(t.elements)) - for _, e := range t.elements { - t.set[e] = struct{}{} - } - }) - return t.set -} - -// Equal on a map list ignores list element order. -func (t *unstructuredSetList) Equal(other ref.Val) ref.Val { - oSetList, ok := other.(traits.Lister) - if !ok { - return types.MaybeNoSuchOverloadErr(other) - } - sz := types.Int(len(t.elements)) - if sz != oSetList.Size() { - return types.False - } - tSet := t.getSet() - for it := oSetList.Iterator(); it.HasNext() == types.True; { - next := it.Next().Value() - _, ok := tSet[next] - if !ok { - return types.False - } - } - return types.True -} - -// Add for a set list `X + Y` performs a union where the array positions of all elements in `X` are preserved and -// non-intersecting elements in `Y` are appended, retaining their partial order. -func (t *unstructuredSetList) Add(other ref.Val) ref.Val { - oSetList, ok := other.(traits.Lister) - if !ok { - return types.MaybeNoSuchOverloadErr(other) - } - elements := t.elements - set := t.getSet() - for it := oSetList.Iterator(); it.HasNext() == types.True; { - next := it.Next().Value() - if _, ok := set[next]; !ok { - set[next] = struct{}{} - elements = append(elements, next) - } - } - return &unstructuredSetList{ - unstructuredList: unstructuredList{elements: elements, itemsSchema: t.itemsSchema}, - escapedKeyProps: t.escapedKeyProps, - } -} - -// unstructuredList represents an unstructured data instance of an OpenAPI array with x-kubernetes-list-type=atomic (the default). -type unstructuredList struct { - elements []interface{} - itemsSchema *structuralschema.Structural -} - -var _ = traits.Lister(&unstructuredList{}) - -func (t *unstructuredList) ConvertToNative(typeDesc reflect.Type) (interface{}, error) { - switch typeDesc.Kind() { - case reflect.Slice: - return t.elements, nil - } - return nil, fmt.Errorf("type conversion error from '%s' to '%s'", t.Type(), typeDesc) -} - -func (t *unstructuredList) ConvertToType(typeValue ref.Type) ref.Val { - switch typeValue { - case types.ListType: - return t - case types.TypeType: - return types.ListType - } - return types.NewErr("type conversion error from '%s' to '%s'", t.Type(), typeValue.TypeName()) -} - -func (t *unstructuredList) Equal(other ref.Val) ref.Val { - oList, ok := other.(traits.Lister) - if !ok { - return types.MaybeNoSuchOverloadErr(other) - } - sz := types.Int(len(t.elements)) - if sz != oList.Size() { - return types.False - } - for i := types.Int(0); i < sz; i++ { - eq := t.Get(i).Equal(oList.Get(i)) - if eq != types.True { - return eq // either false or error - } - } - return types.True -} - -func (t *unstructuredList) Type() ref.Type { - return types.ListType -} - -func (t *unstructuredList) Value() interface{} { - return t.elements -} - -func (t *unstructuredList) Add(other ref.Val) ref.Val { - oList, ok := other.(traits.Lister) - if !ok { - return types.MaybeNoSuchOverloadErr(other) - } - elements := t.elements - for it := oList.Iterator(); it.HasNext() == types.True; { - next := it.Next().Value() - elements = append(elements, next) - } - - return &unstructuredList{elements: elements, itemsSchema: t.itemsSchema} -} - -func (t *unstructuredList) Contains(val ref.Val) ref.Val { - if types.IsUnknownOrError(val) { - return val - } - var err ref.Val - sz := len(t.elements) - for i := 0; i < sz; i++ { - elem := UnstructuredToVal(t.elements[i], t.itemsSchema) - cmp := elem.Equal(val) - b, ok := cmp.(types.Bool) - if !ok && err == nil { - err = types.MaybeNoSuchOverloadErr(cmp) - } - if b == types.True { - return types.True - } - } - if err != nil { - return err - } - return types.False -} - -func (t *unstructuredList) Get(idx ref.Val) ref.Val { - iv, isInt := idx.(types.Int) - if !isInt { - return types.ValOrErr(idx, "unsupported index: %v", idx) - } - i := int(iv) - if i < 0 || i >= len(t.elements) { - return types.NewErr("index out of bounds: %v", idx) - } - return UnstructuredToVal(t.elements[i], t.itemsSchema) -} - -func (t *unstructuredList) Iterator() traits.Iterator { - items := make([]ref.Val, len(t.elements)) - for i, item := range t.elements { - itemCopy := item - items[i] = UnstructuredToVal(itemCopy, t.itemsSchema) - } - return &listIterator{unstructuredList: t, items: items} -} - -type listIterator struct { - *unstructuredList - items []ref.Val - idx int -} - -func (it *listIterator) HasNext() ref.Val { - return types.Bool(it.idx < len(it.items)) -} - -func (it *listIterator) Next() ref.Val { - item := it.items[it.idx] - it.idx++ - return item -} - -func (t *unstructuredList) Size() ref.Val { - return types.Int(len(t.elements)) -} - -// unstructuredMap represented an unstructured data instance of an OpenAPI object. -type unstructuredMap struct { - value map[string]interface{} - schema *structuralschema.Structural - // propSchema finds the schema to use for a particular map key. - propSchema func(key string) (*structuralschema.Structural, bool) -} - -var _ = traits.Mapper(&unstructuredMap{}) - -func (t *unstructuredMap) ConvertToNative(typeDesc reflect.Type) (interface{}, error) { - switch typeDesc.Kind() { - case reflect.Map: - return t.value, nil - } - return nil, fmt.Errorf("type conversion error from '%s' to '%s'", t.Type(), typeDesc) -} - -func (t *unstructuredMap) ConvertToType(typeValue ref.Type) ref.Val { - switch typeValue { - case types.MapType: - return t - case types.TypeType: - return types.MapType - } - return types.NewErr("type conversion error from '%s' to '%s'", t.Type(), typeValue.TypeName()) -} - -func (t *unstructuredMap) Equal(other ref.Val) ref.Val { - oMap, isMap := other.(traits.Mapper) - if !isMap { - return types.MaybeNoSuchOverloadErr(other) - } - if t.Size() != oMap.Size() { - return types.False - } - for key, value := range t.value { - if propSchema, ok := t.propSchema(key); ok { - ov, found := oMap.Find(types.String(key)) - if !found { - return types.False - } - v := UnstructuredToVal(value, propSchema) - vEq := v.Equal(ov) - if vEq != types.True { - return vEq // either false or error - } - } else { - // Must be an object with properties. - // Since we've encountered an unknown field, fallback to unstructured equality checking. - ouMap, ok := other.(*unstructuredMap) - if !ok { - // The compiler ensures equality is against the same type of object, so this should be unreachable - return types.MaybeNoSuchOverloadErr(other) - } - if oValue, ok := ouMap.value[key]; ok { - if !equality.Semantic.DeepEqual(value, oValue) { - return types.False - } - } - } - } - return types.True -} - -func (t *unstructuredMap) Type() ref.Type { - return types.MapType -} - -func (t *unstructuredMap) Value() interface{} { - return t.value -} - -func (t *unstructuredMap) Contains(key ref.Val) ref.Val { - v, found := t.Find(key) - if v != nil && types.IsUnknownOrError(v) { - return v - } - - return types.Bool(found) -} - -func (t *unstructuredMap) Get(key ref.Val) ref.Val { - v, found := t.Find(key) - if found { - return v - } - return types.ValOrErr(key, "no such key: %v", key) -} - -func (t *unstructuredMap) Iterator() traits.Iterator { - isObject := t.schema.Properties != nil - keys := make([]ref.Val, len(t.value)) - i := 0 - for k := range t.value { - if _, ok := t.propSchema(k); ok { - mapKey := k - if isObject { - if escaped, ok := cel.Escape(k); ok { - mapKey = escaped - } - } - keys[i] = types.String(mapKey) - i++ - } - } - return &mapIterator{unstructuredMap: t, keys: keys} -} - -type mapIterator struct { - *unstructuredMap - keys []ref.Val - idx int -} - -func (it *mapIterator) HasNext() ref.Val { - return types.Bool(it.idx < len(it.keys)) -} - -func (it *mapIterator) Next() ref.Val { - key := it.keys[it.idx] - it.idx++ - return key -} - -func (t *unstructuredMap) Size() ref.Val { - return types.Int(len(t.value)) -} - -func (t *unstructuredMap) Find(key ref.Val) (ref.Val, bool) { - isObject := t.schema.Properties != nil - keyStr, ok := key.(types.String) - if !ok { - return types.MaybeNoSuchOverloadErr(key), true - } - k := keyStr.Value().(string) - if isObject { - k, ok = cel.Unescape(k) - if !ok { - return nil, false - } - } - if v, ok := t.value[k]; ok { - // If this is an object with properties, not an object with additionalProperties, - // then null valued nullable fields are treated the same as absent optional fields. - if isObject && v == nil { - return nil, false - } - if propSchema, ok := t.propSchema(k); ok { - return UnstructuredToVal(v, propSchema), true - } - } - - return nil, false + return celopenapi.UnstructuredToVal(unstructured, schema.ToKubeOpenAPI()) } diff --git a/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist.go b/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist.go index d018fc8a10d..ff67c9eed92 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist.go @@ -23,12 +23,12 @@ import ( "k8s.io/kube-openapi/pkg/validation/spec" ) -// mapList provides a "lookup by key" operation for lists (arrays) with x-kubernetes-list-type=map. -type mapList interface { - // get returns the first element having given key, for all - // x-kubernetes-list-map-keys, to the provided object. If the provided object isn't itself a valid mapList element, +// MapList provides a "lookup by key" operation for lists (arrays) with x-kubernetes-list-type=map. +type MapList interface { + // Get returns the first element having given key, for all + // x-kubernetes-list-map-keys, to the provided object. If the provided object isn't itself a valid MapList element, // get returns nil. - get(interface{}) interface{} + Get(interface{}) interface{} } type keyStrategy interface { @@ -91,10 +91,10 @@ func (ks *multiKeyStrategy) CompositeKeyFor(obj map[string]interface{}) (interfa return delimited.String(), true } -// emptyMapList is a mapList containing no elements. +// emptyMapList is a MapList containing no elements. type emptyMapList struct{} -func (emptyMapList) get(interface{}) interface{} { +func (emptyMapList) Get(interface{}) interface{} { return nil } @@ -107,7 +107,7 @@ type mapListImpl struct { unkeyedItems []interface{} } -func (a *mapListImpl) get(obj interface{}) interface{} { +func (a *mapListImpl) Get(obj interface{}) interface{} { mobj, ok := obj.(map[string]interface{}) if !ok { return nil @@ -162,10 +162,10 @@ func makeKeyStrategy(sts *spec.Schema) keyStrategy { } } -// makeMapList returns a queryable interface over the provided x-kubernetes-list-type=map +// MakeMapList returns a queryable interface over the provided x-kubernetes-list-type=map // keyedItems. If the provided schema is _not_ an array with x-kubernetes-list-type=map, returns an // empty mapList. -func makeMapList(sts *spec.Schema, items []interface{}) (rv mapList) { +func MakeMapList(sts *spec.Schema, items []interface{}) (rv MapList) { if !sts.Type.Contains("array") || getXListType(sts) != "map" || len(getXListMapKeys(sts)) == 0 { return emptyMapList{} } diff --git a/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist_test.go b/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist_test.go index 134a0f2cdbb..62d303d1505 100644 --- a/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist_test.go +++ b/staging/src/k8s.io/apiserver/pkg/cel/openapi/maplist_test.go @@ -296,11 +296,11 @@ func TestMapList(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - mapList := makeMapList(tc.sts, tc.items) + mapList := MakeMapList(tc.sts, tc.items) for _, warmUp := range tc.warmUpQueries { - mapList.get(warmUp) + mapList.Get(warmUp) } - actual := mapList.get(tc.query) + actual := mapList.Get(tc.query) if !reflect.DeepEqual(tc.expected, actual) { t.Errorf("got: %v, expected %v", actual, tc.expected) }