Add quantity library to CEL (#118803)

* add quantity library to CEL

* add more tests to quantity

* use 1.29 env for quantity

* set CEL default env to 1.28 for 1.28 release

* add compare function

* docs and arith lib

* fixup addInt and subInt overload, add docs

* more tests

* cleanup docs

* remove old comments

* remove unnecessary cast

* add isInteger

* add overflow tests

* boilerplate

* refactor expectedResult for tests

* doc typo fix

* returns bool

* add docs link

* different dos link

* add isInteger true case

* expand iff

* add quantity back to 1.28 version, and revert change to DefaultCompatibilityVersion

* formatting
This commit is contained in:
Alex Zielenski 2023-07-13 14:43:56 -07:00 committed by GitHub
parent fc798a8dc1
commit 423f4dfc79
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 747 additions and 0 deletions

View File

@ -78,6 +78,7 @@ var baseOpts = []VersionedOptions{
EnvOptions: []cel.EnvOption{
cel.CrossTypeNumericComparisons(true),
cel.OptionalTypes(),
library.Quantity(),
},
},
// TODO: switch to ext.Strings version 2 once format() is fixed to work with HomogeneousAggregateLiterals.

View File

@ -0,0 +1,375 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package library
import (
"errors"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"k8s.io/apimachinery/pkg/api/resource"
apiservercel "k8s.io/apiserver/pkg/cel"
)
// Quantity provides a CEL function library extension of Kubernetes
// resource.Quantity parsing functions. See `resource.Quantity`
// documentation for more detailed information about the format itself:
// https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#Quantity
//
// quantity
//
// Converts a string to a Quantity or results in an error if the string is not a valid Quantity. Refer
// to resource.Quantity documentation for information on accepted patterns.
//
// quantity(<string>) <Quantity>
//
// Examples:
//
// quantity('1.5G') // returns a Quantity
// quantity('200k') // returns a Quantity
// quantity('200K') // error
// quantity('Three') // error
// quantity('Mi') // error
//
// isQuantity
//
// Returns true if a string is a valid Quantity. isQuantity returns true if and
// only if quantity does not result in error.
//
// isQuantity( <string>) <bool>
//
// Examples:
//
// isQuantity('1.3G') // returns true
// isQuantity('1.3Gi') // returns true
// isQuantity('1,3G') // returns false
// isQuantity('10000k') // returns true
// isQuantity('200K') // returns false
// isQuantity('Three') // returns false
// isQuantity('Mi') // returns false
//
// Conversion to Scalars:
//
// - isInteger: returns true if and only if asInteger is safe to call without an error
//
// - asInteger: returns a representation of the current value as an int64 if
// possible or results in an error if conversion would result in overflow
// or loss of precision.
//
// - asApproximateFloat: returns a float64 representation of the quantity which may
// lose precision. If the value of the quantity is outside the range of a float64
// +Inf/-Inf will be returned.
//
// <Quantity>.isInteger() <bool>
// <Quantity>.asInteger() <int>
// <Quantity>.asApproximateFloat() <float>
//
// Examples:
//
// quantity("50000000G").isInteger() // returns true
// quantity("50k").isInteger() // returns true
// quantity("9999999999999999999999999999999999999G").asInteger() // error: cannot convert value to integer
// quantity("9999999999999999999999999999999999999G").isInteger() // returns false
// quantity("50k").asInteger() == 50000 // returns true
// quantity("50k").sub(20000).asApproximateFloat() == 30000 // returns true
//
// Arithmetic
//
// - sign: Returns `1` if the quantity is positive, `-1` if it is negative. `0` if it is zero
//
// - add: Returns sum of two quantities or a quantity and an integer
//
// - sub: Returns difference between two quantities or a quantity and an integer
//
// <Quantity>.sign() <int>
// <Quantity>.add(<quantity>) <quantity>
// <Quantity>.add(<integer>) <quantity>
// <Quantity>.sub(<quantity>) <quantity>
// <Quantity>.sub(<integer>) <quantity>
//
// Examples:
//
// quantity("50k").add("20k") == quantity("70k") // returns true
// quantity("50k").add(20) == quantity("50020") // returns true
// quantity("50k").sub("20k") == quantity("30k") // returns true
// quantity("50k").sub(20000) == quantity("30k") // returns true
// quantity("50k").add(20).sub(quantity("100k")).sub(-50000) == quantity("20") // returns true
//
// Comparisons
//
// - isGreaterThan: Returns true if and only if the receiver is greater than the operand
//
// - isLessThan: Returns true if and only if the receiver is less than the operand
//
// - compareTo: Compares receiver to operand and returns 0 if they are equal, 1 if the receiver is greater, or -1 if the receiver is less than the operand
//
//
// <Quantity>.isLessThan(<quantity>) <bool>
// <Quantity>.isGreaterThan(<quantity>) <bool>
// <Quantity>.compareTo(<quantity>) <int>
//
// Examples:
//
// quantity("200M").compareTo(quantity("0.2G")) // returns 0
// quantity("50M").compareTo(quantity("50Mi")) // returns -1
// quantity("50Mi").compareTo(quantity("50M")) // returns 1
// quantity("150Mi").isGreaterThan(quantity("100Mi")) // returns true
// quantity("50Mi").isGreaterThan(quantity("100Mi")) // returns false
// quantity("50M").isLessThan(quantity("100M")) // returns true
// quantity("100M").isLessThan(quantity("50M")) // returns false
func Quantity() cel.EnvOption {
return cel.Lib(quantityLib)
}
var quantityLib = &quantity{}
type quantity struct{}
var quantityLibraryDecls = map[string][]cel.FunctionOpt{
"quantity": {
cel.Overload("string_to_quantity", []*cel.Type{cel.StringType}, apiservercel.QuantityType, cel.UnaryBinding((stringToQuantity))),
},
"isQuantity": {
cel.Overload("is_quantity_string", []*cel.Type{cel.StringType}, cel.BoolType, cel.UnaryBinding(isQuantity)),
},
"sign": {
cel.Overload("quantity_sign", []*cel.Type{apiservercel.QuantityType}, cel.IntType, cel.UnaryBinding(quantityGetSign)),
},
"isGreaterThan": {
cel.MemberOverload("quantity_is_greater_than", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, cel.BoolType, cel.BinaryBinding(quantityIsGreaterThan)),
},
"isLessThan": {
cel.MemberOverload("quantity_is_less_than", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, cel.BoolType, cel.BinaryBinding(quantityIsLessThan)),
},
"compareTo": {
cel.MemberOverload("quantity_compare_to", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, cel.IntType, cel.BinaryBinding(quantityCompareTo)),
},
"asApproximateFloat": {
cel.MemberOverload("quantity_get_float", []*cel.Type{apiservercel.QuantityType}, cel.DoubleType, cel.UnaryBinding(quantityGetApproximateFloat)),
},
"asInteger": {
cel.MemberOverload("quantity_get_int", []*cel.Type{apiservercel.QuantityType}, cel.IntType, cel.UnaryBinding(quantityGetValue)),
},
"isInteger": {
cel.MemberOverload("quantity_is_integer", []*cel.Type{apiservercel.QuantityType}, cel.BoolType, cel.UnaryBinding(quantityCanValue)),
},
"add": {
cel.MemberOverload("quantity_add", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, apiservercel.QuantityType, cel.BinaryBinding(quantityAdd)),
cel.MemberOverload("quantity_add_int", []*cel.Type{apiservercel.QuantityType, cel.IntType}, apiservercel.QuantityType, cel.BinaryBinding(quantityAddInt)),
},
"sub": {
cel.MemberOverload("quantity_sub", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, apiservercel.QuantityType, cel.BinaryBinding(quantitySub)),
cel.MemberOverload("quantity_sub_int", []*cel.Type{apiservercel.QuantityType, cel.IntType}, apiservercel.QuantityType, cel.BinaryBinding(quantitySubInt)),
},
}
func (*quantity) CompileOptions() []cel.EnvOption {
options := make([]cel.EnvOption, 0, len(quantityLibraryDecls))
for name, overloads := range quantityLibraryDecls {
options = append(options, cel.Function(name, overloads...))
}
return options
}
func (*quantity) ProgramOptions() []cel.ProgramOption {
return []cel.ProgramOption{}
}
func isQuantity(arg ref.Val) ref.Val {
str, ok := arg.Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
_, err := resource.ParseQuantity(str)
if err != nil {
return types.Bool(false)
}
return types.Bool(true)
}
func stringToQuantity(arg ref.Val) ref.Val {
str, ok := arg.Value().(string)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q, err := resource.ParseQuantity(str)
if err != nil {
return types.WrapErr(err)
}
return apiservercel.Quantity{Quantity: &q}
}
func quantityGetApproximateFloat(arg ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.Double(q.AsApproximateFloat64())
}
func quantityCanValue(arg ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
_, success := q.AsInt64()
return types.Bool(success)
}
func quantityGetValue(arg ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
v, success := q.AsInt64()
if !success {
return types.WrapErr(errors.New("cannot convert value to integer"))
}
return types.Int(v)
}
func quantityGetSign(arg ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.Int(q.Sign())
}
func quantityIsGreaterThan(arg ref.Val, other ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q2, ok := other.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.Bool(q.Cmp(*q2) == 1)
}
func quantityIsLessThan(arg ref.Val, other ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q2, ok := other.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.Bool(q.Cmp(*q2) == -1)
}
func quantityCompareTo(arg ref.Val, other ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q2, ok := other.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
return types.Int(q.Cmp(*q2))
}
func quantityAdd(arg ref.Val, other ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q2, ok := other.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
copy := *q
copy.Add(*q2)
return &apiservercel.Quantity{
Quantity: &copy,
}
}
func quantityAddInt(arg ref.Val, other ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q2, ok := other.Value().(int64)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q2Converted := *resource.NewQuantity(q2, resource.DecimalExponent)
copy := *q
copy.Add(q2Converted)
return &apiservercel.Quantity{
Quantity: &copy,
}
}
func quantitySub(arg ref.Val, other ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q2, ok := other.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
copy := *q
copy.Sub(*q2)
return &apiservercel.Quantity{
Quantity: &copy,
}
}
func quantitySubInt(arg ref.Val, other ref.Val) ref.Val {
q, ok := arg.Value().(*resource.Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q2, ok := other.Value().(int64)
if !ok {
return types.MaybeNoSuchOverloadErr(arg)
}
q2Converted := *resource.NewQuantity(q2, resource.DecimalExponent)
copy := *q
copy.Sub(q2Converted)
return &apiservercel.Quantity{
Quantity: &copy,
}
}

View File

@ -0,0 +1,295 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package library_test
import (
"regexp"
"testing"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"github.com/google/cel-go/ext"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/sets"
apiservercel "k8s.io/apiserver/pkg/cel"
"k8s.io/apiserver/pkg/cel/library"
)
func testQuantity(t *testing.T, expr string, expectResult ref.Val, expectRuntimeErrPattern string, expectCompileErrs []string) {
env, err := cel.NewEnv(
ext.Strings(),
library.URLs(),
library.Regex(),
library.Lists(),
library.Quantity(),
)
if err != nil {
t.Fatalf("%v", err)
}
compiled, issues := env.Compile(expr)
if len(expectCompileErrs) > 0 {
missingCompileErrs := []string{}
matchedCompileErrs := sets.New[int]()
for _, expectedCompileErr := range expectCompileErrs {
compiledPattern, err := regexp.Compile(expectedCompileErr)
if err != nil {
t.Fatalf("failed to compile expected err regex: %v", err)
}
didMatch := false
for i, compileError := range issues.Errors() {
if compiledPattern.Match([]byte(compileError.Message)) {
didMatch = true
matchedCompileErrs.Insert(i)
}
}
if !didMatch {
missingCompileErrs = append(missingCompileErrs, expectedCompileErr)
} else if len(matchedCompileErrs) != len(issues.Errors()) {
unmatchedErrs := []common.Error{}
for i, issue := range issues.Errors() {
if !matchedCompileErrs.Has(i) {
unmatchedErrs = append(unmatchedErrs, issue)
}
}
require.Empty(t, unmatchedErrs, "unexpected compilation errors")
}
}
require.Empty(t, missingCompileErrs, "expected compilation errors")
return
} else if len(issues.Errors()) > 0 {
t.Fatalf("%v", issues.Errors())
}
prog, err := env.Program(compiled)
if err != nil {
t.Fatalf("%v", err)
}
res, _, err := prog.Eval(map[string]interface{}{})
if len(expectRuntimeErrPattern) > 0 {
if err == nil {
t.Fatalf("no runtime error thrown. Expected: %v", expectRuntimeErrPattern)
} else if matched, regexErr := regexp.MatchString(expectRuntimeErrPattern, err.Error()); regexErr != nil {
t.Fatalf("failed to compile expected err regex: %v", regexErr)
} else if !matched {
t.Fatalf("unexpected err: %v", err)
}
} else if err != nil {
t.Fatalf("%v", err)
} else if expectResult != nil {
converted := res.Equal(expectResult).Value().(bool)
require.True(t, converted, "expectation not equal to output")
} else {
t.Fatal("expected result must not be nil")
}
}
func TestQuantity(t *testing.T) {
twelveMi := resource.MustParse("12Mi")
trueVal := types.Bool(true)
falseVal := types.Bool(false)
cases := []struct {
name string
expr string
expectValue ref.Val
expectedCompileErr []string
expectedRuntimeErr string
}{
{
name: "parse",
expr: `quantity("12Mi")`,
expectValue: apiservercel.Quantity{Quantity: &twelveMi},
},
{
name: "parseInvalidSuffix",
expr: `quantity("10Mo")`,
expectedRuntimeErr: "quantities must match the regular expression.*",
},
{
// The above case fails due to a regex check. This case passes the
// regex check and fails a suffix check
name: "parseInvalidSuffixPassesRegex",
expr: `quantity("10Mm")`,
expectedRuntimeErr: "unable to parse quantity's suffix",
},
{
name: "isQuantity",
expr: `isQuantity("20")`,
expectValue: trueVal,
},
{
name: "isQuantity_megabytes",
expr: `isQuantity("20M")`,
expectValue: trueVal,
},
{
name: "isQuantity_mebibytes",
expr: `isQuantity("20Mi")`,
expectValue: trueVal,
},
{
name: "isQuantity_invalidSuffix",
expr: `isQuantity("20Mo")`,
expectValue: falseVal,
},
{
name: "isQuantity_passingRegex",
expr: `isQuantity("10Mm")`,
expectValue: falseVal,
},
{
name: "isQuantity_noOverload",
expr: `isQuantity([1, 2, 3])`,
expectedCompileErr: []string{"found no matching overload for 'isQuantity' applied to.*"},
},
{
name: "equality_reflexivity",
expr: `quantity("200M") == quantity("200M")`,
expectValue: trueVal,
},
{
name: "equality_symmetry",
expr: `quantity("200M") == quantity("0.2G") && quantity("0.2G") == quantity("200M")`,
expectValue: trueVal,
},
{
name: "equality_transitivity",
expr: `quantity("2M") == quantity("0.002G") && quantity("2000k") == quantity("2M") && quantity("0.002G") == quantity("2000k")`,
expectValue: trueVal,
},
{
name: "inequality",
expr: `quantity("200M") == quantity("0.3G")`,
expectValue: falseVal,
},
{
name: "quantity_less",
expr: `quantity("50M").isLessThan(quantity("50Mi"))`,
expectValue: trueVal,
},
{
name: "quantity_less_obvious",
expr: `quantity("50M").isLessThan(quantity("100M"))`,
expectValue: trueVal,
},
{
name: "quantity_less_false",
expr: `quantity("100M").isLessThan(quantity("50M"))`,
expectValue: falseVal,
},
{
name: "quantity_greater",
expr: `quantity("50Mi").isGreaterThan(quantity("50M"))`,
expectValue: trueVal,
},
{
name: "quantity_greater_obvious",
expr: `quantity("150Mi").isGreaterThan(quantity("100Mi"))`,
expectValue: trueVal,
},
{
name: "quantity_greater_false",
expr: `quantity("50M").isGreaterThan(quantity("100M"))`,
expectValue: falseVal,
},
{
name: "compare_equal",
expr: `quantity("200M").compareTo(quantity("0.2G"))`,
expectValue: types.Int(0),
},
{
name: "compare_less",
expr: `quantity("50M").compareTo(quantity("50Mi"))`,
expectValue: types.Int(-1),
},
{
name: "compare_greater",
expr: `quantity("50Mi").compareTo(quantity("50M"))`,
expectValue: types.Int(1),
},
{
name: "add_quantity",
expr: `quantity("50k").add(quantity("20")) == quantity("50.02k")`,
expectValue: trueVal,
},
{
name: "add_int",
expr: `quantity("50k").add(20).isLessThan(quantity("50020"))`,
expectValue: falseVal,
},
{
name: "sub_quantity",
expr: `quantity("50k").sub(quantity("20")) == quantity("49.98k")`,
expectValue: trueVal,
},
{
name: "sub_int",
expr: `quantity("50k").sub(20) == quantity("49980")`,
expectValue: trueVal,
},
{
name: "arith_chain_1",
expr: `quantity("50k").add(20).sub(quantity("100k")).asInteger()`,
expectValue: types.Int(-49980),
},
{
name: "arith_chain",
expr: `quantity("50k").add(20).sub(quantity("100k")).sub(-50000).asInteger()`,
expectValue: types.Int(20),
},
{
name: "as_integer",
expr: `quantity("50k").asInteger()`,
expectValue: types.Int(50000),
},
{
name: "as_integer_error",
expr: `quantity("9999999999999999999999999999999999999G").asInteger()`,
expectedRuntimeErr: `cannot convert value to integer`,
},
{
name: "is_integer",
expr: `quantity("9999999999999999999999999999999999999G").isInteger()`,
expectValue: falseVal,
},
{
name: "is_integer",
expr: `quantity("50").isInteger()`,
expectValue: trueVal,
},
{
name: "as_float",
expr: `quantity("50.703k").asApproximateFloat()`,
expectValue: types.Double(50703),
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
testQuantity(t, c.expr, c.expectValue, c.expectedRuntimeErr, c.expectedCompileErr)
})
}
}

View File

@ -0,0 +1,76 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cel
import (
"fmt"
"reflect"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/checker/decls"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"k8s.io/apimachinery/pkg/api/resource"
)
var (
QuantityObject = decls.NewObjectType("kubernetes.Quantity")
quantityTypeValue = types.NewTypeValue("kubernetes.Quantity")
QuantityType = cel.ObjectType("kubernetes.Quantity")
)
// Quantity provdes a CEL representation of a resource.Quantity
type Quantity struct {
*resource.Quantity
}
func (d Quantity) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
if reflect.TypeOf(d.Quantity).AssignableTo(typeDesc) {
return d.Quantity, nil
}
if reflect.TypeOf("").AssignableTo(typeDesc) {
return d.Quantity.String(), nil
}
return nil, fmt.Errorf("type conversion error from 'Quantity' to '%v'", typeDesc)
}
func (d Quantity) ConvertToType(typeVal ref.Type) ref.Val {
switch typeVal {
case typeValue:
return d
case types.TypeType:
return quantityTypeValue
default:
return types.NewErr("type conversion error from '%s' to '%s'", quantityTypeValue, typeVal)
}
}
func (d Quantity) Equal(other ref.Val) ref.Val {
otherDur, ok := other.(Quantity)
if !ok {
return types.MaybeNoSuchOverloadErr(other)
}
return types.Bool(d.Quantity.Equal(*otherDur.Quantity))
}
func (d Quantity) Type() ref.Type {
return quantityTypeValue
}
func (d Quantity) Value() interface{} {
return d.Quantity
}