github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 04:06:03 +00:00
Code Issues Projects Releases Wiki Activity

implmementing type checking

Browse Source 
with multi-type support.
This commit is contained in:
Jiahui Feng 2023-03-07 15:49:19 -08:00 committed by Indeed
parent 54283a1d38
commit feb18b3f5f
11 changed files with 1059 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/kube-apiserver/app/server.go
View File

@ -40,6 +40,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
"k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
@ -56,6 +57,7 @@ import (
"k8s.io/apiserver/pkg/util/webhook"
clientgoinformers "k8s.io/client-go/informers"
clientgoclientset "k8s.io/client-go/kubernetes"
k8sscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/keyutil"
cliflag "k8s.io/component-base/cli/flag"
@ -452,7 +454,8 @@ func buildGenericConfig(
CloudConfigFile: s.CloudProvider.CloudConfigFile,
}
serviceResolver = buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers)
pluginInitializers, admissionPostStartHook, err = admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider)
schemaResolver := resolver.NewDefinitionsSchemaResolver(k8sscheme.Scheme, genericConfig.OpenAPIConfig.GetDefinitions)
pluginInitializers, admissionPostStartHook, err = admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider, schemaResolver)
if err != nil {
lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err)
return

5
pkg/kubeapiserver/admission/config.go
View File

@ -28,6 +28,7 @@ import (
utilwait "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/admission"
webhookinit "k8s.io/apiserver/pkg/admission/plugin/webhook/initializer"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
genericapiserver "k8s.io/apiserver/pkg/server"
egressselector "k8s.io/apiserver/pkg/server/egressselector"
"k8s.io/apiserver/pkg/util/webhook"
@ -47,7 +48,7 @@ type Config struct {
}
// New sets up the plugins and admission start hooks needed for admission
func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselector.EgressSelector, serviceResolver webhook.ServiceResolver, tp trace.TracerProvider) ([]admission.PluginInitializer, genericapiserver.PostStartHookFunc, error) {
func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselector.EgressSelector, serviceResolver webhook.ServiceResolver, tp trace.TracerProvider, schemaResolver resolver.SchemaResolver) ([]admission.PluginInitializer, genericapiserver.PostStartHookFunc, error) {
webhookAuthResolverWrapper := webhook.NewDefaultAuthenticationInfoResolverWrapper(proxyTransport, egressSelector, c.LoopbackClientConfig, tp)
webhookPluginInitializer := webhookinit.NewPluginInitializer(webhookAuthResolverWrapper, serviceResolver)
@ -63,13 +64,13 @@ func (c *Config) New(proxyTransport *http.Transport, egressSelector *egressselec
if err != nil {
return nil, nil, err
}
discoveryClient := cacheddiscovery.NewMemCacheClient(clientset.Discovery())
discoveryRESTMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
kubePluginInitializer := NewPluginInitializer(
cloudConfig,
discoveryRESTMapper,
quotainstall.NewQuotaConfigurationForAdmission(),
schemaResolver,
)
admissionPostStartHook := func(context genericapiserver.PostStartHookContext) error {

8
pkg/kubeapiserver/admission/initializer.go
View File

@ -20,6 +20,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/initializer"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
quota "k8s.io/apiserver/pkg/quota/v1"
)
@ -35,6 +36,7 @@ type PluginInitializer struct {
cloudConfig []byte
restMapper meta.RESTMapper
quotaConfiguration quota.Configuration
schemaResolver resolver.SchemaResolver
}
var _ admission.PluginInitializer = &PluginInitializer{}
@ -46,11 +48,13 @@ func NewPluginInitializer(
cloudConfig []byte,
restMapper meta.RESTMapper,
quotaConfiguration quota.Configuration,
schemaResolver resolver.SchemaResolver,
) *PluginInitializer {
return &PluginInitializer{
cloudConfig: cloudConfig,
restMapper: restMapper,
quotaConfiguration: quotaConfiguration,
schemaResolver: schemaResolver,
}
}
@ -68,4 +72,8 @@ func (i *PluginInitializer) Initialize(plugin admission.Interface) {
if wants, ok := plugin.(initializer.WantsQuotaConfiguration); ok {
wants.SetQuotaConfiguration(i.quotaConfiguration)
}
if wants, ok := plugin.(initializer.WantsSchemaResolver); ok {
wants.SetSchemaResolver(i.schemaResolver)
}
}

8
staging/src/k8s.io/apiserver/pkg/admission/initializer/interfaces.go
View File

@ -20,6 +20,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
quota "k8s.io/apiserver/pkg/quota/v1"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
@ -81,3 +82,10 @@ type WantsRESTMapper interface {
SetRESTMapper(meta.RESTMapper)
admission.InitializationValidator
}
// WantsSchemaResolver defines a function which sets the SchemaResolver for
// an admission plugin that needs it.
type WantsSchemaResolver interface {
SetSchemaResolver(resolver resolver.SchemaResolver)
admission.InitializationValidator
}

6
staging/src/k8s.io/apiserver/pkg/admission/plugin/cel/compile.go
View File

@ -81,7 +81,7 @@ func buildRequiredVarsEnv() (*cel.Env, error) {
var propDecls []cel.EnvOption
reg := apiservercel.NewRegistry(baseEnv)
requestType := buildRequestType()
requestType := BuildRequestType()
rt, err := apiservercel.NewRuleTypes(requestType.TypeName(), requestType, reg)
if err != nil {
return nil, err
@ -134,11 +134,11 @@ func buildWithOptionalVarsEnvs(requiredVarsEnv *cel.Env) (envs, error) {
return envs, nil
}
// buildRequestType generates a DeclType for AdmissionRequest. This may be replaced with a utility that
// BuildRequestType generates a DeclType for AdmissionRequest. This may be replaced with a utility that
// converts the native type definition to apiservercel.DeclType once such a utility becomes available.
// The 'uid' field is omitted since it is not needed for in-process admission review.
// The 'object' and 'oldObject' fields are omitted since they are exposed as root level CEL variables.
func buildRequestType() *apiservercel.DeclType {
func BuildRequestType() *apiservercel.DeclType {
field := func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField {
return apiservercel.NewDeclField(name, declType, required, nil, nil)
}

10
staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/admission.go
View File

@ -24,6 +24,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
"k8s.io/apiserver/pkg/features"
"k8s.io/client-go/dynamic"
"k8s.io/component-base/featuregate"
@ -73,6 +74,7 @@ type celAdmissionPlugin struct {
dynamicClient dynamic.Interface
stopCh <-chan struct{}
authorizer authorizer.Authorizer
schemaResolver resolver.SchemaResolver
}
var _ initializer.WantsExternalKubeInformerFactory = &celAdmissionPlugin{}
@ -81,7 +83,7 @@ var _ initializer.WantsRESTMapper = &celAdmissionPlugin{}
var _ initializer.WantsDynamicClient = &celAdmissionPlugin{}
var _ initializer.WantsDrainedNotification = &celAdmissionPlugin{}
var _ initializer.WantsAuthorizer = &celAdmissionPlugin{}
var _ initializer.WantsSchemaResolver = &celAdmissionPlugin{}
var _ admission.InitializationValidator = &celAdmissionPlugin{}
var _ admission.ValidationInterface = &celAdmissionPlugin{}
@ -115,6 +117,10 @@ func (c *celAdmissionPlugin) SetAuthorizer(authorizer authorizer.Authorizer) {
c.authorizer = authorizer
}
func (c *celAdmissionPlugin) SetSchemaResolver(resolver resolver.SchemaResolver) {
c.schemaResolver = resolver
}
func (c *celAdmissionPlugin) InspectFeatureGates(featureGates featuregate.FeatureGate) {
if featureGates.Enabled(features.ValidatingAdmissionPolicy) {
c.enabled = true
@ -148,7 +154,7 @@ func (c *celAdmissionPlugin) ValidateInitialization() error {
if c.authorizer == nil {
return errors.New("missing authorizer")
}
c.evaluator = NewAdmissionController(c.informerFactory, c.client, c.restMapper, c.dynamicClient, c.authorizer)
c.evaluator = NewAdmissionController(c.informerFactory, c.client, c.restMapper, c.schemaResolver /* (optional) */, c.dynamicClient, c.authorizer)
if err := c.evaluator.ValidateInitialization(); err != nil {
return err
}

7
staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller.go
View File

@ -43,6 +43,7 @@ import (
"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matching"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
"k8s.io/apiserver/pkg/warning"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
@ -124,15 +125,21 @@ func NewAdmissionController(
informerFactory informers.SharedInformerFactory,
client kubernetes.Interface,
restMapper meta.RESTMapper,
schemaResolver resolver.SchemaResolver,
dynamicClient dynamic.Interface,
authz authorizer.Authorizer,
) CELPolicyEvaluator {
var typeChecker *TypeChecker
if schemaResolver != nil {
typeChecker = &TypeChecker{schemaResolver: schemaResolver, restMapper: restMapper}
}
return &celAdmissionController{
definitions: atomic.Value{},
policyController: newPolicyController(
restMapper,
client,
dynamicClient,
typeChecker,
cel.NewFilterCompiler(),
NewMatcher(matching.NewMatcher(informerFactory.Core().V1().Namespaces().Lister(), client)),
generic.NewInformer[*v1alpha1.ValidatingAdmissionPolicy](

43
staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller_reconcile.go
View File

@ -27,8 +27,10 @@ import (
corev1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
celmetrics "k8s.io/apiserver/pkg/admission/cel"
"k8s.io/apiserver/pkg/admission/plugin/cel"
@ -59,6 +61,11 @@ type policyController struct {
newValidator
// The TypeCheck checks the policy's expressions for type errors.
// Type of params is defined in policy.Spec.ParamsKind
// Types of object are calculated from policy.Spec.MatchingConstraints
typeChecker *TypeChecker
// Lock which protects:
// - cachedPolicies
// - paramCRDControllers
@ -98,6 +105,7 @@ func newPolicyController(
restMapper meta.RESTMapper,
client kubernetes.Interface,
dynamicClient dynamic.Interface,
typeChecker *TypeChecker,
filterCompiler cel.FilterCompiler,
matcher Matcher,
policiesInformer generic.Informer[*v1alpha1.ValidatingAdmissionPolicy],
@ -107,6 +115,7 @@ func newPolicyController(
res := &policyController{}
*res = policyController{
filterCompiler: filterCompiler,
typeChecker: typeChecker,
definitionInfo: make(map[namespacedName]*definitionInfo),
bindingInfos: make(map[namespacedName]*bindingInfo),
paramsCRDControllers: make(map[v1alpha1.ParamKind]*paramInfo),
@ -168,7 +177,17 @@ func (c *policyController) HasSynced() bool {
func (c *policyController) reconcilePolicyDefinition(namespace, name string, definition *v1alpha1.ValidatingAdmissionPolicy) error {
c.mutex.Lock()
defer c.mutex.Unlock()
err := c.reconcilePolicyDefinitionSpec(namespace, name, definition)
if err != nil {
return err
}
if c.typeChecker != nil {
err = c.reconcilePolicyStatus(namespace, name, definition)
}
return err
}
func (c *policyController) reconcilePolicyDefinitionSpec(namespace, name string, definition *v1alpha1.ValidatingAdmissionPolicy) error {
c.cachedPolicies = nil // invalidate cachedPolicies
// Namespace for policydefinition is empty.
@ -423,6 +442,30 @@ func (c *policyController) reconcilePolicyBinding(namespace, name string, bindin
return nil
}
func (c *policyController) reconcilePolicyStatus(namespace, name string, definition *v1alpha1.ValidatingAdmissionPolicy) error {
if definition != nil && definition.Status.ObservedGeneration < definition.Generation {
st := c.calculatePolicyStatus(definition)
newDefinition := definition.DeepCopy()
newDefinition.Status = *st
_, err := c.client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies().UpdateStatus(c.context, newDefinition, metav1.UpdateOptions{})
if err != nil {
// ignore error when the controller is not able to
// mutate the definition, and to avoid infinite requeue.
utilruntime.HandleError(err)
}
}
return nil
}
func (c *policyController) calculatePolicyStatus(definition *v1alpha1.ValidatingAdmissionPolicy) *v1alpha1.ValidatingAdmissionPolicyStatus {
expressionWarnings := c.typeChecker.Check(definition)
// modifying a deepcopy of the original status, preserving unrelated existing data
status := definition.Status.DeepCopy()
status.ObservedGeneration = definition.Generation
status.TypeChecking = &v1alpha1.TypeChecking{ExpressionWarnings: expressionWarnings}
return status
}
func (c *policyController) reconcileParams(namespace, name string, params runtime.Object) error {
// Do nothing.
// When we add informational type checking we will need to compile in the

435
staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/typechecking.go Normal file
View File

@ -0,0 +1,435 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"errors"
"fmt"
"sort"
"strings"
"sync"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/common/types/ref"
"k8s.io/api/admissionregistration/v1alpha1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
plugincel "k8s.io/apiserver/pkg/admission/plugin/cel"
apiservercel "k8s.io/apiserver/pkg/cel"
"k8s.io/apiserver/pkg/cel/common"
"k8s.io/apiserver/pkg/cel/library"
"k8s.io/apiserver/pkg/cel/openapi"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
"k8s.io/klog/v2"
)
const maxTypesToCheck = 10
type TypeChecker struct {
schemaResolver resolver.SchemaResolver
restMapper meta.RESTMapper
}
type typeOverwrite struct {
object *apiservercel.DeclType
params *apiservercel.DeclType
}
// typeCheckingResult holds the issues found during type checking, any returned
// error, and the gvk that the type checking is performed against.
type typeCheckingResult struct {
gvk schema.GroupVersionKind
issues *cel.Issues
err error
}
// Check preforms the type check against the given policy, and format the result
// as []ExpressionWarning that is ready to be set in policy.Status
// The result is nil if type checking returns no warning.
// The policy object is NOT mutated. The caller should update Status accordingly
func (c *TypeChecker) Check(policy *v1alpha1.ValidatingAdmissionPolicy) []v1alpha1.ExpressionWarning {
exps := make([]string, 0, len(policy.Spec.Validations))
// check main validation expressions, located in spec.validations[*]
fieldRef := field.NewPath("spec", "validations")
for _, v := range policy.Spec.Validations {
exps = append(exps, v.Expression)
}
msgs := c.CheckExpressions(exps, policy.Spec.ParamKind != nil, policy)
var results []v1alpha1.ExpressionWarning // intentionally not setting capacity
for i, msg := range msgs {
if msg != "" {
results = append(results, v1alpha1.ExpressionWarning{
FieldRef: fieldRef.Index(i).Child("expression").String(),
Warning: msg,
})
}
}
return results
}
// CheckExpressions checks a set of compiled CEL programs against the GVKs defined in
// policy.Spec.MatchConstraints
// The result is a human-readable form that describe which expressions
// violate what types at what place. The indexes of the return []string
// matches these of the input expressions.
// TODO: It is much more useful to have machine-readable output and let the
// client format it. That requires an update to the KEP, probably in coming
// releases.
func (c *TypeChecker) CheckExpressions(expressions []string, hasParams bool, policy *v1alpha1.ValidatingAdmissionPolicy) []string {
var allWarnings []string
allGvks := c.typesToCheck(policy)
gvks := make([]schema.GroupVersionKind, 0, len(allGvks))
schemas := make([]common.Schema, 0, len(allGvks))
for _, gvk := range allGvks {
s, err := c.schemaResolver.ResolveSchema(gvk)
if err != nil {
// type checking errors MUST NOT alter the behavior of the policy
// even if an error occurs.
if !errors.Is(err, resolver.ErrSchemaNotFound) {
// Anything except ErrSchemaNotFound is an internal error
klog.ErrorS(err, "internal error: schema resolution failure", "gvk", gvk)
}
// skip if an unrecoverable error occurs.
continue
}
gvks = append(gvks, gvk)
schemas = append(schemas, &openapi.Schema{Schema: s})
}
paramsType := c.paramsType(policy)
paramsDeclType, err := c.declType(paramsType)
if err != nil {
if !errors.Is(err, resolver.ErrSchemaNotFound) {
klog.V(2).ErrorS(err, "cannot resolve schema for params", "gvk", paramsType)
}
paramsDeclType = nil
}
for _, exp := range expressions {
var results []typeCheckingResult
for i, gvk := range gvks {
s := schemas[i]
issues, err := c.checkExpression(exp, hasParams, typeOverwrite{
object: common.SchemaDeclType(s, true),
params: paramsDeclType,
})
// save even if no issues are found, for the sake of formatting.
results = append(results, typeCheckingResult{
gvk: gvk,
issues: issues,
err: err,
})
}
allWarnings = append(allWarnings, c.formatWarning(results))
}
return allWarnings
}
// formatWarning converts the resulting issues and possible error during
// type checking into a human-readable string
func (c *TypeChecker) formatWarning(results []typeCheckingResult) string {
var sb strings.Builder
for _, result := range results {
if result.issues == nil && result.err == nil {
continue
}
if result.err != nil {
sb.WriteString(fmt.Sprintf("%v: type checking error: %v\n", result.gvk, result.err))
} else {
sb.WriteString(fmt.Sprintf("%v: %s\n", result.gvk, result.issues))
}
}
return strings.TrimSuffix(sb.String(), "\n")
}
func (c *TypeChecker) declType(gvk schema.GroupVersionKind) (*apiservercel.DeclType, error) {
if gvk.Empty() {
return nil, nil
}
s, err := c.schemaResolver.ResolveSchema(gvk)
if err != nil {
return nil, err
}
return common.SchemaDeclType(&openapi.Schema{Schema: s}, true), nil
}
func (c *TypeChecker) paramsType(policy *v1alpha1.ValidatingAdmissionPolicy) schema.GroupVersionKind {
if policy.Spec.ParamKind == nil {
return schema.GroupVersionKind{}
}
gv, err := schema.ParseGroupVersion(policy.Spec.ParamKind.APIVersion)
if err != nil {
return schema.GroupVersionKind{}
}
return gv.WithKind(policy.Spec.ParamKind.Kind)
}
func (c *TypeChecker) checkExpression(expression string, hasParams bool, types typeOverwrite) (*cel.Issues, error) {
env, err := buildEnv(hasParams, types)
if err != nil {
return nil, err
}
// We cannot reuse an AST that is parsed by another env, so reparse it here.
// Compile = Parse + Check, we especially want the results of Check.
//
// Paradoxically, we discard the type-checked result and let the admission
// controller use the dynamic typed program.
// This is a compromise that is defined in the KEP. We can revisit this
// decision and expect a change with limited size.
_, issues := env.Compile(expression)
return issues, nil
}
// typesToCheck extracts a list of GVKs that needs type checking from the policy
// the result is sorted in the order of Group, Version, and Kind
func (c *TypeChecker) typesToCheck(p *v1alpha1.ValidatingAdmissionPolicy) []schema.GroupVersionKind {
gvks := sets.New[schema.GroupVersionKind]()
if p.Spec.MatchConstraints == nil || len(p.Spec.MatchConstraints.ResourceRules) == 0 {
return nil
}
for _, rule := range p.Spec.MatchConstraints.ResourceRules {
groups := extractGroups(&rule.Rule)
if len(groups) == 0 {
continue
}
versions := extractVersions(&rule.Rule)
if len(versions) == 0 {
continue
}
resources := extractResources(&rule.Rule)
if len(resources) == 0 {
continue
}
// sort GVRs so that the loop below provides
// consistent results.
sort.Strings(groups)
sort.Strings(versions)
sort.Strings(resources)
count := 0
for _, group := range groups {
for _, version := range versions {
for _, resource := range resources {
gvr := schema.GroupVersionResource{
Group: group,
Version: version,
Resource: resource,
}
resolved, err := c.restMapper.KindsFor(gvr)
if err != nil {
continue
}
for _, r := range resolved {
if !r.Empty() {
gvks.Insert(r)
count++
// early return if maximum number of types are already
// collected
if count == maxTypesToCheck {
if gvks.Len() == 0 {
return nil
}
return sortGVKList(gvks.UnsortedList())
}
}
}
}
}
}
}
if gvks.Len() == 0 {
return nil
}
return sortGVKList(gvks.UnsortedList())
}
func extractGroups(rule *v1alpha1.Rule) []string {
groups := make([]string, 0, len(rule.APIGroups))
for _, group := range rule.APIGroups {
// give up if wildcard
if strings.ContainsAny(group, "*") {
return nil
}
groups = append(groups, group)
}
return groups
}
func extractVersions(rule *v1alpha1.Rule) []string {
versions := make([]string, 0, len(rule.APIVersions))
for _, version := range rule.APIVersions {
if strings.ContainsAny(version, "*") {
return nil
}
versions = append(versions, version)
}
return versions
}
func extractResources(rule *v1alpha1.Rule) []string {
resources := make([]string, 0, len(rule.Resources))
for _, resource := range rule.Resources {
// skip wildcard and subresources
if strings.ContainsAny(resource, "*/") {
continue
}
resources = append(resources, resource)
}
return resources
}
// sortGVKList sorts the list by Group, Version, and Kind
// returns the list itself.
func sortGVKList(list []schema.GroupVersionKind) []schema.GroupVersionKind {
sort.Slice(list, func(i, j int) bool {
if g := strings.Compare(list[i].Group, list[j].Group); g != 0 {
return g < 0
}
if v := strings.Compare(list[i].Version, list[j].Version); v != 0 {
return v < 0
}
return strings.Compare(list[i].Kind, list[j].Kind) < 0
})
return list
}
func buildEnv(hasParams bool, types typeOverwrite) (*cel.Env, error) {
baseEnv, err := getBaseEnv()
if err != nil {
return nil, err
}
reg := apiservercel.NewRegistry(baseEnv)
requestType := plugincel.BuildRequestType()
var varOpts []cel.EnvOption
var rts []*apiservercel.RuleTypes
// request, hand-crafted type
rt, opts, err := createRuleTypesAndOptions(reg, requestType, plugincel.RequestVarName)
if err != nil {
return nil, err
}
rts = append(rts, rt)
varOpts = append(varOpts, opts...)
// object and oldObject, same type, type(s) resolved from constraints
rt, opts, err = createRuleTypesAndOptions(reg, types.object, plugincel.ObjectVarName, plugincel.OldObjectVarName)
if err != nil {
return nil, err
}
rts = append(rts, rt)
varOpts = append(varOpts, opts...)
// params, defined by ParamKind
if hasParams {
rt, opts, err := createRuleTypesAndOptions(reg, types.params, plugincel.ParamsVarName)
if err != nil {
return nil, err
}
rts = append(rts, rt)
varOpts = append(varOpts, opts...)
}
opts, err = ruleTypesOpts(rts, baseEnv.TypeProvider())
if err != nil {
return nil, err
}
opts = append(opts, varOpts...) // add variables after ruleTypes.
env, err := baseEnv.Extend(opts...)
if err != nil {
return nil, err
}
return env, nil
}
// createRuleTypeAndOptions creates the cel RuleTypes and a slice of EnvOption
// that can be used for creating a CEL env containing variables of declType.
// declType can be nil, in which case the variables will be of DynType.
func createRuleTypesAndOptions(registry *apiservercel.Registry, declType *apiservercel.DeclType, variables ...string) (*apiservercel.RuleTypes, []cel.EnvOption, error) {
opts := make([]cel.EnvOption, 0, len(variables))
// untyped, use DynType
if declType == nil {
for _, v := range variables {
opts = append(opts, cel.Variable(v, cel.DynType))
}
return nil, opts, nil
}
// create a RuleType for the given type
rt, err := apiservercel.NewRuleTypes(declType.TypeName(), declType, registry)
if err != nil {
return nil, nil, err
}
if rt == nil {
return nil, nil, nil
}
for _, v := range variables {
opts = append(opts, cel.Variable(v, declType.CelType()))
}
return rt, opts, nil
}
func ruleTypesOpts(ruleTypes []*apiservercel.RuleTypes, underlyingTypeProvider ref.TypeProvider) ([]cel.EnvOption, error) {
var providers []ref.TypeProvider // may be unused, too small to matter
var adapters []ref.TypeAdapter
for _, rt := range ruleTypes {
if rt != nil {
withTP, err := rt.WithTypeProvider(underlyingTypeProvider)
if err != nil {
return nil, err
}
providers = append(providers, withTP)
adapters = append(adapters, withTP)
}
}
var tp ref.TypeProvider
var ta ref.TypeAdapter
switch len(providers) {
case 0:
return nil, nil
case 1:
tp = providers[0]
ta = adapters[0]
default:
tp = &apiservercel.CompositedTypeProvider{Providers: providers}
ta = &apiservercel.CompositedTypeAdapter{Adapters: adapters}
}
return []cel.EnvOption{cel.CustomTypeProvider(tp), cel.CustomTypeAdapter(ta)}, nil
}
func getBaseEnv() (*cel.Env, error) {
typeCheckingBaseEnvInit.Do(func() {
var opts []cel.EnvOption
opts = append(opts, cel.HomogeneousAggregateLiterals())
// Validate function declarations once during base env initialization,
// so they don't need to be evaluated each time a CEL rule is compiled.
// This is a relatively expensive operation.
opts = append(opts, cel.EagerlyValidateDeclarations(true), cel.DefaultUTCTimeZone(true))
opts = append(opts, library.ExtensionLibs...)
typeCheckingBaseEnv, typeCheckingBaseEnvError = cel.NewEnv(opts...)
})
return typeCheckingBaseEnv, typeCheckingBaseEnvError
}
var typeCheckingBaseEnv *cel.Env
var typeCheckingBaseEnvError error
var typeCheckingBaseEnvInit sync.Once

409
staging/src/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/typechecking_test.go Normal file
View File

@ -0,0 +1,409 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validatingadmissionpolicy
import (
"fmt"
"reflect"
"strings"
"testing"
"k8s.io/api/admissionregistration/v1alpha1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/cel/openapi/resolver"
"k8s.io/kube-openapi/pkg/validation/spec"
)
func TestExtractTypeNames(t *testing.T) {
for _, tc := range []struct {
name string
policy *v1alpha1.ValidatingAdmissionPolicy
expected []schema.GroupVersionKind // must be sorted
}{
{
name: "empty",
policy: &v1alpha1.ValidatingAdmissionPolicy{},
expected: nil,
},
{
name: "specific",
policy: &v1alpha1.ValidatingAdmissionPolicy{Spec: v1alpha1.ValidatingAdmissionPolicySpec{
MatchConstraints: &v1alpha1.MatchResources{ResourceRules: []v1alpha1.NamedRuleWithOperations{
{
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
Resources: []string{"deployments"},
},
},
},
}},
}},
expected: []schema.GroupVersionKind{{
Group: "apps",
Version: "v1",
Kind: "Deployment",
}},
},
{
name: "multiple",
policy: &v1alpha1.ValidatingAdmissionPolicy{Spec: v1alpha1.ValidatingAdmissionPolicySpec{
MatchConstraints: &v1alpha1.MatchResources{ResourceRules: []v1alpha1.NamedRuleWithOperations{
{
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
Resources: []string{"deployments"},
},
},
}, {
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{""},
APIVersions: []string{"v1"},
Resources: []string{"pods"},
},
},
},
}},
}},
expected: []schema.GroupVersionKind{
{
Version: "v1",
Kind: "Pod",
}, {
Group: "apps",
Version: "v1",
Kind: "Deployment",
}},
},
{
name: "all resources",
policy: &v1alpha1.ValidatingAdmissionPolicy{Spec: v1alpha1.ValidatingAdmissionPolicySpec{
MatchConstraints: &v1alpha1.MatchResources{ResourceRules: []v1alpha1.NamedRuleWithOperations{
{
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
Resources: []string{"*"},
},
},
},
}},
}},
expected: nil,
},
{
name: "sub resources",
policy: &v1alpha1.ValidatingAdmissionPolicy{Spec: v1alpha1.ValidatingAdmissionPolicySpec{
MatchConstraints: &v1alpha1.MatchResources{ResourceRules: []v1alpha1.NamedRuleWithOperations{
{
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
Resources: []string{"pods/*"},
},
},
},
}},
}},
expected: nil,
},
{
name: "mixtures",
policy: &v1alpha1.ValidatingAdmissionPolicy{Spec: v1alpha1.ValidatingAdmissionPolicySpec{
MatchConstraints: &v1alpha1.MatchResources{ResourceRules: []v1alpha1.NamedRuleWithOperations{
{
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
Resources: []string{"deployments"},
},
},
},
{
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{"apps"},
APIVersions: []string{"*"},
Resources: []string{"deployments"},
},
},
},
}},
}},
expected: []schema.GroupVersionKind{{
Group: "apps",
Version: "v1",
Kind: "Deployment",
}},
},
} {
t.Run(tc.name, func(t *testing.T) {
typeChecker := buildTypeChecker(nil)
got := typeChecker.typesToCheck(tc.policy)
if !reflect.DeepEqual(tc.expected, got) {
t.Errorf("expected %v but got %v", tc.expected, got)
}
})
}
}
func TestTypeCheck(t *testing.T) {
deploymentPolicy := &v1alpha1.ValidatingAdmissionPolicy{Spec: v1alpha1.ValidatingAdmissionPolicySpec{
Validations: []v1alpha1.Validation{
{
Expression: "object.foo == 'bar'",
},
},
MatchConstraints: &v1alpha1.MatchResources{ResourceRules: []v1alpha1.NamedRuleWithOperations{
{
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
Resources: []string{"deployments"},
},
},
},
}},
}}
multiExpressionPolicy := &v1alpha1.ValidatingAdmissionPolicy{Spec: v1alpha1.ValidatingAdmissionPolicySpec{
Validations: []v1alpha1.Validation{
{
Expression: "object.foo == 'bar'",
},
{
Expression: "object.bar == 'foo'",
},
},
MatchConstraints: &v1alpha1.MatchResources{ResourceRules: []v1alpha1.NamedRuleWithOperations{
{
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
Resources: []string{"deployments"},
},
},
},
}},
}}
paramsRefPolicy := &v1alpha1.ValidatingAdmissionPolicy{Spec: v1alpha1.ValidatingAdmissionPolicySpec{
ParamKind: &v1alpha1.ParamKind{
APIVersion: "v1",
Kind: "DoesNotMatter",
},
Validations: []v1alpha1.Validation{
{
Expression: "object.foo == params.bar",
},
},
MatchConstraints: &v1alpha1.MatchResources{ResourceRules: []v1alpha1.NamedRuleWithOperations{
{
RuleWithOperations: v1alpha1.RuleWithOperations{
Rule: v1alpha1.Rule{
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
Resources: []string{"deployments"},
},
},
},
}},
}}
for _, tc := range []struct {
name string
schemaToReturn *spec.Schema
policy *v1alpha1.ValidatingAdmissionPolicy
assertions []assertionFunc
}{
{
name: "empty",
policy: &v1alpha1.ValidatingAdmissionPolicy{},
assertions: []assertionFunc{toBeEmpty},
},
{
name: "unresolved schema",
policy: deploymentPolicy,
schemaToReturn: nil,
assertions: []assertionFunc{toBeEmpty},
},
{
name: "passed check",
policy: deploymentPolicy,
schemaToReturn: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"foo": *spec.StringProperty(),
},
},
},
assertions: []assertionFunc{toBeEmpty},
},
{
name: "undefined field",
policy: deploymentPolicy,
schemaToReturn: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"bar": *spec.StringProperty(),
},
},
},
assertions: []assertionFunc{
toHaveFieldRef("spec.validations[0].expression"),
toContain(`undefined field 'foo'`),
},
},
{
name: "field type mismatch",
policy: deploymentPolicy,
schemaToReturn: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"foo": *spec.Int64Property(),
},
},
},
assertions: []assertionFunc{
toHaveFieldRef("spec.validations[0].expression"),
toContain(`found no matching overload`),
},
},
{
name: "params",
policy: paramsRefPolicy,
schemaToReturn: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"foo": *spec.StringProperty(),
},
},
},
assertions: []assertionFunc{
toHaveFieldRef("spec.validations[0].expression"),
toContain(`undefined field 'bar'`),
},
},
{
name: "multiple expressions",
policy: multiExpressionPolicy,
schemaToReturn: &spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"foo": *spec.StringProperty(),
},
},
},
assertions: []assertionFunc{
toHaveFieldRef("spec.validations[1].expression"), // expressions[0] is okay, [1] is wrong
toHaveLengthOf(1),
},
},
} {
t.Run(tc.name, func(t *testing.T) {
typeChecker := buildTypeChecker(tc.schemaToReturn)
warnings := typeChecker.Check(tc.policy)
for _, a := range tc.assertions {
a(warnings, t)
}
})
}
}
func buildTypeChecker(schemaToReturn *spec.Schema) *TypeChecker {
restMapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{
{
Group: "",
Version: "v1",
},
})
restMapper.Add(must3(scheme.ObjectKinds(&corev1.Pod{}))[0], meta.RESTScopeRoot)
restMapper.Add(must3(scheme.ObjectKinds(&appsv1.Deployment{}))[0], meta.RESTScopeRoot)
return &TypeChecker{
schemaResolver: &fakeSchemaResolver{schemaToReturn: schemaToReturn},
restMapper: restMapper,
}
}
type fakeSchemaResolver struct {
schemaToReturn *spec.Schema
}
func (r *fakeSchemaResolver) ResolveSchema(gvk schema.GroupVersionKind) (*spec.Schema, error) {
if r.schemaToReturn == nil {
return nil, fmt.Errorf("cannot resolve for %v: %w", gvk, resolver.ErrSchemaNotFound)
}
return r.schemaToReturn, nil
}
func toBeEmpty(warnings []v1alpha1.ExpressionWarning, t *testing.T) {
if len(warnings) != 0 {
t.Fatalf("expected empty but got %v", warnings)
}
}
func toContain(substring string) func(warnings []v1alpha1.ExpressionWarning, t *testing.T) {
return func(warnings []v1alpha1.ExpressionWarning, t *testing.T) {
if len(warnings) == 0 {
t.Errorf("expected containing %q but got empty", substring)
}
for i, w := range warnings {
if !strings.Contains(w.Warning, substring) {
t.Errorf("warning %d does not contain %q, got %v", i, substring, w)
}
}
}
}
func toHaveLengthOf(expected int) func(warnings []v1alpha1.ExpressionWarning, t *testing.T) {
return func(warnings []v1alpha1.ExpressionWarning, t *testing.T) {
got := len(warnings)
if expected != got {
t.Errorf("expect warnings to have length of %d, but got %d", expected, got)
}
}
}
func toHaveFieldRef(paths ...string) func(warnings []v1alpha1.ExpressionWarning, t *testing.T) {
return func(warnings []v1alpha1.ExpressionWarning, t *testing.T) {
if len(paths) != len(warnings) {
t.Errorf("expect warnings to have length of %d, but got %d", len(paths), len(warnings))
}
for i := range paths {
if paths[i] != warnings[i].FieldRef {
t.Errorf("wrong fieldRef at %d, expected %q but got %q", i, paths[i], warnings[i].FieldRef)
}
}
}
}
type assertionFunc func(warnings []v1alpha1.ExpressionWarning, t *testing.T)

131
test/integration/apiserver/cel/validatingadmissionpolicy_test.go
View File

@ -2611,6 +2611,26 @@ func withPolicyExistsLabels(labels []string, policy *admissionregistrationv1alph
return policy
}
func withGVRMatch(groups []string, versions []string, resources []string, policy *admissionregistrationv1alpha1.ValidatingAdmissionPolicy) *admissionregistrationv1alpha1.ValidatingAdmissionPolicy {
policy.Spec.MatchConstraints = &admissionregistrationv1alpha1.MatchResources{
ResourceRules: []admissionregistrationv1alpha1.NamedRuleWithOperations{
{
RuleWithOperations: admissionregistrationv1alpha1.RuleWithOperations{
Operations: []admissionregistrationv1.OperationType{
"*",
},
Rule: admissionregistrationv1.Rule{
APIGroups: groups,
APIVersions: versions,
Resources: resources,
},
},
},
},
}
return policy
}
func withValidations(validations []admissionregistrationv1alpha1.Validation, policy *admissionregistrationv1alpha1.ValidatingAdmissionPolicy) *admissionregistrationv1alpha1.ValidatingAdmissionPolicy {
policy.Spec.Validations = validations
return policy
@ -2885,3 +2905,114 @@ rules:
resources: ["configmaps"]
`
)
func TestValidatingAdmissionPolicyTypeChecking(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.ValidatingAdmissionPolicy, true)()
server, err := apiservertesting.StartTestServer(t, nil, []string{
"--enable-admission-plugins", "ValidatingAdmissionPolicy",
}, framework.SharedEtcd())
if err != nil {
t.Fatal(err)
}
defer server.TearDownFn()
config := server.ClientConfig
client, err := clientset.NewForConfig(config)
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
policy *admissionregistrationv1alpha1.ValidatingAdmissionPolicy
assertFieldRef func(warnings []admissionregistrationv1alpha1.ExpressionWarning, t *testing.T) // warning.fieldRef
assertWarnings func(warnings []admissionregistrationv1alpha1.ExpressionWarning, t *testing.T) // warning.warning
}{
{
name: "deployment with correct expression",
policy: withGVRMatch([]string{"apps"}, []string{"v1"}, []string{"deployments"}, withValidations([]admissionregistrationv1alpha1.Validation{
{
Expression: "object.spec.replicas > 1",
},
}, makePolicy("replicated-deployment"))),
assertFieldRef: toHasLengthOf(0),
assertWarnings: toHasLengthOf(0),
},
{
name: "deployment with type confusion",
policy: withGVRMatch([]string{"apps"}, []string{"v1"}, []string{"deployments"}, withValidations([]admissionregistrationv1alpha1.Validation{
{
Expression: "object.spec.replicas < 100", // this one passes
},
{
Expression: "object.spec.replicas > '1'", // '1' should be int
},
}, makePolicy("confused-deployment"))),
assertFieldRef: toBe("spec.validations[1].expression"),
assertWarnings: toHasSubstring(`found no matching overload for '_>_' applied to '(int, string)'`),
},
} {
t.Run(tc.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
policy, err := client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies().Create(ctx, tc.policy, metav1.CreateOptions{})
if err != nil {
t.Fatal(err)
}
defer client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies().Delete(context.Background(), policy.Name, metav1.DeleteOptions{})
err = wait.PollImmediateWithContext(ctx, time.Second, time.Minute, func(ctx context.Context) (done bool, err error) {
name := policy.Name
// wait until the typeChecking is set, which means the type checking
// is complete.
updated, err := client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies().Get(ctx, name, metav1.GetOptions{})
if err != nil {
return false, err
}
if updated.Status.TypeChecking != nil {
policy = updated
return true, nil
}
return false, nil
})
if err != nil {
t.Fatal(err)
}
tc.assertFieldRef(policy.Status.TypeChecking.ExpressionWarnings, t)
tc.assertWarnings(policy.Status.TypeChecking.ExpressionWarnings, t)
})
}
}
func toBe(expected ...string) func(warnings []admissionregistrationv1alpha1.ExpressionWarning, t *testing.T) {
return func(warnings []admissionregistrationv1alpha1.ExpressionWarning, t *testing.T) {
if len(expected) != len(warnings) {
t.Fatalf("mismatched length, expect %d, got %d", len(expected), len(warnings))
}
for i := range expected {
if expected[i] != warnings[i].FieldRef {
t.Errorf("expected %q but got %q", expected[i], warnings[i].FieldRef)
}
}
}
}
func toHasSubstring(substrings ...string) func(warnings []admissionregistrationv1alpha1.ExpressionWarning, t *testing.T) {
return func(warnings []admissionregistrationv1alpha1.ExpressionWarning, t *testing.T) {
if len(substrings) != len(warnings) {
t.Fatalf("mismatched length, expect %d, got %d", len(substrings), len(warnings))
}
for i := range substrings {
if !strings.Contains(warnings[i].Warning, substrings[i]) {
t.Errorf("missing expected substring %q in %v", substrings[i], warnings[i])
}
}
}
}
func toHasLengthOf(n int) func(warnings []admissionregistrationv1alpha1.ExpressionWarning, t *testing.T) {
return func(warnings []admissionregistrationv1alpha1.ExpressionWarning, t *testing.T) {
if n != len(warnings) {
t.Fatalf("mismatched length, expect %d, got %d", n, len(warnings))
}
}
}