From 247dab3578d2fc36001c99bd57b8d563051c41fc Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Sat, 25 May 2019 12:24:59 -0400 Subject: [PATCH] introduce RuntimeClass admission controller Signed-off-by: Eric Ernst --- .../pkg/admission/runtimeclass/admission.go | 214 +++++++++++++++ .../admission/runtimeclass/admission_test.go | 243 ++++++++++++++++++ 2 files changed, 457 insertions(+) create mode 100644 plugin/pkg/admission/runtimeclass/admission.go create mode 100644 plugin/pkg/admission/runtimeclass/admission_test.go diff --git a/plugin/pkg/admission/runtimeclass/admission.go b/plugin/pkg/admission/runtimeclass/admission.go new file mode 100644 index 00000000000..aa06f9343b5 --- /dev/null +++ b/plugin/pkg/admission/runtimeclass/admission.go @@ -0,0 +1,214 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package runtimeclass contains an admission controller for modifying and validating new Pods to +// take RuntimeClass into account. For RuntimeClass definitions which describe an overhead associated +// with running a pod, this admission controller will set the pod.Spec.Overhead field accordingly. This +// field should only be set through this controller, so vaidation will be carried out to ensure the pod's +// value matches what is defined in the coresponding RuntimeClass. +package runtimeclass + +import ( + "fmt" + "io" + + v1beta1 "k8s.io/api/node/v1beta1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apiserver/pkg/admission" + genericadmissioninitailizer "k8s.io/apiserver/pkg/admission/initializer" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" + nodev1beta1listers "k8s.io/client-go/listers/node/v1beta1" + api "k8s.io/kubernetes/pkg/apis/core" + node "k8s.io/kubernetes/pkg/apis/node" + nodev1beta1 "k8s.io/kubernetes/pkg/apis/node/v1beta1" + "k8s.io/kubernetes/pkg/features" +) + +// PluginName indicates name of admission plugin. +const PluginName = "RuntimeClass" + +// Register registers a plugin +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { + return NewRuntimeClass(), nil + }) +} + +// RuntimeClass is an implementation of admission.Interface. +// It looks at all new pods and sets pod.Spec.Overhead if a RuntimeClass is specified which +// defines an Overhead. If pod.Spec.Overhead is set but a RuntimeClass with matching overhead is +// not specified, the pod is rejected. +type RuntimeClass struct { + *admission.Handler + runtimeClassLister nodev1beta1listers.RuntimeClassLister +} + +var _ admission.MutationInterface = &RuntimeClass{} +var _ admission.ValidationInterface = &RuntimeClass{} + +var _ genericadmissioninitailizer.WantsExternalKubeInformerFactory = &RuntimeClass{} + +// SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface. +func (r *RuntimeClass) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) { + runtimeClassInformer := f.Node().V1beta1().RuntimeClasses() + r.SetReadyFunc(runtimeClassInformer.Informer().HasSynced) + r.runtimeClassLister = runtimeClassInformer.Lister() +} + +// ValidateInitialization implements the WantsExternalKubeInformerFactory interface. +func (r *RuntimeClass) ValidateInitialization() error { + if r.runtimeClassLister == nil { + return fmt.Errorf("missing RuntimeClass lister") + } + return nil +} + +// Admit makes an admission decision based on the request attributes +func (r *RuntimeClass) Admit(attributes admission.Attributes, o admission.ObjectInterfaces) error { + + // Ignore all calls to subresources or resources other than pods. + if shouldIgnore(attributes) { + return nil + } + + pod, runtimeClass, err := r.prepareObjects(attributes) + if err != nil { + return err + } + if utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { + err = setOverhead(attributes, pod, runtimeClass) + if err != nil { + return err + } + } + + return nil +} + +// Validate makes sure that pod adhere's to RuntimeClass's definition +func (r *RuntimeClass) Validate(attributes admission.Attributes, o admission.ObjectInterfaces) error { + + // Ignore all calls to subresources or resources other than pods. + if shouldIgnore(attributes) { + return nil + } + + pod, runtimeClass, err := r.prepareObjects(attributes) + if err != nil { + return err + } + + if utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { + err = validateOverhead(attributes, pod, runtimeClass) + } + + return nil +} + +// NewRuntimeClass creates a new RuntimeClass admission control handler +func NewRuntimeClass() *RuntimeClass { + return &RuntimeClass{ + Handler: admission.NewHandler(admission.Create), + } +} + +// admissionAction handles Admit and Validate phases of admission, switching based on the admissionPhase parameter +func (r *RuntimeClass) prepareObjects(attributes admission.Attributes) (pod *api.Pod, runtimeClass *v1beta1.RuntimeClass, err error) { + + pod, ok := attributes.GetObject().(*api.Pod) + if !ok { + return nil, nil, apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted") + } + + // get RuntimeClass object + runtimeClass, err = r.getRuntimeClass(pod, pod.Spec.RuntimeClassName) + if err != nil { + return pod, nil, err + } + + // return the pod and runtimeClass. If no RuntimeClass is specified in PodSpec, runtimeClass will be nil + return pod, runtimeClass, nil +} + +// getRuntimeClass will return a reference to the RuntimeClass object if it is found. If it cannot be found, or a RuntimeClassName +// is not provided in the pod spec, *node.RuntimeClass returned will be nil +func (r *RuntimeClass) getRuntimeClass(pod *api.Pod, runtimeClassName *string) (runtimeClass *v1beta1.RuntimeClass, err error) { + + runtimeClass = nil + + if runtimeClassName != nil { + runtimeClass, err = r.runtimeClassLister.Get(*runtimeClassName) + } + + return runtimeClass, err +} + +func setOverhead(a admission.Attributes, pod *api.Pod, runtimeClass *v1beta1.RuntimeClass) (err error) { + + if runtimeClass != nil { + if runtimeClass.Overhead != nil { + + // convert to internal type and assign to pod's Overhead + nodeOverhead := &node.Overhead{} + err := nodev1beta1.Convert_v1beta1_Overhead_To_node_Overhead(runtimeClass.Overhead, nodeOverhead, nil) + if err != nil { + return err + } + + // reject pod if Overhead is already set that differs from what is defined in RuntimeClass + if pod.Spec.Overhead != nil && !apiequality.Semantic.DeepEqual(nodeOverhead.PodFixed, pod.Spec.Overhead) { + return admission.NewForbidden(a, fmt.Errorf("pod rejected: Pod's Overhead doesn't match RuntimeClass's defined Overhead")) + } + + pod.Spec.Overhead = nodeOverhead.PodFixed + } + } + + return nil +} + +func validateOverhead(a admission.Attributes, pod *api.Pod, runtimeClass *v1beta1.RuntimeClass) (err error) { + + if runtimeClass != nil && runtimeClass.Overhead != nil { + // If the Overhead set doesn't match what is provided in the RuntimeClass definition, reject the pod + nodeOverhead := &node.Overhead{} + err := nodev1beta1.Convert_v1beta1_Overhead_To_node_Overhead(runtimeClass.Overhead, nodeOverhead, nil) + if err != nil { + return err + } + if !apiequality.Semantic.DeepEqual(nodeOverhead.PodFixed, pod.Spec.Overhead) { + return admission.NewForbidden(a, fmt.Errorf("pod rejected: Pod's Overhead doesn't match RuntimeClass's defined Overhead")) + } + } else { + // If RuntimeClass with Overhead is not defined but an Overhead is set for pod, reject the pod + if pod.Spec.Overhead != nil { + return admission.NewForbidden(a, fmt.Errorf("pod rejected: Pod Overhead set without corresponding RuntimeClass defined Overhead")) + } + } + + return nil +} + +func shouldIgnore(attributes admission.Attributes) bool { + // Ignore all calls to subresources or resources other than pods. + if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != api.Resource("pods") { + return true + } + + return false +} diff --git a/plugin/pkg/admission/runtimeclass/admission_test.go b/plugin/pkg/admission/runtimeclass/admission_test.go new file mode 100644 index 00000000000..4f856964ea9 --- /dev/null +++ b/plugin/pkg/admission/runtimeclass/admission_test.go @@ -0,0 +1,243 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtimeclass + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/api/node/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/authentication/user" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func validPod(name string, numContainers int, resources core.ResourceRequirements, setOverhead bool) *core.Pod { + pod := &core.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "test"}, + Spec: core.PodSpec{}, + } + pod.Spec.Containers = make([]core.Container, 0, numContainers) + for i := 0; i < numContainers; i++ { + pod.Spec.Containers = append(pod.Spec.Containers, core.Container{ + Image: "foo:V" + strconv.Itoa(i), + Resources: resources, + Name: "foo-" + strconv.Itoa(i), + }) + } + + if setOverhead { + pod.Spec.Overhead = core.ResourceList{ + core.ResourceName(core.ResourceCPU): resource.MustParse("100m"), + core.ResourceName(core.ResourceMemory): resource.MustParse("1"), + } + } + return pod +} + +func getGuaranteedRequirements() core.ResourceRequirements { + resources := core.ResourceList{ + core.ResourceName(core.ResourceCPU): resource.MustParse("1"), + core.ResourceName(core.ResourceMemory): resource.MustParse("10"), + } + + return core.ResourceRequirements{Limits: resources, Requests: resources} +} + +func TestSetOverhead(t *testing.T) { + + tests := []struct { + name string + runtimeClass *v1beta1.RuntimeClass + pod *core.Pod + expectError bool + expectedPod *core.Pod + }{ + { + name: "overhead, no container requirements", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, false), + expectError: false, + expectedPod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, true), + }, + { + name: "overhead, guaranteed pod", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("guaranteed", 1, getGuaranteedRequirements(), false), + expectError: false, + expectedPod: validPod("guaranteed", 1, core.ResourceRequirements{}, true), + }, + { + name: "overhead, pod with differing overhead already set", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("10"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("10G"), + }, + }, + }, + pod: validPod("empty-requiremennts-overhead", 1, core.ResourceRequirements{}, true), + expectError: true, + expectedPod: nil, + }, + { + name: "overhead, pod with same overhead already set", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("empty-requiremennts-overhead", 1, core.ResourceRequirements{}, true), + expectError: false, + expectedPod: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + + attrs := admission.NewAttributesRecord(tc.pod, nil, core.Kind("Pod").WithVersion("version"), tc.pod.Namespace, tc.pod.Name, core.Resource("pods").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, &user.DefaultInfo{}) + + errs := setOverhead(attrs, tc.pod, tc.runtimeClass) + if tc.expectError { + assert.NotEmpty(t, errs) + } else { + assert.Empty(t, errs) + } + }) + } +} + +func TestValidateOverhead(t *testing.T) { + + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodOverhead, true)() + + tests := []struct { + name string + runtimeClass *v1beta1.RuntimeClass + pod *core.Pod + expectError bool + }{ + { + name: "Overhead part of RuntimeClass, no Overhead defined in pod", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("no-requirements", 1, core.ResourceRequirements{}, false), + expectError: true, + }, + { + name: "No Overhead in RunntimeClass, Overhead set in pod", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + }, + pod: validPod("no-resource-req-no-overhead", 1, getGuaranteedRequirements(), true), + expectError: true, + }, + { + name: "No RunntimeClass, Overhead set in pod", + runtimeClass: nil, + pod: validPod("no-resource-req-no-overhead", 1, getGuaranteedRequirements(), true), + expectError: true, + }, + { + name: "Non-matching Overheads", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("10"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("10G"), + }, + }, + }, + pod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, true), + expectError: true, + }, + { + name: "Matching Overheads", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, true), + expectError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + + attrs := admission.NewAttributesRecord(tc.pod, nil, core.Kind("Pod").WithVersion("version"), tc.pod.Namespace, tc.pod.Name, core.Resource("pods").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, &user.DefaultInfo{}) + + errs := validateOverhead(attrs, tc.pod, tc.runtimeClass) + if tc.expectError { + assert.NotEmpty(t, errs) + } else { + assert.Empty(t, errs) + } + }) + } +}