diff --git a/pkg/kubeapiserver/options/BUILD b/pkg/kubeapiserver/options/BUILD index 37b8328b53d..22447c7d09a 100644 --- a/pkg/kubeapiserver/options/BUILD +++ b/pkg/kubeapiserver/options/BUILD @@ -43,6 +43,7 @@ go_library( "//plugin/pkg/admission/podtolerationrestriction:go_default_library", "//plugin/pkg/admission/priority:go_default_library", "//plugin/pkg/admission/resourcequota:go_default_library", + "//plugin/pkg/admission/runtimeclass:go_default_library", "//plugin/pkg/admission/security/podsecuritypolicy:go_default_library", "//plugin/pkg/admission/securitycontext/scdeny:go_default_library", "//plugin/pkg/admission/serviceaccount:go_default_library", diff --git a/pkg/kubeapiserver/options/plugins.go b/pkg/kubeapiserver/options/plugins.go index 3e11594b031..af743669104 100644 --- a/pkg/kubeapiserver/options/plugins.go +++ b/pkg/kubeapiserver/options/plugins.go @@ -41,6 +41,7 @@ import ( "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction" podpriority "k8s.io/kubernetes/plugin/pkg/admission/priority" "k8s.io/kubernetes/plugin/pkg/admission/resourcequota" + "k8s.io/kubernetes/plugin/pkg/admission/runtimeclass" "k8s.io/kubernetes/plugin/pkg/admission/security/podsecuritypolicy" "k8s.io/kubernetes/plugin/pkg/admission/securitycontext/scdeny" "k8s.io/kubernetes/plugin/pkg/admission/serviceaccount" @@ -89,6 +90,7 @@ var AllOrderedPlugins = []string{ resize.PluginName, // PersistentVolumeClaimResize mutatingwebhook.PluginName, // MutatingAdmissionWebhook validatingwebhook.PluginName, // ValidatingAdmissionWebhook + runtimeclass.PluginName, //RuntimeClass resourcequota.PluginName, // ResourceQuota deny.PluginName, // AlwaysDeny } @@ -115,6 +117,7 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { podnodeselector.Register(plugins) podpreset.Register(plugins) podtolerationrestriction.Register(plugins) + runtimeclass.Register(plugins) resourcequota.Register(plugins) podsecuritypolicy.Register(plugins) podpriority.Register(plugins) @@ -145,5 +148,9 @@ func DefaultOffAdmissionPlugins() sets.String { defaultOnPlugins.Insert(nodetaint.PluginName) //TaintNodesByCondition } + if utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { + defaultOnPlugins.Insert(runtimeclass.PluginName) //RuntimeClass + } + return sets.NewString(AllOrderedPlugins...).Difference(defaultOnPlugins) } diff --git a/plugin/BUILD b/plugin/BUILD index 36d7d2744ad..23dd6fa3047 100644 --- a/plugin/BUILD +++ b/plugin/BUILD @@ -31,6 +31,7 @@ filegroup( "//plugin/pkg/admission/podtolerationrestriction:all-srcs", "//plugin/pkg/admission/priority:all-srcs", "//plugin/pkg/admission/resourcequota:all-srcs", + "//plugin/pkg/admission/runtimeclass:all-srcs", "//plugin/pkg/admission/security:all-srcs", "//plugin/pkg/admission/securitycontext/scdeny:all-srcs", "//plugin/pkg/admission/serviceaccount:all-srcs", diff --git a/plugin/pkg/admission/runtimeclass/BUILD b/plugin/pkg/admission/runtimeclass/BUILD new file mode 100644 index 00000000000..2769ae8862f --- /dev/null +++ b/plugin/pkg/admission/runtimeclass/BUILD @@ -0,0 +1,55 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["admission.go"], + importpath = "k8s.io/kubernetes/plugin/pkg/admission/runtimeclass", + visibility = ["//visibility:public"], + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/apis/node:go_default_library", + "//pkg/apis/node/v1beta1:go_default_library", + "//pkg/features:go_default_library", + "//staging/src/k8s.io/api/node/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/admission/initializer:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/client-go/informers:go_default_library", + "//staging/src/k8s.io/client-go/listers/node/v1beta1:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["admission_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/features:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/api/node/v1beta1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//vendor/github.com/stretchr/testify/assert:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/plugin/pkg/admission/runtimeclass/OWNERS b/plugin/pkg/admission/runtimeclass/OWNERS new file mode 100644 index 00000000000..767618f5bfd --- /dev/null +++ b/plugin/pkg/admission/runtimeclass/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +- tallclair +reviewers: +- egernst diff --git a/plugin/pkg/admission/runtimeclass/admission.go b/plugin/pkg/admission/runtimeclass/admission.go new file mode 100644 index 00000000000..aa06f9343b5 --- /dev/null +++ b/plugin/pkg/admission/runtimeclass/admission.go @@ -0,0 +1,214 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package runtimeclass contains an admission controller for modifying and validating new Pods to +// take RuntimeClass into account. For RuntimeClass definitions which describe an overhead associated +// with running a pod, this admission controller will set the pod.Spec.Overhead field accordingly. This +// field should only be set through this controller, so vaidation will be carried out to ensure the pod's +// value matches what is defined in the coresponding RuntimeClass. +package runtimeclass + +import ( + "fmt" + "io" + + v1beta1 "k8s.io/api/node/v1beta1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apiserver/pkg/admission" + genericadmissioninitailizer "k8s.io/apiserver/pkg/admission/initializer" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" + nodev1beta1listers "k8s.io/client-go/listers/node/v1beta1" + api "k8s.io/kubernetes/pkg/apis/core" + node "k8s.io/kubernetes/pkg/apis/node" + nodev1beta1 "k8s.io/kubernetes/pkg/apis/node/v1beta1" + "k8s.io/kubernetes/pkg/features" +) + +// PluginName indicates name of admission plugin. +const PluginName = "RuntimeClass" + +// Register registers a plugin +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { + return NewRuntimeClass(), nil + }) +} + +// RuntimeClass is an implementation of admission.Interface. +// It looks at all new pods and sets pod.Spec.Overhead if a RuntimeClass is specified which +// defines an Overhead. If pod.Spec.Overhead is set but a RuntimeClass with matching overhead is +// not specified, the pod is rejected. +type RuntimeClass struct { + *admission.Handler + runtimeClassLister nodev1beta1listers.RuntimeClassLister +} + +var _ admission.MutationInterface = &RuntimeClass{} +var _ admission.ValidationInterface = &RuntimeClass{} + +var _ genericadmissioninitailizer.WantsExternalKubeInformerFactory = &RuntimeClass{} + +// SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface. +func (r *RuntimeClass) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) { + runtimeClassInformer := f.Node().V1beta1().RuntimeClasses() + r.SetReadyFunc(runtimeClassInformer.Informer().HasSynced) + r.runtimeClassLister = runtimeClassInformer.Lister() +} + +// ValidateInitialization implements the WantsExternalKubeInformerFactory interface. +func (r *RuntimeClass) ValidateInitialization() error { + if r.runtimeClassLister == nil { + return fmt.Errorf("missing RuntimeClass lister") + } + return nil +} + +// Admit makes an admission decision based on the request attributes +func (r *RuntimeClass) Admit(attributes admission.Attributes, o admission.ObjectInterfaces) error { + + // Ignore all calls to subresources or resources other than pods. + if shouldIgnore(attributes) { + return nil + } + + pod, runtimeClass, err := r.prepareObjects(attributes) + if err != nil { + return err + } + if utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { + err = setOverhead(attributes, pod, runtimeClass) + if err != nil { + return err + } + } + + return nil +} + +// Validate makes sure that pod adhere's to RuntimeClass's definition +func (r *RuntimeClass) Validate(attributes admission.Attributes, o admission.ObjectInterfaces) error { + + // Ignore all calls to subresources or resources other than pods. + if shouldIgnore(attributes) { + return nil + } + + pod, runtimeClass, err := r.prepareObjects(attributes) + if err != nil { + return err + } + + if utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) { + err = validateOverhead(attributes, pod, runtimeClass) + } + + return nil +} + +// NewRuntimeClass creates a new RuntimeClass admission control handler +func NewRuntimeClass() *RuntimeClass { + return &RuntimeClass{ + Handler: admission.NewHandler(admission.Create), + } +} + +// admissionAction handles Admit and Validate phases of admission, switching based on the admissionPhase parameter +func (r *RuntimeClass) prepareObjects(attributes admission.Attributes) (pod *api.Pod, runtimeClass *v1beta1.RuntimeClass, err error) { + + pod, ok := attributes.GetObject().(*api.Pod) + if !ok { + return nil, nil, apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted") + } + + // get RuntimeClass object + runtimeClass, err = r.getRuntimeClass(pod, pod.Spec.RuntimeClassName) + if err != nil { + return pod, nil, err + } + + // return the pod and runtimeClass. If no RuntimeClass is specified in PodSpec, runtimeClass will be nil + return pod, runtimeClass, nil +} + +// getRuntimeClass will return a reference to the RuntimeClass object if it is found. If it cannot be found, or a RuntimeClassName +// is not provided in the pod spec, *node.RuntimeClass returned will be nil +func (r *RuntimeClass) getRuntimeClass(pod *api.Pod, runtimeClassName *string) (runtimeClass *v1beta1.RuntimeClass, err error) { + + runtimeClass = nil + + if runtimeClassName != nil { + runtimeClass, err = r.runtimeClassLister.Get(*runtimeClassName) + } + + return runtimeClass, err +} + +func setOverhead(a admission.Attributes, pod *api.Pod, runtimeClass *v1beta1.RuntimeClass) (err error) { + + if runtimeClass != nil { + if runtimeClass.Overhead != nil { + + // convert to internal type and assign to pod's Overhead + nodeOverhead := &node.Overhead{} + err := nodev1beta1.Convert_v1beta1_Overhead_To_node_Overhead(runtimeClass.Overhead, nodeOverhead, nil) + if err != nil { + return err + } + + // reject pod if Overhead is already set that differs from what is defined in RuntimeClass + if pod.Spec.Overhead != nil && !apiequality.Semantic.DeepEqual(nodeOverhead.PodFixed, pod.Spec.Overhead) { + return admission.NewForbidden(a, fmt.Errorf("pod rejected: Pod's Overhead doesn't match RuntimeClass's defined Overhead")) + } + + pod.Spec.Overhead = nodeOverhead.PodFixed + } + } + + return nil +} + +func validateOverhead(a admission.Attributes, pod *api.Pod, runtimeClass *v1beta1.RuntimeClass) (err error) { + + if runtimeClass != nil && runtimeClass.Overhead != nil { + // If the Overhead set doesn't match what is provided in the RuntimeClass definition, reject the pod + nodeOverhead := &node.Overhead{} + err := nodev1beta1.Convert_v1beta1_Overhead_To_node_Overhead(runtimeClass.Overhead, nodeOverhead, nil) + if err != nil { + return err + } + if !apiequality.Semantic.DeepEqual(nodeOverhead.PodFixed, pod.Spec.Overhead) { + return admission.NewForbidden(a, fmt.Errorf("pod rejected: Pod's Overhead doesn't match RuntimeClass's defined Overhead")) + } + } else { + // If RuntimeClass with Overhead is not defined but an Overhead is set for pod, reject the pod + if pod.Spec.Overhead != nil { + return admission.NewForbidden(a, fmt.Errorf("pod rejected: Pod Overhead set without corresponding RuntimeClass defined Overhead")) + } + } + + return nil +} + +func shouldIgnore(attributes admission.Attributes) bool { + // Ignore all calls to subresources or resources other than pods. + if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != api.Resource("pods") { + return true + } + + return false +} diff --git a/plugin/pkg/admission/runtimeclass/admission_test.go b/plugin/pkg/admission/runtimeclass/admission_test.go new file mode 100644 index 00000000000..4f856964ea9 --- /dev/null +++ b/plugin/pkg/admission/runtimeclass/admission_test.go @@ -0,0 +1,243 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtimeclass + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/api/node/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/authentication/user" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func validPod(name string, numContainers int, resources core.ResourceRequirements, setOverhead bool) *core.Pod { + pod := &core.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "test"}, + Spec: core.PodSpec{}, + } + pod.Spec.Containers = make([]core.Container, 0, numContainers) + for i := 0; i < numContainers; i++ { + pod.Spec.Containers = append(pod.Spec.Containers, core.Container{ + Image: "foo:V" + strconv.Itoa(i), + Resources: resources, + Name: "foo-" + strconv.Itoa(i), + }) + } + + if setOverhead { + pod.Spec.Overhead = core.ResourceList{ + core.ResourceName(core.ResourceCPU): resource.MustParse("100m"), + core.ResourceName(core.ResourceMemory): resource.MustParse("1"), + } + } + return pod +} + +func getGuaranteedRequirements() core.ResourceRequirements { + resources := core.ResourceList{ + core.ResourceName(core.ResourceCPU): resource.MustParse("1"), + core.ResourceName(core.ResourceMemory): resource.MustParse("10"), + } + + return core.ResourceRequirements{Limits: resources, Requests: resources} +} + +func TestSetOverhead(t *testing.T) { + + tests := []struct { + name string + runtimeClass *v1beta1.RuntimeClass + pod *core.Pod + expectError bool + expectedPod *core.Pod + }{ + { + name: "overhead, no container requirements", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, false), + expectError: false, + expectedPod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, true), + }, + { + name: "overhead, guaranteed pod", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("guaranteed", 1, getGuaranteedRequirements(), false), + expectError: false, + expectedPod: validPod("guaranteed", 1, core.ResourceRequirements{}, true), + }, + { + name: "overhead, pod with differing overhead already set", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("10"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("10G"), + }, + }, + }, + pod: validPod("empty-requiremennts-overhead", 1, core.ResourceRequirements{}, true), + expectError: true, + expectedPod: nil, + }, + { + name: "overhead, pod with same overhead already set", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("empty-requiremennts-overhead", 1, core.ResourceRequirements{}, true), + expectError: false, + expectedPod: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + + attrs := admission.NewAttributesRecord(tc.pod, nil, core.Kind("Pod").WithVersion("version"), tc.pod.Namespace, tc.pod.Name, core.Resource("pods").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, &user.DefaultInfo{}) + + errs := setOverhead(attrs, tc.pod, tc.runtimeClass) + if tc.expectError { + assert.NotEmpty(t, errs) + } else { + assert.Empty(t, errs) + } + }) + } +} + +func TestValidateOverhead(t *testing.T) { + + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodOverhead, true)() + + tests := []struct { + name string + runtimeClass *v1beta1.RuntimeClass + pod *core.Pod + expectError bool + }{ + { + name: "Overhead part of RuntimeClass, no Overhead defined in pod", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("no-requirements", 1, core.ResourceRequirements{}, false), + expectError: true, + }, + { + name: "No Overhead in RunntimeClass, Overhead set in pod", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + }, + pod: validPod("no-resource-req-no-overhead", 1, getGuaranteedRequirements(), true), + expectError: true, + }, + { + name: "No RunntimeClass, Overhead set in pod", + runtimeClass: nil, + pod: validPod("no-resource-req-no-overhead", 1, getGuaranteedRequirements(), true), + expectError: true, + }, + { + name: "Non-matching Overheads", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("10"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("10G"), + }, + }, + }, + pod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, true), + expectError: true, + }, + { + name: "Matching Overheads", + runtimeClass: &v1beta1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "foo"}, + Handler: "bar", + Overhead: &v1beta1.Overhead{ + PodFixed: corev1.ResourceList{ + corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"), + corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"), + }, + }, + }, + pod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, true), + expectError: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + + attrs := admission.NewAttributesRecord(tc.pod, nil, core.Kind("Pod").WithVersion("version"), tc.pod.Namespace, tc.pod.Name, core.Resource("pods").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, &user.DefaultInfo{}) + + errs := validateOverhead(attrs, tc.pod, tc.runtimeClass) + if tc.expectError { + assert.NotEmpty(t, errs) + } else { + assert.Empty(t, errs) + } + }) + } +}