introduce RuntimeClass admission controller

Signed-off-by: Eric Ernst <eric.ernst@intel.com>
This commit is contained in:
Eric Ernst 2019-05-25 12:24:59 -04:00
parent d59bd7364c
commit 247dab3578
2 changed files with 457 additions and 0 deletions

View File

@ -0,0 +1,214 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package runtimeclass contains an admission controller for modifying and validating new Pods to
// take RuntimeClass into account. For RuntimeClass definitions which describe an overhead associated
// with running a pod, this admission controller will set the pod.Spec.Overhead field accordingly. This
// field should only be set through this controller, so vaidation will be carried out to ensure the pod's
// value matches what is defined in the coresponding RuntimeClass.
package runtimeclass
import (
"fmt"
"io"
v1beta1 "k8s.io/api/node/v1beta1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apiserver/pkg/admission"
genericadmissioninitailizer "k8s.io/apiserver/pkg/admission/initializer"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers"
nodev1beta1listers "k8s.io/client-go/listers/node/v1beta1"
api "k8s.io/kubernetes/pkg/apis/core"
node "k8s.io/kubernetes/pkg/apis/node"
nodev1beta1 "k8s.io/kubernetes/pkg/apis/node/v1beta1"
"k8s.io/kubernetes/pkg/features"
)
// PluginName indicates name of admission plugin.
const PluginName = "RuntimeClass"
// Register registers a plugin
func Register(plugins *admission.Plugins) {
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
return NewRuntimeClass(), nil
})
}
// RuntimeClass is an implementation of admission.Interface.
// It looks at all new pods and sets pod.Spec.Overhead if a RuntimeClass is specified which
// defines an Overhead. If pod.Spec.Overhead is set but a RuntimeClass with matching overhead is
// not specified, the pod is rejected.
type RuntimeClass struct {
*admission.Handler
runtimeClassLister nodev1beta1listers.RuntimeClassLister
}
var _ admission.MutationInterface = &RuntimeClass{}
var _ admission.ValidationInterface = &RuntimeClass{}
var _ genericadmissioninitailizer.WantsExternalKubeInformerFactory = &RuntimeClass{}
// SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface.
func (r *RuntimeClass) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) {
runtimeClassInformer := f.Node().V1beta1().RuntimeClasses()
r.SetReadyFunc(runtimeClassInformer.Informer().HasSynced)
r.runtimeClassLister = runtimeClassInformer.Lister()
}
// ValidateInitialization implements the WantsExternalKubeInformerFactory interface.
func (r *RuntimeClass) ValidateInitialization() error {
if r.runtimeClassLister == nil {
return fmt.Errorf("missing RuntimeClass lister")
}
return nil
}
// Admit makes an admission decision based on the request attributes
func (r *RuntimeClass) Admit(attributes admission.Attributes, o admission.ObjectInterfaces) error {
// Ignore all calls to subresources or resources other than pods.
if shouldIgnore(attributes) {
return nil
}
pod, runtimeClass, err := r.prepareObjects(attributes)
if err != nil {
return err
}
if utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) {
err = setOverhead(attributes, pod, runtimeClass)
if err != nil {
return err
}
}
return nil
}
// Validate makes sure that pod adhere's to RuntimeClass's definition
func (r *RuntimeClass) Validate(attributes admission.Attributes, o admission.ObjectInterfaces) error {
// Ignore all calls to subresources or resources other than pods.
if shouldIgnore(attributes) {
return nil
}
pod, runtimeClass, err := r.prepareObjects(attributes)
if err != nil {
return err
}
if utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) {
err = validateOverhead(attributes, pod, runtimeClass)
}
return nil
}
// NewRuntimeClass creates a new RuntimeClass admission control handler
func NewRuntimeClass() *RuntimeClass {
return &RuntimeClass{
Handler: admission.NewHandler(admission.Create),
}
}
// admissionAction handles Admit and Validate phases of admission, switching based on the admissionPhase parameter
func (r *RuntimeClass) prepareObjects(attributes admission.Attributes) (pod *api.Pod, runtimeClass *v1beta1.RuntimeClass, err error) {
pod, ok := attributes.GetObject().(*api.Pod)
if !ok {
return nil, nil, apierrors.NewBadRequest("Resource was marked with kind Pod but was unable to be converted")
}
// get RuntimeClass object
runtimeClass, err = r.getRuntimeClass(pod, pod.Spec.RuntimeClassName)
if err != nil {
return pod, nil, err
}
// return the pod and runtimeClass. If no RuntimeClass is specified in PodSpec, runtimeClass will be nil
return pod, runtimeClass, nil
}
// getRuntimeClass will return a reference to the RuntimeClass object if it is found. If it cannot be found, or a RuntimeClassName
// is not provided in the pod spec, *node.RuntimeClass returned will be nil
func (r *RuntimeClass) getRuntimeClass(pod *api.Pod, runtimeClassName *string) (runtimeClass *v1beta1.RuntimeClass, err error) {
runtimeClass = nil
if runtimeClassName != nil {
runtimeClass, err = r.runtimeClassLister.Get(*runtimeClassName)
}
return runtimeClass, err
}
func setOverhead(a admission.Attributes, pod *api.Pod, runtimeClass *v1beta1.RuntimeClass) (err error) {
if runtimeClass != nil {
if runtimeClass.Overhead != nil {
// convert to internal type and assign to pod's Overhead
nodeOverhead := &node.Overhead{}
err := nodev1beta1.Convert_v1beta1_Overhead_To_node_Overhead(runtimeClass.Overhead, nodeOverhead, nil)
if err != nil {
return err
}
// reject pod if Overhead is already set that differs from what is defined in RuntimeClass
if pod.Spec.Overhead != nil && !apiequality.Semantic.DeepEqual(nodeOverhead.PodFixed, pod.Spec.Overhead) {
return admission.NewForbidden(a, fmt.Errorf("pod rejected: Pod's Overhead doesn't match RuntimeClass's defined Overhead"))
}
pod.Spec.Overhead = nodeOverhead.PodFixed
}
}
return nil
}
func validateOverhead(a admission.Attributes, pod *api.Pod, runtimeClass *v1beta1.RuntimeClass) (err error) {
if runtimeClass != nil && runtimeClass.Overhead != nil {
// If the Overhead set doesn't match what is provided in the RuntimeClass definition, reject the pod
nodeOverhead := &node.Overhead{}
err := nodev1beta1.Convert_v1beta1_Overhead_To_node_Overhead(runtimeClass.Overhead, nodeOverhead, nil)
if err != nil {
return err
}
if !apiequality.Semantic.DeepEqual(nodeOverhead.PodFixed, pod.Spec.Overhead) {
return admission.NewForbidden(a, fmt.Errorf("pod rejected: Pod's Overhead doesn't match RuntimeClass's defined Overhead"))
}
} else {
// If RuntimeClass with Overhead is not defined but an Overhead is set for pod, reject the pod
if pod.Spec.Overhead != nil {
return admission.NewForbidden(a, fmt.Errorf("pod rejected: Pod Overhead set without corresponding RuntimeClass defined Overhead"))
}
}
return nil
}
func shouldIgnore(attributes admission.Attributes) bool {
// Ignore all calls to subresources or resources other than pods.
if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != api.Resource("pods") {
return true
}
return false
}

View File

@ -0,0 +1,243 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package runtimeclass
import (
corev1 "k8s.io/api/core/v1"
"k8s.io/api/node/v1beta1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authentication/user"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/features"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
)
func validPod(name string, numContainers int, resources core.ResourceRequirements, setOverhead bool) *core.Pod {
pod := &core.Pod{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "test"},
Spec: core.PodSpec{},
}
pod.Spec.Containers = make([]core.Container, 0, numContainers)
for i := 0; i < numContainers; i++ {
pod.Spec.Containers = append(pod.Spec.Containers, core.Container{
Image: "foo:V" + strconv.Itoa(i),
Resources: resources,
Name: "foo-" + strconv.Itoa(i),
})
}
if setOverhead {
pod.Spec.Overhead = core.ResourceList{
core.ResourceName(core.ResourceCPU): resource.MustParse("100m"),
core.ResourceName(core.ResourceMemory): resource.MustParse("1"),
}
}
return pod
}
func getGuaranteedRequirements() core.ResourceRequirements {
resources := core.ResourceList{
core.ResourceName(core.ResourceCPU): resource.MustParse("1"),
core.ResourceName(core.ResourceMemory): resource.MustParse("10"),
}
return core.ResourceRequirements{Limits: resources, Requests: resources}
}
func TestSetOverhead(t *testing.T) {
tests := []struct {
name string
runtimeClass *v1beta1.RuntimeClass
pod *core.Pod
expectError bool
expectedPod *core.Pod
}{
{
name: "overhead, no container requirements",
runtimeClass: &v1beta1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
Handler: "bar",
Overhead: &v1beta1.Overhead{
PodFixed: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"),
},
},
},
pod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, false),
expectError: false,
expectedPod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, true),
},
{
name: "overhead, guaranteed pod",
runtimeClass: &v1beta1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
Handler: "bar",
Overhead: &v1beta1.Overhead{
PodFixed: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"),
},
},
},
pod: validPod("guaranteed", 1, getGuaranteedRequirements(), false),
expectError: false,
expectedPod: validPod("guaranteed", 1, core.ResourceRequirements{}, true),
},
{
name: "overhead, pod with differing overhead already set",
runtimeClass: &v1beta1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
Handler: "bar",
Overhead: &v1beta1.Overhead{
PodFixed: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("10"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("10G"),
},
},
},
pod: validPod("empty-requiremennts-overhead", 1, core.ResourceRequirements{}, true),
expectError: true,
expectedPod: nil,
},
{
name: "overhead, pod with same overhead already set",
runtimeClass: &v1beta1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
Handler: "bar",
Overhead: &v1beta1.Overhead{
PodFixed: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"),
},
},
},
pod: validPod("empty-requiremennts-overhead", 1, core.ResourceRequirements{}, true),
expectError: false,
expectedPod: nil,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
attrs := admission.NewAttributesRecord(tc.pod, nil, core.Kind("Pod").WithVersion("version"), tc.pod.Namespace, tc.pod.Name, core.Resource("pods").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, &user.DefaultInfo{})
errs := setOverhead(attrs, tc.pod, tc.runtimeClass)
if tc.expectError {
assert.NotEmpty(t, errs)
} else {
assert.Empty(t, errs)
}
})
}
}
func TestValidateOverhead(t *testing.T) {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodOverhead, true)()
tests := []struct {
name string
runtimeClass *v1beta1.RuntimeClass
pod *core.Pod
expectError bool
}{
{
name: "Overhead part of RuntimeClass, no Overhead defined in pod",
runtimeClass: &v1beta1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
Handler: "bar",
Overhead: &v1beta1.Overhead{
PodFixed: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"),
},
},
},
pod: validPod("no-requirements", 1, core.ResourceRequirements{}, false),
expectError: true,
},
{
name: "No Overhead in RunntimeClass, Overhead set in pod",
runtimeClass: &v1beta1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
Handler: "bar",
},
pod: validPod("no-resource-req-no-overhead", 1, getGuaranteedRequirements(), true),
expectError: true,
},
{
name: "No RunntimeClass, Overhead set in pod",
runtimeClass: nil,
pod: validPod("no-resource-req-no-overhead", 1, getGuaranteedRequirements(), true),
expectError: true,
},
{
name: "Non-matching Overheads",
runtimeClass: &v1beta1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
Handler: "bar",
Overhead: &v1beta1.Overhead{
PodFixed: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("10"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("10G"),
},
},
},
pod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, true),
expectError: true,
},
{
name: "Matching Overheads",
runtimeClass: &v1beta1.RuntimeClass{
ObjectMeta: metav1.ObjectMeta{Name: "foo"},
Handler: "bar",
Overhead: &v1beta1.Overhead{
PodFixed: corev1.ResourceList{
corev1.ResourceName(corev1.ResourceCPU): resource.MustParse("100m"),
corev1.ResourceName(corev1.ResourceMemory): resource.MustParse("1"),
},
},
},
pod: validPod("no-resource-req-no-overhead", 1, core.ResourceRequirements{}, true),
expectError: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
attrs := admission.NewAttributesRecord(tc.pod, nil, core.Kind("Pod").WithVersion("version"), tc.pod.Namespace, tc.pod.Name, core.Resource("pods").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, &user.DefaultInfo{})
errs := validateOverhead(attrs, tc.pod, tc.runtimeClass)
if tc.expectError {
assert.NotEmpty(t, errs)
} else {
assert.Empty(t, errs)
}
})
}
}