From cbfa0cc85a2b94cc58506ce331b4afc5addddcb4 Mon Sep 17 00:00:00 2001 From: NickrenREN Date: Wed, 24 Jan 2018 18:21:35 +0800 Subject: [PATCH] reuse PVC protection admission plugin for PV protection --- cluster/centos/config-default.sh | 2 +- cluster/gce/config-default.sh | 2 +- pkg/kubeapiserver/options/BUILD | 2 +- pkg/kubeapiserver/options/plugins.go | 6 +- pkg/volume/util/finalizer.go | 3 + plugin/BUILD | 2 +- .../pvcprotection/admission.go | 111 ------------- .../storageprotection}/BUILD | 5 +- .../storage/storageprotection/admission.go | 156 ++++++++++++++++++ .../storageprotection}/admission_test.go | 55 +++++- 10 files changed, 219 insertions(+), 125 deletions(-) delete mode 100644 plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go rename plugin/pkg/admission/{persistentvolumeclaim/pvcprotection => storage/storageprotection}/BUILD (87%) create mode 100644 plugin/pkg/admission/storage/storageprotection/admission.go rename plugin/pkg/admission/{persistentvolumeclaim/pvcprotection => storage/storageprotection}/admission_test.go (69%) diff --git a/cluster/centos/config-default.sh b/cluster/centos/config-default.sh index eca05cb3cc9..d75d1b606d3 100755 --- a/cluster/centos/config-default.sh +++ b/cluster/centos/config-default.sh @@ -120,7 +120,7 @@ export FLANNEL_NET=${FLANNEL_NET:-"172.16.0.0/16"} # Admission Controllers to invoke prior to persisting objects in cluster # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely. -export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultTolerationSeconds,Priority,PVCProtection,ResourceQuota"} +export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultTolerationSeconds,Priority,StorageProtection,ResourceQuota"} # Extra options to set on the Docker command line. # This is useful for setting --insecure-registry for local registries. diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index f569d4863aa..1c5612067b9 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -298,7 +298,7 @@ if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then fi # Admission Controllers to invoke prior to persisting objects in cluster -ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,NodeRestriction,Priority,PVCProtection +ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,NodeRestriction,Priority,StorageProtection if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then ADMISSION_CONTROL="${ADMISSION_CONTROL},PodSecurityPolicy" diff --git a/pkg/kubeapiserver/options/BUILD b/pkg/kubeapiserver/options/BUILD index 0c918f81fb6..04a579d01fb 100644 --- a/pkg/kubeapiserver/options/BUILD +++ b/pkg/kubeapiserver/options/BUILD @@ -44,7 +44,6 @@ go_library( "//plugin/pkg/admission/noderestriction:go_default_library", "//plugin/pkg/admission/persistentvolume/label:go_default_library", "//plugin/pkg/admission/persistentvolume/resize:go_default_library", - "//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:go_default_library", "//plugin/pkg/admission/podnodeselector:go_default_library", "//plugin/pkg/admission/podpreset:go_default_library", "//plugin/pkg/admission/podtolerationrestriction:go_default_library", @@ -53,6 +52,7 @@ go_library( "//plugin/pkg/admission/security/podsecuritypolicy:go_default_library", "//plugin/pkg/admission/securitycontext/scdeny:go_default_library", "//plugin/pkg/admission/serviceaccount:go_default_library", + "//plugin/pkg/admission/storage/storageprotection:go_default_library", "//plugin/pkg/admission/storageclass/setdefault:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", diff --git a/pkg/kubeapiserver/options/plugins.go b/pkg/kubeapiserver/options/plugins.go index e8f1749f1b9..ed73e58104f 100644 --- a/pkg/kubeapiserver/options/plugins.go +++ b/pkg/kubeapiserver/options/plugins.go @@ -41,7 +41,6 @@ import ( "k8s.io/kubernetes/plugin/pkg/admission/noderestriction" "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label" "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/resize" - "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection" "k8s.io/kubernetes/plugin/pkg/admission/podnodeselector" "k8s.io/kubernetes/plugin/pkg/admission/podpreset" "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction" @@ -50,6 +49,7 @@ import ( "k8s.io/kubernetes/plugin/pkg/admission/security/podsecuritypolicy" "k8s.io/kubernetes/plugin/pkg/admission/securitycontext/scdeny" "k8s.io/kubernetes/plugin/pkg/admission/serviceaccount" + "k8s.io/kubernetes/plugin/pkg/admission/storage/storageprotection" "k8s.io/kubernetes/plugin/pkg/admission/storageclass/setdefault" "k8s.io/apimachinery/pkg/util/sets" @@ -86,7 +86,7 @@ var AllOrderedPlugins = []string{ extendedresourcetoleration.PluginName, // ExtendedResourceToleration label.PluginName, // PersistentVolumeLabel setdefault.PluginName, // DefaultStorageClass - pvcprotection.PluginName, // PVCProtection + storageprotection.PluginName, // StorageProtection gc.PluginName, // OwnerReferencesPermissionEnforcement resize.PluginName, // PersistentVolumeClaimResize mutatingwebhook.PluginName, // MutatingAdmissionWebhook @@ -125,7 +125,7 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { serviceaccount.Register(plugins) setdefault.Register(plugins) resize.Register(plugins) - pvcprotection.Register(plugins) + storageprotection.Register(plugins) } // DefaultOffAdmissionPlugins get admission plugins off by default for kube-apiserver. diff --git a/pkg/volume/util/finalizer.go b/pkg/volume/util/finalizer.go index 1bc03ad8e78..92d3c2bdd57 100644 --- a/pkg/volume/util/finalizer.go +++ b/pkg/volume/util/finalizer.go @@ -19,4 +19,7 @@ package util const ( // Name of finalizer on PVCs that have a running pod. PVCProtectionFinalizer = "kubernetes.io/pvc-protection" + + // Name of finalizer on PVs that are bound by PVCs + PVProtectionFinalizer = "kubernetes.io/pv-protection" ) diff --git a/plugin/BUILD b/plugin/BUILD index 2d7dd155c1c..275372c9cb9 100644 --- a/plugin/BUILD +++ b/plugin/BUILD @@ -28,7 +28,6 @@ filegroup( "//plugin/pkg/admission/noderestriction:all-srcs", "//plugin/pkg/admission/persistentvolume/label:all-srcs", "//plugin/pkg/admission/persistentvolume/resize:all-srcs", - "//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:all-srcs", "//plugin/pkg/admission/podnodeselector:all-srcs", "//plugin/pkg/admission/podpreset:all-srcs", "//plugin/pkg/admission/podtolerationrestriction:all-srcs", @@ -37,6 +36,7 @@ filegroup( "//plugin/pkg/admission/security:all-srcs", "//plugin/pkg/admission/securitycontext/scdeny:all-srcs", "//plugin/pkg/admission/serviceaccount:all-srcs", + "//plugin/pkg/admission/storage/storageprotection:all-srcs", "//plugin/pkg/admission/storageclass/setdefault:all-srcs", "//plugin/pkg/auth:all-srcs", ], diff --git a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go deleted file mode 100644 index 00c70568db3..00000000000 --- a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pvcprotection - -import ( - "fmt" - "io" - - "github.com/golang/glog" - - admission "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/util/feature" - api "k8s.io/kubernetes/pkg/apis/core" - informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" - corelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion" - "k8s.io/kubernetes/pkg/features" - kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" - volumeutil "k8s.io/kubernetes/pkg/volume/util" -) - -const ( - // PluginName is the name of this admission controller plugin - PluginName = "PVCProtection" -) - -// Register registers a plugin -func Register(plugins *admission.Plugins) { - plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { - plugin := newPlugin() - return plugin, nil - }) -} - -// pvcProtectionPlugin holds state for and implements the admission plugin. -type pvcProtectionPlugin struct { - *admission.Handler - lister corelisters.PersistentVolumeClaimLister -} - -var _ admission.Interface = &pvcProtectionPlugin{} -var _ = kubeapiserveradmission.WantsInternalKubeInformerFactory(&pvcProtectionPlugin{}) - -// newPlugin creates a new admission plugin. -func newPlugin() *pvcProtectionPlugin { - return &pvcProtectionPlugin{ - Handler: admission.NewHandler(admission.Create), - } -} - -func (c *pvcProtectionPlugin) SetInternalKubeInformerFactory(f informers.SharedInformerFactory) { - informer := f.Core().InternalVersion().PersistentVolumeClaims() - c.lister = informer.Lister() - c.SetReadyFunc(informer.Informer().HasSynced) -} - -// ValidateInitialization ensures lister is set. -func (c *pvcProtectionPlugin) ValidateInitialization() error { - if c.lister == nil { - return fmt.Errorf("missing lister") - } - return nil -} - -// Admit sets finalizer on all PVCs. The finalizer is removed by -// PVCProtectionController when it's not referenced by any pod. -// -// This prevents users from deleting a PVC that's used by a running pod. -func (c *pvcProtectionPlugin) Admit(a admission.Attributes) error { - if !feature.DefaultFeatureGate.Enabled(features.StorageProtection) { - return nil - } - - if a.GetResource().GroupResource() != api.Resource("persistentvolumeclaims") { - return nil - } - - if len(a.GetSubresource()) != 0 { - return nil - } - - pvc, ok := a.GetObject().(*api.PersistentVolumeClaim) - // if we can't convert then we don't handle this object so just return - if !ok { - return nil - } - - for _, f := range pvc.Finalizers { - if f == volumeutil.PVCProtectionFinalizer { - // Finalizer is already present, nothing to do - return nil - } - } - - glog.V(4).Infof("adding PVC protection finalizer to %s/%s", pvc.Namespace, pvc.Name) - pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer) - return nil -} diff --git a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD b/plugin/pkg/admission/storage/storageprotection/BUILD similarity index 87% rename from plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD rename to plugin/pkg/admission/storage/storageprotection/BUILD index c7cece04286..71e8fd8b2bb 100644 --- a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD +++ b/plugin/pkg/admission/storage/storageprotection/BUILD @@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = ["admission.go"], - importpath = "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection", + importpath = "k8s.io/kubernetes/plugin/pkg/admission/storage/storageprotection", visibility = ["//visibility:public"], deps = [ "//pkg/apis/core:go_default_library", @@ -22,7 +22,7 @@ go_test( name = "go_default_test", srcs = ["admission_test.go"], embed = [":go_default_library"], - importpath = "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection", + importpath = "k8s.io/kubernetes/plugin/pkg/admission/storage/storageprotection", deps = [ "//pkg/apis/core:go_default_library", "//pkg/client/informers/informers_generated/internalversion:go_default_library", @@ -31,6 +31,7 @@ go_test( "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", ], diff --git a/plugin/pkg/admission/storage/storageprotection/admission.go b/plugin/pkg/admission/storage/storageprotection/admission.go new file mode 100644 index 00000000000..5419a7609a1 --- /dev/null +++ b/plugin/pkg/admission/storage/storageprotection/admission.go @@ -0,0 +1,156 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storageprotection + +import ( + "fmt" + "io" + + "github.com/golang/glog" + + admission "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/util/feature" + api "k8s.io/kubernetes/pkg/apis/core" + informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" + corelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion" + "k8s.io/kubernetes/pkg/features" + kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" + volumeutil "k8s.io/kubernetes/pkg/volume/util" +) + +const ( + // PluginName is the name of this admission controller plugin + PluginName = "StorageProtection" +) + +// Register registers a plugin +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { + plugin := newPlugin() + return plugin, nil + }) +} + +// storageProtectionPlugin holds state for and implements the admission plugin. +type storageProtectionPlugin struct { + *admission.Handler + + pvcLister corelisters.PersistentVolumeClaimLister + pvLister corelisters.PersistentVolumeLister +} + +var _ admission.Interface = &storageProtectionPlugin{} +var _ = kubeapiserveradmission.WantsInternalKubeInformerFactory(&storageProtectionPlugin{}) + +// newPlugin creates a new admission plugin. +func newPlugin() *storageProtectionPlugin { + return &storageProtectionPlugin{ + Handler: admission.NewHandler(admission.Create), + } +} + +func (c *storageProtectionPlugin) SetInternalKubeInformerFactory(f informers.SharedInformerFactory) { + pvcInformer := f.Core().InternalVersion().PersistentVolumeClaims() + c.pvcLister = pvcInformer.Lister() + pvInformer := f.Core().InternalVersion().PersistentVolumes() + c.pvLister = pvInformer.Lister() + c.SetReadyFunc(func() bool { + return pvcInformer.Informer().HasSynced() && pvInformer.Informer().HasSynced() + }) +} + +// ValidateInitialization ensures lister is set. +func (c *storageProtectionPlugin) ValidateInitialization() error { + if c.pvcLister == nil { + return fmt.Errorf("missing PVC lister") + } + if c.pvLister == nil { + return fmt.Errorf("missing PV lister") + } + return nil +} + +var ( + pvResource = api.Resource("persistentvolumes") + pvcResource = api.Resource("persistentvolumeclaims") +) + +// Admit sets finalizer on all PVCs(PVs). The finalizer is removed by +// PVCProtectionController(PVProtectionController) when it's not referenced. +// +// This prevents users from deleting a PVC that's used by a running pod. +// This also prevents admin from deleting a PV that's bound by a PVC +func (c *storageProtectionPlugin) Admit(a admission.Attributes) error { + if !feature.DefaultFeatureGate.Enabled(features.StorageProtection) { + return nil + } + + switch a.GetResource().GroupResource() { + case pvResource: + return c.admitPV(a) + case pvcResource: + return c.admitPVC(a) + + default: + return nil + } +} + +func (c *storageProtectionPlugin) admitPV(a admission.Attributes) error { + if len(a.GetSubresource()) != 0 { + return nil + } + + pv, ok := a.GetObject().(*api.PersistentVolume) + // if we can't convert the obj to PV, just return + if !ok { + return nil + } + for _, f := range pv.Finalizers { + if f == volumeutil.PVProtectionFinalizer { + // Finalizer is already present, nothing to do + return nil + } + } + glog.V(4).Infof("adding PV protection finalizer to %s", pv.Name) + pv.Finalizers = append(pv.Finalizers, volumeutil.PVProtectionFinalizer) + + return nil +} + +func (c *storageProtectionPlugin) admitPVC(a admission.Attributes) error { + if len(a.GetSubresource()) != 0 { + return nil + } + + pvc, ok := a.GetObject().(*api.PersistentVolumeClaim) + // if we can't convert the obj to PVC, just return + if !ok { + return nil + } + + for _, f := range pvc.Finalizers { + if f == volumeutil.PVCProtectionFinalizer { + // Finalizer is already present, nothing to do + return nil + } + } + + glog.V(4).Infof("adding PVC protection finalizer to %s/%s", pvc.Namespace, pvc.Name) + pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer) + return nil +} diff --git a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go b/plugin/pkg/admission/storage/storageprotection/admission_test.go similarity index 69% rename from plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go rename to plugin/pkg/admission/storage/storageprotection/admission_test.go index 9202e4dffb0..60ec8b822ad 100644 --- a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go +++ b/plugin/pkg/admission/storage/storageprotection/admission_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package pvcprotection +package storageprotection import ( "fmt" @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/util/feature" api "k8s.io/kubernetes/pkg/apis/core" @@ -43,32 +44,76 @@ func TestAdmit(t *testing.T) { Namespace: "ns", }, } + + pv := &api.PersistentVolume{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolume", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pv", + }, + } claimWithFinalizer := claim.DeepCopy() claimWithFinalizer.Finalizers = []string{volumeutil.PVCProtectionFinalizer} + pvWithFinalizer := pv.DeepCopy() + pvWithFinalizer.Finalizers = []string{volumeutil.PVProtectionFinalizer} + tests := []struct { name string + resource schema.GroupVersionResource object runtime.Object expectedObject runtime.Object featureEnabled bool + namespace string }{ { "create -> add finalizer", + api.SchemeGroupVersion.WithResource("persistentvolumeclaims"), claim, claimWithFinalizer, true, + claim.Namespace, }, { "finalizer already exists -> no new finalizer", + api.SchemeGroupVersion.WithResource("persistentvolumeclaims"), claimWithFinalizer, claimWithFinalizer, true, + claimWithFinalizer.Namespace, }, { "disabled feature -> no finalizer", + api.SchemeGroupVersion.WithResource("persistentvolumeclaims"), claim, claim, false, + claim.Namespace, + }, + { + "create -> add finalizer", + api.SchemeGroupVersion.WithResource("persistentvolumes"), + pv, + pvWithFinalizer, + true, + pv.Namespace, + }, + { + "finalizer already exists -> no new finalizer", + api.SchemeGroupVersion.WithResource("persistentvolumes"), + pvWithFinalizer, + pvWithFinalizer, + true, + pvWithFinalizer.Namespace, + }, + { + "disabled feature -> no finalizer", + api.SchemeGroupVersion.WithResource("persistentvolumes"), + pv, + pv, + false, + pv.Namespace, }, } @@ -82,10 +127,10 @@ func TestAdmit(t *testing.T) { attrs := admission.NewAttributesRecord( obj, // new object obj.DeepCopyObject(), // old object, copy to be sure it's not modified - api.Kind("PersistentVolumeClaim").WithVersion("version"), - claim.Namespace, - claim.Name, - api.Resource("persistentvolumeclaims").WithVersion("version"), + schema.GroupVersionKind{}, + test.namespace, + "foo", + test.resource, "", // subresource admission.Create, nil, // userInfo