From c74b3937719c588d0b5e11cbab5528de6f39587e Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Fri, 1 Apr 2022 23:57:04 +0800 Subject: [PATCH] csi: add nodeExpandSecret support for CSI client CSI spec 1.5 enhanced the spec to add optional secrets field to NodeExpandVolumeRequest. This commit adds NodeExpandSecret to the CSI PV source and also derive the expansion secret in csiclient to send it out as part of the nodeexpand request. Signed-off-by: Humble Chirammal Signed-off-by: zhucan --- pkg/api/persistentvolume/util.go | 44 +++++++++++++++++++ pkg/api/v1/persistentvolume/util.go | 5 +++ pkg/apis/core/types.go | 11 ++++- pkg/apis/core/validation/validation.go | 12 +++++ pkg/features/kube_features.go | 8 ++++ .../core/persistentvolume/strategy.go | 7 +++ pkg/volume/csi/csi_client.go | 2 + pkg/volume/csi/expander.go | 14 ++++++ staging/src/k8s.io/api/core/v1/types.go | 11 ++++- 9 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 pkg/api/persistentvolume/util.go diff --git a/pkg/api/persistentvolume/util.go b/pkg/api/persistentvolume/util.go new file mode 100644 index 00000000000..844e709417e --- /dev/null +++ b/pkg/api/persistentvolume/util.go @@ -0,0 +1,44 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package persistentvolume + +import ( + utilfeature "k8s.io/apiserver/pkg/util/feature" + api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" +) + +// DropDisabledFields removes disabled fields from the pv spec. +// This should be called from PrepareForCreate/PrepareForUpdate for all resources containing a pv spec. +func DropDisabledFields(pvSpec *api.PersistentVolumeSpec, oldPVSpec *api.PersistentVolumeSpec) { + if !utilfeature.DefaultFeatureGate.Enabled(features.CSINodeExpandSecret) && !hasNodeExpansionSecrets(oldPVSpec) { + if pvSpec.CSI != nil { + pvSpec.CSI.NodeExpandSecretRef = nil + } + } +} + +func hasNodeExpansionSecrets(oldPVSpec *api.PersistentVolumeSpec) bool { + if oldPVSpec == nil || oldPVSpec.CSI == nil { + return false + } + + if oldPVSpec.CSI.NodeExpandSecretRef != nil { + return true + } + return false +} diff --git a/pkg/api/v1/persistentvolume/util.go b/pkg/api/v1/persistentvolume/util.go index 376021a292b..7d55e7778aa 100644 --- a/pkg/api/v1/persistentvolume/util.go +++ b/pkg/api/v1/persistentvolume/util.go @@ -147,6 +147,11 @@ func VisitPVSecretNames(pv *corev1.PersistentVolume, visitor Visitor) bool { return false } } + if source.CSI.NodeExpandSecretRef != nil { + if !visitor(source.CSI.NodeExpandSecretRef.Namespace, source.CSI.NodeExpandSecretRef.Name, true /* kubeletVisible */) { + return false + } + } } return true } diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index 5519db6cdf9..4a2a4e082ab 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -1716,11 +1716,20 @@ type CSIPersistentVolumeSource struct { // ControllerExpandSecretRef is a reference to the secret object containing // sensitive information to pass to the CSI driver to complete the CSI // ControllerExpandVolume call. - // This is an alpha field and requires enabling ExpandCSIVolumes feature gate. + // This is an beta field and requires enabling ExpandCSIVolumes feature gate. // This field is optional, and may be empty if no secret is required. If the // secret object contains more than one secret, all secrets are passed. // +optional ControllerExpandSecretRef *SecretReference + + // NodeExpandSecretRef is a reference to the secret object containing + // sensitive information to pass to the CSI driver to complete the CSI + // NodeExpandVolume call. + // This is an alpha field and requires enabling CSINodeExpandSecret feature gate. + // This field is optional, may be omitted if no secret is required. If the + // secret object contains more than one secret, all secrets are passed. + // +optional + NodeExpandSecretRef *SecretReference } // CSIVolumeSource represents a source location of a volume to mount, managed by an external CSI driver diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index 6357a631f94..de1ba1444d9 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -1595,6 +1595,18 @@ func validateCSIPersistentVolumeSource(csi *core.CSIPersistentVolumeSource, fldP allErrs = append(allErrs, ValidateDNS1123Label(csi.NodePublishSecretRef.Namespace, fldPath.Child("namespace"))...) } } + if csi.NodeExpandSecretRef != nil { + if len(csi.NodeExpandSecretRef.Name) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("nodeExpandSecretRef", "name"), "")) + } else { + allErrs = append(allErrs, ValidateDNS1123Subdomain(csi.NodeExpandSecretRef.Name, fldPath.Child("name"))...) + } + if len(csi.NodeExpandSecretRef.Namespace) == 0 { + allErrs = append(allErrs, field.Required(fldPath.Child("nodeExpandSecretRef", "namespace"), "")) + } else { + allErrs = append(allErrs, ValidateDNS1123Label(csi.NodeExpandSecretRef.Namespace, fldPath.Child("namespace"))...) + } + } return allErrs } diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 550870e9425..d4fe46210d8 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -882,6 +882,13 @@ const ( // // Enables support for time zones in CronJobs. CronJobTimeZone featuregate.Feature = "CronJobTimeZone" + + // owner: @humblec, @zhucan + // kep: http://kep.k8s.io/3171 + // alpha: v1.24 + // + // Enables SecretRef field in CSI NodeExpandVolume request. + CSINodeExpandSecret featuregate.Feature = "CSINodeExpandSecret" ) func init() { @@ -990,6 +997,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS ReadWriteOncePod: {Default: false, PreRelease: featuregate.Alpha}, CSRDuration: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.26 DelegateFSGroupToCSIDriver: {Default: true, PreRelease: featuregate.Beta}, + CSINodeExpandSecret: {Default: false, PreRelease: featuregate.Alpha}, KubeletInUserNamespace: {Default: false, PreRelease: featuregate.Alpha}, MemoryQoS: {Default: false, PreRelease: featuregate.Alpha}, CPUManagerPolicyOptions: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/registry/core/persistentvolume/strategy.go b/pkg/registry/core/persistentvolume/strategy.go index fdccda2a6f3..89b5a0332fd 100644 --- a/pkg/registry/core/persistentvolume/strategy.go +++ b/pkg/registry/core/persistentvolume/strategy.go @@ -28,6 +28,7 @@ import ( "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/names" "k8s.io/kubernetes/pkg/api/legacyscheme" + pvutil "k8s.io/kubernetes/pkg/api/persistentvolume" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/validation" volumevalidation "k8s.io/kubernetes/pkg/volume/validation" @@ -62,6 +63,10 @@ func (persistentvolumeStrategy) GetResetFields() map[fieldpath.APIVersion]*field // ResetBeforeCreate clears the Status field which is not allowed to be set by end users on creation. func (persistentvolumeStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { + pv := obj.(*api.PersistentVolume) + pv.Status = api.PersistentVolumeStatus{} + + pvutil.DropDisabledFields(&pv.Spec, nil) } func (persistentvolumeStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { @@ -89,6 +94,8 @@ func (persistentvolumeStrategy) PrepareForUpdate(ctx context.Context, obj, old r newPv := obj.(*api.PersistentVolume) oldPv := old.(*api.PersistentVolume) newPv.Status = oldPv.Status + + pvutil.DropDisabledFields(&newPv.Spec, &oldPv.Spec) } func (persistentvolumeStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { diff --git a/pkg/volume/csi/csi_client.go b/pkg/volume/csi/csi_client.go index d3d474f2942..fc5a6a42c50 100644 --- a/pkg/volume/csi/csi_client.go +++ b/pkg/volume/csi/csi_client.go @@ -120,6 +120,7 @@ type csiResizeOptions struct { accessMode api.PersistentVolumeAccessMode newSize resource.Quantity mountOptions []string + secrets map[string]string } var _ csiClient = &csiDriverClient{} @@ -320,6 +321,7 @@ func (c *csiDriverClient) NodeExpandVolume(ctx context.Context, opts csiResizeOp Mode: accessModeMapper(opts.accessMode), }, }, + Secrets: opts.secrets, } // not all CSI drivers support NodeStageUnstage and hence the StagingTargetPath diff --git a/pkg/volume/csi/expander.go b/pkg/volume/csi/expander.go index 22692f8e688..206eac9a1fa 100644 --- a/pkg/volume/csi/expander.go +++ b/pkg/volume/csi/expander.go @@ -23,7 +23,9 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" api "k8s.io/api/core/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util" volumetypes "k8s.io/kubernetes/pkg/volume/util/types" @@ -77,6 +79,17 @@ func (c *csiPlugin) nodeExpandWithClient( if pv == nil { return false, fmt.Errorf("Expander.NodeExpand failed to find associated PersistentVolume for plugin %s", c.GetPluginName()) } + nodeExpandSecrets := map[string]string{} + expandClient := c.host.GetKubeClient() + if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeExpandSecret) { + if csiSource.NodeExpandSecretRef != nil { + nodeExpandSecrets, err = getCredentialsFromSecret(expandClient, csiSource.NodeExpandSecretRef) + if err != nil { + return false, fmt.Errorf("expander.NodeExpand failed to get NodeExpandSecretRef %s/%s: %v", + csiSource.NodeExpandSecretRef.Namespace, csiSource.NodeExpandSecretRef.Name, err) + } + } + } opts := csiResizeOptions{ volumePath: resizeOptions.DeviceMountPath, @@ -86,6 +99,7 @@ func (c *csiPlugin) nodeExpandWithClient( fsType: csiSource.FSType, accessMode: api.ReadWriteOnce, mountOptions: pv.Spec.MountOptions, + secrets: nodeExpandSecrets, } if !fsVolume { diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index c2a3c67b205..c09501d1d2c 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -1800,11 +1800,20 @@ type CSIPersistentVolumeSource struct { // controllerExpandSecretRef is a reference to the secret object containing // sensitive information to pass to the CSI driver to complete the CSI // ControllerExpandVolume call. - // This is an alpha field and requires enabling ExpandCSIVolumes feature gate. + // This is an beta field and requires enabling ExpandCSIVolumes feature gate. // This field is optional, and may be empty if no secret is required. If the // secret object contains more than one secret, all secrets are passed. // +optional ControllerExpandSecretRef *SecretReference `json:"controllerExpandSecretRef,omitempty" protobuf:"bytes,9,opt,name=controllerExpandSecretRef"` + + // nodeExpandSecretRef is a reference to the secret object containing + // sensitive information to pass to the CSI driver to complete the CSI + // NodeExpandVolume call. + // This is an alpha field and requires enabling CSINodeExpandSecret feature gate. + // This field is optional, may be omitted if no secret is required. If the + // secret object contains more than one secret, all secrets are passed. + // +optional + NodeExpandSecretRef *SecretReference `json:"nodeExpandSecretRef,omitempty" protobuf:"bytes,10,opt,name=nodeExpandSecretRef"` } // Represents a source location of a volume to mount, managed by an external CSI driver