From 029fd47757518ee525eb679001479a7aa99e4f89 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 24 Jul 2019 08:16:43 +0200 Subject: [PATCH 1/6] storage: introduce CSIDriver.Spec.VolumeLifecycleModes Using a "normal" CSI driver for an inline ephemeral volume may have unexpected and potentially harmful effects when the driver gets a NodePublishVolume call that it isn't expecting. To prevent that mistake, driver deployments for a driver that supports such volumes must: - deploy a CSIDriver object for the driver - list "ephemeral" as one of the supported modes The default is "persistent", so existing deployments continue to work and are automatically protected against incorrect usage. This commit contains the API change. Generated code and manual code which uses the new API follow. --- api/api-rules/violation_exceptions.list | 1 + pkg/apis/storage/fuzzer/fuzzer.go | 37 ++++ pkg/apis/storage/types.go | 39 ++++ pkg/apis/storage/v1beta1/defaults.go | 5 + pkg/apis/storage/v1beta1/defaults_test.go | 30 ++++ pkg/registry/storage/csidriver/strategy.go | 17 +- .../storage/csidriver/strategy_test.go | 167 ++++++++++++++++++ .../src/k8s.io/api/storage/v1beta1/types.go | 49 +++++ 8 files changed, 343 insertions(+), 2 deletions(-) diff --git a/api/api-rules/violation_exceptions.list b/api/api-rules/violation_exceptions.list index b39a57e9212..be1f4356458 100644 --- a/api/api-rules/violation_exceptions.list +++ b/api/api-rules/violation_exceptions.list @@ -344,6 +344,7 @@ API rule violation: list_type_missing,k8s.io/api/storage/v1,StorageClassList,Ite API rule violation: list_type_missing,k8s.io/api/storage/v1,VolumeAttachmentList,Items API rule violation: list_type_missing,k8s.io/api/storage/v1alpha1,VolumeAttachmentList,Items API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSIDriverList,Items +API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSIDriverSpec,VolumeLifecycleModes API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSINodeDriver,TopologyKeys API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSINodeList,Items API rule violation: list_type_missing,k8s.io/api/storage/v1beta1,CSINodeSpec,Drivers diff --git a/pkg/apis/storage/fuzzer/fuzzer.go b/pkg/apis/storage/fuzzer/fuzzer.go index 2081ffe846f..ea1de3ddacf 100644 --- a/pkg/apis/storage/fuzzer/fuzzer.go +++ b/pkg/apis/storage/fuzzer/fuzzer.go @@ -17,6 +17,7 @@ limitations under the License. package fuzzer import ( + "fmt" fuzz "github.com/google/gofuzz" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" @@ -37,6 +38,37 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { func(obj *storage.CSIDriver, c fuzz.Continue) { c.FuzzNoCustom(obj) // fuzz self without calling this function again + // Custom fuzzing for volume modes. + switch c.Rand.Intn(7) { + case 0: + obj.Spec.VolumeLifecycleModes = nil + case 1: + obj.Spec.VolumeLifecycleModes = []storage.VolumeLifecycleMode{} + case 2: + // Invalid mode. + obj.Spec.VolumeLifecycleModes = []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleMode(fmt.Sprintf("%d", c.Rand.Int31())), + } + case 3: + obj.Spec.VolumeLifecycleModes = []storage.VolumeLifecycleMode{ + storage.VolumeLifecyclePersistent, + } + case 4: + obj.Spec.VolumeLifecycleModes = []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleEphemeral, + } + case 5: + obj.Spec.VolumeLifecycleModes = []storage.VolumeLifecycleMode{ + storage.VolumeLifecyclePersistent, + storage.VolumeLifecycleEphemeral, + } + case 6: + obj.Spec.VolumeLifecycleModes = []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleEphemeral, + storage.VolumeLifecyclePersistent, + } + } + // match defaulting if obj.Spec.AttachRequired == nil { obj.Spec.AttachRequired = new(bool) @@ -46,6 +78,11 @@ var Funcs = func(codecs runtimeserializer.CodecFactory) []interface{} { obj.Spec.PodInfoOnMount = new(bool) *(obj.Spec.PodInfoOnMount) = false } + if obj.Spec.VolumeLifecycleModes == nil { + obj.Spec.VolumeLifecycleModes = []storage.VolumeLifecycleMode{ + storage.VolumeLifecyclePersistent, + } + } }, } } diff --git a/pkg/apis/storage/types.go b/pkg/apis/storage/types.go index 9a77746a2be..70851be6cda 100644 --- a/pkg/apis/storage/types.go +++ b/pkg/apis/storage/types.go @@ -293,8 +293,47 @@ type CSIDriverSpec struct { // "csi.storage.k8s.io/pod.uid": string(pod.UID) // +optional PodInfoOnMount *bool + + // VolumeLifecycleModes defines what kind of volumes this CSI volume driver supports. + // The default if the list is empty is "Persistent", which is the usage + // defined by the CSI specification and implemented in Kubernetes via the usual + // PV/PVC mechanism. + // The other mode is "Ephemeral". In this mode, volumes are defined inline + // inside the pod spec with CSIVolumeSource and their lifecycle is tied to + // the lifecycle of that pod. A driver has to be aware of this + // because it is only going to get a NodePublishVolume call for such a volume. + // For more information about implementing this mode, see + // https://kubernetes-csi.github.io/docs/ephemeral-local-volumes.html + // A driver can support one or more of these mode and + // more modes may be added in the future. + // +optional + VolumeLifecycleModes []VolumeLifecycleMode } +// VolumeLifecycleMode specifies how a CSI volume is used in Kubernetes. +// More modes may be added in the future. +type VolumeLifecycleMode string + +const ( + // VolumeLifecyclePersistent explicitly confirms that the driver implements + // the full CSI spec. It is the default when CSIDriverSpec.VolumeLifecycleModes is not + // set. Such volumes are managed in Kubernetes via the persistent volume + // claim mechanism and have a lifecycle that is independent of the pods which + // use them. + VolumeLifecyclePersistent VolumeLifecycleMode = "Persistent" + // VolumeLifecycleEphemeral indicates that the driver can be used for + // ephemeral inline volumes. Such volumes are specified inside the pod + // spec with a CSIVolumeSource and, as far as Kubernetes is concerned, have + // a lifecycle that is tied to the lifecycle of the pod. For example, such + // a volume might contain data that gets created specifically for that pod, + // like secrets. + // But how the volume actually gets created and managed is entirely up to + // the driver. It might also use reference counting to share the same volume + // instance among different pods if the CSIVolumeSource of those pods is + // identical. + VolumeLifecycleEphemeral VolumeLifecycleMode = "Ephemeral" +) + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // CSINode holds information about all CSI drivers installed on a node. diff --git a/pkg/apis/storage/v1beta1/defaults.go b/pkg/apis/storage/v1beta1/defaults.go index 30803281659..50cc4fe15c0 100644 --- a/pkg/apis/storage/v1beta1/defaults.go +++ b/pkg/apis/storage/v1beta1/defaults.go @@ -20,6 +20,8 @@ import ( "k8s.io/api/core/v1" storagev1beta1 "k8s.io/api/storage/v1beta1" "k8s.io/apimachinery/pkg/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/kubernetes/pkg/features" ) func addDefaultingFuncs(scheme *runtime.Scheme) error { @@ -47,4 +49,7 @@ func SetDefaults_CSIDriver(obj *storagev1beta1.CSIDriver) { obj.Spec.PodInfoOnMount = new(bool) *(obj.Spec.PodInfoOnMount) = false } + if len(obj.Spec.VolumeLifecycleModes) == 0 && utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { + obj.Spec.VolumeLifecycleModes = append(obj.Spec.VolumeLifecycleModes, storagev1beta1.VolumeLifecyclePersistent) + } } diff --git a/pkg/apis/storage/v1beta1/defaults_test.go b/pkg/apis/storage/v1beta1/defaults_test.go index d1cebcf7b90..a76efa5de24 100644 --- a/pkg/apis/storage/v1beta1/defaults_test.go +++ b/pkg/apis/storage/v1beta1/defaults_test.go @@ -22,8 +22,11 @@ import ( storagev1beta1 "k8s.io/api/storage/v1beta1" "k8s.io/apimachinery/pkg/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" _ "k8s.io/kubernetes/pkg/apis/storage/install" + "k8s.io/kubernetes/pkg/features" ) func roundTrip(t *testing.T, obj runtime.Object) runtime.Object { @@ -81,3 +84,30 @@ func TestSetDefaultAttachRequired(t *testing.T) { t.Errorf("Expected PodInfoOnMount to be defaulted to: %+v, got: %+v", defaultPodInfo, outPodInfo) } } + +func TestSetDefaultVolumeLifecycleModesEnabled(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)() + driver := &storagev1beta1.CSIDriver{} + + // field should be defaulted + defaultMode := storagev1beta1.VolumeLifecyclePersistent + output := roundTrip(t, runtime.Object(driver)).(*storagev1beta1.CSIDriver) + outModes := output.Spec.VolumeLifecycleModes + if len(outModes) != 1 { + t.Errorf("Expected VolumeLifecycleModes to be defaulted to: %+v, got: %+v", defaultMode, outModes) + } else if outModes[0] != defaultMode { + t.Errorf("Expected VolumeLifecycleModes to be defaulted to: %+v, got: %+v", defaultMode, outModes) + } +} + +func TestSetDefaultVolumeLifecycleModesDisabled(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, false)() + driver := &storagev1beta1.CSIDriver{} + + // field should not be defaulted + output := roundTrip(t, runtime.Object(driver)).(*storagev1beta1.CSIDriver) + outModes := output.Spec.VolumeLifecycleModes + if outModes != nil { + t.Errorf("Expected VolumeLifecycleModes to remain nil, got: %+v", outModes) + } +} diff --git a/pkg/registry/storage/csidriver/strategy.go b/pkg/registry/storage/csidriver/strategy.go index 1f534b2e764..a968fcbaf17 100644 --- a/pkg/registry/storage/csidriver/strategy.go +++ b/pkg/registry/storage/csidriver/strategy.go @@ -22,9 +22,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/storage/names" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/storage" "k8s.io/kubernetes/pkg/apis/storage/validation" + "k8s.io/kubernetes/pkg/features" ) // csiDriverStrategy implements behavior for CSIDriver objects @@ -41,8 +43,12 @@ func (csiDriverStrategy) NamespaceScoped() bool { return false } -// ResetBeforeCreate clears the Status field which is not allowed to be set by end users on creation. +// PrepareForCreate clears the VolumeLifecycleModes field if the corresponding feature is disabled. func (csiDriverStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { + if !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { + csiDriver := obj.(*storage.CSIDriver) + csiDriver.Spec.VolumeLifecycleModes = nil + } } func (csiDriverStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { @@ -62,8 +68,15 @@ func (csiDriverStrategy) AllowCreateOnUpdate() bool { return false } -// PrepareForUpdate sets the Status fields which is not allowed to be set by an end user updating a CSIDriver +// PrepareForUpdate clears the VolumeLifecycleModes field if the corresponding feature is disabled and +// existing object does not already have that field set. This allows the field to remain when +// downgrading to a version that has the feature disabled. func (csiDriverStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { + if old.(*storage.CSIDriver).Spec.VolumeLifecycleModes == nil && + !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { + newCSIDriver := obj.(*storage.CSIDriver) + newCSIDriver.Spec.VolumeLifecycleModes = nil + } } func (csiDriverStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { diff --git a/pkg/registry/storage/csidriver/strategy_test.go b/pkg/registry/storage/csidriver/strategy_test.go index e5334fc1edc..442b1005e8d 100644 --- a/pkg/registry/storage/csidriver/strategy_test.go +++ b/pkg/registry/storage/csidriver/strategy_test.go @@ -19,10 +19,14 @@ package csidriver import ( "testing" + "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/apis/storage" + "k8s.io/kubernetes/pkg/features" ) func getValidCSIDriver(name string) *storage.CSIDriver { @@ -74,6 +78,169 @@ func TestCSIDriverStrategy(t *testing.T) { } } +func TestCSIDriverPrepareForCreate(t *testing.T) { + ctx := genericapirequest.WithRequestInfo(genericapirequest.NewContext(), &genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1beta1", + Resource: "csidrivers", + }) + + attachRequired := true + podInfoOnMount := true + csiDriver := &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachRequired, + PodInfoOnMount: &podInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecyclePersistent, + }, + }, + } + + tests := []struct { + name string + withInline bool + }{ + { + name: "inline enabled", + withInline: true, + }, + { + name: "inline disabled", + withInline: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, test.withInline)() + + Strategy.PrepareForCreate(ctx, csiDriver) + errs := Strategy.Validate(ctx, csiDriver) + if len(errs) != 0 { + t.Errorf("unexpected validating errors: %v", errs) + } + if test.withInline { + if len(csiDriver.Spec.VolumeLifecycleModes) != 1 { + t.Errorf("VolumeLifecycleModes modified: %v", csiDriver.Spec) + } + } else { + if len(csiDriver.Spec.VolumeLifecycleModes) != 0 { + t.Errorf("VolumeLifecycleModes not stripped: %v", csiDriver.Spec) + } + } + }) + } +} + +func TestCSIDriverPrepareForUpdate(t *testing.T) { + ctx := genericapirequest.WithRequestInfo(genericapirequest.NewContext(), &genericapirequest.RequestInfo{ + APIGroup: "storage.k8s.io", + APIVersion: "v1beta1", + Resource: "csidrivers", + }) + + attachRequired := true + podInfoOnMount := true + driverWithoutModes := &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachRequired, + PodInfoOnMount: &podInfoOnMount, + }, + } + driverWithPersistent := &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachRequired, + PodInfoOnMount: &podInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecyclePersistent, + }, + }, + } + driverWithEphemeral := &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachRequired, + PodInfoOnMount: &podInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleEphemeral, + }, + }, + } + var resultEmpty []storage.VolumeLifecycleMode + resultPersistent := []storage.VolumeLifecycleMode{storage.VolumeLifecyclePersistent} + resultEphemeral := []storage.VolumeLifecycleMode{storage.VolumeLifecycleEphemeral} + + tests := []struct { + name string + old, update *storage.CSIDriver + withInline, withoutInline []storage.VolumeLifecycleMode + }{ + { + name: "before: no mode, update: no mode", + old: driverWithoutModes, + update: driverWithoutModes, + withInline: resultEmpty, + withoutInline: resultEmpty, + }, + { + name: "before: no mode, update: persistent", + old: driverWithoutModes, + update: driverWithPersistent, + withInline: resultPersistent, + withoutInline: resultEmpty, + }, + { + name: "before: persistent, update: ephemeral", + old: driverWithPersistent, + update: driverWithEphemeral, + withInline: resultEphemeral, + withoutInline: resultEphemeral, + }, + { + name: "before: persistent, update: no mode", + old: driverWithPersistent, + update: driverWithoutModes, + withInline: resultEmpty, + withoutInline: resultEmpty, + }, + } + + runAll := func(t *testing.T, withInline bool) { + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, withInline)() + + csiDriver := test.update.DeepCopy() + Strategy.PrepareForUpdate(ctx, csiDriver, test.old) + if withInline { + require.Equal(t, csiDriver.Spec.VolumeLifecycleModes, test.withInline) + } else { + require.Equal(t, csiDriver.Spec.VolumeLifecycleModes, test.withoutInline) + } + }) + } + } + + t.Run("with inline volumes", func(t *testing.T) { + runAll(t, true) + }) + t.Run("without inline volumes", func(t *testing.T) { + runAll(t, false) + }) +} + func TestCSIDriverValidation(t *testing.T) { attachRequired := true notAttachRequired := false diff --git a/staging/src/k8s.io/api/storage/v1beta1/types.go b/staging/src/k8s.io/api/storage/v1beta1/types.go index 762fcfcd001..130f6a4c01e 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/types.go +++ b/staging/src/k8s.io/api/storage/v1beta1/types.go @@ -291,10 +291,59 @@ type CSIDriverSpec struct { // "csi.storage.k8s.io/pod.name": pod.Name // "csi.storage.k8s.io/pod.namespace": pod.Namespace // "csi.storage.k8s.io/pod.uid": string(pod.UID) + // "csi.storage.k8s.io/ephemeral": "true" iff the volume is an ephemeral inline volume + // defined by a CSIVolumeSource, otherwise "false" + // + // "csi.storage.k8s.io/ephemeral" is a new feature in Kubernetes 1.16. It is only + // required for drivers which support both the "Persistent" and "Ephemeral" VolumeLifecycleMode. + // Other drivers can leave pod info disabled and/or ignore this field. + // As Kubernetes 1.15 doesn't support this field, drivers can only support one mode when + // deployed on such a cluster and the deployment determines which mode that is, for example + // via a command line parameter of the driver. // +optional PodInfoOnMount *bool `json:"podInfoOnMount,omitempty" protobuf:"bytes,2,opt,name=podInfoOnMount"` + + // VolumeLifecycleModes defines what kind of volumes this CSI volume driver supports. + // The default if the list is empty is "Persistent", which is the usage + // defined by the CSI specification and implemented in Kubernetes via the usual + // PV/PVC mechanism. + // The other mode is "Ephemeral". In this mode, volumes are defined inline + // inside the pod spec with CSIVolumeSource and their lifecycle is tied to + // the lifecycle of that pod. A driver has to be aware of this + // because it is only going to get a NodePublishVolume call for such a volume. + // For more information about implementing this mode, see + // https://kubernetes-csi.github.io/docs/ephemeral-local-volumes.html + // A driver can support one or more of these modes and + // more modes may be added in the future. + // +optional + VolumeLifecycleModes []VolumeLifecycleMode `json:"volumeLifecycleModes,omitempty" protobuf:"bytes,3,opt,name=volumeLifecycleModes"` } +// VolumeLifecycleMode is an enumeration of possible usage modes for a volume +// provided by a CSI driver. More modes may be added in the future. +type VolumeLifecycleMode string + +const ( + // VolumeLifecyclePersistent explicitly confirms that the driver implements + // the full CSI spec. It is the default when CSIDriverSpec.VolumeLifecycleModes is not + // set. Such volumes are managed in Kubernetes via the persistent volume + // claim mechanism and have a lifecycle that is independent of the pods which + // use them. + VolumeLifecyclePersistent VolumeLifecycleMode = "Persistent" + + // VolumeLifecycleEphemeral indicates that the driver can be used for + // ephemeral inline volumes. Such volumes are specified inside the pod + // spec with a CSIVolumeSource and, as far as Kubernetes is concerned, have + // a lifecycle that is tied to the lifecycle of the pod. For example, such + // a volume might contain data that gets created specifically for that pod, + // like secrets. + // But how the volume actually gets created and managed is entirely up to + // the driver. It might also use reference counting to share the same volume + // instance among different pods if the CSIVolumeSource of those pods is + // identical. + VolumeLifecycleEphemeral VolumeLifecycleMode = "Ephemeral" +) + // +genclient // +genclient:nonNamespaced // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object From f7742fd31ac581d12f3aa93436fd3fed5efed543 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 24 Jul 2019 08:16:43 +0200 Subject: [PATCH 2/6] storage: generated files for CSIDriver.Spec.VolumeLifecycleModes --- api/openapi-spec/swagger.json | 9 +- pkg/apis/storage/v1beta1/BUILD | 5 + .../v1beta1/zz_generated.conversion.go | 2 + pkg/apis/storage/zz_generated.deepcopy.go | 5 + pkg/registry/storage/csidriver/BUILD | 6 + .../api/storage/v1beta1/generated.pb.go | 216 +++++++++++------- .../api/storage/v1beta1/generated.proto | 24 ++ .../v1beta1/types_swagger_doc_generated.go | 7 +- .../storage/v1beta1/zz_generated.deepcopy.go | 5 + .../storage.k8s.io.v1beta1.CSIDriver.json | 5 +- .../HEAD/storage.k8s.io.v1beta1.CSIDriver.pb | Bin 222 -> 257 bytes .../storage.k8s.io.v1beta1.CSIDriver.yaml | 2 + 12 files changed, 198 insertions(+), 88 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index b77e3c3d0b9..6782ea60c49 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -16766,8 +16766,15 @@ "type": "boolean" }, "podInfoOnMount": { - "description": "If set to true, podInfoOnMount indicates this CSI volume driver requires additional pod information (like podName, podUID, etc.) during mount operations. If set to false, pod information will not be passed on mount. Default is false. The CSI driver specifies podInfoOnMount as part of driver deployment. If true, Kubelet will pass pod information as VolumeContext in the CSI NodePublishVolume() calls. The CSI driver is responsible for parsing and validating the information passed in as VolumeContext. The following VolumeConext will be passed if podInfoOnMount is set to true. This list might grow, but the prefix will be used. \"csi.storage.k8s.io/pod.name\": pod.Name \"csi.storage.k8s.io/pod.namespace\": pod.Namespace \"csi.storage.k8s.io/pod.uid\": string(pod.UID)", + "description": "If set to true, podInfoOnMount indicates this CSI volume driver requires additional pod information (like podName, podUID, etc.) during mount operations. If set to false, pod information will not be passed on mount. Default is false. The CSI driver specifies podInfoOnMount as part of driver deployment. If true, Kubelet will pass pod information as VolumeContext in the CSI NodePublishVolume() calls. The CSI driver is responsible for parsing and validating the information passed in as VolumeContext. The following VolumeConext will be passed if podInfoOnMount is set to true. This list might grow, but the prefix will be used. \"csi.storage.k8s.io/pod.name\": pod.Name \"csi.storage.k8s.io/pod.namespace\": pod.Namespace \"csi.storage.k8s.io/pod.uid\": string(pod.UID) \"csi.storage.k8s.io/ephemeral\": \"true\" iff the volume is an ephemeral inline volume\n defined by a CSIVolumeSource, otherwise \"false\"\n\n\"csi.storage.k8s.io/ephemeral\" is a new feature in Kubernetes 1.16. It is only required for drivers which support both the \"Persistent\" and \"Ephemeral\" VolumeLifecycleMode. Other drivers can leave pod info disabled and/or ignore this field. As Kubernetes 1.15 doesn't support this field, drivers can only support one mode when deployed on such a cluster and the deployment determines which mode that is, for example via a command line parameter of the driver.", "type": "boolean" + }, + "volumeLifecycleModes": { + "description": "VolumeLifecycleModes defines what kind of volumes this CSI volume driver supports. The default if the list is empty is \"Persistent\", which is the usage defined by the CSI specification and implemented in Kubernetes via the usual PV/PVC mechanism. The other mode is \"Ephemeral\". In this mode, volumes are defined inline inside the pod spec with CSIVolumeSource and their lifecycle is tied to the lifecycle of that pod. A driver has to be aware of this because it is only going to get a NodePublishVolume call for such a volume. For more information about implementing this mode, see https://kubernetes-csi.github.io/docs/ephemeral-local-volumes.html A driver can support one or more of these modes and more modes may be added in the future.", + "items": { + "type": "string" + }, + "type": "array" } }, "type": "object" diff --git a/pkg/apis/storage/v1beta1/BUILD b/pkg/apis/storage/v1beta1/BUILD index fc1f7a4ce9b..ebc6624e70c 100644 --- a/pkg/apis/storage/v1beta1/BUILD +++ b/pkg/apis/storage/v1beta1/BUILD @@ -20,11 +20,13 @@ go_library( "//pkg/apis/core:go_default_library", "//pkg/apis/core/v1:go_default_library", "//pkg/apis/storage:go_default_library", + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", ], ) @@ -51,7 +53,10 @@ go_test( deps = [ "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/storage/install:go_default_library", + "//pkg/features:go_default_library", "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", ], ) diff --git a/pkg/apis/storage/v1beta1/zz_generated.conversion.go b/pkg/apis/storage/v1beta1/zz_generated.conversion.go index 516710c07fa..7994f458a8f 100644 --- a/pkg/apis/storage/v1beta1/zz_generated.conversion.go +++ b/pkg/apis/storage/v1beta1/zz_generated.conversion.go @@ -253,6 +253,7 @@ func Convert_storage_CSIDriverList_To_v1beta1_CSIDriverList(in *storage.CSIDrive func autoConvert_v1beta1_CSIDriverSpec_To_storage_CSIDriverSpec(in *v1beta1.CSIDriverSpec, out *storage.CSIDriverSpec, s conversion.Scope) error { out.AttachRequired = (*bool)(unsafe.Pointer(in.AttachRequired)) out.PodInfoOnMount = (*bool)(unsafe.Pointer(in.PodInfoOnMount)) + out.VolumeLifecycleModes = *(*[]storage.VolumeLifecycleMode)(unsafe.Pointer(&in.VolumeLifecycleModes)) return nil } @@ -264,6 +265,7 @@ func Convert_v1beta1_CSIDriverSpec_To_storage_CSIDriverSpec(in *v1beta1.CSIDrive func autoConvert_storage_CSIDriverSpec_To_v1beta1_CSIDriverSpec(in *storage.CSIDriverSpec, out *v1beta1.CSIDriverSpec, s conversion.Scope) error { out.AttachRequired = (*bool)(unsafe.Pointer(in.AttachRequired)) out.PodInfoOnMount = (*bool)(unsafe.Pointer(in.PodInfoOnMount)) + out.VolumeLifecycleModes = *(*[]v1beta1.VolumeLifecycleMode)(unsafe.Pointer(&in.VolumeLifecycleModes)) return nil } diff --git a/pkg/apis/storage/zz_generated.deepcopy.go b/pkg/apis/storage/zz_generated.deepcopy.go index 55c5583cf2e..5c9ee27e278 100644 --- a/pkg/apis/storage/zz_generated.deepcopy.go +++ b/pkg/apis/storage/zz_generated.deepcopy.go @@ -98,6 +98,11 @@ func (in *CSIDriverSpec) DeepCopyInto(out *CSIDriverSpec) { *out = new(bool) **out = **in } + if in.VolumeLifecycleModes != nil { + in, out := &in.VolumeLifecycleModes, &out.VolumeLifecycleModes + *out = make([]VolumeLifecycleMode, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/registry/storage/csidriver/BUILD b/pkg/registry/storage/csidriver/BUILD index 41953486c85..f565a857d82 100644 --- a/pkg/registry/storage/csidriver/BUILD +++ b/pkg/registry/storage/csidriver/BUILD @@ -12,9 +12,11 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/storage:go_default_library", "//pkg/apis/storage/validation:go_default_library", + "//pkg/features:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", ], ) @@ -41,8 +43,12 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/apis/storage:go_default_library", + "//pkg/features:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library", + "//vendor/github.com/stretchr/testify/require:go_default_library", ], ) diff --git a/staging/src/k8s.io/api/storage/v1beta1/generated.pb.go b/staging/src/k8s.io/api/storage/v1beta1/generated.pb.go index 677d366f5ef..cd35af34f86 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/generated.pb.go +++ b/staging/src/k8s.io/api/storage/v1beta1/generated.pb.go @@ -520,89 +520,91 @@ func init() { } var fileDescriptor_7d2980599fd0de80 = []byte{ - // 1311 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x4d, 0x6f, 0x1b, 0x45, - 0x18, 0xce, 0xc6, 0xf9, 0x1c, 0x27, 0xad, 0x33, 0x8d, 0xc0, 0xf8, 0x60, 0x47, 0x46, 0xd0, 0xb4, - 0x6a, 0xd7, 0x6d, 0x55, 0xaa, 0xaa, 0x12, 0x87, 0x6c, 0x1a, 0x09, 0xb7, 0x75, 0x1a, 0x26, 0x51, - 0x85, 0x2a, 0x0e, 0x8c, 0x77, 0xdf, 0x3a, 0xdb, 0x78, 0x77, 0xb6, 0x33, 0x63, 0x43, 0x6e, 0x9c, - 0xe0, 0x8a, 0x38, 0xf0, 0x0b, 0xf8, 0x0b, 0x20, 0xc1, 0x85, 0x23, 0x3d, 0xa1, 0x8a, 0x53, 0x4f, - 0x16, 0x5d, 0x7e, 0x02, 0xb7, 0x88, 0x03, 0x9a, 0xd9, 0x89, 0x77, 0xfd, 0xd5, 0x24, 0x1c, 0x72, - 0xf3, 0xbc, 0x1f, 0xcf, 0xfb, 0xf5, 0xcc, 0x3b, 0x6b, 0xb4, 0x79, 0x70, 0x57, 0xd8, 0x3e, 0xab, - 0x1d, 0x74, 0x9a, 0xc0, 0x43, 0x90, 0x20, 0x6a, 0x5d, 0x08, 0x3d, 0xc6, 0x6b, 0x46, 0x41, 0x23, - 0xbf, 0x26, 0x24, 0xe3, 0xb4, 0x05, 0xb5, 0xee, 0xcd, 0x26, 0x48, 0x7a, 0xb3, 0xd6, 0x82, 0x10, - 0x38, 0x95, 0xe0, 0xd9, 0x11, 0x67, 0x92, 0xe1, 0x52, 0x62, 0x6b, 0xd3, 0xc8, 0xb7, 0x8d, 0xad, - 0x6d, 0x6c, 0x4b, 0xd7, 0x5b, 0xbe, 0xdc, 0xef, 0x34, 0x6d, 0x97, 0x05, 0xb5, 0x16, 0x6b, 0xb1, - 0x9a, 0x76, 0x69, 0x76, 0x9e, 0xe9, 0x93, 0x3e, 0xe8, 0x5f, 0x09, 0x54, 0xa9, 0x9a, 0x09, 0xeb, - 0x32, 0xae, 0x62, 0x0e, 0x87, 0x2b, 0xdd, 0x4e, 0x6d, 0x02, 0xea, 0xee, 0xfb, 0x21, 0xf0, 0xc3, - 0x5a, 0x74, 0xd0, 0x52, 0x02, 0x51, 0x0b, 0x40, 0xd2, 0x71, 0x5e, 0xb5, 0x49, 0x5e, 0xbc, 0x13, - 0x4a, 0x3f, 0x80, 0x11, 0x87, 0x3b, 0x27, 0x39, 0x08, 0x77, 0x1f, 0x02, 0x3a, 0xec, 0x57, 0xfd, - 0xd5, 0x42, 0x8b, 0x9b, 0xbb, 0xf5, 0xfb, 0xdc, 0xef, 0x02, 0xc7, 0x5f, 0xa0, 0x05, 0x95, 0x91, - 0x47, 0x25, 0x2d, 0x5a, 0x6b, 0xd6, 0x7a, 0xfe, 0xd6, 0x0d, 0x3b, 0x6d, 0x57, 0x1f, 0xd8, 0x8e, - 0x0e, 0x5a, 0x4a, 0x20, 0x6c, 0x65, 0x6d, 0x77, 0x6f, 0xda, 0x8f, 0x9b, 0xcf, 0xc1, 0x95, 0x0d, - 0x90, 0xd4, 0xc1, 0x2f, 0x7b, 0x95, 0xa9, 0xb8, 0x57, 0x41, 0xa9, 0x8c, 0xf4, 0x51, 0xf1, 0x43, - 0x34, 0x23, 0x22, 0x70, 0x8b, 0xd3, 0x1a, 0xfd, 0x8a, 0x3d, 0x79, 0x18, 0x76, 0x3f, 0xad, 0xdd, - 0x08, 0x5c, 0x67, 0xc9, 0xc0, 0xce, 0xa8, 0x13, 0xd1, 0x20, 0xd5, 0x5f, 0x2c, 0xb4, 0xdc, 0xb7, - 0x7a, 0xe4, 0x0b, 0x89, 0x3f, 0x1f, 0x29, 0xc0, 0x3e, 0x5d, 0x01, 0xca, 0x5b, 0xa7, 0x5f, 0x30, - 0x71, 0x16, 0x8e, 0x25, 0x99, 0xe4, 0x1f, 0xa0, 0x59, 0x5f, 0x42, 0x20, 0x8a, 0xd3, 0x6b, 0xb9, - 0xf5, 0xfc, 0xad, 0x0f, 0x4e, 0x95, 0xbd, 0xb3, 0x6c, 0x10, 0x67, 0xeb, 0xca, 0x97, 0x24, 0x10, - 0xd5, 0x6f, 0xb3, 0xb9, 0xab, 0x9a, 0xf0, 0x3d, 0x74, 0x81, 0x4a, 0x49, 0xdd, 0x7d, 0x02, 0x2f, - 0x3a, 0x3e, 0x07, 0x4f, 0x57, 0xb0, 0xe0, 0xe0, 0xb8, 0x57, 0xb9, 0xb0, 0x31, 0xa0, 0x21, 0x43, - 0x96, 0xca, 0x37, 0x62, 0x5e, 0x3d, 0x7c, 0xc6, 0x1e, 0x87, 0x0d, 0xd6, 0x09, 0xa5, 0x6e, 0xb0, - 0xf1, 0xdd, 0x19, 0xd0, 0x90, 0x21, 0xcb, 0xea, 0xcf, 0x16, 0x9a, 0xdf, 0xdc, 0xad, 0x6f, 0x33, - 0x0f, 0xce, 0x81, 0x00, 0xf5, 0x01, 0x02, 0x5c, 0x3e, 0xa1, 0x85, 0x2a, 0xa9, 0x89, 0xe3, 0xff, - 0x27, 0x69, 0xa1, 0xb2, 0x31, 0xfc, 0x5d, 0x43, 0x33, 0x21, 0x0d, 0x40, 0xa7, 0xbe, 0x98, 0xfa, - 0x6c, 0xd3, 0x00, 0x88, 0xd6, 0xe0, 0x0f, 0xd1, 0x5c, 0xc8, 0x3c, 0xa8, 0xdf, 0xd7, 0x09, 0x2c, - 0x3a, 0x17, 0x8c, 0xcd, 0xdc, 0xb6, 0x96, 0x12, 0xa3, 0xc5, 0xb7, 0xd1, 0x92, 0x64, 0x11, 0x6b, - 0xb3, 0xd6, 0xe1, 0x43, 0x38, 0x14, 0xc5, 0xdc, 0x5a, 0x6e, 0x7d, 0xd1, 0x29, 0xc4, 0xbd, 0xca, - 0xd2, 0x5e, 0x46, 0x4e, 0x06, 0xac, 0x70, 0x13, 0xe5, 0x69, 0xbb, 0xcd, 0x5c, 0x2a, 0x69, 0xb3, - 0x0d, 0xc5, 0x19, 0x5d, 0x63, 0xed, 0x6d, 0x35, 0x3e, 0x61, 0xed, 0x4e, 0x00, 0x2a, 0x38, 0x01, - 0xc1, 0x3a, 0xdc, 0x05, 0xe1, 0x5c, 0x8c, 0x7b, 0x95, 0xfc, 0x46, 0x8a, 0x43, 0xb2, 0xa0, 0xd5, - 0x9f, 0x2c, 0x94, 0x37, 0x55, 0x9f, 0x03, 0xe5, 0x3f, 0x19, 0xa4, 0xfc, 0xfb, 0xa7, 0x98, 0xd7, - 0x04, 0xc2, 0xbb, 0xfd, 0xb4, 0x35, 0xdb, 0xf7, 0xd0, 0xbc, 0xa7, 0x87, 0x26, 0x8a, 0x96, 0x86, - 0xbe, 0x72, 0x0a, 0x68, 0x73, 0xa3, 0x2e, 0x9a, 0x00, 0xf3, 0xc9, 0x59, 0x90, 0x63, 0xa8, 0xea, - 0xf7, 0x73, 0x68, 0x69, 0x37, 0xf1, 0xdd, 0x6c, 0x53, 0x21, 0xce, 0x81, 0xd0, 0x1f, 0xa1, 0x7c, - 0xc4, 0x59, 0xd7, 0x17, 0x3e, 0x0b, 0x81, 0x1b, 0x5a, 0x5d, 0x32, 0x2e, 0xf9, 0x9d, 0x54, 0x45, - 0xb2, 0x76, 0xb8, 0x8d, 0x50, 0x44, 0x39, 0x0d, 0x40, 0xaa, 0x16, 0xe4, 0x74, 0x0b, 0xee, 0xbe, - 0xad, 0x05, 0xd9, 0xb2, 0xec, 0x9d, 0xbe, 0xeb, 0x56, 0x28, 0xf9, 0x61, 0x9a, 0x62, 0xaa, 0x20, - 0x19, 0x7c, 0x7c, 0x80, 0x96, 0x39, 0xb8, 0x6d, 0xea, 0x07, 0x3b, 0xac, 0xed, 0xbb, 0x87, 0x9a, - 0x9a, 0x8b, 0xce, 0x56, 0xdc, 0xab, 0x2c, 0x93, 0xac, 0xe2, 0xa8, 0x57, 0xb9, 0x31, 0xfa, 0xaa, - 0xd9, 0x3b, 0xc0, 0x85, 0x2f, 0x24, 0x84, 0x32, 0x21, 0xec, 0x80, 0x0f, 0x19, 0xc4, 0x56, 0x77, - 0x27, 0x50, 0x9b, 0xe5, 0x71, 0x24, 0x7d, 0x16, 0x8a, 0xe2, 0x6c, 0x7a, 0x77, 0x1a, 0x19, 0x39, - 0x19, 0xb0, 0xc2, 0x8f, 0xd0, 0xaa, 0xa2, 0xf9, 0x97, 0x49, 0x80, 0xad, 0xaf, 0x22, 0x1a, 0xaa, - 0x56, 0x15, 0xe7, 0xf4, 0x22, 0x2b, 0xc6, 0xbd, 0xca, 0xea, 0xc6, 0x18, 0x3d, 0x19, 0xeb, 0x85, - 0x3f, 0x43, 0x2b, 0x5d, 0x2d, 0x72, 0xfc, 0xd0, 0xf3, 0xc3, 0x56, 0x83, 0x79, 0x50, 0x9c, 0xd7, - 0x45, 0x5f, 0x8d, 0x7b, 0x95, 0x95, 0x27, 0xc3, 0xca, 0xa3, 0x71, 0x42, 0x32, 0x0a, 0x82, 0x5f, - 0xa0, 0x15, 0x1d, 0x11, 0x3c, 0xb3, 0x08, 0x7c, 0x10, 0xc5, 0x05, 0x3d, 0xbf, 0xf5, 0xec, 0xfc, - 0x54, 0xeb, 0x14, 0x91, 0x8e, 0xd7, 0xc5, 0x2e, 0xb4, 0xc1, 0x95, 0x8c, 0xef, 0x01, 0x0f, 0x9c, - 0xf7, 0xcc, 0xbc, 0x56, 0x36, 0x86, 0xa1, 0xc8, 0x28, 0x7a, 0xe9, 0x63, 0x74, 0x71, 0x68, 0xe0, - 0xb8, 0x80, 0x72, 0x07, 0x70, 0x98, 0x2c, 0x3a, 0xa2, 0x7e, 0xe2, 0x55, 0x34, 0xdb, 0xa5, 0xed, - 0x0e, 0x24, 0x0c, 0x24, 0xc9, 0xe1, 0xde, 0xf4, 0x5d, 0xab, 0xfa, 0x9b, 0x85, 0x0a, 0x59, 0xf6, - 0x9c, 0xc3, 0xda, 0x68, 0x0c, 0xae, 0x8d, 0xf5, 0xd3, 0x12, 0x7b, 0xc2, 0xee, 0xf8, 0x71, 0x1a, - 0x15, 0x92, 0xe1, 0x24, 0xef, 0x60, 0x00, 0xa1, 0x3c, 0x87, 0xab, 0x4d, 0x06, 0xde, 0xaa, 0x1b, - 0x27, 0xef, 0xf1, 0x34, 0xbb, 0x49, 0x8f, 0x16, 0x7e, 0x8a, 0xe6, 0x84, 0xa4, 0xb2, 0xa3, 0xee, - 0xbc, 0x42, 0xbd, 0x75, 0x26, 0x54, 0xed, 0x99, 0x3e, 0x5a, 0xc9, 0x99, 0x18, 0xc4, 0xea, 0xef, - 0x16, 0x5a, 0x1d, 0x76, 0x39, 0x87, 0x61, 0x7f, 0x3a, 0x38, 0xec, 0x6b, 0x67, 0xa9, 0x68, 0xc2, - 0xc0, 0xff, 0xb4, 0xd0, 0x3b, 0x23, 0xc5, 0xeb, 0xe7, 0x51, 0xed, 0x89, 0x68, 0x68, 0x1b, 0x6d, - 0xa7, 0x6f, 0xbe, 0xde, 0x13, 0x3b, 0x63, 0xf4, 0x64, 0xac, 0x17, 0x7e, 0x8e, 0x0a, 0x7e, 0xd8, - 0xf6, 0x43, 0x48, 0x64, 0xbb, 0xe9, 0xb8, 0xc7, 0x5e, 0xe6, 0x61, 0x64, 0x3d, 0xe6, 0xd5, 0xb8, - 0x57, 0x29, 0xd4, 0x87, 0x50, 0xc8, 0x08, 0x6e, 0xf5, 0x8f, 0x31, 0xe3, 0xd1, 0x6f, 0xe1, 0x35, - 0xb4, 0x90, 0x7c, 0xcf, 0x01, 0x37, 0x65, 0xf4, 0xdb, 0xbd, 0x61, 0xe4, 0xa4, 0x6f, 0xa1, 0x19, - 0xa4, 0x5b, 0x61, 0x12, 0x3d, 0x1b, 0x83, 0xb4, 0x67, 0x86, 0x41, 0xfa, 0x4c, 0x0c, 0xa2, 0xca, - 0x44, 0x7d, 0x00, 0xe9, 0x86, 0xe6, 0x06, 0x33, 0xd9, 0x36, 0x72, 0xd2, 0xb7, 0xa8, 0xfe, 0x9b, - 0x1b, 0x33, 0x25, 0x4d, 0xc5, 0x4c, 0x49, 0xc7, 0x9f, 0xb1, 0xc3, 0x25, 0x79, 0xfd, 0x92, 0x3c, - 0xfc, 0x83, 0x85, 0x30, 0xed, 0x43, 0x34, 0x8e, 0xa9, 0x9a, 0xf0, 0xe9, 0xc1, 0xd9, 0x6f, 0x88, - 0xbd, 0x31, 0x02, 0x96, 0xbc, 0x93, 0x25, 0x93, 0x04, 0x1e, 0x35, 0x20, 0x63, 0x32, 0xc0, 0x3e, - 0xca, 0x27, 0xd2, 0x2d, 0xce, 0x19, 0x37, 0x57, 0xf6, 0xf2, 0xc9, 0x09, 0x69, 0x73, 0xa7, 0xac, - 0x3f, 0xe4, 0x52, 0xff, 0xa3, 0x5e, 0x25, 0x9f, 0xd1, 0x93, 0x2c, 0xb6, 0x0a, 0xe5, 0x41, 0x1a, - 0x6a, 0xe6, 0x7f, 0x84, 0xba, 0x0f, 0x93, 0x43, 0x65, 0xb0, 0x4b, 0x5b, 0xe8, 0xdd, 0x09, 0x0d, - 0x3a, 0xd3, 0xbb, 0xf2, 0x8d, 0x85, 0xb2, 0x31, 0xf0, 0x23, 0x34, 0xa3, 0xfe, 0x6a, 0x9a, 0x0d, - 0x73, 0xf5, 0x74, 0x1b, 0x66, 0xcf, 0x0f, 0x20, 0x5d, 0x94, 0xea, 0x44, 0x34, 0x0a, 0xbe, 0x82, - 0xe6, 0x03, 0x10, 0x82, 0xb6, 0x4c, 0xe4, 0xf4, 0xab, 0xaf, 0x91, 0x88, 0xc9, 0xb1, 0xbe, 0x7a, - 0x07, 0x5d, 0x1a, 0xf3, 0x1d, 0x8d, 0x2b, 0x68, 0xd6, 0xd5, 0xff, 0x85, 0x54, 0x42, 0xb3, 0xce, - 0xa2, 0xda, 0x32, 0x9b, 0xfa, 0x2f, 0x50, 0x22, 0x77, 0xae, 0xbf, 0x7c, 0x53, 0x9e, 0x7a, 0xf5, - 0xa6, 0x3c, 0xf5, 0xfa, 0x4d, 0x79, 0xea, 0xeb, 0xb8, 0x6c, 0xbd, 0x8c, 0xcb, 0xd6, 0xab, 0xb8, - 0x6c, 0xbd, 0x8e, 0xcb, 0xd6, 0x5f, 0x71, 0xd9, 0xfa, 0xee, 0xef, 0xf2, 0xd4, 0xd3, 0x79, 0xd3, - 0xef, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0xce, 0x65, 0xbb, 0xc7, 0x7f, 0x10, 0x00, 0x00, + // 1344 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0xbd, 0x6f, 0xdb, 0x46, + 0x1b, 0x37, 0x2d, 0x7f, 0x9e, 0xec, 0x44, 0xbe, 0x18, 0xef, 0xab, 0x57, 0x83, 0x64, 0xe8, 0x45, + 0x1b, 0x27, 0x48, 0xc8, 0x24, 0x48, 0x83, 0x20, 0x40, 0x07, 0xd3, 0x31, 0x50, 0x25, 0x96, 0xe3, + 0x9e, 0x8d, 0xa0, 0x08, 0x3a, 0xf4, 0x44, 0x3e, 0x91, 0x19, 0x93, 0x3c, 0x86, 0x3c, 0xa9, 0xd5, + 0xd6, 0xa9, 0x73, 0xd1, 0xa1, 0x7f, 0x41, 0xff, 0x85, 0x16, 0x68, 0x97, 0x8e, 0xcd, 0x54, 0x04, + 0x9d, 0x32, 0x09, 0x0d, 0xbb, 0x76, 0xeb, 0x66, 0x74, 0x28, 0xee, 0x78, 0x12, 0x29, 0x89, 0x8a, + 0xed, 0x0e, 0xde, 0x78, 0xcf, 0xc7, 0xef, 0xf9, 0x7e, 0xee, 0x88, 0xb6, 0x8f, 0xef, 0x47, 0xba, + 0xc3, 0x8c, 0xe3, 0x4e, 0x0b, 0x42, 0x1f, 0x38, 0x44, 0x46, 0x17, 0x7c, 0x9b, 0x85, 0x86, 0x62, + 0xd0, 0xc0, 0x31, 0x22, 0xce, 0x42, 0xda, 0x06, 0xa3, 0x7b, 0xbb, 0x05, 0x9c, 0xde, 0x36, 0xda, + 0xe0, 0x43, 0x48, 0x39, 0xd8, 0x7a, 0x10, 0x32, 0xce, 0x70, 0x25, 0x91, 0xd5, 0x69, 0xe0, 0xe8, + 0x4a, 0x56, 0x57, 0xb2, 0x95, 0x9b, 0x6d, 0x87, 0x1f, 0x75, 0x5a, 0xba, 0xc5, 0x3c, 0xa3, 0xcd, + 0xda, 0xcc, 0x90, 0x2a, 0xad, 0xce, 0x73, 0x79, 0x92, 0x07, 0xf9, 0x95, 0x40, 0x55, 0xea, 0x19, + 0xb3, 0x16, 0x0b, 0x85, 0xcd, 0x71, 0x73, 0x95, 0xbb, 0xa9, 0x8c, 0x47, 0xad, 0x23, 0xc7, 0x87, + 0xb0, 0x67, 0x04, 0xc7, 0x6d, 0x41, 0x88, 0x0c, 0x0f, 0x38, 0xcd, 0xd3, 0x32, 0xa6, 0x69, 0x85, + 0x1d, 0x9f, 0x3b, 0x1e, 0x4c, 0x28, 0xdc, 0x3b, 0x4d, 0x21, 0xb2, 0x8e, 0xc0, 0xa3, 0xe3, 0x7a, + 0xf5, 0x9f, 0x34, 0xb4, 0xbc, 0x7d, 0xd0, 0x78, 0x18, 0x3a, 0x5d, 0x08, 0xf1, 0x67, 0x68, 0x49, + 0x78, 0x64, 0x53, 0x4e, 0xcb, 0xda, 0x86, 0xb6, 0x59, 0xbc, 0x73, 0x4b, 0x4f, 0xd3, 0x35, 0x04, + 0xd6, 0x83, 0xe3, 0xb6, 0x20, 0x44, 0xba, 0x90, 0xd6, 0xbb, 0xb7, 0xf5, 0x27, 0xad, 0x17, 0x60, + 0xf1, 0x26, 0x70, 0x6a, 0xe2, 0x57, 0xfd, 0xda, 0x4c, 0xdc, 0xaf, 0xa1, 0x94, 0x46, 0x86, 0xa8, + 0xf8, 0x31, 0x9a, 0x8b, 0x02, 0xb0, 0xca, 0xb3, 0x12, 0xfd, 0x9a, 0x3e, 0xbd, 0x18, 0xfa, 0xd0, + 0xad, 0x83, 0x00, 0x2c, 0x73, 0x45, 0xc1, 0xce, 0x89, 0x13, 0x91, 0x20, 0xf5, 0x1f, 0x35, 0xb4, + 0x3a, 0x94, 0xda, 0x75, 0x22, 0x8e, 0x3f, 0x9d, 0x08, 0x40, 0x3f, 0x5b, 0x00, 0x42, 0x5b, 0xba, + 0x5f, 0x52, 0x76, 0x96, 0x06, 0x94, 0x8c, 0xf3, 0x8f, 0xd0, 0xbc, 0xc3, 0xc1, 0x8b, 0xca, 0xb3, + 0x1b, 0x85, 0xcd, 0xe2, 0x9d, 0xf7, 0xce, 0xe4, 0xbd, 0xb9, 0xaa, 0x10, 0xe7, 0x1b, 0x42, 0x97, + 0x24, 0x10, 0xf5, 0x3f, 0xb3, 0xbe, 0x8b, 0x98, 0xf0, 0x03, 0x74, 0x89, 0x72, 0x4e, 0xad, 0x23, + 0x02, 0x2f, 0x3b, 0x4e, 0x08, 0xb6, 0x8c, 0x60, 0xc9, 0xc4, 0x71, 0xbf, 0x76, 0x69, 0x6b, 0x84, + 0x43, 0xc6, 0x24, 0x85, 0x6e, 0xc0, 0xec, 0x86, 0xff, 0x9c, 0x3d, 0xf1, 0x9b, 0xac, 0xe3, 0x73, + 0x99, 0x60, 0xa5, 0xbb, 0x3f, 0xc2, 0x21, 0x63, 0x92, 0xd8, 0x42, 0xeb, 0x5d, 0xe6, 0x76, 0x3c, + 0xd8, 0x75, 0x9e, 0x83, 0xd5, 0xb3, 0x5c, 0x68, 0x32, 0x1b, 0xa2, 0x72, 0x61, 0xa3, 0xb0, 0xb9, + 0x6c, 0x1a, 0x71, 0xbf, 0xb6, 0xfe, 0x34, 0x87, 0x7f, 0xd2, 0xaf, 0x5d, 0xc9, 0xa1, 0x93, 0x5c, + 0xb0, 0xfa, 0x0f, 0x1a, 0x5a, 0xdc, 0x3e, 0x68, 0xec, 0x31, 0x1b, 0x2e, 0xa0, 0xcb, 0x1a, 0x23, + 0x5d, 0x76, 0xf5, 0x94, 0x3a, 0x09, 0xa7, 0xa6, 0xf6, 0xd8, 0x5f, 0x49, 0x9d, 0x84, 0x8c, 0x1a, + 0x92, 0x0d, 0x34, 0xe7, 0x53, 0x0f, 0xa4, 0xeb, 0xcb, 0xa9, 0xce, 0x1e, 0xf5, 0x80, 0x48, 0x0e, + 0x7e, 0x1f, 0x2d, 0xf8, 0xcc, 0x86, 0xc6, 0x43, 0xe9, 0xc0, 0xb2, 0x79, 0x49, 0xc9, 0x2c, 0xec, + 0x49, 0x2a, 0x51, 0x5c, 0x7c, 0x17, 0xad, 0x70, 0x16, 0x30, 0x97, 0xb5, 0x7b, 0x8f, 0xa1, 0x37, + 0xc8, 0x78, 0x29, 0xee, 0xd7, 0x56, 0x0e, 0x33, 0x74, 0x32, 0x22, 0x85, 0x5b, 0xa8, 0x48, 0x5d, + 0x97, 0x59, 0x94, 0xd3, 0x96, 0x0b, 0xe5, 0x39, 0x19, 0xa3, 0xf1, 0xae, 0x18, 0x93, 0x32, 0x09, + 0xe3, 0x04, 0x22, 0xd6, 0x09, 0x2d, 0x88, 0xcc, 0xcb, 0x71, 0xbf, 0x56, 0xdc, 0x4a, 0x71, 0x48, + 0x16, 0xb4, 0xfe, 0xbd, 0x86, 0x8a, 0x2a, 0xea, 0x0b, 0x98, 0xab, 0x8f, 0x46, 0xe7, 0xea, 0xff, + 0x67, 0xa8, 0xd7, 0x94, 0xa9, 0xb2, 0x86, 0x6e, 0xcb, 0x91, 0x3a, 0x44, 0x8b, 0xb6, 0x2c, 0x5a, + 0x54, 0xd6, 0x24, 0xf4, 0xb5, 0x33, 0x40, 0xab, 0xb1, 0xbd, 0xac, 0x0c, 0x2c, 0x26, 0xe7, 0x88, + 0x0c, 0xa0, 0xea, 0xdf, 0x2c, 0xa0, 0x95, 0x83, 0x44, 0x77, 0xdb, 0xa5, 0x51, 0x74, 0x01, 0x0d, + 0xfd, 0x01, 0x2a, 0x06, 0x21, 0xeb, 0x3a, 0x91, 0xc3, 0x7c, 0x08, 0x55, 0x5b, 0x5d, 0x51, 0x2a, + 0xc5, 0xfd, 0x94, 0x45, 0xb2, 0x72, 0xd8, 0x45, 0x28, 0xa0, 0x21, 0xf5, 0x80, 0x8b, 0x14, 0x14, + 0x64, 0x0a, 0xee, 0xbf, 0x2b, 0x05, 0xd9, 0xb0, 0xf4, 0xfd, 0xa1, 0xea, 0x8e, 0xcf, 0xc3, 0x5e, + 0xea, 0x62, 0xca, 0x20, 0x19, 0x7c, 0x7c, 0x8c, 0x56, 0x43, 0xb0, 0x5c, 0xea, 0x78, 0xfb, 0xcc, + 0x75, 0xac, 0x9e, 0x6c, 0xcd, 0x65, 0x73, 0x27, 0xee, 0xd7, 0x56, 0x49, 0x96, 0x71, 0xd2, 0xaf, + 0xdd, 0x9a, 0xbc, 0x3a, 0xf5, 0x7d, 0x08, 0x23, 0x27, 0xe2, 0xe0, 0xf3, 0xa4, 0x61, 0x47, 0x74, + 0xc8, 0x28, 0xb6, 0x98, 0x1d, 0x4f, 0xac, 0xaf, 0x27, 0x01, 0x77, 0x98, 0x1f, 0x95, 0xe7, 0xd3, + 0xd9, 0x69, 0x66, 0xe8, 0x64, 0x44, 0x0a, 0xef, 0xa2, 0x75, 0xd1, 0xe6, 0x9f, 0x27, 0x06, 0x76, + 0xbe, 0x08, 0xa8, 0x2f, 0x52, 0x55, 0x5e, 0x90, 0xdb, 0xb2, 0x2c, 0x76, 0xdd, 0x56, 0x0e, 0x9f, + 0xe4, 0x6a, 0xe1, 0x4f, 0xd0, 0x5a, 0xb2, 0xec, 0x4c, 0xc7, 0xb7, 0x1d, 0xbf, 0x2d, 0x56, 0x5d, + 0x79, 0x51, 0x06, 0x7d, 0x3d, 0xee, 0xd7, 0xd6, 0x9e, 0x8e, 0x33, 0x4f, 0xf2, 0x88, 0x64, 0x12, + 0x04, 0xbf, 0x44, 0x6b, 0xd2, 0x22, 0xd8, 0x6a, 0x11, 0x38, 0x10, 0x95, 0x97, 0x64, 0xfd, 0x36, + 0xb3, 0xf5, 0x13, 0xa9, 0x13, 0x8d, 0x34, 0x58, 0x17, 0x07, 0xe0, 0x82, 0xc5, 0x59, 0x78, 0x08, + 0xa1, 0x67, 0xfe, 0x4f, 0xd5, 0x6b, 0x6d, 0x6b, 0x1c, 0x8a, 0x4c, 0xa2, 0x57, 0x3e, 0x44, 0x97, + 0xc7, 0x0a, 0x8e, 0x4b, 0xa8, 0x70, 0x0c, 0xbd, 0x64, 0xd1, 0x11, 0xf1, 0x89, 0xd7, 0xd1, 0x7c, + 0x97, 0xba, 0x1d, 0x48, 0x3a, 0x90, 0x24, 0x87, 0x07, 0xb3, 0xf7, 0xb5, 0xfa, 0xcf, 0x1a, 0x2a, + 0x65, 0xbb, 0xe7, 0x02, 0xd6, 0x46, 0x73, 0x74, 0x6d, 0x6c, 0x9e, 0xb5, 0xb1, 0xa7, 0xec, 0x8e, + 0xef, 0x66, 0x51, 0x29, 0x29, 0x4e, 0x72, 0xd9, 0x7a, 0xe0, 0xf3, 0x0b, 0x18, 0x6d, 0x32, 0x72, + 0x57, 0xdd, 0x3a, 0x7d, 0x8f, 0xa7, 0xde, 0x4d, 0xbb, 0xb4, 0xf0, 0x33, 0xb4, 0x10, 0x71, 0xca, + 0x3b, 0x62, 0xe6, 0x05, 0xea, 0x9d, 0x73, 0xa1, 0x4a, 0xcd, 0xf4, 0xd2, 0x4a, 0xce, 0x44, 0x21, + 0xd6, 0x7f, 0xd1, 0xd0, 0xfa, 0xb8, 0xca, 0x05, 0x14, 0xfb, 0xe3, 0xd1, 0x62, 0xdf, 0x38, 0x4f, + 0x44, 0x53, 0x0a, 0xfe, 0x9b, 0x86, 0xfe, 0x33, 0x11, 0xbc, 0xbc, 0x1e, 0xc5, 0x9e, 0x08, 0xc6, + 0xb6, 0xd1, 0x5e, 0x7a, 0xe7, 0xcb, 0x3d, 0xb1, 0x9f, 0xc3, 0x27, 0xb9, 0x5a, 0xf8, 0x05, 0x2a, + 0x39, 0xbe, 0xeb, 0xf8, 0x90, 0xd0, 0x0e, 0xd2, 0x72, 0xe7, 0x0e, 0xf3, 0x38, 0xb2, 0x2c, 0xf3, + 0x7a, 0xdc, 0xaf, 0x95, 0x1a, 0x63, 0x28, 0x64, 0x02, 0xb7, 0xfe, 0x6b, 0x4e, 0x79, 0xe4, 0x5d, + 0x78, 0x03, 0x2d, 0x25, 0x8f, 0x46, 0x08, 0x55, 0x18, 0xc3, 0x74, 0x6f, 0x29, 0x3a, 0x19, 0x4a, + 0xc8, 0x0e, 0x92, 0xa9, 0x50, 0x8e, 0x9e, 0xaf, 0x83, 0xa4, 0x66, 0xa6, 0x83, 0xe4, 0x99, 0x28, + 0x44, 0xe1, 0x89, 0x78, 0x00, 0xc9, 0x84, 0x16, 0x46, 0x3d, 0xd9, 0x53, 0x74, 0x32, 0x94, 0xa8, + 0xff, 0x5d, 0xc8, 0xa9, 0x92, 0x6c, 0xc5, 0x4c, 0x48, 0x83, 0xb7, 0xf2, 0x78, 0x48, 0xf6, 0x30, + 0x24, 0x1b, 0x7f, 0xab, 0x21, 0x4c, 0x87, 0x10, 0xcd, 0x41, 0xab, 0x26, 0xfd, 0xf4, 0xe8, 0xfc, + 0x13, 0xa2, 0x6f, 0x4d, 0x80, 0x25, 0xf7, 0x64, 0x45, 0x39, 0x81, 0x27, 0x05, 0x48, 0x8e, 0x07, + 0xd8, 0x41, 0xc5, 0x84, 0xba, 0x13, 0x86, 0x2c, 0x54, 0x23, 0x7b, 0xf5, 0x74, 0x87, 0xa4, 0xb8, + 0x59, 0x95, 0x0f, 0xb9, 0x54, 0xff, 0xa4, 0x5f, 0x2b, 0x66, 0xf8, 0x24, 0x8b, 0x2d, 0x4c, 0xd9, + 0x90, 0x9a, 0x9a, 0xfb, 0x17, 0xa6, 0x1e, 0xc2, 0x74, 0x53, 0x19, 0xec, 0xca, 0x0e, 0xfa, 0xef, + 0x94, 0x04, 0x9d, 0xeb, 0x5e, 0xf9, 0x4a, 0x43, 0x59, 0x1b, 0x78, 0x17, 0xcd, 0x89, 0xff, 0x59, + 0xb5, 0x61, 0xae, 0x9f, 0x6d, 0xc3, 0x1c, 0x3a, 0x1e, 0xa4, 0x8b, 0x52, 0x9c, 0x88, 0x44, 0xc1, + 0xd7, 0xd0, 0xa2, 0x07, 0x51, 0x44, 0xdb, 0xca, 0x72, 0xfa, 0xea, 0x6b, 0x26, 0x64, 0x32, 0xe0, + 0xd7, 0xef, 0xa1, 0x2b, 0x39, 0xef, 0x68, 0x5c, 0x43, 0xf3, 0x96, 0xfc, 0xe1, 0x12, 0x0e, 0xcd, + 0x9b, 0xcb, 0x62, 0xcb, 0x6c, 0xcb, 0xff, 0xac, 0x84, 0x6e, 0xde, 0x7c, 0xf5, 0xb6, 0x3a, 0xf3, + 0xfa, 0x6d, 0x75, 0xe6, 0xcd, 0xdb, 0xea, 0xcc, 0x97, 0x71, 0x55, 0x7b, 0x15, 0x57, 0xb5, 0xd7, + 0x71, 0x55, 0x7b, 0x13, 0x57, 0xb5, 0xdf, 0xe3, 0xaa, 0xf6, 0xf5, 0x1f, 0xd5, 0x99, 0x67, 0x8b, + 0x2a, 0xdf, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0x72, 0xff, 0xde, 0x2e, 0xe4, 0x10, 0x00, 0x00, } func (m *CSIDriver) Marshal() (dAtA []byte, err error) { @@ -715,6 +717,15 @@ func (m *CSIDriverSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.VolumeLifecycleModes) > 0 { + for iNdEx := len(m.VolumeLifecycleModes) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.VolumeLifecycleModes[iNdEx]) + copy(dAtA[i:], m.VolumeLifecycleModes[iNdEx]) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.VolumeLifecycleModes[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } if m.PodInfoOnMount != nil { i-- if *m.PodInfoOnMount { @@ -1458,6 +1469,12 @@ func (m *CSIDriverSpec) Size() (n int) { if m.PodInfoOnMount != nil { n += 2 } + if len(m.VolumeLifecycleModes) > 0 { + for _, s := range m.VolumeLifecycleModes { + l = len(s) + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -1745,6 +1762,7 @@ func (this *CSIDriverSpec) String() string { s := strings.Join([]string{`&CSIDriverSpec{`, `AttachRequired:` + valueToStringGenerated(this.AttachRequired) + `,`, `PodInfoOnMount:` + valueToStringGenerated(this.PodInfoOnMount) + `,`, + `VolumeLifecycleModes:` + fmt.Sprintf("%v", this.VolumeLifecycleModes) + `,`, `}`, }, "") return s @@ -2265,6 +2283,38 @@ func (m *CSIDriverSpec) Unmarshal(dAtA []byte) error { } b := bool(v != 0) m.PodInfoOnMount = &b + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeLifecycleModes", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VolumeLifecycleModes = append(m.VolumeLifecycleModes, VolumeLifecycleMode(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/staging/src/k8s.io/api/storage/v1beta1/generated.proto b/staging/src/k8s.io/api/storage/v1beta1/generated.proto index 3bcc2139cad..83fbb6039c7 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/generated.proto +++ b/staging/src/k8s.io/api/storage/v1beta1/generated.proto @@ -93,8 +93,32 @@ message CSIDriverSpec { // "csi.storage.k8s.io/pod.name": pod.Name // "csi.storage.k8s.io/pod.namespace": pod.Namespace // "csi.storage.k8s.io/pod.uid": string(pod.UID) + // "csi.storage.k8s.io/ephemeral": "true" iff the volume is an ephemeral inline volume + // defined by a CSIVolumeSource, otherwise "false" + // + // "csi.storage.k8s.io/ephemeral" is a new feature in Kubernetes 1.16. It is only + // required for drivers which support both the "Persistent" and "Ephemeral" VolumeLifecycleMode. + // Other drivers can leave pod info disabled and/or ignore this field. + // As Kubernetes 1.15 doesn't support this field, drivers can only support one mode when + // deployed on such a cluster and the deployment determines which mode that is, for example + // via a command line parameter of the driver. // +optional optional bool podInfoOnMount = 2; + + // VolumeLifecycleModes defines what kind of volumes this CSI volume driver supports. + // The default if the list is empty is "Persistent", which is the usage + // defined by the CSI specification and implemented in Kubernetes via the usual + // PV/PVC mechanism. + // The other mode is "Ephemeral". In this mode, volumes are defined inline + // inside the pod spec with CSIVolumeSource and their lifecycle is tied to + // the lifecycle of that pod. A driver has to be aware of this + // because it is only going to get a NodePublishVolume call for such a volume. + // For more information about implementing this mode, see + // https://kubernetes-csi.github.io/docs/ephemeral-local-volumes.html + // A driver can support one or more of these modes and + // more modes may be added in the future. + // +optional + repeated string volumeLifecycleModes = 3; } // CSINode holds information about all CSI drivers installed on a node. diff --git a/staging/src/k8s.io/api/storage/v1beta1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/storage/v1beta1/types_swagger_doc_generated.go index 0bc3456b977..8c4f208a37b 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/storage/v1beta1/types_swagger_doc_generated.go @@ -48,9 +48,10 @@ func (CSIDriverList) SwaggerDoc() map[string]string { } var map_CSIDriverSpec = map[string]string{ - "": "CSIDriverSpec is the specification of a CSIDriver.", - "attachRequired": "attachRequired indicates this CSI volume driver requires an attach operation (because it implements the CSI ControllerPublishVolume() method), and that the Kubernetes attach detach controller should call the attach volume interface which checks the volumeattachment status and waits until the volume is attached before proceeding to mounting. The CSI external-attacher coordinates with CSI volume driver and updates the volumeattachment status when the attach operation is complete. If the CSIDriverRegistry feature gate is enabled and the value is specified to false, the attach operation will be skipped. Otherwise the attach operation will be called.", - "podInfoOnMount": "If set to true, podInfoOnMount indicates this CSI volume driver requires additional pod information (like podName, podUID, etc.) during mount operations. If set to false, pod information will not be passed on mount. Default is false. The CSI driver specifies podInfoOnMount as part of driver deployment. If true, Kubelet will pass pod information as VolumeContext in the CSI NodePublishVolume() calls. The CSI driver is responsible for parsing and validating the information passed in as VolumeContext. The following VolumeConext will be passed if podInfoOnMount is set to true. This list might grow, but the prefix will be used. \"csi.storage.k8s.io/pod.name\": pod.Name \"csi.storage.k8s.io/pod.namespace\": pod.Namespace \"csi.storage.k8s.io/pod.uid\": string(pod.UID)", + "": "CSIDriverSpec is the specification of a CSIDriver.", + "attachRequired": "attachRequired indicates this CSI volume driver requires an attach operation (because it implements the CSI ControllerPublishVolume() method), and that the Kubernetes attach detach controller should call the attach volume interface which checks the volumeattachment status and waits until the volume is attached before proceeding to mounting. The CSI external-attacher coordinates with CSI volume driver and updates the volumeattachment status when the attach operation is complete. If the CSIDriverRegistry feature gate is enabled and the value is specified to false, the attach operation will be skipped. Otherwise the attach operation will be called.", + "podInfoOnMount": "If set to true, podInfoOnMount indicates this CSI volume driver requires additional pod information (like podName, podUID, etc.) during mount operations. If set to false, pod information will not be passed on mount. Default is false. The CSI driver specifies podInfoOnMount as part of driver deployment. If true, Kubelet will pass pod information as VolumeContext in the CSI NodePublishVolume() calls. The CSI driver is responsible for parsing and validating the information passed in as VolumeContext. The following VolumeConext will be passed if podInfoOnMount is set to true. This list might grow, but the prefix will be used. \"csi.storage.k8s.io/pod.name\": pod.Name \"csi.storage.k8s.io/pod.namespace\": pod.Namespace \"csi.storage.k8s.io/pod.uid\": string(pod.UID) \"csi.storage.k8s.io/ephemeral\": \"true\" iff the volume is an ephemeral inline volume\n defined by a CSIVolumeSource, otherwise \"false\"\n\n\"csi.storage.k8s.io/ephemeral\" is a new feature in Kubernetes 1.16. It is only required for drivers which support both the \"Persistent\" and \"Ephemeral\" VolumeLifecycleMode. Other drivers can leave pod info disabled and/or ignore this field. As Kubernetes 1.15 doesn't support this field, drivers can only support one mode when deployed on such a cluster and the deployment determines which mode that is, for example via a command line parameter of the driver.", + "volumeLifecycleModes": "VolumeLifecycleModes defines what kind of volumes this CSI volume driver supports. The default if the list is empty is \"Persistent\", which is the usage defined by the CSI specification and implemented in Kubernetes via the usual PV/PVC mechanism. The other mode is \"Ephemeral\". In this mode, volumes are defined inline inside the pod spec with CSIVolumeSource and their lifecycle is tied to the lifecycle of that pod. A driver has to be aware of this because it is only going to get a NodePublishVolume call for such a volume. For more information about implementing this mode, see https://kubernetes-csi.github.io/docs/ephemeral-local-volumes.html A driver can support one or more of these modes and more modes may be added in the future.", } func (CSIDriverSpec) SwaggerDoc() map[string]string { diff --git a/staging/src/k8s.io/api/storage/v1beta1/zz_generated.deepcopy.go b/staging/src/k8s.io/api/storage/v1beta1/zz_generated.deepcopy.go index 6b4726559fa..52433fcdf2c 100644 --- a/staging/src/k8s.io/api/storage/v1beta1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/storage/v1beta1/zz_generated.deepcopy.go @@ -98,6 +98,11 @@ func (in *CSIDriverSpec) DeepCopyInto(out *CSIDriverSpec) { *out = new(bool) **out = **in } + if in.VolumeLifecycleModes != nil { + in, out := &in.VolumeLifecycleModes, &out.VolumeLifecycleModes + *out = make([]VolumeLifecycleMode, len(*in)) + copy(*out, *in) + } return } diff --git a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.json b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.json index 018ca5aa33f..8a0b0595ea0 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.json +++ b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.json @@ -41,6 +41,9 @@ }, "spec": { "attachRequired": false, - "podInfoOnMount": false + "podInfoOnMount": false, + "volumeLifecycleModes": [ + "r鯹)晿\u003co,c鮽ort昍řČ扷5ƗǸ" + ] } } \ No newline at end of file diff --git a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.pb b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.pb index c8376073855f09f5255dbf23e34724bb80088e00..ebf9afd36771a19505a67d10b1dbfecf980afcc7 100644 GIT binary patch delta 56 zcmV-80LTB{0f7RLDgn`vE2=Cf2mlZO8XD`K=9rDenZ%6d OiMKVzm&dpo03raA+Zi+f delta 21 ccmZosQVdEA07IGtlK=n! diff --git a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.yaml b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.yaml index a67ac9494e6..3ab93186ba5 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/storage.k8s.io.v1beta1.CSIDriver.yaml @@ -31,3 +31,5 @@ metadata: spec: attachRequired: false podInfoOnMount: false + volumeLifecycleModes: + - r鯹)晿 Date: Fri, 26 Jul 2019 13:52:37 +0200 Subject: [PATCH 3/6] storage: validate CSIDriver.Spec.VolumeLifecycleModes This ensures that users get a good error message early on when trying to do something that isn't okay: $ kubectl create -f csi-hostpath-driverinfo.yaml The CSIDriver "hostpath.csi.k8s.io" is invalid: spec.volumeLifecycleModes: Unsupported value: "foobar": supported values: "persistent", "ephemeral" --- pkg/apis/storage/validation/validation.go | 19 ++++++ .../storage/validation/validation_test.go | 54 +++++++++++++++ .../storage/csidriver/strategy_test.go | 65 +++++++++++++++++++ 3 files changed, 138 insertions(+) diff --git a/pkg/apis/storage/validation/validation.go b/pkg/apis/storage/validation/validation.go index fd8e9984135..2f3b438af27 100644 --- a/pkg/apis/storage/validation/validation.go +++ b/pkg/apis/storage/validation/validation.go @@ -419,6 +419,7 @@ func validateCSIDriverSpec( allErrs := field.ErrorList{} allErrs = append(allErrs, validateAttachRequired(spec.AttachRequired, fldPath.Child("attachedRequired"))...) allErrs = append(allErrs, validatePodInfoOnMount(spec.PodInfoOnMount, fldPath.Child("podInfoOnMount"))...) + allErrs = append(allErrs, validateVolumeLifecycleModes(spec.VolumeLifecycleModes, fldPath.Child("volumeLifecycleModes"))...) return allErrs } @@ -441,3 +442,21 @@ func validatePodInfoOnMount(podInfoOnMount *bool, fldPath *field.Path) field.Err return allErrs } + +// validateVolumeLifecycleModes tests if mode has one of the allowed values. +func validateVolumeLifecycleModes(modes []storage.VolumeLifecycleMode, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + for _, mode := range modes { + switch mode { + case storage.VolumeLifecyclePersistent, storage.VolumeLifecycleEphemeral: + default: + allErrs = append(allErrs, field.NotSupported(fldPath, mode, + []string{ + string(storage.VolumeLifecyclePersistent), + string(storage.VolumeLifecycleEphemeral), + })) + } + } + + return allErrs +} diff --git a/pkg/apis/storage/validation/validation_test.go b/pkg/apis/storage/validation/validation_test.go index 15f54b419e9..81f5c0eca6a 100644 --- a/pkg/apis/storage/validation/validation_test.go +++ b/pkg/apis/storage/validation/validation_test.go @@ -1726,6 +1726,49 @@ func TestCSIDriverValidation(t *testing.T) { PodInfoOnMount: ¬PodInfoOnMount, }, }, + { + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: ¬PodInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecyclePersistent, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: ¬PodInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleEphemeral, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: ¬PodInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleEphemeral, + storage.VolumeLifecyclePersistent, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: ¬PodInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleEphemeral, + storage.VolumeLifecyclePersistent, + storage.VolumeLifecycleEphemeral, + }, + }, + }, } for _, csiDriver := range successCases { @@ -1764,6 +1807,17 @@ func TestCSIDriverValidation(t *testing.T) { PodInfoOnMount: nil, }, }, + { + // invalid mode + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: ¬PodInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + "no-such-mode", + }, + }, + }, } for _, csiDriver := range errorCases { diff --git a/pkg/registry/storage/csidriver/strategy_test.go b/pkg/registry/storage/csidriver/strategy_test.go index 442b1005e8d..75c227b2c83 100644 --- a/pkg/registry/storage/csidriver/strategy_test.go +++ b/pkg/registry/storage/csidriver/strategy_test.go @@ -296,6 +296,71 @@ func TestCSIDriverValidation(t *testing.T) { }, true, }, + { + "invalid volume mode", + &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachRequired, + PodInfoOnMount: &podInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleMode("no-such-mode"), + }, + }, + }, + true, + }, + { + "persistent volume mode", + &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachRequired, + PodInfoOnMount: &podInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecyclePersistent, + }, + }, + }, + false, + }, + { + "ephemeral volume mode", + &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachRequired, + PodInfoOnMount: &podInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleEphemeral, + }, + }, + }, + false, + }, + { + "both volume modes", + &storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachRequired, + PodInfoOnMount: &podInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecyclePersistent, + storage.VolumeLifecycleEphemeral, + }, + }, + }, + false, + }, } for _, test := range tests { From 7bbc06fcd436fda2bc82205598e1a91ce568b36f Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 24 Jul 2019 08:16:43 +0200 Subject: [PATCH 4/6] storage: check CSIDriver.Spec.VolumeLifecycleModes Using a "normal" CSI driver for an inline ephemeral volume may have unexpected and potentially harmful effects when the driver gets a NodePublishVolume call that it isn't expecting. To prevent that mistake, driver deployments for a driver that supports such volumes must: - deploy a CSIDriver object for the driver - set CSIDriver.Spec.VolumeLifecycleModes such that it contains "ephemeral" The default for that field is "persistent", so existing deployments continue to work and are automatically protected against incorrect usage. For the E2E tests we need a way to specify the driver mode. The existing cluster-driver-registrar doesn't support that and also was deprecated, so we stop using it altogether and instead deploy and patch a CSIDriver object. --- pkg/volume/csi/csi_attacher_test.go | 12 +- pkg/volume/csi/csi_mounter.go | 41 +-- pkg/volume/csi/csi_mounter_test.go | 57 ++-- pkg/volume/csi/csi_plugin.go | 134 +++++--- pkg/volume/csi/csi_plugin_test.go | 286 +++++++++++------- pkg/volume/csi/csi_test.go | 150 ++++++++- pkg/volume/csi/csi_util_test.go | 7 +- test/e2e/storage/drivers/csi.go | 24 +- test/e2e/storage/testsuites/testdriver.go | 4 +- test/e2e/storage/utils/deployment.go | 33 +- .../cluster-driver-registrar/README.md | 1 - .../cluster-driver-registrar/rbac.yaml | 38 --- .../csi-mock-cluster-driver-registrar.yaml | 36 --- .../storage-csi/mock/csi-mock-driver.yaml | 2 +- .../storage-csi/mock/csi-mock-driverinfo.yaml | 7 + 15 files changed, 522 insertions(+), 310 deletions(-) delete mode 100644 test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/README.md delete mode 100644 test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml delete mode 100644 test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml create mode 100644 test/e2e/testing-manifests/storage-csi/mock/csi-mock-driverinfo.yaml diff --git a/pkg/volume/csi/csi_attacher_test.go b/pkg/volume/csi/csi_attacher_test.go index 2e3b177643c..6dff65ae663 100644 --- a/pkg/volume/csi/csi_attacher_test.go +++ b/pkg/volume/csi/csi_attacher_test.go @@ -342,9 +342,9 @@ func TestAttacherWithCSIDriver(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClient := fakeclient.NewSimpleClientset( - getTestCSIDriver("not-attachable", nil, &bFalse), - getTestCSIDriver("attachable", nil, &bTrue), - getTestCSIDriver("nil", nil, nil), + getTestCSIDriver("not-attachable", nil, &bFalse, nil), + getTestCSIDriver("attachable", nil, &bTrue, nil), + getTestCSIDriver("nil", nil, nil, nil), ) plug, fakeWatcher, tmpDir, _ := newTestWatchPlugin(t, fakeClient) defer os.RemoveAll(tmpDir) @@ -430,9 +430,9 @@ func TestAttacherWaitForVolumeAttachmentWithCSIDriver(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { fakeClient := fakeclient.NewSimpleClientset( - getTestCSIDriver("not-attachable", nil, &bFalse), - getTestCSIDriver("attachable", nil, &bTrue), - getTestCSIDriver("nil", nil, nil), + getTestCSIDriver("not-attachable", nil, &bFalse, nil), + getTestCSIDriver("attachable", nil, &bTrue, nil), + getTestCSIDriver("nil", nil, nil, nil), ) plug, tmpDir := newTestPlugin(t, fakeClient) defer os.RemoveAll(tmpDir) diff --git a/pkg/volume/csi/csi_mounter.go b/pkg/volume/csi/csi_mounter.go index d436f8d24bf..b14476a4c1f 100644 --- a/pkg/volume/csi/csi_mounter.go +++ b/pkg/volume/csi/csi_mounter.go @@ -29,6 +29,7 @@ import ( "k8s.io/klog" api "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1beta1" apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -46,32 +47,32 @@ var ( driverName, nodeName, attachmentID, - csiVolumeMode string + volumeLifecycleMode string }{ "specVolID", "volumeHandle", "driverName", "nodeName", "attachmentID", - "csiVolumeMode", + "volumeLifecycleMode", } ) type csiMountMgr struct { csiClientGetter - k8s kubernetes.Interface - plugin *csiPlugin - driverName csiDriverName - csiVolumeMode csiVolumeMode - volumeID string - specVolumeID string - readOnly bool - spec *volume.Spec - pod *api.Pod - podUID types.UID - options volume.VolumeOptions - publishContext map[string]string - kubeVolHost volume.KubeletVolumeHost + k8s kubernetes.Interface + plugin *csiPlugin + driverName csiDriverName + volumeLifecycleMode storage.VolumeLifecycleMode + volumeID string + specVolumeID string + readOnly bool + spec *volume.Spec + pod *api.Pod + podUID types.UID + options volume.VolumeOptions + publishContext map[string]string + kubeVolHost volume.KubeletVolumeHost volume.MetricsProvider } @@ -145,8 +146,8 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error if !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { return fmt.Errorf("CSIInlineVolume feature required") } - if c.csiVolumeMode != ephemeralVolumeMode { - return fmt.Errorf("unexpected volume mode: %s", c.csiVolumeMode) + if c.volumeLifecycleMode != storage.VolumeLifecycleEphemeral { + return fmt.Errorf("unexpected volume mode: %s", c.volumeLifecycleMode) } if volSrc.FSType != nil { fsType = *volSrc.FSType @@ -160,8 +161,8 @@ func (c *csiMountMgr) SetUpAt(dir string, mounterArgs volume.MounterArgs) error secretRef = &api.SecretReference{Name: secretName, Namespace: ns} } case pvSrc != nil: - if c.csiVolumeMode != persistentVolumeMode { - return fmt.Errorf("unexpected driver mode: %s", c.csiVolumeMode) + if c.volumeLifecycleMode != storage.VolumeLifecyclePersistent { + return fmt.Errorf("unexpected driver mode: %s", c.volumeLifecycleMode) } fsType = pvSrc.FSType @@ -319,7 +320,7 @@ func (c *csiMountMgr) podAttributes() (map[string]string, error) { "csi.storage.k8s.io/serviceAccount.name": c.pod.Spec.ServiceAccountName, } if utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { - attrs["csi.storage.k8s.io/ephemeral"] = strconv.FormatBool(c.csiVolumeMode == ephemeralVolumeMode) + attrs["csi.storage.k8s.io/ephemeral"] = strconv.FormatBool(c.volumeLifecycleMode == storage.VolumeLifecycleEphemeral) } klog.V(4).Infof(log("CSIDriver %q requires pod information", c.driverName)) diff --git a/pkg/volume/csi/csi_mounter_test.go b/pkg/volume/csi/csi_mounter_test.go index ae0fc6576e6..c93fc55c808 100644 --- a/pkg/volume/csi/csi_mounter_test.go +++ b/pkg/volume/csi/csi_mounter_test.go @@ -28,6 +28,7 @@ import ( api "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" + storagev1beta1 "k8s.io/api/storage/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -151,13 +152,16 @@ func MounterSetUpTests(t *testing.T, podInfoEnabled bool) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { klog.Infof("Starting test %s", test.name) + // Modes must be set if (and only if) CSIInlineVolume is enabled. + var modes []storagev1beta1.VolumeLifecycleMode if test.csiInlineVolume { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)() + modes = append(modes, storagev1beta1.VolumeLifecyclePersistent) } fakeClient := fakeclient.NewSimpleClientset( - getTestCSIDriver("no-info", &noPodMountInfo, nil), - getTestCSIDriver("info", ¤tPodInfoMount, nil), - getTestCSIDriver("nil", nil, nil), + getTestCSIDriver("no-info", &noPodMountInfo, nil, modes), + getTestCSIDriver("info", ¤tPodInfoMount, nil, modes), + getTestCSIDriver("nil", nil, nil, modes), ) plug, tmpDir := newTestPlugin(t, fakeClient) defer os.RemoveAll(tmpDir) @@ -278,16 +282,16 @@ func TestMounterSetUpSimple(t *testing.T) { testCases := []struct { name string podUID types.UID - mode csiVolumeMode + mode storagev1beta1.VolumeLifecycleMode fsType string options []string spec func(string, []string) *volume.Spec shouldFail bool }{ { - name: "setup with vol source", + name: "setup with ephemeral source", podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - mode: ephemeralVolumeMode, + mode: storagev1beta1.VolumeLifecycleEphemeral, fsType: "ext4", shouldFail: true, spec: func(fsType string, options []string) *volume.Spec { @@ -299,7 +303,7 @@ func TestMounterSetUpSimple(t *testing.T) { { name: "setup with persistent source", podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - mode: persistentVolumeMode, + mode: storagev1beta1.VolumeLifecyclePersistent, fsType: "zfs", spec: func(fsType string, options []string) *volume.Spec { pvSrc := makeTestPV("pv1", 20, testDriver, "vol1") @@ -311,7 +315,7 @@ func TestMounterSetUpSimple(t *testing.T) { { name: "setup with persistent source without unspecified fstype and options", podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - mode: persistentVolumeMode, + mode: storagev1beta1.VolumeLifecyclePersistent, spec: func(fsType string, options []string) *volume.Spec { return volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol2"), false) }, @@ -345,8 +349,8 @@ func TestMounterSetUpSimple(t *testing.T) { csiMounter := mounter.(*csiMountMgr) csiMounter.csiClient = setupClient(t, true) - if csiMounter.csiVolumeMode != persistentVolumeMode { - t.Fatal("unexpected volume mode: ", csiMounter.csiVolumeMode) + if csiMounter.volumeLifecycleMode != storagev1beta1.VolumeLifecyclePersistent { + t.Fatal("unexpected volume mode: ", csiMounter.volumeLifecycleMode) } attachID := getAttachmentName(csiMounter.volumeID, string(csiMounter.driverName), string(plug.host.GetNodeName())) @@ -397,14 +401,10 @@ func TestMounterSetUpSimple(t *testing.T) { func TestMounterSetUpWithInline(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)() - fakeClient := fakeclient.NewSimpleClientset() - plug, tmpDir := newTestPlugin(t, fakeClient) - defer os.RemoveAll(tmpDir) - testCases := []struct { name string podUID types.UID - mode csiVolumeMode + mode storagev1beta1.VolumeLifecycleMode fsType string options []string spec func(string, []string) *volume.Spec @@ -413,7 +413,7 @@ func TestMounterSetUpWithInline(t *testing.T) { { name: "setup with vol source", podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - mode: ephemeralVolumeMode, + mode: storagev1beta1.VolumeLifecycleEphemeral, fsType: "ext4", spec: func(fsType string, options []string) *volume.Spec { volSrc := makeTestVol("pv1", testDriver) @@ -424,7 +424,7 @@ func TestMounterSetUpWithInline(t *testing.T) { { name: "setup with persistent source", podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - mode: persistentVolumeMode, + mode: storagev1beta1.VolumeLifecyclePersistent, fsType: "zfs", spec: func(fsType string, options []string) *volume.Spec { pvSrc := makeTestPV("pv1", 20, testDriver, "vol1") @@ -436,7 +436,7 @@ func TestMounterSetUpWithInline(t *testing.T) { { name: "setup with persistent source without unspecified fstype and options", podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - mode: persistentVolumeMode, + mode: storagev1beta1.VolumeLifecyclePersistent, spec: func(fsType string, options []string) *volume.Spec { return volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol2"), false) }, @@ -449,6 +449,15 @@ func TestMounterSetUpWithInline(t *testing.T) { } for _, tc := range testCases { + // The fake driver currently supports all modes. + volumeLifecycleModes := []storagev1beta1.VolumeLifecycleMode{ + storagev1beta1.VolumeLifecycleEphemeral, + storagev1beta1.VolumeLifecyclePersistent, + } + driver := getTestCSIDriver(testDriver, nil, nil, volumeLifecycleModes) + fakeClient := fakeclient.NewSimpleClientset(driver) + plug, tmpDir := newTestPlugin(t, fakeClient) + defer os.RemoveAll(tmpDir) registerFakePlugin(testDriver, "endpoint", []string{"1.0.0"}, t) t.Run(tc.name, func(t *testing.T) { mounter, err := plug.NewMounter( @@ -470,15 +479,15 @@ func TestMounterSetUpWithInline(t *testing.T) { csiMounter := mounter.(*csiMountMgr) csiMounter.csiClient = setupClient(t, true) - if csiMounter.csiVolumeMode != tc.mode { - t.Fatal("unexpected volume mode: ", csiMounter.csiVolumeMode) + if csiMounter.volumeLifecycleMode != tc.mode { + t.Fatal("unexpected volume mode: ", csiMounter.volumeLifecycleMode) } - if csiMounter.csiVolumeMode == ephemeralVolumeMode && csiMounter.volumeID != makeVolumeHandle(string(tc.podUID), csiMounter.specVolumeID) { + if csiMounter.volumeLifecycleMode == storagev1beta1.VolumeLifecycleEphemeral && csiMounter.volumeID != makeVolumeHandle(string(tc.podUID), csiMounter.specVolumeID) { t.Fatal("unexpected generated volumeHandle:", csiMounter.volumeID) } - if csiMounter.csiVolumeMode == persistentVolumeMode { + if csiMounter.volumeLifecycleMode == storagev1beta1.VolumeLifecyclePersistent { attachID := getAttachmentName(csiMounter.volumeID, string(csiMounter.driverName), string(plug.host.GetNodeName())) attachment := makeTestAttachment(attachID, "test-node", csiMounter.spec.Name()) _, err = csiMounter.k8s.StorageV1().VolumeAttachments().Create(attachment) @@ -503,10 +512,10 @@ func TestMounterSetUpWithInline(t *testing.T) { } // validate stagingTargetPath - if tc.mode == ephemeralVolumeMode && vol.DeviceMountPath != "" { + if tc.mode == storagev1beta1.VolumeLifecycleEphemeral && vol.DeviceMountPath != "" { t.Errorf("unexpected devicePathTarget sent to driver: %s", vol.DeviceMountPath) } - if tc.mode == persistentVolumeMode { + if tc.mode == storagev1beta1.VolumeLifecyclePersistent { devicePath, err := makeDeviceMountPath(plug, csiMounter.spec) if err != nil { t.Fatal(err) diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 57932d84973..f90878302ba 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -30,6 +30,7 @@ import ( "k8s.io/klog" api "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1beta1" apierrs "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -67,18 +68,6 @@ type csiPlugin struct { csiDriverLister storagelisters.CSIDriverLister } -//TODO (vladimirvivien) add this type to storage api -type driverMode string - -const persistentDriverMode driverMode = "persistent" -const ephemeralDriverMode driverMode = "ephemeral" -const combinedDriverMode driverMode = "persistent+ephemeral" - -type csiVolumeMode string - -const persistentVolumeMode csiVolumeMode = "persistent" -const ephemeralVolumeMode csiVolumeMode = "ephemeral" - // ProbeVolumePlugins returns implemented plugins func ProbeVolumePlugins() []volume.VolumePlugin { p := &csiPlugin{ @@ -373,15 +362,16 @@ func (p *csiPlugin) NewMounter( return nil, errors.New(log("volume source not found in volume.Spec")) } - csiVolumeMode, err := p.getCSIVolumeMode(spec) + volumeLifecycleMode, err := p.getVolumeLifecycleMode(spec) if err != nil { return nil, err } - // TODO(pohly): check CSIDriver.Spec.Mode to ensure that the CSI driver - // supports the current csiVolumeMode. - // In alpha it is assumed that drivers are used correctly without - // the additional sanity check. + // Check CSIDriver.Spec.Mode to ensure that the CSI driver + // supports the current volumeLifecycleMode. + if err := p.supportsVolumeLifecycleMode(driverName, volumeLifecycleMode); err != nil { + return nil, err + } k8s := p.host.GetKubeClient() if k8s == nil { @@ -394,17 +384,17 @@ func (p *csiPlugin) NewMounter( } mounter := &csiMountMgr{ - plugin: p, - k8s: k8s, - spec: spec, - pod: pod, - podUID: pod.UID, - driverName: csiDriverName(driverName), - csiVolumeMode: csiVolumeMode, - volumeID: volumeHandle, - specVolumeID: spec.Name(), - readOnly: readOnly, - kubeVolHost: kvh, + plugin: p, + k8s: k8s, + spec: spec, + pod: pod, + podUID: pod.UID, + driverName: csiDriverName(driverName), + volumeLifecycleMode: volumeLifecycleMode, + volumeID: volumeHandle, + specVolumeID: spec.Name(), + readOnly: readOnly, + kubeVolHost: kvh, } mounter.csiClientGetter.driverName = csiDriverName(driverName) @@ -422,11 +412,11 @@ func (p *csiPlugin) NewMounter( // persist volume info data for teardown node := string(p.host.GetNodeName()) volData := map[string]string{ - volDataKey.specVolID: spec.Name(), - volDataKey.volHandle: volumeHandle, - volDataKey.driverName: driverName, - volDataKey.nodeName: node, - volDataKey.csiVolumeMode: string(csiVolumeMode), + volDataKey.specVolID: spec.Name(), + volDataKey.volHandle: volumeHandle, + volDataKey.driverName: driverName, + volDataKey.nodeName: node, + volDataKey.volumeLifecycleMode: string(volumeLifecycleMode), } attachID := getAttachmentName(volumeHandle, driverName, node) @@ -486,11 +476,11 @@ func (p *csiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.S var spec *volume.Spec inlineEnabled := utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) - // If inlineEnabled is true and mode is ephemeralVolumeMode, + // If inlineEnabled is true and mode is VolumeLifecycleEphemeral, // use constructVolSourceSpec to construct volume source spec. - // If inlineEnabled is false or mode is persistentVolumeMode, + // If inlineEnabled is false or mode is VolumeLifecyclePersistent, // use constructPVSourceSpec to construct volume construct pv source spec. - if inlineEnabled && csiVolumeMode(volData[volDataKey.csiVolumeMode]) == ephemeralVolumeMode { + if inlineEnabled && storage.VolumeLifecycleMode(volData[volDataKey.volumeLifecycleMode]) == storage.VolumeLifecycleEphemeral { spec = p.constructVolSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName]) return spec, nil } @@ -565,12 +555,12 @@ func (p *csiPlugin) NewDetacher() (volume.Detacher, error) { func (p *csiPlugin) CanAttach(spec *volume.Spec) (bool, error) { inlineEnabled := utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) if inlineEnabled { - csiVolumeMode, err := p.getCSIVolumeMode(spec) + volumeLifecycleMode, err := p.getVolumeLifecycleMode(spec) if err != nil { return false, err } - if csiVolumeMode == ephemeralVolumeMode { + if volumeLifecycleMode == storage.VolumeLifecycleEphemeral { klog.V(5).Info(log("plugin.CanAttach = false, ephemeral mode detected for spec %v", spec.Name())) return false, nil } @@ -599,12 +589,12 @@ func (p *csiPlugin) CanDeviceMount(spec *volume.Spec) (bool, error) { return true, nil } - csiVolumeMode, err := p.getCSIVolumeMode(spec) + volumeLifecycleMode, err := p.getVolumeLifecycleMode(spec) if err != nil { return false, err } - if csiVolumeMode == ephemeralVolumeMode { + if volumeLifecycleMode == storage.VolumeLifecycleEphemeral { klog.V(5).Info(log("plugin.CanDeviceMount skipped ephemeral mode detected for spec %v", spec.Name())) return false, nil } @@ -775,11 +765,67 @@ func (p *csiPlugin) skipAttach(driver string) (bool, error) { return false, nil } -// getCSIVolumeMode returns the mode for the specified spec: {persistent|ephemeral}. +// supportsVolumeMode checks whether the CSI driver supports a volume in the given mode. +// An error indicates that it isn't supported and explains why. +func (p *csiPlugin) supportsVolumeLifecycleMode(driver string, volumeMode storage.VolumeLifecycleMode) error { + if !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { + // Feature disabled, therefore only "persistent" volumes are supported. + if volumeMode != storage.VolumeLifecyclePersistent { + return fmt.Errorf("CSIInlineVolume feature not enabled, %q volumes not supported", volumeMode) + } + return nil + } + + // Retrieve CSIDriver. It's not an error if that isn't + // possible (we don't have the lister if CSIDriverRegistry is + // disabled) or the driver isn't found (CSIDriver is + // optional), but then only persistent volumes are supported. + var csiDriver *storage.CSIDriver + if p.csiDriverLister != nil { + kletHost, ok := p.host.(volume.KubeletVolumeHost) + if ok { + kletHost.WaitForCacheSync() + } + + c, err := p.csiDriverLister.Get(driver) + if err != nil && !apierrs.IsNotFound(err) { + // Some internal error. + return err + } + csiDriver = c + } + + // The right response depends on whether we have information + // about the driver and the volume mode. + switch { + case csiDriver == nil && volumeMode == storage.VolumeLifecyclePersistent: + // No information, but that's okay for persistent volumes (and only those). + return nil + case csiDriver == nil: + return fmt.Errorf("volume mode %q not supported by driver %s (no CSIDriver object)", volumeMode, driver) + case containsVolumeMode(csiDriver.Spec.VolumeLifecycleModes, volumeMode): + // Explicitly listed. + return nil + default: + return fmt.Errorf("volume mode %q not supported by driver %s (only supports %q)", volumeMode, driver, csiDriver.Spec.VolumeLifecycleModes) + } +} + +// containsVolumeMode checks whether the given volume mode is listed. +func containsVolumeMode(modes []storage.VolumeLifecycleMode, mode storage.VolumeLifecycleMode) bool { + for _, m := range modes { + if m == mode { + return true + } + } + return false +} + +// getVolumeLifecycleMode returns the mode for the specified spec: {persistent|ephemeral}. // 1) If mode cannot be determined, it will default to "persistent". // 2) If Mode cannot be resolved to either {persistent | ephemeral}, an error is returned // See https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md -func (p *csiPlugin) getCSIVolumeMode(spec *volume.Spec) (csiVolumeMode, error) { +func (p *csiPlugin) getVolumeLifecycleMode(spec *volume.Spec) (storage.VolumeLifecycleMode, error) { // 1) if volume.Spec.Volume.CSI != nil -> mode is ephemeral // 2) if volume.Spec.PersistentVolume.Spec.CSI != nil -> persistent volSrc, _, err := getSourceFromSpec(spec) @@ -788,9 +834,9 @@ func (p *csiPlugin) getCSIVolumeMode(spec *volume.Spec) (csiVolumeMode, error) { } if volSrc != nil && utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { - return ephemeralVolumeMode, nil + return storage.VolumeLifecycleEphemeral, nil } - return persistentVolumeMode, nil + return storage.VolumeLifecyclePersistent, nil } func (p *csiPlugin) getPublishContext(client clientset.Interface, handle, driver, nodeName string) (map[string]string, error) { diff --git a/pkg/volume/csi/csi_plugin_test.go b/pkg/volume/csi/csi_plugin_test.go index cd2d8f6e45f..90733581037 100644 --- a/pkg/volume/csi/csi_plugin_test.go +++ b/pkg/volume/csi/csi_plugin_test.go @@ -25,6 +25,7 @@ import ( "testing" api "k8s.io/api/core/v1" + storagev1beta1 "k8s.io/api/storage/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -178,7 +179,12 @@ func TestPluginGetVolumeNameWithInline(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)() - plug, tmpDir := newTestPlugin(t, nil) + modes := []storagev1beta1.VolumeLifecycleMode{ + storagev1beta1.VolumeLifecyclePersistent, + } + driver := getTestCSIDriver(testDriver, nil, nil, modes) + client := fakeclient.NewSimpleClientset(driver) + plug, tmpDir := newTestPlugin(t, client) defer os.RemoveAll(tmpDir) testCases := []struct { name string @@ -403,9 +409,6 @@ func TestPluginConstructVolumeSpecWithInline(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)() - plug, tmpDir := newTestPlugin(t, nil) - defer os.RemoveAll(tmpDir) - testCases := []struct { name string originSpec *volume.Spec @@ -413,6 +416,7 @@ func TestPluginConstructVolumeSpecWithInline(t *testing.T) { volHandle string podUID types.UID shouldFail bool + modes []storagev1beta1.VolumeLifecycleMode }{ { name: "construct spec1 from persistent spec", @@ -420,6 +424,7 @@ func TestPluginConstructVolumeSpecWithInline(t *testing.T) { volHandle: "testvol-handle1", originSpec: volume.NewSpecFromPersistentVolume(makeTestPV("test.vol.id", 20, testDriver, "testvol-handle1"), true), podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + modes: []storagev1beta1.VolumeLifecycleMode{storagev1beta1.VolumeLifecyclePersistent}, }, { name: "construct spec2 from persistent spec", @@ -427,18 +432,38 @@ func TestPluginConstructVolumeSpecWithInline(t *testing.T) { volHandle: "handle2", originSpec: volume.NewSpecFromPersistentVolume(makeTestPV("spec2", 20, testDriver, "handle2"), true), podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + modes: []storagev1beta1.VolumeLifecycleMode{storagev1beta1.VolumeLifecyclePersistent}, + }, + { + name: "construct spec2 from persistent spec, missing mode", + specVolID: "spec2", + volHandle: "handle2", + originSpec: volume.NewSpecFromPersistentVolume(makeTestPV("spec2", 20, testDriver, "handle2"), true), + podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + modes: []storagev1beta1.VolumeLifecycleMode{}, + shouldFail: true, }, { name: "construct spec from volume spec", specVolID: "volspec", originSpec: volume.NewSpecFromVolume(makeTestVol("volspec", testDriver)), podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + modes: []storagev1beta1.VolumeLifecycleMode{storagev1beta1.VolumeLifecycleEphemeral}, }, { name: "construct spec from volume spec2", specVolID: "volspec2", originSpec: volume.NewSpecFromVolume(makeTestVol("volspec2", testDriver)), podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + modes: []storagev1beta1.VolumeLifecycleMode{storagev1beta1.VolumeLifecycleEphemeral}, + }, + { + name: "construct spec from volume spec2, missing mode", + specVolID: "volspec2", + originSpec: volume.NewSpecFromVolume(makeTestVol("volspec2", testDriver)), + podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + modes: []storagev1beta1.VolumeLifecycleMode{}, + shouldFail: true, }, { name: "missing spec", @@ -451,6 +476,11 @@ func TestPluginConstructVolumeSpecWithInline(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + driver := getTestCSIDriver(testDriver, nil, nil, tc.modes) + client := fakeclient.NewSimpleClientset(driver) + plug, tmpDir := newTestPlugin(t, client) + defer os.RemoveAll(tmpDir) + mounter, err := plug.NewMounter( tc.originSpec, &api.Pod{ObjectMeta: meta.ObjectMeta{UID: tc.podUID, Namespace: testns}}, @@ -520,27 +550,27 @@ func TestPluginNewMounter(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() tests := []struct { - name string - spec *volume.Spec - podUID types.UID - namespace string - csiVolumeMode csiVolumeMode - shouldFail bool + name string + spec *volume.Spec + podUID types.UID + namespace string + volumeLifecycleMode storagev1beta1.VolumeLifecycleMode + shouldFail bool }{ { - name: "mounter from persistent volume source", - spec: volume.NewSpecFromPersistentVolume(makeTestPV("test-pv1", 20, testDriver, testVol), true), - podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - namespace: "test-ns1", - csiVolumeMode: persistentVolumeMode, + name: "mounter from persistent volume source", + spec: volume.NewSpecFromPersistentVolume(makeTestPV("test-pv1", 20, testDriver, testVol), true), + podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + namespace: "test-ns1", + volumeLifecycleMode: storagev1beta1.VolumeLifecyclePersistent, }, { - name: "mounter from volume source", - spec: volume.NewSpecFromVolume(makeTestVol("test-vol1", testDriver)), - podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - namespace: "test-ns2", - csiVolumeMode: ephemeralVolumeMode, - shouldFail: true, // csi inline not enabled + name: "mounter from volume source", + spec: volume.NewSpecFromVolume(makeTestVol("test-vol1", testDriver)), + podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + namespace: "test-ns2", + volumeLifecycleMode: storagev1beta1.VolumeLifecycleEphemeral, + shouldFail: true, // csi inline not enabled }, { name: "mounter from no spec provided", @@ -589,8 +619,8 @@ func TestPluginNewMounter(t *testing.T) { if csiClient == nil { t.Error("mounter csiClient is nil") } - if csiMounter.csiVolumeMode != test.csiVolumeMode { - t.Error("unexpected driver mode:", csiMounter.csiVolumeMode) + if csiMounter.volumeLifecycleMode != test.volumeLifecycleMode { + t.Error("unexpected driver mode:", csiMounter.volumeLifecycleMode) } // ensure data file is created @@ -619,8 +649,8 @@ func TestPluginNewMounter(t *testing.T) { if data[volDataKey.nodeName] != string(csiMounter.plugin.host.GetNodeName()) { t.Error("volume data file unexpected nodeName:", data[volDataKey.nodeName]) } - if data[volDataKey.csiVolumeMode] != string(test.csiVolumeMode) { - t.Error("volume data file unexpected csiVolumeMode:", data[volDataKey.csiVolumeMode]) + if data[volDataKey.volumeLifecycleMode] != string(test.volumeLifecycleMode) { + t.Error("volume data file unexpected volumeLifecycleMode:", data[volDataKey.volumeLifecycleMode]) } }) } @@ -629,13 +659,23 @@ func TestPluginNewMounter(t *testing.T) { func TestPluginNewMounterWithInline(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIBlockVolume, true)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)() + bothModes := []storagev1beta1.VolumeLifecycleMode{ + storagev1beta1.VolumeLifecycleEphemeral, + storagev1beta1.VolumeLifecyclePersistent, + } + persistentMode := []storagev1beta1.VolumeLifecycleMode{ + storagev1beta1.VolumeLifecyclePersistent, + } + ephemeralMode := []storagev1beta1.VolumeLifecycleMode{ + storagev1beta1.VolumeLifecycleEphemeral, + } tests := []struct { - name string - spec *volume.Spec - podUID types.UID - namespace string - csiVolumeMode csiVolumeMode - shouldFail bool + name string + spec *volume.Spec + podUID types.UID + namespace string + volumeLifecycleMode storagev1beta1.VolumeLifecycleMode + shouldFail bool }{ { name: "mounter with missing spec", @@ -651,97 +691,119 @@ func TestPluginNewMounterWithInline(t *testing.T) { shouldFail: true, }, { - name: "mounter with persistent volume source", - spec: volume.NewSpecFromPersistentVolume(makeTestPV("test-pv1", 20, testDriver, testVol), true), - podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - namespace: "test-ns1", - csiVolumeMode: persistentVolumeMode, + name: "mounter with persistent volume source", + spec: volume.NewSpecFromPersistentVolume(makeTestPV("test-pv1", 20, testDriver, testVol), true), + podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + namespace: "test-ns1", + volumeLifecycleMode: storagev1beta1.VolumeLifecyclePersistent, }, { - name: "mounter with volume source", - spec: volume.NewSpecFromVolume(makeTestVol("test-vol1", testDriver)), - podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), - namespace: "test-ns2", - csiVolumeMode: ephemeralVolumeMode, + name: "mounter with volume source", + spec: volume.NewSpecFromVolume(makeTestVol("test-vol1", testDriver)), + podUID: types.UID(fmt.Sprintf("%08X", rand.Uint64())), + namespace: "test-ns2", + volumeLifecycleMode: storagev1beta1.VolumeLifecycleEphemeral, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - plug, tmpDir := newTestPlugin(t, nil) - defer os.RemoveAll(tmpDir) + runAll := func(t *testing.T, supported []storagev1beta1.VolumeLifecycleMode) { + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + driver := getTestCSIDriver(testDriver, nil, nil, supported) + fakeClient := fakeclient.NewSimpleClientset(driver) + plug, tmpDir := newTestPlugin(t, fakeClient) + defer os.RemoveAll(tmpDir) - registerFakePlugin(testDriver, "endpoint", []string{"1.2.0"}, t) - mounter, err := plug.NewMounter( - test.spec, - &api.Pod{ObjectMeta: meta.ObjectMeta{UID: test.podUID, Namespace: test.namespace}}, - volume.VolumeOptions{}, - ) - if test.shouldFail != (err != nil) { - t.Fatal("Unexpected error:", err) - } - if test.shouldFail && err != nil { - t.Log(err) - return - } + registerFakePlugin(testDriver, "endpoint", []string{"1.2.0"}, t) - if mounter == nil { - t.Fatal("failed to create CSI mounter") - } - csiMounter := mounter.(*csiMountMgr) + mounter, err := plug.NewMounter( + test.spec, + &api.Pod{ObjectMeta: meta.ObjectMeta{UID: test.podUID, Namespace: test.namespace}}, + volume.VolumeOptions{}, + ) - // validate mounter fields - if string(csiMounter.driverName) != testDriver { - t.Error("mounter driver name not set") - } - if csiMounter.volumeID == "" { - t.Error("mounter volume id not set") - } - if csiMounter.pod == nil { - t.Error("mounter pod not set") - } - if string(csiMounter.podUID) != string(test.podUID) { - t.Error("mounter podUID not set") - } - csiClient, err := csiMounter.csiClientGetter.Get() - if csiClient == nil { - t.Error("mounter csiClient is nil") - } - if csiMounter.csiVolumeMode != test.csiVolumeMode { - t.Error("unexpected driver mode:", csiMounter.csiVolumeMode) - } + // Some test cases are meant to fail because their input data is broken. + shouldFail := test.shouldFail + // Others fail if the driver does not support the volume mode. + if !containsVolumeMode(supported, test.volumeLifecycleMode) { + shouldFail = true + } + if shouldFail != (err != nil) { + t.Fatal("Unexpected error:", err) + } + if shouldFail && err != nil { + t.Log(err) + return + } - // ensure data file is created - dataDir := path.Dir(mounter.GetPath()) - dataFile := filepath.Join(dataDir, volDataFileName) - if _, err := os.Stat(dataFile); err != nil { - if os.IsNotExist(err) { - t.Errorf("data file not created %s", dataFile) - } else { + if mounter == nil { + t.Fatal("failed to create CSI mounter") + } + csiMounter := mounter.(*csiMountMgr) + + // validate mounter fields + if string(csiMounter.driverName) != testDriver { + t.Error("mounter driver name not set") + } + if csiMounter.volumeID == "" { + t.Error("mounter volume id not set") + } + if csiMounter.pod == nil { + t.Error("mounter pod not set") + } + if string(csiMounter.podUID) != string(test.podUID) { + t.Error("mounter podUID not set") + } + csiClient, err := csiMounter.csiClientGetter.Get() + if csiClient == nil { + t.Error("mounter csiClient is nil") + } + if csiMounter.volumeLifecycleMode != test.volumeLifecycleMode { + t.Error("unexpected driver mode:", csiMounter.volumeLifecycleMode) + } + + // ensure data file is created + dataDir := path.Dir(mounter.GetPath()) + dataFile := filepath.Join(dataDir, volDataFileName) + if _, err := os.Stat(dataFile); err != nil { + if os.IsNotExist(err) { + t.Errorf("data file not created %s", dataFile) + } else { + t.Fatal(err) + } + } + data, err := loadVolumeData(dataDir, volDataFileName) + if err != nil { t.Fatal(err) } - } - data, err := loadVolumeData(dataDir, volDataFileName) - if err != nil { - t.Fatal(err) - } - if data[volDataKey.specVolID] != csiMounter.spec.Name() { - t.Error("volume data file unexpected specVolID:", data[volDataKey.specVolID]) - } - if data[volDataKey.volHandle] != csiMounter.volumeID { - t.Error("volume data file unexpected volHandle:", data[volDataKey.volHandle]) - } - if data[volDataKey.driverName] != string(csiMounter.driverName) { - t.Error("volume data file unexpected driverName:", data[volDataKey.driverName]) - } - if data[volDataKey.nodeName] != string(csiMounter.plugin.host.GetNodeName()) { - t.Error("volume data file unexpected nodeName:", data[volDataKey.nodeName]) - } - if data[volDataKey.csiVolumeMode] != string(csiMounter.csiVolumeMode) { - t.Error("volume data file unexpected csiVolumeMode:", data[volDataKey.csiVolumeMode]) - } - }) + if data[volDataKey.specVolID] != csiMounter.spec.Name() { + t.Error("volume data file unexpected specVolID:", data[volDataKey.specVolID]) + } + if data[volDataKey.volHandle] != csiMounter.volumeID { + t.Error("volume data file unexpected volHandle:", data[volDataKey.volHandle]) + } + if data[volDataKey.driverName] != string(csiMounter.driverName) { + t.Error("volume data file unexpected driverName:", data[volDataKey.driverName]) + } + if data[volDataKey.nodeName] != string(csiMounter.plugin.host.GetNodeName()) { + t.Error("volume data file unexpected nodeName:", data[volDataKey.nodeName]) + } + if data[volDataKey.volumeLifecycleMode] != string(csiMounter.volumeLifecycleMode) { + t.Error("volume data file unexpected volumeLifecycleMode:", data[volDataKey.volumeLifecycleMode]) + } + }) + } } + + t.Run("both supported", func(t *testing.T) { + runAll(t, bothModes) + }) + t.Run("persistent supported", func(t *testing.T) { + runAll(t, persistentMode) + }) + t.Run("ephemeral supported", func(t *testing.T) { + runAll(t, ephemeralMode) + }) } func TestPluginNewUnmounter(t *testing.T) { @@ -871,8 +933,8 @@ func TestPluginCanAttach(t *testing.T) { } for _, test := range tests { - csiDriver := getTestCSIDriver(test.driverName, nil, &test.canAttach) t.Run(test.name, func(t *testing.T) { + csiDriver := getTestCSIDriver(test.driverName, nil, &test.canAttach, nil) fakeCSIClient := fakeclient.NewSimpleClientset(csiDriver) plug, tmpDir := newTestPlugin(t, fakeCSIClient) defer os.RemoveAll(tmpDir) @@ -932,7 +994,7 @@ func TestPluginFindAttachablePlugin(t *testing.T) { } defer os.RemoveAll(tmpDir) - client := fakeclient.NewSimpleClientset(getTestCSIDriver(test.driverName, nil, &test.canAttach)) + client := fakeclient.NewSimpleClientset(getTestCSIDriver(test.driverName, nil, &test.canAttach, nil)) factory := informers.NewSharedInformerFactory(client, CsiResyncPeriod) host := volumetest.NewFakeVolumeHostWithCSINodeName( tmpDir, diff --git a/pkg/volume/csi/csi_test.go b/pkg/volume/csi/csi_test.go index b2865453e93..2e198f257ab 100644 --- a/pkg/volume/csi/csi_test.go +++ b/pkg/volume/csi/csi_test.go @@ -27,7 +27,10 @@ import ( api "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" + storagebeta1 "k8s.io/api/storage/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" @@ -45,6 +48,7 @@ import ( // based on operations from the volume manager/reconciler/operation executor func TestCSI_VolumeAll(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)() + tests := []struct { name string specName string @@ -54,6 +58,7 @@ func TestCSI_VolumeAll(t *testing.T) { podFunc func() *api.Pod isInline bool shouldFail bool + driverSpec *storagebeta1.CSIDriverSpec }{ { name: "PersistentVolume", @@ -69,7 +74,42 @@ func TestCSI_VolumeAll(t *testing.T) { }, }, { - name: "ephermeral inline", + name: "PersistentVolume with driver info", + specName: "pv2", + driver: "simple-driver", + volName: "vol2", + specFunc: func(specName, driver, volName string) *volume.Spec { + return volume.NewSpecFromPersistentVolume(makeTestPV(specName, 20, driver, volName), false) + }, + podFunc: func() *api.Pod { + podUID := types.UID(fmt.Sprintf("%08X", rand.Uint64())) + return &api.Pod{ObjectMeta: meta.ObjectMeta{UID: podUID, Namespace: testns}} + }, + driverSpec: &storagebeta1.CSIDriverSpec{ + // Required for the driver to be accepted for the persistent volume. + VolumeLifecycleModes: []storagebeta1.VolumeLifecycleMode{storagebeta1.VolumeLifecyclePersistent}, + }, + }, + { + name: "PersistentVolume with wrong mode in driver info", + specName: "pv2", + driver: "simple-driver", + volName: "vol2", + specFunc: func(specName, driver, volName string) *volume.Spec { + return volume.NewSpecFromPersistentVolume(makeTestPV(specName, 20, driver, volName), false) + }, + podFunc: func() *api.Pod { + podUID := types.UID(fmt.Sprintf("%08X", rand.Uint64())) + return &api.Pod{ObjectMeta: meta.ObjectMeta{UID: podUID, Namespace: testns}} + }, + driverSpec: &storagebeta1.CSIDriverSpec{ + // This will cause the volume to be rejected. + VolumeLifecycleModes: []storagebeta1.VolumeLifecycleMode{storagebeta1.VolumeLifecycleEphemeral}, + }, + shouldFail: true, + }, + { + name: "ephemeral inline supported", driver: "inline-driver-1", volName: "test.vol2", specFunc: func(specName, driver, volName string) *volume.Spec { @@ -80,6 +120,75 @@ func TestCSI_VolumeAll(t *testing.T) { return &api.Pod{ObjectMeta: meta.ObjectMeta{UID: podUID, Namespace: testns}} }, isInline: true, + driverSpec: &storagebeta1.CSIDriverSpec{ + // Required for the driver to be accepted for the inline volume. + VolumeLifecycleModes: []storagebeta1.VolumeLifecycleMode{storagebeta1.VolumeLifecycleEphemeral}, + }, + }, + { + name: "ephemeral inline also supported", + driver: "inline-driver-1", + volName: "test.vol2", + specFunc: func(specName, driver, volName string) *volume.Spec { + return volume.NewSpecFromVolume(makeTestVol(specName, driver)) + }, + podFunc: func() *api.Pod { + podUID := types.UID(fmt.Sprintf("%08X", rand.Uint64())) + return &api.Pod{ObjectMeta: meta.ObjectMeta{UID: podUID, Namespace: testns}} + }, + isInline: true, + driverSpec: &storagebeta1.CSIDriverSpec{ + // Required for the driver to be accepted for the inline volume. + VolumeLifecycleModes: []storagebeta1.VolumeLifecycleMode{storagebeta1.VolumeLifecyclePersistent, storagebeta1.VolumeLifecycleEphemeral}, + }, + }, + { + name: "ephemeral inline without CSIDriver info", + driver: "inline-driver-2", + volName: "test.vol3", + specFunc: func(specName, driver, volName string) *volume.Spec { + return volume.NewSpecFromVolume(makeTestVol(specName, driver)) + }, + podFunc: func() *api.Pod { + podUID := types.UID(fmt.Sprintf("%08X", rand.Uint64())) + return &api.Pod{ObjectMeta: meta.ObjectMeta{UID: podUID, Namespace: testns}} + }, + isInline: true, + }, + { + name: "ephemeral inline with driver that has no mode", + driver: "inline-driver-3", + volName: "test.vol4", + specFunc: func(specName, driver, volName string) *volume.Spec { + return volume.NewSpecFromVolume(makeTestVol(specName, driver)) + }, + podFunc: func() *api.Pod { + podUID := types.UID(fmt.Sprintf("%08X", rand.Uint64())) + return &api.Pod{ObjectMeta: meta.ObjectMeta{UID: podUID, Namespace: testns}} + }, + isInline: true, + driverSpec: &storagebeta1.CSIDriverSpec{ + // This means the driver *cannot* handle the inline volume because + // the default is "persistent". + VolumeLifecycleModes: nil, + }, + }, + { + name: "ephemeral inline with driver that has wrong mode", + driver: "inline-driver-3", + volName: "test.vol4", + specFunc: func(specName, driver, volName string) *volume.Spec { + return volume.NewSpecFromVolume(makeTestVol(specName, driver)) + }, + podFunc: func() *api.Pod { + podUID := types.UID(fmt.Sprintf("%08X", rand.Uint64())) + return &api.Pod{ObjectMeta: meta.ObjectMeta{UID: podUID, Namespace: testns}} + }, + isInline: true, + driverSpec: &storagebeta1.CSIDriverSpec{ + // This means the driver *cannot* handle the inline volume. + VolumeLifecycleModes: []storagebeta1.VolumeLifecycleMode{storagebeta1.VolumeLifecyclePersistent}, + }, }, { name: "missing spec", @@ -118,10 +227,27 @@ func TestCSI_VolumeAll(t *testing.T) { t.Fatalf("can't create temp dir: %v", err) } defer os.RemoveAll(tmpDir) - client := fakeclient.NewSimpleClientset() + + var driverInfo *storagebeta1.CSIDriver + objs := []runtime.Object{} + if test.driverSpec != nil { + driverInfo = &storagebeta1.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: test.driver, + }, + Spec: *test.driverSpec, + } + objs = append(objs, driverInfo) + } + + client := fakeclient.NewSimpleClientset(objs...) fakeWatcher := watch.NewRaceFreeFake() - factory := informers.NewSharedInformerFactory(client, CsiResyncPeriod) + factory := informers.NewSharedInformerFactory(client, time.Hour /* disable resync */) + csiDriverInformer := factory.Storage().V1beta1().CSIDrivers() + if driverInfo != nil { + csiDriverInformer.Informer().GetStore().Add(driverInfo) + } factory.Start(wait.NeverStop) host := volumetest.NewFakeVolumeHostWithCSINodeName( @@ -129,7 +255,7 @@ func TestCSI_VolumeAll(t *testing.T) { client, nil, "csi-node", - factory.Storage().V1beta1().CSIDrivers().Lister(), + csiDriverInformer.Lister(), ) plugMgr := &volume.VolumePluginMgr{} @@ -253,6 +379,22 @@ func TestCSI_VolumeAll(t *testing.T) { } mounter, err := volPlug.NewMounter(volSpec, pod, volume.VolumeOptions{}) + if test.isInline && (test.driverSpec == nil || !containsVolumeMode(test.driverSpec.VolumeLifecycleModes, storagebeta1.VolumeLifecycleEphemeral)) { + // This *must* fail because a CSIDriver.Spec.VolumeLifecycleModes entry "ephemeral" + // is required. + if err == nil || mounter != nil { + t.Fatalf("csiTest.VolumeAll volPlugin.NewMounter should have failed for inline volume due to lack of support for inline volumes, got: %+v, %s", mounter, err) + } + return + } + if !test.isInline && test.driverSpec != nil && !containsVolumeMode(test.driverSpec.VolumeLifecycleModes, storagebeta1.VolumeLifecyclePersistent) { + // This *must* fail because a CSIDriver.Spec.VolumeLifecycleModes entry "persistent" + // is required when a driver object is available. + if err == nil || mounter != nil { + t.Fatalf("csiTest.VolumeAll volPlugin.NewMounter should have failed for persistent volume due to lack of support for persistent volumes, got: %+v, %s", mounter, err) + } + return + } if err != nil || mounter == nil { t.Fatalf("csiTest.VolumeAll volPlugin.NewMounter is nil or error: %s", err) } diff --git a/pkg/volume/csi/csi_util_test.go b/pkg/volume/csi/csi_util_test.go index 6f2919f0085..8909102092e 100644 --- a/pkg/volume/csi/csi_util_test.go +++ b/pkg/volume/csi/csi_util_test.go @@ -84,14 +84,15 @@ func makeTestVol(name string, driverName string) *api.Volume { } } -func getTestCSIDriver(name string, podInfoMount *bool, attachable *bool) *storagev1beta1.CSIDriver { +func getTestCSIDriver(name string, podInfoMount *bool, attachable *bool, volumeLifecycleModes []storagev1beta1.VolumeLifecycleMode) *storagev1beta1.CSIDriver { return &storagev1beta1.CSIDriver{ ObjectMeta: meta.ObjectMeta{ Name: name, }, Spec: storagev1beta1.CSIDriverSpec{ - PodInfoOnMount: podInfoMount, - AttachRequired: attachable, + PodInfoOnMount: podInfoMount, + AttachRequired: attachable, + VolumeLifecycleModes: volumeLifecycleModes, }, } } diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 73e491d6049..0aecd5c31c4 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -43,6 +43,7 @@ import ( "github.com/onsi/ginkgo" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + storagev1beta1 "k8s.io/api/storage/v1beta1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/test/e2e/framework" @@ -195,7 +196,6 @@ var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{} // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface func InitMockCSIDriver(driverOpts CSIMockDriverOpts) testsuites.TestDriver { driverManifests := []string{ - "test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", @@ -206,7 +206,7 @@ func InitMockCSIDriver(driverOpts CSIMockDriverOpts) testsuites.TestDriver { } if driverOpts.RegisterDriver { - driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml") + driverManifests = append(driverManifests, "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driverinfo.yaml") } if !driverOpts.DisableAttach { @@ -288,14 +288,18 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest } o := utils.PatchCSIOptions{ - OldDriverName: "csi-mock", - NewDriverName: "csi-mock-" + f.UniqueName, - DriverContainerName: "mock", - DriverContainerArguments: containerArgs, - ProvisionerContainerName: "csi-provisioner", - ClusterRegistrarContainerName: "csi-cluster-driver-registrar", - NodeName: config.ClientNodeName, - PodInfo: m.podInfo, + OldDriverName: "csi-mock", + NewDriverName: "csi-mock-" + f.UniqueName, + DriverContainerName: "mock", + DriverContainerArguments: containerArgs, + ProvisionerContainerName: "csi-provisioner", + NodeName: config.ClientNodeName, + PodInfo: m.podInfo, + CanAttach: &m.attachable, + VolumeLifecycleModes: []storagev1beta1.VolumeLifecycleMode{ + storagev1beta1.VolumeLifecyclePersistent, + storagev1beta1.VolumeLifecycleEphemeral, + }, } cleanup, err := f.CreateFromManifests(func(item interface{}) error { return utils.PatchCSIDeployment(f, o, item) diff --git a/test/e2e/storage/testsuites/testdriver.go b/test/e2e/storage/testsuites/testdriver.go index 6a9b30e1094..ed00729fe69 100644 --- a/test/e2e/storage/testsuites/testdriver.go +++ b/test/e2e/storage/testsuites/testdriver.go @@ -114,7 +114,9 @@ type EphemeralTestDriver interface { // GetCSIDriverName returns the name that was used when registering with // kubelet. Depending on how the driver was deployed, this can be different - // from DriverInfo.Name. + // from DriverInfo.Name. Starting with Kubernetes 1.16, there must also + // be a CSIDriver object under the same name with a "mode" field that enables + // usage of the driver for ephemeral inline volumes. GetCSIDriverName(config *PerTestConfig) string } diff --git a/test/e2e/storage/utils/deployment.go b/test/e2e/storage/utils/deployment.go index b1a3aed5af5..067e8cd20f4 100644 --- a/test/e2e/storage/utils/deployment.go +++ b/test/e2e/storage/utils/deployment.go @@ -18,12 +18,12 @@ package utils import ( "path" - "strconv" "strings" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + storagev1beta1 "k8s.io/api/storage/v1beta1" "k8s.io/kubernetes/test/e2e/framework" ) @@ -94,10 +94,6 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf // Driver name is expected to be the same // as the snapshotter here. container.Args = append(container.Args, "--snapshotter="+o.NewDriverName) - case o.ClusterRegistrarContainerName: - if o.PodInfo != nil { - container.Args = append(container.Args, "--pod-info-mount="+strconv.FormatBool(*o.PodInfo)) - } } } } @@ -125,6 +121,17 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf // as the provisioner name here. object.Provisioner = o.NewDriverName } + case *storagev1beta1.CSIDriver: + if o.NewDriverName != "" { + object.Name = o.NewDriverName + } + if o.PodInfo != nil { + object.Spec.PodInfoOnMount = o.PodInfo + } + if o.CanAttach != nil { + object.Spec.AttachRequired = o.CanAttach + } + object.Spec.VolumeLifecycleModes = o.VolumeLifecycleModes } return nil @@ -154,12 +161,18 @@ type PatchCSIOptions struct { // If non-empty, --snapshotter with new name will be appended // to the argument list. SnapshotterContainerName string - // The name of the container which has the cluster-driver-registrar - // binary. - ClusterRegistrarContainerName string // If non-empty, all pods are forced to run on this node. NodeName string - // If not nil, the argument to pass to the cluster-driver-registrar's - // pod-info-mount argument. + // If not nil, the value to use for the CSIDriver.Spec.PodInfo + // field *if* the driver deploys a CSIDriver object. Ignored + // otherwise. PodInfo *bool + // If not nil, the value to use for the CSIDriver.Spec.CanAttach + // field *if* the driver deploys a CSIDriver object. Ignored + // otherwise. + CanAttach *bool + // The value to use for the CSIDriver.Spec.VolumeLifecycleModes + // field *if* the driver deploys a CSIDriver object. Ignored + // otherwise. + VolumeLifecycleModes []storagev1beta1.VolumeLifecycleMode } diff --git a/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/README.md b/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/README.md deleted file mode 100644 index 6247e1c98d9..00000000000 --- a/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/README.md +++ /dev/null @@ -1 +0,0 @@ -The original file is (or will be) https://github.com/kubernetes-csi/cluster-driver-registrar/blob/master/deploy/kubernetes/rbac.yaml diff --git a/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml b/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml deleted file mode 100644 index 0dd03f7f96d..00000000000 --- a/test/e2e/testing-manifests/storage-csi/cluster-driver-registrar/rbac.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# This YAML file contains all RBAC objects that are necessary to run -# cluster-driver-registrar. -# -# In production, each CSI driver deployment has to be customized: -# - to avoid conflicts, use non-default namespace and different names -# for non-namespaced entities like the ClusterRole - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-cluster-driver-registrar - # replace with non-default namespace name - namespace: default - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cluster-driver-registrar-runner -rules: - - apiGroups: ["storage.k8s.io"] - resources: ["csidrivers"] - verbs: ["create", "delete"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-cluster-driver-registrar-role -subjects: - - kind: ServiceAccount - name: csi-cluster-driver-registrar - # replace with non-default namespace name - namespace: default -roleRef: - kind: ClusterRole - name: cluster-driver-registrar-runner - apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml deleted file mode 100644 index e266c6c3c6b..00000000000 --- a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-cluster-driver-registrar.yaml +++ /dev/null @@ -1,36 +0,0 @@ -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-mockplugin-cluster-driver-registrar -spec: - selector: - matchLabels: - app: csi-mockplugin-cluster-driver-registrar - replicas: 1 - template: - metadata: - labels: - app: csi-mockplugin-cluster-driver-registrar - spec: - serviceAccountName: csi-mock - containers: - - name: csi-cluster-driver-registrar - # TODO: replace with official image - image: quay.io/k8scsi/csi-cluster-driver-registrar:tmp-test-1.14 - args: - - --v=5 - - --csi-address=$(ADDRESS) - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: Always - securityContext: - privileged: true - volumeMounts: - - mountPath: /csi - name: socket-dir - volumes: - - hostPath: - path: /var/lib/kubelet/plugins/csi-mock - type: DirectoryOrCreate - name: socket-dir diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml index 447adc3fdf4..3c588a971e8 100644 --- a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml @@ -18,7 +18,6 @@ spec: # TODO: replace with official 1.4.0 release when ready image: quay.io/k8scsi/csi-provisioner:v1.4.0-rc1 args: - - "--provisioner=csi-hostpath" - "--csi-address=$(ADDRESS)" - "--connection-timeout=15s" env: @@ -51,6 +50,7 @@ spec: - name: mock image: quay.io/k8scsi/mock-driver:v2.1.0 args: + - "--name=mock.storage.k8s.io" - "--permissive-target-path" # because of https://github.com/kubernetes/kubernetes/issues/75535 env: - name: CSI_ENDPOINT diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driverinfo.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driverinfo.yaml new file mode 100644 index 00000000000..5c5977bd673 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driverinfo.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: mock.storage.k8s.io +# Intentionally no spec. All values in the +# spec will be inserted dynamically by PatchCSIDeployment() +# in test/e2e/storage/utils/deployment.go. From 1db126347698482931bfdbbf4b51ce648db045fa Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 8 Aug 2019 21:55:01 +0200 Subject: [PATCH 5/6] storage: add test for ValidateCSIDriverUpdate Adding this test was forgotten when adding CSIDriver. --- .../storage/validation/validation_test.go | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/pkg/apis/storage/validation/validation_test.go b/pkg/apis/storage/validation/validation_test.go index 81f5c0eca6a..8363b201cbf 100644 --- a/pkg/apis/storage/validation/validation_test.go +++ b/pkg/apis/storage/validation/validation_test.go @@ -1826,3 +1826,94 @@ func TestCSIDriverValidation(t *testing.T) { } } } + +func TestCSIDriverValidationUpdate(t *testing.T) { + driverName := "test-driver" + longName := "my-a-b-c-d-c-f-g-h-i-j-k-l-m-n-o-p-q-r-s-t-u-v-w-x-y-z-ABCDEFGHIJKLMNOPQRSTUVWXYZ-driver" + invalidName := "-invalid-@#$%^&*()-" + attachRequired := true + attachNotRequired := false + podInfoOnMount := true + notPodInfoOnMount := false + old := storage.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: ¬PodInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleEphemeral, + storage.VolumeLifecyclePersistent, + }, + }, + } + + // Currently there is only one success case: exactly the same + // as the existing object. + successCases := []storage.CSIDriver{old} + for _, csiDriver := range successCases { + if errs := ValidateCSIDriverUpdate(&csiDriver, &old); len(errs) != 0 { + t.Errorf("expected success for %+v: %v", csiDriver, errs) + } + } + + errorCases := []storage.CSIDriver{ + { + ObjectMeta: metav1.ObjectMeta{Name: invalidName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachRequired, + PodInfoOnMount: &podInfoOnMount, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: longName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: ¬PodInfoOnMount, + }, + }, + { + // AttachRequired not set + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: nil, + PodInfoOnMount: &podInfoOnMount, + }, + }, + { + // AttachRequired not set + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: nil, + }, + }, + { + // invalid mode + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: ¬PodInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + "no-such-mode", + }, + }, + }, + { + // different modes + ObjectMeta: metav1.ObjectMeta{Name: driverName}, + Spec: storage.CSIDriverSpec{ + AttachRequired: &attachNotRequired, + PodInfoOnMount: ¬PodInfoOnMount, + VolumeLifecycleModes: []storage.VolumeLifecycleMode{ + storage.VolumeLifecycleEphemeral, + }, + }, + }, + } + + for _, csiDriver := range errorCases { + if errs := ValidateCSIDriverUpdate(&csiDriver, &old); len(errs) == 0 { + t.Errorf("Expected failure for test: %v", csiDriver) + } + } +} From 8270fe81e4916b599c8a9a83552a5381b434dd7f Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 25 Jul 2019 10:26:26 +0200 Subject: [PATCH 6/6] bazel update --- pkg/volume/csi/BUILD | 1 + test/e2e/storage/drivers/BUILD | 1 + test/e2e/storage/utils/BUILD | 1 + 3 files changed, 3 insertions(+) diff --git a/pkg/volume/csi/BUILD b/pkg/volume/csi/BUILD index 80beee758e6..097c2ad94ed 100644 --- a/pkg/volume/csi/BUILD +++ b/pkg/volume/csi/BUILD @@ -23,6 +23,7 @@ go_library( "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", + "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/test/e2e/storage/drivers/BUILD b/test/e2e/storage/drivers/BUILD index b1f489cb788..5dbd9054686 100644 --- a/test/e2e/storage/drivers/BUILD +++ b/test/e2e/storage/drivers/BUILD @@ -13,6 +13,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", + "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", diff --git a/test/e2e/storage/utils/BUILD b/test/e2e/storage/utils/BUILD index ae5f2542d93..0e63b5d0c60 100644 --- a/test/e2e/storage/utils/BUILD +++ b/test/e2e/storage/utils/BUILD @@ -20,6 +20,7 @@ go_library( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/rbac/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", + "//staging/src/k8s.io/api/storage/v1beta1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",