From 3595dee6ccc31364660ab8bb431370875b6d8343 Mon Sep 17 00:00:00 2001 From: David Zhu Date: Thu, 8 Nov 2018 13:12:35 -0800 Subject: [PATCH] Add fields available and volumePluginMechanism to CSINodeInfo CRD API Object. Split CSINodeInfo into Spec and Status. --- .../csi-api/pkg/apis/csi/v1alpha1/README.md | 51 ++++++++++++ .../csi-api/pkg/apis/csi/v1alpha1/types.go | 64 +++++++++++++-- .../csi/v1alpha1/zz_generated.deepcopy.go | 77 ++++++++++++++++--- .../typed/csi/v1alpha1/csinodeinfo.go | 16 ++++ .../csi/v1alpha1/fake/fake_csinodeinfo.go | 11 +++ .../pkg/crd/manifests/csinodeinfo.yaml | 54 +++++++++---- 6 files changed, 238 insertions(+), 35 deletions(-) create mode 100644 staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/README.md diff --git a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/README.md b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/README.md new file mode 100644 index 00000000000..d77a72925ac --- /dev/null +++ b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/README.md @@ -0,0 +1,51 @@ +# CSINodeInfo and CSIDriverInfo Usage and Lifecycle + +CSINodeInfo is an API object representing CSI Information at the Node level. +CSINodeInfo contains a Spec and a Status, each containing Drivers represent the +driver spec and status respectively. The CSIDriverInfoSpec represents the +specification of the driver and is generally not changed, whereas the +CSIDriverInfoStatus represents the current status of the driver and can be +updated and reacted to by various components. + +## Who creates it and when + +CSINodeInfo is created by Kubelet when the first CSI Driver is installed to the +cluster and it is registered through the Kubelet device registration mechanism. + +## Who updates CSIDriverInfo Spec and when + +The CSIDriverInfoSpec ror a driver is created upon installation of the CSI +Driver to the cluster and it is registered through the Kubelet device +registration mechanism. The spec is populated with information about the driver +through the nodeinfomanager and will remain unchanged from then on. + +## Who updates Status and when + +The CSIDriverInfoStatus for the driver is created upon installation of the CSI +Driver to the cluster (the same time as the spec) and it is registered through +the Kubelet device registration mechanism. The Status contains information about +installation and the required Volume Plugin Mechanism of the driver. When the +driver is installed/uninstalled through the Kubelet device registration +mechanism the Available flag is flipped from true/false respectively. The +migration status will also be updated when the flags for migration are set to +true/false on the Kubelet for that Driver on that node. + +## Consumers of Status and Spec + +Currently the only consumer of CSINodeInfo/CSIDriverInfo is the +csi-external-provisioner. In the future, the Attach Detach Controller (ADC) will +need to read this object to determine migration status on a per driver per node +basis. The creation of the CSINodeInfo object could possibly race with the +Attach/Detach controller as for CSI Migration the controller depend on the +existence of the API object for the driver but it will not have been created +yet. The ADC is expected to fail (and retry with exponential backoff) the +operation if it is expecting the object and it has not yet been created. + +## Creation of CSINodeInfo object on Kubelet startup + +For CSI Migration Alpha we expect any user who turns on the feature has both +Kubelet and ADC at a version where the CSINodeInfo's are being created on +Kubelet startup. We will not promote the feature to Beta (on by default) until +the CSINodeInfo's are being created on Kubelet startup for a 2 version skew to +prevent the case where the CSINodeInfo does not exist when the ADC depends on +it. This prevents the race described above becoming a permanent bad state. diff --git a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/types.go b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/types.go index 635f22af8dd..6232b6b4d8e 100644 --- a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/types.go +++ b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/types.go @@ -18,6 +18,16 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// VolumePluginMechanism is the type of mechanism components should use for volume operations +type VolumePluginMechanism string + +const ( + // VolumePluginMechanismInTree means components should use the in-tree plugin for volume operations + VolumePluginMechanismInTree VolumePluginMechanism = "in-tree" + // VolumePluginMechanismCSI means components should use the CSI Driver for volume operations + VolumePluginMechanismCSI VolumePluginMechanism = "csi" +) + // +genclient // +genclient:nonNamespaced // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -91,18 +101,56 @@ type CSINodeInfo struct { // metadata.name must be the Kubernetes node name. metav1.ObjectMeta `json:"metadata,omitempty"` - // List of CSI drivers running on the node and their properties. - // +patchMergeKey=driver - // +patchStrategy=merge - CSIDrivers []CSIDriverInfo `json:"csiDrivers" patchStrategy:"merge" patchMergeKey:"driver"` + // spec is the specification of CSINodeInfo + Spec CSINodeInfoSpec `json:"spec"` + + // status is the current status of CSINodeInfo + Status CSINodeInfoStatus `json:"status"` } -// CSIDriverInfo contains information about one CSI driver installed on a node. -type CSIDriverInfo struct { - // driver is the name of the CSI driver that this object refers to. +// CSINodeInfoSpec holds information about the specification of all CSI drivers installed on a node +type CSINodeInfoSpec struct { + // drivers is a list of specifications of CSIDriverInfo + // +patchMergeKey=name + // +patchStrategy=merge + Drivers []CSIDriverInfoSpec `json:"drivers" patchStrategy:"merge" patchMergeKey:"name"` +} + +// CSINodeInfoStatus holds information about the status of all CSI drivers installed on a node +type CSINodeInfoStatus struct { + // drivers is a list of the statuses of CSIDriverInfo + // +patchMergeKey=name + // +patchStrategy=merge + Drivers []CSIDriverInfoStatus `json:"drivers" patchStrategy:"merge" patchMergeKey:"name"` +} + +// CSIDriverInfoStatus holds information about the status of one CSI driver installed on a node +type CSIDriverInfoStatus struct { + // name is the name of the CSI driver that this object refers to. // This MUST be the same name returned by the CSI GetPluginName() call for // that driver. - Driver string `json:"driver"` + Name string `json:"name"` + + // available is a boolean representing whether the driver has been installed + // on this node or not. + Available bool `json:"available"` + + // volumePluginMechanism announces what mechanism underlies volume plugins. + // It is set by Kubelet. It is used by the attach/detach controller, which + // needs to know how to perform attachments. The allowed values are: + // * "in-tree": the volume operation (e.g., attach/detach) ought to be + // directly performed by the attach/detach controller. + // * "csi-plugin": the attach/detach controller ought to request + // the csi plugin to perform the volume operation rather than perform it directly. + VolumePluginMechanism VolumePluginMechanism `json:"volumePluginMechanism"` +} + +// CSIDriverInfoSpec holds information about the specification of one CSI driver installed on a node +type CSIDriverInfoSpec struct { + // name is the name of the CSI driver that this object refers to. + // This MUST be the same name returned by the CSI GetPluginName() call for + // that driver. + Name string `json:"name"` // nodeID of the node from the driver point of view. // This field enables Kubernetes to communicate with storage systems that do diff --git a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/zz_generated.deepcopy.go b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/zz_generated.deepcopy.go index aa9ccbbaae8..915ef995a01 100644 --- a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/zz_generated.deepcopy.go @@ -52,7 +52,7 @@ func (in *CSIDriver) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CSIDriverInfo) DeepCopyInto(out *CSIDriverInfo) { +func (in *CSIDriverInfoSpec) DeepCopyInto(out *CSIDriverInfoSpec) { *out = *in if in.TopologyKeys != nil { in, out := &in.TopologyKeys, &out.TopologyKeys @@ -62,12 +62,28 @@ func (in *CSIDriverInfo) DeepCopyInto(out *CSIDriverInfo) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverInfo. -func (in *CSIDriverInfo) DeepCopy() *CSIDriverInfo { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverInfoSpec. +func (in *CSIDriverInfoSpec) DeepCopy() *CSIDriverInfoSpec { if in == nil { return nil } - out := new(CSIDriverInfo) + out := new(CSIDriverInfoSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSIDriverInfoStatus) DeepCopyInto(out *CSIDriverInfoStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverInfoStatus. +func (in *CSIDriverInfoStatus) DeepCopy() *CSIDriverInfoStatus { + if in == nil { + return nil + } + out := new(CSIDriverInfoStatus) in.DeepCopyInto(out) return out } @@ -136,13 +152,8 @@ func (in *CSINodeInfo) DeepCopyInto(out *CSINodeInfo) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - if in.CSIDrivers != nil { - in, out := &in.CSIDrivers, &out.CSIDrivers - *out = make([]CSIDriverInfo, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) return } @@ -196,3 +207,47 @@ func (in *CSINodeInfoList) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSINodeInfoSpec) DeepCopyInto(out *CSINodeInfoSpec) { + *out = *in + if in.Drivers != nil { + in, out := &in.Drivers, &out.Drivers + *out = make([]CSIDriverInfoSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSINodeInfoSpec. +func (in *CSINodeInfoSpec) DeepCopy() *CSINodeInfoSpec { + if in == nil { + return nil + } + out := new(CSINodeInfoSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSINodeInfoStatus) DeepCopyInto(out *CSINodeInfoStatus) { + *out = *in + if in.Drivers != nil { + in, out := &in.Drivers, &out.Drivers + *out = make([]CSIDriverInfoStatus, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSINodeInfoStatus. +func (in *CSINodeInfoStatus) DeepCopy() *CSINodeInfoStatus { + if in == nil { + return nil + } + out := new(CSINodeInfoStatus) + in.DeepCopyInto(out) + return out +} diff --git a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csinodeinfo.go b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csinodeinfo.go index 8580127c02f..2efc934691e 100644 --- a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csinodeinfo.go +++ b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csinodeinfo.go @@ -37,6 +37,7 @@ type CSINodeInfosGetter interface { type CSINodeInfoInterface interface { Create(*v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error) Update(*v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error) + UpdateStatus(*v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error) Delete(name string, options *v1.DeleteOptions) error DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error Get(name string, options v1.GetOptions) (*v1alpha1.CSINodeInfo, error) @@ -113,6 +114,21 @@ func (c *cSINodeInfos) Update(cSINodeInfo *v1alpha1.CSINodeInfo) (result *v1alph return } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *cSINodeInfos) UpdateStatus(cSINodeInfo *v1alpha1.CSINodeInfo) (result *v1alpha1.CSINodeInfo, err error) { + result = &v1alpha1.CSINodeInfo{} + err = c.client.Put(). + Resource("csinodeinfos"). + Name(cSINodeInfo.Name). + SubResource("status"). + Body(cSINodeInfo). + Do(). + Into(result) + return +} + // Delete takes name of the cSINodeInfo and deletes it. Returns an error if one occurs. func (c *cSINodeInfos) Delete(name string, options *v1.DeleteOptions) error { return c.client.Delete(). diff --git a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/fake/fake_csinodeinfo.go b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/fake/fake_csinodeinfo.go index babda7c171c..6e44b9d3850 100644 --- a/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/fake/fake_csinodeinfo.go +++ b/staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/fake/fake_csinodeinfo.go @@ -94,6 +94,17 @@ func (c *FakeCSINodeInfos) Update(cSINodeInfo *v1alpha1.CSINodeInfo) (result *v1 return obj.(*v1alpha1.CSINodeInfo), err } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeCSINodeInfos) UpdateStatus(cSINodeInfo *v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(csinodeinfosResource, "status", cSINodeInfo), &v1alpha1.CSINodeInfo{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.CSINodeInfo), err +} + // Delete takes name of the cSINodeInfo and deletes it. Returns an error if one occurs. func (c *FakeCSINodeInfos) Delete(name string, options *v1.DeleteOptions) error { _, err := c.Fake. diff --git a/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml index a9556fc1e82..7b12ba57666 100644 --- a/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml +++ b/staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml @@ -13,20 +13,42 @@ spec: validation: openAPIV3Schema: properties: - csiDrivers: - description: List of CSI drivers running on the node and their properties. - items: - properties: - driver: - description: The CSI driver that this object refers to. - type: string - nodeID: - description: The node from the driver point of view. - type: string - topologyKeys: - description: List of keys supported by the driver. - items: - type: string - type: array - type: array + spec: + description: Specification of CSINodeInfo + properties: + drivers: + description: List of CSI drivers running on the node and their specs. + type: array + items: + properties: + name: + description: The CSI driver that this object refers to. + type: string + nodeID: + description: The node from the driver point of view. + type: string + topologyKeys: + description: List of keys supported by the driver. + items: + type: string + type: array + status: + description: Status of CSINodeInfo + properties: + drivers: + description: List of CSI drivers running on the node and their statuses. + type: array + items: + properties: + name: + description: The CSI driver that this object refers to. + type: string + available: + description: Whether the CSI driver is installed. + type: boolean + volumePluginMechanism: + description: Indicates to external components the required mechanism + to use for any in-tree plugins replaced by this driver. + pattern: in-tree|csi + type: string version: v1alpha1