diff --git a/cluster/addons/volumesnapshots/OWNERS b/cluster/addons/volumesnapshots/OWNERS new file mode 100644 index 00000000000..3d1fa9a70b6 --- /dev/null +++ b/cluster/addons/volumesnapshots/OWNERS @@ -0,0 +1,7 @@ +approvers: +- saad-ali +- msau42 +reviewers: +- jingxu97 +- xing-yang +- yuxiangqian diff --git a/cluster/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml b/cluster/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml new file mode 100644 index 00000000000..90f6b6bd7f0 --- /dev/null +++ b/cluster/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml @@ -0,0 +1,68 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotclasses.snapshot.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + preserveUnknownFields: false + validation: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created + through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. Supported values are "Retain" and "Delete". "Retain" means + that the VolumeSnapshotContent and its physical snapshot on underlying + storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/cluster/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml b/cluster/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml new file mode 100644 index 00000000000..8114e8e5433 --- /dev/null +++ b/cluster/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml @@ -0,0 +1,197 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshotcontents.snapshot.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + subresources: + status: {} + preserveUnknownFields: false + validation: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. "Delete" + means that the VolumeSnapshotContent and its physical snapshot on + underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For + pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be the + same as the name returned by the CSI GetPluginName() call for that + driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a + pre-existing snapshot on the underlying storage system. This field + is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume + from which a snapshot should be dynamically taken from. This field + is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to + which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be provided + for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on + the underlying storage system. If not specified, it indicates that + dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/cluster/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshots.yaml b/cluster/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshots.yaml new file mode 100644 index 00000000000..1e16e3e7a6a --- /dev/null +++ b/cluster/addons/volumesnapshots/crd/snapshot.storage.k8s.io_volumesnapshots.yaml @@ -0,0 +1,144 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: volumesnapshots.snapshot.storage.k8s.io + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + subresources: + status: {} + preserveUnknownFields: false + validation: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. This + field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing + VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is no + default snapshot class, dynamic snapshot creation will fail. Empty + string is not allowed for this field. TODO(xiangqian): a webhook validation + on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. NOTE: + status can be modified by sources other than system controllers, and must + not be depended upon for accuracy. Controllers should only use information + from the VolumeSnapshotContent object after verifying that the binding + is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot object + has not been successfully bound to a VolumeSnapshotContent object + yet. NOTE: Specified boundVolumeSnapshotContentName alone does not + mean binding is valid. Controllers MUST always verify bidirectional + binding between VolumeSnapshot and VolumeSnapshotContent to + avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates that the creation time of the snapshot + is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/cluster/addons/volumesnapshots/volume-snapshot-controller/rbac-volume-snapshot-controller.yaml b/cluster/addons/volumesnapshots/volume-snapshot-controller/rbac-volume-snapshot-controller.yaml new file mode 100644 index 00000000000..8d92502d343 --- /dev/null +++ b/cluster/addons/volumesnapshots/volume-snapshot-controller/rbac-volume-snapshot-controller.yaml @@ -0,0 +1,99 @@ +# RBAC file for the volume snapshot controller. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: volume-snapshot-controller + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # rename if there are conflicts + name: volume-snapshot-controller-runner + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete", "get", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: volume-snapshot-controller-role + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +subjects: + - kind: ServiceAccount + name: volume-snapshot-controller + namespace: kube-system +roleRef: + kind: ClusterRole + # change the name also here if the ClusterRole gets renamed + name: volume-snapshot-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: volume-snapshot-controller-leaderelection + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: volume-snapshot-controller-leaderelection + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +subjects: + - kind: ServiceAccount + name: volume-snapshot-controller + namespace: kube-system +roleRef: + kind: Role + name: volume-snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + diff --git a/cluster/addons/volumesnapshots/volume-snapshot-controller/volume-snapshot-controller-deployment.yaml b/cluster/addons/volumesnapshots/volume-snapshot-controller/volume-snapshot-controller-deployment.yaml new file mode 100644 index 00000000000..2315c855305 --- /dev/null +++ b/cluster/addons/volumesnapshots/volume-snapshot-controller/volume-snapshot-controller-deployment.yaml @@ -0,0 +1,29 @@ +# This YAML file shows how to deploy the volume snapshot controller + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: volume-snapshot-controller + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + serviceName: "volume-snapshot-controller" + replicas: 1 + selector: + matchLabels: + app: volume-snapshot-controller + template: + metadata: + labels: + app: volume-snapshot-controller + spec: + serviceAccount: volume-snapshot-controller + containers: + - name: volume-snapshot-controller + # TODO(xyang): Replace with an official image when it is released + image: quay.io/k8scsi/snapshot-controller:v2.0.0-rc2 + args: + - "--v=5" + imagePullPolicy: Always diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 4d79d73068f..ddc097241b8 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -407,6 +407,9 @@ SCHEDULING_ALGORITHM_PROVIDER="${SCHEDULING_ALGORITHM_PROVIDER:-}" # Optional: install a default StorageClass ENABLE_DEFAULT_STORAGE_CLASS="${ENABLE_DEFAULT_STORAGE_CLASS:-true}" +# Optional: install volume snapshot CRDs +ENABLE_VOLUME_SNAPSHOTS="${ENABLE_VOLUME_SNAPSHOTS:-true}" + # Optional: Enable legacy ABAC policy that makes all service accounts superusers. ENABLE_LEGACY_ABAC="${ENABLE_LEGACY_ABAC:-false}" # true, false diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index f17c013f4e4..4d766a94c53 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -444,6 +444,9 @@ SCHEDULING_ALGORITHM_PROVIDER="${SCHEDULING_ALGORITHM_PROVIDER:-}" # Optional: install a default StorageClass ENABLE_DEFAULT_STORAGE_CLASS="${ENABLE_DEFAULT_STORAGE_CLASS:-true}" +# Optional: install volume snapshot CRDs +ENABLE_VOLUME_SNAPSHOTS="${ENABLE_VOLUME_SNAPSHOTS:-true}" + # Optional: Enable legacy ABAC policy that makes all service accounts superusers. # Disabling this by default in tests ensures default RBAC policies are sufficient from 1.6+ # Upgrade test jobs that go from a version < 1.6 to a version >= 1.6 should override this to be true. diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 748aef03258..aae6a8f906c 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2076,6 +2076,45 @@ function start-fluentd-resource-update { wait-for-apiserver-and-update-fluentd & } +# VolumeSnapshot CRDs and controller are installed by cluster addon manager, +# which may not be available at this point. Run this as a background process. +function wait-for-volumesnapshot-crd-and-controller { + # Wait until volumesnapshot CRDs and controller are in place. + echo "Wait until volume snapshot CRDs are installed" + until kubectl get volumesnapshotclasses.snapshot.storage.k8s.io + do + sleep 10 + done + + until kubectl get volumesnapshotcontents.snapshot.storage.k8s.io + do + sleep 10 + done + + until kubectl get volumesnapshots.snapshot.storage.k8s.io + do + sleep 10 + done + + echo "Wait until volume snapshot RBAC rules are installed" + until kubectl get clusterrolebinding volume-snapshot-controller-role + do + sleep 10 + done + + echo "Wait until volume snapshot controller is installed" + until kubectl get statefulset volume-snapshot-controller | grep volume-snapshot-controller | grep "1/1" + do + sleep 10 + done +} + +# Trigger background process that will wait for volumesnapshot CRDs +# and snapshot-controller to be installed +function start-volumesnapshot-crd-and-controller { + wait-for-volumesnapshot-crd-and-controller & +} + # Update {{ fluentd_container_runtime_service }} with actual container runtime name, # and {{ container_runtime_endpoint }} with actual container runtime # endpoint. @@ -2444,6 +2483,11 @@ EOF if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then setup-addon-manifests "addons" "storage-class/gce" fi + if [[ "${ENABLE_VOLUME_SNAPSHOTS:-}" == "true" ]]; then + setup-addon-manifests "addons" "volumesnapshots/crd" + setup-addon-manifests "addons" "volumesnapshots/volume-snapshot-controller" + start-volumesnapshot-crd-and-controller + fi if [[ "${ENABLE_IP_MASQ_AGENT:-}" == "true" ]]; then setup-addon-manifests "addons" "ip-masq-agent" fi diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 6d6632e3218..f777ed5bd74 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1171,6 +1171,7 @@ MULTIZONE: $(yaml-quote ${MULTIZONE:-}) MULTIMASTER: $(yaml-quote ${MULTIMASTER:-}) NON_MASQUERADE_CIDR: $(yaml-quote ${NON_MASQUERADE_CIDR:-}) ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote ${ENABLE_DEFAULT_STORAGE_CLASS:-}) +ENABLE_VOLUME_SNAPSHOTS: $(yaml-quote ${ENABLE_VOLUME_SNAPSHOTS:-}) ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-}) ENABLE_APISERVER_DYNAMIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_DYNAMIC_AUDIT:-}) ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false}) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 09dfe1efc37..3158286c859 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -349,6 +349,7 @@ const ( // owner: @xing-yang // alpha: v1.12 + // beta: v1.17 // // Enable volume snapshot data source support. VolumeSnapshotDataSource featuregate.Feature = "VolumeSnapshotDataSource" @@ -558,7 +559,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS RuntimeClass: {Default: true, PreRelease: featuregate.Beta}, NodeLease: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, SCTPSupport: {Default: false, PreRelease: featuregate.Alpha}, - VolumeSnapshotDataSource: {Default: false, PreRelease: featuregate.Alpha}, + VolumeSnapshotDataSource: {Default: true, PreRelease: featuregate.Beta}, ProcMountType: {Default: false, PreRelease: featuregate.Alpha}, TTLAfterFinished: {Default: false, PreRelease: featuregate.Alpha}, KubeletPodResources: {Default: true, PreRelease: featuregate.Beta}, diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index 82dfb5b673b..4f0b809ea80 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -483,10 +483,9 @@ func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.U "namespace": ns, }, "spec": map[string]interface{}{ - "snapshotClassName": snapshotClassName, + "volumeSnapshotClassName": snapshotClassName, "source": map[string]interface{}{ - "name": claimName, - "kind": "PersistentVolumeClaim", + "persistentVolumeClaimName": claimName, }, }, }, diff --git a/test/e2e/storage/testsuites/driveroperations.go b/test/e2e/storage/testsuites/driveroperations.go index 4d4298e61a7..1c5bc5c38f7 100644 --- a/test/e2e/storage/testsuites/driveroperations.go +++ b/test/e2e/storage/testsuites/driveroperations.go @@ -97,8 +97,9 @@ func GetSnapshotClass( // Name must be unique, so let's base it on namespace name "name": ns + "-" + suffix, }, - "snapshotter": snapshotter, - "parameters": parameters, + "driver": snapshotter, + "parameters": parameters, + "deletionPolicy": "Delete", }, } diff --git a/test/e2e/storage/testsuites/snapshottable.go b/test/e2e/storage/testsuites/snapshottable.go index 8fce14c57d3..fdc131ee168 100644 --- a/test/e2e/storage/testsuites/snapshottable.go +++ b/test/e2e/storage/testsuites/snapshottable.go @@ -37,12 +37,12 @@ import ( const snapshotGroup = "snapshot.storage.k8s.io" // snapshot CRD api version -const snapshotAPIVersion = "snapshot.storage.k8s.io/v1alpha1" +const snapshotAPIVersion = "snapshot.storage.k8s.io/v1beta1" var ( - snapshotGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshots"} - snapshotClassGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotclasses"} - snapshotContentGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1alpha1", Resource: "volumesnapshotcontents"} + snapshotGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1beta1", Resource: "volumesnapshots"} + snapshotClassGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1beta1", Resource: "volumesnapshotclasses"} + snapshotContentGVR = schema.GroupVersionResource{Group: snapshotGroup, Version: "v1beta1", Resource: "volumesnapshotcontents"} ) type snapshottableTestSuite struct { @@ -152,7 +152,7 @@ func (s *snapshottableTestSuite) defineTests(driver TestDriver, pattern testpatt framework.ExpectNoError(err) // Get the bound PV - pv, err := cs.CoreV1().PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{}) + _, err = cs.CoreV1().PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{}) framework.ExpectNoError(err) ginkgo.By("creating a SnapshotClass") @@ -185,21 +185,19 @@ func (s *snapshottableTestSuite) defineTests(driver TestDriver, pattern testpatt framework.ExpectNoError(err) // Get the bound snapshotContent - snapshotSpec := snapshot.Object["spec"].(map[string]interface{}) - snapshotContentName := snapshotSpec["snapshotContentName"].(string) + snapshotStatus := snapshot.Object["status"].(map[string]interface{}) + snapshotContentName := snapshotStatus["boundVolumeSnapshotContentName"].(string) snapshotContent, err := dc.Resource(snapshotContentGVR).Get(snapshotContentName, metav1.GetOptions{}) framework.ExpectNoError(err) snapshotContentSpec := snapshotContent.Object["spec"].(map[string]interface{}) volumeSnapshotRef := snapshotContentSpec["volumeSnapshotRef"].(map[string]interface{}) - persistentVolumeRef := snapshotContentSpec["persistentVolumeRef"].(map[string]interface{}) // Check SnapshotContent properties ginkgo.By("checking the SnapshotContent") - framework.ExpectEqual(snapshotContentSpec["snapshotClassName"], vsc.GetName()) + framework.ExpectEqual(snapshotContentSpec["volumeSnapshotClassName"], vsc.GetName()) framework.ExpectEqual(volumeSnapshotRef["name"], snapshot.GetName()) framework.ExpectEqual(volumeSnapshotRef["namespace"], snapshot.GetNamespace()) - framework.ExpectEqual(persistentVolumeRef["name"], pv.Name) }) } diff --git a/test/e2e/storage/utils/deployment.go b/test/e2e/storage/utils/deployment.go index c8161f7f057..f421dc3107b 100644 --- a/test/e2e/storage/utils/deployment.go +++ b/test/e2e/storage/utils/deployment.go @@ -90,10 +90,6 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf // Driver name is expected to be the same // as the provisioner here. container.Args = append(container.Args, "--provisioner="+o.NewDriverName) - case o.SnapshotterContainerName: - // Driver name is expected to be the same - // as the snapshotter here. - container.Args = append(container.Args, "--snapshotter="+o.NewDriverName) } } } diff --git a/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml index 6a07ee32141..ca4a54b6e05 100644 --- a/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml @@ -20,15 +20,6 @@ metadata: # rename if there are conflicts name: external-snapshotter-runner rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] @@ -42,14 +33,8 @@ rules: resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] + resources: ["volumesnapshotcontents/status"] verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] --- kind: ClusterRoleBinding diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml index d7b73daac3f..f18f45a455c 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml @@ -40,7 +40,8 @@ spec: serviceAccountName: csi-provisioner containers: - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.4.0 + # TODO: replace with official 1.5.0 release when ready + image: quay.io/k8scsi/csi-provisioner:v1.5.0-rc1 args: - -v=5 - --csi-address=/csi/csi.sock diff --git a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml index f2dd767d016..e3ebf2a7823 100644 --- a/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml +++ b/test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml @@ -31,10 +31,10 @@ spec: containers: - name: csi-snapshotter # TODO: replace with official 2.0.0 release when ready - image: quay.io/k8scsi/csi-snapshotter:v2.0.0-rc1 + image: quay.io/k8scsi/csi-snapshotter:v2.0.0-rc2 args: + - "--v=5" - "--csi-address=$(ADDRESS)" - - "--connection-timeout=15s" env: - name: ADDRESS value: /csi/csi.sock