github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-01 07:47:56 +00:00
Code Issues Projects Releases Wiki Activity

Add crd from external snapshotter

Browse Source 
Signed-off-by: Manish Yathnalli <myathnal@redhat.com>
This commit is contained in:
Manish Yathnalli 2024-11-26 11:41:37 +05:30
parent 8770bd58d0
commit 446252b0c3
9 changed files with 94 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
test/e2e/storage/framework/volume_group_snapshot_resource.go
View File

@ -21,7 +21,7 @@ import (
"fmt"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/kubernetes/test/e2e/framework"
@ -56,15 +56,15 @@ type VolumeGroupSnapshotResource struct {
Config *PerTestConfig
Pattern TestPattern
Vgs *unstructured.Unstructured
Vgscontent *unstructured.Unstructured
Vgsclass *unstructured.Unstructured
VGS *unstructured.Unstructured
VGSContent *unstructured.Unstructured
VGSClass *unstructured.Unstructured
}
// CreateVolumeGroupSnapshot creates a VolumeGroupSnapshotClass with given SnapshotDeletionPolicy and a VolumeGroupSnapshot
// from the VolumeGroupSnapshotClass using a dynamic client.
// Returns the unstructured VolumeGroupSnapshotClass and VolumeGroupSnapshot objects.
func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, groupName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured) {
func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, groupName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured, *unstructured.Unstructured) {
defer ginkgo.GinkgoRecover()
var err error
if pattern.SnapshotType != VolumeGroupSnapshot {
@ -99,28 +99,35 @@ func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshott
ginkgo.By("Getting group snapshot and content")
volumeGroupSnapshot, err = dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(volumeGroupSnapshot.GetNamespace()).Get(ctx, volumeGroupSnapshot.GetName(), metav1.GetOptions{})
framework.ExpectNoError(err, "Failed to get volume group snapshot after creation")
return gsclass, volumeGroupSnapshot
status := volumeGroupSnapshot.Object["status"]
err = framework.Gomega().Expect(status).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "Failed to get status of volume group snapshot")
vgscName := status.(map[string]interface{})["boundVolumeGroupSnapshotContentName"].(string)
err = framework.Gomega().Expect(vgscName).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "Failed to get content name of volume group snapshot")
vgsc, err := dc.Resource(utils.VolumeGroupSnapshotContentGVR).Get(ctx, vgscName, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to get content of group snapshot")
return gsclass, volumeGroupSnapshot, vgsc
}
// CleanupResource deletes the VolumeGroupSnapshotClass and VolumeGroupSnapshot objects using a dynamic client.
func (r *VolumeGroupSnapshotResource) CleanupResource(ctx context.Context, timeouts *framework.TimeoutContext) error {
defer ginkgo.GinkgoRecover()
dc := r.Config.Framework.DynamicClient
err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, r.Vgsclass.GetName(), metav1.DeleteOptions{})
err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, r.VGSClass.GetName(), metav1.DeleteOptions{})
framework.ExpectNoError(err, "Failed to delete volume group snapshot class")
return nil
}
// CreateVolumeGroupSnapshotResource creates a VolumeGroupSnapshotResource object with the given parameters.
func CreateVolumeGroupSnapshotResource(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *VolumeGroupSnapshotResource {
vgsclass, snapshot := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters)
vgsClass, snapshot, vgsc := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters)
vgs := &VolumeGroupSnapshotResource{
Config: config,
Pattern: pattern,
Vgs: snapshot,
Vgsclass: vgsclass,
Vgscontent: nil,
VGS: snapshot,
VGSClass: vgsClass,
VGSContent: vgsc,
}
return vgs
}

31
test/e2e/storage/testsuites/volume_group_snapshottable.go
View File

@ -18,6 +18,8 @@ package testsuites
import (
"context"
"crypto/sha256"
"fmt"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
@ -176,18 +178,30 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
snapshot := storageframework.CreateVolumeGroupSnapshotResource(ctx, snapshottableDriver, groupTest.config, pattern, labelValue, groupTest.volumeGroup[0][0].Pvc.GetNamespace(), f.Timeouts, map[string]string{"deletionPolicy": pattern.SnapshotDeletionPolicy.String()})
groupTest.snapshots = append(groupTest.snapshots, snapshot)
ginkgo.By("verifying the snapshots in the group are ready to use")
status := snapshot.Vgs.Object["status"]
status := snapshot.VGS.Object["status"]
err := framework.Gomega().Expect(status).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get status of group snapshot")
volumes := status.(map[string]interface{})["pvcVolumeSnapshotRefList"]
err = framework.Gomega().Expect(volumes).NotTo(gomega.BeNil())
volumeListMap := snapshot.VGSContent.Object["status"].(map[string]interface{})
err = framework.Gomega().Expect(volumeListMap).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get volume snapshot list")
volumeList := volumes.([]interface{})
err = framework.Gomega().Expect(len(volumeList)).To(gomega.Equal(groupTest.numVolumes))
volumeSnapshotHandlePairList := volumeListMap["volumeSnapshotHandlePairList"].([]interface{})
err = framework.Gomega().Expect(volumeSnapshotHandlePairList).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get volume snapshot list")
err = framework.Gomega().Expect(len(volumeSnapshotHandlePairList)).To(gomega.Equal(groupTest.numVolumes))
framework.ExpectNoError(err, "failed to get volume snapshot list")
claimSize := groupTest.volumeGroup[0][0].Pvc.Spec.Resources.Requests.Storage().String()
for _, volume := range volumeList {
for _, volume := range volumeSnapshotHandlePairList {
// Create a PVC from the snapshot
volumeHandle := volume.(map[string]interface{})["volumeHandle"].(string)
err = framework.Gomega().Expect(volumeHandle).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get volume handle from volume")
uid := snapshot.VGSContent.Object["metadata"].(map[string]interface{})["uid"].(string)
err = framework.Gomega().Expect(uid).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get uuid from content")
volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
uid+volumeHandle)))
pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
StorageClassName: &groupTest.volumeGroup[0][0].Sc.Name,
ClaimSize: claimSize,
@ -198,7 +212,7 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
pvc.Spec.DataSource = &v1.TypedLocalObjectReference{
APIGroup: &group,
Kind: "VolumeSnapshot",
Name: volume.(map[string]interface{})["volumeSnapshotRef"].(map[string]interface{})["name"].(string),
Name: volumeSnapshotName,
}
volSrc := v1.VolumeSource{
@ -208,8 +222,9 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
},
},
}
pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{})
pvc, err = cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create PVC from snapshot")
pod := StartInPodWithVolumeSource(ctx, cs, volSrc, pvc.Namespace, "snapshot-pod", "sleep 300", groupTest.config.ClientNodeSelection)
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, cs, pod)
framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(ctx, cs, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow), "Pod did not start in expected time")

7
test/e2e/storage/utils/volume_group_snapshot.go
View File

@ -33,15 +33,16 @@ const (
// VolumeGroupSnapshot is the group snapshot api
VolumeGroupSnapshotAPIGroup = "groupsnapshot.storage.k8s.io"
// VolumeGroupSnapshotAPIVersion is the group snapshot api version
VolumeGroupSnapshotAPIVersion = "groupsnapshot.storage.k8s.io/v1alpha1"
VolumeGroupSnapshotAPIVersion = "groupsnapshot.storage.k8s.io/v1beta1"
)
var (
// VolumeGroupSnapshotGVR is GroupVersionResource for volumegroupsnapshots
VolumeGroupSnapshotGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1alpha1", Resource: "volumegroupsnapshots"}
VolumeGroupSnapshotGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1beta1", Resource: "volumegroupsnapshots"}
// VolumeGroupSnapshotClassGVR is GroupVersionResource for volumegroupsnapshotsclasses
VolumeGroupSnapshotClassGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1alpha1", Resource: "volumegroupsnapshotclasses"}
VolumeGroupSnapshotClassGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1beta1", Resource: "volumegroupsnapshotclasses"}
VolumeGroupSnapshotContentGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1beta1", Resource: "volumegroupsnapshotcontents"}
)
// WaitForVolumeGroupSnapshotReady waits for a VolumeGroupSnapshot to be ready to use or until timeout occurs, whichever comes first.

4
test/e2e/testing-manifests/storage-csi/external-snapshotter/groupsnapshot.storage.k8s.io_volumegroupsnapshotclasses.yaml
View File

@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814"
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150"
controller-gen.kubebuilder.io/version: v0.15.0
name: volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io
spec:
@ -31,7 +31,7 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
name: v1beta1
schema:
openAPIV3Schema:
description: |-

73
test/e2e/testing-manifests/storage-csi/external-snapshotter/groupsnapshot.storage.k8s.io_volumegroupsnapshotcontents.yaml
View File

@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1068"
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150"
controller-gen.kubebuilder.io/version: v0.15.0
name: volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io
spec:
@ -53,7 +53,7 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
name: v1beta1
schema:
openAPIV3Schema:
description: |-
@ -237,8 +237,6 @@ spec:
- message: both volumeGroupSnapshotRef.name and volumeGroupSnapshotRef.namespace
must be set
rule: has(self.name) && has(self.__namespace__)
- message: volumeGroupSnapshotRef is immutable
rule: self == oldSelf
required:
- deletionPolicy
- driver
@ -257,8 +255,9 @@ spec:
The format of this field is a Unix nanoseconds time encoded as an int64.
On Unix, the command date +%s%N returns the current time in nanoseconds
since 1970-01-01 00:00:00 UTC.
format: int64
type: integer
This field is the source for the CreationTime field in VolumeGroupSnapshotStatus
format: date-time
type: string
error:
description: |-
Error is the last observed error during group snapshot creation, if any.
@ -276,42 +275,6 @@ spec:
format: date-time
type: string
type: object
pvVolumeSnapshotContentList:
description: |-
PVVolumeSnapshotContentList is the list of pairs of PV and
VolumeSnapshotContent for this group snapshot
The maximum number of allowed snapshots in the group is 100.
items:
description: |-
PVVolumeSnapshotContentPair represent a pair of PV names and
VolumeSnapshotContent names
properties:
persistentVolumeRef:
description: PersistentVolumeRef is a reference to the persistent
volume resource
properties:
name:
description: |-
Name of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?
type: string
type: object
x-kubernetes-map-type: atomic
volumeSnapshotContentRef:
description: VolumeSnapshotContentRef is a reference to the
volume snapshot content resource
properties:
name:
description: |-
Name of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?
type: string
type: object
x-kubernetes-map-type: atomic
type: object
type: array
readyToUse:
description: |-
ReadyToUse indicates if all the individual snapshots in the group are ready to be
@ -325,6 +288,32 @@ spec:
If a storage system does not provide such an id, the
CSI driver can choose to return the VolumeGroupSnapshot name.
type: string
volumeSnapshotHandlePairList:
description: |-
VolumeSnapshotHandlePairList is a list of CSI "volume_id" and "snapshot_id"
pair returned by the CSI driver to identify snapshots and their source volumes
on the storage system.
items:
description: VolumeSnapshotHandlePair defines a pair of a source
volume handle and a snapshot handle
properties:
snapshotHandle:
description: |-
SnapshotHandle is a unique id returned by the CSI driver to identify a volume
snapshot on the storage system
Required.
type: string
volumeHandle:
description: |-
VolumeHandle is a unique id returned by the CSI driver to identify a volume
on the storage system
Required.
type: string
required:
- snapshotHandle
- volumeHandle
type: object
type: array
type: object
required:
- spec

40
test/e2e/testing-manifests/storage-csi/external-snapshotter/groupsnapshot.storage.k8s.io_volumegroupsnapshots.yaml
View File

@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1068"
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150"
controller-gen.kubebuilder.io/version: v0.15.0
name: volumegroupsnapshots.groupsnapshot.storage.k8s.io
spec:
@ -43,7 +43,7 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
name: v1beta1
schema:
openAPIV3Schema:
description: |-
@ -198,6 +198,7 @@ spec:
The format of this field is a Unix nanoseconds time encoded as an int64.
On Unix, the command date +%s%N returns the current time in nanoseconds
since 1970-01-01 00:00:00 UTC.
This field is updated based on the CreationTime field in VolumeGroupSnapshotContentStatus
format: date-time
type: string
error:
@ -221,41 +222,6 @@ spec:
format: date-time
type: string
type: object
pvcVolumeSnapshotRefList:
description: |-
VolumeSnapshotRefList is the list of PVC and VolumeSnapshot pairs that
is part of this group snapshot.
The maximum number of allowed snapshots in the group is 100.
items:
description: PVCVolumeSnapshotPair defines a pair of a PVC reference
and a Volume Snapshot Reference
properties:
persistentVolumeClaimRef:
description: PersistentVolumeClaimRef is a reference to the
PVC this pair is referring to
properties:
name:
description: |-
Name of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?
type: string
type: object
x-kubernetes-map-type: atomic
volumeSnapshotRef:
description: VolumeSnapshotRef is a reference to the VolumeSnapshot
this pair is referring to
properties:
name:
description: |-
Name of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?
type: string
type: object
x-kubernetes-map-type: atomic
type: object
type: array
readyToUse:
description: |-
ReadyToUse indicates if all the individual snapshots in the group are ready

6
test/e2e/testing-manifests/storage-csi/external-snapshotter/volume-group-snapshots/csi-hostpath-plugin.yaml
View File

@ -219,7 +219,7 @@ spec:
serviceAccountName: csi-hostpathplugin-sa
containers:
- name: hostpath
image: registry.k8s.io/sig-storage/hostpathplugin:v1.14.0
image: registry.k8s.io/sig-storage/hostpathplugin:v1.15.0
args:
- "--drivername=hostpath.csi.k8s.io"
- "--v=5"
@ -354,11 +354,11 @@ spec:
name: socket-dir
- name: csi-snapshotter
image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0
args:
- -v=5
- --csi-address=/csi/csi.sock
- --enable-volume-group-snapshots=true
- --feature-gates=CSIVolumeGroupSnapshot=true
securityContext:
# This is necessary only for systems with SELinux, where
# non-privileged sidecar containers cannot access unix domain socket

11
test/e2e/testing-manifests/storage-csi/external-snapshotter/volume-group-snapshots/run_group_snapshot_e2e.sh
View File

@ -278,10 +278,15 @@ run_tests() {
kubectl apply -f test/e2e/testing-manifests/storage-csi/external-snapshotter/groupsnapshot.storage.k8s.io_volumegroupsnapshotcontents.yaml || exit 1
kubectl apply -f test/e2e/testing-manifests/storage-csi/external-snapshotter/groupsnapshot.storage.k8s.io_volumegroupsnapshots.yaml || exit 1
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/refs/tags/v8.2.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml || exit 1
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/refs/tags/v8.2.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml || exit 1
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/refs/tags/v8.2.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml || exit 1
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.0.0/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml || exit 1
curl -s https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/release-8.1/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | \
awk '/--leader-election=true/ {print; print " - \"--enable-volume-group-snapshots=true\""; next}1' | \
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/refs/tags/v8.2.0/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml || exit 1
curl -s https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/refs/tags/v8.2.0/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | \
awk '/--leader-election=true/ {print; print " - \"--feature-gates=CSIVolumeGroupSnapshot=true\""; next}1' | \
sed 's|image: registry.k8s.io/sig-storage/snapshot-controller:v8.0.1|image: registry.k8s.io/sig-storage/snapshot-controller:v8.2.0|' | \
kubectl apply -f - || exit 1

2
test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml
View File

@ -354,7 +354,7 @@ spec:
name: socket-dir
- name: csi-snapshotter
image: registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0
image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0
args:
- -v=5
- --csi-address=/csi/csi.sock