Merge pull request #137711 from duanwei33/add-volumegroupsnapshotclass-test

[e2e] Add volume_group_snapshot_class cases
This commit is contained in:
Kubernetes Prow Robot
2026-03-18 23:21:05 +05:30
committed by GitHub
3 changed files with 367 additions and 14 deletions

View File

@@ -34,8 +34,21 @@ import (
"k8s.io/kubernetes/test/e2e/storage/utils"
)
func getVolumeGroupSnapshot(labels map[string]interface{}, ns, snapshotClassName string) *unstructured.Unstructured {
snapshot := &unstructured.Unstructured{
// GetVolumeGroupSnapshot constructs a VolumeGroupSnapshot object with a label selector.
// vgsclassName is optional; if empty, the cluster's default VolumeGroupSnapshotClass
// will be used by the controller.
func GetVolumeGroupSnapshot(ns string, matchLabels map[string]interface{}, vgsclassName string) *unstructured.Unstructured {
spec := map[string]interface{}{
"source": map[string]interface{}{
"selector": map[string]interface{}{
"matchLabels": matchLabels,
},
},
}
if vgsclassName != "" {
spec["volumeGroupSnapshotClassName"] = vgsclassName
}
return &unstructured.Unstructured{
Object: map[string]interface{}{
"kind": "VolumeGroupSnapshot",
"apiVersion": utils.VolumeGroupSnapshotAPIVersion,
@@ -43,18 +56,9 @@ func getVolumeGroupSnapshot(labels map[string]interface{}, ns, snapshotClassName
"generateName": "group-snapshot-",
"namespace": ns,
},
"spec": map[string]interface{}{
"volumeGroupSnapshotClassName": snapshotClassName,
"source": map[string]interface{}{
"selector": map[string]interface{}{
"matchLabels": labels,
},
},
},
"spec": spec,
},
}
return snapshot
}
// VolumeGroupSnapshotResource represents a volumegroupsnapshot class, a volumegroupsnapshot and its bound contents for a specific test case
@@ -98,9 +102,9 @@ func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VolumeGroupSnapshott
ginkgo.By("creating a dynamic VolumeGroupSnapshot")
// Prepare a dynamically provisioned group volume snapshot with certain data
volumeGroupSnapshot := getVolumeGroupSnapshot(map[string]interface{}{
volumeGroupSnapshot := GetVolumeGroupSnapshot(pvcNamespace, map[string]interface{}{
"group": groupName,
}, pvcNamespace, gsclass.GetName())
}, gsclass.GetName())
volumeGroupSnapshot, err = dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(volumeGroupSnapshot.GetNamespace()).Create(ctx, volumeGroupSnapshot, metav1.CreateOptions{})
framework.ExpectNoError(err, "Failed to create volume group snapshot")

View File

@@ -69,6 +69,7 @@ var BaseSuites = []func() storageframework.TestSuite{
InitVolumeStressTestSuite,
InitFsGroupChangePolicyTestSuite,
InitVolumeGroupSnapshottableTestSuite,
InitVolumeGroupSnapshotClassTestSuite,
func() storageframework.TestSuite {
return InitCustomEphemeralTestSuite(GenericEphemeralTestPatterns())
},

View File

@@ -0,0 +1,348 @@
/*
Copyright The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package testsuites
import (
"context"
"fmt"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/feature"
"k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
"k8s.io/kubernetes/test/e2e/storage/utils"
admissionapi "k8s.io/pod-security-admission/api"
)
// Error substrings emitted by the group-snapshot sidecar controller when default
// VolumeGroupSnapshotClass selection fails. Sourced from:
// https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/common-controller/groupsnapshot_controller_helper.go
const (
errNoDefaultVGSClass = "cannot find default group snapshot class"
errMultipleDefaultVGSClasses = "default snapshot classes were found"
defaultVGSClassAnnotationKey = "groupsnapshot.storage.kubernetes.io/is-default-class"
)
// VolumeGroupSnapshotClassTestSuite tests VolumeGroupSnapshotClass default class
// selection behavior: whether the controller correctly picks a default class when
// no className is specified in the VolumeGroupSnapshot, and whether it rejects
// ambiguous configurations (no default or multiple defaults).
type VolumeGroupSnapshotClassTestSuite struct {
tsInfo storageframework.TestSuiteInfo
}
// InitVolumeGroupSnapshotClassTestSuite initializes the test suite for
// VolumeGroupSnapshotClass default class selection tests.
func InitVolumeGroupSnapshotClassTestSuite() storageframework.TestSuite {
return &VolumeGroupSnapshotClassTestSuite{
tsInfo: storageframework.TestSuiteInfo{
Name: "volumegroupsnapshotclass",
// A single pattern is sufficient: these tests exercise class selection
// logic and do not depend on snapshot deletion policy.
TestPatterns: []storageframework.TestPattern{
storageframework.VolumeGroupSnapshotDelete,
},
SupportedSizeRange: e2evolume.SizeRange{
Min: "1Mi",
},
TestTags: []interface{}{feature.VolumeGroupSnapshotDataSource},
},
}
}
// GetTestSuiteInfo returns the test suite information.
func (s *VolumeGroupSnapshotClassTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
return s.tsInfo
}
// SkipUnsupportedTests skips tests if the driver does not support group snapshots.
func (s *VolumeGroupSnapshotClassTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
dInfo := driver.GetDriverInfo()
_, ok := driver.(storageframework.VolumeGroupSnapshottableTestDriver)
if !dInfo.Capabilities[storageframework.CapVolumeGroupSnapshot] || !ok {
e2eskipper.Skipf("Driver %q does not support group snapshots - skipping", dInfo.Name)
}
}
// DefineTests defines the test cases for VolumeGroupSnapshotClass default class selection.
func (s *VolumeGroupSnapshotClassTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
labelKey := "pvc-group"
labelValue := "test-vgsclass"
f := framework.NewDefaultFramework("volumegroupsnapshotclass")
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
ginkgo.Describe("VolumeGroupSnapshotClass", func() {
var (
snapshottableDriver storageframework.VolumeGroupSnapshottableTestDriver
cs clientset.Interface
config *storageframework.PerTestConfig
)
ginkgo.BeforeEach(func(ctx context.Context) {
snapshottableDriver = driver.(storageframework.VolumeGroupSnapshottableTestDriver)
cs = f.ClientSet
config = driver.PrepareTest(ctx, f)
})
// createPVCsWithPod creates two labeled PVCs and a pod that mounts them,
// ensuring the PVCs enter the Bound state.
createPVCsWithPod := func(ctx context.Context) (*storageframework.VolumeResource, *storageframework.VolumeResource, *v1.Pod) {
ginkgo.By("creating two PVCs with labels for the VGS selector")
vr1 := storageframework.CreateVolumeResource(ctx, driver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange)
vr2 := storageframework.CreateVolumeResource(ctx, driver, config, pattern, s.GetTestSuiteInfo().SupportedSizeRange)
for _, vr := range []*storageframework.VolumeResource{vr1, vr2} {
patchData := fmt.Appendf(nil, `{"metadata":{"labels":{"%s":"%s"}}}`, labelKey, labelValue)
updatedPVC, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Patch(ctx, vr.Pvc.Name, types.MergePatchType, patchData, metav1.PatchOptions{})
framework.ExpectNoError(err, "failed to add label to PVC %s", vr.Pvc.Name)
vr.Pvc = updatedPVC
}
ginkgo.By("creating a pod to bind the PVCs")
podConfig := e2epod.Config{
NS: f.Namespace.Name,
PVCs: []*v1.PersistentVolumeClaim{vr1.Pvc, vr2.Pvc},
SeLinuxLabel: e2epv.SELinuxLabel,
}
pod, err := e2epod.MakeSecPod(&podConfig)
framework.ExpectNoError(err, "failed to make pod config")
pod, err = cs.CoreV1().Pods(f.Namespace.Name).Create(ctx, pod, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create pod")
framework.ExpectNoError(
e2epod.WaitTimeoutForPodRunningInNamespace(ctx, cs, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow),
"pod did not become running in time",
)
return vr1, vr2, pod
}
// createDefaultVGSClass creates a VolumeGroupSnapshotClass with the default
// annotation and registers a DeferCleanup to delete it.
createDefaultVGSClass := func(ctx context.Context) *unstructured.Unstructured {
vgsclassSpec := snapshottableDriver.GetVolumeGroupSnapshotClass(ctx, config, map[string]string{"deletionPolicy": "Delete"})
annotations := vgsclassSpec.GetAnnotations()
if annotations == nil {
annotations = make(map[string]string)
}
annotations[defaultVGSClassAnnotationKey] = "true"
vgsclassSpec.SetAnnotations(annotations)
vgsclass, err := f.DynamicClient.Resource(utils.VolumeGroupSnapshotClassGVR).Create(ctx, vgsclassSpec, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create VolumeGroupSnapshotClass")
ginkgo.DeferCleanup(func(ctx context.Context) {
err := f.DynamicClient.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, vgsclass.GetName(), metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
framework.ExpectNoError(err, "failed to delete VolumeGroupSnapshotClass %s", vgsclass.GetName())
}
})
return vgsclass
}
// createVGSWithoutClassName creates a VolumeGroupSnapshot without specifying
// volumeGroupSnapshotClassName and registers a DeferCleanup to delete it.
createVGSWithoutClassName := func(ctx context.Context) *unstructured.Unstructured {
vgs, err := f.DynamicClient.Resource(utils.VolumeGroupSnapshotGVR).Namespace(f.Namespace.Name).Create(ctx,
storageframework.GetVolumeGroupSnapshot(f.Namespace.Name, map[string]interface{}{labelKey: labelValue}, ""),
metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create VolumeGroupSnapshot")
ginkgo.DeferCleanup(func(ctx context.Context) {
err := f.DynamicClient.Resource(utils.VolumeGroupSnapshotGVR).Namespace(f.Namespace.Name).Delete(ctx, vgs.GetName(), metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
framework.ExpectNoError(err, "failed to delete VolumeGroupSnapshot %s", vgs.GetName())
}
})
return vgs
}
// removeAndRestoreDefaultVGSClasses removes the default annotation from any
// existing VolumeGroupSnapshotClasses for the same CSI driver, and restores
// them via DeferCleanup after the test. This prevents interference with
// default class selection tests.
removeAndRestoreDefaultVGSClasses := func(ctx context.Context) {
driverName := driver.GetDriverInfo().Name
list, err := f.DynamicClient.Resource(utils.VolumeGroupSnapshotClassGVR).List(ctx, metav1.ListOptions{})
framework.ExpectNoError(err, "failed to list VolumeGroupSnapshotClasses")
for _, item := range list.Items {
classDriver, _, _ := unstructured.NestedString(item.Object, "driver")
if classDriver != driverName {
continue
}
if item.GetAnnotations()[defaultVGSClassAnnotationKey] != "true" {
continue
}
name := item.GetName()
framework.Logf("removing default annotation from VolumeGroupSnapshotClass %s for test isolation", name)
removePatch := []byte(`{"metadata":{"annotations":{"groupsnapshot.storage.kubernetes.io/is-default-class":null}}}`)
_, err := f.DynamicClient.Resource(utils.VolumeGroupSnapshotClassGVR).Patch(ctx, name, types.MergePatchType, removePatch, metav1.PatchOptions{})
framework.ExpectNoError(err, "failed to remove default annotation from VolumeGroupSnapshotClass %s", name)
ginkgo.DeferCleanup(func(ctx context.Context) {
framework.Logf("restoring default annotation on VolumeGroupSnapshotClass %s", name)
restorePatch := []byte(`{"metadata":{"annotations":{"groupsnapshot.storage.kubernetes.io/is-default-class":"true"}}}`)
_, err := f.DynamicClient.Resource(utils.VolumeGroupSnapshotClassGVR).Patch(ctx, name, types.MergePatchType, restorePatch, metav1.PatchOptions{})
framework.ExpectNoError(err, "failed to restore default annotation on VolumeGroupSnapshotClass %s", name)
})
}
}
// expectVolumeGroupSnapshotClassName fetches the named VolumeGroupSnapshot and
// asserts spec.volumeGroupSnapshotClassName against expectedClassName.
// Pass nil to assert the field is absent.
// Pass a pointer to a string to assert the field equals that exact value,
// including "" for the edge case where a VolumeGroupSnapshotClass carries a
// "groupsnapshot.storage.kubernetes.io/is-default-class"="" annotation.
expectVolumeGroupSnapshotClassName := func(ctx context.Context, vgsName string, expectedClassName *string) {
ginkgo.GinkgoHelper()
vgs, err := f.DynamicClient.Resource(utils.VolumeGroupSnapshotGVR).Namespace(f.Namespace.Name).Get(ctx, vgsName, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to get VolumeGroupSnapshot %s", vgsName)
spec, ok := vgs.Object["spec"].(map[string]interface{})
if !ok {
ginkgo.Fail("failed to get VolumeGroupSnapshot spec: spec field is not a map[string]interface{}")
}
if expectedClassName == nil {
gomega.Expect(spec["volumeGroupSnapshotClassName"]).To(gomega.BeNil(),
"VolumeGroupSnapshot %s spec.volumeGroupSnapshotClassName should not be set by the controller", vgsName)
} else {
gomega.Expect(spec["volumeGroupSnapshotClassName"]).To(gomega.Equal(*expectedClassName),
"VolumeGroupSnapshot %s spec.volumeGroupSnapshotClassName should be %q", vgsName, *expectedClassName)
}
}
f.It("should use default VolumeGroupSnapshotClass when no className is specified", f.WithSerial(), func(ctx context.Context) {
removeAndRestoreDefaultVGSClasses(ctx)
vr1, vr2, pod := createPVCsWithPod(ctx)
ginkgo.DeferCleanup(func(ctx context.Context) {
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, cs, pod))
framework.ExpectNoError(vr1.CleanupResource(ctx))
framework.ExpectNoError(vr2.CleanupResource(ctx))
})
ginkgo.By("creating a VolumeGroupSnapshotClass with the default annotation")
vgsclass := createDefaultVGSClass(ctx)
ginkgo.By("creating a VolumeGroupSnapshot without specifying volumeGroupSnapshotClassName")
vgs := createVGSWithoutClassName(ctx)
ginkgo.DeferCleanup(func(ctx context.Context) {
sr := &storageframework.VolumeGroupSnapshotResource{
Config: config,
Pattern: storageframework.VolumeGroupSnapshotDelete,
VGS: vgs,
}
framework.ExpectNoError(sr.CleanupResource(ctx, f.Timeouts), "failed to clean up VolumeGroupSnapshot resources")
})
ginkgo.By("verifying the VolumeGroupSnapshot becomes ready using the default class")
framework.ExpectNoError(utils.WaitForVolumeGroupSnapshotReady(ctx, f.DynamicClient, f.Namespace.Name, vgs.GetName(), framework.Poll, f.Timeouts.SnapshotCreate))
ginkgo.By("verifying the default VolumeGroupSnapshotClass name was written back to the VolumeGroupSnapshot spec")
expectedClassName := vgsclass.GetName()
expectVolumeGroupSnapshotClassName(ctx, vgs.GetName(), &expectedClassName)
})
f.It("should report error when VolumeGroupSnapshot is created without className and no default class exists", f.WithSerial(), func(ctx context.Context) {
removeAndRestoreDefaultVGSClasses(ctx)
vr1, vr2, pod := createPVCsWithPod(ctx)
ginkgo.DeferCleanup(func(ctx context.Context) {
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, cs, pod))
framework.ExpectNoError(vr1.CleanupResource(ctx))
framework.ExpectNoError(vr2.CleanupResource(ctx))
})
ginkgo.By("creating a VolumeGroupSnapshot without className and without any default VolumeGroupSnapshotClass")
vgs := createVGSWithoutClassName(ctx)
ginkgo.By("verifying the VolumeGroupSnapshot enters an error state")
framework.ExpectNoError(waitForVolumeGroupSnapshotError(ctx, f.DynamicClient, f.Namespace.Name, vgs.GetName(), framework.Poll, f.Timeouts.SnapshotCreate, errNoDefaultVGSClass))
ginkgo.By("verifying spec.volumeGroupSnapshotClassName was not set by the controller")
expectVolumeGroupSnapshotClassName(ctx, vgs.GetName(), nil)
})
f.It("should report error when VolumeGroupSnapshot is created without className and multiple default classes exist", f.WithSerial(), func(ctx context.Context) {
removeAndRestoreDefaultVGSClasses(ctx)
vr1, vr2, pod := createPVCsWithPod(ctx)
ginkgo.DeferCleanup(func(ctx context.Context) {
framework.ExpectNoError(e2epod.DeletePodWithWait(ctx, cs, pod))
framework.ExpectNoError(vr1.CleanupResource(ctx))
framework.ExpectNoError(vr2.CleanupResource(ctx))
})
ginkgo.By("creating two VolumeGroupSnapshotClasses both with the default annotation")
createDefaultVGSClass(ctx)
createDefaultVGSClass(ctx)
ginkgo.By("creating a VolumeGroupSnapshot without specifying className")
vgs := createVGSWithoutClassName(ctx)
ginkgo.By("verifying the VolumeGroupSnapshot enters an error state due to multiple default classes")
framework.ExpectNoError(waitForVolumeGroupSnapshotError(ctx, f.DynamicClient, f.Namespace.Name, vgs.GetName(), framework.Poll, f.Timeouts.SnapshotCreate, errMultipleDefaultVGSClasses))
ginkgo.By("verifying spec.volumeGroupSnapshotClassName was not set by the controller")
expectVolumeGroupSnapshotClassName(ctx, vgs.GetName(), nil)
})
})
}
// waitForVolumeGroupSnapshotError polls until the given VolumeGroupSnapshot has a
// non-nil error field in its status, then immediately asserts that the message
// contains expectedErrSubstr. Separating the two phases ensures that a controller
// message change surfaces as an immediate assertion failure rather than a silent
// timeout. Use the package-level error constants as expectedErrSubstr.
func waitForVolumeGroupSnapshotError(ctx context.Context, dc dynamic.Interface, ns, name string, poll, timeout time.Duration, expectedErrSubstr string) error {
framework.Logf("Waiting up to %v for VolumeGroupSnapshot %s/%s to enter an error state", timeout, ns, name)
var errorMessage string
if successful := utils.WaitUntil(poll, timeout, func() bool {
vgs, err := dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
if err != nil {
framework.Logf("Failed to get VolumeGroupSnapshot %q, retrying: %v", name, err)
return false
}
status, ok := vgs.Object["status"].(map[string]interface{})
if !ok || status == nil {
return false
}
errObj, ok := status["error"].(map[string]interface{})
if !ok || errObj == nil {
return false
}
errorMessage, _ = errObj["message"].(string)
return true
}); !successful {
return fmt.Errorf("VolumeGroupSnapshot %s/%s did not enter an error state within %v", ns, name, timeout)
}
gomega.Expect(errorMessage).To(gomega.ContainSubstring(expectedErrSubstr),
"VolumeGroupSnapshot %s/%s has error status but message %q does not contain expected substring %q",
ns, name, errorMessage, expectedErrSubstr)
return nil
}