diff --git a/test/e2e/storage/testsuites/provisioning.go b/test/e2e/storage/testsuites/provisioning.go index 6a085b85a23..b083b029414 100644 --- a/test/e2e/storage/testsuites/provisioning.go +++ b/test/e2e/storage/testsuites/provisioning.go @@ -19,18 +19,22 @@ package testsuites import ( "context" "fmt" + "strings" "sync" "time" "github.com/onsi/ginkgo" "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" @@ -239,6 +243,181 @@ func (p *provisioningTestSuite) DefineTests(driver storageframework.TestDriver, l.testCase.TestDynamicProvisioning() }) + ginkgo.It("should provision storage with any volume data source [Serial]", func() { + if len(dInfo.InTreePluginName) != 0 { + e2eskipper.Skipf("AnyVolumeDataSource feature only works with CSI drivers - skipping") + } + if pattern.VolMode == v1.PersistentVolumeBlock { + e2eskipper.Skipf("Test for Block volumes is not implemented - skipping") + } + + init() + defer cleanup() + + ginkgo.By("Creating validator namespace") + valNamespace, err := f.CreateNamespace(fmt.Sprintf("%s-val", f.Namespace.Name), map[string]string{ + "e2e-framework": f.BaseName, + "e2e-test-namespace": f.Namespace.Name, + }) + framework.ExpectNoError(err) + + defer func() { + f.DeleteNamespace(valNamespace.Name) + }() + + ginkgo.By("Deploying validator") + valManifests := []string{ + "test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/populator.storage.k8s.io_volumepopulators.yaml", + "test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/rbac-data-source-validator.yaml", + "test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/setup-data-source-validator.yaml", + } + valCleanup, err := storageutils.CreateFromManifests(f, valNamespace, + func(item interface{}) error { return nil }, + valManifests...) + + framework.ExpectNoError(err) + defer valCleanup() + + ginkgo.By("Creating populator namespace") + popNamespace, err := f.CreateNamespace(fmt.Sprintf("%s-pop", f.Namespace.Name), map[string]string{ + "e2e-framework": f.BaseName, + "e2e-test-namespace": f.Namespace.Name, + }) + framework.ExpectNoError(err) + + defer func() { + f.DeleteNamespace(popNamespace.Name) + }() + + ginkgo.By("Deploying hello-populator") + popManifests := []string{ + "test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/hello-populator-crd.yaml", + "test/e2e/testing-manifests/storage-csi/any-volume-datasource/hello-populator-deploy.yaml", + } + popCleanup, err := storageutils.CreateFromManifests(f, popNamespace, + func(item interface{}) error { + switch item := item.(type) { + case *appsv1.Deployment: + for i, container := range item.Spec.Template.Spec.Containers { + switch container.Name { + case "hello": + var found bool + args := []string{} + for _, arg := range container.Args { + if strings.HasPrefix(arg, "--namespace=") { + args = append(args, fmt.Sprintf("--namespace=%s", popNamespace.Name)) + found = true + } else { + args = append(args, arg) + } + } + if !found { + args = append(args, fmt.Sprintf("--namespace=%s", popNamespace.Name)) + framework.Logf("container name: %s", container.Name) + } + container.Args = args + item.Spec.Template.Spec.Containers[i] = container + default: + } + } + } + return nil + }, + popManifests...) + + framework.ExpectNoError(err) + defer popCleanup() + + dc := l.config.Framework.DynamicClient + + // Make hello-populator handle Hello resource in hello.example.com group + ginkgo.By("Creating VolumePopulator CR datasource") + volumePopulatorGVR := schema.GroupVersionResource{Group: "populator.storage.k8s.io", Version: "v1beta1", Resource: "volumepopulators"} + helloPopulatorCR := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "VolumePopulator", + "apiVersion": "populator.storage.k8s.io/v1beta1", + "metadata": map[string]interface{}{ + "name": fmt.Sprintf("%s-%s", "hello-populator", f.Namespace.Name), + }, + "sourceKind": map[string]interface{}{ + "group": "hello.example.com", + "kind": "Hello", + }, + }, + } + + _, err = dc.Resource(volumePopulatorGVR).Create(context.TODO(), helloPopulatorCR, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + defer func() { + framework.Logf("deleting VolumePopulator CR datasource %q/%q", helloPopulatorCR.GetNamespace(), helloPopulatorCR.GetName()) + err = dc.Resource(volumePopulatorGVR).Delete(context.TODO(), helloPopulatorCR.GetName(), metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Failf("Error deleting VolumePopulator CR datasource %q. Error: %v", helloPopulatorCR.GetName(), err) + } + }() + + // Create Hello CR datasource + ginkgo.By("Creating Hello CR datasource") + helloCRName := "example-hello" + fileName := fmt.Sprintf("example-%s.txt", f.Namespace.Name) + expectedContent := fmt.Sprintf("Hello from namespace %s", f.Namespace.Name) + helloGVR := schema.GroupVersionResource{Group: "hello.example.com", Version: "v1alpha1", Resource: "hellos"} + helloCR := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "kind": "Hello", + "apiVersion": "hello.example.com/v1alpha1", + "metadata": map[string]interface{}{ + "name": helloCRName, + "namespace": f.Namespace.Name, + }, + "spec": map[string]interface{}{ + "fileName": fileName, + "fileContents": expectedContent, + }, + }, + } + + _, err = dc.Resource(helloGVR).Namespace(f.Namespace.Name).Create(context.TODO(), helloCR, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + defer func() { + framework.Logf("deleting Hello CR datasource %q/%q", helloCR.GetNamespace(), helloCR.GetName()) + err = dc.Resource(helloGVR).Namespace(helloCR.GetNamespace()).Delete(context.TODO(), helloCR.GetName(), metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + framework.Failf("Error deleting Hello CR datasource %q. Error: %v", helloCR.GetName(), err) + } + }() + + apiGroup := "hello.example.com" + l.pvc.Spec.DataSourceRef = &v1.TypedLocalObjectReference{ + APIGroup: &apiGroup, + Kind: "Hello", + Name: helloCRName, + } + + testConfig := storageframework.ConvertTestConfig(l.config) + l.testCase.NodeSelection = testConfig.ClientNodeSelection + l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim) { + ginkgo.By("checking whether the created volume has the pre-populated data") + tests := []e2evolume.Test{ + { + Volume: *storageutils.CreateVolumeSource(claim.Name, false /* readOnly */), + Mode: pattern.VolMode, + File: fileName, + ExpectedContent: expectedContent, + }, + } + e2evolume.TestVolumeClientSlow(f, testConfig, nil, "", tests) + } + + _, clearProvisionedStorageClass := SetupStorageClass(l.testCase.Client, l.testCase.Class) + defer clearProvisionedStorageClass() + + l.testCase.TestDynamicProvisioning() + }) + ginkgo.It("should provision storage with pvc data source", func() { if !dInfo.Capabilities[storageframework.CapPVCDataSource] { e2eskipper.Skipf("Driver %q does not support cloning - skipping", dInfo.Name) diff --git a/test/e2e/storage/utils/create.go b/test/e2e/storage/utils/create.go index c9a97aff842..ba765317db1 100644 --- a/test/e2e/storage/utils/create.go +++ b/test/e2e/storage/utils/create.go @@ -27,8 +27,10 @@ import ( v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" @@ -269,17 +271,19 @@ func describeItem(item interface{}) string { var errorItemNotSupported = errors.New("not supported") var factories = map[What]ItemFactory{ - {"ClusterRole"}: &clusterRoleFactory{}, - {"ClusterRoleBinding"}: &clusterRoleBindingFactory{}, - {"CSIDriver"}: &csiDriverFactory{}, - {"DaemonSet"}: &daemonSetFactory{}, - {"Role"}: &roleFactory{}, - {"RoleBinding"}: &roleBindingFactory{}, - {"Secret"}: &secretFactory{}, - {"Service"}: &serviceFactory{}, - {"ServiceAccount"}: &serviceAccountFactory{}, - {"StatefulSet"}: &statefulSetFactory{}, - {"StorageClass"}: &storageClassFactory{}, + {"ClusterRole"}: &clusterRoleFactory{}, + {"ClusterRoleBinding"}: &clusterRoleBindingFactory{}, + {"CSIDriver"}: &csiDriverFactory{}, + {"DaemonSet"}: &daemonSetFactory{}, + {"Role"}: &roleFactory{}, + {"RoleBinding"}: &roleBindingFactory{}, + {"Secret"}: &secretFactory{}, + {"Service"}: &serviceFactory{}, + {"ServiceAccount"}: &serviceAccountFactory{}, + {"StatefulSet"}: &statefulSetFactory{}, + {"Deployment"}: &deploymentFactory{}, + {"StorageClass"}: &storageClassFactory{}, + {"CustomResourceDefinition"}: &customResourceDefinitionFactory{}, } // PatchName makes the name of some item unique by appending the @@ -362,6 +366,14 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace, if err := patchContainerImages(item.Spec.Template.Spec.InitContainers); err != nil { return err } + case *appsv1.Deployment: + PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace) + if err := patchContainerImages(item.Spec.Template.Spec.Containers); err != nil { + return err + } + if err := patchContainerImages(item.Spec.Template.Spec.InitContainers); err != nil { + return err + } case *appsv1.DaemonSet: PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace) if err := patchContainerImages(item.Spec.Template.Spec.Containers); err != nil { @@ -370,6 +382,8 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace, if err := patchContainerImages(item.Spec.Template.Spec.InitContainers); err != nil { return err } + case *apiextensionsv1.CustomResourceDefinition: + // Do nothing. Patching name to all CRDs won't always be the expected behavior. default: return fmt.Errorf("missing support for patching item of type %T", item) } @@ -528,6 +542,27 @@ func (*statefulSetFactory) Create(f *framework.Framework, ns *v1.Namespace, i in }, nil } +type deploymentFactory struct{} + +func (f *deploymentFactory) New() runtime.Object { + return &appsv1.Deployment{} +} + +func (*deploymentFactory) Create(f *framework.Framework, ns *v1.Namespace, i interface{}) (func() error, error) { + item, ok := i.(*appsv1.Deployment) + if !ok { + return nil, errorItemNotSupported + } + + client := f.ClientSet.AppsV1().Deployments(ns.Name) + if _, err := client.Create(context.TODO(), item, metav1.CreateOptions{}); err != nil { + return nil, fmt.Errorf("create Deployment: %w", err) + } + return func() error { + return client.Delete(context.TODO(), item.GetName(), metav1.DeleteOptions{}) + }, nil +} + type daemonSetFactory struct{} func (f *daemonSetFactory) New() runtime.Object { @@ -612,6 +647,35 @@ func (*secretFactory) Create(f *framework.Framework, ns *v1.Namespace, i interfa }, nil } +type customResourceDefinitionFactory struct{} + +func (f *customResourceDefinitionFactory) New() runtime.Object { + return &apiextensionsv1.CustomResourceDefinition{} +} + +func (*customResourceDefinitionFactory) Create(f *framework.Framework, ns *v1.Namespace, i interface{}) (func() error, error) { + var err error + unstructCRD := &unstructured.Unstructured{} + gvr := schema.GroupVersionResource{Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions"} + + item, ok := i.(*apiextensionsv1.CustomResourceDefinition) + if !ok { + return nil, errorItemNotSupported + } + + unstructCRD.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(i) + if err != nil { + return nil, err + } + + if _, err = f.DynamicClient.Resource(gvr).Create(context.TODO(), unstructCRD, metav1.CreateOptions{}); err != nil { + return nil, fmt.Errorf("create CustomResourceDefinition: %w", err) + } + return func() error { + return f.DynamicClient.Resource(gvr).Delete(context.TODO(), item.GetName(), metav1.DeleteOptions{}) + }, nil +} + // PrettyPrint returns a human-readable representation of an item. func PrettyPrint(item interface{}) string { data, err := json.MarshalIndent(item, "", " ") diff --git a/test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/hello-populator-crd.yaml b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/hello-populator-crd.yaml new file mode 100644 index 00000000000..2f8dc976b10 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/hello-populator-crd.yaml @@ -0,0 +1,50 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: hellos.hello.example.com +spec: + group: hello.example.com + names: + kind: Hello + listKind: HelloList + plural: hellos + singular: hello + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Hello is a specification for a Hello resource + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: HelloSpec is the spec for a Hello resource + properties: + fileContents: + type: string + fileName: + type: string + required: + - fileContents + - fileName + type: object + required: + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/populator.storage.k8s.io_volumepopulators.yaml b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/populator.storage.k8s.io_volumepopulators.yaml new file mode 100644 index 00000000000..98cd376a1cb --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/populator.storage.k8s.io_volumepopulators.yaml @@ -0,0 +1,57 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.0 + api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/2934 + creationTimestamp: null + name: volumepopulators.populator.storage.k8s.io +spec: + group: populator.storage.k8s.io + names: + kind: VolumePopulator + listKind: VolumePopulatorList + plural: volumepopulators + singular: volumepopulator + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .sourceKind + name: SourceKind + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumePopulator represents the registration for a volume populator. VolumePopulators are cluster scoped. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + sourceKind: + description: Kind of the data source this populator supports + properties: + group: + type: string + kind: + type: string + required: + - group + - kind + type: object + required: + - sourceKind + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/test/e2e/testing-manifests/storage-csi/any-volume-datasource/hello-populator-deploy.yaml b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/hello-populator-deploy.yaml new file mode 100644 index 00000000000..256d087235d --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/hello-populator-deploy.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hello-account + namespace: hello +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hello-role +rules: + - apiGroups: [""] + resources: [persistentvolumes] + verbs: [get, list, watch, patch] + - apiGroups: [""] + resources: [persistentvolumeclaims] + verbs: [get, list, watch, patch, create, delete] + - apiGroups: [""] + resources: [pods] + verbs: [get, list, watch, create, delete] + - apiGroups: [storage.k8s.io] + resources: [storageclasses] + verbs: [get, list, watch] + + - apiGroups: [hello.example.com] + resources: [hellos] + verbs: [get, list, watch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hello-binding +subjects: + - kind: ServiceAccount + name: hello-account + namespace: hello +roleRef: + kind: ClusterRole + name: hello-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-populator + namespace: hello +spec: + selector: + matchLabels: + app: hello + template: + metadata: + labels: + app: hello + spec: + serviceAccount: hello-account + containers: + - name: hello + image: k8s.gcr.io/sig-storage/hello-populator:v1.0.1 + imagePullPolicy: IfNotPresent + args: + - --mode=controller + - --image-name=k8s.gcr.io/sig-storage/hello-populator:v1.0.1 + - --http-endpoint=:8080 + ports: + - containerPort: 8080 + name: http-endpoint + protocol: TCP diff --git a/test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/rbac-data-source-validator.yaml b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/rbac-data-source-validator.yaml new file mode 100644 index 00000000000..677185cb770 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/rbac-data-source-validator.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: volume-data-source-validator + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: volume-data-source-validator +rules: + - apiGroups: [populator.storage.k8s.io] + resources: [volumepopulators] + verbs: [get, list, watch] + - apiGroups: [""] + resources: [persistentvolumeclaims] + verbs: [get, list, watch] + - apiGroups: [""] + resources: [events] + verbs: [list, watch, create, update, patch] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: volume-data-source-validator + labels: + addonmanager.kubernetes.io/mode: Reconcile +subjects: + - kind: ServiceAccount + name: volume-data-source-validator + namespace: kube-system +roleRef: + kind: ClusterRole + name: volume-data-source-validator + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/setup-data-source-validator.yaml b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/setup-data-source-validator.yaml new file mode 100644 index 00000000000..ca081407977 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/setup-data-source-validator.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: volume-data-source-validator + namespace: kube-system +spec: + serviceName: volume-data-source-validator + replicas: 1 + selector: + matchLabels: + app: volume-data-source-validator + template: + metadata: + labels: + app: volume-data-source-validator + spec: + serviceAccount: volume-data-source-validator + containers: + - name: volume-data-source-validator + image: k8s.gcr.io/sig-storage/volume-data-source-validator:v1.0.0 + args: + - "--v=5" + - "--leader-election=false" + imagePullPolicy: Always diff --git a/test/utils/image/csi_manifests_test.go b/test/utils/image/csi_manifests_test.go index d522c0c04b4..e80f2557369 100644 --- a/test/utils/image/csi_manifests_test.go +++ b/test/utils/image/csi_manifests_test.go @@ -50,6 +50,10 @@ func TestCSIImageConfigs(t *testing.T) { // For some hostpath tests. "socat", "busybox", + + // For AnyVolumeDataSource feature tests. + "volume-data-source-validator", + "hello-populator", } actualImages := sets.NewString() for _, config := range configs {