Implement initial E2E test for AnyVolumeDataSource

test/e2e/storage/testsuites/provisioning.go

@@ -19,18 +19,22 @@ package testsuites
 import (
 	"context"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
 
+	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -239,6 +243,181 @@ func (p *provisioningTestSuite) DefineTests(driver storageframework.TestDriver,
 		l.testCase.TestDynamicProvisioning()
 	})
 
+	ginkgo.It("should provision storage with any volume data source [Serial]", func() {
+		if len(dInfo.InTreePluginName) != 0 {
+			e2eskipper.Skipf("AnyVolumeDataSource feature only works with CSI drivers - skipping")
+		}
+		if pattern.VolMode == v1.PersistentVolumeBlock {
+			e2eskipper.Skipf("Test for Block volumes is not implemented - skipping")
+		}
+
+		init()
+		defer cleanup()
+
+		ginkgo.By("Creating validator namespace")
+		valNamespace, err := f.CreateNamespace(fmt.Sprintf("%s-val", f.Namespace.Name), map[string]string{
+			"e2e-framework":      f.BaseName,
+			"e2e-test-namespace": f.Namespace.Name,
+		})
+		framework.ExpectNoError(err)
+
+		defer func() {
+			f.DeleteNamespace(valNamespace.Name)
+		}()
+
+		ginkgo.By("Deploying validator")
+		valManifests := []string{
+			"test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/populator.storage.k8s.io_volumepopulators.yaml",
+			"test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/rbac-data-source-validator.yaml",
+			"test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/setup-data-source-validator.yaml",
+		}
+		valCleanup, err := storageutils.CreateFromManifests(f, valNamespace,
+			func(item interface{}) error { return nil },
+			valManifests...)
+
+		framework.ExpectNoError(err)
+		defer valCleanup()
+
+		ginkgo.By("Creating populator namespace")
+		popNamespace, err := f.CreateNamespace(fmt.Sprintf("%s-pop", f.Namespace.Name), map[string]string{
+			"e2e-framework":      f.BaseName,
+			"e2e-test-namespace": f.Namespace.Name,
+		})
+		framework.ExpectNoError(err)
+
+		defer func() {
+			f.DeleteNamespace(popNamespace.Name)
+		}()
+
+		ginkgo.By("Deploying hello-populator")
+		popManifests := []string{
+			"test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/hello-populator-crd.yaml",
+			"test/e2e/testing-manifests/storage-csi/any-volume-datasource/hello-populator-deploy.yaml",
+		}
+		popCleanup, err := storageutils.CreateFromManifests(f, popNamespace,
+			func(item interface{}) error {
+				switch item := item.(type) {
+				case *appsv1.Deployment:
+					for i, container := range item.Spec.Template.Spec.Containers {
+						switch container.Name {
+						case "hello":
+							var found bool
+							args := []string{}
+							for _, arg := range container.Args {
+								if strings.HasPrefix(arg, "--namespace=") {
+									args = append(args, fmt.Sprintf("--namespace=%s", popNamespace.Name))
+									found = true
+								} else {
+									args = append(args, arg)
+								}
+							}
+							if !found {
+								args = append(args, fmt.Sprintf("--namespace=%s", popNamespace.Name))
+								framework.Logf("container name: %s", container.Name)
+							}
+							container.Args = args
+							item.Spec.Template.Spec.Containers[i] = container
+						default:
+						}
+					}
+				}
+				return nil
+			},
+			popManifests...)
+
+		framework.ExpectNoError(err)
+		defer popCleanup()
+
+		dc := l.config.Framework.DynamicClient
+
+		// Make hello-populator handle Hello resource in hello.example.com group
+		ginkgo.By("Creating VolumePopulator CR datasource")
+		volumePopulatorGVR := schema.GroupVersionResource{Group: "populator.storage.k8s.io", Version: "v1beta1", Resource: "volumepopulators"}
+		helloPopulatorCR := &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"kind":       "VolumePopulator",
+				"apiVersion": "populator.storage.k8s.io/v1beta1",
+				"metadata": map[string]interface{}{
+					"name": fmt.Sprintf("%s-%s", "hello-populator", f.Namespace.Name),
+				},
+				"sourceKind": map[string]interface{}{
+					"group": "hello.example.com",
+					"kind":  "Hello",
+				},
+			},
+		}
+
+		_, err = dc.Resource(volumePopulatorGVR).Create(context.TODO(), helloPopulatorCR, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+
+		defer func() {
+			framework.Logf("deleting VolumePopulator CR datasource %q/%q", helloPopulatorCR.GetNamespace(), helloPopulatorCR.GetName())
+			err = dc.Resource(volumePopulatorGVR).Delete(context.TODO(), helloPopulatorCR.GetName(), metav1.DeleteOptions{})
+			if err != nil && !apierrors.IsNotFound(err) {
+				framework.Failf("Error deleting VolumePopulator CR datasource %q. Error: %v", helloPopulatorCR.GetName(), err)
+			}
+		}()
+
+		// Create Hello CR datasource
+		ginkgo.By("Creating Hello CR datasource")
+		helloCRName := "example-hello"
+		fileName := fmt.Sprintf("example-%s.txt", f.Namespace.Name)
+		expectedContent := fmt.Sprintf("Hello from namespace %s", f.Namespace.Name)
+		helloGVR := schema.GroupVersionResource{Group: "hello.example.com", Version: "v1alpha1", Resource: "hellos"}
+		helloCR := &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"kind":       "Hello",
+				"apiVersion": "hello.example.com/v1alpha1",
+				"metadata": map[string]interface{}{
+					"name":      helloCRName,
+					"namespace": f.Namespace.Name,
+				},
+				"spec": map[string]interface{}{
+					"fileName":     fileName,
+					"fileContents": expectedContent,
+				},
+			},
+		}
+
+		_, err = dc.Resource(helloGVR).Namespace(f.Namespace.Name).Create(context.TODO(), helloCR, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+
+		defer func() {
+			framework.Logf("deleting Hello CR datasource %q/%q", helloCR.GetNamespace(), helloCR.GetName())
+			err = dc.Resource(helloGVR).Namespace(helloCR.GetNamespace()).Delete(context.TODO(), helloCR.GetName(), metav1.DeleteOptions{})
+			if err != nil && !apierrors.IsNotFound(err) {
+				framework.Failf("Error deleting Hello CR datasource %q. Error: %v", helloCR.GetName(), err)
+			}
+		}()
+
+		apiGroup := "hello.example.com"
+		l.pvc.Spec.DataSourceRef = &v1.TypedLocalObjectReference{
+			APIGroup: &apiGroup,
+			Kind:     "Hello",
+			Name:     helloCRName,
+		}
+
+		testConfig := storageframework.ConvertTestConfig(l.config)
+		l.testCase.NodeSelection = testConfig.ClientNodeSelection
+		l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim) {
+			ginkgo.By("checking whether the created volume has the pre-populated data")
+			tests := []e2evolume.Test{
+				{
+					Volume:          *storageutils.CreateVolumeSource(claim.Name, false /* readOnly */),
+					Mode:            pattern.VolMode,
+					File:            fileName,
+					ExpectedContent: expectedContent,
+				},
+			}
+			e2evolume.TestVolumeClientSlow(f, testConfig, nil, "", tests)
+		}
+
+		_, clearProvisionedStorageClass := SetupStorageClass(l.testCase.Client, l.testCase.Class)
+		defer clearProvisionedStorageClass()
+
+		l.testCase.TestDynamicProvisioning()
+	})
+
 	ginkgo.It("should provision storage with pvc data source", func() {
 		if !dInfo.Capabilities[storageframework.CapPVCDataSource] {
 			e2eskipper.Skipf("Driver %q does not support cloning - skipping", dInfo.Name)

test/e2e/storage/utils/create.go

@@ -27,8 +27,10 @@ import (
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	storagev1 "k8s.io/api/storage/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -279,7 +281,9 @@ var factories = map[What]ItemFactory{
 	{"Service"}:                  &serviceFactory{},
 	{"ServiceAccount"}:           &serviceAccountFactory{},
 	{"StatefulSet"}:              &statefulSetFactory{},
+	{"Deployment"}:               &deploymentFactory{},
 	{"StorageClass"}:             &storageClassFactory{},
+	{"CustomResourceDefinition"}: &customResourceDefinitionFactory{},
 }
 
 // PatchName makes the name of some item unique by appending the
@@ -362,6 +366,14 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace,
 		if err := patchContainerImages(item.Spec.Template.Spec.InitContainers); err != nil {
 			return err
 		}
+	case *appsv1.Deployment:
+		PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace)
+		if err := patchContainerImages(item.Spec.Template.Spec.Containers); err != nil {
+			return err
+		}
+		if err := patchContainerImages(item.Spec.Template.Spec.InitContainers); err != nil {
+			return err
+		}
 	case *appsv1.DaemonSet:
 		PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace)
 		if err := patchContainerImages(item.Spec.Template.Spec.Containers); err != nil {
@@ -370,6 +382,8 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace,
 		if err := patchContainerImages(item.Spec.Template.Spec.InitContainers); err != nil {
 			return err
 		}
+	case *apiextensionsv1.CustomResourceDefinition:
+		// Do nothing. Patching name to all CRDs won't always be the expected behavior.
 	default:
 		return fmt.Errorf("missing support for patching item of type %T", item)
 	}
@@ -528,6 +542,27 @@ func (*statefulSetFactory) Create(f *framework.Framework, ns *v1.Namespace, i in
 	}, nil
 }
 
+type deploymentFactory struct{}
+
+func (f *deploymentFactory) New() runtime.Object {
+	return &appsv1.Deployment{}
+}
+
+func (*deploymentFactory) Create(f *framework.Framework, ns *v1.Namespace, i interface{}) (func() error, error) {
+	item, ok := i.(*appsv1.Deployment)
+	if !ok {
+		return nil, errorItemNotSupported
+	}
+
+	client := f.ClientSet.AppsV1().Deployments(ns.Name)
+	if _, err := client.Create(context.TODO(), item, metav1.CreateOptions{}); err != nil {
+		return nil, fmt.Errorf("create Deployment: %w", err)
+	}
+	return func() error {
+		return client.Delete(context.TODO(), item.GetName(), metav1.DeleteOptions{})
+	}, nil
+}
+
 type daemonSetFactory struct{}
 
 func (f *daemonSetFactory) New() runtime.Object {
@@ -612,6 +647,35 @@ func (*secretFactory) Create(f *framework.Framework, ns *v1.Namespace, i interfa
 	}, nil
 }
 
+type customResourceDefinitionFactory struct{}
+
+func (f *customResourceDefinitionFactory) New() runtime.Object {
+	return &apiextensionsv1.CustomResourceDefinition{}
+}
+
+func (*customResourceDefinitionFactory) Create(f *framework.Framework, ns *v1.Namespace, i interface{}) (func() error, error) {
+	var err error
+	unstructCRD := &unstructured.Unstructured{}
+	gvr := schema.GroupVersionResource{Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions"}
+
+	item, ok := i.(*apiextensionsv1.CustomResourceDefinition)
+	if !ok {
+		return nil, errorItemNotSupported
+	}
+
+	unstructCRD.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(i)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = f.DynamicClient.Resource(gvr).Create(context.TODO(), unstructCRD, metav1.CreateOptions{}); err != nil {
+		return nil, fmt.Errorf("create CustomResourceDefinition: %w", err)
+	}
+	return func() error {
+		return f.DynamicClient.Resource(gvr).Delete(context.TODO(), item.GetName(), metav1.DeleteOptions{})
+	}, nil
+}
+
 // PrettyPrint returns a human-readable representation of an item.
 func PrettyPrint(item interface{}) string {
 	data, err := json.MarshalIndent(item, "", "  ")

test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/hello-populator-crd.yaml

@@ -0,0 +1,50 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: hellos.hello.example.com
+spec:
+  group: hello.example.com
+  names:
+    kind: Hello
+    listKind: HelloList
+    plural: hellos
+    singular: hello
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: Hello is a specification for a Hello resource
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          spec:
+            description: HelloSpec is the spec for a Hello resource
+            properties:
+              fileContents:
+                type: string
+              fileName:
+                type: string
+            required:
+            - fileContents
+            - fileName
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

test/e2e/testing-manifests/storage-csi/any-volume-datasource/crd/populator.storage.k8s.io_volumepopulators.yaml

@@ -0,0 +1,57 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.5.0
+    api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/2934
+  creationTimestamp: null
+  name: volumepopulators.populator.storage.k8s.io
+spec:
+  group: populator.storage.k8s.io
+  names:
+    kind: VolumePopulator
+    listKind: VolumePopulatorList
+    plural: volumepopulators
+    singular: volumepopulator
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .sourceKind
+      name: SourceKind
+      type: string
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: VolumePopulator represents the registration for a volume populator. VolumePopulators are cluster scoped.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          sourceKind:
+            description: Kind of the data source this populator supports
+            properties:
+              group:
+                type: string
+              kind:
+                type: string
+            required:
+            - group
+            - kind
+            type: object
+        required:
+        - sourceKind
+        type: object
+    served: true
+    storage: true
+    subresources: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

test/e2e/testing-manifests/storage-csi/any-volume-datasource/hello-populator-deploy.yaml

@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: hello-account
+  namespace: hello
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: hello-role
+rules:
+  - apiGroups: [""]
+    resources: [persistentvolumes]
+    verbs: [get, list, watch, patch]
+  - apiGroups: [""]
+    resources: [persistentvolumeclaims]
+    verbs: [get, list, watch, patch, create, delete]
+  - apiGroups: [""]
+    resources: [pods]
+    verbs: [get, list, watch, create, delete]
+  - apiGroups: [storage.k8s.io]
+    resources: [storageclasses]
+    verbs: [get, list, watch]
+
+  - apiGroups: [hello.example.com]
+    resources: [hellos]
+    verbs: [get, list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: hello-binding
+subjects:
+  - kind: ServiceAccount
+    name: hello-account
+    namespace: hello
+roleRef:
+  kind: ClusterRole
+  name: hello-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hello-populator
+  namespace: hello
+spec:
+  selector:
+    matchLabels:
+      app: hello
+  template:
+    metadata:
+      labels:
+        app: hello
+    spec:
+      serviceAccount: hello-account
+      containers:
+        - name: hello
+          image: k8s.gcr.io/sig-storage/hello-populator:v1.0.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - --mode=controller
+            - --image-name=k8s.gcr.io/sig-storage/hello-populator:v1.0.1
+            - --http-endpoint=:8080
+          ports:
+            - containerPort: 8080
+              name: http-endpoint
+              protocol: TCP

test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/rbac-data-source-validator.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: volume-data-source-validator
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: volume-data-source-validator
+rules:
+  - apiGroups: [populator.storage.k8s.io]
+    resources: [volumepopulators]
+    verbs: [get, list, watch]
+  - apiGroups: [""]
+    resources: [persistentvolumeclaims]
+    verbs: [get, list, watch]
+  - apiGroups: [""]
+    resources: [events]
+    verbs: [list, watch, create, update, patch]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: volume-data-source-validator
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+subjects:
+  - kind: ServiceAccount
+    name: volume-data-source-validator
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: volume-data-source-validator
+  apiGroup: rbac.authorization.k8s.io

test/e2e/testing-manifests/storage-csi/any-volume-datasource/volume-data-source-validator/setup-data-source-validator.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: volume-data-source-validator
+  namespace: kube-system
+spec:
+  serviceName: volume-data-source-validator
+  replicas: 1
+  selector:
+    matchLabels:
+      app: volume-data-source-validator
+  template:
+    metadata:
+      labels:
+        app: volume-data-source-validator
+    spec:
+      serviceAccount: volume-data-source-validator
+      containers:
+        - name: volume-data-source-validator
+          image: k8s.gcr.io/sig-storage/volume-data-source-validator:v1.0.0
+          args:
+            - "--v=5"
+            - "--leader-election=false"
+          imagePullPolicy: Always

test/utils/image/csi_manifests_test.go

@@ -50,6 +50,10 @@ func TestCSIImageConfigs(t *testing.T) {
 		// For some hostpath tests.
 		"socat",
 		"busybox",
+
+		// For AnyVolumeDataSource feature tests.
+		"volume-data-source-validator",
+		"hello-populator",
 	}
 	actualImages := sets.NewString()
 	for _, config := range configs {