Merge pull request #70439 from jsafrane/worlload-test

CSI: Add test for passing Pod information in NodePublish call

test/e2e/storage/csi_volumes.go

@@ -18,13 +18,16 @@ package storage
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"regexp"
+	"strings"
 
 	"k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
 	csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
 	csiclient "k8s.io/csi-api/pkg/client/clientset/versioned"
@@ -48,6 +51,7 @@ var csiTestDrivers = []func(config testsuites.TestConfig) testsuites.TestDriver{
 	drivers.InitGcePDCSIDriver,
 	drivers.InitGcePDExternalCSIDriver,
 	drivers.InitHostPathV0CSIDriver,
+	// Don't run tests with mock driver (drivers.InitMockCSIDriver), it does not provide persistent storage.
 }
 
 // List of testSuites to be executed in below loop
@@ -80,6 +84,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 	var (
 		cancel context.CancelFunc
 		cs     clientset.Interface
+		csics  csiclient.Interface
 		ns     *v1.Namespace
 		// Common configuration options for each driver.
 		config = testsuites.TestConfig{
@@ -92,6 +97,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 		ctx, c := context.WithCancel(context.Background())
 		cancel = c
 		cs = f.ClientSet
+		csics = f.CSIClientSet
 		ns = f.Namespace
 
 		// Debugging of the following tests heavily depends on the log output
@@ -150,14 +156,10 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 	// The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12.
 	Context("CSI attach test using HostPath driver [Feature:CSIDriverRegistry]", func() {
 		var (
-			cs     clientset.Interface
-			csics  csiclient.Interface
 			driver testsuites.TestDriver
 		)
 
 		BeforeEach(func() {
-			cs = f.ClientSet
-			csics = f.CSIClientSet
 			config := testsuites.TestConfig{
 				Framework: f,
 				Prefix:    "csi-attach",
@@ -199,7 +201,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 			test := t
 			It(test.name, func() {
 				if test.driverExists {
-					csiDriver := createCSIDriver(csics, testsuites.GetUniqueDriverName(driver), test.driverAttachable)
+					csiDriver := createCSIDriver(csics, testsuites.GetUniqueDriverName(driver), test.driverAttachable, nil)
 					if csiDriver != nil {
 						defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil)
 					}
@@ -258,16 +260,129 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 			})
 		}
 	})
+
+	Context("CSI workload information [Feature:CSIDriverRegistry]", func() {
+		var (
+			driver         testsuites.TestDriver
+			podInfoV1      = "v1"
+			podInfoUnknown = "unknown"
+			podInfoEmpty   = ""
+		)
+
+		BeforeEach(func() {
+			config := testsuites.TestConfig{
+				Framework: f,
+				Prefix:    "csi-workload",
+			}
+			driver = drivers.InitMockCSIDriver(config)
+			driver.CreateDriver()
+		})
+
+		AfterEach(func() {
+			driver.CleanupDriver()
+		})
+
+		tests := []struct {
+			name                  string
+			podInfoOnMountVersion *string
+			driverExists          bool
+			expectPodInfo         bool
+		}{
+			{
+				name:                  "should not be passed when podInfoOnMountVersion=nil",
+				podInfoOnMountVersion: nil,
+				driverExists:          true,
+				expectPodInfo:         false,
+			},
+			{
+				name:                  "should be passed when podInfoOnMountVersion=v1",
+				podInfoOnMountVersion: &podInfoV1,
+				driverExists:          true,
+				expectPodInfo:         true,
+			},
+			{
+				name:                  "should not be passed when podInfoOnMountVersion=<empty string>",
+				podInfoOnMountVersion: &podInfoEmpty,
+				driverExists:          true,
+				expectPodInfo:         false,
+			},
+			{
+				name:                  "should not be passed when podInfoOnMountVersion=<unknown string>",
+				podInfoOnMountVersion: &podInfoUnknown,
+				driverExists:          true,
+				expectPodInfo:         false,
+			},
+			{
+				name:          "should not be passed when CSIDriver does not exist",
+				driverExists:  false,
+				expectPodInfo: false,
+			},
+		}
+		for _, t := range tests {
+			test := t
+			It(test.name, func() {
+				if test.driverExists {
+					csiDriver := createCSIDriver(csics, testsuites.GetUniqueDriverName(driver), true, test.podInfoOnMountVersion)
+					if csiDriver != nil {
+						defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil)
+					}
+				}
+
+				By("Creating pod")
+				var sc *storagev1.StorageClass
+				if dDriver, ok := driver.(testsuites.DynamicPVTestDriver); ok {
+					sc = dDriver.GetDynamicProvisionStorageClass("")
+				}
+				nodeName := driver.GetDriverInfo().Config.ClientNodeName
+				scTest := testsuites.StorageClassTest{
+					Name:         driver.GetDriverInfo().Name,
+					Parameters:   sc.Parameters,
+					ClaimSize:    "1Gi",
+					ExpectedSize: "1Gi",
+					// The mock driver only works when everything runs on a single node.
+					NodeName: nodeName,
+					// Provisioner and storage class name must match what's used in
+					// csi-storageclass.yaml, plus the test-specific suffix.
+					Provisioner:      sc.Provisioner,
+					StorageClassName: "csi-mock-sc-" + f.UniqueName,
+					// Mock driver does not provide any persistency.
+					SkipWriteReadCheck: true,
+				}
+				class, claim, pod := startPausePod(cs, scTest, ns.Name)
+				if class != nil {
+					defer cs.StorageV1().StorageClasses().Delete(class.Name, nil)
+				}
+				if claim != nil {
+					defer cs.CoreV1().PersistentVolumeClaims(ns.Name).Delete(claim.Name, nil)
+				}
+				if pod != nil {
+					// Fully delete (=unmount) the pod before deleting CSI driver
+					defer framework.DeletePodWithWait(f, cs, pod)
+				}
+				if pod == nil {
+					return
+				}
+				err := framework.WaitForPodNameRunningInNamespace(cs, pod.Name, pod.Namespace)
+				framework.ExpectNoError(err, "Failed to start pod: %v", err)
+				By("Checking CSI driver logs")
+				// The driver is deployed as a statefulset with stable pod names
+				driverPodName := "csi-mockplugin-0"
+				err = checkPodInfo(cs, f.Namespace.Name, driverPodName, "mock", pod, test.expectPodInfo)
+				framework.ExpectNoError(err)
+			})
+		}
+	})
 })
 
-func createCSIDriver(csics csiclient.Interface, name string, attachable bool) *csiv1alpha1.CSIDriver {
+func createCSIDriver(csics csiclient.Interface, name string, attachable bool, podInfoOnMountVersion *string) *csiv1alpha1.CSIDriver {
 	By("Creating CSIDriver instance")
 	driver := &csiv1alpha1.CSIDriver{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
 		},
 		Spec: csiv1alpha1.CSIDriverSpec{
-			AttachRequired: &attachable,
+			AttachRequired:        &attachable,
+			PodInfoOnMountVersion: podInfoOnMountVersion,
 		},
 	}
 	driver, err := csics.CsiV1alpha1().CSIDrivers().Create(driver)
@@ -343,3 +458,65 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, ns str
 	framework.ExpectNoError(err, "Failed to create pod: %v", err)
 	return class, claim, pod
 }
+
+// checkPodInfo tests that NodePublish was called with expected volume_context
+func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo bool) error {
+	expectedAttributes := map[string]string{
+		"csi.storage.k8s.io/pod.name":            pod.Name,
+		"csi.storage.k8s.io/pod.namespace":       namespace,
+		"csi.storage.k8s.io/pod.uid":             string(pod.UID),
+		"csi.storage.k8s.io/serviceAccount.name": "default",
+	}
+	// Load logs of driver pod
+	log, err := framework.GetPodLogs(cs, namespace, driverPodName, driverContainerName)
+	if err != nil {
+		return fmt.Errorf("could not load CSI driver logs: %s", err)
+	}
+	framework.Logf("CSI driver logs:\n%s", log)
+	// Find NodePublish in the logs
+	foundAttributes := sets.NewString()
+	logLines := strings.Split(log, "\n")
+	for _, line := range logLines {
+		if !strings.HasPrefix(line, "gRPCCall:") {
+			continue
+		}
+		line = strings.TrimPrefix(line, "gRPCCall:")
+		// Dummy structure that parses just volume_attributes out of logged CSI call
+		type MockCSICall struct {
+			Method  string
+			Request struct {
+				VolumeContext map[string]string `json:"volume_context"`
+			}
+		}
+		var call MockCSICall
+		err := json.Unmarshal([]byte(line), &call)
+		if err != nil {
+			framework.Logf("Could not parse CSI driver log line %q: %s", line, err)
+			continue
+		}
+		if call.Method != "/csi.v1.Node/NodePublishVolume" {
+			continue
+		}
+		// Check that NodePublish had expected attributes
+		for k, v := range expectedAttributes {
+			vv, found := call.Request.VolumeContext[k]
+			if found && v == vv {
+				foundAttributes.Insert(k)
+				framework.Logf("Found volume attribute %s: %s", k, v)
+			}
+		}
+		// Process just the first NodePublish, the rest of the log is useless.
+		break
+	}
+	if expectPodInfo {
+		if foundAttributes.Len() != len(expectedAttributes) {
+			return fmt.Errorf("number of found volume attributes does not match, expected %d, got %d", len(expectedAttributes), foundAttributes.Len())
+		}
+		return nil
+	} else {
+		if foundAttributes.Len() != 0 {
+			return fmt.Errorf("some unexpected volume attributes were found: %+v", foundAttributes.List())
+		}
+		return nil
+	}
+}

test/e2e/storage/drivers/csi.go

@@ -126,6 +126,7 @@ func (h *hostpathCSIDriver) CreateDriver() {
 		OldDriverName:            h.driverInfo.Name,
 		NewDriverName:            testsuites.GetUniqueDriverName(h),
 		DriverContainerName:      "hostpath",
+		DriverContainerArguments: []string{"--drivername=csi-hostpath-" + f.UniqueName},
 		ProvisionerContainerName: "csi-provisioner",
 		NodeName:                 nodeName,
 	}
@@ -146,6 +147,98 @@ func (h *hostpathCSIDriver) CleanupDriver() {
 	}
 }
 
+// mockCSI
+type mockCSIDriver struct {
+	cleanup    func()
+	driverInfo testsuites.DriverInfo
+}
+
+var _ testsuites.TestDriver = &mockCSIDriver{}
+var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{}
+
+// InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface
+func InitMockCSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
+	return &mockCSIDriver{
+		driverInfo: testsuites.DriverInfo{
+			Name:        "csi-mock",
+			FeatureTag:  "",
+			MaxFileSize: testpatterns.FileSizeMedium,
+			SupportedFsType: sets.NewString(
+				"", // Default fsType
+			),
+			Capabilities: map[testsuites.Capability]bool{
+				testsuites.CapPersistence: false,
+				testsuites.CapFsGroup:     false,
+				testsuites.CapExec:        false,
+			},
+			Config: config,
+		},
+	}
+}
+
+func (m *mockCSIDriver) GetDriverInfo() *testsuites.DriverInfo {
+	return &m.driverInfo
+}
+
+func (m *mockCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+}
+
+func (m *mockCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass {
+	provisioner := testsuites.GetUniqueDriverName(m)
+	parameters := map[string]string{}
+	ns := m.driverInfo.Config.Framework.Namespace.Name
+	suffix := fmt.Sprintf("%s-sc", provisioner)
+
+	return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
+}
+
+func (m *mockCSIDriver) GetClaimSize() string {
+	return "5Gi"
+}
+
+func (m *mockCSIDriver) CreateDriver() {
+	By("deploying csi mock driver")
+	f := m.driverInfo.Config.Framework
+	cs := f.ClientSet
+
+	// pods should be scheduled on the node
+	nodes := framework.GetReadySchedulableNodesOrDie(cs)
+	node := nodes.Items[rand.Intn(len(nodes.Items))]
+	m.driverInfo.Config.ClientNodeName = node.Name
+
+	// TODO (?): the storage.csi.image.version and storage.csi.image.registry
+	// settings are ignored for this test. We could patch the image definitions.
+	o := utils.PatchCSIOptions{
+		OldDriverName:            "csi-mock",
+		NewDriverName:            "csi-mock-" + f.UniqueName,
+		DriverContainerName:      "mock",
+		DriverContainerArguments: []string{"--name=csi-mock-" + f.UniqueName},
+		ProvisionerContainerName: "csi-provisioner",
+		NodeName:                 m.driverInfo.Config.ClientNodeName,
+	}
+	cleanup, err := f.CreateFromManifests(func(item interface{}) error {
+		return utils.PatchCSIDeployment(f, o, item)
+	},
+		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml",
+		"test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml",
+		"test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml",
+	)
+	m.cleanup = cleanup
+	if err != nil {
+		framework.Failf("deploying csi mock driver: %v", err)
+	}
+}
+
+func (m *mockCSIDriver) CleanupDriver() {
+	if m.cleanup != nil {
+		By("uninstalling csi mock driver")
+		m.cleanup()
+	}
+}
+
 // InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests.
 func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver {
 	return initHostPathCSIDriver("csi-hostpath-v0", config,

test/e2e/storage/utils/deployment.go

@@ -88,7 +88,7 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf
 			// value.
 			switch container.Name {
 			case o.DriverContainerName:
-				container.Args = append(container.Args, "--drivername="+o.NewDriverName)
+				container.Args = append(container.Args, o.DriverContainerArguments...)
 			case o.ProvisionerContainerName:
 				// Driver name is expected to be the same
 				// as the provisioner here.
@@ -135,9 +135,12 @@ type PatchCSIOptions struct {
 	// in existing fields).
 	NewDriverName string
 	// The name of the container which has the CSI driver binary.
-	// If non-empty, --drivername with the new name will be
-	// appended to the argument list.
+	// If non-empty, DriverContainerArguments are added to argument
+	// list in container with that name.
 	DriverContainerName string
+	// List of arguments to add to container with
+	// DriverContainerName.
+	DriverContainerArguments []string
 	// The name of the container which has the provisioner binary.
 	// If non-empty, --provisioner with new name will be appended
 	// to the argument list.

test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml

@@ -42,6 +42,12 @@ rules:
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
     verbs: ["get", "list"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
 
 ---
 kind: ClusterRoleBinding

test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml

@@ -0,0 +1,97 @@
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-mockplugin
+spec:
+  selector:
+    matchLabels:
+      app: csi-mockplugin
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: csi-mockplugin
+    spec:
+      serviceAccountName: csi-mock
+      containers:
+
+
+        - name: csi-attacher
+          image: quay.io/k8scsi/csi-attacher:v1.0.1
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+        - name: csi-provisioner
+          image: quay.io/k8scsi/csi-provisioner:v1.0.1
+          args:
+            - "--provisioner=csi-hostpath"
+            - "--csi-address=$(ADDRESS)"
+            - "--connection-timeout=15s"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+
+
+        - name: driver-registrar
+          image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
+          args:
+            - --v=5
+            - --csi-address=/csi/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-mock/csi.sock
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          imagePullPolicy: Always
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+          - mountPath: /registration
+            name: registration-dir
+
+
+        - name: mock
+          image: quay.io/k8scsi/mock-driver:v1.0.0-1
+          env:
+            - name: CSI_ENDPOINT
+              value: /csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          securityContext:
+            privileged: true
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+            - mountPath: /var/lib/kubelet/pods
+              mountPropagation: Bidirectional
+              name: mountpoint-dir
+      volumes:
+        - hostPath:
+            path: /var/lib/kubelet/plugins/csi-mock
+            type: DirectoryOrCreate
+          name: socket-dir
+        - hostPath:
+            path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+          name: mountpoint-dir
+        - hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+          name: registration-dir

test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-mock
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-controller-attacher-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-mock
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-attacher-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-controller-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-mock
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: external-provisioner-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# priviledged Pod Security Policy, previously defined via PrivilegedTestPSPClusterRoleBinding()
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: psp-csi-controller-driver-registrar-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-mock
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: e2e-test-privileged-psp
+  apiGroup: rbac.authorization.k8s.io

test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml

@@ -0,0 +1,7 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-mock-sc
+provisioner: csi-mock
+reclaimPolicy: Delete
+volumeBindingMode: Immediate

test/e2e/testing-manifests/storage-csi/mock/e2e-test-rbac.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: psp-csi-mock-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-driver-registrar
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: e2e-test-privileged-psp
+  apiGroup: rbac.authorization.k8s.io