diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index 5c5477119c2..00c5ce9147c 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -18,13 +18,16 @@ package storage import ( "context" + "encoding/json" "fmt" "regexp" + "strings" "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" @@ -48,6 +51,7 @@ var csiTestDrivers = []func(config testsuites.TestConfig) testsuites.TestDriver{ drivers.InitGcePDCSIDriver, drivers.InitGcePDExternalCSIDriver, drivers.InitHostPathV0CSIDriver, + // Don't run tests with mock driver (drivers.InitMockCSIDriver), it does not provide persistent storage. } // List of testSuites to be executed in below loop @@ -80,6 +84,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { var ( cancel context.CancelFunc cs clientset.Interface + csics csiclient.Interface ns *v1.Namespace // Common configuration options for each driver. config = testsuites.TestConfig{ @@ -92,6 +97,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { ctx, c := context.WithCancel(context.Background()) cancel = c cs = f.ClientSet + csics = f.CSIClientSet ns = f.Namespace // Debugging of the following tests heavily depends on the log output @@ -150,14 +156,10 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { // The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. Context("CSI attach test using HostPath driver [Feature:CSIDriverRegistry]", func() { var ( - cs clientset.Interface - csics csiclient.Interface driver testsuites.TestDriver ) BeforeEach(func() { - cs = f.ClientSet - csics = f.CSIClientSet config := testsuites.TestConfig{ Framework: f, Prefix: "csi-attach", @@ -199,7 +201,7 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { test := t It(test.name, func() { if test.driverExists { - csiDriver := createCSIDriver(csics, testsuites.GetUniqueDriverName(driver), test.driverAttachable) + csiDriver := createCSIDriver(csics, testsuites.GetUniqueDriverName(driver), test.driverAttachable, nil) if csiDriver != nil { defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil) } @@ -258,16 +260,129 @@ var _ = utils.SIGDescribe("CSI Volumes", func() { }) } }) + + Context("CSI workload information [Feature:CSIDriverRegistry]", func() { + var ( + driver testsuites.TestDriver + podInfoV1 = "v1" + podInfoUnknown = "unknown" + podInfoEmpty = "" + ) + + BeforeEach(func() { + config := testsuites.TestConfig{ + Framework: f, + Prefix: "csi-workload", + } + driver = drivers.InitMockCSIDriver(config) + driver.CreateDriver() + }) + + AfterEach(func() { + driver.CleanupDriver() + }) + + tests := []struct { + name string + podInfoOnMountVersion *string + driverExists bool + expectPodInfo bool + }{ + { + name: "should not be passed when podInfoOnMountVersion=nil", + podInfoOnMountVersion: nil, + driverExists: true, + expectPodInfo: false, + }, + { + name: "should be passed when podInfoOnMountVersion=v1", + podInfoOnMountVersion: &podInfoV1, + driverExists: true, + expectPodInfo: true, + }, + { + name: "should not be passed when podInfoOnMountVersion=", + podInfoOnMountVersion: &podInfoEmpty, + driverExists: true, + expectPodInfo: false, + }, + { + name: "should not be passed when podInfoOnMountVersion=", + podInfoOnMountVersion: &podInfoUnknown, + driverExists: true, + expectPodInfo: false, + }, + { + name: "should not be passed when CSIDriver does not exist", + driverExists: false, + expectPodInfo: false, + }, + } + for _, t := range tests { + test := t + It(test.name, func() { + if test.driverExists { + csiDriver := createCSIDriver(csics, testsuites.GetUniqueDriverName(driver), true, test.podInfoOnMountVersion) + if csiDriver != nil { + defer csics.CsiV1alpha1().CSIDrivers().Delete(csiDriver.Name, nil) + } + } + + By("Creating pod") + var sc *storagev1.StorageClass + if dDriver, ok := driver.(testsuites.DynamicPVTestDriver); ok { + sc = dDriver.GetDynamicProvisionStorageClass("") + } + nodeName := driver.GetDriverInfo().Config.ClientNodeName + scTest := testsuites.StorageClassTest{ + Name: driver.GetDriverInfo().Name, + Parameters: sc.Parameters, + ClaimSize: "1Gi", + ExpectedSize: "1Gi", + // The mock driver only works when everything runs on a single node. + NodeName: nodeName, + // Provisioner and storage class name must match what's used in + // csi-storageclass.yaml, plus the test-specific suffix. + Provisioner: sc.Provisioner, + StorageClassName: "csi-mock-sc-" + f.UniqueName, + // Mock driver does not provide any persistency. + SkipWriteReadCheck: true, + } + class, claim, pod := startPausePod(cs, scTest, ns.Name) + if class != nil { + defer cs.StorageV1().StorageClasses().Delete(class.Name, nil) + } + if claim != nil { + defer cs.CoreV1().PersistentVolumeClaims(ns.Name).Delete(claim.Name, nil) + } + if pod != nil { + // Fully delete (=unmount) the pod before deleting CSI driver + defer framework.DeletePodWithWait(f, cs, pod) + } + if pod == nil { + return + } + err := framework.WaitForPodNameRunningInNamespace(cs, pod.Name, pod.Namespace) + framework.ExpectNoError(err, "Failed to start pod: %v", err) + By("Checking CSI driver logs") + // The driver is deployed as a statefulset with stable pod names + driverPodName := "csi-mockplugin-0" + err = checkPodInfo(cs, f.Namespace.Name, driverPodName, "mock", pod, test.expectPodInfo) + framework.ExpectNoError(err) + }) + } + }) }) -func createCSIDriver(csics csiclient.Interface, name string, attachable bool) *csiv1alpha1.CSIDriver { +func createCSIDriver(csics csiclient.Interface, name string, attachable bool, podInfoOnMountVersion *string) *csiv1alpha1.CSIDriver { By("Creating CSIDriver instance") driver := &csiv1alpha1.CSIDriver{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: csiv1alpha1.CSIDriverSpec{ - AttachRequired: &attachable, + AttachRequired: &attachable, + PodInfoOnMountVersion: podInfoOnMountVersion, }, } driver, err := csics.CsiV1alpha1().CSIDrivers().Create(driver) @@ -343,3 +458,65 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, ns str framework.ExpectNoError(err, "Failed to create pod: %v", err) return class, claim, pod } + +// checkPodInfo tests that NodePublish was called with expected volume_context +func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo bool) error { + expectedAttributes := map[string]string{ + "csi.storage.k8s.io/pod.name": pod.Name, + "csi.storage.k8s.io/pod.namespace": namespace, + "csi.storage.k8s.io/pod.uid": string(pod.UID), + "csi.storage.k8s.io/serviceAccount.name": "default", + } + // Load logs of driver pod + log, err := framework.GetPodLogs(cs, namespace, driverPodName, driverContainerName) + if err != nil { + return fmt.Errorf("could not load CSI driver logs: %s", err) + } + framework.Logf("CSI driver logs:\n%s", log) + // Find NodePublish in the logs + foundAttributes := sets.NewString() + logLines := strings.Split(log, "\n") + for _, line := range logLines { + if !strings.HasPrefix(line, "gRPCCall:") { + continue + } + line = strings.TrimPrefix(line, "gRPCCall:") + // Dummy structure that parses just volume_attributes out of logged CSI call + type MockCSICall struct { + Method string + Request struct { + VolumeContext map[string]string `json:"volume_context"` + } + } + var call MockCSICall + err := json.Unmarshal([]byte(line), &call) + if err != nil { + framework.Logf("Could not parse CSI driver log line %q: %s", line, err) + continue + } + if call.Method != "/csi.v1.Node/NodePublishVolume" { + continue + } + // Check that NodePublish had expected attributes + for k, v := range expectedAttributes { + vv, found := call.Request.VolumeContext[k] + if found && v == vv { + foundAttributes.Insert(k) + framework.Logf("Found volume attribute %s: %s", k, v) + } + } + // Process just the first NodePublish, the rest of the log is useless. + break + } + if expectPodInfo { + if foundAttributes.Len() != len(expectedAttributes) { + return fmt.Errorf("number of found volume attributes does not match, expected %d, got %d", len(expectedAttributes), foundAttributes.Len()) + } + return nil + } else { + if foundAttributes.Len() != 0 { + return fmt.Errorf("some unexpected volume attributes were found: %+v", foundAttributes.List()) + } + return nil + } +} diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 357ef2318ce..17e5cfc9a14 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -126,6 +126,7 @@ func (h *hostpathCSIDriver) CreateDriver() { OldDriverName: h.driverInfo.Name, NewDriverName: testsuites.GetUniqueDriverName(h), DriverContainerName: "hostpath", + DriverContainerArguments: []string{"--drivername=csi-hostpath-" + f.UniqueName}, ProvisionerContainerName: "csi-provisioner", NodeName: nodeName, } @@ -146,6 +147,98 @@ func (h *hostpathCSIDriver) CleanupDriver() { } } +// mockCSI +type mockCSIDriver struct { + cleanup func() + driverInfo testsuites.DriverInfo +} + +var _ testsuites.TestDriver = &mockCSIDriver{} +var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{} + +// InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface +func InitMockCSIDriver(config testsuites.TestConfig) testsuites.TestDriver { + return &mockCSIDriver{ + driverInfo: testsuites.DriverInfo{ + Name: "csi-mock", + FeatureTag: "", + MaxFileSize: testpatterns.FileSizeMedium, + SupportedFsType: sets.NewString( + "", // Default fsType + ), + Capabilities: map[testsuites.Capability]bool{ + testsuites.CapPersistence: false, + testsuites.CapFsGroup: false, + testsuites.CapExec: false, + }, + Config: config, + }, + } +} + +func (m *mockCSIDriver) GetDriverInfo() *testsuites.DriverInfo { + return &m.driverInfo +} + +func (m *mockCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) { +} + +func (m *mockCSIDriver) GetDynamicProvisionStorageClass(fsType string) *storagev1.StorageClass { + provisioner := testsuites.GetUniqueDriverName(m) + parameters := map[string]string{} + ns := m.driverInfo.Config.Framework.Namespace.Name + suffix := fmt.Sprintf("%s-sc", provisioner) + + return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix) +} + +func (m *mockCSIDriver) GetClaimSize() string { + return "5Gi" +} + +func (m *mockCSIDriver) CreateDriver() { + By("deploying csi mock driver") + f := m.driverInfo.Config.Framework + cs := f.ClientSet + + // pods should be scheduled on the node + nodes := framework.GetReadySchedulableNodesOrDie(cs) + node := nodes.Items[rand.Intn(len(nodes.Items))] + m.driverInfo.Config.ClientNodeName = node.Name + + // TODO (?): the storage.csi.image.version and storage.csi.image.registry + // settings are ignored for this test. We could patch the image definitions. + o := utils.PatchCSIOptions{ + OldDriverName: "csi-mock", + NewDriverName: "csi-mock-" + f.UniqueName, + DriverContainerName: "mock", + DriverContainerArguments: []string{"--name=csi-mock-" + f.UniqueName}, + ProvisionerContainerName: "csi-provisioner", + NodeName: m.driverInfo.Config.ClientNodeName, + } + cleanup, err := f.CreateFromManifests(func(item interface{}) error { + return utils.PatchCSIDeployment(f, o, item) + }, + "test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml", + "test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml", + "test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml", + "test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml", + ) + m.cleanup = cleanup + if err != nil { + framework.Failf("deploying csi mock driver: %v", err) + } +} + +func (m *mockCSIDriver) CleanupDriver() { + if m.cleanup != nil { + By("uninstalling csi mock driver") + m.cleanup() + } +} + // InitHostPathV0CSIDriver returns a variant of hostpathCSIDriver with different manifests. func InitHostPathV0CSIDriver(config testsuites.TestConfig) testsuites.TestDriver { return initHostPathCSIDriver("csi-hostpath-v0", config, diff --git a/test/e2e/storage/utils/deployment.go b/test/e2e/storage/utils/deployment.go index 49d0f2ea6eb..fa7ddd5995f 100644 --- a/test/e2e/storage/utils/deployment.go +++ b/test/e2e/storage/utils/deployment.go @@ -88,7 +88,7 @@ func PatchCSIDeployment(f *framework.Framework, o PatchCSIOptions, object interf // value. switch container.Name { case o.DriverContainerName: - container.Args = append(container.Args, "--drivername="+o.NewDriverName) + container.Args = append(container.Args, o.DriverContainerArguments...) case o.ProvisionerContainerName: // Driver name is expected to be the same // as the provisioner here. @@ -135,9 +135,12 @@ type PatchCSIOptions struct { // in existing fields). NewDriverName string // The name of the container which has the CSI driver binary. - // If non-empty, --drivername with the new name will be - // appended to the argument list. + // If non-empty, DriverContainerArguments are added to argument + // list in container with that name. DriverContainerName string + // List of arguments to add to container with + // DriverContainerName. + DriverContainerArguments []string // The name of the container which has the provisioner binary. // If non-empty, --provisioner with new name will be appended // to the argument list. diff --git a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml index 65bd2e42e97..e727ee2902e 100644 --- a/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml +++ b/test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml @@ -42,6 +42,12 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["get", "list"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml new file mode 100644 index 00000000000..1592ccda52d --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-driver.yaml @@ -0,0 +1,97 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-mockplugin +spec: + selector: + matchLabels: + app: csi-mockplugin + replicas: 1 + template: + metadata: + labels: + app: csi-mockplugin + spec: + serviceAccountName: csi-mock + containers: + + + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.0.1 + args: + - --v=5 + - --csi-address=$(ADDRESS) + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.0.1 + args: + - "--provisioner=csi-hostpath" + - "--csi-address=$(ADDRESS)" + - "--connection-timeout=15s" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + + + - name: driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2 + args: + - --v=5 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-mock/csi.sock + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + imagePullPolicy: Always + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /registration + name: registration-dir + + + - name: mock + image: quay.io/k8scsi/mock-driver:v1.0.0-1 + env: + - name: CSI_ENDPOINT + value: /csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-mock + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml new file mode 100644 index 00000000000..dd09a7dc44a --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-mock-rbac.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-mock + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-controller-attacher-role +subjects: + - kind: ServiceAccount + name: csi-mock + namespace: default +roleRef: + kind: ClusterRole + name: external-attacher-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-controller-provisioner-role +subjects: + - kind: ServiceAccount + name: csi-mock + namespace: default +roleRef: + kind: ClusterRole + name: external-provisioner-runner + apiGroup: rbac.authorization.k8s.io + +--- +# priviledged Pod Security Policy, previously defined via PrivilegedTestPSPClusterRoleBinding() +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-csi-controller-driver-registrar-role +subjects: + - kind: ServiceAccount + name: csi-mock + namespace: default +roleRef: + kind: ClusterRole + name: e2e-test-privileged-psp + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml b/test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml new file mode 100644 index 00000000000..89c47dccbec --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/mock/csi-storageclass.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: csi-mock-sc +provisioner: csi-mock +reclaimPolicy: Delete +volumeBindingMode: Immediate diff --git a/test/e2e/testing-manifests/storage-csi/mock/e2e-test-rbac.yaml b/test/e2e/testing-manifests/storage-csi/mock/e2e-test-rbac.yaml new file mode 100644 index 00000000000..0c4449272b8 --- /dev/null +++ b/test/e2e/testing-manifests/storage-csi/mock/e2e-test-rbac.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-csi-mock-role +subjects: + - kind: ServiceAccount + name: csi-driver-registrar + namespace: default +roleRef: + kind: ClusterRole + name: e2e-test-privileged-psp + apiGroup: rbac.authorization.k8s.io