Add GCE-PD CSI Driver test to E2E test suite

test/e2e/framework/util.go

@@ -376,6 +376,22 @@ func SkipUnlessNodeOSDistroIs(supportedNodeOsDistros ...string) {
 	}
 }
 
+func SkipUnlessSecretExistsAfterWait(c clientset.Interface, name, namespace string, timeout time.Duration) {
+	Logf("Waiting for secret %v in namespace %v to exist in duration %v", name, namespace, timeout)
+	start := time.Now()
+	if wait.PollImmediate(15*time.Second, timeout, func() (bool, error) {
+		_, err := c.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{})
+		if err != nil {
+			Logf("Secret %v in namespace %v still does not exist after duration %v", name, namespace, time.Since(start))
+			return false, nil
+		}
+		return true, nil
+	}) != nil {
+		Skipf("Secret %v in namespace %v did not exist after timeout of %v", name, namespace, timeout)
+	}
+	Logf("Secret %v in namespace %v found after duration %v", name, namespace, time.Since(start))
+}
+
 func SkipIfContainerRuntimeIs(runtimes ...string) {
 	for _, runtime := range runtimes {
 		if runtime == TestContext.ContainerRuntime {

test/e2e/manifest/manifest.go

@@ -125,3 +125,20 @@ func StatefulSetFromManifest(fileName, ns string) (*apps.StatefulSet, error) {
 	}
 	return &ss, nil
 }
+
+// DaemonSetFromManifest returns a DaemonSet from a manifest stored in fileName in the Namespace indicated by ns.
+func DaemonSetFromManifest(fileName, ns string) (*apps.DaemonSet, error) {
+	var ds apps.DaemonSet
+	data := generated.ReadOrDie(fileName)
+
+	json, err := utilyaml.ToJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	err = runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &ds)
+	if err != nil {
+		return nil, err
+	}
+	ds.Namespace = ns
+	return &ds, nil
+}

test/e2e/storage/BUILD

@@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 go_library(
     name = "go_default_library",
     srcs = [
-        "csi_hostpath.go",
+        "csi_defs.go",
         "csi_volumes.go",
         "empty_dir_wrapper.go",
         "flexvolume.go",
@@ -39,6 +39,7 @@ go_library(
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
         "//test/e2e/generated:go_default_library",
+        "//test/e2e/manifest:go_default_library",
         "//test/e2e/storage/utils:go_default_library",
         "//test/e2e/storage/vsphere:go_default_library",
         "//test/utils/image:go_default_library",

test/e2e/storage/csi_defs.go

@@ -21,10 +21,12 @@ package storage
 
 import (
 	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/manifest"
 )
 
 const (
@@ -197,3 +199,59 @@ func csiHostPathPod(
 	framework.ExpectNoError(framework.WaitForPodRunningInNamespace(client, ret))
 	return ret
 }
+
+func csiGCEPDSetup(
+	client clientset.Interface,
+	config framework.VolumeTestConfig,
+	teardown bool,
+	f *framework.Framework,
+	nodeSA *v1.ServiceAccount,
+	controllerSA *v1.ServiceAccount,
+) {
+	// Get API Objects from manifests
+	nodeds, err := manifest.DaemonSetFromManifest("test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml", config.Namespace)
+	framework.ExpectNoError(err, "Failed to create DaemonSet from manifest")
+	nodeds.Spec.Template.Spec.ServiceAccountName = nodeSA.GetName()
+
+	controllerss, err := manifest.StatefulSetFromManifest("test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml", config.Namespace)
+	framework.ExpectNoError(err, "Failed to create StatefulSet from manifest")
+	controllerss.Spec.Template.Spec.ServiceAccountName = controllerSA.GetName()
+
+	controllerservice, err := manifest.SvcFromManifest("test/e2e/testing-manifests/storage-csi/gce-pd/controller_service.yaml")
+	framework.ExpectNoError(err, "Failed to create Service from manifest")
+
+	// Got all objects from manifests now try to delete objects
+	err = client.CoreV1().Services(config.Namespace).Delete(controllerservice.GetName(), nil)
+	if err != nil {
+		if !apierrs.IsNotFound(err) {
+			framework.ExpectNoError(err, "Failed to delete Service: %v", controllerservice.GetName())
+		}
+	}
+
+	err = client.AppsV1().StatefulSets(config.Namespace).Delete(controllerss.Name, nil)
+	if err != nil {
+		if !apierrs.IsNotFound(err) {
+			framework.ExpectNoError(err, "Failed to delete StatefulSet: %v", controllerss.GetName())
+		}
+	}
+	err = client.AppsV1().DaemonSets(config.Namespace).Delete(nodeds.Name, nil)
+	if err != nil {
+		if !apierrs.IsNotFound(err) {
+			framework.ExpectNoError(err, "Failed to delete DaemonSet: %v", nodeds.GetName())
+		}
+	}
+	if teardown {
+		return
+	}
+
+	// Create new API Objects through client
+	_, err = client.CoreV1().Services(config.Namespace).Create(controllerservice)
+	framework.ExpectNoError(err, "Failed to create Service: %v", controllerservice.Name)
+
+	_, err = client.AppsV1().StatefulSets(config.Namespace).Create(controllerss)
+	framework.ExpectNoError(err, "Failed to create StatefulSet: %v", controllerss.Name)
+
+	_, err = client.AppsV1().DaemonSets(config.Namespace).Create(nodeds)
+	framework.ExpectNoError(err, "Failed to create DaemonSet: %v", nodeds.Name)
+
+}

test/e2e/storage/csi_volumes.go

@@ -17,6 +17,7 @@ limitations under the License.
 package storage
 
 import (
+	"fmt"
 	"math/rand"
 	"time"
 
@@ -28,10 +29,12 @@ import (
 
 	clientset "k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
 
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
 const (
@@ -95,7 +98,7 @@ func csiServiceAccount(
 	componentName string,
 	teardown bool,
 ) *v1.ServiceAccount {
-	By("Creating a CSI service account")
+	By(fmt.Sprintf("Creating a CSI service account for %v", componentName))
 	serviceAccountName := config.Prefix + "-" + componentName + "-service-account"
 	serviceAccountClient := client.CoreV1().ServiceAccounts(config.Namespace)
 	sa := &v1.ServiceAccount{
@@ -130,7 +133,7 @@ func csiClusterRoleBindings(
 	sa *v1.ServiceAccount,
 	clusterRolesNames []string,
 ) {
-	By("Binding cluster roles to the CSI service account")
+	By(fmt.Sprintf("Binding cluster roles %v to the CSI service account %v", clusterRolesNames, sa.GetName()))
 	clusterRoleBindingClient := client.RbacV1().ClusterRoleBindings()
 	for _, clusterRoleName := range clusterRolesNames {
 
@@ -237,4 +240,61 @@ var _ = utils.SIGDescribe("CSI Volumes [Flaky]", func() {
 			testDynamicProvisioning(t, cs, claim, class)
 		})
 	})
+
+	Describe("[Feature: CSI] Sanity CSI plugin test using GCE-PD CSI driver", func() {
+		var (
+			controllerClusterRoles []string = []string{
+				csiExternalAttacherClusterRoleName,
+				csiExternalProvisionerClusterRoleName,
+			}
+			nodeClusterRoles []string = []string{
+				csiDriverRegistrarClusterRoleName,
+			}
+			controllerServiceAccount *v1.ServiceAccount
+			nodeServiceAccount       *v1.ServiceAccount
+		)
+
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce", "gke")
+			// Currently you will need to manually add the required GCP Credentials as a secret "cloud-sa"
+			// kubectl create generic cloud-sa --from-file=PATH/TO/cloud-sa.json --namespace={{config.Namespace}}
+			// TODO(GITHUBISSUE): Inject the necessary credentials automatically to the driver containers in e2e test
+			framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute)
+
+			By("deploying gce-pd driver")
+			controllerServiceAccount = csiServiceAccount(cs, config, "gce-controller", false)
+			nodeServiceAccount = csiServiceAccount(cs, config, "gce-node", false)
+			csiClusterRoleBindings(cs, config, false, controllerServiceAccount, controllerClusterRoles)
+			csiClusterRoleBindings(cs, config, false, nodeServiceAccount, nodeClusterRoles)
+			csiGCEPDSetup(cs, config, false, f, nodeServiceAccount, controllerServiceAccount)
+		})
+
+		AfterEach(func() {
+			By("uninstalling gce-pd driver")
+			csiGCEPDSetup(cs, config, true, f, nodeServiceAccount, controllerServiceAccount)
+			csiClusterRoleBindings(cs, config, true, controllerServiceAccount, controllerClusterRoles)
+			csiClusterRoleBindings(cs, config, true, nodeServiceAccount, nodeClusterRoles)
+			csiServiceAccount(cs, config, "gce-controller", true)
+			csiServiceAccount(cs, config, "gce-node", true)
+		})
+
+		It("should provision storage with a GCE-PD CSI driver", func() {
+			nodeZone, ok := node.GetLabels()[kubeletapis.LabelZoneFailureDomain]
+			Expect(ok).To(BeTrue(), "Could not get label %v from node %v", kubeletapis.LabelZoneFailureDomain, node.GetName())
+			t := storageClassTest{
+				name:         "csi-gce-pd",
+				provisioner:  "csi-gce-pd",
+				parameters:   map[string]string{"type": "pd-standard", "zone": nodeZone},
+				claimSize:    "5Gi",
+				expectedSize: "5Gi",
+				nodeName:     node.Name,
+			}
+
+			claim := newClaim(t, ns.GetName(), "")
+			class := newStorageClass(t, ns.GetName(), "")
+			claim.Spec.StorageClassName = &class.ObjectMeta.Name
+			testDynamicProvisioning(t, cs, claim, class)
+		})
+
+	})
 })

test/e2e/testing-manifests/storage-csi/gce-pd/controller_service.yaml

@@ -0,0 +1,13 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-gce-pd
+  labels:
+    app: csi-gce-pd
+spec:
+  selector:
+    app: csi-gce-pd
+  ports:
+    - name: dummy
+      port: 12345
+ 

test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml

@@ -0,0 +1,75 @@
+kind: StatefulSet
+apiVersion: apps/v1beta1
+metadata:
+  name: csi-gce-controller
+spec:
+  serviceName: "csi-gce-pd"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-gce-pd-driver
+  template:
+    metadata:
+      labels:
+        app: csi-gce-pd-driver
+    spec:
+      serviceAccount: csi-gce-pd
+      containers:
+        - name: csi-external-provisioner
+          imagePullPolicy: Always
+          image: quay.io/k8scsi/csi-provisioner:v0.2.0
+          args:
+            - "--v=5"
+            - "--provisioner=csi-gce-pd"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-attacher
+          imagePullPolicy: Always
+          image: quay.io/k8scsi/csi-attacher:v0.2.0
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: gce-driver
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          imagePullPolicy: Always
+          image: gcr.io/google-containers/volume-csi/compute-persistent-disk-csi-driver:v0.2.0.alpha
+          args:
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodeid=$(KUBE_NODE_NAME)"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: "/etc/service-account/cloud-sa.json"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: cloud-sa-volume
+              readOnly: true
+              mountPath: "/etc/service-account"
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: cloud-sa-volume
+          secret:
+            secretName: cloud-sa

test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml

@@ -0,0 +1,72 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-gce-node
+spec:
+  selector:
+    matchLabels:
+      app: csi-gce-driver
+  serviceName: csi-gce
+  template:
+    metadata:
+      labels:
+        app: csi-gce-driver
+    spec:
+      serviceAccount: csi-gce-pd
+      containers:
+        - name: csi-driver-registrar
+          imagePullPolicy: Always
+          image: quay.io/k8scsi/driver-registrar:v0.2.0
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+        - name: gce-driver
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          imagePullPolicy: Always
+          image: gcr.io/google-containers/volume-csi/compute-persistent-disk-csi-driver:v0.2.0.alpha
+          args:
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--nodeid=$(KUBE_NODE_NAME)"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: device-dir
+              mountPath: /host/dev
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/com.google.csi.gcepd/
+            type: DirectoryOrCreate
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory