From f33f850ff1d53c2bf70376afaf90ac221382425f Mon Sep 17 00:00:00 2001 From: David Zhu Date: Thu, 20 Sep 2018 13:23:34 -0700 Subject: [PATCH] Update GCE PD CSI Driver to run by default and automatically set up secrets with GCP Service Account Key --- test/e2e/storage/csi_objects.go | 74 ++++++++++++++++++- test/e2e/storage/csi_volumes.go | 14 ++-- .../storage-csi/gce-pd/controller_ss.yaml | 3 +- .../storage-csi/gce-pd/node_ds.yaml | 3 +- 4 files changed, 81 insertions(+), 13 deletions(-) diff --git a/test/e2e/storage/csi_objects.go b/test/e2e/storage/csi_objects.go index 4239d7cdcea..4b719517d7f 100644 --- a/test/e2e/storage/csi_objects.go +++ b/test/e2e/storage/csi_objects.go @@ -21,12 +21,17 @@ package storage import ( "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" "time" "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" @@ -121,7 +126,8 @@ func csiServiceAccount( serviceAccountClient := client.CoreV1().ServiceAccounts(config.Namespace) sa := &v1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceAccountName, + Name: serviceAccountName, + Namespace: config.Namespace, }, } @@ -158,14 +164,13 @@ func csiClusterRoleBindings( By(fmt.Sprintf("%v cluster roles %v to the CSI service account %v", bindingString, clusterRolesNames, sa.GetName())) clusterRoleBindingClient := client.RbacV1().ClusterRoleBindings() for _, clusterRoleName := range clusterRolesNames { - binding := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: config.Prefix + "-" + clusterRoleName + "-" + config.Namespace + "-role-binding", + Name: clusterRoleName + "-" + config.Namespace + "-" + string(uuid.NewUUID()), }, Subjects: []rbacv1.Subject{ { - Kind: "ServiceAccount", + Kind: rbacv1.ServiceAccountKind, Name: sa.GetName(), Namespace: sa.GetNamespace(), }, @@ -454,3 +459,64 @@ func deleteCSICRDs(c apiextensionsclient.Interface) { err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiNodeInfoCRDName, &metav1.DeleteOptions{}) framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiNodeInfoCRDName, err) } + +func shredFile(filePath string) { + if _, err := os.Stat(filePath); os.IsNotExist(err) { + framework.Logf("File %v was not found, skipping shredding", filePath) + return + } + framework.Logf("Shredding file %v", filePath) + _, _, err := framework.RunCmd("shred", "--remove", filePath) + if err != nil { + framework.Logf("Failed to shred file %v: %v", filePath, err) + } + if _, err := os.Stat(filePath); os.IsNotExist(err) { + framework.Logf("File %v successfully shredded", filePath) + return + } + // Shred failed Try to remove the file for good meausure + err = os.Remove(filePath) + framework.ExpectNoError(err, "Failed to remove service account file %s", filePath) + +} + +// createGCESecrets downloads the GCP IAM Key for the default compute service account +// and puts it in a secret for the GCE PD CSI Driver to consume +func createGCESecrets(client clientset.Interface, config framework.VolumeTestConfig) { + saEnv := "E2E_GOOGLE_APPLICATION_CREDENTIALS" + saFile := fmt.Sprintf("/tmp/%s/cloud-sa.json", string(uuid.NewUUID())) + + os.MkdirAll(path.Dir(saFile), 0750) + defer os.Remove(path.Dir(saFile)) + + premadeSAFile, ok := os.LookupEnv(saEnv) + if !ok { + framework.Logf("Could not find env var %v, please either create cloud-sa"+ + " secret manually or rerun test after setting %v to the filepath of"+ + " the GCP Service Account to give to the GCE Persistent Disk CSI Driver", saEnv, saEnv) + return + } + + framework.Logf("Found CI service account key at %v", premadeSAFile) + // Need to copy it saFile + stdout, stderr, err := framework.RunCmd("cp", premadeSAFile, saFile) + framework.ExpectNoError(err, "error copying service account key: %s\nstdout: %s\nstderr: %s", err, stdout, stderr) + defer shredFile(saFile) + // Create Secret with this Service Account + fileBytes, err := ioutil.ReadFile(saFile) + framework.ExpectNoError(err, "Failed to read file %v", saFile) + + s := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-sa", + Namespace: config.Namespace, + }, + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{ + filepath.Base(saFile): fileBytes, + }, + } + + _, err = client.CoreV1().Secrets(config.Namespace).Create(s) + framework.ExpectNoError(err, "Failed to create Secret %v", s.GetName()) +} diff --git a/test/e2e/storage/csi_volumes.go b/test/e2e/storage/csi_volumes.go index 20ac056e93f..4c3d428dbb8 100644 --- a/test/e2e/storage/csi_volumes.go +++ b/test/e2e/storage/csi_volumes.go @@ -55,8 +55,7 @@ type csiTestDriver interface { var csiTestDrivers = map[string]func(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver{ "hostPath": initCSIHostpath, - // Feature tag to skip test in CI, pending fix of #62237 - "[Feature: GCE PD CSI Plugin] gcePD": initCSIgcePD, + "gcePD": initCSIgcePD, } var _ = utils.SIGDescribe("[Serial] CSI Volumes", func() { @@ -361,9 +360,10 @@ type gcePDCSIDriver struct { func initCSIgcePD(f *framework.Framework, config framework.VolumeTestConfig) csiTestDriver { cs := f.ClientSet framework.SkipUnlessProviderIs("gce", "gke") - // Currently you will need to manually add the required GCP Credentials as a secret "cloud-sa" - // kubectl create generic cloud-sa --from-file=PATH/TO/cloud-sa.json --namespace={{config.Namespace}} - // TODO(#62561): Inject the necessary credentials automatically to the driver containers in e2e test + + // TODO(#62561): Use credentials through external pod identity when that goes GA instead of downloading keys. + createGCESecrets(cs, config) + framework.SkipUnlessSecretExistsAfterWait(cs, "cloud-sa", config.Namespace, 3*time.Minute) return &gcePDCSIDriver{ @@ -402,6 +402,8 @@ func (g *gcePDCSIDriver) createCSIDriver() { g.nodeServiceAccount = csiServiceAccount(cs, config, "gce-node", false /* teardown */) csiClusterRoleBindings(cs, config, false /* teardown */, g.controllerServiceAccount, g.controllerClusterRoles) csiClusterRoleBindings(cs, config, false /* teardown */, g.nodeServiceAccount, g.nodeClusterRoles) + utils.PrivilegedTestPSPClusterRoleBinding(cs, config.Namespace, + false /* teardown */, []string{g.controllerServiceAccount.Name, g.nodeServiceAccount.Name}) deployGCEPDCSIDriver(cs, config, false /* teardown */, f, g.nodeServiceAccount, g.controllerServiceAccount) } @@ -413,6 +415,8 @@ func (g *gcePDCSIDriver) cleanupCSIDriver() { deployGCEPDCSIDriver(cs, config, true /* teardown */, f, g.nodeServiceAccount, g.controllerServiceAccount) csiClusterRoleBindings(cs, config, true /* teardown */, g.controllerServiceAccount, g.controllerClusterRoles) csiClusterRoleBindings(cs, config, true /* teardown */, g.nodeServiceAccount, g.nodeClusterRoles) + utils.PrivilegedTestPSPClusterRoleBinding(cs, config.Namespace, + true /* teardown */, []string{g.controllerServiceAccount.Name, g.nodeServiceAccount.Name}) csiServiceAccount(cs, config, "gce-controller", true /* teardown */) csiServiceAccount(cs, config, "gce-node", true /* teardown */) } diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml index 76f1ca9b723..36a43ce4b80 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml @@ -13,7 +13,6 @@ spec: labels: app: csi-gce-pd-driver spec: - serviceAccount: csi-gce-pd containers: - name: csi-external-provisioner imagePullPolicy: Always @@ -42,7 +41,7 @@ spec: mountPath: /csi - name: gce-driver imagePullPolicy: Always - image: gcr.io/google-containers/volume-csi/compute-persistent-disk-csi-driver:v0.2.0.alpha + image: gcr.io/google-containers/volume-csi/gcp-compute-persistent-disk-csi-driver:v0.1.0.alpha args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" diff --git a/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml b/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml index 8bfa28bbd0b..85be59f7a11 100644 --- a/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml +++ b/test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml @@ -12,7 +12,6 @@ spec: labels: app: csi-gce-driver spec: - serviceAccount: csi-gce-pd containers: - name: csi-driver-registrar imagePullPolicy: Always @@ -39,7 +38,7 @@ spec: securityContext: privileged: true imagePullPolicy: Always - image: gcr.io/google-containers/volume-csi/compute-persistent-disk-csi-driver:v0.2.0.alpha + image: gcr.io/google-containers/volume-csi/gcp-compute-persistent-disk-csi-driver:v0.1.0.alpha args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)"