Merge pull request #70193 from saad-ali/csiCRDAddon

Register CSI CRDs as addon

cluster/addons/storage-crds/OWNERS

@@ -0,0 +1,6 @@
+approvers:
+- saad-ali
+- jsafrane
+- msau42
+reviewers:
+- davidz627

cluster/addons/storage-crds/README.md

@@ -0,0 +1,13 @@
+# Kubernetes CSI CRDs
+
+The Kubernetes Container Storage Interface implementation defines some API objects as CRDs that Kubernetes components
+including the Attach/Detach controller depend on.
+
+If you are using CSI, it is recommended that you enable the relevant feature gates (e.g. `CSIDriverRegistry`, `CSINodeInfo`, etc.), and ensure the CRDs in this directory are installed.
+
+These objects and their CRDs are defined in `staging/src/k8s.io/csi-api/pkg/crd/manifests`, the source of truth.
+They are copied from that CRD manifest directory to this addon directory.
+A unit test in `staging/src/k8s.io/csi-api/pkg/crd` verifies that this (and any other) copies of the manifest outside of `staging/src/k8s.io/csi-api/pkg/crd/manifests` do not drift from that source of truth.
+If you need to make changes please make changes in the `staging/src/k8s.io/csi-api/pkg/crd/manifests` directory and then update this copy.
+
+For more information, see: https://kubernetes-csi.github.io/docs/

cluster/addons/storage-crds/csidriver.yaml

@@ -1,8 +1,9 @@
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
-  creationTimestamp: null
   name: csidrivers.csi.storage.k8s.io
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   group: csi.storage.k8s.io
   names:
@@ -25,9 +26,3 @@ spec:
                 information (like podName, podUID, etc.) during mount operations.
               type: string
   version: v1alpha1
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: null
-  storedVersions: null

cluster/addons/storage-crds/csinodeinfo.yaml

@@ -1,8 +1,9 @@
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
-  creationTimestamp: null
   name: csinodeinfos.csi.storage.k8s.io
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   group: csi.storage.k8s.io
   names:
@@ -29,9 +30,3 @@ spec:
                 type: array
           type: array
   version: v1alpha1
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: null
-  storedVersions: null

cluster/gce/config-default.sh

@@ -246,10 +246,6 @@ fi
 # Optional: customize runtime config
 RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
 
-if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then
-  RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-api/all=true}"
-fi
-
 # Optional: set feature gates
 FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"
 
@@ -260,6 +256,18 @@ if [[ ! -z "${NODE_ACCELERATORS}" ]]; then
     fi
 fi
 
+if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then
+  RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-api/all=true}"
+  if ! [[ "${KUBE_FEATURE_GATES:-}" =~ "CSIDriverRegistry" ]]; then
+    # If not explicitly specified, default to true.
+    FEATURE_GATES="${FEATURE_GATES},CSIDriverRegistry=true"
+  fi
+  if ! [[ "${KUBE_FEATURE_GATES:-}" =~ "CSINodeInfo" ]]; then
+    # If not explicitly specified, default to true.
+    FEATURE_GATES="${FEATURE_GATES},CSINodeInfo=true"
+  fi
+fi
+
 # Optional: Install cluster DNS.
 # Set CLUSTER_DNS_CORE_DNS to 'false' to install kube-dns instead of CoreDNS.
 CLUSTER_DNS_CORE_DNS="${CLUSTER_DNS_CORE_DNS:-true}"

cluster/gce/gci/configure-helper.sh

@@ -2567,6 +2567,9 @@ EOF
   if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
     setup-addon-manifests "addons" "storage-class/gce"
   fi
+  if [[ "${FEATURE_GATES:-}" =~ "CSIDriverRegistry=true" || "${FEATURE_GATES:-}" =~ "CSINodeInfo=true" ]]; then
+    setup-addon-manifests "addons" "storage-crds"
+  fi
   if [[ "${ENABLE_IP_MASQ_AGENT:-}" == "true" ]]; then
     setup-addon-manifests "addons" "ip-masq-agent"
   fi

staging/src/k8s.io/csi-api/Godeps/Godeps.json

@@ -298,18 +298,6 @@
 			"ImportPath": "k8s.io/api/storage/v1beta1",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
-		{
-			"ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions",
-			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-		},
-		{
-			"ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1",
-			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-		},
-		{
-			"ImportPath": "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme",
-			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/errors",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

staging/src/k8s.io/csi-api/pkg/crd/BUILD

@@ -6,25 +6,17 @@ go_library(
     importmap = "k8s.io/kubernetes/vendor/k8s.io/csi-api/pkg/crd",
     importpath = "k8s.io/csi-api/pkg/crd",
     visibility = ["//visibility:public"],
-    deps = [
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
-    ],
 )
 
 go_test(
     name = "go_default_test",
     srcs = ["crd_test.go"],
-    data = glob(["testdata/**"]),
-    embed = [":go_default_library"],
-    deps = [
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
-        "//vendor/github.com/ghodss/yaml:go_default_library",
+    data = [
+        "//cluster/addons:addon-srcs",
+        "//staging/src/k8s.io/csi-api/pkg/crd:manifests",
     ],
+    embed = [":go_default_library"],
+    deps = ["//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library"],
 )
 
 filegroup(
@@ -34,6 +26,14 @@ filegroup(
     visibility = ["//visibility:private"],
 )
 
+filegroup(
+    name = "csi-manifests",
+    srcs = [
+        "//staging/src/k8s.io/csi-api/pkg/crd:manifests",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 filegroup(
     name = "all-srcs",
     srcs = [":package-srcs"],

staging/src/k8s.io/csi-api/pkg/crd/crd.go

@@ -14,106 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package crd is only for running tests to verify the manifest files
+// in this package and the addons are in sync.
+// The manifest file is currently manually generated, in the future, we
+// should invest in tooling that will automatically generate the CRD
+// manifest from the CR schema.
 package crd
-
-import (
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
-	"reflect"
-)
-
-// NOTE: the CRD functions here and the associated unit tests are non-ideal temporary measures in
-// release 1.12 in order to aid manual CRD installation. This installation will be automated in
-// subsequent releases and as a result this package will be removed.
-
-// CSIDriverCRD returns the CustomResourceDefinition for CSIDriver object.
-func CSIDriverCRD() *apiextensionsv1beta1.CustomResourceDefinition {
-	return &apiextensionsv1beta1.CustomResourceDefinition{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName,
-		},
-		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   csiapiv1alpha1.GroupName,
-			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
-			Scope:   apiextensionsv1beta1.ClusterScoped,
-			Validation: &apiextensionsv1beta1.CustomResourceValidation{
-				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
-					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
-						"spec": {
-							Description: "Specification of the CSI Driver.",
-							Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
-								"attachRequired": {
-									Description: "Indicates this CSI volume driver requires an attach operation," +
-										" and that Kubernetes should call attach and wait for any attach operation to" +
-										" complete before proceeding to mount.",
-									Type: "boolean",
-								},
-								"podInfoOnMountVersion": {
-									Description: "Indicates this CSI volume driver requires additional pod" +
-										" information (like podName, podUID, etc.) during mount operations.",
-									Type: "string",
-								},
-							},
-						},
-					},
-				},
-			},
-			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
-				Plural: csiapiv1alpha1.CsiDriverResourcePlural,
-				Kind:   reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(),
-			},
-		},
-	}
-}
-
-// CSINodeInfoCRD returns the CustomResourceDefinition for CSINodeInfo object.
-func CSINodeInfoCRD() *apiextensionsv1beta1.CustomResourceDefinition {
-	return &apiextensionsv1beta1.CustomResourceDefinition{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName,
-		},
-		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   csiapiv1alpha1.GroupName,
-			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
-			Scope:   apiextensionsv1beta1.ClusterScoped,
-			Validation: &apiextensionsv1beta1.CustomResourceValidation{
-				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
-					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
-						"csiDrivers": {
-							Description: "List of CSI drivers running on the node and their properties.",
-							Type:        "array",
-							Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{
-								Schema: &apiextensionsv1beta1.JSONSchemaProps{
-									Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
-										"driver": {
-											Description: "The CSI driver that this object refers to.",
-											Type:        "string",
-										},
-										"nodeID": {
-											Description: "The node from the driver point of view.",
-											Type:        "string",
-										},
-										"topologyKeys": {
-											Description: "List of keys supported by the driver.",
-											Type:        "array",
-											Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{
-												Schema: &apiextensionsv1beta1.JSONSchemaProps{
-													Type: "string",
-												},
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
-				Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural,
-				Kind:   reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(),
-			},
-		},
-	}
-}

staging/src/k8s.io/csi-api/pkg/crd/crd_test.go

@@ -14,56 +14,69 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// These tests verify the manifest files in this package and the
+// addons directory are in sync.
 package crd_test
 
 import (
+	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 
-	"github.com/ghodss/yaml"
-	"io/ioutil"
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	apiextensionsscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/diff"
-	"k8s.io/csi-api/pkg/crd"
-	"os"
 )
 
 func TestBootstrapCRDs(t *testing.T) {
-	testObjects(t, crd.CSIDriverCRD(), "csidriver.yaml")
-	testObjects(t, crd.CSINodeInfoCRD(), "csinodeinfo.yaml")
+	verifyCopiesAreInSync(
+		t,
+		"csidriver.yaml", /* filename */
+		"manifests",      /* sourceOfTruthDir */
+		[]string{"../../../../../../cluster/addons/storage-crds"}, /* copyDirs */
+	)
+	verifyCopiesAreInSync(
+		t,
+		"csinodeinfo.yaml", /* filename */
+		"manifests",        /* sourceOfTruthDir */
+		[]string{"../../../../../../cluster/addons/storage-crds"}, /* copyDirs */
+	)
 }
 
-func testObjects(t *testing.T, crd *apiextensionsv1beta1.CustomResourceDefinition, fixtureFilename string) {
-	filename := filepath.Join("testdata", fixtureFilename)
-	expectedYAML, err := ioutil.ReadFile(filename)
+// verifyCopiesAreInSync fails if any copies are different from source of truth.
+func verifyCopiesAreInSync(t *testing.T, filename string, sourceOfTruthDir string, copyDirs []string) {
+	sourceOfTruthFilename := filepath.Join(sourceOfTruthDir, filename)
+
+	if len(copyDirs) <= 0 {
+		t.Fatalf("copyDirs is empty. There are no copies to validate.")
+	}
+
+	expectedYAML, err := ioutil.ReadFile(sourceOfTruthFilename)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	jsonData, err := runtime.Encode(apiextensionsscheme.Codecs.LegacyCodec(apiextensionsv1beta1.SchemeGroupVersion), crd)
-	if err != nil {
-		t.Fatal(err)
-	}
-	yamlData, err := yaml.JSONToYAML(jsonData)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if string(yamlData) != string(expectedYAML) {
-		t.Errorf("Bootstrap CRD data does not match the test fixture in %s", filename)
+	for _, copyDir := range copyDirs {
+		copyFilename := filepath.Join(copyDir, filename)
+		actualYAML, err := ioutil.ReadFile(copyFilename)
+		if err != nil {
+			t.Fatal(err)
+		}
 
-		const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA"
-		if os.Getenv(updateEnvVar) == "true" {
-			if err := ioutil.WriteFile(filename, []byte(yamlData), os.FileMode(0755)); err == nil {
-				t.Logf("Updated data in %s", filename)
-				t.Logf("Verify the diff, commit changes, and rerun the tests")
+		if string(actualYAML) != string(expectedYAML) {
+			t.Errorf("Data in %q does not match source of truth in %q.", copyFilename, sourceOfTruthFilename)
+
+			const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA"
+			if os.Getenv(updateEnvVar) == "true" {
+				if err := ioutil.WriteFile(copyFilename, []byte(expectedYAML), os.FileMode(0755)); err == nil {
+					t.Logf("Updated data in %s", copyFilename)
+					t.Logf("Verify the diff, commit changes, and rerun the tests")
+				} else {
+					t.Logf("Could not update data in %s: %v", copyFilename, err)
+				}
 			} else {
-				t.Logf("Could not update data in %s: %v", filename, err)
+				t.Logf("Diff between source of truth data and copy data in %s:\n-------------\n%s", copyFilename, diff.StringDiff(string(actualYAML), string(expectedYAML)))
+				t.Logf("If the change is expected, re-run with %s=true to update the copy data", updateEnvVar)
 			}
-		} else {
-			t.Logf("Diff between data in code and fixture data in %s:\n-------------\n%s", filename, diff.StringDiff(string(yamlData), string(expectedYAML)))
-			t.Logf("If the change is expected, re-run with %s=true to update the fixtures", updateEnvVar)
 		}
 	}
 }

staging/src/k8s.io/csi-api/pkg/crd/manifests/csidriver.yaml

@@ -0,0 +1,28 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: csidrivers.csi.storage.k8s.io
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  group: csi.storage.k8s.io
+  names:
+    kind: CSIDriver
+    plural: csidrivers
+  scope: Cluster
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          description: Specification of the CSI Driver.
+          properties:
+            attachRequired:
+              description: Indicates this CSI volume driver requires an attach operation,
+                and that Kubernetes should call attach and wait for any attach operation
+                to complete before proceeding to mount.
+              type: boolean
+            podInfoOnMountVersion:
+              description: Indicates this CSI volume driver requires additional pod
+                information (like podName, podUID, etc.) during mount operations.
+              type: string
+  version: v1alpha1

staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml

@@ -0,0 +1,32 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: csinodeinfos.csi.storage.k8s.io
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  group: csi.storage.k8s.io
+  names:
+    kind: CSINodeInfo
+    plural: csinodeinfos
+  scope: Cluster
+  validation:
+    openAPIV3Schema:
+      properties:
+        csiDrivers:
+          description: List of CSI drivers running on the node and their properties.
+          items:
+            properties:
+              driver:
+                description: The CSI driver that this object refers to.
+                type: string
+              nodeID:
+                description: The node from the driver point of view.
+                type: string
+              topologyKeys:
+                description: List of keys supported by the driver.
+                items:
+                  type: string
+                type: array
+          type: array
+  version: v1alpha1

test/e2e/storage/BUILD

@@ -45,8 +45,6 @@ go_library(
         "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -68,7 +66,6 @@ go_library(
         "//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
-        "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
         "//test/e2e/framework/podlogs:go_default_library",

test/e2e/storage/csi_objects.go

@@ -28,18 +28,11 @@ import (
 	"path/filepath"
 
 	"k8s.io/api/core/v1"
-	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
 
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
-
-	. "github.com/onsi/ginkgo"
-
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
-	csicrd "k8s.io/csi-api/pkg/crd"
 )
 
 var (
@@ -64,25 +57,6 @@ func csiContainerImage(image string) string {
 	return fullName
 }
 
-func createCSICRDs(c apiextensionsclient.Interface) {
-	By("Creating CSI CRDs")
-	crds := []*apiextensionsv1beta1.CustomResourceDefinition{
-		csicrd.CSIDriverCRD(),
-		csicrd.CSINodeInfoCRD(),
-	}
-
-	for _, crd := range crds {
-		_, err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
-		if err == nil {
-			continue
-		} else if !apierrs.IsNotFound(err) {
-			framework.ExpectNoError(err, "Failed to check for existing of CSI CRD %q: %v", crd.Name, err)
-		}
-		_, err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
-		framework.ExpectNoError(err, "Failed to create CSI CRD %q: %v", crd.Name, err)
-	}
-}
-
 func shredFile(filePath string) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		framework.Logf("File %v was not found, skipping shredding", filePath)

test/e2e/storage/csi_volumes.go

@@ -25,7 +25,6 @@ import (
 
 	"k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
@@ -60,13 +59,12 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 	f := framework.NewDefaultFramework("csi-volumes")
 
 	var (
-		cancel    context.CancelFunc
-		cs        clientset.Interface
-		crdclient apiextensionsclient.Interface
-		csics     csiclient.Interface
-		ns        *v1.Namespace
-		node      v1.Node
-		config    framework.VolumeTestConfig
+		cancel context.CancelFunc
+		cs     clientset.Interface
+		csics  csiclient.Interface
+		ns     *v1.Namespace
+		node   v1.Node
+		config framework.VolumeTestConfig
 	)
 
 	BeforeEach(func() {
@@ -74,7 +72,6 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 		cancel = c
 
 		cs = f.ClientSet
-		crdclient = f.APIExtensionsClientSet
 		csics = f.CSIClientSet
 		ns = f.Namespace
 
@@ -114,7 +111,6 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 			ServerNodeName:    node.Name,
 			WaitForCompletion: true,
 		}
-		createCSICRDs(crdclient)
 	})
 
 	AfterEach(func() {

test/integration/auth/BUILD

@@ -17,6 +17,9 @@ go_test(
         "rbac_test.go",
         "svcaccttoken_test.go",
     ],
+    data = [
+        "//staging/src/k8s.io/csi-api/pkg/crd:csi-manifests",
+    ],
     tags = ["integration"],
     deps = [
         "//cmd/kube-apiserver/app/testing:go_default_library",
@@ -49,11 +52,13 @@ go_test(
         "//staging/src/k8s.io/api/authentication/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
@@ -81,7 +86,6 @@ go_test(
         "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
-        "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library",
         "//test/e2e/lifecycle/bootstrap:go_default_library",
         "//test/integration:go_default_library",
         "//test/integration/etcd:go_default_library",

test/integration/auth/node_test.go

@@ -24,9 +24,11 @@ import (
 	"time"
 
 	storagev1beta1 "k8s.io/api/storage/v1beta1"
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -34,8 +36,8 @@ import (
 	externalclientset "k8s.io/client-go/kubernetes"
 	csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
 	csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned"
-	csicrd "k8s.io/csi-api/pkg/crd"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/apis/coordination"
 	"k8s.io/kubernetes/pkg/apis/core"
 	api "k8s.io/kubernetes/pkg/apis/core"
@@ -156,7 +158,12 @@ func TestNodeAuthorizer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	etcd.CreateTestCRDs(t, superuserCRDClient, false, csicrd.CSINodeInfoCRD())
+	csiNodeInfoCRD, err := crdFromManifest("../../../staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	etcd.CreateTestCRDs(t, superuserCRDClient, false, csiNodeInfoCRD)
 
 	getSecret := func(client clientset.Interface) func() error {
 		return func() error {
@@ -664,3 +671,17 @@ func expectAllowed(t *testing.T, f func() error) {
 		t.Errorf("Expected no error, got %v", err)
 	}
 }
+
+// crdFromManifest reads a .json/yaml file and returns the CRD in it.
+func crdFromManifest(filename string) (*apiextensionsv1beta1.CustomResourceDefinition, error) {
+	var crd apiextensionsv1beta1.CustomResourceDefinition
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), data, &crd); err != nil {
+		return nil, err
+	}
+	return &crd, nil
+}