diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index 7fbb43e3716..8126c0f8890 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -106,6 +106,7 @@ go_library( "//pkg/volume/util:go_default_library", "//pkg/volume/vsphere_volume:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", diff --git a/cmd/kube-controller-manager/app/core.go b/cmd/kube-controller-manager/app/core.go index c7a93e0e3b1..ce8157de074 100644 --- a/cmd/kube-controller-manager/app/core.go +++ b/cmd/kube-controller-manager/app/core.go @@ -31,6 +31,7 @@ import ( "net/http" "k8s.io/api/core/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/runtime/schema" utilfeature "k8s.io/apiserver/pkg/util/feature" cacheddiscovery "k8s.io/client-go/discovery/cached" @@ -197,10 +198,13 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err // csiClient works with CRDs that support json only csiClientConfig.ContentType = "application/json" + crdClientConfig := ctx.ClientBuilder.ConfigOrDie("attachdetach-controller") + attachDetachController, attachDetachControllerErr := attachdetach.NewAttachDetachController( ctx.ClientBuilder.ClientOrDie("attachdetach-controller"), csiclientset.NewForConfigOrDie(csiClientConfig), + apiextensionsclient.NewForConfigOrDie(crdClientConfig), ctx.InformerFactory.Core().V1().Pods(), ctx.InformerFactory.Core().V1().Nodes(), ctx.InformerFactory.Core().V1().PersistentVolumeClaims(), diff --git a/pkg/controller/volume/attachdetach/BUILD b/pkg/controller/volume/attachdetach/BUILD index b3e919746bf..13da43f203d 100644 --- a/pkg/controller/volume/attachdetach/BUILD +++ b/pkg/controller/volume/attachdetach/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/controller/volume/attachdetach/reconciler:go_default_library", "//pkg/controller/volume/attachdetach/statusupdater:go_default_library", "//pkg/controller/volume/attachdetach/util:go_default_library", + "//pkg/features:go_default_library", "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/util:go_default_library", @@ -26,11 +27,15 @@ go_library( "//pkg/volume/util/volumepathhandler:go_default_library", "//staging/src/k8s.io/api/authentication/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//staging/src/k8s.io/client-go/informers/core/v1:go_default_library", "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", @@ -39,6 +44,7 @@ go_library( "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/record:go_default_library", "//staging/src/k8s.io/client-go/util/workqueue:go_default_library", + "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library", "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library", "//vendor/github.com/golang/glog:go_default_library", ], @@ -54,6 +60,7 @@ go_test( "//pkg/controller/volume/attachdetach/testing:go_default_library", "//pkg/volume:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index 774bcc6fb9f..b0b941008ad 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -21,16 +21,21 @@ package attachdetach import ( "fmt" "net" + "reflect" "time" "github.com/golang/glog" authenticationv1 "k8s.io/api/authentication/v1" "k8s.io/api/core/v1" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -39,7 +44,8 @@ import ( kcache "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" - csiclientset "k8s.io/csi-api/pkg/client/clientset/versioned" + csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1" + csiclient "k8s.io/csi-api/pkg/client/clientset/versioned" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" @@ -48,6 +54,7 @@ import ( "k8s.io/kubernetes/pkg/controller/volume/attachdetach/reconciler" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/statusupdater" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/util" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" volumeutil "k8s.io/kubernetes/pkg/volume/util" @@ -97,7 +104,8 @@ type AttachDetachController interface { // NewAttachDetachController returns a new instance of AttachDetachController. func NewAttachDetachController( kubeClient clientset.Interface, - csiClient csiclientset.Interface, + csiClient csiclient.Interface, + crdClient apiextensionsclient.Interface, podInformer coreinformers.PodInformer, nodeInformer coreinformers.NodeInformer, pvcInformer coreinformers.PersistentVolumeClaimInformer, @@ -125,6 +133,7 @@ func NewAttachDetachController( adc := &attachDetachController{ kubeClient: kubeClient, csiClient: csiClient, + crdClient: crdClient, pvcLister: pvcInformer.Lister(), pvcsSynced: pvcInformer.Informer().HasSynced, pvLister: pvInformer.Lister(), @@ -138,6 +147,11 @@ func NewAttachDetachController( pvcQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pvcs"), } + // Install required CSI CRDs on API server + if utilfeature.DefaultFeatureGate.Enabled(features.CSICrdAutoInstall) { + adc.installCRDs() + } + if err := adc.volumePluginMgr.InitPlugins(plugins, prober, adc); err != nil { return nil, fmt.Errorf("Could not initialize volume plugins for Attach/Detach Controller: %+v", err) } @@ -240,9 +254,13 @@ type attachDetachController struct { // the API server. kubeClient clientset.Interface - // csiClient is the csi.storage.k8s.io API client used by volumehost to communicate with - // the API server. - csiClient csiclientset.Interface + // csiClient is the client used to read/write csi.storage.k8s.io API objects + // from the API server. + csiClient csiclient.Interface + + // crdClient is the client used to read/write apiextensions.k8s.io objects + // from the API server. + crdClient apiextensionsclient.Interface // pvcLister is the shared PVC lister used to fetch and store PVC // objects from the API server. It is shared with other controllers and @@ -649,6 +667,65 @@ func (adc *attachDetachController) processVolumesInUse( } } +// installCRDs creates the specified CustomResourceDefinition for the CSIDrivers object. +func (adc *attachDetachController) installCRDs() error { + crd := &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName, + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: csiapiv1alpha1.GroupName, + Version: csiapiv1alpha1.SchemeGroupVersion.Version, + Scope: apiextensionsv1beta1.ClusterScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: csiapiv1alpha1.CsiDriverResourcePlural, + Kind: reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(), + }, + }, + } + res, err := adc.crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) + + if err != nil && !apierrors.IsAlreadyExists(err) { + glog.Errorf("failed to create CSIDrivers CRD: %#v, err: %#v", + res, err) + } else if apierrors.IsAlreadyExists(err) { + glog.Warningf("CSIDrivers CRD already exists: %#v, err: %#v", + res, err) + } else { + glog.Infof("CSIDrivers CRD created successfully: %#v", + res) + } + + crd = &apiextensionsv1beta1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName, + }, + Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ + Group: csiapiv1alpha1.GroupName, + Version: csiapiv1alpha1.SchemeGroupVersion.Version, + Scope: apiextensionsv1beta1.ClusterScoped, + Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural, + Kind: reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(), + }, + }, + } + res, err = adc.crdClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd) + + if err != nil && !apierrors.IsAlreadyExists(err) { + glog.Errorf("failed to create CSINodeInfo CRD: %#v, err: %#v", + res, err) + } else if apierrors.IsAlreadyExists(err) { + glog.Warningf("CSINodeInfo CRD already exists: %#v, err: %#v", + res, err) + } else { + glog.Infof("CSINodeInfo CRD created successfully: %#v", + res) + } + + return nil +} + // VolumeHost implementation // This is an unfortunate requirement of the current factoring of volume plugin // initializing code. It requires kubelet specific methods used by the mounting @@ -759,6 +836,6 @@ func (adc *attachDetachController) GetEventRecorder() record.EventRecorder { return adc.recorder } -func (adc *attachDetachController) GetCSIClient() csiclientset.Interface { +func (adc *attachDetachController) GetCSIClient() csiclient.Interface { return adc.csiClient } diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go index 84cdd95c8c9..1ded525fd8f 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/api/core/v1" + fakeapiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -36,11 +37,13 @@ func Test_NewAttachDetachController_Positive(t *testing.T) { // Arrange fakeKubeClient := controllervolumetesting.CreateTestClient() informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, controller.NoResyncPeriodFunc()) + fakeApiExtensionsClient := fakeapiextensionsclient.NewSimpleClientset() // Act _, err := NewAttachDetachController( fakeKubeClient, nil, /* csiClient */ + fakeApiExtensionsClient, /* crdClient */ informerFactory.Core().V1().Pods(), informerFactory.Core().V1().Nodes(), informerFactory.Core().V1().PersistentVolumeClaims(), @@ -146,6 +149,7 @@ func Test_AttachDetachControllerRecovery(t *testing.T) { func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 []*v1.Pod) { fakeKubeClient := controllervolumetesting.CreateTestClient() + fakeApiExtensionsClient := fakeapiextensionsclient.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1) //informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1) plugins := controllervolumetesting.CreateTestPlugin() @@ -215,7 +219,8 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 // Create the controller adcObj, err := NewAttachDetachController( fakeKubeClient, - nil, + nil, /* csiClient */ + fakeApiExtensionsClient, /* crdClient */ informerFactory.Core().V1().Pods(), informerFactory.Core().V1().Nodes(), informerFactory.Core().V1().PersistentVolumeClaims(), diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index a570a9f6b4e..e48a2b5c057 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -194,11 +194,17 @@ const ( VolumeScheduling utilfeature.Feature = "VolumeScheduling" // owner: @vladimirvivien - // alpha: v1.9 + // beta: v1.10 // // Enable mount/attachment of Container Storage Interface (CSI) backed PVs CSIPersistentVolume utilfeature.Feature = "CSIPersistentVolume" + // owner: @saad-ali + // alpha: v1.12 + // + // Enable automatic installation of CRD for csi.storage.k8s.io API objects. + CSICrdAutoInstall utilfeature.Feature = "CSICrdAutoInstall" + // owner @MrHohn // beta: v1.10 // @@ -400,6 +406,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS MountContainers: {Default: false, PreRelease: utilfeature.Alpha}, VolumeScheduling: {Default: true, PreRelease: utilfeature.Beta}, CSIPersistentVolume: {Default: true, PreRelease: utilfeature.Beta}, + CSICrdAutoInstall: {Default: false, PreRelease: utilfeature.Alpha}, CustomPodDNS: {Default: true, PreRelease: utilfeature.Beta}, BlockVolume: {Default: false, PreRelease: utilfeature.Alpha}, StorageObjectInUseProtection: {Default: true, PreRelease: utilfeature.GA}, diff --git a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/register.go b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/register.go index 3bf48fa7e50..b2ffe63ab2b 100644 --- a/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/register.go +++ b/staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/register.go @@ -22,8 +22,16 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) -// GroupName is the group name use in this package -const GroupName = "csi.storage.k8s.io" +const ( + // GroupName is the group name use in this package + GroupName string = "csi.storage.k8s.io" + + // CsiDriverResourcePlural is the plural name of the CSIDriver resource + CsiDriverResourcePlural string = "csidrivers" + + // CsiNodeInfoResourcePlural is the plural name of the CSINode resource + CsiNodeInfoResourcePlural string = "csinodeinfos" +) // SchemeGroupVersion is group version used to register these objects var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"} diff --git a/test/cmd/crd.sh b/test/cmd/crd.sh index 3a2de16f267..7fafafffc04 100755 --- a/test/cmd/crd.sh +++ b/test/cmd/crd.sh @@ -44,7 +44,7 @@ run_crd_tests() { __EOF__ # Post-Condition: assertion object exist - kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{$id_field}}:{{end}}" 'foos.company.com:' + kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{if eq $id_field \\\"foos.company.com\\\"}}{{$id_field}}:{{end}}{{end}}" 'foos.company.com:' kubectl "${kube_flags_with_token[@]}" create -f - << __EOF__ { @@ -66,7 +66,7 @@ __EOF__ __EOF__ # Post-Condition: assertion object exist - kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{$id_field}}:{{end}}" 'bars.company.com:foos.company.com:' + kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{if eq $id_field \\\"foos.company.com\\\" \\\"bars.company.com\\\"}}{{$id_field}}:{{end}}{{end}}" 'bars.company.com:foos.company.com:' # This test ensures that the name printer is able to output a resource # in the proper "kind.group/resource_name" format, and that the @@ -93,7 +93,7 @@ __EOF__ __EOF__ # Post-Condition: assertion crd with non-matching kind and resource exists - kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{$id_field}}:{{end}}" 'bars.company.com:foos.company.com:resources.mygroup.example.com:' + kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{if eq $id_field \\\"foos.company.com\\\" \\\"bars.company.com\\\" \\\"resources.mygroup.example.com\\\"}}{{$id_field}}:{{end}}{{end}}" 'bars.company.com:foos.company.com:resources.mygroup.example.com:' # This test ensures that we can create complex validation without client-side validation complaining kubectl "${kube_flags_with_token[@]}" create -f - << __EOF__ @@ -128,7 +128,7 @@ __EOF__ __EOF__ # Post-Condition: assertion crd with non-matching kind and resource exists - kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{$id_field}}:{{end}}" 'bars.company.com:foos.company.com:resources.mygroup.example.com:validfoos.company.com:' + kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{if eq $id_field \\\"foos.company.com\\\" \\\"bars.company.com\\\" \\\"resources.mygroup.example.com\\\" \\\"validfoos.company.com\\\"}}{{$id_field}}:{{end}}{{end}}" 'bars.company.com:foos.company.com:resources.mygroup.example.com:validfoos.company.com:' run_non_native_resource_tests diff --git a/test/cmd/old-print.sh b/test/cmd/old-print.sh index 0011862e836..f881e50e85d 100755 --- a/test/cmd/old-print.sh +++ b/test/cmd/old-print.sh @@ -117,7 +117,7 @@ run_kubectl_old_print_tests() { __EOF__ # Post-Condition: assertion object exists - kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{$id_field}}:{{end}}" 'foos.company.com:' + kube::test::get_object_assert customresourcedefinitions "{{range.items}}{{if eq $id_field \\\"foos.company.com\\\"}}{{$id_field}}:{{end}}{{end}}" 'foos.company.com:' # Test that we can list this new CustomResource kube::test::get_object_assert foos "{{range.items}}{{$id_field}}:{{end}}" '' diff --git a/test/integration/volume/BUILD b/test/integration/volume/BUILD index 0e6231fde6c..156b58ab2d4 100644 --- a/test/integration/volume/BUILD +++ b/test/integration/volume/BUILD @@ -26,6 +26,7 @@ go_test( "//pkg/volume/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/api/storage/v1:go_default_library", + "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/test/integration/volume/attach_detach_test.go b/test/integration/volume/attach_detach_test.go index aa72838c058..956b90c587e 100644 --- a/test/integration/volume/attach_detach_test.go +++ b/test/integration/volume/attach_detach_test.go @@ -23,6 +23,7 @@ import ( "time" "k8s.io/api/core/v1" + fakeapiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -393,6 +394,7 @@ func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, sy } resyncPeriod := 12 * time.Hour testClient := clientset.NewForConfigOrDie(&config) + fakeApiExtensionsClient := fakeapiextensionsclient.NewSimpleClientset() host := volumetest.NewFakeVolumeHost("/tmp/fake", nil, nil) plugin := &volumetest.FakeVolumePlugin{ @@ -413,6 +415,7 @@ func createAdClients(ns *v1.Namespace, t *testing.T, server *httptest.Server, sy ctrl, err := attachdetach.NewAttachDetachController( testClient, nil, /* csiClient */ + fakeApiExtensionsClient, /* crdClient */ informers.Core().V1().Pods(), informers.Core().V1().Nodes(), informers.Core().V1().PersistentVolumeClaims(),