Merge pull request #68200 from RenaudWasTaken/pluginwatcher-beta

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

KubeletPluginsWatcher feature is beta in 1.12 release

*What this PR does / why we need it:*
Graduates DevicePlugins feature to beta.

*Which issue(s) this PR fixes:*
Related but does not fix: https://github.com/kubernetes/features/issues/595 as well as https://github.com/kubernetes/kubernetes/issues/65773

*Special notes for your reviewer:*
Includes upgrading the gRPC pluginwatcher API to beta. Based on the [device plugin model](https://github.com/kubernetes/kubernetes/pull/59588).

*Depends on https://github.com/kubernetes/kubernetes/pull/64621 being merged* 

Release note:

```release-note
KubeletPluginsWatcher feature graduates to beta.
```

/sig node
/sig storage

/cc @vladimirvivien @sbezverk @vikaschoudhary16 @saad-ali @vishh @jiayingz

hack/.golint_failures

@@ -186,6 +186,7 @@ pkg/kubelet/apis/cri/testing
 pkg/kubelet/apis/deviceplugin/v1alpha
 pkg/kubelet/apis/deviceplugin/v1beta1
 pkg/kubelet/apis/pluginregistration/v1alpha1
+pkg/kubelet/apis/pluginregistration/v1beta1
 pkg/kubelet/cadvisor
 pkg/kubelet/cadvisor/testing
 pkg/kubelet/checkpoint

hack/update-generated-kubelet-plugin-registration-dockerized.sh

@@ -19,11 +19,13 @@ set -o nounset
 set -o pipefail
 
 KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/../" && pwd -P)"
-KUBELET_PLUGIN_REGISTRATION_ROOT="${KUBE_ROOT}/pkg/kubelet/apis/pluginregistration/v1alpha1/"
+KUBELET_PLUGIN_REGISTRATION_V1ALPHA="${KUBE_ROOT}/pkg/kubelet/apis/pluginregistration/v1alpha1/"
+KUBELET_PLUGIN_REGISTRATION_V1BETA="${KUBE_ROOT}/pkg/kubelet/apis/pluginregistration/v1beta1/"
 KUBELET_EXAMPLE_PLUGIN_V1BETA1="${KUBE_ROOT}/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta1/"
 KUBELET_EXAMPLE_PLUGIN_V1BETA2="${KUBE_ROOT}/pkg/kubelet/util/pluginwatcher/example_plugin_apis/v1beta2/"
 
 source "${KUBE_ROOT}/hack/lib/protoc.sh"
-kube::protoc::generate_proto ${KUBELET_PLUGIN_REGISTRATION_ROOT}
+kube::protoc::generate_proto ${KUBELET_PLUGIN_REGISTRATION_V1ALPHA}
+kube::protoc::generate_proto ${KUBELET_PLUGIN_REGISTRATION_V1BETA}
 kube::protoc::generate_proto ${KUBELET_EXAMPLE_PLUGIN_V1BETA1}
 kube::protoc::generate_proto ${KUBELET_EXAMPLE_PLUGIN_V1BETA2}

hack/verify-generated-kubelet-plugin-registration.sh

@@ -20,20 +20,31 @@ set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 ERROR="Kubelet Plugin Registration api is out of date. Please run hack/update-generated-kubelet-plugin-registration.sh"
-KUBELET_PLUGIN_REGISTRATION_ROOT="${KUBE_ROOT}/pkg/kubelet/apis/pluginregistration/v1alpha1/"
+KUBELET_PLUGIN_REGISTRATION_V1ALPHA="${KUBE_ROOT}/pkg/kubelet/apis/pluginregistration/v1alpha1/"
+KUBELET_PLUGIN_REGISTRATION_V1BETA="${KUBE_ROOT}/pkg/kubelet/apis/pluginregistration/v1beta1/"
 
 source "${KUBE_ROOT}/hack/lib/protoc.sh"
 kube::golang::setup_env
 
 function cleanup {
-	rm -rf ${KUBELET_PLUGIN_REGISTRATION_ROOT}/_tmp/
+	rm -rf ${KUBELET_PLUGIN_REGISTRATION_V1ALPHA}/_tmp/
+	rm -rf ${KUBELET_PLUGIN_REGISTRATION_V1BETA}/_tmp/
 }
 
 trap cleanup EXIT
 
-mkdir -p ${KUBELET_PLUGIN_REGISTRATION_ROOT}/_tmp
-cp ${KUBELET_PLUGIN_REGISTRATION_ROOT}/api.pb.go ${KUBELET_PLUGIN_REGISTRATION_ROOT}/_tmp/
+mkdir -p ${KUBELET_PLUGIN_REGISTRATION_V1ALPHA}/_tmp
+mkdir -p ${KUBELET_PLUGIN_REGISTRATION_V1BETA}/_tmp
 
+cp ${KUBELET_PLUGIN_REGISTRATION_V1ALPHA}/api.pb.go ${KUBELET_PLUGIN_REGISTRATION_V1ALPHA}/_tmp/
+cp ${KUBELET_PLUGIN_REGISTRATION_V1BETA}/api.pb.go ${KUBELET_PLUGIN_REGISTRATION_V1BETA}/_tmp/
+
+# Check V1Alpha
 KUBE_VERBOSE=3 "${KUBE_ROOT}/hack/update-generated-kubelet-plugin-registration.sh"
-kube::protoc::diff "${KUBELET_PLUGIN_REGISTRATION_ROOT}/api.pb.go" "${KUBELET_PLUGIN_REGISTRATION_ROOT}/_tmp/api.pb.go" ${ERROR}
+kube::protoc::diff "${KUBELET_PLUGIN_REGISTRATION_V1ALPHA}/api.pb.go" "${KUBELET_PLUGIN_REGISTRATION_V1ALPHA}/_tmp/api.pb.go" ${ERROR}
+echo "Generated Kubelet Plugin Registration api is up to date."
+
+# Check V1Beta
+KUBE_VERBOSE=3 "${KUBE_ROOT}/hack/update-generated-kubelet-plugin-registration.sh"
+kube::protoc::diff "${KUBELET_PLUGIN_REGISTRATION_V1BETA}/api.pb.go" "${KUBELET_PLUGIN_REGISTRATION_V1BETA}/_tmp/api.pb.go" ${ERROR}
 echo "Generated Kubelet Plugin Registration api is up to date."

pkg/features/kube_features.go

@@ -442,7 +442,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	BalanceAttachedNodeVolumes:                  {Default: false, PreRelease: utilfeature.Alpha},
 	PodReadinessGates:                           {Default: true, PreRelease: utilfeature.Beta},
 	VolumeSubpathEnvExpansion:                   {Default: false, PreRelease: utilfeature.Alpha},
-	KubeletPluginsWatcher:                       {Default: false, PreRelease: utilfeature.Alpha},
+	KubeletPluginsWatcher:                       {Default: true, PreRelease: utilfeature.Beta},
 	ResourceQuotaScopeSelectors:                 {Default: true, PreRelease: utilfeature.Beta},
 	CSIBlockVolume:                              {Default: false, PreRelease: utilfeature.Alpha},
 	RuntimeClass:                                {Default: false, PreRelease: utilfeature.Alpha},

pkg/kubelet/apis/BUILD

@@ -38,6 +38,7 @@ filegroup(
         "//pkg/kubelet/apis/deviceplugin/v1alpha:all-srcs",
         "//pkg/kubelet/apis/deviceplugin/v1beta1:all-srcs",
         "//pkg/kubelet/apis/pluginregistration/v1alpha1:all-srcs",
+        "//pkg/kubelet/apis/pluginregistration/v1beta1:all-srcs",
         "//pkg/kubelet/apis/stats/v1alpha1:all-srcs",
     ],
     tags = ["automanaged"],

pkg/kubelet/apis/pluginregistration/v1alpha1/api.proto

@@ -22,12 +22,12 @@ message PluginInfo {
 	// should follow the extended resource name convention.
 	// For CSI, this is the CSI driver registrar name.
 	string name = 2;
-        // Optional endpoint location. If found set by Kubelet component,
-        // Kubelet component will use this endpoint for specific requests.
-        // This allows the plugin to register using one endpoint and possibly use
-        // a different socket for control operations. CSI uses this model to delegate
-        // its registration external from the plugin.
-        string endpoint = 3;
+	// Optional endpoint location. If found set by Kubelet component,
+	// Kubelet component will use this endpoint for specific requests.
+	// This allows the plugin to register using one endpoint and possibly use
+	// a different socket for control operations. CSI uses this model to delegate
+	// its registration external from the plugin.
+	string endpoint = 3;
 	// Plugin service API versions the plugin supports.
 	// For DevicePlugin, this maps to the deviceplugin API versions the
 	// plugin supports at the given socket.

pkg/kubelet/apis/pluginregistration/v1beta1/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api.pb.go",
+        "constants.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1beta1",
+    deps = [
+        "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/google.golang.org/grpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/apis/pluginregistration/v1beta1/api.proto

@@ -0,0 +1,60 @@
+// To regenerate api.pb.go run hack/update-generated-kubelet-plugin-registration.sh
+syntax = 'proto3';
+
+package pluginregistration;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) =  true;
+option (gogoproto.goproto_getters_all) = true;
+option (gogoproto.marshaler_all) = true;
+option (gogoproto.sizer_all) = true;
+option (gogoproto.unmarshaler_all) = true;
+option (gogoproto.goproto_unrecognized_all) = false;
+
+// PluginInfo is the message sent from a plugin to the Kubelet pluginwatcher for plugin registration
+message PluginInfo {
+	// Type of the Plugin. CSIPlugin or DevicePlugin
+	string type = 1;
+	// Plugin name that uniquely identifies the plugin for the given plugin type.
+	// For DevicePlugin, this is the resource name that the plugin manages and
+	// should follow the extended resource name convention.
+	// For CSI, this is the CSI driver registrar name.
+	string name = 2;
+	// Optional endpoint location. If found set by Kubelet component,
+	// Kubelet component will use this endpoint for specific requests.
+	// This allows the plugin to register using one endpoint and possibly use
+	// a different socket for control operations. CSI uses this model to delegate
+	// its registration external from the plugin.
+	string endpoint = 3;
+	// Plugin service API versions the plugin supports.
+	// For DevicePlugin, this maps to the deviceplugin API versions the
+	// plugin supports at the given socket.
+	// The Kubelet component communicating with the plugin should be able
+	// to choose any preferred version from this list, or returns an error
+	// if none of the listed versions is supported.
+	repeated string supported_versions = 4;
+}
+
+// RegistrationStatus is the message sent from Kubelet pluginwatcher to the plugin for notification on registration status
+message RegistrationStatus {
+	// True if plugin gets registered successfully at Kubelet
+	bool plugin_registered  = 1;
+	// Error message in case plugin fails to register, empty string otherwise
+	string error  = 2;
+}
+
+// RegistrationStatusResponse is sent by plugin to kubelet in response to RegistrationStatus RPC
+message RegistrationStatusResponse {
+}
+
+// InfoRequest is the empty request message from Kubelet
+message InfoRequest {
+}
+
+// Registration is the service advertised by the Plugins.
+service Registration {
+	rpc GetInfo(InfoRequest) returns (PluginInfo) {}
+	rpc NotifyRegistrationStatus(RegistrationStatus) returns (RegistrationStatusResponse) {}
+}

pkg/kubelet/apis/pluginregistration/v1beta1/constants.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pluginregistration
+
+const (
+	CSIPlugin    = "CSIPlugin"
+	DevicePlugin = "DevicePlugin"
+)

test/e2e/storage/csi_objects.go

@@ -41,7 +41,7 @@ var csiImageVersions = map[string]string{
 	"hostpathplugin":   "v0.2.0",
 	"csi-attacher":     "v0.2.0",
 	"csi-provisioner":  "v0.2.1",
-	"driver-registrar": "v0.2.0",
+	"driver-registrar": "v0.3.0",
 }
 
 func csiContainerImage(image string) string {
@@ -239,6 +239,7 @@ func csiHostPathPod(
 					Args: []string{
 						"--v=5",
 						"--csi-address=/csi/csi.sock",
+						"--kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock",
 					},
 					Env: []v1.EnvVar{
 						{
@@ -255,6 +256,10 @@ func csiHostPathPod(
 							Name:      "socket-dir",
 							MountPath: "/csi",
 						},
+						{
+							Name:      "registration-dir",
+							MountPath: "/registration",
+						},
 					},
 				},
 				{
@@ -327,6 +332,15 @@ func csiHostPathPod(
 						},
 					},
 				},
+				{
+					Name: "registration-dir",
+					VolumeSource: v1.VolumeSource{
+						HostPath: &v1.HostPathVolumeSource{
+							Path: "/var/lib/kubelet/plugins",
+							Type: &hostPathType,
+						},
+					},
+				},
 				{
 					Name: "mountpoint-dir",
 					VolumeSource: v1.VolumeSource{

test/e2e/testing-manifests/storage-csi/gce-pd/node_ds.yaml

@@ -16,13 +16,16 @@ spec:
       containers:
         - name: csi-driver-registrar
           imagePullPolicy: Always
-          image: quay.io/k8scsi/driver-registrar:v0.2.0
+          image: quay.io/k8scsi/driver-registrar:v0.3.0
           args:
             - "--v=5"
             - "--csi-address=$(ADDRESS)"
+            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
           env:
             - name: ADDRESS
               value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins/com.google.csi.gcepd/csi.sock
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
@@ -30,6 +33,8 @@ spec:
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
         - name: gce-driver
           securityContext:
             privileged: true
@@ -63,6 +68,10 @@ spec:
           hostPath:
             path: /var/lib/kubelet/plugins/com.google.csi.gcepd/
             type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/
+            type: Directory
         - name: device-dir
           hostPath:
             path: /dev