Merge pull request #40760 from mikedanese/gce

Automatic merge from submit-queue (batch tested with PRs 40760, 46706, 46783, 46742, 46751)

enable kubelet csr bootstrap in GCE/GKE

@jcbsmpsn @pipejakob 

Fixes https://github.com/kubernetes/kubernetes/issues/31168

```release-note
Enable kubelet csr bootstrap in GCE/GKE
```

cluster/addons/rbac/kubelet-certificate-management.yaml

@@ -0,0 +1,61 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: gce:beta:kubelet-certificate-bootstrap
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: gce:beta:kubelet-certificate-bootstrap
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: kubelet
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: gce:beta:kubelet-certificate-rotation
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: gce:beta:kubelet-certificate-rotation
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: gce:beta:kubelet-certificate-bootstrap
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - "certificates.k8s.io"
+  resources:
+  - certificatesigningrequests/nodeclient
+  verbs:
+  - "create"
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: gce:beta:kubelet-certificate-rotation
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - "certificates.k8s.io"
+  resources:
+  - certificatesigningrequests/selfnodeclient
+  verbs:
+  - "create"

cluster/gce/configure-vm.sh

@@ -616,7 +616,7 @@ function convert-bytes-gce-kube() {
 #    connect to the apiserver.
 
 function create-salt-kubelet-auth() {
-  local -r kubelet_kubeconfig_file="/srv/salt-overlay/salt/kubelet/kubeconfig"
+  local -r kubelet_kubeconfig_file="/srv/salt-overlay/salt/kubelet/bootstrap-kubeconfig"
   if [ ! -e "${kubelet_kubeconfig_file}" ]; then
     mkdir -p /srv/salt-overlay/salt/kubelet
     (umask 077;
@@ -631,7 +631,7 @@ users:
 clusters:
 - name: local
   cluster:
-    server: https://kubernetes-master
+    server: https://${KUBERNETES_MASTER_NAME}
     certificate-authority: ${CA_CERT_BUNDLE_PATH}
 contexts:
 - context:

cluster/gce/gci/configure-helper.sh

@@ -441,7 +441,7 @@ EOF
 
 function create-kubelet-kubeconfig {
   echo "Creating kubelet kubeconfig file"
-  cat <<EOF >/var/lib/kubelet/kubeconfig
+  cat <<EOF >/var/lib/kubelet/bootstrap-kubeconfig
 apiVersion: v1
 kind: Config
 users:
@@ -453,6 +453,7 @@ clusters:
 - name: local
   cluster:
     certificate-authority: ${CA_CERT_BUNDLE_PATH}
+    server: https://${KUBERNETES_MASTER_NAME}
 contexts:
 - context:
     cluster: local
@@ -703,7 +704,11 @@ function start-kubelet {
     flags+=" --enable-debugging-handlers=false"
     flags+=" --hairpin-mode=none"
     if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
-      flags+=" --api-servers=https://${KUBELET_APISERVER}"
+      #TODO(mikedanese): allow static pods to start before creating a client
+      #flags+=" --experimental-bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
+      #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
+      flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
+      flags+=" --require-kubeconfig"
       flags+=" --register-schedulable=false"
     else
       # Standalone mode (not widely used?)
@@ -712,7 +717,9 @@ function start-kubelet {
   else # For nodes
     flags+="${NODE_KUBELET_TEST_ARGS:-}"
     flags+=" --enable-debugging-handlers=true"
-    flags+=" --api-servers=https://${KUBERNETES_MASTER_NAME}"
+    flags+=" --experimental-bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
+    flags+=" --require-kubeconfig"
+    flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig"
     if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \
        [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \
        [[ "${HAIRPIN_MODE:-}" == "none" ]]; then

cluster/saltbase/salt/kubelet/default

@@ -38,6 +38,10 @@
   {% endif -%}
 {% endif -%}
 
+{% if grains.cloud == 'gce' -%}
+  {% set api_servers = "--experimental-bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig --require-kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig" -%}
+{% endif -%}
+
 {% set cloud_provider = "" -%}
 {% if grains.cloud is defined and grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%}
   {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}

cluster/saltbase/salt/kubelet/init.sls

@@ -23,9 +23,9 @@
 # won't be able to parse it as JSON and it will not be able to publish events
 # to the apiserver. You'll see a single error line in the kubelet start up file
 # about this.
-/var/lib/kubelet/kubeconfig:
+/var/lib/kubelet/bootstrap-kubeconfig:
   file.managed:
-    - source: salt://kubelet/kubeconfig
+    - source: salt://kubelet/bootstrap-kubeconfig
     - user: root
     - group: root
     - mode: 400
@@ -60,7 +60,7 @@ fix-service-kubelet:
       - file: /usr/local/bin/kubelet
       - file: {{ pillar.get('systemd_system_path') }}/kubelet.service
       - file: {{ environment_file }}
-      - file: /var/lib/kubelet/kubeconfig
+      - file: /var/lib/kubelet/bootstrap-kubeconfig
 {% if grains.cloud != 'gce' %}
       - file: /var/lib/kubelet/ca.crt
 {% endif %}
@@ -90,7 +90,7 @@ kubelet:
       - file: /usr/lib/systemd/system/kubelet.service
 {% endif %}
       - file: {{ environment_file }}
-      - file: /var/lib/kubelet/kubeconfig
+      - file: /var/lib/kubelet/bootstrap-kubeconfig
 {% if grains.cloud != 'gce' %}
       - file: /var/lib/kubelet/ca.crt
 {% endif %}

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go

@@ -278,6 +278,7 @@ func init() {
 		Rules: []rbac.PolicyRule{
 			rbac.NewRule("get", "list", "watch").Groups(certificatesGroup).Resources("certificatesigningrequests").RuleOrDie(),
 			rbac.NewRule("update").Groups(certificatesGroup).Resources("certificatesigningrequests/status", "certificatesigningrequests/approval").RuleOrDie(),
+			rbac.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews").RuleOrDie(),
 			eventsRule(),
 		},
 	})

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml

@@ -73,6 +73,12 @@ items:
     - certificatesigningrequests/status
     verbs:
     - update
+  - apiGroups:
+    - authorization.k8s.io
+    resources:
+    - subjectaccessreviews
+    verbs:
+    - create
   - apiGroups:
     - ""
     resources: