From ae91ecb62e7c67308d7ef1f902fed0a941180885 Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Mon, 22 May 2017 19:34:37 -0700 Subject: [PATCH 1/2] enable tls bootstrap in GCE/GKE --- .../rbac/kubelet-certificate-management.yaml | 61 +++++++++++++++++++ cluster/gce/configure-vm.sh | 4 +- cluster/gce/gci/configure-helper.sh | 13 +++- cluster/saltbase/salt/kubelet/default | 4 ++ cluster/saltbase/salt/kubelet/init.sls | 8 +-- 5 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 cluster/addons/rbac/kubelet-certificate-management.yaml diff --git a/cluster/addons/rbac/kubelet-certificate-management.yaml b/cluster/addons/rbac/kubelet-certificate-management.yaml new file mode 100644 index 00000000000..4b4b3d7381c --- /dev/null +++ b/cluster/addons/rbac/kubelet-certificate-management.yaml @@ -0,0 +1,61 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: gce:beta:kubelet-certificate-bootstrap + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gce:beta:kubelet-certificate-bootstrap +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kubelet +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: gce:beta:kubelet-certificate-rotation + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gce:beta:kubelet-certificate-rotation +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: gce:beta:kubelet-certificate-bootstrap + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "certificates.k8s.io" + resources: + - certificatesigningrequests/nodeclient + verbs: + - "create" +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: gce:beta:kubelet-certificate-rotation + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "certificates.k8s.io" + resources: + - certificatesigningrequests/selfnodeclient + verbs: + - "create" diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 2d1dad42284..f556589ad80 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -615,7 +615,7 @@ function convert-bytes-gce-kube() { # connect to the apiserver. function create-salt-kubelet-auth() { - local -r kubelet_kubeconfig_file="/srv/salt-overlay/salt/kubelet/kubeconfig" + local -r kubelet_kubeconfig_file="/srv/salt-overlay/salt/kubelet/bootstrap-kubeconfig" if [ ! -e "${kubelet_kubeconfig_file}" ]; then mkdir -p /srv/salt-overlay/salt/kubelet (umask 077; @@ -630,7 +630,7 @@ users: clusters: - name: local cluster: - server: https://kubernetes-master + server: https://${KUBERNETES_MASTER_NAME} certificate-authority: ${CA_CERT_BUNDLE_PATH} contexts: - context: diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index b4e5becf3bd..634b543bc62 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -427,7 +427,7 @@ EOF function create-kubelet-kubeconfig { echo "Creating kubelet kubeconfig file" - cat </var/lib/kubelet/kubeconfig + cat </var/lib/kubelet/bootstrap-kubeconfig apiVersion: v1 kind: Config users: @@ -439,6 +439,7 @@ clusters: - name: local cluster: certificate-authority: ${CA_CERT_BUNDLE_PATH} + server: https://${KUBERNETES_MASTER_NAME} contexts: - context: cluster: local @@ -689,7 +690,11 @@ function start-kubelet { flags+=" --enable-debugging-handlers=false" flags+=" --hairpin-mode=none" if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then - flags+=" --api-servers=https://${KUBELET_APISERVER}" + #TODO(mikedanese): allow static pods to start before creating a client + #flags+=" --experimental-bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" + #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" + flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" + flags+=" --require-kubeconfig" flags+=" --register-schedulable=false" else # Standalone mode (not widely used?) @@ -698,7 +703,9 @@ function start-kubelet { else # For nodes flags+="${NODE_KUBELET_TEST_ARGS:-}" flags+=" --enable-debugging-handlers=true" - flags+=" --api-servers=https://${KUBERNETES_MASTER_NAME}" + flags+=" --experimental-bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" + flags+=" --require-kubeconfig" + flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \ [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \ [[ "${HAIRPIN_MODE:-}" == "none" ]]; then diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 1f258139d94..d1309b7c81a 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -38,6 +38,10 @@ {% endif -%} {% endif -%} +{% if grains.cloud == 'gce' -%} + {% set api_servers = "--experimental-bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig --require-kubeconfig --kubeconfig=/var/lib/kubelet/kubeconfig" -%} +{% endif -%} + {% set cloud_provider = "" -%} {% if grains.cloud is defined and grains.cloud not in ['vagrant', 'photon-controller', 'azure-legacy'] -%} {% set cloud_provider = "--cloud-provider=" + grains.cloud -%} diff --git a/cluster/saltbase/salt/kubelet/init.sls b/cluster/saltbase/salt/kubelet/init.sls index 86c1f82bbd3..21114f99377 100644 --- a/cluster/saltbase/salt/kubelet/init.sls +++ b/cluster/saltbase/salt/kubelet/init.sls @@ -23,9 +23,9 @@ # won't be able to parse it as JSON and it will not be able to publish events # to the apiserver. You'll see a single error line in the kubelet start up file # about this. -/var/lib/kubelet/kubeconfig: +/var/lib/kubelet/bootstrap-kubeconfig: file.managed: - - source: salt://kubelet/kubeconfig + - source: salt://kubelet/bootstrap-kubeconfig - user: root - group: root - mode: 400 @@ -60,7 +60,7 @@ fix-service-kubelet: - file: /usr/local/bin/kubelet - file: {{ pillar.get('systemd_system_path') }}/kubelet.service - file: {{ environment_file }} - - file: /var/lib/kubelet/kubeconfig + - file: /var/lib/kubelet/bootstrap-kubeconfig {% if grains.cloud != 'gce' %} - file: /var/lib/kubelet/ca.crt {% endif %} @@ -90,7 +90,7 @@ kubelet: - file: /usr/lib/systemd/system/kubelet.service {% endif %} - file: {{ environment_file }} - - file: /var/lib/kubelet/kubeconfig + - file: /var/lib/kubelet/bootstrap-kubeconfig {% if grains.cloud != 'gce' %} - file: /var/lib/kubelet/ca.crt {% endif %} From f533bf729f09e2b109458a6a4abe4192fdb35675 Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Thu, 1 Jun 2017 09:16:53 -0700 Subject: [PATCH 2/2] allow sarapprover to create subject access reviews --- .../authorizer/rbac/bootstrappolicy/controller_policy.go | 1 + .../rbac/bootstrappolicy/testdata/controller-roles.yaml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go index ddf83429177..643629edb1c 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go @@ -277,6 +277,7 @@ func init() { Rules: []rbac.PolicyRule{ rbac.NewRule("get", "list", "watch").Groups(certificatesGroup).Resources("certificatesigningrequests").RuleOrDie(), rbac.NewRule("update").Groups(certificatesGroup).Resources("certificatesigningrequests/status", "certificatesigningrequests/approval").RuleOrDie(), + rbac.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews").RuleOrDie(), eventsRule(), }, }) diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml index 9a9e031bdd0..7c4b4b0a64b 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml @@ -73,6 +73,12 @@ items: - certificatesigningrequests/status verbs: - update + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - "" resources: