From c70b554af9594effebda6905589c33cddec77392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Go=C5=82=C4=85b?= Date: Tue, 29 May 2018 15:16:27 +0200 Subject: [PATCH 1/4] Create system:cluster-autoscaler account & role and introduce it to CA start-up script --- .../cluster-autoscaler-rbac.yaml | 64 +++++++++++++++++++ cluster/gce/gci/configure-helper.sh | 33 +++++++++- .../gce/manifests/cluster-autoscaler.manifest | 13 +++- 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 cluster/addons/rbac/cluster-autoscaler/cluster-autoscaler-rbac.yaml diff --git a/cluster/addons/rbac/cluster-autoscaler/cluster-autoscaler-rbac.yaml b/cluster/addons/rbac/cluster-autoscaler/cluster-autoscaler-rbac.yaml new file mode 100644 index 00000000000..970e1c90695 --- /dev/null +++ b/cluster/addons/rbac/cluster-autoscaler/cluster-autoscaler-rbac.yaml @@ -0,0 +1,64 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cluster-autoscaler + labels: + addonmanager.kubernetes.io/mode: Reconcile +rules: + # leader election + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["create"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update", "patch", "delete"] + # accessing & modifying cluster state (nodes & pods) + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + # read-only access to cluster state + - apiGroups: [""] + resources: ["services", "replicationcontrollers", "persistentvolumes", "persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps", "extensions"] + resources: ["daemonsets", "replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + # misc access + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cluster-autoscaler + labels: + addonmanager.kubernetes.io/mode: Reconcile +subjects: + - kind: User + name: cluster-autoscaler + namespace: kube-system +roleRef: + kind: ClusterRole + name: cluster-autoscaler + apiGroup: rbac.authorization.k8s.io + diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 0935fc37f2c..4ed509b0700 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -545,6 +545,9 @@ function create-master-auth { if [[ -n "${KUBE_SCHEDULER_TOKEN:-}" ]]; then append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_SCHEDULER_TOKEN}," "system:kube-scheduler,uid:system:kube-scheduler" fi + if [[ -n "${KUBE_CLUSTER_AUTOSCALER_TOKEN:-}" ]]; then + append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_CLUSTER_AUTOSCALER_TOKEN}," "cluster-autoscaler,uid:cluster-autoscaler" + fi if [[ -n "${KUBE_PROXY_TOKEN:-}" ]]; then append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_PROXY_TOKEN}," "system:kube-proxy,uid:kube_proxy" fi @@ -1006,6 +1009,30 @@ current-context: kube-scheduler EOF } +function create-clusterautoscaler-kubeconfig { + echo "Creating cluster-autoscaler kubeconfig file" + mkdir -p /etc/srv/kubernetes/cluster-autoscaler + cat </etc/srv/kubernetes/cluster-autoscaler/kubeconfig +apiVersion: v1 +kind: Config +users: +- name: cluster-autoscaler + user: + token: ${KUBE_CLUSTER_AUTOSCALER_TOKEN} +clusters: +- name: local + cluster: + insecure-skip-tls-verify: true + server: https://localhost:443 +contexts: +- context: + cluster: local + user: cluster-autoscaler + name: cluster-autoscaler +current-context: cluster-autoscaler +EOF +} + function create-kubescheduler-policy-config { echo "Creating kube-scheduler policy config file" mkdir -p /etc/srv/kubernetes/kube-scheduler @@ -1970,12 +1997,15 @@ function start-kube-scheduler { function start-cluster-autoscaler { if [[ "${ENABLE_CLUSTER_AUTOSCALER:-}" == "true" ]]; then echo "Start kubernetes cluster autoscaler" + setup-addon-manifests "addons" "rbac/cluster-autoscaler" + create-clusterautoscaler-kubeconfig prepare-log-file /var/log/cluster-autoscaler.log # Remove salt comments and replace variables with values local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/cluster-autoscaler.manifest" local params="${AUTOSCALER_MIG_CONFIG} ${CLOUD_CONFIG_OPT} ${AUTOSCALER_EXPANDER_CONFIG:---expander=price}" + params+=" --kubeconfig=/etc/srv/kubernetes/cluster-autoscaler/kubeconfig" sed -i -e "s@{{params}}@${params}@g" "${src_file}" sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}" sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}" @@ -2570,9 +2600,10 @@ function main() { fi fi - # generate the controller manager and scheduler tokens here since they are only used on the master. + # generate the controller manager, scheduler and CA tokens here since they are only used on the master. KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + KUBE_CLUSTER_AUTOSCALER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) setup-os-params config-ip-firewall diff --git a/cluster/gce/manifests/cluster-autoscaler.manifest b/cluster/gce/manifests/cluster-autoscaler.manifest index fc1b406ea6f..69318036282 100644 --- a/cluster/gce/manifests/cluster-autoscaler.manifest +++ b/cluster/gce/manifests/cluster-autoscaler.manifest @@ -25,7 +25,7 @@ }, "command": [ "./run.sh", - "--kubernetes=http://127.0.0.1:8080?inClusterConfig=f", + "--kubernetes=https://127.0.0.1:443", "--v=4", "--logtostderr=true", "--write-status-configmap=true", @@ -56,6 +56,11 @@ "readOnly": true, "mountPath": "/usr/share/ca-certificates" }, + { + "name": "srvkube", + "readOnly": true, + "mountPath": "/etc/srv/kubernetes" + }, { "name": "logfile", "mountPath": "/var/log/cluster-autoscaler.log", @@ -80,6 +85,12 @@ "path": "/usr/share/ca-certificates" } }, + { + "name": "srvkube", + "hostPath": { + "path": "/etc/srv/kubernetes" + } + }, { "name": "logfile", "hostPath": { From f8fa60e9eaba8fb77bf145a3e31be9c2d6b31498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Go=C5=82=C4=85b?= Date: Fri, 8 Jun 2018 18:02:37 +0200 Subject: [PATCH 2/4] Limit access to configmaps --- .../rbac/cluster-autoscaler/cluster-autoscaler-rbac.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cluster/addons/rbac/cluster-autoscaler/cluster-autoscaler-rbac.yaml b/cluster/addons/rbac/cluster-autoscaler/cluster-autoscaler-rbac.yaml index 970e1c90695..822f3c7b4c1 100644 --- a/cluster/addons/rbac/cluster-autoscaler/cluster-autoscaler-rbac.yaml +++ b/cluster/addons/rbac/cluster-autoscaler/cluster-autoscaler-rbac.yaml @@ -45,7 +45,11 @@ rules: verbs: ["create", "update", "patch"] - apiGroups: [""] resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] + verbs: ["create"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status"] + verbs: ["get", "update", "patch", "delete"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 From faa4dc39c4052882153d767d737aba5dcfc77ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Go=C5=82=C4=85b?= Date: Mon, 11 Jun 2018 10:56:02 +0200 Subject: [PATCH 3/4] Disambiguate a comment --- cluster/gce/gci/configure-helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 4ed509b0700..2cd3e4110f9 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2600,7 +2600,7 @@ function main() { fi fi - # generate the controller manager, scheduler and CA tokens here since they are only used on the master. + # generate the controller manager, scheduler and cluster autoscaler tokens here since they are only used on the master. KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) KUBE_CLUSTER_AUTOSCALER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) From 9e2fa69d209bfa134513e32773671b6c7dcb0c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Go=C5=82=C4=85b?= Date: Mon, 11 Jun 2018 18:52:03 +0200 Subject: [PATCH 4/4] Limit the mounted directory to cluster-autoscaler/ --- cluster/gce/manifests/cluster-autoscaler.manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster/gce/manifests/cluster-autoscaler.manifest b/cluster/gce/manifests/cluster-autoscaler.manifest index 69318036282..3546a862207 100644 --- a/cluster/gce/manifests/cluster-autoscaler.manifest +++ b/cluster/gce/manifests/cluster-autoscaler.manifest @@ -59,7 +59,7 @@ { "name": "srvkube", "readOnly": true, - "mountPath": "/etc/srv/kubernetes" + "mountPath": "/etc/srv/kubernetes/cluster-autoscaler" }, { "name": "logfile", @@ -88,7 +88,7 @@ { "name": "srvkube", "hostPath": { - "path": "/etc/srv/kubernetes" + "path": "/etc/srv/kubernetes/cluster-autoscaler" } }, {