Use a token for the scheduler, controller-manager, proxy and kubelet

Before we used the kubecfg certificate for everything. Mint one token
for each service and push it around where it belongs.

contrib/ansible/roles/kubernetes-addons/tasks/main.yml

@@ -33,13 +33,10 @@
 - name: HACK | copy local kube-addon-update.sh
   copy: src=kube-addon-update.sh dest={{ kube_script_dir }}/kube-addon-update.sh mode=0755
 
-- name: Copy script to create known_tokens.csv
-  copy: src=kube-gen-token.sh dest={{ kube_script_dir }}/kube-gen-token.sh mode=0755
-
-- name: Run kube-gen-token script to create {{ kube_config_dir }}/known_tokens.csv
+- name: Run kube-gen-token script to create {{ kube_token_dir }}/known_tokens.csv
   command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item }}"
   environment:
-    TOKEN_DIR: "{{ kube_config_dir }}"
+    TOKEN_DIR: "{{ kube_token_dir }}"
   with_items:
     - "system:dns"
   register: gentoken

contrib/ansible/roles/kubernetes-addons/templates/kube-addons.service.j2

@@ -3,7 +3,7 @@ Description=Kubernetes Addon Object Manager
 Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 
 [Service]
-Environment="TOKEN_DIR={{ kube_config_dir }}"
+Environment="TOKEN_DIR={{ kube_token_dir }}"
 Environment="KUBECTL_BIN=/usr/bin/kubectl"
 Environment="KUBERNETES_MASTER_NAME={{ groups['masters'][0] }}"
 ExecStart={{ kube_script_dir }}/kube-addons.sh

contrib/ansible/roles/kubernetes/defaults/main.yml

@@ -14,6 +14,8 @@ kube_config_dir: /etc/kubernetes
 # This is where all the cert scripts and certs will be located
 kube_cert_dir: "{{ kube_config_dir }}/certs"
 
+# This is where all of the bearer tokens will be stored
+kube_token_dir: "{{ kube_config_dir }}/tokens"
 
 # This is the group that the cert creation scripts chgrp the
 # cert files to. Not really changable...

contrib/ansible/roles/kubernetes/files/kube-gen-token.sh

@@ -21,10 +21,11 @@ create_accounts=($@)
 
 touch "${token_file}"
 for account in "${create_accounts[@]}"; do
-  if grep "${account}" "${token_file}" ; then
+  if grep ",${account}," "${token_file}" ; then
     continue
   fi
   token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
   echo "${token},${account},${account}" >> "${token_file}"
+  echo "${token}" > "${token_dir}/${account}.token"
   echo "Added ${account}"
 done

contrib/ansible/roles/kubernetes/tasks/gen_tokens.yml

@@ -0,0 +1,30 @@
+---
+- name: Copy the token gen script
+  copy:
+    src=kube-gen-token.sh
+    dest={{ kube_script_dir }}
+    mode=u+x
+
+- name: Generate tokens for master components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_items:
+    - "system:controller_manager"
+    - "system:scheduler"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  notify:
+    - restart daemons
+
+- name: Generate tokens for node components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_items:
+    - "system:kubelet"
+    - "system:proxy"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  notify:
+    - restart daemons

contrib/ansible/roles/kubernetes/tasks/main.yml

@@ -18,6 +18,6 @@
   notify:
     - restart daemons
 
-- include: certs.yml
+- include: secrets.yml
   tags:
-    certs
+    secrets

contrib/ansible/roles/kubernetes/tasks/place_certs.yml

@@ -1,26 +0,0 @@
----
-- name: place ssh public key on other nodes so apiserver can push certs
-  authorized_key: user=root key="{{ item }}" state=present
-  with_file:
-    - '/tmp/id_rsa.pub'
-  changed_when: false
-
-- name: Copy certificates directly from the apiserver to nodes
-  synchronize:
-    src={{ kube_cert_dir }}/{{ item }}
-    dest={{ kube_cert_dir }}/{{ item }}
-    rsync_timeout=30
-    set_remote_user=no
-  delegate_to: "{{ groups['masters'][0] }}"
-  with_items:
-    - "ca.crt"
-    - "kubecfg.crt"
-    - "kubecfg.key"
-  notify:
-    - restart daemons
-
-- name: remove ssh public key so apiserver can not push stuff
-  authorized_key: user=root key="{{ item }}" state=absent
-  with_file:
-    - '/tmp/id_rsa.pub'
-  changed_when: false

contrib/ansible/roles/kubernetes/tasks/place_secrets.yml

@@ -0,0 +1,40 @@
+---
+- name: place ssh public key so apiserver can push certs
+  authorized_key: user=root key="{{ item }}" state=present
+  with_file:
+    - '/tmp/id_rsa.pub'
+  changed_when: false
+
+- name: Copy certificates directly from the apiserver to nodes
+  synchronize: src={{ kube_cert_dir }}/{{ item }} dest={{ kube_cert_dir }}/{{ item }}
+  delegate_to: "{{ groups['masters'][0] }}"
+  with_items:
+    - "ca.crt"
+  notify:
+    - restart daemons
+
+- name: Copy master tokens to the masters
+  synchronize: src={{ kube_token_dir }}/{{ item }} dest={{ kube_token_dir }}/{{ item }}
+  delegate_to: "{{ groups['masters'][0] }}"
+  with_items:
+    - "system:controller_manager.token"
+    - "system:scheduler.token"
+  notify:
+    - restart daemons
+  when: inventory_hostname in groups['masters']
+
+- name: Copy node tokens to the nodes
+  synchronize: src={{ kube_token_dir }}/{{ item }} dest={{ kube_token_dir }}/{{ item }}
+  delegate_to: "{{ groups['masters'][0] }}"
+  with_items:
+    - "system:kubelet.token"
+    - "system:proxy.token"
+  notify:
+    - restart daemons
+  when: inventory_hostname in groups['nodes']
+
+- name: remove ssh public key so apiserver can not push stuff
+  authorized_key: user=root key="{{ item }}" state=absent
+  with_file:
+    - '/tmp/id_rsa.pub'
+  changed_when: false

contrib/ansible/roles/kubernetes/tasks/secrets.yml

@@ -18,14 +18,27 @@
     mode=o-rwx
     group={{ kube_cert_group }}
 
-- name: Install rsync to push certs around
+- name: make sure the tokens directory exits
+  file:
+    path={{ kube_token_dir }}
+    state=directory
+    mode=o-rwx
+    group={{ kube_cert_group }}
+
+- include: gen_certs.yml
+  when: inventory_hostname == groups['masters'][0]
+
+- include: gen_tokens.yml
+  when: inventory_hostname == groups['masters'][0]
+
+- name: Install rsync to push secrets around
   action: "{{ ansible_pkg_mgr }}"
   args:
     name: rsync
     state: latest
   when: not is_atomic
 
-- name: Generating RSA key for cert node to push to others
+- name: Generating RSA key for master node to push to others
   user: name=root generate_ssh_key=yes
   run_once: true
   delegate_to: "{{ groups['masters'][0] }}"
@@ -40,10 +53,7 @@
   delegate_to: "{{ groups['masters'][0] }}"
   changed_when: false
 
-- include: gen_certs.yml
-  when: inventory_hostname == groups['masters'][0]
-
-- include: place_certs.yml
+- include: place_secrets.yml
 
 - name: Delete the downloaded pub key
   local_action: file path=/tmp/id_rsa.pub state=absent

contrib/ansible/roles/master/tasks/main.yml

@@ -11,39 +11,49 @@
     - restart apiserver
 
 - name: Ensure that a token auth file exists (addons may populate it)
-  file: path={{kube_config_dir }}/known_tokens.csv state=touch
+  file: path={{kube_token_dir }}/known_tokens.csv state=touch
   changed_when: false
 
+- name: add cap_net_bind_service to kube-apiserver
+  capabilities: path=/usr/bin/kube-apiserver capability=cap_net_bind_service=ep state=present
+  when: not is_atomic
+
+- name: Enable apiserver
+  service: name=kube-apiserver enabled=yes state=started
+
 - name: write the config file for the controller-manager
   template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager
   notify:
     - restart controller-manager
 
-- name: write the config file for the scheduler
-  template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler
-  notify:
-    - restart scheduler
-
-- name: add cap_net_bind_service to kube-apiserver
-  capabilities: path=/usr/bin/kube-apiserver capability=cap_net_bind_service=ep state=present
-  when: not is_atomic
+- name: Get the controller-manager token value
+  slurp:
+    src: "{{ kube_token_dir }}/system:controller_manager.token"
+  register: controller_manager_token
 
 - name: write the kubecfg (auth) file for controller-manager
   template: src=controller-manager.kubeconfig.j2 dest={{ kube_config_dir }}/controller-manager.kubeconfig
   notify:
     - restart controller-manager
 
+- name: Enable controller-manager
+  service: name=kube-controller-manager enabled=yes state=started
+
+- name: write the config file for the scheduler
+  template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler
+  notify:
+    - restart scheduler
+
+- name: Get the scheduler token value
+  slurp:
+    src: "{{ kube_token_dir }}/system:scheduler.token"
+  register: scheduler_token
+
 - name: write the kubecfg (auth) file for scheduler
   template: src=scheduler.kubeconfig.j2 dest={{ kube_config_dir }}/scheduler.kubeconfig
   notify:
     - restart scheduler
 
-- name: Enable apiserver
-  service: name=kube-apiserver enabled=yes state=started
-
-- name: Enable controller-manager
-  service: name=kube-controller-manager enabled=yes state=started
-
 - name: Enable scheduler
   service: name=kube-scheduler enabled=yes state=started
 

contrib/ansible/roles/master/templates/apiserver.j2

@@ -23,4 +23,4 @@ KUBE_ETCD_SERVERS="--etcd_servers={% for node in groups['etcd'] %}http://{{ node
 KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
 
 # Add your own!
-KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.cert --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_config_dir }}/known_tokens.csv"
+KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.cert --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_token_dir }}/known_tokens.csv"

contrib/ansible/roles/master/templates/controller-manager.kubeconfig.j2

@@ -1,19 +1,18 @@
 apiVersion: v1
+kind: Config
+current-context: controller-manager-to-{{ cluster_name }}
+preferences: {}
 clusters:
 - cluster:
     certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: http://{{ groups['masters'][0] }}:443
+    server: https://{{ groups['masters'][0] }}:443
   name: {{ cluster_name }}
 contexts:
 - context:
     cluster: {{ cluster_name }}
-    user: kubelet
-  name: kubelet-to-{{ cluster_name }}
-current-context: kubelet-to-{{ cluster_name }}
-kind: Config
-preferences: {}
+    user: controller-manager
+  name: controller-manager-to-{{ cluster_name }}
 users:
-- name: kubelet
+- name: controller-manager
   user:
-    client-certificate: {{ kube_cert_dir }}/kubecfg.crt
-    client-key: {{ kube_cert_dir }}/kubecfg.key
+    token: {{ controller_manager_token.content|b64decode }}

contrib/ansible/roles/master/templates/scheduler.kubeconfig.j2

@@ -1,19 +1,18 @@
 apiVersion: v1
+kind: Config
+current-context: scheduler-to-{{ cluster_name }}
+preferences: {}
 clusters:
 - cluster:
     certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: http://{{ groups['masters'][0] }}:443
+    server: https://{{ groups['masters'][0] }}:443
   name: {{ cluster_name }}
 contexts:
 - context:
     cluster: {{ cluster_name }}
-    user: kubelet
-  name: kubelet-to-{{ cluster_name }}
-current-context: kubelet-to-{{ cluster_name }}
-kind: Config
-preferences: {}
+    user: scheduler
+  name: scheduler-to-{{ cluster_name }}
 users:
-- name: kubelet
+- name: scheduler
   user:
-    client-certificate: {{ kube_cert_dir }}/kubecfg.crt
-    client-key: {{ kube_cert_dir }}/kubecfg.key
+    token: {{ scheduler_token.content|b64decode }}

contrib/ansible/roles/node/tasks/main.yml

@@ -19,24 +19,34 @@
   notify:
     - restart kubelet
 
-- name: write the config files for proxy
-  template: src=proxy.j2 dest={{ kube_config_dir }}/proxy
-  notify:
-    - restart proxy
+- name: Get the kubelet token value
+  slurp:
+    src: "{{ kube_token_dir }}/system:kubelet.token"
+  register: kubelet_token
 
 - name: write the kubecfg (auth) file for kubelet
   template: src=kubelet.kubeconfig.j2 dest={{ kube_config_dir }}/kubelet.kubeconfig
   notify:
     - restart kubelet
 
+- name: Enable kubelet
+  service: name=kubelet enabled=yes state=started
+
+- name: write the config files for proxy
+  template: src=proxy.j2 dest={{ kube_config_dir }}/proxy
+  notify:
+    - restart proxy
+
+- name: Get the proxy token value
+  slurp:
+    src: "{{ kube_token_dir }}/system:proxy.token"
+  register: proxy_token
+
 - name: write the kubecfg (auth) file for kube-proxy
   template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig
   notify:
     - restart proxy
 
-- name: Enable kubelet
-  service: name=kubelet enabled=yes state=started
-
 - name: Enable proxy
   service: name=kube-proxy enabled=yes state=started
 

contrib/ansible/roles/node/templates/kubelet.kubeconfig.j2

@@ -1,19 +1,18 @@
 apiVersion: v1
+kind: Config
+current-context: kubelet-to-{{ cluster_name }}
+preferences: {}
 clusters:
 - cluster:
     certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: http://{{ groups['masters'][0] }}:443
+    server: https://{{ groups['masters'][0] }}:443
   name: {{ cluster_name }}
 contexts:
 - context:
     cluster: {{ cluster_name }}
     user: kubelet
   name: kubelet-to-{{ cluster_name }}
-current-context: kubelet-to-{{ cluster_name }}
-kind: Config
-preferences: {}
 users:
 - name: kubelet
   user:
-    client-certificate: {{ kube_cert_dir }}/kubecfg.crt
-    client-key: {{ kube_cert_dir }}/kubecfg.key
+    token: {{ kubelet_token.content|b64decode }}

contrib/ansible/roles/node/templates/proxy.kubeconfig.j2

@@ -1,19 +1,18 @@
 apiVersion: v1
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: http://{{ groups['masters'][0] }}:443
-  name: {{ cluster_name }}
+kind: Config
+current-context: proxy-to-{{ cluster_name }}
+preferences: {}
 contexts:
 - context:
     cluster: {{ cluster_name }}
-    user: kubelet
-  name: kubelet-to-{{ cluster_name }}
-current-context: kubelet-to-{{ cluster_name }}
-kind: Config
-preferences: {}
+    user: proxy
+  name: proxy-to-{{ cluster_name }}
+clusters:
+- cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.crt
+    server: https://{{ groups['masters'][0] }}:443
+  name: {{ cluster_name }}
 users:
-- name: kubelet
+- name: proxy
   user:
-    client-certificate: {{ kube_cert_dir }}/kubecfg.crt
-    client-key: {{ kube_cert_dir }}/kubecfg.key
+    token: {{ proxy_token.content|b64decode }}