diff --git a/contrib/ansible/roles/kubernetes-addons/tasks/main.yml b/contrib/ansible/roles/kubernetes-addons/tasks/main.yml index 3da6617954c..67802394f35 100644 --- a/contrib/ansible/roles/kubernetes-addons/tasks/main.yml +++ b/contrib/ansible/roles/kubernetes-addons/tasks/main.yml @@ -33,13 +33,10 @@ - name: HACK | copy local kube-addon-update.sh copy: src=kube-addon-update.sh dest={{ kube_script_dir }}/kube-addon-update.sh mode=0755 -- name: Copy script to create known_tokens.csv - copy: src=kube-gen-token.sh dest={{ kube_script_dir }}/kube-gen-token.sh mode=0755 - -- name: Run kube-gen-token script to create {{ kube_config_dir }}/known_tokens.csv +- name: Run kube-gen-token script to create {{ kube_token_dir }}/known_tokens.csv command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item }}" environment: - TOKEN_DIR: "{{ kube_config_dir }}" + TOKEN_DIR: "{{ kube_token_dir }}" with_items: - "system:dns" register: gentoken diff --git a/contrib/ansible/roles/kubernetes-addons/templates/kube-addons.service.j2 b/contrib/ansible/roles/kubernetes-addons/templates/kube-addons.service.j2 index 38db35c8e5a..0bd58b057cd 100644 --- a/contrib/ansible/roles/kubernetes-addons/templates/kube-addons.service.j2 +++ b/contrib/ansible/roles/kubernetes-addons/templates/kube-addons.service.j2 @@ -3,7 +3,7 @@ Description=Kubernetes Addon Object Manager Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] -Environment="TOKEN_DIR={{ kube_config_dir }}" +Environment="TOKEN_DIR={{ kube_token_dir }}" Environment="KUBECTL_BIN=/usr/bin/kubectl" Environment="KUBERNETES_MASTER_NAME={{ groups['masters'][0] }}" ExecStart={{ kube_script_dir }}/kube-addons.sh diff --git a/contrib/ansible/roles/kubernetes/defaults/main.yml b/contrib/ansible/roles/kubernetes/defaults/main.yml index 805b069b476..3a4eedc71cd 100644 --- a/contrib/ansible/roles/kubernetes/defaults/main.yml +++ b/contrib/ansible/roles/kubernetes/defaults/main.yml @@ -14,6 +14,8 @@ kube_config_dir: /etc/kubernetes # This is where all the cert scripts and certs will be located kube_cert_dir: "{{ kube_config_dir }}/certs" +# This is where all of the bearer tokens will be stored +kube_token_dir: "{{ kube_config_dir }}/tokens" # This is the group that the cert creation scripts chgrp the # cert files to. Not really changable... diff --git a/contrib/ansible/roles/kubernetes-addons/files/kube-gen-token.sh b/contrib/ansible/roles/kubernetes/files/kube-gen-token.sh similarity index 91% rename from contrib/ansible/roles/kubernetes-addons/files/kube-gen-token.sh rename to contrib/ansible/roles/kubernetes/files/kube-gen-token.sh index baa950c0129..fa6a5ddc752 100644 --- a/contrib/ansible/roles/kubernetes-addons/files/kube-gen-token.sh +++ b/contrib/ansible/roles/kubernetes/files/kube-gen-token.sh @@ -21,10 +21,11 @@ create_accounts=($@) touch "${token_file}" for account in "${create_accounts[@]}"; do - if grep "${account}" "${token_file}" ; then + if grep ",${account}," "${token_file}" ; then continue fi token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) echo "${token},${account},${account}" >> "${token_file}" + echo "${token}" > "${token_dir}/${account}.token" echo "Added ${account}" done diff --git a/contrib/ansible/roles/kubernetes/tasks/gen_tokens.yml b/contrib/ansible/roles/kubernetes/tasks/gen_tokens.yml new file mode 100644 index 00000000000..fc13e74db66 --- /dev/null +++ b/contrib/ansible/roles/kubernetes/tasks/gen_tokens.yml @@ -0,0 +1,30 @@ +--- +- name: Copy the token gen script + copy: + src=kube-gen-token.sh + dest={{ kube_script_dir }} + mode=u+x + +- name: Generate tokens for master components + command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item }}" + environment: + TOKEN_DIR: "{{ kube_token_dir }}" + with_items: + - "system:controller_manager" + - "system:scheduler" + register: gentoken + changed_when: "'Added' in gentoken.stdout" + notify: + - restart daemons + +- name: Generate tokens for node components + command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item }}" + environment: + TOKEN_DIR: "{{ kube_token_dir }}" + with_items: + - "system:kubelet" + - "system:proxy" + register: gentoken + changed_when: "'Added' in gentoken.stdout" + notify: + - restart daemons diff --git a/contrib/ansible/roles/kubernetes/tasks/main.yml b/contrib/ansible/roles/kubernetes/tasks/main.yml index f1008991833..ce7699f2ac5 100644 --- a/contrib/ansible/roles/kubernetes/tasks/main.yml +++ b/contrib/ansible/roles/kubernetes/tasks/main.yml @@ -18,6 +18,6 @@ notify: - restart daemons -- include: certs.yml +- include: secrets.yml tags: - certs + secrets diff --git a/contrib/ansible/roles/kubernetes/tasks/place_certs.yml b/contrib/ansible/roles/kubernetes/tasks/place_certs.yml deleted file mode 100644 index 2271fd1115a..00000000000 --- a/contrib/ansible/roles/kubernetes/tasks/place_certs.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- name: place ssh public key on other nodes so apiserver can push certs - authorized_key: user=root key="{{ item }}" state=present - with_file: - - '/tmp/id_rsa.pub' - changed_when: false - -- name: Copy certificates directly from the apiserver to nodes - synchronize: - src={{ kube_cert_dir }}/{{ item }} - dest={{ kube_cert_dir }}/{{ item }} - rsync_timeout=30 - set_remote_user=no - delegate_to: "{{ groups['masters'][0] }}" - with_items: - - "ca.crt" - - "kubecfg.crt" - - "kubecfg.key" - notify: - - restart daemons - -- name: remove ssh public key so apiserver can not push stuff - authorized_key: user=root key="{{ item }}" state=absent - with_file: - - '/tmp/id_rsa.pub' - changed_when: false diff --git a/contrib/ansible/roles/kubernetes/tasks/place_secrets.yml b/contrib/ansible/roles/kubernetes/tasks/place_secrets.yml new file mode 100644 index 00000000000..e832b968b00 --- /dev/null +++ b/contrib/ansible/roles/kubernetes/tasks/place_secrets.yml @@ -0,0 +1,40 @@ +--- +- name: place ssh public key so apiserver can push certs + authorized_key: user=root key="{{ item }}" state=present + with_file: + - '/tmp/id_rsa.pub' + changed_when: false + +- name: Copy certificates directly from the apiserver to nodes + synchronize: src={{ kube_cert_dir }}/{{ item }} dest={{ kube_cert_dir }}/{{ item }} + delegate_to: "{{ groups['masters'][0] }}" + with_items: + - "ca.crt" + notify: + - restart daemons + +- name: Copy master tokens to the masters + synchronize: src={{ kube_token_dir }}/{{ item }} dest={{ kube_token_dir }}/{{ item }} + delegate_to: "{{ groups['masters'][0] }}" + with_items: + - "system:controller_manager.token" + - "system:scheduler.token" + notify: + - restart daemons + when: inventory_hostname in groups['masters'] + +- name: Copy node tokens to the nodes + synchronize: src={{ kube_token_dir }}/{{ item }} dest={{ kube_token_dir }}/{{ item }} + delegate_to: "{{ groups['masters'][0] }}" + with_items: + - "system:kubelet.token" + - "system:proxy.token" + notify: + - restart daemons + when: inventory_hostname in groups['nodes'] + +- name: remove ssh public key so apiserver can not push stuff + authorized_key: user=root key="{{ item }}" state=absent + with_file: + - '/tmp/id_rsa.pub' + changed_when: false diff --git a/contrib/ansible/roles/kubernetes/tasks/certs.yml b/contrib/ansible/roles/kubernetes/tasks/secrets.yml similarity index 74% rename from contrib/ansible/roles/kubernetes/tasks/certs.yml rename to contrib/ansible/roles/kubernetes/tasks/secrets.yml index 338e975f61d..b52f4a179f1 100644 --- a/contrib/ansible/roles/kubernetes/tasks/certs.yml +++ b/contrib/ansible/roles/kubernetes/tasks/secrets.yml @@ -18,14 +18,27 @@ mode=o-rwx group={{ kube_cert_group }} -- name: Install rsync to push certs around +- name: make sure the tokens directory exits + file: + path={{ kube_token_dir }} + state=directory + mode=o-rwx + group={{ kube_cert_group }} + +- include: gen_certs.yml + when: inventory_hostname == groups['masters'][0] + +- include: gen_tokens.yml + when: inventory_hostname == groups['masters'][0] + +- name: Install rsync to push secrets around action: "{{ ansible_pkg_mgr }}" args: name: rsync state: latest when: not is_atomic -- name: Generating RSA key for cert node to push to others +- name: Generating RSA key for master node to push to others user: name=root generate_ssh_key=yes run_once: true delegate_to: "{{ groups['masters'][0] }}" @@ -40,10 +53,7 @@ delegate_to: "{{ groups['masters'][0] }}" changed_when: false -- include: gen_certs.yml - when: inventory_hostname == groups['masters'][0] - -- include: place_certs.yml +- include: place_secrets.yml - name: Delete the downloaded pub key local_action: file path=/tmp/id_rsa.pub state=absent diff --git a/contrib/ansible/roles/master/tasks/main.yml b/contrib/ansible/roles/master/tasks/main.yml index 3bbd9e204e2..a1b4511fcde 100644 --- a/contrib/ansible/roles/master/tasks/main.yml +++ b/contrib/ansible/roles/master/tasks/main.yml @@ -11,39 +11,49 @@ - restart apiserver - name: Ensure that a token auth file exists (addons may populate it) - file: path={{kube_config_dir }}/known_tokens.csv state=touch + file: path={{kube_token_dir }}/known_tokens.csv state=touch changed_when: false +- name: add cap_net_bind_service to kube-apiserver + capabilities: path=/usr/bin/kube-apiserver capability=cap_net_bind_service=ep state=present + when: not is_atomic + +- name: Enable apiserver + service: name=kube-apiserver enabled=yes state=started + - name: write the config file for the controller-manager template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager notify: - restart controller-manager -- name: write the config file for the scheduler - template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler - notify: - - restart scheduler - -- name: add cap_net_bind_service to kube-apiserver - capabilities: path=/usr/bin/kube-apiserver capability=cap_net_bind_service=ep state=present - when: not is_atomic +- name: Get the controller-manager token value + slurp: + src: "{{ kube_token_dir }}/system:controller_manager.token" + register: controller_manager_token - name: write the kubecfg (auth) file for controller-manager template: src=controller-manager.kubeconfig.j2 dest={{ kube_config_dir }}/controller-manager.kubeconfig notify: - restart controller-manager +- name: Enable controller-manager + service: name=kube-controller-manager enabled=yes state=started + +- name: write the config file for the scheduler + template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler + notify: + - restart scheduler + +- name: Get the scheduler token value + slurp: + src: "{{ kube_token_dir }}/system:scheduler.token" + register: scheduler_token + - name: write the kubecfg (auth) file for scheduler template: src=scheduler.kubeconfig.j2 dest={{ kube_config_dir }}/scheduler.kubeconfig notify: - restart scheduler -- name: Enable apiserver - service: name=kube-apiserver enabled=yes state=started - -- name: Enable controller-manager - service: name=kube-controller-manager enabled=yes state=started - - name: Enable scheduler service: name=kube-scheduler enabled=yes state=started diff --git a/contrib/ansible/roles/master/templates/apiserver.j2 b/contrib/ansible/roles/master/templates/apiserver.j2 index c0787286da7..03252ba31b6 100644 --- a/contrib/ansible/roles/master/templates/apiserver.j2 +++ b/contrib/ansible/roles/master/templates/apiserver.j2 @@ -23,4 +23,4 @@ KUBE_ETCD_SERVERS="--etcd_servers={% for node in groups['etcd'] %}http://{{ node KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota" # Add your own! -KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.cert --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_config_dir }}/known_tokens.csv" +KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.cert --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_token_dir }}/known_tokens.csv" diff --git a/contrib/ansible/roles/master/templates/controller-manager.kubeconfig.j2 b/contrib/ansible/roles/master/templates/controller-manager.kubeconfig.j2 index adf812936b1..d36522091c6 100644 --- a/contrib/ansible/roles/master/templates/controller-manager.kubeconfig.j2 +++ b/contrib/ansible/roles/master/templates/controller-manager.kubeconfig.j2 @@ -1,19 +1,18 @@ apiVersion: v1 +kind: Config +current-context: controller-manager-to-{{ cluster_name }} +preferences: {} clusters: - cluster: certificate-authority: {{ kube_cert_dir }}/ca.crt - server: http://{{ groups['masters'][0] }}:443 + server: https://{{ groups['masters'][0] }}:443 name: {{ cluster_name }} contexts: - context: cluster: {{ cluster_name }} - user: kubelet - name: kubelet-to-{{ cluster_name }} -current-context: kubelet-to-{{ cluster_name }} -kind: Config -preferences: {} + user: controller-manager + name: controller-manager-to-{{ cluster_name }} users: -- name: kubelet +- name: controller-manager user: - client-certificate: {{ kube_cert_dir }}/kubecfg.crt - client-key: {{ kube_cert_dir }}/kubecfg.key + token: {{ controller_manager_token.content|b64decode }} diff --git a/contrib/ansible/roles/master/templates/scheduler.kubeconfig.j2 b/contrib/ansible/roles/master/templates/scheduler.kubeconfig.j2 index adf812936b1..d8031f761cb 100644 --- a/contrib/ansible/roles/master/templates/scheduler.kubeconfig.j2 +++ b/contrib/ansible/roles/master/templates/scheduler.kubeconfig.j2 @@ -1,19 +1,18 @@ apiVersion: v1 +kind: Config +current-context: scheduler-to-{{ cluster_name }} +preferences: {} clusters: - cluster: certificate-authority: {{ kube_cert_dir }}/ca.crt - server: http://{{ groups['masters'][0] }}:443 + server: https://{{ groups['masters'][0] }}:443 name: {{ cluster_name }} contexts: - context: cluster: {{ cluster_name }} - user: kubelet - name: kubelet-to-{{ cluster_name }} -current-context: kubelet-to-{{ cluster_name }} -kind: Config -preferences: {} + user: scheduler + name: scheduler-to-{{ cluster_name }} users: -- name: kubelet +- name: scheduler user: - client-certificate: {{ kube_cert_dir }}/kubecfg.crt - client-key: {{ kube_cert_dir }}/kubecfg.key + token: {{ scheduler_token.content|b64decode }} diff --git a/contrib/ansible/roles/node/tasks/main.yml b/contrib/ansible/roles/node/tasks/main.yml index 08c815878a6..f23bf1787fd 100644 --- a/contrib/ansible/roles/node/tasks/main.yml +++ b/contrib/ansible/roles/node/tasks/main.yml @@ -19,24 +19,34 @@ notify: - restart kubelet -- name: write the config files for proxy - template: src=proxy.j2 dest={{ kube_config_dir }}/proxy - notify: - - restart proxy +- name: Get the kubelet token value + slurp: + src: "{{ kube_token_dir }}/system:kubelet.token" + register: kubelet_token - name: write the kubecfg (auth) file for kubelet template: src=kubelet.kubeconfig.j2 dest={{ kube_config_dir }}/kubelet.kubeconfig notify: - restart kubelet +- name: Enable kubelet + service: name=kubelet enabled=yes state=started + +- name: write the config files for proxy + template: src=proxy.j2 dest={{ kube_config_dir }}/proxy + notify: + - restart proxy + +- name: Get the proxy token value + slurp: + src: "{{ kube_token_dir }}/system:proxy.token" + register: proxy_token + - name: write the kubecfg (auth) file for kube-proxy template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig notify: - restart proxy -- name: Enable kubelet - service: name=kubelet enabled=yes state=started - - name: Enable proxy service: name=kube-proxy enabled=yes state=started diff --git a/contrib/ansible/roles/node/templates/kubelet.kubeconfig.j2 b/contrib/ansible/roles/node/templates/kubelet.kubeconfig.j2 index adf812936b1..1c15a436542 100644 --- a/contrib/ansible/roles/node/templates/kubelet.kubeconfig.j2 +++ b/contrib/ansible/roles/node/templates/kubelet.kubeconfig.j2 @@ -1,19 +1,18 @@ apiVersion: v1 +kind: Config +current-context: kubelet-to-{{ cluster_name }} +preferences: {} clusters: - cluster: certificate-authority: {{ kube_cert_dir }}/ca.crt - server: http://{{ groups['masters'][0] }}:443 + server: https://{{ groups['masters'][0] }}:443 name: {{ cluster_name }} contexts: - context: cluster: {{ cluster_name }} user: kubelet name: kubelet-to-{{ cluster_name }} -current-context: kubelet-to-{{ cluster_name }} -kind: Config -preferences: {} users: - name: kubelet user: - client-certificate: {{ kube_cert_dir }}/kubecfg.crt - client-key: {{ kube_cert_dir }}/kubecfg.key + token: {{ kubelet_token.content|b64decode }} diff --git a/contrib/ansible/roles/node/templates/proxy.kubeconfig.j2 b/contrib/ansible/roles/node/templates/proxy.kubeconfig.j2 index adf812936b1..35018bea3f0 100644 --- a/contrib/ansible/roles/node/templates/proxy.kubeconfig.j2 +++ b/contrib/ansible/roles/node/templates/proxy.kubeconfig.j2 @@ -1,19 +1,18 @@ apiVersion: v1 -clusters: -- cluster: - certificate-authority: {{ kube_cert_dir }}/ca.crt - server: http://{{ groups['masters'][0] }}:443 - name: {{ cluster_name }} +kind: Config +current-context: proxy-to-{{ cluster_name }} +preferences: {} contexts: - context: cluster: {{ cluster_name }} - user: kubelet - name: kubelet-to-{{ cluster_name }} -current-context: kubelet-to-{{ cluster_name }} -kind: Config -preferences: {} + user: proxy + name: proxy-to-{{ cluster_name }} +clusters: +- cluster: + certificate-authority: {{ kube_cert_dir }}/ca.crt + server: https://{{ groups['masters'][0] }}:443 + name: {{ cluster_name }} users: -- name: kubelet +- name: proxy user: - client-certificate: {{ kube_cert_dir }}/kubecfg.crt - client-key: {{ kube_cert_dir }}/kubecfg.key + token: {{ proxy_token.content|b64decode }}