Merge pull request #10627 from eparis/total-ansible

Ansible: Get DNS working - Add kubectl kubeconfig file

contrib/ansible/roles/kubernetes/files/make-ca-cert.sh

@@ -18,13 +18,32 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-cert_ip=$1
-cert_dir=${CERT_DIR:-/srv/kubernetes}
-cert_group=${CERT_GROUP:-kube-cert}
+# Caller should set in the ev:
+# MASTER_IP - this may be an ip or things like "_use_gce_external_ip_"
+# DNS_DOMAIN - which will be passed to minions in --cluster_domain
+# SERVICE_CLUSTER_IP_RANGE - where all service IPs are allocated
+# MASTER_NAME - I'm not sure what it is...
 
-mkdir -p "$cert_dir"
+# Also the following will be respected
+# CERT_DIR - where to place the finished certs
+# CERT_GROUP - who the group owner of the cert files should be
 
-use_cn=false
+cert_ip="${MASTER_IP:="${1}"}"
+master_name="${MASTER_NAME:="kubernetes"}"
+service_range="${SERVICE_CLUSTER_IP_RANGE:="10.0.0.0/16"}"
+dns_domain="${DNS_DOMAIN:="cluster.local"}"
+cert_dir="${CERT_DIR:-"/srv/kubernetes"}"
+cert_group="${CERT_GROUP:="kube-cert"}"
+
+# The following certificate pairs are created:
+#
+#  - ca (the cluster's certificate authority)
+#  - server
+#  - kubelet
+#  - kubecfg (for kubectl)
+#
+# TODO(roberthbailey): Replace easyrsa with a simple Go program to generate
+# the certs that we need.
 
 # TODO: Add support for discovery on other providers?
 if [ "$cert_ip" == "_use_gce_external_ip_" ]; then
@@ -37,10 +56,9 @@ fi
 
 if [ "$cert_ip" == "_use_azure_dns_name_" ]; then
   cert_ip=$(uname -n | awk -F. '{ print $2 }').cloudapp.net
-  use_cn=true
 fi
 
-tmpdir=$(mktemp -d -t kubernetes_cacert.XXXXXX)
+tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
 trap 'rm -rf "${tmpdir}"' EXIT
 cd "${tmpdir}"
 
@@ -56,25 +74,42 @@ cd "${tmpdir}"
 #
 # Due to GCS caching of public objects, it may take time for this to be widely
 # distributed.
-curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1
-tar xzf easy-rsa.tar.gz > /dev/null 2>&1
 
+# Calculate the first ip address in the service range
+octects=($(echo "${service_range}" | sed -e 's|/.*||' -e 's/\./ /g'))
+((octects[3]+=1))
+service_ip=$(echo "${octects[*]}" | sed 's/ /./g')
+
+# Determine appropriete subject alt names
+sans="IP:${cert_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${dns_domain},DNS:${master_name}"
+
+curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1
+tar xzf easy-rsa.tar.gz > /dev/null
 cd easy-rsa-master/easyrsa3
-./easyrsa init-pki > /dev/null 2>&1
-./easyrsa --batch "--req-cn=$cert_ip@`date +%s`" build-ca nopass > /dev/null 2>&1
-if [ $use_cn = "true" ]; then
-    ./easyrsa build-server-full $cert_ip nopass > /dev/null 2>&1
-    cp -p pki/issued/$cert_ip.crt "${cert_dir}/server.cert" > /dev/null 2>&1
-    cp -p pki/private/$cert_ip.key "${cert_dir}/server.key" > /dev/null 2>&1
-else
-    ./easyrsa --subject-alt-name=IP:$cert_ip build-server-full kubernetes-master nopass > /dev/null 2>&1
-    cp -p pki/issued/kubernetes-master.crt "${cert_dir}/server.cert" > /dev/null 2>&1
-    cp -p pki/private/kubernetes-master.key "${cert_dir}/server.key" > /dev/null 2>&1
-fi
-./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1
+
+(./easyrsa init-pki > /dev/null 2>&1
+ ./easyrsa --batch "--req-cn=${cert_ip}@$(date +%s)" build-ca nopass > /dev/null 2>&1
+ ./easyrsa --subject-alt-name="${sans}" build-server-full "${master_name}" nopass > /dev/null 2>&1
+ ./easyrsa build-client-full kubelet nopass > /dev/null 2>&1
+ ./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1) || {
+ # If there was an error in the subshell, just die.
+ # TODO(roberthbailey): add better error handling here
+ echo "=== Failed to generate certificates: Aborting ==="
+ exit 2
+ }
+
+mkdir -p "$cert_dir"
+
 cp -p pki/ca.crt "${cert_dir}/ca.crt"
+cp -p "pki/issued/${master_name}.crt" "${cert_dir}/server.crt" > /dev/null 2>&1
+cp -p "pki/private/${master_name}.key" "${cert_dir}/server.key" > /dev/null 2>&1
 cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt"
 cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key"
-# Make server certs accessible to apiserver.
-chgrp $cert_group "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt"
-chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt"
+cp -p pki/issued/kubelet.crt "${cert_dir}/kubelet.crt"
+cp -p pki/private/kubelet.key "${cert_dir}/kubelet.key"
+
+CERTS=("ca.crt" "server.key" "server.crt" "kubelet.key" "kubelet.crt" "kubecfg.key" "kubecfg.crt")
+for cert in "${CERTS[@]}"; do
+  chgrp "${cert_group}" "${cert_dir}/${cert}"
+  chmod 660 "${cert_dir}/${cert}"
+done

contrib/ansible/roles/kubernetes/files/make-cert.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cert_dir=${CERT_DIR:-/srv/kubernetes}
-cert_group=${CERT_GROUP:-kube-cert}
-
-mkdir -p "$cert_dir"
-
-openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
-  -subj "/CN=kubernetes.invalid/O=Kubernetes" \
-  -keyout "${cert_dir}/server.key" -out "${cert_dir}/server.cert"
-chgrp $cert_group "${cert_dir}/server.key" "${cert_dir}/server.cert"
-chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert"

contrib/ansible/roles/kubernetes/tasks/gen_certs.yml

@@ -1,23 +1,10 @@
 ---
-#- name: Get create cert script from Kubernetes
-#  get_url:
-#    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/saltbase/salt/generate-cert/make-cert.sh
-#    dest={{ kube_script_dir }}/make-cert.sh mode=0500
-#    force=yes
-
 #- name: Get create ca cert script from Kubernetes
 #  get_url:
 #    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/saltbase/salt/generate-cert/make-ca-cert.sh
 #    dest={{ kube_script_dir }}/make-ca-cert.sh mode=0500
 #    force=yes
 
-- name: HACK | overwrite make-cert.sh from local copy
-  copy:
-    src=make-cert.sh
-    dest={{ kube_script_dir }}
-    mode=0500
-  changed_when: false
-
 - name: HACK | overwrite make-ca-cert.sh from local copy
   copy:
     src=make-ca-cert.sh
@@ -30,8 +17,12 @@
   command:
     "{{ kube_script_dir }}/make-ca-cert.sh {{ inventory_hostname }}"
   args:
-    creates: "{{ kube_cert_dir }}/server.cert"
+    creates: "{{ kube_cert_dir }}/server.crt"
   environment:
+    MASTER_IP: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}"
+    MASTER_NAME: "kubernetes"
+    DNS_DOMAIN: "{{ dns_domain }}"
+    SERVICE_CLUSTER_IP_RANGE: "{{ kube_service_addresses }}"
     CERT_DIR: "{{ kube_cert_dir }}"
     CERT_GROUP: "{{ kube_cert_group }}"
 
@@ -43,7 +34,7 @@
     mode=0440
   with_items:
     - "{{ kube_cert_dir }}/ca.crt"
-    - "{{ kube_cert_dir }}/server.cert"
+    - "{{ kube_cert_dir }}/server.crt"
     - "{{ kube_cert_dir }}/server.key"
     - "{{ kube_cert_dir }}/kubecfg.crt"
     - "{{ kube_cert_dir }}/kubecfg.key"

contrib/ansible/roles/kubernetes/tasks/gen_tokens.yml

@@ -10,7 +10,7 @@
   environment:
     TOKEN_DIR: "{{ kube_token_dir }}"
   with_nested:
-    - [ "system:controller_manager", "system:scheduler" ]
+    - [ "system:controller_manager", "system:scheduler", "system:kubectl" ]
     - "{{ groups['masters'] }}"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"

contrib/ansible/roles/kubernetes/tasks/secrets.yml

@@ -35,8 +35,12 @@
   run_once: true
   delegate_to: "{{ groups['masters'][0] }}"
 
+- name: Register the CA certificate as a fact so it can be used later
+  set_fact:
+    kube_ca_cert: "{{ ca_cert.content|b64decode }}"
+
 - name: Place CA certificate everywhere
-  copy: content="{{ ca_cert.content|b64decode }}" dest="{{ kube_cert_dir }}/ca.crt"
+  copy: content="{{ kube_ca_cert }}" dest="{{ kube_cert_dir }}/ca.crt"
   notify:
     - restart daemons
 

contrib/ansible/roles/master/tasks/main.yml

@@ -11,7 +11,7 @@
     - restart apiserver
 
 - name: Ensure that a token auth file exists (addons may populate it)
-  file: path={{kube_token_dir }}/known_tokens.csv state=touch
+  file: path={{ kube_token_dir }}/known_tokens.csv state=touch
   changed_when: false
 
 - name: add cap_net_bind_service to kube-apiserver
@@ -27,6 +27,7 @@
   with_items:
     - "system:controller_manager"
     - "system:scheduler"
+    - "system:kubectl"
   register: tokens
   delegate_to: "{{ groups['masters'][0] }}"
 
@@ -34,6 +35,7 @@
   set_fact:
     controller_manager_token: "{{ tokens.results[0].content|b64decode }}"
     scheduler_token: "{{ tokens.results[1].content|b64decode }}"
+    kubectl_token: "{{ tokens.results[2].content|b64decode }}"
 
 - name: write the config file for the controller-manager
   template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager
@@ -61,6 +63,9 @@
 - name: Enable scheduler
   service: name=kube-scheduler enabled=yes state=started
 
+- name: write the kubecfg (auth) file for kubectl
+  template: src=kubectl.kubeconfig.j2 dest={{ kube_config_dir }}/kubectl.kubeconfig
+
 - include: firewalld.yml
   when: has_firewalld
 

contrib/ansible/roles/master/templates/apiserver.j2

@@ -23,4 +23,4 @@ KUBE_ETCD_SERVERS="--etcd_servers={% for node in groups['etcd'] %}http://{{ node
 KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
 
 # Add your own!
-KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.cert --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_token_dir }}/known_tokens.csv --service_account_key_file={{ kube_cert_dir }}/server.cert"
+KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.crt --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_token_dir }}/known_tokens.csv --service_account_key_file={{ kube_cert_dir }}/server.crt"

contrib/ansible/roles/master/templates/kubectl.kubeconfig.j2

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+current-context: kubectl-to-{{ cluster_name }}
+preferences: {}
+clusters:
+- cluster:
+    certificate-authority-data: {{ kube_ca_cert|b64encode }}
+    server: https://{{ groups['masters'][0] }}:443
+  name: {{ cluster_name }}
+contexts:
+- context:
+    cluster: {{ cluster_name }}
+    user: kubectl
+  name: kubectl-to-{{ cluster_name }}
+users:
+- name: kubectl
+  user:
+    token: {{ kubectl_token }}