diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 06c2012c238..7cdd30fa10e 100644 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -227,7 +227,7 @@ function kube-up { if [ ! -f $AWS_SSH_KEY ]; then ssh-keygen -f $AWS_SSH_KEY -N '' fi - + $AWS_CMD import-key-pair --key-name kubernetes --public-key-material file://$AWS_SSH_KEY.pub > /dev/null 2>&1 || true VPC_ID=$($AWS_CMD create-vpc --cidr-block 172.20.0.0/16 | json_val '["Vpc"]["VpcId"]') $AWS_CMD modify-vpc-attribute --vpc-id $VPC_ID --enable-dns-support '{"Value": true}' > /dev/null @@ -294,14 +294,14 @@ function kube-up { --security-group-ids $SEC_GROUP_ID \ --associate-public-ip-address \ --user-data file://${KUBE_TEMP}/minion-start-${i}.sh | json_val '["Instances"][0]["InstanceId"]') - sleep 3 + sleep 3 n=0 until [ $n -ge 5 ]; do $AWS_CMD create-tags --resources $minion_id --tags Key=Name,Value=${MINION_NAMES[$i]} > /dev/null && break n=$[$n+1] sleep 15 done - + sleep 3 n=0 until [ $n -ge 5 ]; do @@ -309,7 +309,7 @@ function kube-up { n=$[$n+1] sleep 15 done - + sleep 3 $AWS_CMD modify-instance-attribute --instance-id $minion_id --source-dest-check '{"Value": false}' > /dev/null @@ -343,7 +343,7 @@ function kube-up { detect-master > /dev/null detect-minions > /dev/null - # Wait 3 minutes for cluster to come up. We hit it with a "highstate" after that to + # Wait 3 minutes for cluster to come up. We hit it with a "highstate" after that to # make sure that everything is well configured. echo "Waiting for cluster to settle" local i @@ -353,7 +353,7 @@ function kube-up { done echo "Re-running salt highstate" ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo salt '*' state.highstate > /dev/null - + echo "Waiting for cluster initialization." echo echo " This will continually check to see if the API for kubernetes is reachable." @@ -400,9 +400,9 @@ function kube-up { # config file. Distribute the same way the htpasswd is done. ( umask 077 - ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /usr/share/nginx/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null - ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /usr/share/nginx/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null - ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /usr/share/nginx/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null + ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null + ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null + ssh -oStrictHostKeyChecking=no -i ~/.ssh/kube_aws_rsa ubuntu@${KUBE_MASTER_IP} sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null cat << EOF > ~/.kubernetes_auth { diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 102336a539a..f1732d7c522 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -422,9 +422,9 @@ function kube-up { # TODO: generate ADMIN (and KUBELET) tokens and put those in the master's # config file. Distribute the same way the htpasswd is done. (umask 077 - gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null - gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null - gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null + gcutil ssh "${MASTER_NAME}" sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null + gcutil ssh "${MASTER_NAME}" sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null + gcutil ssh "${MASTER_NAME}" sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null cat << EOF > ~/.kubernetes_auth { diff --git a/cluster/saltbase/salt/apiserver/default b/cluster/saltbase/salt/apiserver/default index e8671d7a979..e39024faf2f 100644 --- a/cluster/saltbase/salt/apiserver/default +++ b/cluster/saltbase/salt/apiserver/default @@ -27,4 +27,7 @@ {% set portal_net = "-portal_net=" + pillar['portal_net'] %} {% endif %} -DAEMON_ARGS="{{daemon_args}} {{address}} {{etcd_servers}} {{ cloud_provider }} --allow_privileged={{pillar['allow_privileged']}} {{portal_net}}" +{% set cert_file = "-tls_cert_file=/srv/kubernetes/server.cert" %} +{% set key_file = "-tls_private_key_file=/srv/kubernetes/server.key" %} + +DAEMON_ARGS="{{daemon_args}} {{address}} {{etcd_servers}} {{ cloud_provider }} --allow_privileged={{pillar['allow_privileged']}} {{portal_net}} {{cert_file}} {{key_file}}" diff --git a/cluster/saltbase/salt/generate-cert/init.sls b/cluster/saltbase/salt/generate-cert/init.sls new file mode 100644 index 00000000000..250d7e6763b --- /dev/null +++ b/cluster/saltbase/salt/generate-cert/init.sls @@ -0,0 +1,38 @@ +{% if grains.cloud is defined %} + {% if grains.cloud == 'gce' %} + {% set cert_ip='_use_gce_external_ip_' %} + {% endif %} + {% if grains.cloud == 'aws' %} + {% set cert_ip='_use_aws_external_ip_' %} + {% endif %} + {% if grains.cloud == 'vagrant' %} + {% set cert_ip=grains.fqdn_ip4 %} + {% endif %} + {% if grains.cloud == 'vsphere' %} + {% set cert_ip=grains.ip_interfaces.eth0[0] %} + {% endif %} +{% endif %} + +# If there is a pillar defined, override any defaults. +{% if pillar['cert_ip'] is defined %} + {% set cert_ip=pillar['cert_ip'] %} +{% endif %} + +{% set certgen="make-cert.sh" %} +{% if cert_ip is defined %} + {% set certgen="make-ca-cert.sh" %} +{% endif %} + +kubernetes-cert: + cmd.script: + - unless: test -f /srv/kubernetes/server.cert + - source: salt://generate-cert/{{certgen}} +{% if cert_ip is defined %} + - args: {{cert_ip}} + - require: + - pkg: curl +{% endif %} + - cwd: / + - user: root + - group: root + - shell: /bin/bash diff --git a/cluster/saltbase/salt/nginx/make-ca-cert.sh b/cluster/saltbase/salt/generate-cert/make-ca-cert.sh similarity index 55% rename from cluster/saltbase/salt/nginx/make-ca-cert.sh rename to cluster/saltbase/salt/generate-cert/make-ca-cert.sh index 80bddc66e2a..35e16df3ea1 100755 --- a/cluster/saltbase/salt/nginx/make-ca-cert.sh +++ b/cluster/saltbase/salt/generate-cert/make-ca-cert.sh @@ -19,6 +19,9 @@ set -o nounset set -o pipefail cert_ip=$1 +cert_dir=/srv/kubernetes + +mkdir -p "$cert_dir" # TODO: Add support for discovery on other providers? if [ "$cert_ip" == "_use_gce_external_ip_" ]; then @@ -33,19 +36,28 @@ tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX) trap 'rm -rf "${tmpdir}"' EXIT cd "${tmpdir}" -# TODO: For now, this is a patched repo that makes subject-alt-name work, when the fix is upstream -# move back to the upstream easyrsa -curl -L -J -O https://github.com/brendandburns/easy-rsa/archive/master.tar.gz > /dev/null 2>&1 -tar xzf easy-rsa-master.tar.gz > /dev/null 2>&1 +# TODO: For now, this is a patched tool that makes subject-alt-name work, when +# the fix is upstream move back to the upstream easyrsa. This is cached in GCS +# but is originally taken from: +# https://github.com/brendandburns/easy-rsa/archive/master.tar.gz +# +# To update, do the following: +# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz +# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz +# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz +# +# Due to GCS caching of public objects, it may take time for this to be widely +# distributed. +curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1 +tar xzf easy-rsa.tar.gz > /dev/null 2>&1 cd easy-rsa-master/easyrsa3 ./easyrsa init-pki > /dev/null 2>&1 ./easyrsa --batch build-ca nopass > /dev/null 2>&1 ./easyrsa --subject-alt-name=IP:$cert_ip build-server-full kubernetes-master nopass > /dev/null 2>&1 ./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1 -cp -p pki/issued/kubernetes-master.crt /usr/share/nginx/server.cert > /dev/null 2>&1 -cp -p pki/private/kubernetes-master.key /usr/share/nginx/server.key > /dev/null 2>&1 -cp -p pki/ca.crt /usr/share/nginx/ca.crt -cp -p pki/issued/kubecfg.crt /usr/share/nginx/kubecfg.crt -cp -p pki/private/kubecfg.key /usr/share/nginx/kubecfg.key - +cp -p pki/issued/kubernetes-master.crt "${cert_dir}/server.cert" > /dev/null 2>&1 +cp -p pki/private/kubernetes-master.key "${cert_dir}/server.key" > /dev/null 2>&1 +cp -p pki/ca.crt "${cert_dir}/ca.crt" +cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt" +cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key" diff --git a/cluster/saltbase/salt/nginx/make-cert.sh b/cluster/saltbase/salt/generate-cert/make-cert.sh similarity index 88% rename from cluster/saltbase/salt/nginx/make-cert.sh rename to cluster/saltbase/salt/generate-cert/make-cert.sh index f67cb908c95..cb4d93ad9f8 100755 --- a/cluster/saltbase/salt/nginx/make-cert.sh +++ b/cluster/saltbase/salt/generate-cert/make-cert.sh @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +cert_dir=/srv/kubernetes + openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \ -subj "/CN=kubernetes.invalid/O=Kubernetes" \ - -keyout /usr/share/nginx/server.key -out /usr/share/nginx/server.cert + -keyout "${cert_dir}/server.key" -out "${cert_dir}/server.cert" diff --git a/cluster/saltbase/salt/nginx/init.sls b/cluster/saltbase/salt/nginx/init.sls index 0abebce4159..bbc4fdeb102 100644 --- a/cluster/saltbase/salt/nginx/init.sls +++ b/cluster/saltbase/salt/nginx/init.sls @@ -8,45 +8,7 @@ nginx: - file: /etc/nginx/nginx.conf - file: /etc/nginx/sites-enabled/default - file: /usr/share/nginx/htpasswd - - cmd: /usr/share/nginx/server.cert - -{% if grains.cloud is defined %} - {% if grains.cloud == 'gce' %} - {% set cert_ip='_use_gce_external_ip_' %} - {% endif %} - {% if grains.cloud == 'aws' %} - {% set cert_ip='_use_aws_external_ip_' %} - {% endif %} - {% if grains.cloud == 'vagrant' %} - {% set cert_ip=grains.fqdn_ip4 %} - {% endif %} - {% if grains.cloud == 'vsphere' %} - {% set cert_ip=grains.ip_interfaces.eth0[0] %} - {% endif %} -{% endif %} -# If there is a pillar defined, override any defaults. -{% if pillar['cert_ip'] is defined %} - {% set cert_ip=pillar['cert_ip'] %} -{% endif %} - -{% set certgen="make-cert.sh" %} -{% if cert_ip is defined %} - {% set certgen="make-ca-cert.sh" %} -{% endif %} - -/usr/share/nginx/server.cert: - cmd.script: - - unless: test -f /usr/share/nginx/server.cert - - source: salt://nginx/{{certgen}} -{% if cert_ip is defined %} - - args: {{cert_ip}} - - require: - - pkg: curl -{% endif %} - - cwd: / - - user: root - - group: root - - shell: /bin/bash + - cmd: kubernetes-cert /etc/nginx/nginx.conf: file: diff --git a/cluster/saltbase/salt/nginx/kubernetes-site b/cluster/saltbase/salt/nginx/kubernetes-site index ef5dc96fe36..a4be99cc0f3 100644 --- a/cluster/saltbase/salt/nginx/kubernetes-site +++ b/cluster/saltbase/salt/nginx/kubernetes-site @@ -33,8 +33,8 @@ server { index index.html index.htm; ssl on; - ssl_certificate /usr/share/nginx/server.cert; - ssl_certificate_key /usr/share/nginx/server.key; + ssl_certificate /srv/kubernetes/server.cert; + ssl_certificate_key /srv/kubernetes/server.key; ssl_session_timeout 5m; @@ -53,7 +53,7 @@ server { proxy_connect_timeout 159s; proxy_send_timeout 600s; proxy_read_timeout 600s; - + # Disable retry proxy_next_upstream off; diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 53fc83c0a8f..187663f88d6 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -26,6 +26,7 @@ base: 'roles:kubernetes-master': - match: grain + - generate-cert - etcd - apiserver - controller-manager diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh index c149bff3e2e..17d77c16c64 100755 --- a/cluster/vsphere/util.sh +++ b/cluster/vsphere/util.sh @@ -397,9 +397,9 @@ function kube-up { ( umask 077 - kube-ssh "${KUBE_MASTER_IP}" sudo cat /usr/share/nginx/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null - kube-ssh "${KUBE_MASTER_IP}" sudo cat /usr/share/nginx/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null - kube-ssh "${KUBE_MASTER_IP}" sudo cat /usr/share/nginx/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null + kube-ssh "${KUBE_MASTER_IP}" sudo cat /srv/kubernetes/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null + kube-ssh "${KUBE_MASTER_IP}" sudo cat /srv/kubernetes/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null + kube-ssh "${KUBE_MASTER_IP}" sudo cat /srv/kubernetes/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null cat << EOF > ~/.kubernetes_auth {