diff --git a/cluster/mesos/docker/common/bin/await-health-check b/cluster/mesos/docker/common/bin/await-health-check index 577f5787ac0..7aa813ea1e3 100755 --- a/cluster/mesos/docker/common/bin/await-health-check +++ b/cluster/mesos/docker/common/bin/await-health-check @@ -33,7 +33,7 @@ fi address=${1:-} [ -z "${address}" ] && echo "No address supplied" && exit 1 -bin=$(cd $(dirname $0) && pwd -P) +bin=$(cd "$(dirname ${BASH_SOURCE})" && pwd -P) echo "Waiting (up to ${duration}s) for ${address} to be healthy" if ! timeout "${duration}" bash -c "while ! ${bin}/health-check ${address}; do sleep 0.5; done"; then diff --git a/cluster/mesos/docker/common/bin/util-ssl.sh b/cluster/mesos/docker/common/bin/util-ssl.sh new file mode 100644 index 00000000000..5caa56c8c85 --- /dev/null +++ b/cluster/mesos/docker/common/bin/util-ssl.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sourcable SSL functions + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +bin="$(cd "$(dirname "${BASH_SOURCE}")" && pwd -P)" +source "${bin}/util-temp-dir.sh" + +function cluster::mesos::docker::find_openssl_config { + for candidate in "/etc/ssl/openssl.cnf" "/System/Library/OpenSSL/openssl.cnf"; do + if [ -f "${candidate}" ]; then + echo "${candidate}" + return 0 + fi + done + echo "ERROR: cannot find openssl.cnf" 1>&2 + return 1 +} + +function cluster::mesos::docker::create_root_certificate_authority { + local out_dir="$1" + + # TODO(karlkfi): extract config + local subject="/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com" + + echo "Creating private key" 1>&2 + openssl genrsa -out "${out_dir}/root-ca.key" 2048 + + echo "Creating certificate sign request" 1>&2 + openssl req -nodes -new -utf8 \ + -key "${out_dir}/root-ca.key" \ + -out "${out_dir}/root-ca.csr" \ + -subj "${subject}" + + echo "Signing new certificate with private key" 1>&2 + openssl x509 -req -days 3650 \ + -in "${out_dir}/root-ca.csr" \ + -out "${out_dir}/root-ca.crt" \ + -signkey "${out_dir}/root-ca.key" + + echo "Key: ${out_dir}/root-ca.key" 1>&2 + echo "Cert: ${out_dir}/root-ca.crt" 1>&2 +} + +# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names. +# Uses the current dir for scratch work. +function cluster::mesos::docker::create_apiserver_cert_inner { + local in_dir="$1" + local out_dir="$2" + local apiserver_ip="$3" + local service_ip="$4" + local workspace="$(pwd)" + + if [ ! -f "${in_dir}/root-ca.key" ]; then + echo "Signing key not found: ${in_dir}/root-ca.key" + return 1 + fi + if [ ! -f "${in_dir}/root-ca.crt" ]; then + echo "Root certificate not found: ${in_dir}/root-ca.key" + return 1 + fi + + mkdir -p "${out_dir}" + + local openssl_cnf=$(cluster::mesos::docker::find_openssl_config) + + # TODO(karlkfi): extract config + local subject="/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com" + local cluster_domain="cluster.local" + local service_name="kubernetes" + local service_namespace="default" + local subject_alt_name="IP:${apiserver_ip},IP:${service_ip},DNS:${service_name},DNS:${service_name}.${service_namespace},DNS:${service_name}.${service_namespace}.svc,DNS:${service_name}.${service_namespace}.svc.${cluster_domain}" + + echo "Creating private key" 1>&2 + openssl genrsa -out "${workspace}/apiserver.key" 2048 + + echo "Creating certificate sign request" 1>&2 + openssl req -nodes -new -utf8 \ + -key "${workspace}/apiserver.key" \ + -out "${workspace}/apiserver.csr" \ + -reqexts SAN \ + -config <(cat "${openssl_cnf}"; echo -e "[SAN]\nsubjectAltName=${subject_alt_name}") \ + -subj "${subject}" + + echo "Validating certificate sign request" 1>&2 + openssl req -text -noout -in "${workspace}/apiserver.csr" | grep -q "${service_name}.${service_namespace}.svc.${cluster_domain}" + + echo "Signing new certificate with root certificate authority key" 1>&2 + mkdir -p "${workspace}/demoCA/newcerts" + touch "${workspace}/demoCA/index.txt" + echo 1000 > "${workspace}/demoCA/serial" + openssl ca -batch \ + -days 3650 \ + -in "${workspace}/apiserver.csr" \ + -cert "${in_dir}/root-ca.crt" \ + -keyfile "${in_dir}/root-ca.key" \ + -config <(sed 's/.*\(copy_extensions = copy\)/\1/' ${openssl_cnf}) + + echo "Validating signed certificate" 1>&2 + openssl x509 -in "${workspace}/demoCA/newcerts/1000.pem" -text -noout | grep -q "${service_name}.${service_namespace}.svc.${cluster_domain}" + + echo "Key: ${out_dir}/apiserver.key" 1>&2 + cp "${workspace}/apiserver.key" "${out_dir}/apiserver.key" + + echo "Cert: ${out_dir}/apiserver.crt" 1>&2 + cp "${workspace}/demoCA/newcerts/1000.pem" "${out_dir}/apiserver.crt" +} + +# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names. +function cluster::mesos::docker::create_apiserver_cert { + local in_dir="$1" # must contain root-ca.crt & root-ca.key + local out_dir="$2" + local apiserver_ip="$3" + local service_ip="$4" + + cluster::mesos::docker::run_in_temp_dir "k8sm-certs" \ + "cluster::mesos::docker::create_apiserver_cert_inner" \ + "${in_dir}" "${out_dir}" "${apiserver_ip}" "${service_ip}" +} + +# Creates an rsa key (for signing service accounts). +function cluster::mesos::docker::create_rsa_key { + local key_file_path="$1" + + # buffer output until failure + local output=$(( + openssl genrsa -out "${key_file_path}" 2048 || exit $? + ) 2>&1) + local exit_status="$?" + if [ "${exit_status}" != 0 ]; then + echo "${output}" 1>&2 + return "${exit_status}" + fi + + echo "Key: ${key_file_path}" 1>&2 +} + +# Creates a k8s token auth user file. +# See /docs/admin/authentication.md +function cluster::mesos::docker::create_token_user { + local user_name="$1" + echo "$(openssl rand -hex 32),${user_name},${user_name}" +} + +# Creates a k8s basic auth user file. +# See /docs/admin/authentication.md +function cluster::mesos::docker::create_basic_user { + local user_name="$1" + local password="$2" + echo "${password},${user_name},${user_name}" +} diff --git a/cluster/mesos/docker/util-temp-dir.sh b/cluster/mesos/docker/common/bin/util-temp-dir.sh similarity index 91% rename from cluster/mesos/docker/util-temp-dir.sh rename to cluster/mesos/docker/common/bin/util-temp-dir.sh index 3e12a050a17..d44d4e2ae33 100644 --- a/cluster/mesos/docker/util-temp-dir.sh +++ b/cluster/mesos/docker/common/bin/util-temp-dir.sh @@ -19,6 +19,7 @@ set -o errexit set -o nounset set -o pipefail +set -o errtrace # Runs the supplied command string in a temporary workspace directory. function cluster::mesos::docker::run_in_temp_dir { @@ -31,15 +32,16 @@ function cluster::mesos::docker::run_in_temp_dir { echo "Workspace created: ${workspace}" 1>&2 cleanup() { + local -r workspace="$1" rm -rf "${workspace}" echo "Workspace deleted: ${workspace}" 1>&2 } - trap 'cleanup' EXIT + trap "cleanup '${workspace}'" EXIT pushd "${workspace}" > /dev/null - (${cmd}) || return $? + ${cmd} popd > /dev/null trap - EXIT - cleanup -} \ No newline at end of file + cleanup "${workspace}" +} diff --git a/cluster/mesos/docker/config-default.sh b/cluster/mesos/docker/config-default.sh index 1a1fd24c138..e56446d1324 100755 --- a/cluster/mesos/docker/config-default.sh +++ b/cluster/mesos/docker/config-default.sh @@ -36,3 +36,34 @@ DNS_REPLICAS=1 # Optional: Deploy cluster web interface. ENABLE_CLUSTER_UI=true + +# Timeout (in seconds) to wait for ssl certs to be generated +KUBE_KEYGEN_TIMEOUT="${KUBE_KEYGEN_TIMEOUT:-60}" + +# Timeout (in seconds) to wait for Etcd to come up +MESOS_DOCKER_ETCD_TIMEOUT="${MESOS_DOCKER_ETCD_TIMEOUT:-60}" + +# Timeout (in seconds) to wait for the Mesos Master to come up +MESOS_DOCKER_MESOS_TIMEOUT="${MESOS_DOCKER_MESOS_TIMEOUT:-60}" + +# Timeout (in seconds) to wait for the API Server to come up +MESOS_DOCKER_API_TIMEOUT="${MESOS_DOCKER_API_TIMEOUT:-180}" + +# Timeout (in seconds) to wait for each addon to come up +MESOS_DOCKER_ADDON_TIMEOUT="${MESOS_DOCKER_ADDON_TIMEOUT:-180}" + +# Path to directory on the host to use as the root for multiple docker volumes. +# ${MESOS_DOCKER_WORK_DIR}/log - storage of component logs (written on deploy failure) +# ${MESOS_DOCKER_WORK_DIR}/auth - storage of SSL certs/keys/tokens +# ${MESOS_DOCKER_WORK_DIR}//mesos - storage of mesos slave work (e.g. task logs) +# If using docker-machine or boot2docker, should be under /Users (which is mounted from the host into the docker vm). +# If running in a container, $HOME should be resolved outside of the container. +MESOS_DOCKER_WORK_DIR="${MESOS_DOCKER_WORK_DIR:-${HOME}/tmp/kubernetes}" + +# Path to directory to store mesos slave docker-in-docker images & volumes. +# Usage: ${MESOS_DOCKER_IMAGE_DIR}//docker +# Must not be either an AUFS mount or an SMB/CIFS mount. +# If using docker-machine or boot2docker, should NOT be under /Users (which is mounted from the host into the docker vm). +MESOS_DOCKER_IMAGE_DIR="${MESOS_DOCKER_IMAGE_DIR:-/var/tmp/kubernetes}" + +DOCKER_DAEMON_ARGS="${DOCKER_DAEMON_ARGS:---log-level=error}" diff --git a/cluster/mesos/docker/deploy-addons.sh b/cluster/mesos/docker/deploy-addons.sh index 30e803c62b3..b18f25de56f 100755 --- a/cluster/mesos/docker/deploy-addons.sh +++ b/cluster/mesos/docker/deploy-addons.sh @@ -23,10 +23,11 @@ set -o errexit set -o nounset set -o pipefail +set -o errtrace KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/${KUBE_CONFIG_FILE-"config-default.sh"}" -source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util-temp-dir.sh" +source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/common/bin/util-temp-dir.sh" kubectl="${KUBE_ROOT}/cluster/kubectl.sh" diff --git a/cluster/mesos/docker/docker-compose.yml b/cluster/mesos/docker/docker-compose.yml index afe20c00e33..fdc48a7bf37 100644 --- a/cluster/mesos/docker/docker-compose.yml +++ b/cluster/mesos/docker/docker-compose.yml @@ -29,7 +29,7 @@ mesosmaster1: mesosslave1: hostname: mesosslave1 privileged: true - image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404 + image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404.docker181 entrypoint: [ "bash", "-c", "wrapdocker mesos-slave --hostname=$(getent hosts mesosslave1 | cut -d' ' -f1 | sort -u | tail -1)" ] command: ~ environment: @@ -40,18 +40,19 @@ mesosslave1: - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099] - MESOS_SWITCH_USER=0 - MESOS_CONTAINERIZERS=docker,mesos + - MESOS_WORK_DIR=/var/tmp/mesos - DOCKER_NETWORK_OFFSET=0.0.1.0 - - DOCKER_DAEMON_ARGS=--log-level=error + - DOCKER_DAEMON_ARGS links: - etcd - mesosmaster1 - "ambassador:apiserver" volumes: - - /var/tmp/mesosslave1:/var/lib/docker + - ${MESOS_DOCKER_WORK_DIR}/mesosslave1/mesos:/var/tmp/mesos mesosslave2: hostname: mesosslave2 privileged: true - image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404 + image: mesosphere/mesos-slave-dind:0.23.0-1.0.ubuntu1404.docker181 entrypoint: [ "bash", "-c", "wrapdocker mesos-slave --hostname=$(getent hosts mesosslave2 | cut -d' ' -f1 | sort -u | tail -1)" ] command: ~ environment: @@ -62,26 +63,27 @@ mesosslave2: - MESOS_RESOURCES=cpus:4;mem:1280;disk:25600;ports:[21000-21099] - MESOS_SWITCH_USER=0 - MESOS_CONTAINERIZERS=docker,mesos + - MESOS_WORK_DIR=/var/tmp/mesos - DOCKER_NETWORK_OFFSET=0.0.2.0 - - DOCKER_DAEMON_ARGS=--log-level=error + - DOCKER_DAEMON_ARGS links: - etcd - mesosmaster1 - "ambassador:apiserver" volumes: - - /var/tmp/mesosslave2:/var/lib/docker + - ${MESOS_DOCKER_WORK_DIR}/mesosslave2/mesos:/var/tmp/mesos apiserver: hostname: apiserver image: mesosphere/kubernetes-mesos entrypoint: - /bin/bash - - "-c" + - "-ceu" - > echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" && (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo " mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) && - await-health-check http://etcd:4001/health && - await-health-check http://mesosmaster1:5050/health && - await-file -t=60 /var/run/kubernetes/auth/apiserver.crt && + await-health-check "-t=${MESOS_DOCKER_ETCD_TIMEOUT}" http://etcd:4001/health && + await-health-check "-t=${MESOS_DOCKER_MESOS_TIMEOUT}" http://mesosmaster1:5050/health && + await-file "-t=${KUBE_KEYGEN_TIMEOUT}" /var/run/kubernetes/auth/apiserver.crt && km apiserver --address=$(resolveip apiserver) --external-hostname=apiserver @@ -98,10 +100,14 @@ apiserver: --cloud-config=/opt/mesos-cloud.conf --tls-cert-file=/var/run/kubernetes/auth/apiserver.crt --tls-private-key-file=/var/run/kubernetes/auth/apiserver.key - --v=2 + --v=4 ports: [ "8888:8888", "6443:6443" ] + environment: + - MESOS_DOCKER_ETCD_TIMEOUT + - MESOS_DOCKER_MESOS_TIMEOUT + - KUBE_KEYGEN_TIMEOUT volumes: - - ./certs/apiserver:/var/run/kubernetes/auth + - ${MESOS_DOCKER_WORK_DIR}/auth:/var/run/kubernetes/auth:ro links: - etcd - mesosmaster1 @@ -110,20 +116,24 @@ controller: image: mesosphere/kubernetes-mesos entrypoint: - /bin/bash - - "-c" + - "-ceu" - > echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" && (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo " mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) && - await-health-check -t=60 http://mesosmaster1:5050/health && - await-health-check -t=60 http://apiserver:8888/healthz && + await-health-check "-t=${MESOS_DOCKER_MESOS_TIMEOUT}" http://mesosmaster1:5050/health && + await-health-check "-t=${MESOS_DOCKER_API_TIMEOUT}" http://apiserver:8888/healthz && km controller-manager + --address=$(resolveip controller) --master=http://apiserver:8888 --cloud-config=/opt/mesos-cloud.conf --service-account-private-key-file=/var/run/kubernetes/auth/service-accounts.key --root-ca-file=/var/run/kubernetes/auth/root-ca.crt - --v=2 + --v=4 + environment: + - MESOS_DOCKER_MESOS_TIMEOUT + - MESOS_DOCKER_API_TIMEOUT volumes: - - ./certs/controller:/var/run/kubernetes/auth + - ${MESOS_DOCKER_WORK_DIR}/auth:/var/run/kubernetes/auth:ro links: - mesosmaster1 - apiserver @@ -132,13 +142,13 @@ scheduler: image: mesosphere/kubernetes-mesos entrypoint: - /bin/bash - - "-c" + - "-ceu" - > echo "Hostname: $(hostname -f) ($(hostname -f | xargs resolveip))" && (grep "mesos-master\s*=" /opt/mesos-cloud.conf || echo " mesos-master = mesosmaster1:5050" >> /opt/mesos-cloud.conf) && - await-health-check http://etcd:4001/health && - await-health-check http://mesosmaster1:5050/health && - await-health-check -t=60 http://apiserver:8888/healthz && + await-health-check "-t=${MESOS_DOCKER_ETCD_TIMEOUT}" http://etcd:4001/health && + await-health-check "-t=${MESOS_DOCKER_MESOS_TIMEOUT}" http://mesosmaster1:5050/health && + await-health-check "-t=${MESOS_DOCKER_API_TIMEOUT}" http://apiserver:8888/healthz && km scheduler --address=$(resolveip scheduler) --hostname-override=scheduler @@ -148,10 +158,23 @@ scheduler: --mesos-master=mesosmaster1:5050 --cluster-dns=10.10.10.10 --cluster-domain=cluster.local - --v=2 + --v=4 + environment: + - MESOS_DOCKER_ETCD_TIMEOUT + - MESOS_DOCKER_MESOS_TIMEOUT + - MESOS_DOCKER_API_TIMEOUT links: - etcd - mesosmaster1 - mesosslave1 - mesosslave2 - apiserver +keygen: + image: mesosphere/kubernetes-mesos-keygen + command: + - apiserver + - /var/run/kubernetes/auth + volumes: + - ${MESOS_DOCKER_WORK_DIR}/auth:/var/run/kubernetes/auth + links: + - apiserver diff --git a/cluster/mesos/docker/keygen/Dockerfile b/cluster/mesos/docker/keygen/Dockerfile new file mode 100644 index 00000000000..59d92371d7a --- /dev/null +++ b/cluster/mesos/docker/keygen/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:14.04.2 +MAINTAINER Mesosphere + +RUN locale-gen en_US.UTF-8 +RUN dpkg-reconfigure locales +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +RUN apt-get update -qq && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \ + curl \ + openssl \ + && \ + apt-get clean + +COPY ./bin/* /usr/local/bin/ + +ENTRYPOINT ["kube-keygen.sh"] diff --git a/cluster/mesos/docker/keygen/bin/kube-cagen.sh b/cluster/mesos/docker/keygen/bin/kube-cagen.sh new file mode 100755 index 00000000000..a68ad432761 --- /dev/null +++ b/cluster/mesos/docker/keygen/bin/kube-cagen.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates root certificate authority crt and key. +# Writes to (use docker volume or docker export to retrieve files). +# Params: +# out_dir - dir to write crt and key to + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +source "util-ssl.sh" + +out_dir="${1:-}" +[ -z "${out_dir}" ] && echo "No out_dir supplied (param 1)" && exit 1 + +cluster::mesos::docker::create_root_certificate_authority "${out_dir}" diff --git a/cluster/mesos/docker/keygen/bin/kube-keygen.sh b/cluster/mesos/docker/keygen/bin/kube-keygen.sh new file mode 100755 index 00000000000..b990950bc18 --- /dev/null +++ b/cluster/mesos/docker/keygen/bin/kube-keygen.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates apiserver crt and key. +# Requires provided hostname to be resolvable (use docker link). +# Requires root certificate in (use docker volume). +# Writes to (use docker volume or docker export to retrieve files). +# Params: +# hostname - host name of the Kubernetes API Server to resolve into an IP +# in_dir - dir to read root certificate from +# out_dir - (Optional) dir to write crt and key to (default=) + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +source "util-ssl.sh" + +hostname="${1:-}" +[ -z "${hostname}" ] && echo "No hostname supplied (param 1)" && exit 1 + +in_dir="${2:-}" +[ -z "${in_dir}" ] && echo "No in_dir supplied (param 2)" && exit 1 + +out_dir="${3:-${in_dir}}" + +# Certificate generation depends on IP being resolvable from the provided hostname. +apiserver_ip="$(resolveip ${hostname})" +apiservice_ip="10.10.10.1" #TODO(karlkfi): extract config + +cluster::mesos::docker::create_apiserver_cert "${in_dir}" "${out_dir}" "${apiserver_ip}" "${apiservice_ip}" diff --git a/cluster/mesos/docker/keygen/build.sh b/cluster/mesos/docker/keygen/build.sh new file mode 100755 index 00000000000..d753d539c60 --- /dev/null +++ b/cluster/mesos/docker/keygen/build.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds a docker image that generates ssl certificates/keys/tokens required by kubernetes + +set -o errexit +set -o nounset +set -o pipefail + +IMAGE_REPO=${IMAGE_REPO:-mesosphere/kubernetes-mesos-keygen} +IMAGE_TAG=${IMAGE_TAG:-latest} + +script_dir=$(cd $(dirname "${BASH_SOURCE}") && pwd -P) +common_bin_path=$(cd ${script_dir}/../common/bin && pwd -P) +KUBE_ROOT=$(cd ${script_dir}/../../../.. && pwd -P) + +source "${common_bin_path}/util-temp-dir.sh" + +cd "${KUBE_ROOT}" + +function build_image { + local -r workspace="$(pwd)" + + echo "Copying files to workspace" + + # binaries & scripts + mkdir -p "${workspace}/bin" + cp -a "${common_bin_path}/"* "${workspace}/bin/" + cp -a "${script_dir}/bin/"* "${workspace}/bin/" + + # docker + cp -a "${script_dir}/Dockerfile" "${workspace}/" + + echo "Building docker image ${IMAGE_REPO}:${IMAGE_TAG}" + set -o xtrace + docker build -t ${IMAGE_REPO}:${IMAGE_TAG} "$@" . + set +o xtrace + echo "Built docker image ${IMAGE_REPO}:${IMAGE_TAG}" +} + +cluster::mesos::docker::run_in_temp_dir 'k8sm-keygen' 'build_image' diff --git a/cluster/mesos/docker/test/Dockerfile b/cluster/mesos/docker/test/Dockerfile index 19cd78a595a..93f5ced8a5d 100644 --- a/cluster/mesos/docker/test/Dockerfile +++ b/cluster/mesos/docker/test/Dockerfile @@ -26,12 +26,13 @@ RUN apt-get update -qq && \ # Install latest Docker # RUN curl -sSL https://get.docker.com/ubuntu/ | sh -# Install Docker 1.6.2 (docker 1.7 has regressions) -RUN echo deb https://get.docker.com/ubuntu docker main > /etc/apt/sources.list.d/docker.list && \ - apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 && \ +# Install Docker 1.8.1 explicitly +RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D && \ + mkdir -p /etc/apt/sources.list.d && \ + echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list && \ apt-get update -qq && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qqy \ - lxc-docker-1.6.2 \ + docker-engine=1.8.1-0~trusty \ && \ apt-get clean diff --git a/cluster/mesos/docker/util-ssl.sh b/cluster/mesos/docker/util-ssl.sh deleted file mode 100644 index e03b084bea1..00000000000 --- a/cluster/mesos/docker/util-ssl.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Sourcable SSL functions - -set -o errexit -set -o nounset -set -o pipefail - -script_dir=$(cd "$(dirname ${BASH_SOURCE})" && pwd -P) -source "${script_dir}/util-temp-dir.sh" - -function cluster::mesos::docker::find_openssl_config { - for candidate in "/etc/ssl/openssl.cnf" "/System/Library/OpenSSL/openssl.cnf"; do - if [ -f "${candidate}" ]; then - echo "${candidate}" - return 0 - fi - done - echo "ERROR: cannot find openssl.cnf" 1>&2 - return 1 -} - -function cluster::mesos::docker::create_root_certificate_authority { - local certdir="$1" - - openssl req -nodes -newkey rsa:2048 \ - -keyout "${certdir}/root-ca.key" \ - -out "${certdir}/root-ca.csr" \ - -subj "/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com" - - openssl x509 -req -days 3650 \ - -in "${certdir}/root-ca.csr" \ - -out "${certdir}/root-ca.crt" \ - -signkey "${certdir}/root-ca.key" -} - -# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names. -# Uses the current dir for scratch work. -function cluster::mesos::docker::create_apiserver_cert_inner { - local in_dir="$1" # must contain root-ca.crt & root-ca.key - local out_dir="$2" - local apiserver_ip="$3" - local service_ip="$4" - local workspace="$(pwd)" - - mkdir -p "${out_dir}" - - local OPENSSL_CNF=$(cluster::mesos::docker::find_openssl_config) - - # create apiserver key and certificate sign request - local SANS="IP:${apiserver_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local" - openssl req -nodes -newkey rsa:2048 \ - -keyout "${workspace}/apiserver.key" -out "${workspace}/apiserver.csr" \ - -reqexts SAN -config <(cat "${OPENSSL_CNF}"; echo -e "[SAN]\nsubjectAltName=$SANS") \ - -subj "/C=GB/ST=London/L=London/O=example/OU=IT/CN=example.com" - - # sign with root-ca - mkdir -p ${workspace}/demoCA/newcerts - touch ${workspace}/demoCA/index.txt - echo 1000 > ${workspace}/demoCA/serial - openssl ca -cert "${in_dir}/root-ca.crt" -keyfile "${in_dir}/root-ca.key" \ - -batch -days 3650 -in "${workspace}/apiserver.csr" \ - -config <(sed 's/.*\(copy_extensions = copy\)/\1/' ${OPENSSL_CNF}) >/dev/null - - # check certificate for subjectAltName extension - if ! openssl x509 -in "${workspace}/demoCA/newcerts/1000.pem" -text -noout | grep -q kubernetes.default.svc.cluster.local; then - echo "ERROR: openssl failed to add subjectAltName extension" 1>&2 - return 1 - fi - - # write to out_dir - cp "${workspace}/demoCA/newcerts/1000.pem" "${out_dir}/apiserver.crt" - cp "${workspace}/apiserver.key" "${out_dir}/" -} - -# Creates an apiserver key and certificate with the given IPs & kubernetes.* domain names. -function cluster::mesos::docker::create_apiserver_cert { - local in_dir="$1" # must contain root-ca.crt & root-ca.key - local out_dir="$2" - local apiserver_ip="$3" - local service_ip="$4" - - cluster::mesos::docker::run_in_temp_dir "k8sm-certs" \ - "cluster::mesos::docker::create_apiserver_cert_inner" \ - "${in_dir}" "${out_dir}" "${apiserver_ip}" "${service_ip}" -} - -# Creates an rsa key (for signing service accounts). -function cluster::mesos::docker::create_rsa_key { - local key_file_path="$1" - openssl genrsa -out "${key_file_path}" 2048 -} - -# Creates a k8s token auth user file. -# See /docs/admin/authentication.md -function cluster::mesos::docker::create_token_user { - local user_name="$1" - echo "$(openssl rand -hex 32),${user_name},${user_name}" -} - -# Creates a k8s basic auth user file. -# See /docs/admin/authentication.md -function cluster::mesos::docker::create_basic_user { - local user_name="$1" - local password="$2" - echo "${password},${user_name},${user_name}" -} \ No newline at end of file diff --git a/cluster/mesos/docker/util.sh b/cluster/mesos/docker/util.sh index b202883c852..ff51e8eee69 100644 --- a/cluster/mesos/docker/util.sh +++ b/cluster/mesos/docker/util.sh @@ -23,6 +23,7 @@ set -o errexit set -o nounset set -o pipefail +set -o errtrace KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE}")/../../.." && pwd) provider_root="${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}" @@ -30,13 +31,13 @@ compose_file="${provider_root}/docker-compose.yml" source "${provider_root}/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" -source "${provider_root}/util-ssl.sh" +source "${provider_root}/common/bin/util-ssl.sh" # Run kubernetes scripts inside docker. # This bypasses the need to set up network routing when running docker in a VM (e.g. boot2docker). # Trap signals and kills the docker container for better signal handing -function cluster::mesos::docker::run_in_docker { +function cluster::mesos::docker::run_in_docker_test { entrypoint="$1" if [[ "${entrypoint}" = "./"* ]]; then # relative to project root @@ -45,12 +46,18 @@ function cluster::mesos::docker::run_in_docker { shift args="$@" + # only mount KUBECONFIG if it exists, otherwise the directory will be created/owned by root + kube_config_mount="" + if [ -n "${KUBECONFIG:-}" ] && [ -e "${KUBECONFIG}" ]; then + kube_config_mount="-v \"$(dirname ${KUBECONFIG}):/root/.kube\"" + fi + container_id=$( docker run \ -d \ -e "KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER}" \ -v "${KUBE_ROOT}:/go/src/github.com/GoogleCloudPlatform/kubernetes" \ - -v "$(dirname ${KUBECONFIG}):/root/.kube" \ + ${kube_config_mount} \ -v "/var/run/docker.sock:/var/run/docker.sock" \ --link docker_mesosmaster1_1:mesosmaster1 \ --link docker_mesosslave1_1:mesosslave1 \ @@ -77,8 +84,39 @@ function cluster::mesos::docker::run_in_docker { return "${exit_status}" } +# Run kube-cagen.sh inside docker. +# Creating and signing in the same environment avoids a subject comparison string_mask issue. +function cluster::mesos::docker::run_in_docker_cagen { + out_dir="$1" + + container_id=$( + docker run \ + -d \ + -v "${out_dir}:/var/run/kubernetes/auth" \ + --entrypoint="kube-cagen.sh" \ + mesosphere/kubernetes-mesos-keygen \ + "/var/run/kubernetes/auth" + ) + + docker logs -f "${container_id}" & + + # trap and kill for better signal handing + trap 'echo "Killing container ${container_id}" 1>&2 && docker kill ${container_id}' INT TERM + exit_status=$(docker wait "${container_id}") + trap - INT TERM + + if [ "$exit_status" != 0 ]; then + echo "Exited ${exit_status}" 1>&2 + fi + + docker rm -f "${container_id}" > /dev/null + + return "${exit_status}" +} + # Generate kubeconfig data for the created cluster. function create-kubeconfig { + local -r auth_dir="${MESOS_DOCKER_WORK_DIR}/auth" local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" export CONTEXT="${KUBERNETES_PROVIDER}" @@ -89,8 +127,8 @@ function create-kubeconfig { touch "${KUBECONFIG}" fi - local token="$(cut -d, -f1 ${provider_root}/certs/apiserver/token-users)" - "${kubectl}" config set-cluster "${CONTEXT}" --server="${KUBE_SERVER}" --certificate-authority="${provider_root}/certs/root-ca.crt" + local token="$(cut -d, -f1 ${auth_dir}/token-users)" + "${kubectl}" config set-cluster "${CONTEXT}" --server="${KUBE_SERVER}" --certificate-authority="${auth_dir}/root-ca.crt" "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="cluster-admin" "${kubectl}" config set-credentials cluster-admin --token="${token}" "${kubectl}" config use-context "${CONTEXT}" --cluster="${CONTEXT}" @@ -117,7 +155,7 @@ function detect-master { docker_id=$(docker ps --filter="name=docker_apiserver" --quiet) if [[ "${docker_id}" == *'\n'* ]]; then - echo "ERROR: Multiple API Servers running in docker" 1>&2 + echo "ERROR: Multiple API Servers running" 1>&2 return 1 fi @@ -135,6 +173,10 @@ function detect-master { # but might not have a Kublet running unless a kubernetes task has been scheduled on them. function detect-minions { docker_ids=$(docker ps --filter="name=docker_mesosslave" --quiet) + if [ -z "${docker_ids}" ]; then + echo "ERROR: Mesos slave(s) not running" 1>&2 + return 1 + fi while read -r docker_id; do minion_ip=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "${docker_id}") KUBE_MINION_IP_ADDRESSES+=("${minion_ip}") @@ -150,70 +192,90 @@ function verify-prereqs { # mesosphere/kubernetes-mesos-test, etc. } -# Instantiate a kubernetes cluster. -function kube-up { - # TODO: version images (k8s version, git sha, and dirty state) to avoid re-building them every time. - if [ "${MESOS_DOCKER_SKIP_BUILD:-false}" != "true" ]; then - echo "Building Docker images" 1>&2 - "${provider_root}/km/build.sh" - "${provider_root}/test/build.sh" - fi +# Initialize +function cluster::mesos::docker::init_auth { + local -r auth_dir="${MESOS_DOCKER_WORK_DIR}/auth" - local certdir="${provider_root}/certs" - - # create mount volume dirs - local apiserver_certs_dir="${certdir}/apiserver" - local controller_certs_dir="${certdir}/controller" - # clean certs directory from previous runs - if [ -d "${apiserver_certs_dir}" ]; then - rm -rf "${apiserver_certs_dir}"/* - fi - mkdir -p "${apiserver_certs_dir}" "${controller_certs_dir}" + #TODO(karlkfi): reuse existing credentials/certs/keys + # Nuke old auth + echo "Creating Auth Dir: ${auth_dir}" 1>&2 + mkdir -p "${auth_dir}" + rm -rf "${auth_dir}"/* echo "Creating root certificate authority" 1>&2 - cluster::mesos::docker::create_root_certificate_authority "${certdir}" - cp "${certdir}/root-ca.crt" "${controller_certs_dir}/" + cluster::mesos::docker::run_in_docker_cagen "${auth_dir}" echo "Creating service-account rsa key" 1>&2 - cluster::mesos::docker::create_rsa_key "${certdir}/service-accounts.key" - cp "${certdir}/service-accounts.key" "${apiserver_certs_dir}/" - cp "${certdir}/service-accounts.key" "${controller_certs_dir}/" + cluster::mesos::docker::create_rsa_key "${auth_dir}/service-accounts.key" echo "Creating cluster-admin token user" 1>&2 - cluster::mesos::docker::create_token_user "cluster-admin" > "${apiserver_certs_dir}/token-users" + cluster::mesos::docker::create_token_user "cluster-admin" > "${auth_dir}/token-users" echo "Creating admin basic auth user" 1>&2 - cluster::mesos::docker::create_basic_user "admin" "admin" > "${apiserver_certs_dir}/basic-users" + cluster::mesos::docker::create_basic_user "admin" "admin" > "${auth_dir}/basic-users" +} - # log dump on failure - trap 'cluster::mesos::docker::dump_logs' ERR +# Instantiate a kubernetes cluster. +function kube-up { + # Nuke old mesos-slave workspaces + for ((i=1; i <= NUM_MINIONS; i++)) do + local work_dir="${MESOS_DOCKER_WORK_DIR}/mesosslave${i}/mesos" + echo "Creating Mesos Work Dir: ${work_dir}" 1>&2 + mkdir -p "${work_dir}" + rm -rf "${work_dir}"/* + done + + local -r log_dir="${MESOS_DOCKER_WORK_DIR}/log" + + # Nuke old logs + mkdir -p "${log_dir}" + rm -rf "${log_dir}"/* + + if [ "${MESOS_DOCKER_SKIP_BUILD:-false}" != "true" ]; then + # Pull before `docker-compose up` to avoid timeouts between container runs. + # Pull before building images (but only if built) to avoid overwriting locally built images. + echo "Pulling docker images" 1>&2 + docker-compose -f "${compose_file}" pull + + echo "Building Docker images" 1>&2 + # TODO: version images (k8s version, git sha, and dirty state) to avoid re-building them every time. + "${provider_root}/km/build.sh" + "${provider_root}/test/build.sh" + "${provider_root}/keygen/build.sh" + fi + + cluster::mesos::docker::init_auth + + # Dump logs on premature exit (errexit triggers exit). + # Trap EXIT instead of ERR, because ERR can trigger multiple times with errtrace enabled. + trap "cluster::mesos::docker::dump_logs '${log_dir}'" EXIT echo "Starting ${KUBERNETES_PROVIDER} cluster" 1>&2 + export MESOS_DOCKER_ETCD_TIMEOUT="${MESOS_DOCKER_ETCD_TIMEOUT}" + export MESOS_DOCKER_MESOS_TIMEOUT="${MESOS_DOCKER_MESOS_TIMEOUT}" + export MESOS_DOCKER_API_TIMEOUT="${MESOS_DOCKER_API_TIMEOUT}" + export KUBE_KEYGEN_TIMEOUT="${KUBE_KEYGEN_TIMEOUT}" + export MESOS_DOCKER_WORK_DIR="${MESOS_DOCKER_WORK_DIR}" + export MESOS_DOCKER_IMAGE_DIR="${MESOS_DOCKER_IMAGE_DIR}" + export DOCKER_DAEMON_ARGS="${DOCKER_DAEMON_ARGS}" docker-compose -f "${compose_file}" up -d + # await-health-check requires GNU timeout + # apiserver hostname resolved by docker + cluster::mesos::docker::run_in_docker_test await-health-check "-t=${MESOS_DOCKER_API_TIMEOUT}" http://apiserver:8888/healthz + detect-master detect-minions - - # The apiserver is waiting for its certificate, which depends on the IP docker chose. - echo "Creating apiserver certificate" 1>&2 - local apiserer_ip="$(cut -f1 -d: <<<${KUBE_MASTER_IP})" - local apiservice_ip="10.10.10.1" - cluster::mesos::docker::create_apiserver_cert \ - "${certdir}" "${apiserver_certs_dir}" "${apiserer_ip}" "${apiservice_ip}" - - # KUBECONFIG needs to exist before run_in_docker mounts it, otherwise it will be owned by root create-kubeconfig - # await-health-check could be run locally, but it would require GNU timeout installed on mac... - # "${provider_root}/common/bin/await-health-check" -t=120 ${KUBE_SERVER}/healthz - cluster::mesos::docker::run_in_docker await-health-check -t=120 http://apiserver:8888/healthz - echo "Deploying Addons" 1>&2 KUBE_SERVER=${KUBE_SERVER} "${provider_root}/deploy-addons.sh" # Wait for addons to deploy - cluster::mesos::docker::await_ready "kube-dns" - cluster::mesos::docker::await_ready "kube-ui" + cluster::mesos::docker::await_ready "kube-dns" "${MESOS_DOCKER_ADDON_TIMEOUT}" + cluster::mesos::docker::await_ready "kube-ui" "${MESOS_DOCKER_ADDON_TIMEOUT}" + + trap - EXIT } function validate-cluster { @@ -236,10 +298,7 @@ function kube-down { } function test-setup { - echo "Building required docker images" 1>&2 - "${KUBE_ROOT}/cluster/mesos/docker/km/build.sh" - "${KUBE_ROOT}/cluster/mesos/docker/test/build.sh" - "${KUBE_ROOT}/cluster/mesos/docker/mesos-slave/build.sh" + echo "TODO: test-setup" 1>&2 } # Execute after running tests to perform any required clean-up @@ -267,8 +326,8 @@ function restart-apiserver { # Waits for a kube-system pod (of the provided name) to have the phase/status "Running". function cluster::mesos::docker::await_ready { - local pod_name=$1 - local max_attempts=60 + local pod_name="$1" + local max_attempts="$2" local phase="Unknown" echo -n "${pod_name}: " local n=0 @@ -287,7 +346,7 @@ function cluster::mesos::docker::await_ready { # Prints the status of the kube-system pod specified function cluster::mesos::docker::addon_status { - local pod_name=$1 + local pod_name="$1" local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" local phase=$("${kubectl}" get pods --namespace=kube-system -l k8s-app=${pod_name} -o template --template="{{(index .items 0).status.phase}}" 2>/dev/null) phase="${phase:-Unknown}" @@ -295,10 +354,10 @@ function cluster::mesos::docker::addon_status { } function cluster::mesos::docker::dump_logs { - local log_dir="${provider_root}/logs" - echo "Dumping logs to '${log_dir}'" 1>&2 - mkdir -p "${log_dir}" + local out_dir="$1" + echo "Dumping logs to '${out_dir}'" 1>&2 + mkdir -p "${out_dir}" while read name; do - docker logs "${name}" &> "${log_dir}/${name}.log" + docker logs "${name}" &> "${out_dir}/${name}.log" done < <(docker-compose -f "${compose_file}" ps -q | xargs docker inspect --format '{{.Name}}') }