diff --git a/build/common.sh b/build/common.sh index dd40b9e65da..cf86b5ed74a 100644 --- a/build/common.sh +++ b/build/common.sh @@ -245,11 +245,16 @@ function kube::build::clean_images() { function kube::build::run_build_command() { [[ -n "$@" ]] || { echo "Invalid input." >&2; return 4; } - local -r docker="docker run --rm --name=${DOCKER_CONTAINER_NAME} -it ${DOCKER_MOUNT} ${KUBE_BUILD_IMAGE}" + local -r docker="docker run --name=${DOCKER_CONTAINER_NAME} -it ${DOCKER_MOUNT} ${KUBE_BUILD_IMAGE}" + # Remove the container if it is left over from some previous aborted run docker rm ${DOCKER_CONTAINER_NAME} >/dev/null 2>&1 || true - ${docker} "$@" + + # Remove the container after we run. '--rm' might be appropriate but it + # appears that sometimes it fails. See + # https://github.com/docker/docker/issues/3968 + docker rm ${DOCKER_CONTAINER_NAME} >/dev/null 2>&1 || true } # If the Docker server is remote, copy the results back out. @@ -264,7 +269,7 @@ function kube::build::copy_output() { # The easiest thing I (jbeda) could figure out was to launch another # container pointed at the same volume, tar the output directory and ship # that tar over stdou. - local -r docker="docker run -a stdout --rm --name=${DOCKER_CONTAINER_NAME} ${DOCKER_MOUNT} ${KUBE_BUILD_IMAGE}" + local -r docker="docker run -a stdout --name=${DOCKER_CONTAINER_NAME} ${DOCKER_MOUNT} ${KUBE_BUILD_IMAGE}" # Kill any leftover container docker rm ${DOCKER_CONTAINER_NAME} >/dev/null 2>&1 || true @@ -275,6 +280,11 @@ function kube::build::copy_output() { ${docker} sh -c "tar c -C ${REMOTE_OUTPUT_DIR} . ; sleep 1" \ | tar xv -C "${LOCAL_OUTPUT_BUILD}" + # Remove the container after we run. '--rm' might be appropriate but it + # appears that sometimes it fails. See + # https://github.com/docker/docker/issues/3968 + docker rm ${DOCKER_CONTAINER_NAME} >/dev/null 2>&1 || true + # I (jbeda) also tried getting rsync working using 'docker run' as the # 'remote shell'. This mostly worked but there was a hang when # closing/finishing things off. Ug. @@ -292,27 +302,10 @@ function kube::release::package_tarballs() { rm -rf "${RELEASE_DIR}" mkdir -p "${RELEASE_DIR}" - kube::release::package_full_tarball kube::release::package_client_tarballs - kube::release::package_server_tarball -} - -# This is all the stuff you need to run/install kubernetes. This includes: -# - precompiled binaries for client and server -# - Cluster spin up/down scripts and configs for various cloud providers -function kube::release::package_full_tarball() { - local release_stage="${LOCAL_OUTPUT_ROOT}/release-stage/full/kubernetes" - rm -rf "${release_stage}" - mkdir -p "${release_stage}" - - cp -R "${LOCAL_OUTPUT_ROOT}/build" "${release_stage}/platforms" - cp -R "${KUBE_REPO_ROOT}/cluster" "${release_stage}/" - cp -R "${KUBE_REPO_ROOT}/examples" "${release_stage}/" - cp "${KUBE_REPO_ROOT}/README.md" "${release_stage}/" - cp "${KUBE_REPO_ROOT}/LICENSE" "${release_stage}/" - - local package_name="${RELEASE_DIR}/kubernetes.tar.gz" - tar czf "${package_name}" -C "${release_stage}/.." . + kube::release::package_server_tarballs + kube::release::package_salt_tarball + kube::release::package_full_tarball } # Package up all of the cross compiled clients. Over time this should grow into @@ -323,43 +316,92 @@ function kube::release::package_client_tarballs() { platforms=($(cd "${LOCAL_OUTPUT_ROOT}/build" ; echo */*)) for platform in "${platforms[@]}" ; do local platform_tag=${platform/\//-} # Replace a "/" for a "-" - echo "+++ Building client tarball for $platform_tag" + echo "+++ Building tarball: client $platform_tag" - local release_stage="${LOCAL_OUTPUT_ROOT}/release-stage/client/${platform_tag}/kubernetes/client" + local release_stage="${LOCAL_OUTPUT_ROOT}/release-stage/client/${platform_tag}/kubernetes" rm -rf "${release_stage}" - mkdir -p "${release_stage}/bin" + mkdir -p "${release_stage}/client/bin" # This fancy expression will expand to prepend a path # (${LOCAL_OUTPUT_ROOT}/build/${platform}/) to every item in the # KUBE_CLIENT_BINARIES array. - cp "${KUBE_CLIENT_BINARIES[@]/#/${LOCAL_OUTPUT_ROOT}/build/${platform}/}" "${release_stage}/bin/" + cp "${KUBE_CLIENT_BINARIES[@]/#/${LOCAL_OUTPUT_ROOT}/build/${platform}/}" \ + "${release_stage}/client/bin/" local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz" - tar czf "${package_name}" -C "${release_stage}/../.." . + tar czf "${package_name}" -C "${release_stage}/.." . done } # Package up all of the server binaries -function kube::release::package_server_tarball() { +function kube::release::package_server_tarballs() { local platform for platform in "${KUBE_SERVER_PLATFORMS[@]}" ; do local platform_tag=${platform/\//-} # Replace a "/" for a "-" - echo "+++ Building server tarball for $platform_tag" + echo "+++ Building tarball: server $platform_tag" - local release_stage="${LOCAL_OUTPUT_ROOT}/release-stage/server/${platform_tag}/kubernetes/server" + local release_stage="${LOCAL_OUTPUT_ROOT}/release-stage/server/${platform_tag}/kubernetes" rm -rf "${release_stage}" - mkdir -p "${release_stage}/bin" + mkdir -p "${release_stage}/server/bin" # This fancy expression will expand to prepend a path # (${LOCAL_OUTPUT_ROOT}/build/${platform}/) to every item in the # KUBE_SERVER_BINARIES array. - cp "${KUBE_SERVER_BINARIES[@]/#/${LOCAL_OUTPUT_ROOT}/build/${platform}/}" "${release_stage}/bin/" + cp "${KUBE_SERVER_BINARIES[@]/#/${LOCAL_OUTPUT_ROOT}/build/${platform}/}" \ + "${release_stage}/server/bin/" local package_name="${RELEASE_DIR}/kubernetes-server-${platform_tag}.tar.gz" - tar czf "${package_name}" -C "${release_stage}/../.." . + tar czf "${package_name}" -C "${release_stage}/.." . done } +# Package up the salt configuration tree. This is an optional helper to getting +# a cluster up and running. +function kube::release::package_salt_tarball() { + echo "+++ Building tarball: salt" + + local release_stage="${LOCAL_OUTPUT_ROOT}/release-stage/salt/kubernetes" + rm -rf "${release_stage}" + mkdir -p "${release_stage}" + + cp -R "${KUBE_REPO_ROOT}/cluster/saltbase" "${release_stage}/" + + local package_name="${RELEASE_DIR}/kubernetes-salt.tar.gz" + tar czf "${package_name}" -C "${release_stage}/.." . +} + +# This is all the stuff you need to run/install kubernetes. This includes: +# - precompiled binaries for client +# - Cluster spin up/down scripts and configs for various cloud providers +# - tarballs for server binary and salt configs that are ready to be uploaded +# to master by whatever means appropriate. +function kube::release::package_full_tarball() { + echo "+++ Building tarball: full" + + local release_stage="${LOCAL_OUTPUT_ROOT}/release-stage/full/kubernetes" + rm -rf "${release_stage}" + mkdir -p "${release_stage}" + + cp -R "${LOCAL_OUTPUT_ROOT}/build" "${release_stage}/platforms" + + # We want everything in /cluster except saltbase. That is only needed on the + # server. + cp -R "${KUBE_REPO_ROOT}/cluster" "${release_stage}/" + rm -rf "${release_stage}/cluster/saltbase" + + mkdir -p "${release_stage}/server" + cp "${RELEASE_DIR}/kubernetes-salt.tar.gz" "${release_stage}/server/" + cp "${RELEASE_DIR}"/kubernetes-server-*.tar.gz "${release_stage}/server/" + + cp -R "${KUBE_REPO_ROOT}/examples" "${release_stage}/" + cp "${KUBE_REPO_ROOT}/README.md" "${release_stage}/" + cp "${KUBE_REPO_ROOT}/LICENSE" "${release_stage}/" + + local package_name="${RELEASE_DIR}/kubernetes.tar.gz" + tar czf "${package_name}" -C "${release_stage}/.." . +} + + # --------------------------------------------------------------------------- # GCS Release diff --git a/cluster/templates/download-release.sh b/cluster/gce/templates/download-release.sh similarity index 72% rename from cluster/templates/download-release.sh rename to cluster/gce/templates/download-release.sh index e1a61f4fe5b..3982dd0f50e 100755 --- a/cluster/templates/download-release.sh +++ b/cluster/gce/templates/download-release.sh @@ -20,13 +20,16 @@ # the release tar to download and unpack. It is meant to be pushed to the # master and run. -echo "Downloading release ($MASTER_RELEASE_TAR)" -gsutil cp $MASTER_RELEASE_TAR master-release.tgz +echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" +gsutil cp "$SERVER_BINARY_TAR_URL" . -echo "Unpacking release" -rm -rf master-release || false -tar xzf master-release.tgz +echo "Downloading binary release tar ($SALT_TAR_URL)" +gsutil cp "$SALT_TAR_URL" . + +echo "Unpacking Salt tree" +rm -rf kubernetes +tar xzf "${SALT_TAR_URL##*/}" echo "Running release install script" -sudo master-release/src/scripts/master-release-install.sh +sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}" diff --git a/cluster/templates/salt-master.sh b/cluster/gce/templates/salt-master.sh similarity index 95% rename from cluster/templates/salt-master.sh rename to cluster/gce/templates/salt-master.sh index e44f793663e..132497774da 100755 --- a/cluster/templates/salt-master.sh +++ b/cluster/gce/templates/salt-master.sh @@ -28,6 +28,10 @@ grains: cloud: gce EOF +cat </srv/pillar/cluster-params.sls +node_instance_prefix: $NODE_INSTANCE_PREFIX +EOF + # Auto accept all keys from minions that try to join mkdir -p /etc/salt/master.d cat </etc/salt/master.d/auto-accept.conf diff --git a/cluster/templates/salt-minion.sh b/cluster/gce/templates/salt-minion.sh similarity index 100% rename from cluster/templates/salt-minion.sh rename to cluster/gce/templates/salt-minion.sh diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 6052a141e7f..d52c1278b6f 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -20,29 +20,51 @@ # config-default.sh. source $(dirname ${BASH_SOURCE})/${KUBE_CONFIG_FILE-"config-default.sh"} -# Find the release to use. If passed in, go with that and validate. If not use -# the release/config.sh version assuming a dev workflow. -function find-release() { - if [ -n "$1" ]; then - RELEASE_NORMALIZED=$1 - else - local RELEASE_CONFIG_SCRIPT=$(dirname $0)/../release/config.sh - if [ -f $(dirname $0)/../release/config.sh ]; then - . $RELEASE_CONFIG_SCRIPT - normalize_release - fi - fi +# Verify prereqs +# +# Vars set: +# KUBE_REPO_ROOT +function verify-prereqs { + KUBE_REPO_ROOT="$(dirname ${BASH_SOURCE})/../.." - # Do one final check that we have a good release - if ! gsutil -q stat $RELEASE_NORMALIZED/master-release.tgz; then - echo "Could not find release tar. If developing, make sure you have run src/release/release.sh to create a release." 1>&2 + for x in gcloud gcutil gsutil; do + if [ "$(which $x)" == "" ]; then + echo "Can't find $x in PATH, please fix and retry." + exit 1 + fi + done +} + +# Verify and find the various tar files that we are going to use on the server. +# +# Vars set: +# SERVER_BINARY_TAR +# SALT_TAR +function find-release-tars { + SERVER_BINARY_TAR="${KUBE_REPO_ROOT}/server/kubernetes-server-linux-amd64.tar.gz" + if [[ ! -f "$SERVER_BINARY_TAR" ]]; then + SERVER_BINARY_TAR="${KUBE_REPO_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz" + fi + if [[ ! -f "$SERVER_BINARY_TAR" ]]; then + echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz" + exit 1 + fi + + SALT_TAR="${KUBE_REPO_ROOT}/server/kubernetes-salt.tar.gz" + if [[ ! -f "$SALT_TAR" ]]; then + SALT_TAR="${KUBE_REPO_ROOT}/_output/release-tars/kubernetes-salt.tar.gz" + fi + if [[ ! -f "$SALT_TAR" ]]; then + echo "!!! Cannot find kubernetes-salt.tar.gz" exit 1 fi - echo "Release: ${RELEASE_NORMALIZED}" } # Use the gcloud defaults to find the project. If it is already set in the # environment then go with that. +# +# Vars set: +# PROJECT function detect-project () { if [ -z "$PROJECT" ]; then PROJECT=$(gcloud config list project | tail -n 1 | cut -f 3 -d ' ') @@ -55,6 +77,52 @@ function detect-project () { echo "Project: $PROJECT (autodetected from gcloud config)" } + +# Take the local tar files and upload them to Google Storage. They will then be +# downloaded by the master as part of the start up script for the master. +# +# Assumed vars: +# PROJECT +# SERVER_BINARY_TAR +# SALT_TAR +# Vars set: +# SERVER_BINARY_TAR_URL +# SALT_TAR_URL +function upload-server-tars() { + SERVER_BINARY_TAR_URL= + SALT_TAR_URL= + + local project_hash + if which md5 > /dev/null 2>&1; then + project_hash=$(md5 -q -s "$PROJECT") + else + project_hash=$(echo -n "$PROJECT" | md5sum) + fi + + local -r staging_bucket="gs://kubernetes-staging-${project_hash}" + + # Ensure the bucket is created + if ! gsutil ls "$staging_bucket" > /dev/null 2>&1 ; then + echo "Creating $staging_bucket" + gsutil mb "${staging_bucket}" + fi + + local -r staging_path="${staging_bucket}/devel" + + echo "+++ Staging server tars to Google Storage: ${staging_path}" + SERVER_BINARY_TAR_URL="${staging_path}/${SERVER_BINARY_TAR##*/}" + gsutil -q cp "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_URL}" + SALT_TAR_URL="${staging_path}/${SALT_TAR##*/}" + gsutil -q cp "${SALT_TAR}" "${SALT_TAR_URL}" +} + +# Detect the information about the minions +# +# Assumed vars: +# MINION_NAMES +# ZONE +# Vars set: +# KUBE_MINION_IP_ADDRESS (array) function detect-minions () { KUBE_MINION_IP_ADDRESSES=() for (( i=0; i<${#MINION_NAMES[@]}; i++)); do @@ -75,6 +143,14 @@ function detect-minions () { fi } +# Detect the IP for the master +# +# Assumed vars: +# MASTER_NAME +# ZONE +# Vars set: +# KUBE_MASTER +# KUBE_MASTER_IP function detect-master () { KUBE_MASTER=${MASTER_NAME} if [ -z "$KUBE_MASTER_IP" ]; then @@ -90,58 +166,53 @@ function detect-master () { echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)" } +# Ensure that we have a password created for validating to the master. Will +# read from $HOME/.kubernetres_auth if available. +# +# Vars set: +# KUBE_USER +# KUBE_PASSWORD function get-password { - file=${HOME}/.kubernetes_auth - if [ -e ${file} ]; then - user=$(cat $file | python -c 'import json,sys;print json.load(sys.stdin)["User"]') - passwd=$(cat $file | python -c 'import json,sys;print json.load(sys.stdin)["Password"]') + local file="$HOME/.kubernetes_auth" + if [[ -r "$file" ]]; then + KUBE_USER=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["User"]') + KUBE_PASSWORD=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["Password"]') return fi - user=admin - passwd=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') + KUBE_USER=admin + KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') # Store password for reuse. - cat << EOF > ~/.kubernetes_auth + cat << EOF > "$file" { - "User": "$user", - "Password": "$passwd" + "User": "$KUBE_USER", + "Password": "$KUBE_PASSWORD" } EOF - chmod 0600 ~/.kubernetes_auth -} - -# Verify prereqs -function verify-prereqs { - for x in gcloud gcutil gsutil; do - if [ "$(which $x)" == "" ]; then - echo "Can't find $x in PATH, please fix and retry." 1>&2 - exit 1 - fi - done + chmod 0600 "$file" } # Instantiate a kubernetes cluster +# +# Assumed vars +# KUBE_REPO_ROOT +# function kube-up { - - # Find the release to use. Generally it will be passed when doing a 'prod' - # install and will default to the release/config.sh version when doing a - # developer up. - find-release $1 - # Detect the project into $PROJECT if it isn't set detect-project - # This will take us up to the git repo root - local base_dir=$(dirname "${BASH_SOURCE}")/../.. + # Make sure we have the tar files staged on Google Storage + find-release-tars + upload-server-tars # Build up start up script for master - KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) - trap 'rm -rf "${KUBE_TEMP}"' EXIT + local kube_temp=$(mktemp -d -t kubernetes.XXXXXX) + trap 'rm -rf "${kube_temp}"' EXIT get-password - python "${base_dir}/third_party/htpasswd/htpasswd.py" -b \ - -c "${KUBE_TEMP}/htpasswd" $user $passwd - HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd") + python "${KUBE_REPO_ROOT}/third_party/htpasswd/htpasswd.py" \ + -b -c "${kube_temp}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD" + local htpasswd=$(cat "${kube_temp}/htpasswd") if ! gcutil getnetwork "${NETWORK}"; then echo "Creating new network for: ${NETWORK}" @@ -175,12 +246,16 @@ function kube-up { ( echo "#! /bin/bash" - echo "MASTER_NAME='${MASTER_NAME}'" - echo "MASTER_RELEASE_TAR=${RELEASE_NORMALIZED}/master-release.tgz" - echo "MASTER_HTPASSWD='${HTPASSWD}'" - grep -v "^#" "${base_dir}/cluster/templates/download-release.sh" - grep -v "^#" "${base_dir}/cluster/templates/salt-master.sh" - ) > "${KUBE_TEMP}/master-start.sh" + echo "mkdir -p /var/cache/kubernetes-install" + echo "cd /var/cache/kubernetes-install" + echo "readonly MASTER_NAME='${MASTER_NAME}'" + echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'" + echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'" + echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" + echo "readonly MASTER_HTPASSWD='${htpasswd}'" + grep -v "^#" "${KUBE_REPO_ROOT}/cluster/gce/templates/download-release.sh" + grep -v "^#" "${KUBE_REPO_ROOT}/cluster/gce/templates/salt-master.sh" + ) > "${kube_temp}/master-start.sh" gcutil addinstance ${MASTER_NAME}\ --project ${PROJECT} \ @@ -193,15 +268,15 @@ function kube-up { --network ${NETWORK} \ --service_account_scopes="storage-ro,compute-rw" \ --automatic_restart \ - --metadata_from_file "startup-script:${KUBE_TEMP}/master-start.sh" & + --metadata_from_file "startup-script:${kube_temp}/master-start.sh" & for (( i=0; i<${#MINION_NAMES[@]}; i++)); do ( echo "#! /bin/bash" echo "MASTER_NAME='${MASTER_NAME}'" echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}" - grep -v "^#" "${base_dir}/cluster/templates/salt-minion.sh" - ) > ${KUBE_TEMP}/minion-start-${i}.sh + grep -v "^#" "${KUBE_REPO_ROOT}/cluster/gce/templates/salt-minion.sh" + ) > "${kube_temp}/minion-start-${i}.sh" gcutil addfirewall ${MINION_NAMES[$i]}-all \ --project ${PROJECT} \ @@ -223,7 +298,7 @@ function kube-up { --service_account_scopes=${MINION_SCOPES} \ --automatic_restart \ --can_ip_forward \ - --metadata_from_file "startup-script:${KUBE_TEMP}/minion-start-${i}.sh" & + --metadata_from_file "startup-script:${kube_temp}/minion-start-${i}.sh" & gcutil addroute ${MINION_NAMES[$i]} ${MINION_IP_RANGES[$i]} \ --project ${PROJECT} \ @@ -233,17 +308,17 @@ function kube-up { --next_hop_instance ${ZONE}/instances/${MINION_NAMES[$i]} & done - FAIL=0 + local fail=0 + local job for job in `jobs -p` do - wait $job || let "FAIL+=1" + wait $job || let "fail+=1" done - if (( $FAIL != 0 )); then - echo "${FAIL} commands failed. Exiting." + if (( $fail != 0 )); then + echo "${fail} commands failed. Exiting." exit 2 fi - detect-master > /dev/null echo "Waiting for cluster initialization." @@ -253,7 +328,7 @@ function kube-up { echo " up." echo - until $(curl --insecure --user ${user}:${passwd} --max-time 5 \ + until $(curl --insecure --user ${KUBE_USER}:${KUBE_PASSWORD} --max-time 5 \ --fail --output /dev/null --silent https://${KUBE_MASTER_IP}/api/v1beta1/pods); do printf "." sleep 2 @@ -264,16 +339,15 @@ function kube-up { sleep 5 - # Don't bail on errors, we want to be able to print some info. - set +e - # Basic sanity checking + local i + local rc # Capture return code without exiting because of errexit bash option for (( i=0; i<${#MINION_NAMES[@]}; i++)); do # Make sure docker is installed - gcutil ssh ${MINION_NAMES[$i]} which docker > /dev/null - if [ "$?" != "0" ]; then - echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly." 1>&2 - echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)" 1>&2 + gcutil ssh ${MINION_NAMES[$i]} which docker >/dev/null && rc=$? || rc=$? + if [[ "$rc" != "0" ]]; then + echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly." + echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)" exit 1 fi done @@ -286,29 +360,28 @@ function kube-up { echo "The user name and password to use is located in ~/.kubernetes_auth." echo - kube_cert=".kubecfg.crt" - kube_key=".kubecfg.key" - ca_cert=".kubernetes.ca.crt" + local kube_cert=".kubecfg.crt" + local kube_key=".kubecfg.key" + local ca_cert=".kubernetes.ca.crt" (umask 077 - gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/kubecfg.crt > "${HOME}/${kube_cert}" - gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/kubecfg.key > "${HOME}/${kube_key}" - gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/ca.crt > "${HOME}/${ca_cert}" + gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null + gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null + gcutil ssh "${MASTER_NAME}" sudo cat /usr/share/nginx/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null cat << EOF > ~/.kubernetes_auth { - "User": "$user", - "Password": "$passwd", + "User": "$KUBE_USER", + "Password": "$KUBE_PASSWORD", "CAFile": "$HOME/$ca_cert", "CertFile": "$HOME/$kube_cert", "KeyFile": "$HOME/$kube_key" } EOF - chmod 0600 ~/.kubernetes_auth - chmod 0600 "${HOME}/${kube_cert}" - chmod 0600 "${HOME}/${kube_key}" - chmod 0600 "${HOME}/${ca_cert}") + chmod 0600 ~/.kubernetes_auth "${HOME}/${kube_cert}" \ + "${HOME}/${kube_key}" "${HOME}/${ca_cert}" + ) } # Delete a kubernetes cluster @@ -362,22 +435,24 @@ function kube-down { # Update a kubernetes cluster with latest source function kube-push { - - # Find the release to use. Generally it will be passed when doing a 'prod' - # install and will default to the release/config.sh version when doing a - # developer up. - find-release $1 - - # Detect the project into $PROJECT + detect-project detect-master + # Make sure we have the tar files staged on Google Storage + find-release-tars + upload-server-tars + ( - echo MASTER_RELEASE_TAR=$RELEASE_NORMALIZED/master-release.tgz - grep -v "^#" $(dirname $0)/templates/download-release.sh + echo "#! /bin/bash" + echo "mkdir -p /var/cache/kubernetes-install" + echo "cd /var/cache/kubernetes-install" + echo "readonly SERVER_BINARY_TAR_URL=${SERVER_BINARY_TAR_URL}" + echo "readonly SALT_TAR_URL=${SALT_TAR_URL}" + grep -v "^#" "${KUBE_REPO_ROOT}/cluster/gce/templates/download-release.sh" echo "echo Executing configuration" echo "sudo salt '*' mine.update" echo "sudo salt --force-color '*' state.highstate" - ) | gcutil ssh --project ${PROJECT} --zone ${ZONE} $KUBE_MASTER bash + ) | gcutil ssh --project $PROJECT --zone $ZONE $KUBE_MASTER sudo bash get-password @@ -391,15 +466,27 @@ function kube-push { } -# Execute prior to running tests to build a release if required for env +# ----------------------------------------------------------------------------- +# Cluster specific test helpers used from hack/e2e-test.sh + +# Execute prior to running tests to build a release if required for env. +# +# Assumed Vars: +# KUBE_REPO_ROOT function test-build-release { # Build source - ${KUBE_REPO_ROOT}/hack/build-go.sh + "${KUBE_REPO_ROOT}/hack/build-go.sh" # Make a release - $(dirname $0)/../release/release.sh + "${KUBE_REPO_ROOT}/release/release.sh" } -# Execute prior to running tests to initialize required structure +# Execute prior to running tests to initialize required structure. This is +# called from hack/e2e-test.sh. +# +# Assumed vars: +# PROJECT +# ALREADY_UP +# Variables from config.sh function test-setup { # Detect the project into $PROJECT if it isn't set @@ -420,7 +507,11 @@ function test-setup { } -# Execute after running tests to perform any required clean-up +# Execute after running tests to perform any required clean-up. This is called +# from hack/e2e-test.sh +# +# Assumed Vars: +# PROJECT function test-teardown { echo "Shutting down test cluster in background." gcutil deletefirewall \ @@ -431,5 +522,3 @@ function test-teardown { ${MINION_TAG}-${INSTANCE_PREFIX}-http-alt || true > /dev/null $(dirname $0)/../cluster/kube-down.sh > /dev/null } - - diff --git a/cluster/saltbase/install.sh b/cluster/saltbase/install.sh new file mode 100755 index 00000000000..0768b3e51e2 --- /dev/null +++ b/cluster/saltbase/install.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script will set up the salt directory on the target server. It takes one +# argument that is a tarball with the pre-compiled kuberntes server binaries. + +set -o errexit +set -o nounset +set -o pipefail + +readonly SALTBASE="$(dirname $0)" + +readonly SERVER_BIN_TAR=${1-} +if [[ -z "$SERVER_BIN_TAR" ]]; then + echo "!!! No binaries specified" + exit 1 +fi + +# Create a temp dir for untaring +KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) +trap "rm -rf ${KUBE_TEMP}" EXIT + +# This file is meant to run on the master. It will install the salt configs +# into the appropriate place on the master. + +echo "+++ Installing salt files" +mkdir -p /srv +# This bash voodoo will prepend $SALTBASE to the start of each item in the +# $SALTDIRS array +readonly SALTDIRS=(salt pillar reactor) +cp -R --preserve=mode "${SALTDIRS[@]/#/${SALTBASE}/}" /srv/ + + +echo "+++ Install binaries from tar: $1" +tar -xz -C "${KUBE_TEMP}" -f "$1" +mkdir -p /srv/salt/kube-bins +cp "${KUBE_TEMP}/kubernetes/server/bin/"* /srv/salt/kube-bins/ diff --git a/cluster/saltbase/pillar/top.sls b/cluster/saltbase/pillar/top.sls index c50c4f4725d..297721a43d4 100644 --- a/cluster/saltbase/pillar/top.sls +++ b/cluster/saltbase/pillar/top.sls @@ -1,5 +1,5 @@ base: '*': - mine - - common + - cluster-params - privilege diff --git a/cluster/saltbase/salt/apiserver/default b/cluster/saltbase/salt/apiserver/default index 7380d60837f..44176f6cfa3 100644 --- a/cluster/saltbase/salt/apiserver/default +++ b/cluster/saltbase/salt/apiserver/default @@ -9,7 +9,7 @@ {% set cloud_provider = "-cloud_provider=" + grains.cloud_provider %} {% endif %} -{% set minion_regexp = "-minion_regexp='" + pillar['instance_prefix'] + ".*'" %} +{% set minion_regexp = "-minion_regexp='" + pillar['node_instance_prefix'] + ".*'" %} {% set address = "-address=127.0.0.1" %} {% if grains.etcd_servers is defined %} diff --git a/cluster/saltbase/salt/apiserver/init.sls b/cluster/saltbase/salt/apiserver/init.sls index 012d990cfc2..2d0909caedb 100644 --- a/cluster/saltbase/salt/apiserver/init.sls +++ b/cluster/saltbase/salt/apiserver/init.sls @@ -1,30 +1,9 @@ -{% set root = '/var/src/apiserver' %} -{% set package = 'github.com/GoogleCloudPlatform/kubernetes' %} -{% set package_dir = root + '/src/' + package %} -{% set go_opt = pillar['go_opt'] %} {% if grains['os_family'] == 'RedHat' %} {% set environment_file = '/etc/sysconfig/apiserver' %} {% else %} {% set environment_file = '/etc/default/apiserver' %} {% endif %} -{{ package_dir }}: - file.recurse: - - source: salt://apiserver/go - - user: root - {% if grains['os_family'] == 'RedHat' %} - - group: root - {% else %} - - group: staff - {% endif %} - - dir_mode: 775 - - file_mode: 664 - - makedirs: True - - recurse: - - user - - group - - mode - {{ environment_file }}: file.managed: - source: salt://apiserver/default @@ -33,22 +12,12 @@ - group: root - mode: 644 -apiserver-build: - cmd.run: - - cwd: {{ root }} - - names: - - go build {{ go_opt }} {{ package }}/cmd/apiserver - - env: - - PATH: {{ grains['path'] }}:/usr/local/bin - - GOPATH: {{ root }}:{{ package_dir }}/Godeps/_workspace - - require: - - file: {{ package_dir }} - /usr/local/bin/apiserver: - file.symlink: - - target: {{ root }}/apiserver - - watch: - - cmd: apiserver-build + file.managed: + - source: salt://kube-bins/apiserver + - user: root + - group: root + - mode: 755 {% if grains['os_family'] == 'RedHat' %} @@ -82,7 +51,6 @@ apiserver: service.running: - enable: True - watch: - - cmd: apiserver-build - file: {{ environment_file }} - file: /usr/local/bin/apiserver {% if grains['os_family'] != 'RedHat' %} diff --git a/cluster/saltbase/salt/controller-manager/init.sls b/cluster/saltbase/salt/controller-manager/init.sls index 289ed825892..790442ee6f6 100644 --- a/cluster/saltbase/salt/controller-manager/init.sls +++ b/cluster/saltbase/salt/controller-manager/init.sls @@ -1,30 +1,9 @@ -{% set root = '/var/src/controller-manager' %} -{% set package = 'github.com/GoogleCloudPlatform/kubernetes' %} -{% set package_dir = root + '/src/' + package %} -{% set go_opt = pillar['go_opt'] %} {% if grains['os_family'] == 'RedHat' %} {% set environment_file = '/etc/sysconfig/controller-manager' %} {% else %} {% set environment_file = '/etc/default/controller-manager' %} {% endif %} -{{ package_dir }}: - file.recurse: - - source: salt://controller-manager/go - - user: root - {% if grains['os_family'] == 'RedHat' %} - - group: root - {% else %} - - group: staff - {% endif %} - - dir_mode: 775 - - file_mode: 664 - - makedirs: True - - recurse: - - user - - group - - mode - {{ environment_file }}: file.managed: - source: salt://controller-manager/default @@ -33,22 +12,12 @@ - group: root - mode: 644 -controller-manager-build: - cmd.run: - - cwd: {{ root }} - - names: - - go build {{ go_opt }} {{ package }}/cmd/controller-manager - - env: - - PATH: {{ grains['path'] }}:/usr/local/bin - - GOPATH: {{ root }}:{{ package_dir }}/Godeps/_workspace - - require: - - file: {{ package_dir }} - /usr/local/bin/controller-manager: - file.symlink: - - target: {{ root }}/controller-manager - - watch: - - cmd: controller-manager-build + file.managed: + - source: salt://kube-bins/controller-manager + - user: root + - group: root + - mode: 755 {% if grains['os_family'] == 'RedHat' %} @@ -82,7 +51,6 @@ controller-manager: service.running: - enable: True - watch: - - cmd: controller-manager-build - file: /usr/local/bin/controller-manager - file: {{ environment_file }} {% if grains['os_family'] != 'RedHat' %} diff --git a/cluster/saltbase/salt/etcd/init.sls b/cluster/saltbase/salt/etcd/init.sls index 3e3df0e3bab..d7359ac2f34 100644 --- a/cluster/saltbase/salt/etcd/init.sls +++ b/cluster/saltbase/salt/etcd/init.sls @@ -1,21 +1,43 @@ -etcd-install: - git.latest: - - target: /var/src/etcd - - name: git://github.com/coreos/etcd - cmd.wait: - - cwd: /var/src/etcd - - name: | - git checkout ab4bcc18694644d12f0c038339d8d039072502b1 - ./build - - env: - - PATH: {{ grains['path'] }}:/usr/local/bin +{% set etcd_version="v0.4.6" %} +{% set etcd_tar_url="https://github.com/coreos/etcd/releases/download/%s/etcd-%s-linux-amd64.tar.gz" + | format(etcd_version, etcd_version) %} +{% set etcd_tar_hash="md5=2949e9163e59dc4f8db9ad92f3245b20" %} + +etcd-tar: + archive: + - extracted + - user: root + - name: /usr/local/src + - source: {{ etcd_tar_url }} + - source_hash: {{ etcd_tar_hash }} + - archive_format: tar + - if_missing: /usr/local/src/etcd-{{ etcd_version }}-linux-amd64 + - tar_options: z + file.directory: + - name: /usr/local/src/etcd-{{ etcd_version }}-linux-amd64 + - user: root + - group: root - watch: - - git: etcd-install + - archive: etcd-tar + - recurse: + - user + - group + +etcd-symlink: file.symlink: - name: /usr/local/bin/etcd - - target: /var/src/etcd/bin/etcd + - target: /usr/local/src/etcd-{{ etcd_version }}-linux-amd64/etcd + - force: true - watch: - - cmd: etcd-install + - archive: etcd-tar + +etcdctl-symlink: + file.symlink: + - name: /usr/local/bin/etcdctl + - target: /usr/local/src/etcd-{{ etcd_version }}-linux-amd64/etcdctl + - force: true + - watch: + - archive: etcd-tar etcd: group.present: @@ -25,8 +47,6 @@ etcd: - gid_from_name: True - shell: /sbin/nologin - home: /var/etcd - - require: - - group: etcd /etc/etcd: file.directory: @@ -84,5 +104,6 @@ etcd-service: - file: /usr/lib/systemd/system/etcd.service - file: /etc/default/etcd {% endif %} - - cmd: etcd-install + - file: etcd-tar + - file: etcd-symlink diff --git a/cluster/saltbase/salt/etcdctl/init.sls b/cluster/saltbase/salt/etcdctl/init.sls deleted file mode 100644 index a6dd90f42cd..00000000000 --- a/cluster/saltbase/salt/etcdctl/init.sls +++ /dev/null @@ -1,16 +0,0 @@ -etcdctl-install: - git.latest: - - target: /var/src/etcdctl - - name: git://github.com/coreos/etcdctl - cmd.wait: - - cwd: /var/src/etcdctl - - name: ./build - - env: - - PATH: {{ grains['path'] }}:/usr/local/bin - - watch: - - git: etcdctl-install - file.symlink: - - name: /usr/local/bin/etcdctl - - target: /var/src/etcdctl/bin/etcdctl - - watch: - - cmd: etcdctl-install diff --git a/cluster/saltbase/salt/golang.sls b/cluster/saltbase/salt/golang.sls deleted file mode 100644 index 557eba62dd0..00000000000 --- a/cluster/saltbase/salt/golang.sls +++ /dev/null @@ -1,23 +0,0 @@ -{% set go_version = '1.2.2' %} -{% set go_arch = 'linux-amd64' %} -{% set go_archive = 'go%s.%s.tar.gz' | format(go_version, go_arch) %} -{% set go_url = 'http://golang.org/dl/' + go_archive %} -{% set go_hash = 'sha1=6bd151ca49c435462c8bf019477a6244b958ebb5' %} - -get-golang: - file.managed: - - name: /var/cache/{{ go_archive }} - - source: {{ go_url }} - - source_hash: {{ go_hash }} - cmd.wait: - - cwd: /usr/local - - name: tar xzf /var/cache/{{ go_archive }} - - watch: - - file: get-golang - -install-golang: - file.symlink: - - name: /usr/local/bin/go - - target: /usr/local/go/bin/go - - watch: - - cmd: get-golang diff --git a/cluster/saltbase/salt/kube-proxy/init.sls b/cluster/saltbase/salt/kube-proxy/init.sls index 79fbd9ebfd6..05dfaf192f8 100644 --- a/cluster/saltbase/salt/kube-proxy/init.sls +++ b/cluster/saltbase/salt/kube-proxy/init.sls @@ -1,46 +1,15 @@ -{% set root = '/var/src/kube-proxy' %} -{% set package = 'github.com/GoogleCloudPlatform/kubernetes' %} -{% set package_dir = root + '/src/' + package %} -{% set go_opt = pillar['go_opt'] %} {% if grains['os_family'] == 'RedHat' %} {% set environment_file = '/etc/sysconfig/kube-proxy' %} {% else %} {% set environment_file = '/etc/default/kube-proxy' %} {% endif %} -{{ package_dir }}: - file.recurse: - - source: salt://kube-proxy/go - - user: root - {% if grains['os_family'] == 'RedHat' %} - - group: root - {% else %} - - group: staff - {% endif %} - - dir_mode: 775 - - file_mode: 664 - - makedirs: True - - recurse: - - user - - group - - mode - -kube-proxy-build: - cmd.run: - - cwd: {{ root }} - - names: - - go build {{ go_opt }} {{ package }}/cmd/proxy - - env: - - PATH: {{ grains['path'] }}:/usr/local/bin - - GOPATH: {{ root }}:{{ package_dir }}/Godeps/_workspace - - require: - - file: {{ package_dir }} - /usr/local/bin/kube-proxy: - file.symlink: - - target: {{ root }}/proxy - - watch: - - cmd: kube-proxy-build + file.managed: + - source: salt://kube-bins/proxy + - user: root + - group: root + - mode: 755 {% if grains['os_family'] == 'RedHat' %} @@ -82,7 +51,6 @@ kube-proxy: service.running: - enable: True - watch: - - cmd: kube-proxy-build - file: {{ environment_file }} {% if grains['os_family'] != 'RedHat' %} - file: /etc/init.d/kube-proxy diff --git a/cluster/saltbase/salt/kubelet/init.sls b/cluster/saltbase/salt/kubelet/init.sls index e1a006e89bb..bd71bb61cef 100644 --- a/cluster/saltbase/salt/kubelet/init.sls +++ b/cluster/saltbase/salt/kubelet/init.sls @@ -1,30 +1,9 @@ -{% set root = '/var/src/kubelet' %} -{% set package = 'github.com/GoogleCloudPlatform/kubernetes' %} -{% set package_dir = root + '/src/' + package %} -{% set go_opt = pillar['go_opt'] %} {% if grains['os_family'] == 'RedHat' %} {% set environment_file = '/etc/sysconfig/kubelet' %} {% else %} {% set environment_file = '/etc/default/kubelet' %} {% endif %} -{{ package_dir }}: - file.recurse: - - source: salt://kubelet/go - - user: root - {% if grains['os_family'] == 'RedHat' %} - - group: root - {% else %} - - group: staff - {% endif %} - - dir_mode: 775 - - file_mode: 664 - - makedirs: True - - recurse: - - user - - group - - mode - {{ environment_file}}: file.managed: - source: salt://kubelet/default @@ -33,22 +12,12 @@ - group: root - mode: 644 -kubelet-build: - cmd.run: - - cwd: {{ root }} - - names: - - go build {{ go_opt }} {{ package }}/cmd/kubelet - - env: - - PATH: {{ grains['path'] }}:/usr/local/bin - - GOPATH: {{ root }}:{{ package_dir }}/Godeps/_workspace - - require: - - file: {{ package_dir }} - /usr/local/bin/kubelet: - file.symlink: - - target: {{ root }}/kubelet - - watch: - - cmd: kubelet-build + file.managed: + - source: salt://kube-bins/kubelet + - user: root + - group: root + - mode: 755 {% if grains['os_family'] == 'RedHat' %} @@ -84,7 +53,6 @@ kubelet: service.running: - enable: True - watch: - - cmd: kubelet-build - file: /usr/local/bin/kubelet {% if grains['os_family'] != 'RedHat' %} - file: /etc/init.d/kubelet diff --git a/cluster/saltbase/salt/scheduler/init.sls b/cluster/saltbase/salt/scheduler/init.sls index 5deacd70c08..cf1b939a4f1 100644 --- a/cluster/saltbase/salt/scheduler/init.sls +++ b/cluster/saltbase/salt/scheduler/init.sls @@ -1,30 +1,9 @@ -{% set root = '/var/src/scheduler' %} -{% set package = 'github.com/GoogleCloudPlatform/kubernetes' %} -{% set package_dir = root + '/src/' + package %} -{% set go_opt = pillar['go_opt'] %} {% if grains['os_family'] == 'RedHat' %} {% set environment_file = '/etc/sysconfig/scheduler' %} {% else %} {% set environment_file = '/etc/default/scheduler' %} {% endif %} -{{ package_dir }}: - file.recurse: - - source: salt://scheduler/go - - user: root - {% if grains['os_family'] == 'RedHat' %} - - group: root - {% else %} - - group: staff - {% endif %} - - dir_mode: 775 - - file_mode: 664 - - makedirs: True - - recurse: - - user - - group - - mode - {{ environment_file }}: file.managed: - source: salt://scheduler/default @@ -33,22 +12,12 @@ - group: root - mode: 644 -scheduler-build: - cmd.run: - - cwd: {{ root }} - - names: - - go build {{ go_opt }} {{ package }}/plugin/cmd/scheduler - - env: - - PATH: {{ grains['path'] }}:/usr/local/bin - - GOPATH: {{ root }}:{{ package_dir }}/Godeps/_workspace - - require: - - file: {{ package_dir }} - /usr/local/bin/scheduler: - file.symlink: - - target: {{ root }}/scheduler - - watch: - - cmd: scheduler-build + file.managed: + - source: salt://kube-bins/scheduler + - user: root + - group: root + - mode: 755 {% if grains['os_family'] == 'RedHat' %} @@ -82,7 +51,6 @@ scheduler: service.running: - enable: True - watch: - - cmd: scheduler-build - file: /usr/local/bin/scheduler - file: {{ environment_file }} {% if grains['os_family'] != 'RedHat' %} diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 59c407f1bc8..69b383b192f 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -4,12 +4,12 @@ base: 'roles:kubernetes-pool': - match: grain - - golang - docker - kubelet - kube-proxy - cadvisor - - nsinit + # We need a binary release of nsinit + # - nsinit - logrotate {% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} - openvpn-client @@ -19,9 +19,7 @@ base: 'roles:kubernetes-master': - match: grain - - golang - etcd - - etcdctl - apiserver - controller-manager - scheduler diff --git a/cluster/validate-cluster.sh b/cluster/validate-cluster.sh index e585e1b3c21..217281b32a9 100755 --- a/cluster/validate-cluster.sh +++ b/cluster/validate-cluster.sh @@ -49,7 +49,7 @@ for (( i=0; i<${#MINION_NAMES[@]}; i++)); do fi # Make sure the kubelet is healthy - if [ "$(curl --insecure --user ${user}:${passwd} https://${KUBE_MASTER_IP}/proxy/minion/${MINION_NAMES[$i]}/healthz)" != "ok" ]; then + if [ "$(curl -s --insecure --user ${KUBE_USER}:${KUBE_PASSWORD} https://${KUBE_MASTER_IP}/proxy/minion/${MINION_NAMES[$i]}/healthz)" != "ok" ]; then echo "Kubelet failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly." echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)" exit 1 diff --git a/release/launch-kubernetes-base.sh b/release/launch-kubernetes-base.sh deleted file mode 100755 index 84fcfac116e..00000000000 --- a/release/launch-kubernetes-base.sh +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Prerequisites -# TODO (bburns): Perhaps install cloud SDK automagically if we can't find it? - -# Exit on any error -set -e - -echo "Auto installer for launching Kubernetes" -echo "Release: $RELEASE_PREFIX$RELEASE_NAME" - -# Make sure that prerequisites are installed. -for x in gcloud gcutil gsutil; do - if [ "$(which $x)" == "" ]; then - echo "Can't find $x in PATH, please fix and retry." - exit 1 - fi -done - -# TODO(jbeda): Provide a way to install this in to someplace beyond a temp dir -# so that users have access to local tools. -TMPDIR=$(mktemp -d /tmp/installer.kubernetes.XXXXXX) - -cd $TMPDIR - -echo "Downloading support files" -gsutil cp $RELEASE_FULL_PATH/launch-kubernetes.tgz . - -tar xzf launch-kubernetes.tgz - -./src/scripts/kube-up.sh $RELEASE_FULL_PATH - -cd / - -# clean up -# rm -rf $TMPDIR