Merge pull request #2692 from jameskyle/develop

Support remote docker hosts on OS X.
This commit is contained in:
Joe Beda 2014-12-01 13:41:03 -08:00
commit 484388479c
2 changed files with 87 additions and 58 deletions

View File

@ -19,6 +19,10 @@ set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
DOCKER_OPTS=${DOCKER_OPTS:-""}
DOCKER_NATIVE=${DOCKER_NATIVE:-""}
DOCKER=(docker ${DOCKER_OPTS})
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
cd "${KUBE_ROOT}" cd "${KUBE_ROOT}"
@ -64,11 +68,13 @@ readonly LOCAL_OUTPUT_SUBPATH="${LOCAL_OUTPUT_ROOT}/dockerized"
readonly LOCAL_OUTPUT_BINPATH="${LOCAL_OUTPUT_SUBPATH}/bin" readonly LOCAL_OUTPUT_BINPATH="${LOCAL_OUTPUT_SUBPATH}/bin"
readonly LOCAL_OUTPUT_IMAGE_STAGING="${LOCAL_OUTPUT_ROOT}/images" readonly LOCAL_OUTPUT_IMAGE_STAGING="${LOCAL_OUTPUT_ROOT}/images"
readonly OUTPUT_BINPATH="${CUSTOM_OUTPUT_BINPATH:-$LOCAL_OUTPUT_BINPATH}"
readonly REMOTE_OUTPUT_ROOT="/go/src/${KUBE_GO_PACKAGE}/_output" readonly REMOTE_OUTPUT_ROOT="/go/src/${KUBE_GO_PACKAGE}/_output"
readonly REMOTE_OUTPUT_SUBPATH="${REMOTE_OUTPUT_ROOT}/dockerized" readonly REMOTE_OUTPUT_SUBPATH="${REMOTE_OUTPUT_ROOT}/dockerized"
readonly REMOTE_OUTPUT_BINPATH="${REMOTE_OUTPUT_SUBPATH}/bin" readonly REMOTE_OUTPUT_BINPATH="${REMOTE_OUTPUT_SUBPATH}/bin"
readonly DOCKER_MOUNT_ARGS_BASE=(--volume "${LOCAL_OUTPUT_BINPATH}:${REMOTE_OUTPUT_BINPATH}") readonly DOCKER_MOUNT_ARGS_BASE=(--volume "${OUTPUT_BINPATH}:${REMOTE_OUTPUT_BINPATH}")
# DOCKER_MOUNT_ARGS=("${DOCKER_MOUNT_ARGS_BASE[@]}" --volumes-from "${KUBE_BUILD_DATA_CONTAINER_NAME}") # DOCKER_MOUNT_ARGS=("${DOCKER_MOUNT_ARGS_BASE[@]}" --volumes-from "${KUBE_BUILD_DATA_CONTAINER_NAME}")
# We create a Docker data container to cache incremental build artifacts. We # We create a Docker data container to cache incremental build artifacts. We
@ -110,6 +116,7 @@ readonly RELEASE_DIR="${LOCAL_OUTPUT_ROOT}/release-tars"
# KUBE_BUILD_DATA_CONTAINER_NAME # KUBE_BUILD_DATA_CONTAINER_NAME
# DOCKER_MOUNT_ARGS # DOCKER_MOUNT_ARGS
function kube::build::verify_prereqs() { function kube::build::verify_prereqs() {
echo "+++ Verifying Prerequisites...."
if [[ -z "$(which docker)" ]]; then if [[ -z "$(which docker)" ]]; then
echo "Can't find 'docker' in PATH, please fix and retry." >&2 echo "Can't find 'docker' in PATH, please fix and retry." >&2
echo "See https://docs.docker.com/installation/#installation for installation instructions." >&2 echo "See https://docs.docker.com/installation/#installation for installation instructions." >&2
@ -117,22 +124,24 @@ function kube::build::verify_prereqs() {
fi fi
if kube::build::is_osx; then if kube::build::is_osx; then
if [[ -z "$(which boot2docker)" ]]; then if [[ -z "$DOCKER_NATIVE" ]];then
echo "It looks like you are running on Mac OS X and boot2docker can't be found." >&2 if [[ -z "$(which boot2docker)" ]]; then
echo "See: https://docs.docker.com/installation/mac/" >&2 echo "It looks like you are running on Mac OS X and boot2docker can't be found." >&2
exit 1 echo "See: https://docs.docker.com/installation/mac/" >&2
fi exit 1
if [[ $(boot2docker status) != "running" ]]; then fi
echo "boot2docker VM isn't started. Please run 'boot2docker start'" >&2 if [[ $(boot2docker status) != "running" ]]; then
exit 1 echo "boot2docker VM isn't started. Please run 'boot2docker start'" >&2
else exit 1
# Reach over and set the clock. After sleep/resume the clock will skew. else
echo "+++ Setting boot2docker clock" # Reach over and set the clock. After sleep/resume the clock will skew.
boot2docker ssh sudo date -u -D "%Y%m%d%H%M.%S" --set "$(date -u +%Y%m%d%H%M.%S)" >/dev/null echo "+++ Setting boot2docker clock"
boot2docker ssh sudo date -u -D "%Y%m%d%H%M.%S" --set "$(date -u +%Y%m%d%H%M.%S)" >/dev/null
fi
fi fi
fi fi
if ! docker info > /dev/null 2>&1 ; then if ! "${DOCKER[@]}" info > /dev/null 2>&1 ; then
{ {
echo "Can't connect to 'docker' daemon. please fix and retry." echo "Can't connect to 'docker' daemon. please fix and retry."
echo echo
@ -173,7 +182,7 @@ function kube::build::clean_output() {
fi fi
echo "+++ Removing data container" echo "+++ Removing data container"
docker rm -v "${KUBE_BUILD_DATA_CONTAINER_NAME}" >/dev/null 2>&1 || true "${DOCKER[@]}" rm -v "${KUBE_BUILD_DATA_CONTAINER_NAME}" >/dev/null 2>&1 || true
echo "+++ Cleaning out local _output directory" echo "+++ Cleaning out local _output directory"
rm -rf "${LOCAL_OUTPUT_ROOT}" rm -rf "${LOCAL_OUTPUT_ROOT}"
@ -213,7 +222,7 @@ function kube::build::docker_image_exists() {
# We cannot just specify the IMAGE here as `docker images` doesn't behave as # We cannot just specify the IMAGE here as `docker images` doesn't behave as
# expected. See: https://github.com/docker/docker/issues/8048 # expected. See: https://github.com/docker/docker/issues/8048
docker images | grep -Eq "^${1}\s+${2}\s+" "${DOCKER[@]}" images | grep -Eq "^${1}\s+${2}\s+"
} }
# Takes $1 and computes a short has for it. Useful for unique tag generation # Takes $1 and computes a short has for it. Useful for unique tag generation
@ -252,7 +261,7 @@ function kube::build::ensure_golang() {
} }
echo "+++ Pulling docker image: golang:${KUBE_BUILD_GOLANG_VERSION}" echo "+++ Pulling docker image: golang:${KUBE_BUILD_GOLANG_VERSION}"
docker pull golang:${KUBE_BUILD_GOLANG_VERSION} "${DOCKER[@]}" pull golang:${KUBE_BUILD_GOLANG_VERSION}
} }
} }
@ -327,7 +336,7 @@ function kube::build::run_image() {
function kube::build::docker_build() { function kube::build::docker_build() {
local -r image=$1 local -r image=$1
local -r context_dir=$2 local -r context_dir=$2
local -ra build_cmd=(docker build -t "${image}" "${context_dir}") local -ra build_cmd=("${DOCKER[@]}" build -t "${image}" "${context_dir}")
echo "+++ Building Docker image ${image}." echo "+++ Building Docker image ${image}."
local docker_output local docker_output
@ -350,7 +359,7 @@ function kube::build::clean_image() {
local -r image=$1 local -r image=$1
echo "+++ Deleting docker image ${image}" echo "+++ Deleting docker image ${image}"
docker rmi ${image} 2> /dev/null || true "${DOCKER[@]}" rmi ${image} 2> /dev/null || true
} }
function kube::build::clean_images() { function kube::build::clean_images() {
@ -364,14 +373,14 @@ function kube::build::clean_images() {
done done
echo "+++ Cleaning all other untagged docker images" echo "+++ Cleaning all other untagged docker images"
docker rmi $(docker images -q --filter 'dangling=true') 2> /dev/null || true "${DOCKER[@]}" rmi $("${DOCKER[@]}" images -q --filter 'dangling=true') 2> /dev/null || true
} }
function kube::build::ensure_data_container() { function kube::build::ensure_data_container() {
if ! docker inspect "${KUBE_BUILD_DATA_CONTAINER_NAME}" >/dev/null 2>&1; then if ! "${DOCKER[@]}" inspect "${KUBE_BUILD_DATA_CONTAINER_NAME}" >/dev/null 2>&1; then
echo "+++ Creating data container" echo "+++ Creating data container"
local -ra docker_cmd=( local -ra docker_cmd=(
docker run "${DOCKER[@]}" run
"${DOCKER_DATA_MOUNT_ARGS[@]}" "${DOCKER_DATA_MOUNT_ARGS[@]}"
--name "${KUBE_BUILD_DATA_CONTAINER_NAME}" --name "${KUBE_BUILD_DATA_CONTAINER_NAME}"
"${KUBE_BUILD_IMAGE}" "${KUBE_BUILD_IMAGE}"
@ -384,6 +393,7 @@ function kube::build::ensure_data_container() {
# Run a command in the kube-build image. This assumes that the image has # Run a command in the kube-build image. This assumes that the image has
# already been built. This will sync out all output data from the build. # already been built. This will sync out all output data from the build.
function kube::build::run_build_command() { function kube::build::run_build_command() {
echo "+++ Running build command...."
[[ $# != 0 ]] || { echo "Invalid input." >&2; return 4; } [[ $# != 0 ]] || { echo "Invalid input." >&2; return 4; }
kube::build::ensure_data_container kube::build::ensure_data_container
@ -405,16 +415,16 @@ function kube::build::run_build_command() {
fi fi
local -ra docker_cmd=( local -ra docker_cmd=(
docker run "${docker_run_opts[@]}" "${KUBE_BUILD_IMAGE}") "${DOCKER[@]}" run "${docker_run_opts[@]}" "${KUBE_BUILD_IMAGE}")
# Remove the container if it is left over from some previous aborted run # Remove the container if it is left over from some previous aborted run
docker rm -v "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true "${DOCKER[@]}" rm -f -v "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true
"${docker_cmd[@]}" "$@" "${docker_cmd[@]}" "$@"
# Remove the container after we run. '--rm' might be appropriate but it # Remove the container after we run. '--rm' might be appropriate but it
# appears that sometimes it fails. See # appears that sometimes it fails. See
# https://github.com/docker/docker/issues/3968 # https://github.com/docker/docker/issues/3968
docker rm -v "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true "${DOCKER[@]}" rm -f -v "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true
} }
# Test if the output directory is remote (and can only be accessed through # Test if the output directory is remote (and can only be accessed through
@ -430,33 +440,48 @@ function kube::build::is_output_remote() {
# If the Docker server is remote, copy the results back out. # If the Docker server is remote, copy the results back out.
function kube::build::copy_output() { function kube::build::copy_output() {
if kube::build::is_output_remote; then if kube::build::is_output_remote; then
# When we are on the Mac with boot2docker (or to a remote Docker in any # At time of this code, docker cp does not work when copying from a volume.
# other situation) we need to copy the results back out. Ideally we would # As a workaround, the binaries are first copied to a local filesystem,
# leave the container around and use 'docker cp' to copy the results out. # /tmp, then docker cp'd to the local binaries output directory.
# However, that doesn't work for mounted volumes currently # The fix for the volume bug has been accepted and once it's widely
# (https://github.com/dotcloud/docker/issues/1992). And it is just plain # deployed the code below should be simplified to a simple docker cp
# broken (https://github.com/dotcloud/docker/issues/6483). # Bug: https://github.com/docker/docker/pull/8509
# local -a docker_run_opts=(
# The easiest thing I (jbeda) could figure out was to launch another "--name=${KUBE_BUILD_CONTAINER_NAME}"
# container pointed at the same volume, tar the output directory and ship "${DOCKER_MOUNT_ARGS[@]}"
# that tar over stdout. -d
)
local -ra docker_cmd=(
"${DOCKER[@]}" run "${docker_run_opts[@]}" "${KUBE_BUILD_IMAGE}"
)
echo "+++ Syncing back _output/dockerized/bin directory from remote Docker" echo "+++ Syncing back _output/dockerized/bin directory from remote Docker"
rm -rf "${LOCAL_OUTPUT_BINPATH}" rm -rf "${LOCAL_OUTPUT_BINPATH}"
mkdir -p "${LOCAL_OUTPUT_BINPATH}" mkdir -p "${LOCAL_OUTPUT_BINPATH}"
# The '</dev/null' here makes us run docker in a "non-interactive" mode. Not # Remove the container if it is left over from some previous aborted run
# doing this corrupts the output stream. "${DOCKER[@]}" rm -f -v "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true
kube::build::run_build_command sh -c "tar c -C ${REMOTE_OUTPUT_BINPATH} . ; sleep 1" </dev/null \ "${docker_cmd[@]}" bash -c "cp -r ${REMOTE_OUTPUT_BINPATH} /tmp/bin;touch /tmp/finished;rm /tmp/bin/test_for_remote;/bin/sleep 600" > /dev/null 2>&1
| tar xv -C "${LOCAL_OUTPUT_BINPATH}"
# I (jbeda) also tried getting rsync working using 'docker run' as the # Wait until binaries have finished coppying
# 'remote shell'. This mostly worked but there was a hang when count=0
# closing/finishing things off. Ug. while true;do
# if docker "${DOCKER_OPTS}" cp "${KUBE_BUILD_CONTAINER_NAME}:/tmp/finished" "${LOCAL_OUTPUT_BINPATH}" > /dev/null 2>&1;then
# local DOCKER="docker run -i --rm --name=${KUBE_BUILD_CONTAINER_NAME} ${DOCKER_MOUNT} ${KUBE_BUILD_IMAGE}" docker "${DOCKER_OPTS}" cp "${KUBE_BUILD_CONTAINER_NAME}:/tmp/bin" "${LOCAL_OUTPUT_SUBPATH}"
# DOCKER+=" bash -c 'shift ; exec \"\$@\"' --" break;
# rsync --blocking-io -av -e "${DOCKER}" foo:${REMOTE_OUTPUT_BINPATH}/ ${LOCAL_OUTPUT_BINPATH} fi
let count=count+1
if [[ $count -eq 60 ]]; then
# break after 5m
echo "!!! Timed out waiting for binaries..."
break
fi
sleep 5
done
"${DOCKER[@]}" rm -f -v "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true
else else
echo "+++ Output directory is local. No need to copy results out." echo "+++ Output directory is local. No need to copy results out."
fi fi
@ -494,11 +519,12 @@ function kube::release::package_client_tarballs() {
client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}") client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}")
fi fi
local bin # This fancy expression will expand to prepend a path
for bin in "${client_bins[@]}"; do # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
cp "${LOCAL_OUTPUT_BINPATH}/${platform}/${bin}" \ # KUBE_CLIENT_BINARIES array.
"${release_stage}/client/bin/" cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
done "${release_stage}/client/bin/"
local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz" local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz"
kube::release::create_tarball "${package_name}" "${release_stage}/.." kube::release::create_tarball "${package_name}" "${release_stage}/.."
@ -679,7 +705,7 @@ function kube::release::gcs::ensure_release_bucket() {
function kube::release::gcs::ensure_docker_registry() { function kube::release::gcs::ensure_docker_registry() {
local -r reg_container_name="gcs-registry" local -r reg_container_name="gcs-registry"
local -r running=$(docker inspect ${reg_container_name} 2>/dev/null \ local -r running=$("${DOCKER[@]}" inspect ${reg_container_name} 2>/dev/null \
| build/json-extractor.py 0.State.Running 2>/dev/null) | build/json-extractor.py 0.State.Running 2>/dev/null)
[[ "$running" != "true" ]] || return 0 [[ "$running" != "true" ]] || return 0
@ -695,11 +721,11 @@ function kube::release::gcs::ensure_docker_registry() {
fi fi
# If we have an old one sitting around, remove it # If we have an old one sitting around, remove it
docker rm ${reg_container_name} >/dev/null 2>&1 || true "${DOCKER[@]}" rm ${reg_container_name} >/dev/null 2>&1 || true
echo "+++ Starting GCS backed Docker registry" echo "+++ Starting GCS backed Docker registry"
local -ra docker_cmd=( local -ra docker_cmd=(
docker run -d "--name=${reg_container_name}" "${DOCKER[@]}" run -d "--name=${reg_container_name}"
-e "GCS_BUCKET=${KUBE_GCS_RELEASE_BUCKET}" -e "GCS_BUCKET=${KUBE_GCS_RELEASE_BUCKET}"
-e "STORAGE_PATH=${KUBE_GCS_DOCKER_REG_PREFIX}" -e "STORAGE_PATH=${KUBE_GCS_DOCKER_REG_PREFIX}"
-e "GCP_OAUTH2_REFRESH_TOKEN=${refresh_token}" -e "GCP_OAUTH2_REFRESH_TOKEN=${refresh_token}"
@ -723,9 +749,9 @@ function kube::release::gcs::push_images() {
for b in "${KUBE_RUN_IMAGES[@]}" ; do for b in "${KUBE_RUN_IMAGES[@]}" ; do
image_name="${KUBE_RUN_IMAGE_BASE}-${b}" image_name="${KUBE_RUN_IMAGE_BASE}-${b}"
echo "+++ Tagging and pushing ${image_name} to GCS bucket ${KUBE_GCS_RELEASE_BUCKET}" echo "+++ Tagging and pushing ${image_name} to GCS bucket ${KUBE_GCS_RELEASE_BUCKET}"
docker tag "${KUBE_RUN_IMAGE_BASE}-$b" "localhost:5000/${image_name}" "${DOCKER[@]}" tag "${KUBE_RUN_IMAGE_BASE}-$b" "localhost:5000/${image_name}"
docker push "localhost:5000/${image_name}" "${DOCKER[@]}" push "localhost:5000/${image_name}"
docker rmi "localhost:5000/${image_name}" "${DOCKER[@]}" rmi "localhost:5000/${image_name}"
done done
} }

View File

@ -17,6 +17,9 @@
# This command builds and runs a local kubernetes cluster. It's just like # This command builds and runs a local kubernetes cluster. It's just like
# local-up.sh, but this one launches the three separate binaries. # local-up.sh, but this one launches the three separate binaries.
# You may need to run this as root to allow kubelet to open docker's socket. # You may need to run this as root to allow kubelet to open docker's socket.
DOCKER_OPTS=${DOCKER_OPTS:-""}
DOCKER_NATIVE=${DOCKER_NATIVE:-""}
DOCKER=(docker ${DOCKER_OPTS})
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
cd "${KUBE_ROOT}" cd "${KUBE_ROOT}"
@ -27,7 +30,7 @@ set -e
source "${KUBE_ROOT}/hack/lib/init.sh" source "${KUBE_ROOT}/hack/lib/init.sh"
"${KUBE_ROOT}/hack/build-go.sh" "${KUBE_ROOT}/hack/build-go.sh"
docker ps 2> /dev/null 1> /dev/null ${DOCKER[@]} ps 2> /dev/null 1> /dev/null
if [ "$?" != "0" ]; then if [ "$?" != "0" ]; then
echo "Failed to successfully run 'docker ps', please verify that docker is installed and \$DOCKER_HOST is set correctly." echo "Failed to successfully run 'docker ps', please verify that docker is installed and \$DOCKER_HOST is set correctly."
exit 1 exit 1