github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-16 14:45:28 +00:00
Code Issues Projects Releases Wiki Activity

Use rsync to get source into build container

Browse Source 
We also add "version" to all docker images and containers

This version is to be incremented manually when we change the shape of the build
image (like changing the golang version or the set of volumes in the data
container).  This will delete all older versions of images and containers when
the version is different.
This commit is contained in:
Joe Beda
2016-08-17 10:45:47 -07:00
parent 22b7d90034
commit dc586ea8f7
12 changed files with 528 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

546
build/common.sh
View File

@@ -55,7 +55,15 @@ readonly KUBE_BUILD_PPC64LE="${KUBE_BUILD_PPC64LE:-n}"
# Constants
readonly KUBE_BUILD_IMAGE_REPO=kube-build
readonly KUBE_BUILD_IMAGE_CROSS_TAG="$(cat ${KUBE_ROOT}/build/build-image/cross/VERSION)"
# KUBE_BUILD_DATA_CONTAINER_NAME=kube-build-data-<hash>"
# This version number is used to cause everyone to rebuild their data containers
# and build image. This is especially useful for automated build systems like
# Jenkins.
#
# Increment/change this number if you change the build image (anything under
# build/build-image) or change the set of volumes in the data container.
readonly KUBE_BUILD_IMAGE_VERSION_BASE="$(cat ${KUBE_ROOT}/build/BUILD_IMAGE_VERSION)"
readonly KUBE_BUILD_IMAGE_VERSION="${KUBE_BUILD_IMAGE_VERSION_BASE}-${KUBE_BUILD_IMAGE_CROSS_TAG}"
# Here we map the output directories across both the local and remote _output
# directories:
@@ -76,17 +84,19 @@ readonly LOCAL_OUTPUT_IMAGE_STAGING="${LOCAL_OUTPUT_ROOT}/images"
# This is a symlink to binaries for "this platform" (e.g. build tools).
readonly THIS_PLATFORM_BIN="${LOCAL_OUTPUT_ROOT}/bin"
readonly REMOTE_OUTPUT_ROOT="/go/src/${KUBE_GO_PACKAGE}/_output"
readonly REMOTE_ROOT="/go/src/${KUBE_GO_PACKAGE}"
readonly REMOTE_OUTPUT_ROOT="${REMOTE_ROOT}/_output"
readonly REMOTE_OUTPUT_SUBPATH="${REMOTE_OUTPUT_ROOT}/dockerized"
readonly REMOTE_OUTPUT_BINPATH="${REMOTE_OUTPUT_SUBPATH}/bin"
readonly REMOTE_OUTPUT_GOPATH="${REMOTE_OUTPUT_SUBPATH}/go"
readonly DOCKER_MOUNT_ARGS_BASE=(
# where the container build will drop output
--volume "${LOCAL_OUTPUT_BINPATH}:${REMOTE_OUTPUT_BINPATH}"
# timezone
--volume /etc/localtime:/etc/localtime:ro
)
# This is the port on the workstation host to expose RSYNC on. Set this if you
# are doing something fancy with ssh tunneling.
readonly KUBE_RSYNC_PORT="${KUBE_RSYNC_PORT:-}"
# This is the port that rsync is running on *inside* the container. This may be
# mapped to KUBE_RSYNC_PORT via docker networking.
readonly KUBE_CONTAINER_RSYNC_PORT=8730
# This is where the final release artifacts are created locally
readonly RELEASE_STAGE="${LOCAL_OUTPUT_ROOT}/release-stage"
@@ -141,10 +151,15 @@ kube::build::get_docker_wrapped_binaries() {
#
# Vars set:
# KUBE_ROOT_HASH
# KUBE_BUILD_IMAGE_TAG_BASE
# KUBE_BUILD_IMAGE_TAG
# KUBE_BUILD_IMAGE
# KUBE_BUILD_CONTAINER_NAME_BASE
# KUBE_BUILD_CONTAINER_NAME
# KUBE_BUILD_DATA_CONTAINER_NAME
# KUBE_DATA_CONTAINER_NAME_BASE
# KUBE_DATA_CONTAINER_NAME
# KUBE_RSYNC_CONTAINER_NAME_BASE
# KUBE_RSYNC_CONTAINER_NAME
# DOCKER_MOUNT_ARGS
# LOCAL_OUTPUT_BUILD_CONTEXT
function kube::build::verify_prereqs() {
@@ -156,13 +171,26 @@ function kube::build::verify_prereqs() {
fi
kube::build::ensure_docker_daemon_connectivity || return 1
if (( ${KUBE_VERBOSE} > 6 )); then
kube::log::status "Docker Version:"
"${DOCKER[@]}" version | kube::log::info_from_stdin
fi
KUBE_ROOT_HASH=$(kube::build::short_hash "${HOSTNAME:-}:${KUBE_ROOT}")
KUBE_BUILD_IMAGE_TAG="build-${KUBE_ROOT_HASH}"
KUBE_BUILD_IMAGE_TAG_BASE="build-${KUBE_ROOT_HASH}"
KUBE_BUILD_IMAGE_TAG="${KUBE_BUILD_IMAGE_TAG_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
KUBE_BUILD_IMAGE="${KUBE_BUILD_IMAGE_REPO}:${KUBE_BUILD_IMAGE_TAG}"
KUBE_BUILD_CONTAINER_NAME="kube-build-${KUBE_ROOT_HASH}"
KUBE_BUILD_DATA_CONTAINER_NAME="kube-build-data-${KUBE_ROOT_HASH}"
DOCKER_MOUNT_ARGS=("${DOCKER_MOUNT_ARGS_BASE[@]}" --volumes-from "${KUBE_BUILD_DATA_CONTAINER_NAME}")
KUBE_BUILD_CONTAINER_NAME_BASE="kube-build-${KUBE_ROOT_HASH}"
KUBE_BUILD_CONTAINER_NAME="${KUBE_BUILD_CONTAINER_NAME_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
KUBE_RSYNC_CONTAINER_NAME_BASE="kube-rsync-${KUBE_ROOT_HASH}"
KUBE_RSYNC_CONTAINER_NAME="${KUBE_RSYNC_CONTAINER_NAME_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
KUBE_DATA_CONTAINER_NAME_BASE="kube-build-data-${KUBE_ROOT_HASH}"
KUBE_DATA_CONTAINER_NAME="${KUBE_DATA_CONTAINER_NAME_BASE}-${KUBE_BUILD_IMAGE_VERSION}"
DOCKER_MOUNT_ARGS=(--volumes-from "${KUBE_DATA_CONTAINER_NAME}")
LOCAL_OUTPUT_BUILD_CONTEXT="${LOCAL_OUTPUT_IMAGE_STAGING}/${KUBE_BUILD_IMAGE}"
kube::version::get_version_vars
kube::version::save_version_vars "${KUBE_ROOT}/.dockerized-kube-version-defs"
}
# ---------------------------------------------------------------------------
@@ -177,8 +205,8 @@ function kube::build::docker_available_on_osx() {
kube::log::status "No docker host is set. Checking options for setting one..."
if [[ -z "$(which docker-machine)" ]]; then
kube::log::status "It looks like you're running Mac OS X, yet neither Docker for Mac or docker-machine can be found."
kube::log::status "See: https://docs.docker.com/machine/ for installation instructions."
kube::log::status "It looks like you're running Mac OS X, yet neither Docker for Mac nor docker-machine can be found."
kube::log::status "See: https://docs.docker.com/engine/installation/mac/ for installation instructions."
return 1
elif [[ -n "$(which docker-machine)" ]]; then
kube::build::prepare_docker_machine
@@ -244,25 +272,23 @@ function kube::build::ensure_docker_in_path() {
function kube::build::ensure_docker_daemon_connectivity {
if ! "${DOCKER[@]}" info > /dev/null 2>&1 ; then
{
echo "Can't connect to 'docker' daemon. please fix and retry."
echo
echo "Possible causes:"
echo " - On Mac OS X, DOCKER_HOST hasn't been set. You may need to: "
echo " - Set up Docker for Mac (https://docs.docker.com/docker-for-mac/)"
echo " - Or, set up docker-machine"
echo " - Create and start your VM using docker-machine: "
echo " - docker-machine create -d ${DOCKER_MACHINE_DRIVER} ${DOCKER_MACHINE_NAME}"
echo " - Set your environment variables using: "
echo " - eval \$(docker-machine env ${DOCKER_MACHINE_NAME})"
echo " - Update your Docker VM"
echo " - Error Message: 'Error response from daemon: client is newer than server (...)' "
echo " - docker-machine upgrade ${DOCKER_MACHINE_NAME}"
echo " - On Linux, user isn't in 'docker' group. Add and relogin."
echo " - Something like 'sudo usermod -a -G docker ${USER-user}'"
echo " - RHEL7 bug and workaround: https://bugzilla.redhat.com/show_bug.cgi?id=1119282#c8"
echo " - On Linux, Docker daemon hasn't been started or has crashed."
} >&2
cat <<'EOF' >&2
Can't connect to 'docker' daemon. please fix and retry.
Possible causes:
- Docker Daemon not started
- Linux: confirm via your init system
- macOS w/ docker-machine: run `docker-machine ls` and `docker-machine start <name>`
- macOS w/ Docker for Mac: Check the menu bar and start the Docker application
- DOCKER_HOST hasn't been set of is set incorrectly
- Linux: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}`
- macOS w/ docker-machine: run `eval "$(docker-machine env <name>)"`
- macOS w/ Docker for Mac: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}`
- Other things to check:
- Linux: User isn't in 'docker' group. Add and relogin.
- Something like 'sudo usermod -a -G docker ${USER}'
- RHEL7 bug and workaround: https://bugzilla.redhat.com/show_bug.cgi?id=1119282#c8
EOF
return 1
fi
}
@@ -288,57 +314,6 @@ function kube::build::ensure_tar() {
fi
}
function kube::build::clean_output() {
# Clean out the output directory if it exists.
if kube::build::has_docker ; then
if kube::build::build_image_built ; then
kube::log::status "Cleaning out _output/dockerized/bin/ via docker build image"
kube::build::run_build_command bash -c "rm -rf '${REMOTE_OUTPUT_BINPATH}'/*"
else
kube::log::error "Build image not built. Cannot clean via docker build image."
fi
kube::log::status "Removing data container ${KUBE_BUILD_DATA_CONTAINER_NAME}"
"${DOCKER[@]}" rm -v "${KUBE_BUILD_DATA_CONTAINER_NAME}" >/dev/null 2>&1 || true
fi
kube::log::status "Removing _output directory"
rm -rf "${LOCAL_OUTPUT_ROOT}"
}
# Make sure the _output directory is created and mountable by docker
function kube::build::prepare_output() {
# See auto-creation of host mounts: https://github.com/docker/docker/pull/21666
# if selinux is enabled, docker run -v /foo:/foo:Z will not autocreate the host dir
mkdir -p "${LOCAL_OUTPUT_SUBPATH}"
mkdir -p "${LOCAL_OUTPUT_BINPATH}"
# On RHEL/Fedora SELinux is enabled by default and currently breaks docker
# volume mounts. We can work around this by explicitly adding a security
# context to the _output directory.
# Details: http://www.projectatomic.io/blog/2015/06/using-volumes-with-docker-can-cause-problems-with-selinux/
if which selinuxenabled &>/dev/null && \
selinuxenabled && \
which chcon >/dev/null ; then
if [[ ! $(ls -Zd "${LOCAL_OUTPUT_ROOT}") =~ svirt_sandbox_file_t ]] ; then
kube::log::status "Applying SELinux policy to '_output' directory."
if ! chcon -Rt svirt_sandbox_file_t "${LOCAL_OUTPUT_ROOT}"; then
echo " ***Failed***. This may be because you have root owned files under _output."
echo " Continuing, but this build may fail later if SELinux prevents access."
fi
fi
number=${#DOCKER_MOUNT_ARGS[@]}
for (( i=0; i<number; i++ )); do
if [[ "${DOCKER_MOUNT_ARGS[i]}" =~ "${KUBE_ROOT}" ]]; then
## Ensure we don't label the argument multiple times
if [[ ! "${DOCKER_MOUNT_ARGS[i]}" == *:Z ]]; then
DOCKER_MOUNT_ARGS[i]="${DOCKER_MOUNT_ARGS[i]}:Z"
fi
fi
done
fi
}
function kube::build::has_docker() {
which docker &> /dev/null
}
@@ -353,9 +328,51 @@ function kube::build::docker_image_exists() {
exit 2
}
# We cannot just specify the IMAGE here as `docker images` doesn't behave as
# expected. See: https://github.com/docker/docker/issues/8048
"${DOCKER[@]}" images | grep -Eq "^(\S+/)?${1}\s+${2}\s+"
[[ $("${DOCKER[@]}" images -q "${1}:${2}") ]]
}
# Delete all images that match a tag prefix except for the "current" version
#
# $1: The image repo/name
# $2: The tag base. We consider any image that matches $2*
# $3: The current image not to delete if provided
function kube::build::docker_delete_old_images() {
# In Docker 1.12, we can replace this with
# docker images "$1" --format "{{.Tag}}"
for tag in $("${DOCKER[@]}" images ${1} | tail -n +2 | awk '{print $2}') ; do
if [[ "${tag}" != "${2}"* ]] ; then
V=6 kube::log::status "Keeping image ${1}:${tag}"
continue
fi
if [[ -z "${3:-}" || "${tag}" != "${3}" ]] ; then
V=2 kube::log::status "Deleting image ${1}:${tag}"
"${DOCKER[@]}" rmi "${1}:${tag}" >/dev/null
else
V=6 kube::log::status "Keeping image ${1}:${tag}"
fi
done
}
# Stop and delete all containers that match a pattern
#
# $1: The base container prefix
# $2: The current container to keep, if provided
function kube::build::docker_delete_old_containers() {
# In Docker 1.12 we can replace this line with
# docker ps -a --format="{{.Names}}"
for container in $("${DOCKER[@]}" ps -a | tail -n +2 | awk '{print $NF}') ; do
if [[ "${container}" != "${1}"* ]] ; then
V=6 kube::log::status "Keeping container ${container}"
continue
fi
if [[ -z "${2:-}" || "${container}" != "${2}" ]] ; then
V=2 kube::log::status "Deleting container ${container}"
kube::build::destroy_container "${container}"
else
V=6 kube::log::status "Keeping container ${container}"
fi
done
}
# Takes $1 and computes a short has for it. Useful for unique tag generation
@@ -420,34 +437,51 @@ function kube::release::parse_and_validate_ci_version() {
# ---------------------------------------------------------------------------
# Building
function kube::build::clean() {
if kube::build::has_docker ; then
kube::build::docker_delete_old_containers "${KUBE_BUILD_CONTAINER_NAME_BASE}"
kube::build::docker_delete_old_containers "${KUBE_RSYNC_CONTAINER_NAME_BASE}"
kube::build::docker_delete_old_containers "${KUBE_DATA_CONTAINER_NAME_BASE}"
kube::build::docker_delete_old_images "${KUBE_BUILD_IMAGE_REPO}" "${KUBE_BUILD_IMAGE_TAG_BASE}"
V=2 kube::log::status "Cleaning all untagged docker images"
"${DOCKER[@]}" rmi $("${DOCKER[@]}" images -q --filter 'dangling=true') 2> /dev/null || true
fi
kube::log::status "Removing _output directory"
rm -rf "${LOCAL_OUTPUT_ROOT}"
}
function kube::build::build_image_built() {
kube::build::docker_image_exists "${KUBE_BUILD_IMAGE_REPO}" "${KUBE_BUILD_IMAGE_TAG}"
}
# The set of source targets to include in the kube-build image
function kube::build::source_targets() {
local targets=(
$(find . -mindepth 1 -maxdepth 1 -not \( \
\( -path ./_\* -o -path ./.git\* \) -prune \
\))
)
echo "${targets[@]}"
}
# Set up the context directory for the kube-build image and build it.
function kube::build::build_image() {
kube::build::ensure_tar
if ! kube::build::build_image_built; then
mkdir -p "${LOCAL_OUTPUT_BUILD_CONTEXT}"
mkdir -p "${LOCAL_OUTPUT_BUILD_CONTEXT}"
"${TAR}" czf "${LOCAL_OUTPUT_BUILD_CONTEXT}/kube-source.tar.gz" $(kube::build::source_targets)
cp /etc/localtime "${LOCAL_OUTPUT_BUILD_CONTEXT}/"
kube::version::get_version_vars
kube::version::save_version_vars "${LOCAL_OUTPUT_BUILD_CONTEXT}/kube-version-defs"
cp build/build-image/Dockerfile "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile"
cp build/build-image/rsyncd.sh "${LOCAL_OUTPUT_BUILD_CONTEXT}/"
dd if=/dev/urandom bs=512 count=1 2>/dev/null | LC_ALL=C tr -dc 'A-Za-z0-9' | dd bs=32 count=1 2>/dev/null > "${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
chmod go= "${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
cp build/build-image/Dockerfile "${LOCAL_OUTPUT_BUILD_CONTEXT}/Dockerfile"
kube::build::update_dockerfile
kube::build::update_dockerfile
kube::build::docker_build "${KUBE_BUILD_IMAGE}" "${LOCAL_OUTPUT_BUILD_CONTEXT}" 'false'
kube::build::docker_build "${KUBE_BUILD_IMAGE}" "${LOCAL_OUTPUT_BUILD_CONTEXT}" 'false'
fi
# Clean up old versions of everything
kube::build::docker_delete_old_containers "${KUBE_BUILD_CONTAINER_NAME_BASE}" "${KUBE_BUILD_CONTAINER_NAME}"
kube::build::docker_delete_old_containers "${KUBE_RSYNC_CONTAINER_NAME_BASE}" "${KUBE_RSYNC_CONTAINER_NAME}"
kube::build::docker_delete_old_containers "${KUBE_DATA_CONTAINER_NAME_BASE}" "${KUBE_DATA_CONTAINER_NAME}"
kube::build::docker_delete_old_images "${KUBE_BUILD_IMAGE_REPO}" "${KUBE_BUILD_IMAGE_TAG_BASE}" "${KUBE_BUILD_IMAGE_TAG}"
kube::build::ensure_data_container
kube::build::sync_to_container
}
# Build a docker image from a Dockerfile.
@@ -477,70 +511,101 @@ EOF
}
}
function kube::build::clean_image() {
local -r image=$1
kube::log::status "Deleting docker image ${image}"
"${DOCKER[@]}" rmi ${image} 2> /dev/null || true
}
function kube::build::clean_images() {
kube::build::has_docker || return 0
kube::build::clean_image "${KUBE_BUILD_IMAGE}"
kube::log::status "Cleaning all other untagged docker images"
"${DOCKER[@]}" rmi $("${DOCKER[@]}" images -q --filter 'dangling=true') 2> /dev/null || true
}
function kube::build::ensure_data_container() {
# If the data container exists AND exited successfully, we can use it.
# Otherwise nuke it and start over.
local ret=0
local code=$(docker inspect \
-f '{{.State.ExitCode}}' \
"${KUBE_BUILD_DATA_CONTAINER_NAME}" 2>/dev/null || ret=$?)
"${KUBE_DATA_CONTAINER_NAME}" 2>/dev/null || ret=$?)
if [[ "${ret}" == 0 && "${code}" != 0 ]]; then
kube::build::destroy_container "${KUBE_BUILD_DATA_CONTAINER_NAME}"
kube::build::destroy_container "${KUBE_DATA_CONTAINER_NAME}"
ret=1
fi
if [[ "${ret}" != 0 ]]; then
kube::log::status "Creating data container ${KUBE_BUILD_DATA_CONTAINER_NAME}"
kube::log::status "Creating data container ${KUBE_DATA_CONTAINER_NAME}"
# We have to ensure the directory exists, or else the docker run will
# create it as root.
mkdir -p "${LOCAL_OUTPUT_GOPATH}"
# We want this to run as root to be able to chown, so non-root users can
# later use the result as a data container. This run both creates the data
# container and chowns the GOPATH.
#
# The data container creates volumes for all of the directories that store
# intermediates for the Go build. This enables incremental builds across
# Docker sessions. The *_cgo paths are re-compiled versions of the go std
# libraries for true static building.
local -ra docker_cmd=(
"${DOCKER[@]}" run
--name "${KUBE_BUILD_DATA_CONTAINER_NAME}"
--volume "${REMOTE_ROOT}" # white-out the whole output dir
--volume /usr/local/go/pkg/linux_386_cgo
--volume /usr/local/go/pkg/linux_amd64_cgo
--volume /usr/local/go/pkg/linux_arm_cgo
--volume /usr/local/go/pkg/linux_arm64_cgo
--volume /usr/local/go/pkg/linux_ppc64le_cgo
--volume /usr/local/go/pkg/darwin_amd64_cgo
--volume /usr/local/go/pkg/darwin_386_cgo
--volume /usr/local/go/pkg/windows_amd64_cgo
--volume /usr/local/go/pkg/windows_386_cgo
--name "${KUBE_DATA_CONTAINER_NAME}"
--hostname "${HOSTNAME}"
--volume "${REMOTE_OUTPUT_ROOT}" # white-out the whole output dir
--volume "${REMOTE_OUTPUT_GOPATH}" # make a non-root owned mountpoint
"${KUBE_BUILD_IMAGE}"
chown -R $(id -u).$(id -g) "${REMOTE_OUTPUT_ROOT}"
chown -R $(id -u).$(id -g)
"${REMOTE_ROOT}"
/usr/local/go/pkg/
)
"${docker_cmd[@]}"
fi
}
# Run a command in the kube-build image. This assumes that the image has
# already been built. This will sync out all output data from the build.
# already been built.
function kube::build::run_build_command() {
kube::log::status "Running build command...."
[[ $# != 0 ]] || { echo "Invalid input - please specify a command to run." >&2; return 4; }
kube::log::status "Running build command..."
kube::build::run_build_command_ex "${KUBE_BUILD_CONTAINER_NAME}" -- "$@"
}
kube::build::ensure_data_container
kube::build::prepare_output
# Run a command in the kube-build image. This assumes that the image has
# already been built.
#
# Arguments are in the form of
# <container name> <extra docker args> -- <command>
function kube::build::run_build_command_ex() {
[[ $# != 0 ]] || { echo "Invalid input - please specify a container name." >&2; return 4; }
local container_name="${1}"
shift
local -a docker_run_opts=(
"--name=${KUBE_BUILD_CONTAINER_NAME}"
"--name=${container_name}"
"--user=$(id -u):$(id -g)"
"--hostname=${HOSTNAME}"
"${DOCKER_MOUNT_ARGS[@]}"
)
local detach=false
[[ $# != 0 ]] || { echo "Invalid input - please specify docker arguments followed by --." >&2; return 4; }
# Everything before "--" is an arg to docker
until [ -z "${1-}" ] ; do
if [[ "$1" == "--" ]]; then
shift
break
fi
docker_run_opts+=("$1")
if [[ "$1" == "-d" || "$1" == "--detach" ]] ; then
detach=true
fi
shift
done
# Everything after "--" is the command to run
[[ $# != 0 ]] || { echo "Invalid input - please specify a command to run." >&2; return 4; }
local -a cmd=()
until [ -z "${1-}" ] ; do
cmd+=("$1")
shift
done
docker_run_opts+=(
--env "KUBE_FASTBUILD=${KUBE_FASTBUILD:-false}"
--env "KUBE_BUILDER_OS=${OSTYPE:-notdetected}"
@@ -553,7 +618,7 @@ function kube::build::run_build_command() {
# attach stderr/stdout but don't bother asking for a tty.
if [[ -t 0 ]]; then
docker_run_opts+=(--interactive --tty)
else
elif [[ "${detach}" == false ]]; then
docker_run_opts+=(--attach=stdout --attach=stderr)
fi
@@ -561,73 +626,154 @@ function kube::build::run_build_command() {
"${DOCKER[@]}" run "${docker_run_opts[@]}" "${KUBE_BUILD_IMAGE}")
# Clean up container from any previous run
kube::build::destroy_container "${KUBE_BUILD_CONTAINER_NAME}"
"${docker_cmd[@]}" "$@"
kube::build::destroy_container "${KUBE_BUILD_CONTAINER_NAME}"
}
# Test if the output directory is remote (and can only be accessed through
# docker) or if it is "local" and we can access the output without going through
# docker.
function kube::build::is_output_remote() {
rm -f "${LOCAL_OUTPUT_SUBPATH}/test_for_remote"
kube::build::run_build_command touch "${REMOTE_OUTPUT_BINPATH}/test_for_remote"
[[ ! -e "${LOCAL_OUTPUT_BINPATH}/test_for_remote" ]]
}
# If the Docker server is remote, copy the results back out.
function kube::build::copy_output() {
if kube::build::is_output_remote; then
# At time of this code, docker cp does not work when copying from a volume.
# As a workaround, the binaries are first copied to a local filesystem,
# /tmp, then docker cp'd to the local binaries output directory.
# The fix for the volume bug has been accepted and once it's widely
# deployed the code below should be simplified to a simple docker cp
# Bug: https://github.com/docker/docker/pull/8509
local -a docker_run_opts=(
"--name=${KUBE_BUILD_CONTAINER_NAME}"
"--user=$(id -u):$(id -g)"
"${DOCKER_MOUNT_ARGS[@]}"
-d
)
local -ra docker_cmd=(
"${DOCKER[@]}" run "${docker_run_opts[@]}" "${KUBE_BUILD_IMAGE}"
)
kube::log::status "Syncing back _output/dockerized/bin directory from remote Docker"
rm -rf "${LOCAL_OUTPUT_BINPATH}"
mkdir -p "${LOCAL_OUTPUT_BINPATH}"
rm -f "${THIS_PLATFORM_BIN}"
ln -s "${LOCAL_OUTPUT_BINPATH}" "${THIS_PLATFORM_BIN}"
kube::build::destroy_container "${KUBE_BUILD_CONTAINER_NAME}"
"${docker_cmd[@]}" bash -c "cp -r ${REMOTE_OUTPUT_BINPATH} /tmp/bin;touch /tmp/finished;rm /tmp/bin/test_for_remote;/bin/sleep 600" > /dev/null 2>&1
# Wait until binaries have finished coppying
count=0
while true;do
if "${DOCKER[@]}" cp "${KUBE_BUILD_CONTAINER_NAME}:/tmp/finished" "${LOCAL_OUTPUT_BINPATH}" > /dev/null 2>&1;then
"${DOCKER[@]}" cp "${KUBE_BUILD_CONTAINER_NAME}:/tmp/bin" "${LOCAL_OUTPUT_SUBPATH}"
break;
fi
let count=count+1
if [[ $count -eq 60 ]]; then
# break after 5m
kube::log::error "Timed out waiting for binaries..."
break
fi
sleep 5
done
"${DOCKER[@]}" rm -f -v "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true
else
kube::log::status "Output directory is local. No need to copy results out."
kube::build::destroy_container "${container_name}"
"${docker_cmd[@]}" "${cmd[@]}"
if [[ "${detach}" == false ]]; then
kube::build::destroy_container "${container_name}"
fi
}
function kube::build::probe_address {
# Apple has an ancient version of netcat with custom timeout flags. This is
# the best way I (jbeda) could find to test for that.
local netcat
if nc 2>&1 | grep -e 'apple' >/dev/null ; then
netcat="nc -G 1"
else
netcat="nc -w 1"
fi
# Wait unil rsync is up and running.
if ! which nc >/dev/null ; then
V=6 kube::log::info "netcat not installed, waiting for 1s"
sleep 1
return 0
fi
local tries=10
while (( ${tries} > 0 )) ; do
if ${netcat} -z "$1" "$2" 2> /dev/null ; then
return 0
fi
tries=$(( ${tries} - 1))
sleep 0.1
done
return 1
}
# Start up the rsync container in the backgound. This should be explicitly
# stoped with kube::build::stop_rsyncd_container.
#
# This will set the global var KUBE_RSYNC_ADDR to the effective port that the
# rsync daemon can be reached out.
function kube::build::start_rsyncd_container() {
kube::build::stop_rsyncd_container
V=6 kube::log::status "Starting rsyncd container"
kube::build::run_build_command_ex \
"${KUBE_RSYNC_CONTAINER_NAME}" -p 127.0.0.1:${KUBE_RSYNC_PORT}:${KUBE_CONTAINER_RSYNC_PORT} -d \
-- /rsyncd.sh >/dev/null
local mapped_port
if ! mapped_port=$("${DOCKER[@]}" port "${KUBE_RSYNC_CONTAINER_NAME}" ${KUBE_CONTAINER_RSYNC_PORT} 2> /dev/null | cut -d: -f 2) ; then
kube:log:error "Could not get effective rsync port"
return 1
fi
local container_ip
container_ip=$("${DOCKER[@]}" inspect --format '{{ .NetworkSettings.IPAddress }}' "${KUBE_RSYNC_CONTAINER_NAME}")
# Sometimes we can reach rsync through localhost and a NAT'd port. Other
# times (when we are running in another docker container on the Jenkins
# machines) we have to talk directly to the container IP. There is no one
# strategy that works in all cases so we test to figure out which situation we
# are in.
if kube::build::probe_address 127.0.0.1 ${mapped_port}; then
KUBE_RSYNC_ADDR="127.0.0.1:${mapped_port}"
sleep 0.5
return 0
elif kube::build::probe_address "${container_ip}" ${KUBE_CONTAINER_RSYNC_PORT}; then
KUBE_RSYNC_ADDR="${container_ip}:${KUBE_CONTAINER_RSYNC_PORT}"
sleep 0.5
return 0
fi
kube::log::error "Could not connect to rsync container. See build/README.md for setting up remote Docker engine."
return 1
}
function kube::build::stop_rsyncd_container() {
V=6 kube::log::status "Stopping any currently running rsyncd container"
unset KUBE_RSYNC_ADDR
kube::build::destroy_container "${KUBE_RSYNC_CONTAINER_NAME}"
}
# This will launch rsyncd in a container and then sync the source tree to the
# container over the local network.
function kube::build::sync_to_container() {
kube::log::status "Syncing sources to container"
kube::build::start_rsyncd_container
local rsync_extra=""
if (( ${KUBE_VERBOSE} >= 6 )); then
rsync_extra="-iv"
fi
# rsync filters are a bit confusing. Here we are syncing everything except
# output only directories and things that are not necessary like the git
# directory. The '- /' filter prevents rsync from trying to set the
# uid/gid/perms on the root of the sync tree.
V=6 kube::log::status "Running rsync"
rsync ${rsync_extra} \
--archive \
--prune-empty-dirs \
--password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" \
--filter='- /.git/' \
--filter='- /.make/' \
--filter='- /_tmp/' \
--filter='- /_output/' \
--filter='- /' \
"${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/"
kube::build::stop_rsyncd_container
}
# Copy all build results back out.
function kube::build::copy_output() {
kube::log::status "Syncing out of container"
kube::build::start_rsyncd_container
local rsync_extra=""
if (( ${KUBE_VERBOSE} >= 6 )); then
rsync_extra="-iv"
fi
# The filter syntax for rsync is a little obscure. It filters on files and
# directories. If you don't go in to a directory you won't find any files
# there. Rules are evaluated in order. The last two rules are a little
# magic. '+ */' says to go in to every directory and '- /**' says to ignore
# any file or directory that isn't already specifically allowed.
#
# We are looking to copy out all of the built binaries along with various
# generated files.
V=6 kube::log::status "Running rsync"
rsync ${rsync_extra} \
--archive \
--prune-empty-dirs \
--password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" \
--filter='- /vendor/' \
--filter='- /_temp/' \
--filter='+ /_output/dockerized/bin/**' \
--filter='+ zz_generated.*' \
--filter='+ */' \
--filter='- /**' \
"rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}"
kube::build::stop_rsyncd_container
}
# ---------------------------------------------------------------------------
# Build final release artifacts
function kube::release::clean_cruft() {