From 570ebf54a94311e18c940b4ebe126c50538b7580 Mon Sep 17 00:00:00 2001 From: Joe Beda Date: Mon, 16 Jun 2014 23:16:40 -0700 Subject: [PATCH] Build Kubernetes in Docker. Scripts and Dockerfile to build a container image, build binaries, run tests, etc. Also copy output back out to the host machine. --- build/README.md | 16 +++ build/build-image/Dockerfile | 65 ++++++++++++ build/build-image/common.sh | 27 +++++ build/build-image/make-binaries.sh | 45 ++++++++ build/build-image/run-integration.sh | 31 ++++++ build/build-image/run-tests.sh | 43 ++++++++ build/common.sh | 148 +++++++++++++++++++++++++++ build/make-binaries.sh | 28 +++++ build/make-build-image.sh | 28 +++++ build/run-integration.sh | 28 +++++ build/run-tests.sh | 27 +++++ build/shell.sh | 30 ++++++ 12 files changed, 516 insertions(+) create mode 100644 build/README.md create mode 100644 build/build-image/Dockerfile create mode 100644 build/build-image/common.sh create mode 100755 build/build-image/make-binaries.sh create mode 100755 build/build-image/run-integration.sh create mode 100755 build/build-image/run-tests.sh create mode 100644 build/common.sh create mode 100755 build/make-binaries.sh create mode 100755 build/make-build-image.sh create mode 100755 build/run-integration.sh create mode 100755 build/run-tests.sh create mode 100755 build/shell.sh diff --git a/build/README.md b/build/README.md new file mode 100644 index 00000000000..0d10510d1d1 --- /dev/null +++ b/build/README.md @@ -0,0 +1,16 @@ +# Building Kubernetes + +To build Kubernetes you need to have access to a Docker installation through either of the following methods: + +## Requirements + +1. Run on Mac OS X. The best way to go is to use `boot2docker`. See instructions [here](https://docs.docker.com/installation/mac/). +2. Run on Linux against a local Docker. Install Docker according to the [instructions](https://docs.docker.com/installation/#installation) for your OS. The scripts here assume that they are using a local Docker server and that they can "reach around" docker and grab results directly from the file system. + +## Basic Flow + +The scripts directly under `build/` are used to build and test. They will ensure that the `kube-build` Docker image is built (based on `build/build-image/Dockerfile`) and then execute the appropriate command in that container. If necessary (for Mac OS X), the scripts will also copy results out. + +The `kube-build` container image is built by first creating a "context" directory in `output/build-image`. It is done there instead of at the root of the Kubernetes repo to minimize the amount of data we need to package up when building the image. + +Everything in `build/build-image/` is meant to be run inside of the container. If it doesn't think it is running in the container it'll throw a warning. While you can run some of that stuff outside of the container, it wasn't built to do so. diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile new file mode 100644 index 00000000000..dcfb9df338a --- /dev/null +++ b/build/build-image/Dockerfile @@ -0,0 +1,65 @@ +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file creates a standard build environment for building Kubernetes +# +# Usage: +# +# # Assemble the full dev environment. This is slow the first time. +# docker build -t kube-build . +# +# # Mount your source in an interactive container for quick testing: +# docker run -v `pwd`:/go/src/github.com/GoogleCloudPlatform/kubernetes -i -t docker bash +# + +FROM google/debian:wheezy +MAINTAINER Joe Beda + +RUN apt-get update -y && apt-get install --no-install-recommends -y -q \ + curl \ + build-essential \ + ca-certificates \ + git \ + mercurial \ + rsync + +# Install Go +# TODO(jbeda) -- we need to verify this against the hash +RUN curl -s https://storage.googleapis.com/golang/go1.2.2.src.tar.gz | tar -C /usr/local -xz +ENV PATH /usr/local/go/bin:$PATH +RUN cd /usr/local/go/src && ./make.bash --no-clean 2>&1 + +# Compile Go for cross compilation +ENV KUBE_CROSSPLATFORMS \ + linux/386 linux/arm \ + darwin/amd64 darwin/386 +# (set an explicit GOARM of 5 for maximum compatibility) +ENV GOARM 5 +RUN cd /usr/local/go/src && \ + bash -xc 'for platform in $KUBE_CROSSPLATFORMS; do GOOS=${platform%/*} GOARCH=${platform##*/} ./make.bash --no-clean 2>&1; done' + +# Set up Go Environment +ENV PATH /usr/local/go/bin:/go/bin:$PATH +ENV GOPATH /go:/go/src/github.com/GoogleCloudPlatform/kubernetes/third_party + +# Mark this as a kube-build container +RUN touch /kube-build-image + +# Get the code coverage tool and etcd for integration tests +RUN go get code.google.com/p/go.tools/cmd/cover github.com/coreos/etcd + +WORKDIR /go/src/github.com/GoogleCloudPlatform/kubernetes + +# Upload Kubernetes +ADD kube-source.tar.gz /go/src/github.com/GoogleCloudPlatform/kubernetes diff --git a/build/build-image/common.sh b/build/build-image/common.sh new file mode 100644 index 00000000000..b3220460fb6 --- /dev/null +++ b/build/build-image/common.sh @@ -0,0 +1,27 @@ +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script sets up a go workspace locally and builds all go components. +# You can 'source' this file if you want to set up GOPATH in your local shell. + +cd $(dirname "${BASH_SOURCE}")/../.. >/dev/null +readonly KUBE_REPO_ROOT="${PWD}" +readonly KUBE_TARGET="${KUBE_REPO_ROOT}/output/build" +readonly KUBE_GO_PACKAGE=github.com/GoogleCloudPlatform/kubernetes + +mkdir -p "${KUBE_TARGET}" + +if [[ ! -f "/kube-build-image" ]]; then + echo "WARNING: This script should be run in the kube-build conrtainer image!" >&2 +fi diff --git a/build/build-image/make-binaries.sh b/build/build-image/make-binaries.sh new file mode 100755 index 00000000000..8781d305750 --- /dev/null +++ b/build/build-image/make-binaries.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This and builds all go components. + +set -e + +source $(dirname $0)/common.sh + +readonly BINARIES=" + proxy + integration + apiserver + controller-manager + kubelet + cloudcfg + localkube" + +if [[ -n $1 ]]; then + echo "+++ Building $1" + go build \ + -o "${KUBE_TARGET}/$1" \ + github.com/GoogleCloudPlatform/kubernetes/cmd/$1 + exit 0 +fi + +for b in ${BINARIES}; do + echo "+++ Building $b" + go build \ + -o "${KUBE_TARGET}/$b" \ + github.com/GoogleCloudPlatform/kubernetes/cmd/$b +done diff --git a/build/build-image/run-integration.sh b/build/build-image/run-integration.sh new file mode 100755 index 00000000000..d007ea1ffb9 --- /dev/null +++ b/build/build-image/run-integration.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +source $(dirname $0)/common.sh + +ETCD_DIR="${KUBE_REPO_ROOT}/output/etcd" +mkdir -p "${ETCD_DIR}" + +etcd -name test -data-dir ${ETCD_DIR} > "${KUBE_REPO_ROOT}/output/etcd.log" & +ETCD_PID=$! + +sleep 5 + +${KUBE_TARGET}/integration + +kill $ETCD_PID diff --git a/build/build-image/run-tests.sh b/build/build-image/run-tests.sh new file mode 100755 index 00000000000..4c9e4fd4660 --- /dev/null +++ b/build/build-image/run-tests.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +source $(dirname $0)/common.sh + +find_test_dirs() { + ( + cd ${KUBE_REPO_ROOT} + find . -not \( \ + \( \ + -wholename './third_party' \ + -o -wholename './output' \ + \) -prune \ + \) -name '*_test.go' -print0 | xargs -0n1 dirname | sort -u + ) +} + +cd "${KUBE_TARGET}" + +if [[ -n "$1" ]]; then + go test -cover -coverprofile="tmp.out" "$KUBE_GO_PACKAGE/$1" + exit 0 +fi + +for package in $(find_test_dirs); do + go test -cover -coverprofile="tmp.out" "${KUBE_GO_PACKAGE}/${package}" +done diff --git a/build/common.sh b/build/common.sh new file mode 100644 index 00000000000..2a83e3923c8 --- /dev/null +++ b/build/common.sh @@ -0,0 +1,148 @@ +#! /bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Common utilties, variables and checks for all build scripts. + +cd $(dirname "${BASH_SOURCE}")/.. +readonly KUBE_REPO_ROOT="${PWD}" + +readonly KUBE_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null) +KUBE_BUILD_IMAGE=kube-build +if [ -n "${KUBE_GIT_BRANCH}" ]; then + KUBE_BUILD_IMAGE="${KUBE_BUILD_IMAGE}:${KUBE_GIT_BRANCH}" +fi +readonly KUBE_BUILD_IMAGE +readonly KUBE_GO_PACKAGE="github.com/GoogleCloudPlatform/kubernetes" + +# We set up a volume so that we have the same output directory from one run of +# the container to the next. +# +# Note that here "LOCAL" is local to the docker daemon. In the boot2docker case +# this is still inside the VM. We use the same directory in both cases though. +readonly LOCAL_OUTPUT_DIR="${KUBE_REPO_ROOT}/output/build" +readonly REMOTE_OUTPUT_DIR="/go/src/${KUBE_GO_PACKAGE}/output/build" +readonly DOCKER_CONTAINER_NAME=kube-build +readonly DOCKER_MOUNT="-v ${LOCAL_OUTPUT_DIR}:${REMOTE_OUTPUT_DIR}" + +# Verify that the right utilitites and such are installed. +if [[ -z "$(which docker)" ]]; then + echo "Can't find 'docker' in PATH, please fix and retry." >&2 + echo "See https://docs.docker.com/installation/#installation for installation instructions." >&2 + exit 1 +fi + +if [[ "$OSTYPE" == "darwin"* ]]; then + if [[ -z "$(which boot2docker)" ]]; then + echo "It looks like you are running on Mac OS X and boot2docker can't be found." >&2 + echo "See: https://docs.docker.com/installation/mac/" >&2 + exit 1 + fi + if [[ $(boot2docker status) != "running" ]]; then + echo "boot2docker VM isn't started. Please run 'boot2docker start'" >&2 + exit 1 + fi +fi + +if ! docker info > /dev/null 2>&1 ; then + echo "Can't connect to 'docker' daemon. please fix and retry." >&2 + echo >&2 + echo "Possible causes:" >&2 + echo " - On Mac OS X, boot2docker VM isn't started" >&2 + echo " - On Mac OS X, DOCKER_HOST env variable isn't set approriately" >&2 + echo " - On Linux, user isn't in 'docker' group. Add and relogin." >&2 + echo " - On Linux, Docker daemon hasn't been started or has crashed" >&2 + exit 1 +fi + +# Set up the context directory for the kube-build image and build it. +function build-image() { + local -r BUILD_CONTEXT_DIR=${KUBE_REPO_ROOT}/output/build-image + local -r SOURCE=" + api + build + cmd + hack + pkg + third_party + LICENSE + " + local -r DOCKER_BUILD_CMD="docker build -t ${KUBE_BUILD_IMAGE} ${BUILD_CONTEXT_DIR}" + + echo "+++ Building Docker image ${KUBE_BUILD_IMAGE}. First run can take minutes." + + mkdir -p ${BUILD_CONTEXT_DIR} + tar czf ${BUILD_CONTEXT_DIR}/kube-source.tar.gz ${SOURCE} + cp build/build-image/Dockerfile ${BUILD_CONTEXT_DIR}/Dockerfile + + set +e # We are handling the error here manually + local -r DOCKER_OUTPUT="$(${DOCKER_BUILD_CMD} 2>&1)" + if [ $? -ne 0 ]; then + echo "+++ Docker build command failed." >&2 + echo >&2 + echo "${DOCKER_OUTPUT}" >&2 + echo >&2 + echo "To retry manually, run:" >&2 + echo >&2 + echo " ${DOCKER_BUILD_CMD}" >&2 + echo >&2 + return 1 + fi +} + +# Run a command in the kube-build image. This assumes that the image has +# already been built. This will sync out all output data from the build. +function run-build-command() { + [[ -n "$@" ]] || { echo "Invalid input." >&2; return 4; } + + local -r DOCKER="docker run --rm --name=${DOCKER_CONTAINER_NAME} -it ${DOCKER_MOUNT} ${KUBE_BUILD_IMAGE}" + + docker rm ${DOCKER_CONTAINER_NAME} >/dev/null 2>&1 + + ${DOCKER} "$@" + +} + +# If the Docker server is remote, copy the results back out. +function copy-output() { + if [[ "$OSTYPE" == "darwin"* ]]; then + # When we are on the Mac with boot2docker Now we need to copy the results + # back out. Ideally we would leave the container around and use 'docker cp' + # to copy the results out. However, that doesn't work for mounted volumes + # currently (https://github.com/dotcloud/docker/issues/1992). And it is + # just plain broken (https://github.com/dotcloud/docker/issues/6483). + # + # The easiest thing I (jbeda) could figure out was to launch another + # container pointed at the same volume, tar the output directory and ship + # that tar over stdou. + local DOCKER="docker run -a stdout --rm --name=${DOCKER_CONTAINER_NAME} ${DOCKER_MOUNT} ${KUBE_BUILD_IMAGE}" + + # Kill any leftover container + docker rm ${DOCKER_CONTAINER_NAME} >/dev/null 2>&1 + + echo "+++ Syncing back output directory from boot2docker VM" + mkdir -p "${LOCAL_OUTPUT_DIR}" + ${DOCKER} sh -c "tar c -C ${REMOTE_OUTPUT_DIR} ." \ + | tar xv -C "${LOCAL_OUTPUT_DIR}" + + # I (jbeda) also tried getting rsync working using 'docker run' as the + # 'remote shell'. This mostly worked but there was a hang when + # closing/finishing things off. Ug. + # + # local DOCKER="docker run -i --rm --name=${DOCKER_CONTAINER_NAME} ${DOCKER_MOUNT} ${KUBE_BUILD_IMAGE}" + # DOCKER+=" bash -c 'shift ; exec \"\$@\"' --" + # rsync --blocking-io -av -e "${DOCKER}" foo:${REMOTE_OUTPUT_DIR}/ ${LOCAL_OUTPUT_DIR} + fi +} diff --git a/build/make-binaries.sh b/build/make-binaries.sh new file mode 100755 index 00000000000..fa5de2ef177 --- /dev/null +++ b/build/make-binaries.sh @@ -0,0 +1,28 @@ +#! /bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make all of the Kubernetes binaries. +# +# This makes the docker build image, builds the binaries and copies them out +# of the docker container. + +set -e + +source $(dirname $0)/common.sh + +build-image +run-build-command build/build-image/make-binaries.sh "$@" +copy-output diff --git a/build/make-build-image.sh b/build/make-build-image.sh new file mode 100755 index 00000000000..d96995dc839 --- /dev/null +++ b/build/make-build-image.sh @@ -0,0 +1,28 @@ +#! /bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the docker image necessary for building Kubernetes +# +# This script will package the parts of the repo that we need to build +# Kubernetes into a tar file and put it in the right place in the output +# directory. It will then copy over the Dockerfile and build the kube-build +# image. + +set -e + +source $(dirname $0)/common.sh + +build-image diff --git a/build/run-integration.sh b/build/run-integration.sh new file mode 100755 index 00000000000..f15f1a4650a --- /dev/null +++ b/build/run-integration.sh @@ -0,0 +1,28 @@ +#! /bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make all of the Kubernetes binaries. +# +# This makes the docker build image, builds the binaries and copies them out +# of the docker container. + +set -e + +source $(dirname $0)/common.sh + +build-image +run-build-command build/build-image/make-binaries.sh "integration" +run-build-command build/build-image/run-integration.sh diff --git a/build/run-tests.sh b/build/run-tests.sh new file mode 100755 index 00000000000..869b5454bb3 --- /dev/null +++ b/build/run-tests.sh @@ -0,0 +1,27 @@ +#! /bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make all of the Kubernetes binaries. +# +# This makes the docker build image, builds the binaries and copies them out +# of the docker container. + +set -e + +source $(dirname $0)/common.sh + +build-image +run-build-command build/build-image/run-tests.sh "$@" diff --git a/build/shell.sh b/build/shell.sh new file mode 100755 index 00000000000..7f313a7eb3d --- /dev/null +++ b/build/shell.sh @@ -0,0 +1,30 @@ +#! /bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the docker image necessary for building Kubernetes +# +# This script will package the parts of the repo that we need to build +# Kubernetes into a tar file and put it in the right place in the output +# directory. It will then copy over the Dockerfile and build the kube-build +# image. + +set -e + +source $(dirname $0)/common.sh + +build-image +run-build-command bash +copy-output