From 04813f0dcdbbc4fb104f5dcf2b0d386687fe73f3 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 2 Apr 2015 13:46:39 -0700 Subject: [PATCH 1/7] Revert "Revert "Parallelize architectures in both the building and packaging phases of `make release`"" This reverts commit 9f60dde3200a10d84cd5638f3ce81a596023519d. --- build/common.sh | 39 ++++++++++--------- hack/lib/golang.sh | 94 ++++++++++++++++++++++++++-------------------- 2 files changed, 76 insertions(+), 57 deletions(-) diff --git a/build/common.sh b/build/common.sh index a85570be748..b0806e21e4d 100644 --- a/build/common.sh +++ b/build/common.sh @@ -518,30 +518,35 @@ function kube::release::package_client_tarballs() { # Find all of the built client binaries local platform platforms platforms=($(cd "${LOCAL_OUTPUT_BINPATH}" ; echo */*)) - for platform in "${platforms[@]}" ; do + for platform in "${platforms[@]}"; do local platform_tag=${platform/\//-} # Replace a "/" for a "-" - kube::log::status "Building tarball: client $platform_tag" + kube::log::status "Starting tarball: client $platform_tag" - local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes" - rm -rf "${release_stage}" - mkdir -p "${release_stage}/client/bin" + ( + local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes" + rm -rf "${release_stage}" + mkdir -p "${release_stage}/client/bin" - local client_bins=("${KUBE_CLIENT_BINARIES[@]}") - if [[ "${platform%/*}" == "windows" ]]; then - client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}") - fi + local client_bins=("${KUBE_CLIENT_BINARIES[@]}") + if [[ "${platform%/*}" == "windows" ]]; then + client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}") + fi - # This fancy expression will expand to prepend a path - # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the - # KUBE_CLIENT_BINARIES array. - cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ - "${release_stage}/client/bin/" + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # KUBE_CLIENT_BINARIES array. + cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + "${release_stage}/client/bin/" - kube::release::clean_cruft + kube::release::clean_cruft - local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz" - kube::release::create_tarball "${package_name}" "${release_stage}/.." + local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz" + kube::release::create_tarball "${package_name}" "${release_stage}/.." + ) & done + + kube::log::status "Waiting on tarballs" + wait || { kube::log::error "client tarball creation failed"; exit 1; } } # Package up all of the server binaries diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index fc21bfd5636..111a6a0a595 100644 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -313,62 +313,76 @@ kube::golang::build_binaries() { local binaries binaries=($(kube::golang::binaries_from_targets "${targets[@]}")) + kube::log::status "Building go targets for ${platforms[@]} in parallel (output will appear in a burst when complete):" "${targets[@]}" local platform - for platform in "${platforms[@]}"; do - kube::golang::set_platform_envs "${platform}" - kube::log::status "Building go targets for ${platform}:" "${targets[@]}" - - local -a statics=() - local -a nonstatics=() - for binary in "${binaries[@]}"; do - if kube::golang::is_statically_linked_library "${binary}"; then - kube::golang::exit_if_stdlib_not_installed; - statics+=($binary) - else - nonstatics+=($binary) - fi - done - - if [[ -n ${use_go_build:-} ]]; then - # Try and replicate the native binary placement of go install without - # calling go install. This means we have to iterate each binary. - local output_path="${KUBE_GOPATH}/bin" - if [[ $platform != $host_platform ]]; then - output_path="${output_path}/${platform//\//_}" - fi + for platform in "${platforms[@]}"; do ( + kube::golang::set_platform_envs "${platform}" + kube::log::status "${platform}: Parallel go build started" + local -a statics=() + local -a nonstatics=() for binary in "${binaries[@]}"; do - local bin=$(basename "${binary}") - if [[ ${GOOS} == "windows" ]]; then - bin="${bin}.exe" - fi - if kube::golang::is_statically_linked_library "${binary}"; then kube::golang::exit_if_stdlib_not_installed; - CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \ - "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${binary}" + statics+=($binary) else - go build -o "${output_path}/${bin}" \ - "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${binary}" + nonstatics+=($binary) fi done - else + + if [[ -n ${use_go_build:-} ]]; then + # Try and replicate the native binary placement of go install without + # calling go install. This means we have to iterate each binary. + local output_path="${KUBE_GOPATH}/bin" + if [[ $platform != $host_platform ]]; then + output_path="${output_path}/${platform//\//_}" + fi + + for binary in "${binaries[@]}"; do + local bin=$(basename "${binary}") + if [[ ${GOOS} == "windows" ]]; then + bin="${bin}.exe" + fi + + if kube::golang::is_statically_linked_library "${binary}"; then + kube::golang::exit_if_stdlib_not_installed; + CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \ + "${goflags[@]:+${goflags[@]}}" \ + -ldflags "${version_ldflags}" \ + "${binary}" + else + go build -o "${output_path}/${bin}" \ + "${goflags[@]:+${goflags[@]}}" \ + -ldflags "${version_ldflags}" \ + "${binary}" + fi + done + else # Use go install. if [[ "${#nonstatics[@]}" != 0 ]]; then go install "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${nonstatics[@]:+${nonstatics[@]}}" + -ldflags "${version_ldflags}" \ + "${nonstatics[@]:+${nonstatics[@]}}" fi if [[ "${#statics[@]}" != 0 ]]; then CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \ -ldflags "${version_ldflags}" \ - "${statics[@]:+${statics[@]}}" + "${statics[@]:+${statics[@]}}" fi - fi + fi + kube::log::status "${platform}: Parallel go build finished" + ) &> "/tmp//${platform//\//_}.build" & done + + local fails=0 + for job in $(jobs -p); do + wait ${job} || let "fails+=1" + done + + for platform in "${platforms[@]}"; do + cat "/tmp//${platform//\//_}.build" + done + + exit ${fails} ) } From e159c02a2a832ef49e0377b318b3b9f597c97458 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 2 Apr 2015 06:51:20 -0700 Subject: [PATCH 2/7] Completely neutral commit that just factors each platform build out to a function (to make it kinder on reviewers for the next piece). --- hack/lib/golang.sh | 111 ++++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index 111a6a0a595..a5a71d49216 100644 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -263,6 +263,64 @@ kube::golang::exit_if_stdlib_not_installed() { exit 0; } +kube::golang::build_binaries_for_platform() { + local platform=$1 + local use_go_build=${2-} + + local -a statics=() + local -a nonstatics=() + for binary in "${binaries[@]}"; do + if kube::golang::is_statically_linked_library "${binary}"; then + kube::golang::exit_if_stdlib_not_installed; + statics+=($binary) + else + nonstatics+=($binary) + fi + done + + if [[ -n ${use_go_build:-} ]]; then + # Try and replicate the native binary placement of go install without + # calling go install. This means we have to iterate each binary. + local output_path="${KUBE_GOPATH}/bin" + if [[ $platform != $host_platform ]]; then + output_path="${output_path}/${platform//\//_}" + fi + + for binary in "${binaries[@]}"; do + local bin=$(basename "${binary}") + if [[ ${GOOS} == "windows" ]]; then + bin="${bin}.exe" + fi + + if kube::golang::is_statically_linked_library "${binary}"; then + kube::golang::exit_if_stdlib_not_installed; + CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \ + "${goflags[@]:+${goflags[@]}}" \ + -ldflags "${version_ldflags}" \ + "${binary}" + else + go build -o "${output_path}/${bin}" \ + "${goflags[@]:+${goflags[@]}}" \ + -ldflags "${version_ldflags}" \ + "${binary}" + fi + done + else + # Use go install. + if [[ "${#nonstatics[@]}" != 0 ]]; then + go install "${goflags[@]:+${goflags[@]}}" \ + -ldflags "${version_ldflags}" \ + "${nonstatics[@]:+${nonstatics[@]}}" + fi + if [[ "${#statics[@]}" != 0 ]]; then + CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \ + -ldflags "${version_ldflags}" \ + "${statics[@]:+${statics[@]}}" + fi + fi + +} + # Build binaries targets specified # # Input: @@ -318,58 +376,7 @@ kube::golang::build_binaries() { for platform in "${platforms[@]}"; do ( kube::golang::set_platform_envs "${platform}" kube::log::status "${platform}: Parallel go build started" - - local -a statics=() - local -a nonstatics=() - for binary in "${binaries[@]}"; do - if kube::golang::is_statically_linked_library "${binary}"; then - kube::golang::exit_if_stdlib_not_installed; - statics+=($binary) - else - nonstatics+=($binary) - fi - done - - if [[ -n ${use_go_build:-} ]]; then - # Try and replicate the native binary placement of go install without - # calling go install. This means we have to iterate each binary. - local output_path="${KUBE_GOPATH}/bin" - if [[ $platform != $host_platform ]]; then - output_path="${output_path}/${platform//\//_}" - fi - - for binary in "${binaries[@]}"; do - local bin=$(basename "${binary}") - if [[ ${GOOS} == "windows" ]]; then - bin="${bin}.exe" - fi - - if kube::golang::is_statically_linked_library "${binary}"; then - kube::golang::exit_if_stdlib_not_installed; - CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \ - "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${binary}" - else - go build -o "${output_path}/${bin}" \ - "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${binary}" - fi - done - else - # Use go install. - if [[ "${#nonstatics[@]}" != 0 ]]; then - go install "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${nonstatics[@]:+${nonstatics[@]}}" - fi - if [[ "${#statics[@]}" != 0 ]]; then - CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${statics[@]:+${statics[@]}}" - fi - fi + kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-} kube::log::status "${platform}: Parallel go build finished" ) &> "/tmp//${platform//\//_}.build" & done From c0ed73a10d007c1088414719c72369753f571f92 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 2 Apr 2015 08:46:28 -0700 Subject: [PATCH 3/7] Only build in parallel if the build machine is >8G --- hack/lib/golang.sh | 86 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index a5a71d49216..5dc923f80e2 100644 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -72,6 +72,11 @@ readonly KUBE_CLIENT_PLATFORMS=( windows/amd64 ) +# Gigabytes desired for parallel platform builds. 8 is fairly +# arbitrary, but is a reasonable splitting point for 2015 +# laptops-versus-not. +readonly KUBE_PARALLEL_BUILD_MEMORY=8 + readonly KUBE_ALL_TARGETS=( "${KUBE_SERVER_TARGETS[@]}" "${KUBE_CLIENT_TARGETS[@]}" @@ -321,6 +326,29 @@ kube::golang::build_binaries_for_platform() { } +# Return approximate physical memory in gigabytes. +kube::golang::get_physmem() { + local mem + + # Linux, in kb + if mem=$(grep MemTotal /proc/meminfo | awk '{ print $2 }'); then + echo $(( ${mem} / 1048576 )) + return + fi + + # OS X, in bytes. Note that get_physmem, as used, should only ever + # run in a Linux container (because it's only used in the multiple + # platform case, which is a Dockerized build), but this is provided + # for completeness. + if mem=$(sysctl -n hw.memsize 2>/dev/null); then + echo $(( ${mem} / 1073741824 )) + return + fi + + # If we can't infer it, just give up and assume a low memory system + echo 1 +} + # Build binaries targets specified # # Input: @@ -371,25 +399,47 @@ kube::golang::build_binaries() { local binaries binaries=($(kube::golang::binaries_from_targets "${targets[@]}")) - kube::log::status "Building go targets for ${platforms[@]} in parallel (output will appear in a burst when complete):" "${targets[@]}" - local platform - for platform in "${platforms[@]}"; do ( + local parallel=false + if [[ ${#platforms[@]} -gt 1 ]]; then + local gigs + gigs=$(kube::golang::get_physmem) + + if [[ ${gigs} -gt ${KUBE_PARALLEL_BUILD_MEMORY} ]]; then + kube::log::status "Multiple platforms requested and available ${gigs}G > threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in parallel" + parallel=true + else + kube::log::status "Multiple platforms requested, but available ${gigs}G < threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in serial" + parallel=false + fi + fi + + if [[ "${parallel}" == "true" ]]; then + kube::log::status "Building go targets for ${platforms[@]} in parallel (output will appear in a burst when complete):" "${targets[@]}" + local platform + for platform in "${platforms[@]}"; do ( + kube::golang::set_platform_envs "${platform}" + kube::log::status "${platform}: go build started" + kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-} + kube::log::status "${platform}: go build finished" + ) &> "/tmp//${platform//\//_}.build" & + done + + local fails=0 + for job in $(jobs -p); do + wait ${job} || let "fails+=1" + done + + for platform in "${platforms[@]}"; do + cat "/tmp//${platform//\//_}.build" + done + + exit ${fails} + else + for platform in "${platforms[@]}"; do + kube::log::status "Building go targets for ${platform}:" "${targets[@]}" kube::golang::set_platform_envs "${platform}" - kube::log::status "${platform}: Parallel go build started" kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-} - kube::log::status "${platform}: Parallel go build finished" - ) &> "/tmp//${platform//\//_}.build" & - done - - local fails=0 - for job in $(jobs -p); do - wait ${job} || let "fails+=1" - done - - for platform in "${platforms[@]}"; do - cat "/tmp//${platform//\//_}.build" - done - - exit ${fails} + done + fi ) } From 03c7182cac9eda4afdea71c89901b03de44796e4 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 2 Apr 2015 10:52:53 -0700 Subject: [PATCH 4/7] Also build entire tarball phases in parallel --- build/common.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/build/common.sh b/build/common.sh index b0806e21e4d..5f52e6381de 100644 --- a/build/common.sh +++ b/build/common.sh @@ -505,11 +505,14 @@ function kube::release::package_tarballs() { # Clean out any old releases rm -rf "${RELEASE_DIR}" mkdir -p "${RELEASE_DIR}" - kube::release::package_client_tarballs - kube::release::package_server_tarballs - kube::release::package_salt_tarball - kube::release::package_test_tarball - kube::release::package_full_tarball + kube::release::package_client_tarballs & + kube::release::package_server_tarballs & + kube::release::package_salt_tarball & + wait || { kube::log::error "previous tarball phase failed"; return 1; } + + kube::release::package_full_tarball & # _full depends on all the previous phases + kube::release::package_test_tarball & # _test doesn't depend on anything + wait || { kube::log::error "previous tarball phase failed"; return 1; } } # Package up all of the cross compiled clients. Over time this should grow into From 9a6e04dc1f896b27ff4193c45eeb3a3107338cfa Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 2 Apr 2015 18:18:24 -0700 Subject: [PATCH 5/7] Cleanup output of Docker builds, fix #6389 --- build/common.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/build/common.sh b/build/common.sh index 5f52e6381de..7548acfcc55 100644 --- a/build/common.sh +++ b/build/common.sh @@ -586,26 +586,34 @@ function kube::release::package_server_tarballs() { done } +function kube::release::md5() { + if which md5 >/dev/null 2>&1; then + md5 -q "$1" + else + md5sum "$1" | awk '{ print $1 }' + fi +} + # This will take binaries that run on master and creates Docker images # that wrap the binary in them. (One docker image per binary) function kube::release::create_docker_images_for_server() { # Create a sub-shell so that we don't pollute the outer environment ( - local binary_name; + local binary_name for binary_name in "${KUBE_DOCKER_WRAPPED_BINARIES[@]}"; do - echo "+++ Building docker image: ${binary_name}"; - local docker_file_path="$1/Dockerfile"; - local binary_file_path="$1/${binary_name}"; + kube::log::status "Building docker image: ${binary_name}" + local docker_file_path="$1/Dockerfile" + local binary_file_path="$1/${binary_name}" if [ -f ${docker_file_path} ]; then - rm ${docker_file_path}; - fi; - printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path}; - local md5_sum=$(md5sum ${binary_file_path} | awk '{print $1}') + rm ${docker_file_path} + fi + printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path} + local md5_sum=$(kube::release::md5 ${binary_file_path}) local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum - docker build -t "${docker_image_tag}" ${1}; - docker save ${docker_image_tag} > ${1}/${binary_name}.tar; - echo $md5_sum > ${1}/${binary_name}.docker_tag; - rm ${docker_file_path}; + docker build -q -t "${docker_image_tag}" ${1} >/dev/null + docker save ${docker_image_tag} > ${1}/${binary_name}.tar + echo $md5_sum > ${1}/${binary_name}.docker_tag + rm ${docker_file_path} done ) } From e57171d2ccf93e27f2d20b4bf64ce6b8d22b7180 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 2 Apr 2015 18:39:40 -0700 Subject: [PATCH 6/7] Parallelize Docker build as well --- build/common.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/build/common.sh b/build/common.sh index 7548acfcc55..238a40ed451 100644 --- a/build/common.sh +++ b/build/common.sh @@ -601,8 +601,10 @@ function kube::release::create_docker_images_for_server() { ( local binary_name for binary_name in "${KUBE_DOCKER_WRAPPED_BINARIES[@]}"; do - kube::log::status "Building docker image: ${binary_name}" - local docker_file_path="$1/Dockerfile" + kube::log::status "Starting Docker build for image: ${binary_name}" + + ( + local docker_file_path="$1/${binary_name}.Dockerfile" local binary_file_path="$1/${binary_name}" if [ -f ${docker_file_path} ]; then rm ${docker_file_path} @@ -610,11 +612,15 @@ function kube::release::create_docker_images_for_server() { printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path} local md5_sum=$(kube::release::md5 ${binary_file_path}) local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum - docker build -q -t "${docker_image_tag}" ${1} >/dev/null + docker build -q -f "${docker_file_path}" -t "${docker_image_tag}" ${1} >/dev/null docker save ${docker_image_tag} > ${1}/${binary_name}.tar echo $md5_sum > ${1}/${binary_name}.docker_tag rm ${docker_file_path} + ) & done + + wait || { kube::log::error "previous Docker build failed"; return 1; } + kube::log::status "Docker builds done" ) } From 5b6c75f98672d4a19ba439d6a0cf006e5e288c97 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 2 Apr 2015 18:40:27 -0700 Subject: [PATCH 7/7] And re-tabify that (to make previous easier to review) --- build/common.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/build/common.sh b/build/common.sh index 238a40ed451..915fe47e203 100644 --- a/build/common.sh +++ b/build/common.sh @@ -604,18 +604,18 @@ function kube::release::create_docker_images_for_server() { kube::log::status "Starting Docker build for image: ${binary_name}" ( - local docker_file_path="$1/${binary_name}.Dockerfile" - local binary_file_path="$1/${binary_name}" - if [ -f ${docker_file_path} ]; then + local docker_file_path="$1/${binary_name}.Dockerfile" + local binary_file_path="$1/${binary_name}" + if [ -f ${docker_file_path} ]; then + rm ${docker_file_path} + fi + printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path} + local md5_sum=$(kube::release::md5 ${binary_file_path}) + local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum + docker build -q -f "${docker_file_path}" -t "${docker_image_tag}" ${1} >/dev/null + docker save ${docker_image_tag} > ${1}/${binary_name}.tar + echo $md5_sum > ${1}/${binary_name}.docker_tag rm ${docker_file_path} - fi - printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path} - local md5_sum=$(kube::release::md5 ${binary_file_path}) - local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum - docker build -q -f "${docker_file_path}" -t "${docker_image_tag}" ${1} >/dev/null - docker save ${docker_image_tag} > ${1}/${binary_name}.tar - echo $md5_sum > ${1}/${binary_name}.docker_tag - rm ${docker_file_path} ) & done