From 04813f0dcdbbc4fb104f5dcf2b0d386687fe73f3 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 2 Apr 2015 13:46:39 -0700 Subject: [PATCH] Revert "Revert "Parallelize architectures in both the building and packaging phases of `make release`"" This reverts commit 9f60dde3200a10d84cd5638f3ce81a596023519d. --- build/common.sh | 39 ++++++++++--------- hack/lib/golang.sh | 94 ++++++++++++++++++++++++++-------------------- 2 files changed, 76 insertions(+), 57 deletions(-) diff --git a/build/common.sh b/build/common.sh index a85570be748..b0806e21e4d 100644 --- a/build/common.sh +++ b/build/common.sh @@ -518,30 +518,35 @@ function kube::release::package_client_tarballs() { # Find all of the built client binaries local platform platforms platforms=($(cd "${LOCAL_OUTPUT_BINPATH}" ; echo */*)) - for platform in "${platforms[@]}" ; do + for platform in "${platforms[@]}"; do local platform_tag=${platform/\//-} # Replace a "/" for a "-" - kube::log::status "Building tarball: client $platform_tag" + kube::log::status "Starting tarball: client $platform_tag" - local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes" - rm -rf "${release_stage}" - mkdir -p "${release_stage}/client/bin" + ( + local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes" + rm -rf "${release_stage}" + mkdir -p "${release_stage}/client/bin" - local client_bins=("${KUBE_CLIENT_BINARIES[@]}") - if [[ "${platform%/*}" == "windows" ]]; then - client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}") - fi + local client_bins=("${KUBE_CLIENT_BINARIES[@]}") + if [[ "${platform%/*}" == "windows" ]]; then + client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}") + fi - # This fancy expression will expand to prepend a path - # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the - # KUBE_CLIENT_BINARIES array. - cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ - "${release_stage}/client/bin/" + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # KUBE_CLIENT_BINARIES array. + cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + "${release_stage}/client/bin/" - kube::release::clean_cruft + kube::release::clean_cruft - local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz" - kube::release::create_tarball "${package_name}" "${release_stage}/.." + local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz" + kube::release::create_tarball "${package_name}" "${release_stage}/.." + ) & done + + kube::log::status "Waiting on tarballs" + wait || { kube::log::error "client tarball creation failed"; exit 1; } } # Package up all of the server binaries diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index fc21bfd5636..111a6a0a595 100644 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -313,62 +313,76 @@ kube::golang::build_binaries() { local binaries binaries=($(kube::golang::binaries_from_targets "${targets[@]}")) + kube::log::status "Building go targets for ${platforms[@]} in parallel (output will appear in a burst when complete):" "${targets[@]}" local platform - for platform in "${platforms[@]}"; do - kube::golang::set_platform_envs "${platform}" - kube::log::status "Building go targets for ${platform}:" "${targets[@]}" - - local -a statics=() - local -a nonstatics=() - for binary in "${binaries[@]}"; do - if kube::golang::is_statically_linked_library "${binary}"; then - kube::golang::exit_if_stdlib_not_installed; - statics+=($binary) - else - nonstatics+=($binary) - fi - done - - if [[ -n ${use_go_build:-} ]]; then - # Try and replicate the native binary placement of go install without - # calling go install. This means we have to iterate each binary. - local output_path="${KUBE_GOPATH}/bin" - if [[ $platform != $host_platform ]]; then - output_path="${output_path}/${platform//\//_}" - fi + for platform in "${platforms[@]}"; do ( + kube::golang::set_platform_envs "${platform}" + kube::log::status "${platform}: Parallel go build started" + local -a statics=() + local -a nonstatics=() for binary in "${binaries[@]}"; do - local bin=$(basename "${binary}") - if [[ ${GOOS} == "windows" ]]; then - bin="${bin}.exe" - fi - if kube::golang::is_statically_linked_library "${binary}"; then kube::golang::exit_if_stdlib_not_installed; - CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \ - "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${binary}" + statics+=($binary) else - go build -o "${output_path}/${bin}" \ - "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${binary}" + nonstatics+=($binary) fi done - else + + if [[ -n ${use_go_build:-} ]]; then + # Try and replicate the native binary placement of go install without + # calling go install. This means we have to iterate each binary. + local output_path="${KUBE_GOPATH}/bin" + if [[ $platform != $host_platform ]]; then + output_path="${output_path}/${platform//\//_}" + fi + + for binary in "${binaries[@]}"; do + local bin=$(basename "${binary}") + if [[ ${GOOS} == "windows" ]]; then + bin="${bin}.exe" + fi + + if kube::golang::is_statically_linked_library "${binary}"; then + kube::golang::exit_if_stdlib_not_installed; + CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \ + "${goflags[@]:+${goflags[@]}}" \ + -ldflags "${version_ldflags}" \ + "${binary}" + else + go build -o "${output_path}/${bin}" \ + "${goflags[@]:+${goflags[@]}}" \ + -ldflags "${version_ldflags}" \ + "${binary}" + fi + done + else # Use go install. if [[ "${#nonstatics[@]}" != 0 ]]; then go install "${goflags[@]:+${goflags[@]}}" \ - -ldflags "${version_ldflags}" \ - "${nonstatics[@]:+${nonstatics[@]}}" + -ldflags "${version_ldflags}" \ + "${nonstatics[@]:+${nonstatics[@]}}" fi if [[ "${#statics[@]}" != 0 ]]; then CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \ -ldflags "${version_ldflags}" \ - "${statics[@]:+${statics[@]}}" + "${statics[@]:+${statics[@]}}" fi - fi + fi + kube::log::status "${platform}: Parallel go build finished" + ) &> "/tmp//${platform//\//_}.build" & done + + local fails=0 + for job in $(jobs -p); do + wait ${job} || let "fails+=1" + done + + for platform in "${platforms[@]}"; do + cat "/tmp//${platform//\//_}.build" + done + + exit ${fails} ) }