Merge pull request #6369 from zmerlynn/i6346

Redo on building platforms in parallel

build/common.sh

@@ -505,11 +505,14 @@ function kube::release::package_tarballs() {
   # Clean out any old releases
   rm -rf "${RELEASE_DIR}"
   mkdir -p "${RELEASE_DIR}"
-  kube::release::package_client_tarballs
+  kube::release::package_client_tarballs &
-  kube::release::package_server_tarballs
+  kube::release::package_server_tarballs &
-  kube::release::package_salt_tarball
+  kube::release::package_salt_tarball &
-  kube::release::package_test_tarball
+  wait || { kube::log::error "previous tarball phase failed"; return 1; }
-  kube::release::package_full_tarball
+
+  kube::release::package_full_tarball & # _full depends on all the previous phases
+  kube::release::package_test_tarball & # _test doesn't depend on anything
+  wait || { kube::log::error "previous tarball phase failed"; return 1; }
 }
 
 # Package up all of the cross compiled clients.  Over time this should grow into
@@ -518,30 +521,35 @@ function kube::release::package_client_tarballs() {
    # Find all of the built client binaries
   local platform platforms
   platforms=($(cd "${LOCAL_OUTPUT_BINPATH}" ; echo */*))
-  for platform in "${platforms[@]}" ; do
+  for platform in "${platforms[@]}"; do
     local platform_tag=${platform/\//-} # Replace a "/" for a "-"
-    kube::log::status "Building tarball: client $platform_tag"
+    kube::log::status "Starting tarball: client $platform_tag"
 
-    local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes"
+    (
-    rm -rf "${release_stage}"
+      local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes"
-    mkdir -p "${release_stage}/client/bin"
+      rm -rf "${release_stage}"
+      mkdir -p "${release_stage}/client/bin"
 
-    local client_bins=("${KUBE_CLIENT_BINARIES[@]}")
+      local client_bins=("${KUBE_CLIENT_BINARIES[@]}")
-    if [[ "${platform%/*}" == "windows" ]]; then
+      if [[ "${platform%/*}" == "windows" ]]; then
-      client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}")
+        client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}")
-    fi
+      fi
 
-    # This fancy expression will expand to prepend a path
+      # This fancy expression will expand to prepend a path
-    # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
+      # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
-    # KUBE_CLIENT_BINARIES array.
+      # KUBE_CLIENT_BINARIES array.
-    cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
+      cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
-      "${release_stage}/client/bin/"
+        "${release_stage}/client/bin/"
 
-    kube::release::clean_cruft
+      kube::release::clean_cruft
 
-    local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz"
+      local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz"
-    kube::release::create_tarball "${package_name}" "${release_stage}/.."
+      kube::release::create_tarball "${package_name}" "${release_stage}/.."
+    ) &
   done
+
+  kube::log::status "Waiting on tarballs"
+  wait || { kube::log::error "client tarball creation failed"; exit 1; }
 }
 
 # Package up all of the server binaries
@@ -578,27 +586,41 @@ function kube::release::package_server_tarballs() {
   done
 }
 
+function kube::release::md5() {
+  if which md5 >/dev/null 2>&1; then
+    md5 -q "$1"
+  else
+    md5sum "$1" | awk '{ print $1 }'
+  fi
+}
+
 # This will take binaries that run on master and creates Docker images
 # that wrap the binary in them. (One docker image per binary)
 function kube::release::create_docker_images_for_server() {
   # Create a sub-shell so that we don't pollute the outer environment
   (
-    local binary_name;
+    local binary_name
     for binary_name in "${KUBE_DOCKER_WRAPPED_BINARIES[@]}"; do
-      echo "+++ Building docker image: ${binary_name}";
+      kube::log::status "Starting Docker build for image: ${binary_name}"
-      local docker_file_path="$1/Dockerfile";
+
-      local binary_file_path="$1/${binary_name}";
+      (
-      if [ -f ${docker_file_path} ]; then
+        local docker_file_path="$1/${binary_name}.Dockerfile"
-        rm ${docker_file_path};
+        local binary_file_path="$1/${binary_name}"
-      fi;
+        if [ -f ${docker_file_path} ]; then
-      printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path};
+          rm ${docker_file_path}
-      local md5_sum=$(md5sum ${binary_file_path} | awk '{print $1}')
+        fi
-      local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum
+        printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path}
-      docker build -t "${docker_image_tag}" ${1};
+        local md5_sum=$(kube::release::md5 ${binary_file_path})
-      docker save ${docker_image_tag} > ${1}/${binary_name}.tar;
+        local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum
-      echo $md5_sum > ${1}/${binary_name}.docker_tag;
+        docker build -q -f "${docker_file_path}" -t "${docker_image_tag}" ${1} >/dev/null
-      rm ${docker_file_path};
+        docker save ${docker_image_tag} > ${1}/${binary_name}.tar
+        echo $md5_sum > ${1}/${binary_name}.docker_tag
+        rm ${docker_file_path}
+      ) &
     done
+
+    wait || { kube::log::error "previous Docker build failed"; return 1; }
+    kube::log::status "Docker builds done"
   )
 }
 

hack/lib/golang.sh

@@ -72,6 +72,11 @@ readonly KUBE_CLIENT_PLATFORMS=(
   windows/amd64
 )
 
+# Gigabytes desired for parallel platform builds. 8 is fairly
+# arbitrary, but is a reasonable splitting point for 2015
+# laptops-versus-not.
+readonly KUBE_PARALLEL_BUILD_MEMORY=8
+
 readonly KUBE_ALL_TARGETS=(
   "${KUBE_SERVER_TARGETS[@]}"
   "${KUBE_CLIENT_TARGETS[@]}"
@@ -263,6 +268,87 @@ kube::golang::exit_if_stdlib_not_installed() {
   exit 0;
 }
 
+kube::golang::build_binaries_for_platform() {
+  local platform=$1
+  local use_go_build=${2-}
+
+  local -a statics=()
+  local -a nonstatics=()
+  for binary in "${binaries[@]}"; do
+    if kube::golang::is_statically_linked_library "${binary}"; then
+      kube::golang::exit_if_stdlib_not_installed;
+      statics+=($binary)
+    else
+      nonstatics+=($binary)
+    fi
+  done
+
+  if [[ -n ${use_go_build:-} ]]; then
+    # Try and replicate the native binary placement of go install without
+    # calling go install.  This means we have to iterate each binary.
+    local output_path="${KUBE_GOPATH}/bin"
+    if [[ $platform != $host_platform ]]; then
+      output_path="${output_path}/${platform//\//_}"
+    fi
+
+    for binary in "${binaries[@]}"; do
+      local bin=$(basename "${binary}")
+      if [[ ${GOOS} == "windows" ]]; then
+        bin="${bin}.exe"
+      fi
+
+      if kube::golang::is_statically_linked_library "${binary}"; then
+        kube::golang::exit_if_stdlib_not_installed;
+        CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \
+          "${goflags[@]:+${goflags[@]}}" \
+          -ldflags "${version_ldflags}" \
+          "${binary}"
+      else
+        go build -o "${output_path}/${bin}" \
+          "${goflags[@]:+${goflags[@]}}" \
+          -ldflags "${version_ldflags}" \
+          "${binary}"
+      fi
+    done
+  else
+    # Use go install.
+    if [[ "${#nonstatics[@]}" != 0 ]]; then
+      go install "${goflags[@]:+${goflags[@]}}" \
+        -ldflags "${version_ldflags}" \
+        "${nonstatics[@]:+${nonstatics[@]}}"
+    fi
+    if [[ "${#statics[@]}" != 0 ]]; then
+      CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \
+        -ldflags "${version_ldflags}" \
+        "${statics[@]:+${statics[@]}}"
+    fi
+  fi
+
+}
+
+# Return approximate physical memory in gigabytes.
+kube::golang::get_physmem() {
+  local mem
+
+  # Linux, in kb
+  if mem=$(grep MemTotal /proc/meminfo | awk '{ print $2 }'); then
+    echo $(( ${mem} / 1048576 ))
+    return
+  fi
+
+  # OS X, in bytes. Note that get_physmem, as used, should only ever
+  # run in a Linux container (because it's only used in the multiple
+  # platform case, which is a Dockerized build), but this is provided
+  # for completeness.
+  if mem=$(sysctl -n hw.memsize 2>/dev/null); then
+    echo $(( ${mem} / 1073741824 ))
+    return
+  fi
+
+  # If we can't infer it, just give up and assume a low memory system
+  echo 1
+}
+
 # Build binaries targets specified
 #
 # Input:
@@ -313,62 +399,47 @@ kube::golang::build_binaries() {
     local binaries
     binaries=($(kube::golang::binaries_from_targets "${targets[@]}"))
 
-    local platform
+    local parallel=false
-    for platform in "${platforms[@]}"; do
+    if [[ ${#platforms[@]} -gt 1 ]]; then
-      kube::golang::set_platform_envs "${platform}"
+      local gigs
-      kube::log::status "Building go targets for ${platform}:" "${targets[@]}"
+      gigs=$(kube::golang::get_physmem)
 
-      local -a statics=()
+      if [[ ${gigs} -gt ${KUBE_PARALLEL_BUILD_MEMORY} ]]; then
-      local -a nonstatics=()
+        kube::log::status "Multiple platforms requested and available ${gigs}G > threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in parallel"
-      for binary in "${binaries[@]}"; do
+        parallel=true
-        if kube::golang::is_statically_linked_library "${binary}"; then
+      else
-          kube::golang::exit_if_stdlib_not_installed;
+        kube::log::status "Multiple platforms requested, but available ${gigs}G < threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in serial"
-          statics+=($binary)
+        parallel=false
-        else
+      fi
-          nonstatics+=($binary)
+    fi
-        fi
+
+    if [[ "${parallel}" == "true" ]]; then
+      kube::log::status "Building go targets for ${platforms[@]} in parallel (output will appear in a burst when complete):" "${targets[@]}"
+      local platform
+      for platform in "${platforms[@]}"; do (
+          kube::golang::set_platform_envs "${platform}"
+          kube::log::status "${platform}: go build started"
+          kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
+          kube::log::status "${platform}: go build finished"
+        ) &> "/tmp//${platform//\//_}.build" &
       done
 
-      if [[ -n ${use_go_build:-} ]]; then
+      local fails=0
-        # Try and replicate the native binary placement of go install without
+      for job in $(jobs -p); do
-        # calling go install.  This means we have to iterate each binary.
+        wait ${job} || let "fails+=1"
-        local output_path="${KUBE_GOPATH}/bin"
+      done
-        if [[ $platform != $host_platform ]]; then
-          output_path="${output_path}/${platform//\//_}"
-        fi
 
-        for binary in "${binaries[@]}"; do
+      for platform in "${platforms[@]}"; do
-          local bin=$(basename "${binary}")
+        cat "/tmp//${platform//\//_}.build"
-          if [[ ${GOOS} == "windows" ]]; then
+      done
-            bin="${bin}.exe"
-          fi
 
-          if kube::golang::is_statically_linked_library "${binary}"; then
+      exit ${fails}
-            kube::golang::exit_if_stdlib_not_installed;
+    else
-            CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \
+      for platform in "${platforms[@]}"; do
-              "${goflags[@]:+${goflags[@]}}" \
+        kube::log::status "Building go targets for ${platform}:" "${targets[@]}"
-              -ldflags "${version_ldflags}" \
+        kube::golang::set_platform_envs "${platform}"
-              "${binary}"
+        kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
-          else
+      done
-            go build -o "${output_path}/${bin}" \
+    fi
-              "${goflags[@]:+${goflags[@]}}" \
-              -ldflags "${version_ldflags}" \
-              "${binary}"
-          fi
-        done
-      else
-          # Use go install.
-          if [[ "${#nonstatics[@]}" != 0 ]]; then
-            go install "${goflags[@]:+${goflags[@]}}" \
-                -ldflags "${version_ldflags}" \
-                "${nonstatics[@]:+${nonstatics[@]}}"
-          fi
-          if [[ "${#statics[@]}" != 0 ]]; then
-            CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \
-              -ldflags "${version_ldflags}" \
-                "${statics[@]:+${statics[@]}}"
-          fi
-      fi
-    done
   )
 }