Merge pull request #6369 from zmerlynn/i6346

Redo on building platforms in parallel

build/common.sh

@@ -505,11 +505,14 @@ function kube::release::package_tarballs() {
   # Clean out any old releases
   rm -rf "${RELEASE_DIR}"
   mkdir -p "${RELEASE_DIR}"
-  kube::release::package_client_tarballs
+  kube::release::package_client_tarballs &
-  kube::release::package_server_tarballs
+  kube::release::package_server_tarballs &
-  kube::release::package_salt_tarball
+  kube::release::package_salt_tarball &
-  kube::release::package_test_tarball
+  wait || { kube::log::error "previous tarball phase failed"; return 1; }
-  kube::release::package_full_tarball
+
+  kube::release::package_full_tarball & # _full depends on all the previous phases
+  kube::release::package_test_tarball & # _test doesn't depend on anything
+  wait || { kube::log::error "previous tarball phase failed"; return 1; }
 }
 
 # Package up all of the cross compiled clients.  Over time this should grow into
@@ -520,8 +523,9 @@ function kube::release::package_client_tarballs() {
   platforms=($(cd "${LOCAL_OUTPUT_BINPATH}" ; echo */*))
   for platform in "${platforms[@]}"; do
     local platform_tag=${platform/\//-} # Replace a "/" for a "-"
-    kube::log::status "Building tarball: client $platform_tag"
+    kube::log::status "Starting tarball: client $platform_tag"
 
+    (
       local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes"
       rm -rf "${release_stage}"
       mkdir -p "${release_stage}/client/bin"
@@ -541,7 +545,11 @@ function kube::release::package_client_tarballs() {
 
       local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz"
       kube::release::create_tarball "${package_name}" "${release_stage}/.."
+    ) &
   done
+
+  kube::log::status "Waiting on tarballs"
+  wait || { kube::log::error "client tarball creation failed"; exit 1; }
 }
 
 # Package up all of the server binaries
@@ -578,27 +586,41 @@ function kube::release::package_server_tarballs() {
   done
 }
 
+function kube::release::md5() {
+  if which md5 >/dev/null 2>&1; then
+    md5 -q "$1"
+  else
+    md5sum "$1" | awk '{ print $1 }'
+  fi
+}
+
 # This will take binaries that run on master and creates Docker images
 # that wrap the binary in them. (One docker image per binary)
 function kube::release::create_docker_images_for_server() {
   # Create a sub-shell so that we don't pollute the outer environment
   (
-    local binary_name;
+    local binary_name
     for binary_name in "${KUBE_DOCKER_WRAPPED_BINARIES[@]}"; do
-      echo "+++ Building docker image: ${binary_name}";
+      kube::log::status "Starting Docker build for image: ${binary_name}"
-      local docker_file_path="$1/Dockerfile";
+
-      local binary_file_path="$1/${binary_name}";
+      (
+        local docker_file_path="$1/${binary_name}.Dockerfile"
+        local binary_file_path="$1/${binary_name}"
         if [ -f ${docker_file_path} ]; then
-        rm ${docker_file_path};
+          rm ${docker_file_path}
-      fi;
+        fi
-      printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path};
+        printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path}
-      local md5_sum=$(md5sum ${binary_file_path} | awk '{print $1}')
+        local md5_sum=$(kube::release::md5 ${binary_file_path})
         local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum
-      docker build -t "${docker_image_tag}" ${1};
+        docker build -q -f "${docker_file_path}" -t "${docker_image_tag}" ${1} >/dev/null
-      docker save ${docker_image_tag} > ${1}/${binary_name}.tar;
+        docker save ${docker_image_tag} > ${1}/${binary_name}.tar
-      echo $md5_sum > ${1}/${binary_name}.docker_tag;
+        echo $md5_sum > ${1}/${binary_name}.docker_tag
-      rm ${docker_file_path};
+        rm ${docker_file_path}
+      ) &
     done
+
+    wait || { kube::log::error "previous Docker build failed"; return 1; }
+    kube::log::status "Docker builds done"
   )
 }
 

hack/lib/golang.sh

@@ -72,6 +72,11 @@ readonly KUBE_CLIENT_PLATFORMS=(
   windows/amd64
 )
 
+# Gigabytes desired for parallel platform builds. 8 is fairly
+# arbitrary, but is a reasonable splitting point for 2015
+# laptops-versus-not.
+readonly KUBE_PARALLEL_BUILD_MEMORY=8
+
 readonly KUBE_ALL_TARGETS=(
   "${KUBE_SERVER_TARGETS[@]}"
   "${KUBE_CLIENT_TARGETS[@]}"
@@ -263,60 +268,9 @@ kube::golang::exit_if_stdlib_not_installed() {
   exit 0;
 }
 
-# Build binaries targets specified
+kube::golang::build_binaries_for_platform() {
-#
+  local platform=$1
-# Input:
+  local use_go_build=${2-}
-#   $@ - targets and go flags.  If no targets are set then all binaries targets
-#     are built.
-#   KUBE_BUILD_PLATFORMS - Incoming variable of targets to build for.  If unset
-#     then just the host architecture is built.
-kube::golang::build_binaries() {
-  # Create a sub-shell so that we don't pollute the outer environment
-  (
-    # Check for `go` binary and set ${GOPATH}.
-    kube::golang::setup_env
-
-    # Fetch the version.
-    local version_ldflags
-    version_ldflags=$(kube::version::ldflags)
-
-    local host_platform
-    host_platform=$(kube::golang::host_platform)
-
-    # Use eval to preserve embedded quoted strings.
-    local goflags
-    eval "goflags=(${KUBE_GOFLAGS:-})"
-
-    local use_go_build
-    local -a targets=()
-    local arg
-    for arg; do
-      if [[ "${arg}" == "--use_go_build" ]]; then
-        use_go_build=true
-      elif [[ "${arg}" == -* ]]; then
-        # Assume arguments starting with a dash are flags to pass to go.
-        goflags+=("${arg}")
-      else
-        targets+=("${arg}")
-      fi
-    done
-
-    if [[ ${#targets[@]} -eq 0 ]]; then
-      targets=("${KUBE_ALL_TARGETS[@]}")
-    fi
-
-    local -a platforms=("${KUBE_BUILD_PLATFORMS[@]:+${KUBE_BUILD_PLATFORMS[@]}}")
-    if [[ ${#platforms[@]} -eq 0 ]]; then
-      platforms=("${host_platform}")
-    fi
-
-    local binaries
-    binaries=($(kube::golang::binaries_from_targets "${targets[@]}"))
-
-    local platform
-    for platform in "${platforms[@]}"; do
-      kube::golang::set_platform_envs "${platform}"
-      kube::log::status "Building go targets for ${platform}:" "${targets[@]}"
 
   local -a statics=()
   local -a nonstatics=()
@@ -369,6 +323,123 @@ kube::golang::build_binaries() {
         "${statics[@]:+${statics[@]}}"
     fi
   fi
+
+}
+
+# Return approximate physical memory in gigabytes.
+kube::golang::get_physmem() {
+  local mem
+
+  # Linux, in kb
+  if mem=$(grep MemTotal /proc/meminfo | awk '{ print $2 }'); then
+    echo $(( ${mem} / 1048576 ))
+    return
+  fi
+
+  # OS X, in bytes. Note that get_physmem, as used, should only ever
+  # run in a Linux container (because it's only used in the multiple
+  # platform case, which is a Dockerized build), but this is provided
+  # for completeness.
+  if mem=$(sysctl -n hw.memsize 2>/dev/null); then
+    echo $(( ${mem} / 1073741824 ))
+    return
+  fi
+
+  # If we can't infer it, just give up and assume a low memory system
+  echo 1
+}
+
+# Build binaries targets specified
+#
+# Input:
+#   $@ - targets and go flags.  If no targets are set then all binaries targets
+#     are built.
+#   KUBE_BUILD_PLATFORMS - Incoming variable of targets to build for.  If unset
+#     then just the host architecture is built.
+kube::golang::build_binaries() {
+  # Create a sub-shell so that we don't pollute the outer environment
+  (
+    # Check for `go` binary and set ${GOPATH}.
+    kube::golang::setup_env
+
+    # Fetch the version.
+    local version_ldflags
+    version_ldflags=$(kube::version::ldflags)
+
+    local host_platform
+    host_platform=$(kube::golang::host_platform)
+
+    # Use eval to preserve embedded quoted strings.
+    local goflags
+    eval "goflags=(${KUBE_GOFLAGS:-})"
+
+    local use_go_build
+    local -a targets=()
+    local arg
+    for arg; do
+      if [[ "${arg}" == "--use_go_build" ]]; then
+        use_go_build=true
+      elif [[ "${arg}" == -* ]]; then
+        # Assume arguments starting with a dash are flags to pass to go.
+        goflags+=("${arg}")
+      else
+        targets+=("${arg}")
+      fi
     done
+
+    if [[ ${#targets[@]} -eq 0 ]]; then
+      targets=("${KUBE_ALL_TARGETS[@]}")
+    fi
+
+    local -a platforms=("${KUBE_BUILD_PLATFORMS[@]:+${KUBE_BUILD_PLATFORMS[@]}}")
+    if [[ ${#platforms[@]} -eq 0 ]]; then
+      platforms=("${host_platform}")
+    fi
+
+    local binaries
+    binaries=($(kube::golang::binaries_from_targets "${targets[@]}"))
+
+    local parallel=false
+    if [[ ${#platforms[@]} -gt 1 ]]; then
+      local gigs
+      gigs=$(kube::golang::get_physmem)
+
+      if [[ ${gigs} -gt ${KUBE_PARALLEL_BUILD_MEMORY} ]]; then
+        kube::log::status "Multiple platforms requested and available ${gigs}G > threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in parallel"
+        parallel=true
+      else
+        kube::log::status "Multiple platforms requested, but available ${gigs}G < threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in serial"
+        parallel=false
+      fi
+    fi
+
+    if [[ "${parallel}" == "true" ]]; then
+      kube::log::status "Building go targets for ${platforms[@]} in parallel (output will appear in a burst when complete):" "${targets[@]}"
+      local platform
+      for platform in "${platforms[@]}"; do (
+          kube::golang::set_platform_envs "${platform}"
+          kube::log::status "${platform}: go build started"
+          kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
+          kube::log::status "${platform}: go build finished"
+        ) &> "/tmp//${platform//\//_}.build" &
+      done
+
+      local fails=0
+      for job in $(jobs -p); do
+        wait ${job} || let "fails+=1"
+      done
+
+      for platform in "${platforms[@]}"; do
+        cat "/tmp//${platform//\//_}.build"
+      done
+
+      exit ${fails}
+    else
+      for platform in "${platforms[@]}"; do
+        kube::log::status "Building go targets for ${platform}:" "${targets[@]}"
+        kube::golang::set_platform_envs "${platform}"
+        kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
+      done
+    fi
   )
 }