Fix subtle build breakage

Repro case:
$ make clean generated_files
$ hack/update-generated-protobuf.sh

This would complain about not finding `fmt`, and it was indicating the wrong
GOROOT.  The problem was that the first step built binaries for generating
code, which *embeds* the value of GOROOT into the binary.  The whole tree was
bind-mounted into the build container and then JUST the dockerized dir was
mounted over it.  The in-container build tried to use the existing binaries,
but GOROOT is wrong.

This change whites-out the whole _output dir.

I first made just an anonymous volume for _output, but docker makes that as
root, which means I can't write to it from our non-root build.  So I just put
it in the data container.  This seems to work.  The biggest change this makes
is that the $GOPATH/bin/ and $GOPATH/pkg/ dirs will persist across dockerized
builds.

build/common.sh

@@ -67,15 +67,15 @@ readonly LOCAL_OUTPUT_IMAGE_STAGING="${LOCAL_OUTPUT_ROOT}/images"
 # This is a symlink to binaries for "this platform" (e.g. build tools).
 readonly THIS_PLATFORM_BIN="${LOCAL_OUTPUT_ROOT}/bin"
 
-readonly OUTPUT_BINPATH="${CUSTOM_OUTPUT_BINPATH:-$LOCAL_OUTPUT_BINPATH}"
-
 readonly REMOTE_OUTPUT_ROOT="/go/src/${KUBE_GO_PACKAGE}/_output"
 readonly REMOTE_OUTPUT_SUBPATH="${REMOTE_OUTPUT_ROOT}/dockerized"
 readonly REMOTE_OUTPUT_BINPATH="${REMOTE_OUTPUT_SUBPATH}/bin"
 readonly REMOTE_OUTPUT_GOPATH="${REMOTE_OUTPUT_SUBPATH}/go"
 
 readonly DOCKER_MOUNT_ARGS_BASE=(
-  --volume "${OUTPUT_BINPATH}:${REMOTE_OUTPUT_BINPATH}"
+  # where the container build will drop output
+  --volume "${LOCAL_OUTPUT_BINPATH}:${REMOTE_OUTPUT_BINPATH}"
+  # timezone
   --volume /etc/localtime:/etc/localtime:ro
 )
 
@@ -568,11 +568,12 @@ function kube::build::ensure_data_container() {
     # container and chowns the GOPATH.
     local -ra docker_cmd=(
       "${DOCKER[@]}" run
-      --volume "${REMOTE_OUTPUT_GOPATH}"
       --name "${KUBE_BUILD_DATA_CONTAINER_NAME}"
       --hostname "${HOSTNAME}"
+      --volume "${REMOTE_OUTPUT_ROOT}"   # white-out the whole output dir
+      --volume "${REMOTE_OUTPUT_GOPATH}" # make a non-root owned mountpoint
       "${KUBE_BUILD_IMAGE}"
-      chown -R $(id -u).$(id -g) "${REMOTE_OUTPUT_GOPATH}"
+      chown -R $(id -u).$(id -g) "${REMOTE_OUTPUT_ROOT}"
     )
     "${docker_cmd[@]}"
   fi

hack/lib/golang.sh

@@ -612,18 +612,18 @@ kube::golang::build_binaries() {
     local use_go_build
     local -a targets=()
     local arg
-    
+
     # Add any files with those //generate annotations in the array below.
     readonly BINDATAS=( "${KUBE_ROOT}/test/e2e/framework/gobindata_util.go" )
     kube::log::status "Generating bindata:" "${BINDATAS[@]}"
     for bindata in ${BINDATAS[@]}; do
-	  # Only try to generate bindata if the file exists, since in some cases
-	  # one-off builds of individual directories may exclude some files.
+          # Only try to generate bindata if the file exists, since in some cases
+          # one-off builds of individual directories may exclude some files.
       if [[ -f $bindata ]]; then
           go generate "${bindata}"
       fi
     done
-    
+
     for arg; do
       if [[ "${arg}" == "--use_go_build" ]]; then
         use_go_build=true

hack/update-generated-protobuf-dockerized.sh

@@ -45,7 +45,7 @@ gotoprotobuf=$(kube::util::find-binary "go-to-protobuf")
 # searches for the protoc-gen-gogo extension in the output directory
 # satisfies import of github.com/gogo/protobuf/gogoproto/gogo.proto and the
 # core Google protobuf types
-PATH="${KUBE_ROOT}/_output/local/go/bin:${PATH}" \
+PATH="${KUBE_ROOT}/_output/bin:${PATH}" \
   "${gotoprotobuf}" \
   --proto-import="${KUBE_ROOT}/vendor" \
   --proto-import="${KUBE_ROOT}/third_party/protobuf" \