From 68232c328fa0f26c6c66c3d1896ce0b9155aeb06 Mon Sep 17 00:00:00 2001 From: Christoph Blecker Date: Fri, 25 Aug 2017 16:51:51 -0700 Subject: [PATCH 1/3] Create kube::util::create-fake-git-tree function --- hack/lib/util.sh | 19 ++++++++++++++++++- hack/update-staging-godeps-dockerized.sh | 23 +++++++---------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/hack/lib/util.sh b/hack/lib/util.sh index ab27382c56e..8e96ad462b1 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -331,7 +331,7 @@ kube::util::group-version-to-pkg-path() { return fi - # "v1" is the API GroupVersion + # "v1" is the API GroupVersion if [[ "${group_version}" == "v1" ]]; then echo "vendor/k8s.io/api/core/v1" return @@ -437,6 +437,23 @@ kube::util::git_upstream_remote_name() { head -n 1 | awk '{print $1}' } +# Ensures the current directory is a git tree for doing things like restoring or +# validating godeps +kube::util::create-fake-git-tree() { + local -r target_dir=${1:-$(pwd)} + + pushd "${target_dir}" >/dev/null + git init >/dev/null + git config --local user.email "nobody@k8s.io" + git config --local user.name "$0" + git add . >/dev/null + git commit -q -m "Snapshot" >/dev/null + if (( ${KUBE_VERBOSE:-5} >= 6 )); then + kube::log::status "${target_dir} is now a git tree." + fi + popd >/dev/null +} + # Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist # and are checked out to the referenced rev. kube::util::godep_restored() { diff --git a/hack/update-staging-godeps-dockerized.sh b/hack/update-staging-godeps-dockerized.sh index 458cbb211f2..bf4bbea8bba 100755 --- a/hack/update-staging-godeps-dockerized.sh +++ b/hack/update-staging-godeps-dockerized.sh @@ -50,11 +50,16 @@ while getopts ":df" opt; do esac done +# Confirm this is running inside a docker container, as this will modify the git tree (unsafe to run outside of container) kube::util::ensure_dockerized kube::golang::setup_env +# Ensure we have a simple gopath so that we can modify it, and that no staging repos have made their way in kube::util::ensure_single_dir_gopath kube::util::ensure_no_staging_repos_in_gopath +# Confirm we have the right godep version installed kube::util::ensure_godep_version v79 +# Create a fake git repo the root of the repo to prevent godeps from complaining +kube::util::create-fake-git-tree "${KUBE_ROOT}" kube::log::status "Checking whether godeps are restored" if ! kube::util::godep_restored 2>&1 | sed 's/^/ /'; then @@ -87,27 +92,13 @@ function diffGodepManifest() { fi } -# Create a fake git repo for staging to prevent godeps from complaining -pushd "${KUBE_ROOT}" >/dev/null - git init >/dev/null - git config --local user.email "nobody@k8s.io" - git config --local user.name "$0" - git add . >/dev/null - git commit -q -m "Snapshot" >/dev/null -popd >/dev/null - # move into staging and save the dependencies for everything in order mkdir -p "${TMP_GOPATH}/src/k8s.io" for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/" - pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null - git init >/dev/null - git config --local user.email "nobody@k8s.io" - git config --local user.name "$0" - git add . >/dev/null - git commit -q -m "Snapshot" >/dev/null - popd >/dev/null + # Create a fake git tree for the staging repo to prevent godeps from complaining + kube::util::create-fake-git-tree "${TMP_GOPATH}/src/k8s.io/${repo}" updateGodepManifest diffGodepManifest From 4d63e13c9fedd75e352e09102497a768f79da225 Mon Sep 17 00:00:00 2001 From: Christoph Blecker Date: Fri, 25 Aug 2017 16:58:06 -0700 Subject: [PATCH 2/3] Add option to copy output when running the build shell --- build/run.sh | 11 ++++++++++- build/shell.sh | 5 +---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/build/run.sh b/build/run.sh index af6e92dfa3f..5304f928037 100755 --- a/build/run.sh +++ b/build/run.sh @@ -25,10 +25,19 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "$KUBE_ROOT/build/common.sh" +KUBE_RUN_COPY_OUTPUT="${KUBE_RUN_COPY_OUTPUT:-y}" + kube::build::verify_prereqs kube::build::build_image + +if [[ ${KUBE_RUN_COPY_OUTPUT} =~ ^[yY]$ ]]; then + kube::log::status "Output from this container will be rsynced out upon completion. Set KUBE_RUN_COPY_OUTPUT=n to disable." +else + kube::log::status "Output from this container will NOT be rsynced out upon completion. Set KUBE_RUN_COPY_OUTPUT=y to enable." +fi + kube::build::run_build_command "$@" -if [[ ${KUBE_RUN_COPY_OUTPUT:-y} =~ ^[yY]$ ]]; then +if [[ ${KUBE_RUN_COPY_OUTPUT} =~ ^[yY]$ ]]; then kube::build::copy_output fi diff --git a/build/shell.sh b/build/shell.sh index 6f546b662de..dac2e494976 100755 --- a/build/shell.sh +++ b/build/shell.sh @@ -24,8 +24,5 @@ set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/build/common.sh" -source "${KUBE_ROOT}/build/lib/release.sh" -kube::build::verify_prereqs -kube::build::build_image -kube::build::run_build_command bash || true +KUBE_RUN_COPY_OUTPUT="${KUBE_RUN_COPY_OUTPUT:-n}" "${KUBE_ROOT}/build/run.sh" bash "$@" From 4f1106c8a536228dd6d401698ee304b5b24383b1 Mon Sep 17 00:00:00 2001 From: Christoph Blecker Date: Fri, 25 Aug 2017 16:58:59 -0700 Subject: [PATCH 3/3] Modify rsync filter to retain output across runs --- build/common.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/build/common.sh b/build/common.sh index 80c6fc7d8a3..2db1fae3226 100755 --- a/build/common.sh +++ b/build/common.sh @@ -654,7 +654,6 @@ function kube::build::stop_rsyncd_container() { function kube::build::rsync { local -a rsync_opts=( --archive - --prune-empty-dirs --password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" ) if (( ${KUBE_VERBOSE} >= 6 )); then @@ -679,16 +678,19 @@ function kube::build::sync_to_container() { # directory and generated files. The '- /' filter prevents rsync # from trying to set the uid/gid/perms on the root of the sync tree. # As an exception, we need to sync generated files in staging/, because - # they will not be re-generated by 'make'. + # they will not be re-generated by 'make'. Note that the 'H' filtered files + # are hidden from rsync so they will be deleted in the target container if + # they exist. This will allow them to be re-created in the container if + # necessary. kube::build::rsync \ --delete \ - --filter='- /.git/' \ + --filter='H /.git/' \ --filter='- /.make/' \ --filter='- /_tmp/' \ --filter='- /_output/' \ --filter='- /' \ - --filter='- zz_generated.*' \ - --filter='- generated.proto' \ + --filter='H zz_generated.*' \ + --filter='H generated.proto' \ "${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" kube::build::stop_rsyncd_container @@ -714,6 +716,7 @@ function kube::build::copy_output() { # We are looking to copy out all of the built binaries along with various # generated files. kube::build::rsync \ + --prune-empty-dirs \ --filter='- /_temp/' \ --filter='+ /vendor/' \ --filter='+ /Godeps/' \