From d39ac98cc5d54568c784ebb31852209a62171921 Mon Sep 17 00:00:00 2001 From: Ji Shan Xing Date: Sat, 21 Sep 2019 05:28:43 -0400 Subject: [PATCH] 76093 restructure LICENSES file generation --- Godeps/OWNERS | 4 -- build/common.sh | 2 - build/lib/release.sh | 9 +++-- build/release-tars/BUILD | 14 ++++++- cluster/gce/gci/configure.sh | 2 +- hack/boilerplate/boilerplate.py | 2 +- hack/update-vendor-licenses.sh | 58 +++++++++++++++++++--------- hack/update-vendor.sh | 5 +-- hack/verify-vendor-licenses.sh | 15 +++++-- hack/verify-vendor.sh | 2 +- staging/publishing/rules-godeps.yaml | 4 +- 11 files changed, 76 insertions(+), 41 deletions(-) delete mode 100644 Godeps/OWNERS diff --git a/Godeps/OWNERS b/Godeps/OWNERS deleted file mode 100644 index 0f5d2f6734e..00000000000 --- a/Godeps/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -approvers: -- dep-approvers diff --git a/build/common.sh b/build/common.sh index 2ae17c9f1a9..8872b6059f6 100755 --- a/build/common.sh +++ b/build/common.sh @@ -729,8 +729,6 @@ function kube::build::copy_output() { --prune-empty-dirs \ --filter='- /_temp/' \ --filter='+ /vendor/' \ - --filter='+ /Godeps/' \ - --filter='+ /staging/***/Godeps/**' \ --filter='+ /_output/dockerized/bin/**' \ --filter='+ zz_generated.*' \ --filter='+ generated.proto' \ diff --git a/build/lib/release.sh b/build/lib/release.sh index b538c1e4880..26ccfd54553 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -187,7 +187,8 @@ function kube::release::package_node_tarballs() { cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ "${release_stage}/node/bin/" - cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/" + cp -R "${KUBE_ROOT}/vendor/ALL_LICENSES" "${release_stage}/" + cp "${KUBE_ROOT}/LICENSE" "${release_stage}/LICENSES" cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/" @@ -252,7 +253,8 @@ function kube::release::package_server_tarballs() { cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ "${release_stage}/server/bin/" - cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/" + cp -R "${KUBE_ROOT}/vendor/ALL_LICENSES" "${release_stage}/" + cp "${KUBE_ROOT}/LICENSE" "${release_stage}/LICENSES" cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/" @@ -565,7 +567,8 @@ EOF cp -R "${KUBE_ROOT}/docs" "${release_stage}/" cp "${KUBE_ROOT}/README.md" "${release_stage}/" - cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/" + cp -R "${KUBE_ROOT}/vendor/ALL_LICENSES" "${release_stage}/" + cp "${KUBE_ROOT}/LICENSE" "${release_stage}/LICENSES" echo "${KUBE_GIT_VERSION}" > "${release_stage}/version" diff --git a/build/release-tars/BUILD b/build/release-tars/BUILD index 013a844beb9..16ef8198e4a 100644 --- a/build/release-tars/BUILD +++ b/build/release-tars/BUILD @@ -69,12 +69,21 @@ pkg_tar( ], ) +filegroup( + name = "all-licenses-files", + srcs = glob([ + "vendor/ALL_LICENSES/*", + ]), + visibility = ["//visibility:private"], +) + # Included in node and server tarballs. filegroup( name = "license-targets", srcs = [ + ":all-licenses-files", ":kubernetes-src.tar.gz", - "//:Godeps/LICENSES", + "//:LICENSE", ], visibility = ["//visibility:private"], ) @@ -235,7 +244,8 @@ pkg_tar( pkg_tar( name = "kubernetes", srcs = [ - "//:Godeps/LICENSES", + ":all-licenses-files", + "//:LICENSE", "//:README.md", "//:version", "//cluster:all-srcs", diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index 639f4a8f4bd..c4f1a20e5c4 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -421,7 +421,7 @@ function install-kube-binary-config { mv "${src_dir}/kubelet" "${KUBE_BIN}" mv "${src_dir}/kubectl" "${KUBE_BIN}" - mv "${KUBE_HOME}/kubernetes/LICENSES" "${KUBE_HOME}" + mv "${KUBE_HOME}/kubernetes/LICENSE" "${KUBE_HOME}" mv "${KUBE_HOME}/kubernetes/kubernetes-src.tar.gz" "${KUBE_HOME}" fi diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py index 2c03451a9b4..5a6d3177eb1 100755 --- a/hack/boilerplate/boilerplate.py +++ b/hack/boilerplate/boilerplate.py @@ -155,7 +155,7 @@ def file_extension(filename): return os.path.splitext(filename)[1].split(".")[-1].lower() -skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', 'cluster/env.sh', +skipped_dirs = ['third_party', '_gopath', '_output', '.git', 'cluster/env.sh', "vendor", "test/e2e/generated/bindata.go", "hack/boilerplate/test", "staging/src/k8s.io/kubectl/pkg/generated/bindata.go"] diff --git a/hack/update-vendor-licenses.sh b/hack/update-vendor-licenses.sh index 99c3411cfae..260d62cc63b 100755 --- a/hack/update-vendor-licenses.sh +++ b/hack/update-vendor-licenses.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Update the Godeps/LICENSES document. +# Update the vendor/ALL_LICENSES document. # Generates a table of Godep dependencies and their license. # # Usage: @@ -24,12 +24,15 @@ # additionally created files into the vendor auto-generated tree. # # Run every time a license file is added/modified within /vendor to -# update /Godeps/LICENSES +# update /vendor/ALL_LICENSES set -o errexit set -o nounset set -o pipefail +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" + export LANG=C export LC_ALL=C @@ -130,8 +133,6 @@ process_content () { ############################################################################# # MAIN ############################################################################# -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${KUBE_ROOT}/hack/lib/init.sh" export GO111MODULE=on @@ -153,8 +154,19 @@ fi LICENSE_ROOT="${LICENSE_ROOT:-${KUBE_ROOT}}" cd "${LICENSE_ROOT}" -VENDOR_LICENSE_FILE="Godeps/LICENSES" -TMP_LICENSE_FILE="/tmp/Godeps.LICENSES.$$" +kube::util::ensure-temp-dir + +# Save the genreated LICENSE file for each package temporarily +TMP_LICENSE_FILE="${KUBE_TEMP}/vendor.LICENSES.$$" + +# The directory to save all the LICENSE files +VENDOR_LICENSE_DIR="${LICENSE_ROOT}/vendor/ALL_LICENSES" + +# The tmp directory to save all the LICENSE files, will move to VENDOR_LICENSE_DIR +TMP_VENDOR_LICENSE_DIR="${KUBE_TEMP}/vendor.LICENSES.DIR.$$" + +mkdir -p "${TMP_VENDOR_LICENSE_DIR}" + DEPS_DIR="vendor" declare -Ag CONTENT @@ -167,7 +179,8 @@ cat "${LICENSE_ROOT}/LICENSE" echo echo "= LICENSE $(kube::util::md5 "${LICENSE_ROOT}/LICENSE")" echo "================================================================================" -) > ${TMP_LICENSE_FILE} +) > "${TMP_LICENSE_FILE}" +mv "${TMP_LICENSE_FILE}" "${TMP_VENDOR_LICENSE_DIR}/LICENSE" # Loop through every vendored package for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do @@ -184,11 +197,10 @@ for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do process_content "${PACKAGE}" COPYRIGHT process_content "${PACKAGE}" COPYING - # display content - echo - echo "================================================================================" - echo "= ${DEPS_DIR}/${PACKAGE} licensed under: =" - echo + # copy content and throw error message + { + echo "= ${DEPS_DIR}/${PACKAGE} licensed under: =" + echo file="" if [[ -n "${CONTENT[${PACKAGE}-LICENSE]-}" ]]; then @@ -211,12 +223,20 @@ Options: __EOF__ exit 9 fi - cat "${file}" + + cat "${file}" + echo + echo "= ${file} $(kube::util::md5 "${file}")" + } >> "${TMP_LICENSE_FILE}" - echo - echo "= ${file} $(kube::util::md5 "${file}")" - echo "================================================================================" - echo -done >> ${TMP_LICENSE_FILE} -cat ${TMP_LICENSE_FILE} > ${VENDOR_LICENSE_FILE} + dest_dir="${TMP_VENDOR_LICENSE_DIR}/${PACKAGE}" + mkdir -p "${dest_dir}" + mv "${TMP_LICENSE_FILE}" "${dest_dir}/LICENSE" +done + +if [[ -d "${VENDOR_LICENSE_DIR}" ]]; then + rm -r "${VENDOR_LICENSE_DIR}" +fi +mv "${TMP_VENDOR_LICENSE_DIR}" "${VENDOR_LICENSE_DIR}" + diff --git a/hack/update-vendor.sh b/hack/update-vendor.sh index 0c8dd87bb01..268b23372ca 100755 --- a/hack/update-vendor.sh +++ b/hack/update-vendor.sh @@ -340,11 +340,10 @@ kube::log::status "vendor: updating LICENSES file" hack/update-vendor-licenses.sh >>"${LOG_FILE}" 2>&1 kube::log::status "vendor: creating OWNERS file" -rm -f "Godeps/OWNERS" "vendor/OWNERS" -cat <<__EOF__ > "Godeps/OWNERS" +rm -f "vendor/OWNERS" +cat <<__EOF__ > "vendor/OWNERS" # See the OWNERS docs at https://go.k8s.io/owners approvers: - dep-approvers __EOF__ -cp "Godeps/OWNERS" "vendor/OWNERS" diff --git a/hack/verify-vendor-licenses.sh b/hack/verify-vendor-licenses.sh index 3f0ee540d1f..00ff556e398 100755 --- a/hack/verify-vendor-licenses.sh +++ b/hack/verify-vendor-licenses.sh @@ -33,16 +33,25 @@ function cleanup { } kube::util::trap_add cleanup EXIT -cp -r "${KUBE_ROOT}/Godeps" "${_tmpdir}/Godeps" +#ln -s all subfolder except ALL_LICENSES in vendor +mkdir -p "${_tmpdir}/vendor" +for child in "${KUBE_ROOT}/vendor"/* +do + if [[ "${child}" == "${KUBE_ROOT}/vendor/ALL_LICENSES" ]] ; then + continue + else + ln -s "${child}" "${_tmpdir}/vendor" + fi +done + ln -s "${KUBE_ROOT}/LICENSE" "${_tmpdir}" -ln -s "${KUBE_ROOT}/vendor" "${_tmpdir}" ln -s "${KUBE_ROOT}/staging" "${_tmpdir}" # Update vendor Licenses LICENSE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-vendor-licenses.sh" # Compare vendor Licenses -if ! _out="$(diff -Naupr "${KUBE_ROOT}/Godeps/LICENSES" "${_tmpdir}/Godeps/LICENSES")"; then +if ! _out="$(diff -Naupr "${KUBE_ROOT}/vendor/ALL_LICENSES" "${_tmpdir}/vendor/ALL_LICENSES")"; then echo "Your vendor licenses file is out of date. Run hack/update-vendor-licenses.sh and commit the results." >&2 echo "${_out}" >&2 exit 1 diff --git a/hack/verify-vendor.sh b/hack/verify-vendor.sh index b3875452ec8..ea3ea01b5bc 100755 --- a/hack/verify-vendor.sh +++ b/hack/verify-vendor.sh @@ -55,7 +55,7 @@ export GO111MODULE=on pushd "${_kubetmp}" > /dev/null 2>&1 # Destroy deps in the copy of the kube tree - rm -rf ./Godeps/LICENSES ./vendor + rm -rf ./vendor # Recreate the vendor tree using the nice clean set we just downloaded hack/update-vendor.sh diff --git a/staging/publishing/rules-godeps.yaml b/staging/publishing/rules-godeps.yaml index d24362636af..9c9d4d799a6 100644 --- a/staging/publishing/rules-godeps.yaml +++ b/staging/publishing/rules-godeps.yaml @@ -121,7 +121,7 @@ rules: go build . # re-create vendor/ and try again godep restore - rm -rf vendor/ Godeps/ + rm -rf vendor/ godep save ./... go build . - destination: sample-controller @@ -150,7 +150,7 @@ rules: # re-create vendor/ and try again godep restore - rm -rf vendor/ Godeps/ + rm -rf vendor/ godep save ./... go build . - destination: apiextensions-apiserver