Restructure licenses again (revert cd4474a)

This moves licenses of vendored code from one monolith file into a tree
of individual files for easier reviews.  This fixes both the bash and
bazel paths.

Godeps/OWNERS

@@ -1,4 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-approvers:
-- dep-approvers

build/lib/release.sh

@@ -203,7 +203,7 @@ function kube::release::package_node_tarballs() {
     cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
       "${release_stage}/node/bin/"
 
-    cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
+    cp -R "${KUBE_ROOT}/LICENSES" "${release_stage}/"
 
     cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/"
 
@@ -275,7 +275,7 @@ function kube::release::package_server_tarballs() {
     cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
       "${release_stage}/server/bin/"
 
-    cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
+    cp -R "${KUBE_ROOT}/LICENSES" "${release_stage}/"
 
     cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/"
 
@@ -559,7 +559,7 @@ EOF
 
   cp -R "${KUBE_ROOT}/docs" "${release_stage}/"
   cp "${KUBE_ROOT}/README.md" "${release_stage}/"
-  cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
+  cp -R "${KUBE_ROOT}/LICENSES" "${release_stage}/"
 
   echo "${KUBE_GIT_VERSION}" > "${release_stage}/version"
 

build/release-tars/BUILD

@@ -70,13 +70,20 @@ pkg_tar(
 )
 
 # Included in node and server tarballs.
-filegroup(
+pkg_tar(
-    name = "license-targets",
+    name = "license-artifacts",
     srcs = [
-        ":kubernetes-src.tar.gz",
+        ":kubernetes-src.tar.gz",  # we want the tar file itself
-        "//:Godeps/LICENSES",
+    ],
+    mode = "0755",
+    tags = [
+        "manual",
+        "no-cache",
     ],
     visibility = ["//visibility:private"],
+    deps = [
+        "//:LICENSES",  # we want to expand this
+    ],
 )
 
 pkg_tar(
@@ -119,7 +126,7 @@ pkg_tar(
 
 [[pkg_tar(
     name = "kubernetes-node-%s-%s" % (os, arch),
-    srcs = [":license-targets"],
+    srcs = [],
     extension = "tar.gz",
     mode = "0644",
     package_dir = "kubernetes",
@@ -127,7 +134,9 @@ pkg_tar(
         "manual",
         "no-cache",
     ],
-    deps = select({go_platform_constraint(os, arch): [":_node-bin"]}),
+    deps = select({go_platform_constraint(os, arch): [":_node-bin"]}) + [
+        ":license-artifacts",
+    ],
 ) for arch in archs] for os, archs in NODE_PLATFORMS.items()]
 
 pkg_tar(
@@ -170,7 +179,7 @@ pkg_tar(
 
 [[pkg_tar(
     name = "kubernetes-server-%s-%s" % (os, arch),
-    srcs = [":license-targets"],
+    srcs = [],
     extension = "tar.gz",
     mode = "0644",
     package_dir = "kubernetes",
@@ -183,7 +192,9 @@ pkg_tar(
             ":_server-addons",
             ":_server-bin",
         ],
-    }),
+    }) + [
+        ":license-artifacts",
+    ],
 ) for arch in archs] for os, archs in SERVER_PLATFORMS.items()]
 
 pkg_tar(
@@ -235,7 +246,6 @@ pkg_tar(
 pkg_tar(
     name = "kubernetes",
     srcs = [
-        "//:Godeps/LICENSES",
         "//:README.md",
         "//:version",
         "//cluster:all-srcs",
@@ -253,6 +263,7 @@ pkg_tar(
     tags = ["no-cache"],
     deps = [
         ":_full_server",
+        ":license-artifacts",
     ],
 )
 

build/root/BUILD.root

@@ -17,6 +17,7 @@
 package(default_visibility = ["//visibility:public"])
 
 load("@io_k8s_repo_infra//defs:build.bzl", "gcs_upload")
+load("@io_k8s_repo_infra//defs:pkg.bzl", "pkg_tar")
 load("//build:platforms.bzl", "for_platforms")
 
 filegroup(
@@ -124,3 +125,18 @@ platform(
       {PARENT_REMOTE_EXECUTION_PROPERTIES}
     """,
 )
+
+pkg_tar(
+    name = "LICENSES",
+    srcs = glob(
+        ["LICENSES/**/LICENSE"],
+        allow_empty = False,
+        exclude_directories = 0,
+    ),
+    mode = "0755",
+    strip_prefix = "//",
+    tags = [
+        "manual",
+        "no-cache",
+    ],
+)

cluster/gce/gci/configure.sh

@@ -322,7 +322,7 @@ function install-exec-auth-plugin {
   local -r license_url="${EXEC_AUTH_PLUGIN_LICENSE_URL}"
   echo "Downloading gke-exec-auth-plugin license"
   download-or-bust "" "${license_url}"
-  mv "${KUBE_HOME}/LICENSE" "${KUBE_BIN}/gke-exec-auth-plugin-license"
+  mv "${KUBE_HOME}/LICENSES/LICENSE" "${KUBE_BIN}/gke-exec-auth-plugin-license"
 }
 
 function install-kube-manifests {

hack/tools/.gitignore

@@ -1 +1 @@
-vendor/
+/vendor/

hack/update-hack-tools.sh

@@ -38,6 +38,6 @@ pushd "${KUBE_ROOT}/hack/tools" >/dev/null
   go mod tidy
   go mod vendor
 
-  LICENSE_ROOT="${PWD}" VENDOR_LICENSE_FILE="LICENSES" "${KUBE_ROOT}/hack/update-vendor-licenses.sh"
+  LICENSE_ROOT="${PWD}" "${KUBE_ROOT}/hack/update-vendor-licenses.sh"
   rm -rf vendor
 popd >/dev/null

hack/update-vendor-licenses.sh

@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Update the Godeps/LICENSES document.
+# Update the LICENSES directory.
-# Generates a table of Godep dependencies and their license.
+# Generates a table of Go dependencies and their licenses.
 #
 # Usage:
 #    $0 [--create-missing] [/path/to/licenses]
@@ -24,12 +24,15 @@
 #    additionally created files into the vendor auto-generated tree.
 #
 #    Run every time a license file is added/modified within /vendor to
-#    update /Godeps/LICENSES
+#    update /LICENSES
 
 set -o errexit
 set -o nounset
 set -o pipefail
 
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
 export LANG=C
 export LC_ALL=C
 
@@ -130,8 +133,6 @@ process_content () {
 #############################################################################
 # MAIN
 #############################################################################
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
 
 export GO111MODULE=on
 
@@ -153,54 +154,66 @@ fi
 LICENSE_ROOT="${LICENSE_ROOT:-${KUBE_ROOT}}"
 cd "${LICENSE_ROOT}"
 
-VENDOR_LICENSE_FILE="${VENDOR_LICENSE_FILE:-"Godeps/LICENSES"}"
+kube::util::ensure-temp-dir
-TMP_LICENSE_FILE="/tmp/Godeps.LICENSES.$$"
+
+# Save the genreated LICENSE file for each package temporarily
+TMP_LICENSE_FILE="${KUBE_TEMP}/LICENSES.$$"
+
+# The directory to save all the LICENSE files
+LICENSES_DIR="${LICENSES_DIR:-${LICENSE_ROOT}/LICENSES}"
+mkdir -p "${LICENSES_DIR}"
+
+# The tmp directory to save all the LICENSE files, will move to LICENSES_DIR
+TMP_LICENSES_DIR="${KUBE_TEMP}/LICENSES.DIR.$$"
+mkdir -p "${TMP_LICENSES_DIR}"
+
 DEPS_DIR="vendor"
 declare -Ag CONTENT
 
-if [[ -f "${LICENSE_ROOT}/LICENSE" ]]; then
+# Put the K8S LICENSE on top
-  # Put the K8S LICENSE on top
+if [ -f "${LICENSE_ROOT}/LICENSE" ]; then
   (
-  echo "================================================================================"
+    echo "================================================================================"
-  echo "= Kubernetes licensed under: ="
+    echo "= Kubernetes licensed under: ="
-  echo
+    echo
-  cat "${LICENSE_ROOT}/LICENSE"
+    cat "${LICENSE_ROOT}/LICENSE"
-  echo
+    echo
-  echo "= LICENSE $(kube::util::md5 "${LICENSE_ROOT}/LICENSE")"
+    echo "= LICENSE $(kube::util::md5 "${LICENSE_ROOT}/LICENSE")"
-  echo "================================================================================"
+    echo "================================================================================"
-  ) > ${TMP_LICENSE_FILE}
+  ) > "${TMP_LICENSE_FILE}"
+  mv "${TMP_LICENSE_FILE}" "${TMP_LICENSES_DIR}/LICENSE"
 fi
 
 # Loop through every vendored package
 for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do
   if [[ -e "staging/src/${PACKAGE}" ]]; then
-    echo "$PACKAGE is a staging package, skipping" > /dev/stderr
+    echo "${PACKAGE} is a staging package, skipping" > /dev/stderr
     continue
   fi
   if [[ ! -e "${DEPS_DIR}/${PACKAGE}" ]]; then
-    echo "$PACKAGE doesn't exist in vendor, skipping" > /dev/stderr
+    echo "${PACKAGE} doesn't exist in ${DEPS_DIR}, skipping" > /dev/stderr
     continue
   fi
+  echo "${PACKAGE}"
 
   process_content "${PACKAGE}" LICENSE
   process_content "${PACKAGE}" COPYRIGHT
   process_content "${PACKAGE}" COPYING
 
-  # display content
+  # copy content and throw error message
-  echo
+  {
-  echo "================================================================================"
+    echo "= ${DEPS_DIR}/${PACKAGE} licensed under: ="
-  echo "= ${DEPS_DIR}/${PACKAGE} licensed under: ="
+    echo
-  echo
 
-  file=""
+    file=""
-  if [[ -n "${CONTENT[${PACKAGE}-LICENSE]-}" ]]; then
+    if [[ -n "${CONTENT[${PACKAGE}-LICENSE]-}" ]]; then
       file="${CONTENT[${PACKAGE}-LICENSE]-}"
-  elif [[ -n "${CONTENT[${PACKAGE}-COPYRIGHT]-}" ]]; then
+    elif [[ -n "${CONTENT[${PACKAGE}-COPYRIGHT]-}" ]]; then
       file="${CONTENT[${PACKAGE}-COPYRIGHT]-}"
-  elif [[ -n "${CONTENT[${PACKAGE}-COPYING]-}" ]]; then
+    elif [[ -n "${CONTENT[${PACKAGE}-COPYING]-}" ]]; then
       file="${CONTENT[${PACKAGE}-COPYING]-}"
-  fi
+    fi
-  if [[ -z "${file}" ]]; then
+    if [[ -z "${file}" ]]; then
       cat > /dev/stderr << __EOF__
 No license could be found for ${PACKAGE} - aborting.
 
@@ -212,13 +225,19 @@ Options:
 3. Do not use this package in Kubernetes.
 __EOF__
       exit 9
-  fi
+    fi
-  cat "${file}"
 
-  echo
+    cat "${file}"
-  echo "= ${file} $(kube::util::md5 "${file}")"
+    echo
-  echo "================================================================================"
+    echo "= ${file} $(kube::util::md5 "${file}")"
-  echo
+  } >> "${TMP_LICENSE_FILE}"
-done >> ${TMP_LICENSE_FILE}
 
-cat ${TMP_LICENSE_FILE} > "${VENDOR_LICENSE_FILE}"
+  dest_dir="${TMP_LICENSES_DIR}/vendor/${PACKAGE}"
+  mkdir -p "${dest_dir}"
+  mv "${TMP_LICENSE_FILE}" "${dest_dir}/LICENSE"
+done
+
+# Leave things like OWNERS alone.
+rm -f "${LICENSES_DIR}/LICENSE"
+rm -rf "${LICENSES_DIR}/vendor"
+mv "${TMP_LICENSES_DIR}"/* "${LICENSES_DIR}"

hack/update-vendor.sh

@@ -319,8 +319,7 @@ for repo in $(kube::util::list_staging_repos); do
 done
 
 
-# Phase 6: rebuild vendor directory
+# Phase 7: rebuild vendor directory
-
 kube::log::status "vendor: running 'go mod vendor'"
 go mod vendor >>"${LOG_FILE}" 2>&1
 
@@ -341,18 +340,19 @@ kube::log::status "vendor: updating BUILD files"
 # Assume that anything imported through vendor doesn't need Bazel to build.
 # Prune out any Bazel build files, since these can break the build due to
 # missing dependencies that aren't included by go mod vendor.
-find vendor/ -type f \( -name BUILD -o -name BUILD.bazel -o -name WORKSPACE \) -exec rm -f {} \;
+find vendor/ -type f \
+    \( -name BUILD -o -name BUILD.bazel -o -name WORKSPACE \) \
+    -exec rm -f {} \;
 hack/update-bazel.sh >>"${LOG_FILE}" 2>&1
 
-kube::log::status "vendor: updating LICENSES file"
+kube::log::status "vendor: updating vendor/LICENSES"
 hack/update-vendor-licenses.sh >>"${LOG_FILE}" 2>&1
 
 kube::log::status "vendor: creating OWNERS file"
-rm -f "Godeps/OWNERS" "vendor/OWNERS"
+rm -f "vendor/OWNERS"
-cat <<__EOF__ > "Godeps/OWNERS"
+cat <<__EOF__ > "vendor/OWNERS"
 # See the OWNERS docs at https://go.k8s.io/owners
 
 approvers:
 - dep-approvers
 __EOF__
-cp "Godeps/OWNERS" "vendor/OWNERS"

hack/verify-vendor-licenses.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script checks whether updating of vendor licenses file is needed
+# This script checks whether updating of licenses files is needed
 # or not. We should run `hack/update-vendor-licenses.sh` and commit the results,
 # if actually updates them.
 # Usage: `hack/verify-vendor-licenses.sh`.
@@ -30,7 +30,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 # must be in the user dir (e.g. KUBE_ROOT) in order for the docker volume mount
 # to work with docker-machine on macs
 mkdir -p "${KUBE_ROOT}/_tmp"
-_tmpdir="$(mktemp -d "${KUBE_ROOT}/_tmp/kube-vendor-licenses.XXXXXX")"
+_tmpdir="$(mktemp -d "${KUBE_ROOT}/_tmp/kube-licenses.XXXXXX")"
 #echo "Created workspace: ${_tmpdir}"
 function cleanup {
   #echo "Removing workspace: ${_tmpdir}"
@@ -38,17 +38,22 @@ function cleanup {
 }
 kube::util::trap_add cleanup EXIT
 
-cp -r "${KUBE_ROOT}/Godeps" "${_tmpdir}/Godeps"
+# symlink all vendor subfolders in temp vendor
+mkdir -p "${_tmpdir}/vendor"
+for child in "${KUBE_ROOT}/vendor"/*
+do
+  ln -s "${child}" "${_tmpdir}/vendor"
+done
+
 ln -s "${KUBE_ROOT}/LICENSE" "${_tmpdir}"
-ln -s "${KUBE_ROOT}/vendor" "${_tmpdir}"
 ln -s "${KUBE_ROOT}/staging" "${_tmpdir}"
 
-# Update vendor Licenses
+# Update licenses
 LICENSE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-vendor-licenses.sh"
 
-# Compare vendor Licenses
+# Compare licenses
-if ! _out="$(diff -Naupr "${KUBE_ROOT}/Godeps/LICENSES" "${_tmpdir}/Godeps/LICENSES")"; then
+if ! _out="$(diff -Naupr -x OWNERS "${KUBE_ROOT}/LICENSES" "${_tmpdir}/LICENSES")"; then
-  echo "Your vendor licenses file is out of date. Run hack/update-vendor-licenses.sh and commit the results." >&2
+  echo "Your LICENSES tree is out of date. Run hack/update-vendor-licenses.sh and commit the results." >&2
   echo "${_out}" >&2
   exit 1
 fi

hack/verify-vendor.sh

@@ -25,7 +25,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-# create a nice clean place to put our new licenses
+# create a nice clean place to put our new vendor tree
 # must be in the user dir (e.g. KUBE_ROOT) in order for the docker volume mount
 # to work with docker-machine on macs
 mkdir -p "${KUBE_ROOT}/_tmp"
@@ -59,7 +59,7 @@ export GO111MODULE=on
 
 pushd "${_kubetmp}" > /dev/null 2>&1
   # Destroy deps in the copy of the kube tree
-  rm -rf ./Godeps/LICENSES ./vendor
+  rm -rf ./vendor ./LICENSES
 
   # Recreate the vendor tree using the nice clean set we just downloaded
   hack/update-vendor.sh