From 84759deac5bde20ae2c88bd35a775935cb0c7069 Mon Sep 17 00:00:00 2001 From: Priyanka Saggu Date: Fri, 8 Apr 2022 00:44:16 +0530 Subject: [PATCH 1/3] add verify-licenses.sh hack script (kubernetes#108942) --- hack/verify-licenses.sh | 131 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100755 hack/verify-licenses.sh diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh new file mode 100755 index 00000000000..61e9d900713 --- /dev/null +++ b/hack/verify-licenses.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: `hack/verify-licenses.sh`. + + +set -o errexit +set -o nounset +set -o pipefail + + +KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +source "${KUBE_ROOT}/hack/lib/init.sh" +source "${KUBE_ROOT}/hack/lib/util.sh" + + +kube::golang::verify_go_version +kube::util::ensure-temp-dir + + +# Ensure that we find the binaries we build before anything else. +export GOBIN="${KUBE_OUTPUT_BINPATH}" +PATH="${GOBIN}:${PATH}" + + +# Explicitly opt into go modules, even though we're inside a GOPATH directory +export GO111MODULE=on + + +allowed_licenses=() +packages_flagged=() +packages_url_missing=() +exit_code=0 + + +git remote add licenses https://github.com/kubernetes/kubernetes >/dev/null 2>&1 || true + + +# Install go-licenses +echo -e "[INFO] Installing go-licenses..." +pushd "${KUBE_TEMP}" >/dev/null + git clone https://github.com/google/go-licenses.git >/dev/null 2>&1 + cd go-licenses + go build -o "${GOPATH}/bin" +popd >/dev/null + + +# Fetching CNCF Approved List Of Licenses +# Refer: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md +curl -s 'https://spdx.org/licenses/licenses.json' > "${KUBE_TEMP}"/licenses.json + + +number_of_licenses=$(jq '.licenses | length' "${KUBE_TEMP}"/licenses.json) +loop_index_length=$(( number_of_licenses - 1 )) + + +echo -e "[INFO] Fetching current list of CNCF approved licenses..." +for index in $(seq 0 $loop_index_length); +do + licenseID=$(jq ".licenses[$index] .licenseId" "${KUBE_TEMP}"/licenses.json) + if [[ $(jq ".licenses[$index] .isDeprecatedLicenseId" "${KUBE_TEMP}"/licenses.json) == false ]] + then + allowed_licenses+=("${licenseID}") + fi +done + + +# Scanning go-packages Under The Project & Verifying Against The CNCF Approved List Of Licenses +echo -e "[INFO] Starting license scan on go-packages..." +go-licenses csv --git_remote "licenses" ./... >> "${KUBE_TEMP}"/licenses.csv 2>/dev/null + + +echo -e "PACKAGE_NAME LICENSE_NAME LICENSE_URL\n" >> "${KUBE_TEMP}"/approved_licenses.dump +while IFS=, read -r GO_PACKAGE LICENSE_URL LICENSE_NAME +do + if [[ " ${allowed_licenses[*]} " == *"${LICENSE_NAME}"* ]]; + then + if [[ "${LICENSE_URL}" == "Unknown" ]]; + then + if [[ "${GO_PACKAGE}" != k8s.io/* ]]; + then + echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump + packages_url_missing+=("${GO_PACKAGE}") + else + LICENSE_URL="https://github.com/kubernetes/kubernetes/blob/master/LICENSE" + echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump + fi + else + echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump + fi + else + echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"notapproved_licenses.dump + packages_flagged+=("${GO_PACKAGE}") + fi +done < "${KUBE_TEMP}"/licenses.csv +awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/approved_licenses.dump + + +# cleanup +git remote remove licenses + + +if [[ ${#packages_url_missing[@]} -gt 0 ]]; then + kube::log::error "[ERROR] The following go-packages in the project have missing or unknown license URL.\n\n" + awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump + exit_code=1 +fi + + +if [[ ${#packages_flagged[@]} -gt 0 ]]; then + kube::log::error "[ERROR] The following go-packages in the project are using non-CNCF approved licenses. Please check the CNCF allowed list of licenses for more details here: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md\n\n" + awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/notapproved_licenses.dump + exit_code=1 +else + kube::log::status "[SUCCESS] Scan complete! All go-packages under the project are using current CNCF approved licenses!" +fi + +exit ${exit_code} From 0e92873ee53c23c84b37abdc9bb1f0c9c2d70a0a Mon Sep 17 00:00:00 2001 From: Priyanka Saggu Date: Sun, 10 Apr 2022 09:56:58 +0530 Subject: [PATCH 2/3] update verify-licenses.sh to make it execute in a different git worktree --- hack/verify-licenses.sh | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 61e9d900713..8eb6e77390e 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -31,6 +31,12 @@ kube::golang::verify_go_version kube::util::ensure-temp-dir +# Creating a new repository tree +# Deleting vendor directory to make go-licenses fetch license URLs from go-packages source repository +git worktree add -f "${KUBE_TEMP}"/tmp_test_licenses/kubernetes HEAD >/dev/null 2>&1 || true +cd "${KUBE_TEMP}"/tmp_test_licenses/kubernetes && rm -rf vendor + + # Ensure that we find the binaries we build before anything else. export GOBIN="${KUBE_OUTPUT_BINPATH}" PATH="${GOBIN}:${PATH}" @@ -50,7 +56,7 @@ git remote add licenses https://github.com/kubernetes/kubernetes >/dev/null 2>&1 # Install go-licenses -echo -e "[INFO] Installing go-licenses..." +echo -e '[INFO] Installing go-licenses...' pushd "${KUBE_TEMP}" >/dev/null git clone https://github.com/google/go-licenses.git >/dev/null 2>&1 cd go-licenses @@ -60,14 +66,14 @@ popd >/dev/null # Fetching CNCF Approved List Of Licenses # Refer: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md -curl -s 'https://spdx.org/licenses/licenses.json' > "${KUBE_TEMP}"/licenses.json +curl -s 'https://spdx.org/licenses/licenses.json' -o "${KUBE_TEMP}"/licenses.json number_of_licenses=$(jq '.licenses | length' "${KUBE_TEMP}"/licenses.json) loop_index_length=$(( number_of_licenses - 1 )) -echo -e "[INFO] Fetching current list of CNCF approved licenses..." +echo -e '[INFO] Fetching current list of CNCF approved licenses...' for index in $(seq 0 $loop_index_length); do licenseID=$(jq ".licenses[$index] .licenseId" "${KUBE_TEMP}"/licenses.json) @@ -78,26 +84,30 @@ do done -# Scanning go-packages Under The Project & Verifying Against The CNCF Approved List Of Licenses -echo -e "[INFO] Starting license scan on go-packages..." +# Scanning go-packages under the project & verifying against the CNCF approved list of licenses +echo -e '[INFO] Starting license scan on go-packages...' go-licenses csv --git_remote "licenses" ./... >> "${KUBE_TEMP}"/licenses.csv 2>/dev/null -echo -e "PACKAGE_NAME LICENSE_NAME LICENSE_URL\n" >> "${KUBE_TEMP}"/approved_licenses.dump +echo -e 'PACKAGE_NAME LICENSE_NAME LICENSE_URL\n' >> "${KUBE_TEMP}"/approved_licenses.dump while IFS=, read -r GO_PACKAGE LICENSE_URL LICENSE_NAME do if [[ " ${allowed_licenses[*]} " == *"${LICENSE_NAME}"* ]]; then - if [[ "${LICENSE_URL}" == "Unknown" ]]; + if [[ "${LICENSE_URL}" == 'Unknown' ]]; then if [[ "${GO_PACKAGE}" != k8s.io/* ]]; then echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump packages_url_missing+=("${GO_PACKAGE}") else - LICENSE_URL="https://github.com/kubernetes/kubernetes/blob/master/LICENSE" + LICENSE_URL='https://github.com/kubernetes/kubernetes/blob/master/LICENSE' echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump fi + elif curl -Is "${LICENSE_URL}" | head -1 | grep -q 404; + then + packages_url_missing+=("${GO_PACKAGE}") + echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump else echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump fi @@ -114,16 +124,18 @@ git remote remove licenses if [[ ${#packages_url_missing[@]} -gt 0 ]]; then - kube::log::error "[ERROR] The following go-packages in the project have missing or unknown license URL.\n\n" + echo -e '\n[ERROR] The following go-packages in the project have unknown or unreachable license URL:' awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump exit_code=1 fi if [[ ${#packages_flagged[@]} -gt 0 ]]; then - kube::log::error "[ERROR] The following go-packages in the project are using non-CNCF approved licenses. Please check the CNCF allowed list of licenses for more details here: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md\n\n" + kube::log::error "[ERROR] The following go-packages in the project are using non-CNCF approved licenses. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md" awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/notapproved_licenses.dump exit_code=1 +elif [[ "${exit_code}" -eq 1 ]]; then + kube::log::status "[ERROR] Project is using go-packages with unknown or unreachable license URLs. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md " else kube::log::status "[SUCCESS] Scan complete! All go-packages under the project are using current CNCF approved licenses!" fi From 31a5e8a0273533d76b311678faba8a2405042636 Mon Sep 17 00:00:00 2001 From: Priyanka Saggu Date: Thu, 5 May 2022 22:30:37 +0530 Subject: [PATCH 3/3] add workaround fix for incorrect license URL --- hack/verify-licenses.sh | 52 ++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 8eb6e77390e..0f64a5fbad5 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -51,12 +51,11 @@ packages_flagged=() packages_url_missing=() exit_code=0 - git remote add licenses https://github.com/kubernetes/kubernetes >/dev/null 2>&1 || true # Install go-licenses -echo -e '[INFO] Installing go-licenses...' +echo '[INFO] Installing go-licenses...' pushd "${KUBE_TEMP}" >/dev/null git clone https://github.com/google/go-licenses.git >/dev/null 2>&1 cd go-licenses @@ -73,7 +72,7 @@ number_of_licenses=$(jq '.licenses | length' "${KUBE_TEMP}"/licenses.json) loop_index_length=$(( number_of_licenses - 1 )) -echo -e '[INFO] Fetching current list of CNCF approved licenses...' +echo '[INFO] Fetching current list of CNCF approved licenses...' for index in $(seq 0 $loop_index_length); do licenseID=$(jq ".licenses[$index] .licenseId" "${KUBE_TEMP}"/licenses.json) @@ -85,34 +84,61 @@ done # Scanning go-packages under the project & verifying against the CNCF approved list of licenses -echo -e '[INFO] Starting license scan on go-packages...' -go-licenses csv --git_remote "licenses" ./... >> "${KUBE_TEMP}"/licenses.csv 2>/dev/null +echo '[INFO] Starting license scan on go-packages...' +go-licenses csv --git_remote licenses ./... >> "${KUBE_TEMP}"/licenses.csv 2>/dev/null echo -e 'PACKAGE_NAME LICENSE_NAME LICENSE_URL\n' >> "${KUBE_TEMP}"/approved_licenses.dump while IFS=, read -r GO_PACKAGE LICENSE_URL LICENSE_NAME do + FORMATTED_LICENSE_URL= if [[ " ${allowed_licenses[*]} " == *"${LICENSE_NAME}"* ]]; then if [[ "${LICENSE_URL}" == 'Unknown' ]]; then if [[ "${GO_PACKAGE}" != k8s.io/* ]]; then - echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump + echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump packages_url_missing+=("${GO_PACKAGE}") else LICENSE_URL='https://github.com/kubernetes/kubernetes/blob/master/LICENSE' - echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump + echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump fi elif curl -Is "${LICENSE_URL}" | head -1 | grep -q 404; then - packages_url_missing+=("${GO_PACKAGE}") - echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump + # Check whether the License URL is incorrectly formed + # TODO: Remove this workaround check once PR https://github.com/google/go-licenses/pull/110 is merged + IFS='/' read -r -a split_license_url <<< ${LICENSE_URL} + for part_of_url in "${split_license_url[@]}" + do + if [[ ${part_of_url} == '' ]] + then + continue + elif [[ ${part_of_url} == 'https:' ]] + then + FORMATTED_LICENSE_URL+='https://' + else + if [[ ${part_of_url} =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]] + then + FORMATTED_LICENSE_URL+="${part_of_url}/${split_license_url[-1]}" + break + else + FORMATTED_LICENSE_URL+="${part_of_url}/" + fi + fi + done + if curl -Is "${FORMATTED_LICENSE_URL}" | head -1 | grep -q 404; + then + packages_url_missing+=("${GO_PACKAGE}") + echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump + else + echo "${GO_PACKAGE} ${LICENSE_NAME} ${FORMATTED_LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump + fi else - echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump + echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump fi else - echo -e "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"notapproved_licenses.dump + echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"notapproved_licenses.dump packages_flagged+=("${GO_PACKAGE}") fi done < "${KUBE_TEMP}"/licenses.csv @@ -135,9 +161,9 @@ if [[ ${#packages_flagged[@]} -gt 0 ]]; then awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/notapproved_licenses.dump exit_code=1 elif [[ "${exit_code}" -eq 1 ]]; then - kube::log::status "[ERROR] Project is using go-packages with unknown or unreachable license URLs. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md " + kube::log::status "[ERROR] Project is using go-packages with unknown or unreachable license URLs. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md" else kube::log::status "[SUCCESS] Scan complete! All go-packages under the project are using current CNCF approved licenses!" fi -exit ${exit_code} +exit "${exit_code}"