Merge pull request #122250 from thockin/fix_verify-licenses

Fix verify-licenses.sh

hack/verify-licenses.sh

@@ -26,26 +26,25 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 source "${KUBE_ROOT}/hack/lib/util.sh"
 
-
 # This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
 kube::golang::setup_env
 kube::util::ensure-temp-dir
 
-
 # Creating a new repository tree 
 # Deleting vendor directory to make go-licenses fetch license URLs from go-packages source repository
 git worktree add -f "${KUBE_TEMP}"/tmp_test_licenses/kubernetes HEAD >/dev/null 2>&1 || true
 cd "${KUBE_TEMP}"/tmp_test_licenses/kubernetes && rm -rf vendor
 
-
 # Ensure that we find the binaries we build before anything else.
 export GOBIN="${KUBE_OUTPUT_BINPATH}"
 PATH="${GOBIN}:${PATH}"
 
-
 # Explicitly opt into go modules, even though we're inside a GOPATH directory
 export GO111MODULE=on
 
+function http_code() {
+    curl -I -s -o /dev/null -w "%{http_code}" "$1"
+}
 
 allowed_licenses=()
 packages_flagged=()
@@ -60,77 +59,59 @@ go install github.com/google/go-licenses@latest
 # Refer: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md
 curl -s 'https://spdx.org/licenses/licenses.json' -o "${KUBE_TEMP}"/licenses.json
 
-number_of_licenses=$(jq '.licenses | length' "${KUBE_TEMP}"/licenses.json)
-loop_index_length=$(( number_of_licenses - 1 ))
-
-
 echo '[INFO] Fetching current list of CNCF approved licenses...'
-for index in $(seq 0 $loop_index_length);
+while read -r L; do
-do
+    allowed_licenses+=("${L}")
-	licenseID=$(jq ".licenses[$index] .licenseId" "${KUBE_TEMP}"/licenses.json)
+done < <(jq -r '.licenses[] | select(.isDeprecatedLicenseId==false) .licenseId' "${KUBE_TEMP}"/licenses.json)
-	if [[ $(jq ".licenses[$index] .isDeprecatedLicenseId" "${KUBE_TEMP}"/licenses.json) == false ]]
-	then
-		allowed_licenses+=("${licenseID}")
-        fi	
-done
-
 
 # Scanning go-packages under the project & verifying against the CNCF approved list of licenses
 echo '[INFO] Starting license scan on go-packages...'
 go-licenses report ./... >> "${KUBE_TEMP}"/licenses.csv
 
 echo -e 'PACKAGE_NAME  LICENSE_NAME  LICENSE_URL\n' >> "${KUBE_TEMP}"/approved_licenses.dump
-while IFS=, read -r GO_PACKAGE LICENSE_URL LICENSE_NAME
+while IFS=, read -r GO_PACKAGE LICENSE_URL LICENSE_NAME; do
-do
+    if ! printf -- "%s\n" "${allowed_licenses[@]}" | grep -q "^${LICENSE_NAME}$"; then
-	FORMATTED_LICENSE_URL=
+        echo "${GO_PACKAGE}  ${LICENSE_NAME}  ${LICENSE_URL}" >> "${KUBE_TEMP}"/notapproved_licenses.dump
-	if [[ " ${allowed_licenses[*]} " == *"${LICENSE_NAME}"* ]];
+        packages_flagged+=("${GO_PACKAGE}")
-	then
+        continue
-		if [[ "${LICENSE_URL}" == 'Unknown' ]];
+    fi
-		then
+
-			if  [[ "${GO_PACKAGE}" != k8s.io/* ]];
+    if [[ "${LICENSE_URL}" == 'Unknown' ]]; then
-			then
+        if  [[ "${GO_PACKAGE}" != k8s.io/* ]]; then
             echo "${GO_PACKAGE}  ${LICENSE_NAME}  ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump
             packages_url_missing+=("${GO_PACKAGE}")
         else
             LICENSE_URL='https://github.com/kubernetes/kubernetes/blob/master/LICENSE'
             echo "${GO_PACKAGE}  ${LICENSE_NAME}  ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump
         fi
-		elif curl -Is "${LICENSE_URL}" | head -1 | grep -q 404;
-		then
-			# Check whether the License URL is incorrectly formed
-			# TODO: Remove this workaround check once PR https://github.com/google/go-licenses/pull/110 is merged
-			IFS='/' read -r -a split_license_url <<< ${LICENSE_URL}
-			for part_of_url in "${split_license_url[@]}"
-			do
-				if  [[ ${part_of_url} == '' ]]
-				then
         continue
-				elif	[[ ${part_of_url} == 'https:' ]]
+    fi
-				then
+
-					FORMATTED_LICENSE_URL+='https://'
+    if [[ "$(http_code "${LICENSE_URL}")" != 404 ]]; then
-				else
+        echo "${GO_PACKAGE}  ${LICENSE_NAME}  ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump
-					if [[ ${part_of_url} =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]
+        continue
-					then
+    fi
-						FORMATTED_LICENSE_URL+="${part_of_url}/${split_license_url[-1]}"
+
+    # The URL 404'ed.  Try parent-paths.
+
+    #echo -e "DBG: err 404 ${LICENSE_URL}"
+    dir="$(dirname "${LICENSE_URL}")"
+    file="$(basename "${LICENSE_URL}")"
+
+    while [[ "${dir}" != "." ]]; do
+        dir="$(dirname "${dir}")"
+        #echo "DBG:     try ${dir}/${file}"
+        if [[ "$(http_code "${dir}/${file}")" != 404 ]]; then
+            #echo "DBG:         it worked"
+            echo "${GO_PACKAGE}  ${LICENSE_NAME}  ${dir}/${file}" >> "${KUBE_TEMP}"/approved_licenses.dump
             break
-					else
-						FORMATTED_LICENSE_URL+="${part_of_url}/"
-					fi
         fi
+        #echo "DBG:         still 404"
     done
-			if curl -Is "${FORMATTED_LICENSE_URL}" | head -1 | grep -q 404;
+    if [[ "${dir}" == "." ]];then
-			then
+        #echo "DBG:     failed to find a license"
         packages_url_missing+=("${GO_PACKAGE}")
         echo "${GO_PACKAGE}  ${LICENSE_NAME}  ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump
-			else
-				echo "${GO_PACKAGE}  ${LICENSE_NAME}  ${FORMATTED_LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump
-			fi
-		else
-			echo "${GO_PACKAGE}  ${LICENSE_NAME}  ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump
-		fi
-	else
-		echo "${GO_PACKAGE}  ${LICENSE_NAME}  ${LICENSE_URL}" >> "${KUBE_TEMP}"/notapproved_licenses.dump
-		packages_flagged+=("${GO_PACKAGE}")
     fi
 done < "${KUBE_TEMP}"/licenses.csv
 awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/approved_licenses.dump