Merge pull request #22681 from ixdy/verify-all

Refactor hack/verify-all.sh and run almost all checks

Makefile

@@ -55,17 +55,7 @@ all:
 #   make verify
 #   make verify BRANCH=branch_x
 verify:
-	hack/verify-gofmt.sh
-	hack/verify-boilerplate.sh
-	hack/verify-codecgen.sh
-	hack/verify-description.sh
-	hack/verify-generated-conversions.sh
-	hack/verify-generated-deep-copies.sh
-	hack/verify-generated-docs.sh
-	hack/verify-swagger-spec.sh
-	hack/verify-flags-underscore.py
-	hack/verify-godeps.sh $(BRANCH)
-	hack/verify-godep-licenses.sh $(BRANCH)
+	KUBE_VERIFY_GIT_BRANCH=$(BRANCH) hack/verify-all.sh -v
 .PHONY: verify
 
 # Build and run tests.

hack/jenkins/gotest-dockerized.sh

@@ -21,9 +21,6 @@ set -o xtrace
 
 export REPO_DIR=${REPO_DIR:-$(pwd)}
 
-# Produce a JUnit-style XML test report for Jenkins.
-export KUBE_JUNIT_REPORT_DIR=${WORKSPACE}/_artifacts
-
 # Run the kubekins container, mapping in docker (so we can launch containers),
 # the repo directory, and the artifacts output directory.
 #
@@ -40,8 +37,9 @@ docker run --rm=true \
   -v /var/run/docker.sock:/var/run/docker.sock \
   -v "$(which docker)":/bin/docker \
   -v "${REPO_DIR}":/go/src/k8s.io/kubernetes \
-  -v "${KUBE_JUNIT_REPORT_DIR}":/workspace/artifacts \
+  -v "${WORKSPACE}/_artifacts":/workspace/artifacts \
   -v /etc/localtime:/etc/localtime:ro \
-  --env REPO_DIR="${REPO_DIR}" \
+  -e "KUBE_VERIFY_GIT_BRANCH=${KUBE_VERIFY_GIT_BRANCH:-}" \
+  -e "REPO_DIR=${REPO_DIR}" \
   -i gcr.io/google_containers/kubekins-test:0.7 \
   bash -c "cd kubernetes && ./hack/jenkins/test-dockerized.sh"

hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml

@@ -9,6 +9,8 @@
     builders:
         - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")'
         - shell: |
+            export KUBE_FORCE_VERIFY_CHECKS='y'
+            export KUBE_VERIFY_GIT_BRANCH='{branch}'
             timeout -k {kill-timeout}m {timeout}m ./hack/jenkins/gotest-dockerized.sh && rc=$? || rc=$?
             {report-rc}
     publishers:

hack/lib/util.sh

@@ -312,4 +312,33 @@ kube::util::gv-to-swagger-name() {
   esac
 }
 
+# Returns the name of the upstream remote repository name for the local git
+# repo, e.g. "upstream" or "origin".
+kube::util::git_upstream_remote_name() {
+  git remote -v | grep fetch |\
+    grep -E 'github.com/kubernetes/kubernetes|k8s.io/kubernetes' |\
+    head -n 1 | awk '{print $1}'
+}
+
+# Checks whether there are any files matching pattern $2 changed between the
+# current branch and upstream branch named by $1.
+# Returns 1 (false) if there are no changes, 0 (true) if there are changes
+# detected.
+kube::util::has_changes_against_upstream_branch() {
+  local -r git_branch=$1
+  local -r pattern=$2
+
+  readonly full_branch="$(kube::util::git_upstream_remote_name)/${git_branch}"
+  echo "Checking for '${pattern}' changes against '${full_branch}'"
+  # make sure the branch is valid, otherwise the check will pass erroneously.
+  if ! git describe "${full_branch}" >/dev/null; then
+    exit 1
+  fi
+  # notice this uses ... to find the first shared ancestor
+  if ! git diff --name-only "${full_branch}...HEAD" | grep "${pattern}" > /dev/null; then
+    echo "No '${pattern}' changes detected."
+    return 1
+  fi
+}
+
 # ex: ts=2 sw=2 et filetype=sh

hack/verify-all.sh

@@ -23,11 +23,17 @@ source "${KUBE_ROOT}/cluster/lib/util.sh"
 
 SILENT=true
 
+# Excluded checks are always skipped.
+EXCLUDED_CHECKS=(
+  "verify-linkcheck.sh"  # runs in separate Jenkins job once per day due to high network usage
+  "verify-generated-protobuf.sh"  # TODO(smarterclayton) add when protobuf is part of direct generation
+  )
+
 function is-excluded {
-  for e in $EXCLUDE; do
-    if [[ $1 -ef ${BASH_SOURCE} ]]; then
-      return
-    fi
+  if [[ $1 -ef ${BASH_SOURCE} ]]; then
+    return
+  fi
+  for e in ${EXCLUDED_CHECKS[@]}; do
     if [[ $1 -ef "$KUBE_ROOT/hack/$e" ]]; then
       return
     fi
@@ -35,7 +41,7 @@ function is-excluded {
   return 1
 }
 
-function run-cmd() {
+function run-cmd {
   if ${SILENT}; then
     "$@" &> /dev/null
   else
@@ -43,55 +49,48 @@ function run-cmd() {
   fi
 }
 
+function run-checks {
+  local -r pattern=$1
+  local -r runner=$2
+
+  for t in $(ls ${pattern})
+  do
+    if is-excluded "${t}" ; then
+      echo "Skipping ${t}"
+      continue
+    fi
+    echo -e "Verifying ${t}"
+    local start=$(date +%s)
+    run-cmd "${runner}" "${t}" && tr=$? || tr=$?
+    local elapsed=$(($(date +%s) - ${start}))
+    if [[ ${tr} -eq 0 ]]; then
+      echo -e "${color_green}SUCCESS${color_norm}  ${t}\t${elapsed}s"
+    else
+      echo -e "${color_red}FAILED${color_norm}   ${t}\t${elapsed}s"
+      ret=1
+    fi
+  done
+}
+
 while getopts ":v" opt; do
-  case $opt in
+  case ${opt} in
     v)
       SILENT=false
       ;;
     \?)
-      echo "Invalid flag: -$OPTARG" >&2
+      echo "Invalid flag: -${OPTARG}" >&2
       exit 1
       ;;
   esac
 done
 
-if $SILENT ; then
+if ${SILENT} ; then
   echo "Running in the silent mode, run with -v if you want to see script logs."
 fi
 
-# remove protobuf until it is part of direct generation
-EXCLUDE="verify-godeps.sh verify-godep-licenses.sh verify-generated-protobuf.sh verify-linkcheck.sh"
-
 ret=0
-for t in `ls $KUBE_ROOT/hack/verify-*.sh`
-do
-  if is-excluded $t ; then
-    echo "Skipping $t"
-    continue
-  fi
-  echo -e "Verifying $t"
-  if run-cmd bash "$t"; then
-    echo -e "${color_green}SUCCESS${color_norm}"
-  else
-    echo -e "${color_red}FAILED${color_norm}"
-    ret=1
-  fi
-done
-
-for t in `ls $KUBE_ROOT/hack/verify-*.py`
-do
-  if is-excluded $t ; then
-    echo "Skipping $t"
-    continue
-  fi
-  echo -e "Verifying $t"
-  if run-cmd python "$t"; then
-    echo -e "${color_green}SUCCESS${color_norm}"
-  else
-    echo -e "${color_red}FAILED${color_norm}"
-    ret=1
-  fi
-done
-exit $ret
+run-checks "${KUBE_ROOT}/hack/verify-*.sh" bash
+run-checks "${KUBE_ROOT}/hack/verify-*.py" python
+exit ${ret}
 
 # ex: ts=2 sw=2 et filetype=sh

hack/verify-godep-licenses.sh

@@ -19,10 +19,11 @@ set -o nounset
 set -o pipefail
 
 KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/.." && pwd -P)"
+source "${KUBE_ROOT}/hack/lib/init.sh"
 
-branch="${1:-master}"
-# notice this uses ... to find the first shared ancestor
-if ! git diff origin/"${branch}"...HEAD | grep 'Godeps/' > /dev/null; then
+readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}}
+if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \
+  ! kube::util::has_changes_against_upstream_branch "${branch}" 'Godeps/'; then
   exit 0
 fi
 

hack/verify-godeps.sh

@@ -39,9 +39,9 @@ preload-dep() {
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-branch="${1:-master}"
-# notice this uses ... to find the first shared ancestor
-if ! git diff origin/"${branch}"...HEAD | grep 'Godeps/' > /dev/null; then
+readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}}
+if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \
+  ! kube::util::has_changes_against_upstream_branch "${branch}" 'Godeps/'; then
   exit 0
 fi
 

shippable.yml

@@ -44,18 +44,7 @@ install:
   - ./hack/build-go.sh
   - godep go install ./...
   - ./hack/install-etcd.sh
-  - ./hack/verify-gofmt.sh
-  - ./hack/verify-boilerplate.sh
-  - ./hack/verify-description.sh
-  - ./hack/verify-flags-underscore.py
-  - ./hack/verify-godeps.sh ${BASE_BRANCH}
-  - ./hack/verify-godep-licenses.sh ${BASE_BRANCH}
-  - ./hack/travis/install-std-race.sh
-  - ./hack/verify-generated-conversions.sh
-  - ./hack/verify-generated-deep-copies.sh
-  - ./hack/verify-generated-docs.sh
-  - ./hack/verify-generated-swagger-docs.sh
-  - ./hack/verify-swagger-spec.sh
+  - make verify BRANCH=${BASE_BRANCH}
 
 script:
   # Disable coverage collection on pull requests