diff --git a/Makefile b/Makefile index d2ed676a092..7b056878855 100644 --- a/Makefile +++ b/Makefile @@ -55,17 +55,7 @@ all: # make verify # make verify BRANCH=branch_x verify: - hack/verify-gofmt.sh - hack/verify-boilerplate.sh - hack/verify-codecgen.sh - hack/verify-description.sh - hack/verify-generated-conversions.sh - hack/verify-generated-deep-copies.sh - hack/verify-generated-docs.sh - hack/verify-swagger-spec.sh - hack/verify-flags-underscore.py - hack/verify-godeps.sh $(BRANCH) - hack/verify-godep-licenses.sh $(BRANCH) + KUBE_VERIFY_GIT_BRANCH=$(BRANCH) hack/verify-all.sh -v .PHONY: verify # Build and run tests. diff --git a/hack/jenkins/gotest-dockerized.sh b/hack/jenkins/gotest-dockerized.sh index e7357420556..77eaf9034e0 100755 --- a/hack/jenkins/gotest-dockerized.sh +++ b/hack/jenkins/gotest-dockerized.sh @@ -21,9 +21,6 @@ set -o xtrace export REPO_DIR=${REPO_DIR:-$(pwd)} -# Produce a JUnit-style XML test report for Jenkins. -export KUBE_JUNIT_REPORT_DIR=${WORKSPACE}/_artifacts - # Run the kubekins container, mapping in docker (so we can launch containers), # the repo directory, and the artifacts output directory. # @@ -40,8 +37,9 @@ docker run --rm=true \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$(which docker)":/bin/docker \ -v "${REPO_DIR}":/go/src/k8s.io/kubernetes \ - -v "${KUBE_JUNIT_REPORT_DIR}":/workspace/artifacts \ + -v "${WORKSPACE}/_artifacts":/workspace/artifacts \ -v /etc/localtime:/etc/localtime:ro \ - --env REPO_DIR="${REPO_DIR}" \ + -e "KUBE_VERIFY_GIT_BRANCH=${KUBE_VERIFY_GIT_BRANCH:-}" \ + -e "REPO_DIR=${REPO_DIR}" \ -i gcr.io/google_containers/kubekins-test:0.7 \ bash -c "cd kubernetes && ./hack/jenkins/test-dockerized.sh" diff --git a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml index 9600e7989d9..434b1e38dcd 100644 --- a/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml +++ b/hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-test-go.yaml @@ -9,6 +9,8 @@ builders: - shell: 'bash <(curl -fsS --retry 3 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-started.sh")' - shell: | + export KUBE_FORCE_VERIFY_CHECKS='y' + export KUBE_VERIFY_GIT_BRANCH='{branch}' timeout -k {kill-timeout}m {timeout}m ./hack/jenkins/gotest-dockerized.sh && rc=$? || rc=$? {report-rc} publishers: diff --git a/hack/lib/util.sh b/hack/lib/util.sh index b42f4fae199..86051b6afa1 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -312,4 +312,33 @@ kube::util::gv-to-swagger-name() { esac } +# Returns the name of the upstream remote repository name for the local git +# repo, e.g. "upstream" or "origin". +kube::util::git_upstream_remote_name() { + git remote -v | grep fetch |\ + grep -E 'github.com/kubernetes/kubernetes|k8s.io/kubernetes' |\ + head -n 1 | awk '{print $1}' +} + +# Checks whether there are any files matching pattern $2 changed between the +# current branch and upstream branch named by $1. +# Returns 1 (false) if there are no changes, 0 (true) if there are changes +# detected. +kube::util::has_changes_against_upstream_branch() { + local -r git_branch=$1 + local -r pattern=$2 + + readonly full_branch="$(kube::util::git_upstream_remote_name)/${git_branch}" + echo "Checking for '${pattern}' changes against '${full_branch}'" + # make sure the branch is valid, otherwise the check will pass erroneously. + if ! git describe "${full_branch}" >/dev/null; then + exit 1 + fi + # notice this uses ... to find the first shared ancestor + if ! git diff --name-only "${full_branch}...HEAD" | grep "${pattern}" > /dev/null; then + echo "No '${pattern}' changes detected." + return 1 + fi +} + # ex: ts=2 sw=2 et filetype=sh diff --git a/hack/verify-all.sh b/hack/verify-all.sh index c5b32958bfa..936848151fc 100755 --- a/hack/verify-all.sh +++ b/hack/verify-all.sh @@ -23,11 +23,17 @@ source "${KUBE_ROOT}/cluster/lib/util.sh" SILENT=true +# Excluded checks are always skipped. +EXCLUDED_CHECKS=( + "verify-linkcheck.sh" # runs in separate Jenkins job once per day due to high network usage + "verify-generated-protobuf.sh" # TODO(smarterclayton) add when protobuf is part of direct generation + ) + function is-excluded { - for e in $EXCLUDE; do - if [[ $1 -ef ${BASH_SOURCE} ]]; then - return - fi + if [[ $1 -ef ${BASH_SOURCE} ]]; then + return + fi + for e in ${EXCLUDED_CHECKS[@]}; do if [[ $1 -ef "$KUBE_ROOT/hack/$e" ]]; then return fi @@ -35,7 +41,7 @@ function is-excluded { return 1 } -function run-cmd() { +function run-cmd { if ${SILENT}; then "$@" &> /dev/null else @@ -43,55 +49,48 @@ function run-cmd() { fi } +function run-checks { + local -r pattern=$1 + local -r runner=$2 + + for t in $(ls ${pattern}) + do + if is-excluded "${t}" ; then + echo "Skipping ${t}" + continue + fi + echo -e "Verifying ${t}" + local start=$(date +%s) + run-cmd "${runner}" "${t}" && tr=$? || tr=$? + local elapsed=$(($(date +%s) - ${start})) + if [[ ${tr} -eq 0 ]]; then + echo -e "${color_green}SUCCESS${color_norm} ${t}\t${elapsed}s" + else + echo -e "${color_red}FAILED${color_norm} ${t}\t${elapsed}s" + ret=1 + fi + done +} + while getopts ":v" opt; do - case $opt in + case ${opt} in v) SILENT=false ;; \?) - echo "Invalid flag: -$OPTARG" >&2 + echo "Invalid flag: -${OPTARG}" >&2 exit 1 ;; esac done -if $SILENT ; then +if ${SILENT} ; then echo "Running in the silent mode, run with -v if you want to see script logs." fi -# remove protobuf until it is part of direct generation -EXCLUDE="verify-godeps.sh verify-godep-licenses.sh verify-generated-protobuf.sh verify-linkcheck.sh" - ret=0 -for t in `ls $KUBE_ROOT/hack/verify-*.sh` -do - if is-excluded $t ; then - echo "Skipping $t" - continue - fi - echo -e "Verifying $t" - if run-cmd bash "$t"; then - echo -e "${color_green}SUCCESS${color_norm}" - else - echo -e "${color_red}FAILED${color_norm}" - ret=1 - fi -done - -for t in `ls $KUBE_ROOT/hack/verify-*.py` -do - if is-excluded $t ; then - echo "Skipping $t" - continue - fi - echo -e "Verifying $t" - if run-cmd python "$t"; then - echo -e "${color_green}SUCCESS${color_norm}" - else - echo -e "${color_red}FAILED${color_norm}" - ret=1 - fi -done -exit $ret +run-checks "${KUBE_ROOT}/hack/verify-*.sh" bash +run-checks "${KUBE_ROOT}/hack/verify-*.py" python +exit ${ret} # ex: ts=2 sw=2 et filetype=sh diff --git a/hack/verify-godep-licenses.sh b/hack/verify-godep-licenses.sh index 5eb081443d1..6d98210984a 100755 --- a/hack/verify-godep-licenses.sh +++ b/hack/verify-godep-licenses.sh @@ -19,10 +19,11 @@ set -o nounset set -o pipefail KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/.." && pwd -P)" +source "${KUBE_ROOT}/hack/lib/init.sh" -branch="${1:-master}" -# notice this uses ... to find the first shared ancestor -if ! git diff origin/"${branch}"...HEAD | grep 'Godeps/' > /dev/null; then +readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}} +if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \ + ! kube::util::has_changes_against_upstream_branch "${branch}" 'Godeps/'; then exit 0 fi diff --git a/hack/verify-godeps.sh b/hack/verify-godeps.sh index e00daa25fc8..a9890a6858d 100755 --- a/hack/verify-godeps.sh +++ b/hack/verify-godeps.sh @@ -39,9 +39,9 @@ preload-dep() { KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" -branch="${1:-master}" -# notice this uses ... to find the first shared ancestor -if ! git diff origin/"${branch}"...HEAD | grep 'Godeps/' > /dev/null; then +readonly branch=${1:-${KUBE_VERIFY_GIT_BRANCH:-master}} +if ! [[ ${KUBE_FORCE_VERIFY_CHECKS:-} =~ ^[yY]$ ]] && \ + ! kube::util::has_changes_against_upstream_branch "${branch}" 'Godeps/'; then exit 0 fi diff --git a/shippable.yml b/shippable.yml index 0c43b9810c7..e30902ac0b1 100644 --- a/shippable.yml +++ b/shippable.yml @@ -44,18 +44,7 @@ install: - ./hack/build-go.sh - godep go install ./... - ./hack/install-etcd.sh - - ./hack/verify-gofmt.sh - - ./hack/verify-boilerplate.sh - - ./hack/verify-description.sh - - ./hack/verify-flags-underscore.py - - ./hack/verify-godeps.sh ${BASE_BRANCH} - - ./hack/verify-godep-licenses.sh ${BASE_BRANCH} - - ./hack/travis/install-std-race.sh - - ./hack/verify-generated-conversions.sh - - ./hack/verify-generated-deep-copies.sh - - ./hack/verify-generated-docs.sh - - ./hack/verify-generated-swagger-docs.sh - - ./hack/verify-swagger-spec.sh + - make verify BRANCH=${BASE_BRANCH} script: # Disable coverage collection on pull requests