diff --git a/hack/verify-e2e-test-ownership.sh b/hack/verify-e2e-test-ownership.sh new file mode 100755 index 00000000000..fae2f3d2725 --- /dev/null +++ b/hack/verify-e2e-test-ownership.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash + +# Copyright 2014 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script verifies the following e2e test ownership policies +# - tests MUST start with [sig-foo] +# - tests MUST use top-level SIGDescribe +# - tests SHOULD NOT have multiple [sig-foo] tags +# - tests MUST NOT use nested SIGDescribe +# TODO: these two can be dropped if KubeDescribe is gone from codebase +# - tests MUST NOT have [k8s.io] in test names +# - tests MUST NOT use KubeDescribe + +set -o errexit +set -o nounset +set -o pipefail + +# This will canonicalize the path +KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P) +source "${KUBE_ROOT}/hack/lib/init.sh" + +# Set REUSE_BUILD_OUTPUT=y to skip rebuilding dependencies if present +REUSE_BUILD_OUTPUT=${REUSE_BUILD_OUTPUT:-n} +# set VERBOSE_OUTPUT=y to output .jq files and shell commands +VERBOSE_OUTPUT=${VERBOSE_OUTPUT:-n} + +if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then + set -x +fi + +pushd "${KUBE_ROOT}" > /dev/null + +# Setup a tmpdir to hold generated scripts and results +readonly tmpdir=$(mktemp -d -t verify-e2e-test-ownership.XXXX) +trap 'rm -rf ${tmpdir}' EXIT + +# input +spec_summaries="${KUBE_ROOT}/_output/specsummaries.json" +# output +results_json="${tmpdir}/results.json" +summary_json="${tmpdir}/summary.json" +failures_json="${tmpdir}/failures.json" + +# rebuild dependencies if necessary +function ensure_dependencies() { + local -r ginkgo="${KUBE_ROOT}/_output/bin/ginkgo" + local -r e2e_test="${KUBE_ROOT}/_output/bin/e2e.test" + if ! { [ -f "${ginkgo}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then + make ginkgo + fi + if ! { [ -f "${e2e_test}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then + hack/make-rules/build.sh test/e2e/e2e.test + fi + if ! { [ -f "${spec_summaries}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then + "${ginkgo}" --dryRun=true "${e2e_test}" -- --spec-dump "${spec_summaries}" > /dev/null + fi +} + +# evaluate ginkgo spec summaries against e2e test ownership polices +# output to ${results_json} +function generate_results_json() { + readonly results_jq=${tmpdir}/results.jq + cat >"${results_jq}" < "${results_json}" +} + +# summarize e2e test policy results +# output to ${summary_json} +function generate_summary_json() { + summary_jq=${tmpdir}/summary.jq + cat >"${summary_jq}" < "${summary_json}" +} + +# filter e2e policy tests results to tests that failed, with the policies they failed +# output to ${failures_json} +function generate_failures_json() { + local -r failures_jq="${tmpdir}/failures.jq" + cat >"${failures_jq}" < "${failures_json}" +} + +function output_results_and_exit_if_failed() { + local -r total_tests=$(<"${spec_summaries}" wc -l | awk '{print $1}') + + # output results to console + ( + echo "run at datetime: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "based on commit: $(git log -n1 --date=iso-strict --pretty='%h - %cd - %s')" + echo + <"${failures_json}" cat + printf "%4s: e2e tests %-40s: %-4d\n" "INFO" "in total" "${total_tests}" + <"${summary_json}" jq -r 'to_entries[].value | + "printf \"%4s: ..failing %-40s: %-4d\\n\" \"\(.log)\" \"\(.reason)\" \"\(.failing)\""' | sh + ) | tee "${tmpdir}/output.txt" + # if we said "FAIL" in that output, we should fail + if <"${tmpdir}/output.txt" grep -q "^FAIL"; then + echo "FAIL" + exit 1 + fi +} + +ensure_dependencies +generate_results_json +generate_failures_json +generate_summary_json +output_results_and_exit_if_failed +echo "PASS"