mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-22 03:11:40 +00:00
Merge pull request #99698 from spiffxp/verify-e2e-test-ownership
add verify-e2e-test-ownership.sh
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
f28cb4659d
223
hack/verify-e2e-test-ownership.sh
Executable file
223
hack/verify-e2e-test-ownership.sh
Executable file
@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2014 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script verifies the following e2e test ownership policies
|
||||
# - tests MUST start with [sig-foo]
|
||||
# - tests MUST use top-level SIGDescribe
|
||||
# - tests SHOULD NOT have multiple [sig-foo] tags
|
||||
# - tests MUST NOT use nested SIGDescribe
|
||||
# TODO: these two can be dropped if KubeDescribe is gone from codebase
|
||||
# - tests MUST NOT have [k8s.io] in test names
|
||||
# - tests MUST NOT use KubeDescribe
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# This will canonicalize the path
|
||||
KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P)
|
||||
source "${KUBE_ROOT}/hack/lib/init.sh"
|
||||
|
||||
# Set REUSE_BUILD_OUTPUT=y to skip rebuilding dependencies if present
|
||||
REUSE_BUILD_OUTPUT=${REUSE_BUILD_OUTPUT:-n}
|
||||
# set VERBOSE_OUTPUT=y to output .jq files and shell commands
|
||||
VERBOSE_OUTPUT=${VERBOSE_OUTPUT:-n}
|
||||
|
||||
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
||||
set -x
|
||||
fi
|
||||
|
||||
pushd "${KUBE_ROOT}" > /dev/null
|
||||
|
||||
# Setup a tmpdir to hold generated scripts and results
|
||||
readonly tmpdir=$(mktemp -d -t verify-e2e-test-ownership.XXXX)
|
||||
trap 'rm -rf ${tmpdir}' EXIT
|
||||
|
||||
# input
|
||||
spec_summaries="${KUBE_ROOT}/_output/specsummaries.json"
|
||||
# output
|
||||
results_json="${tmpdir}/results.json"
|
||||
summary_json="${tmpdir}/summary.json"
|
||||
failures_json="${tmpdir}/failures.json"
|
||||
|
||||
# rebuild dependencies if necessary
|
||||
function ensure_dependencies() {
|
||||
local -r ginkgo="${KUBE_ROOT}/_output/bin/ginkgo"
|
||||
local -r e2e_test="${KUBE_ROOT}/_output/bin/e2e.test"
|
||||
if ! { [ -f "${ginkgo}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
|
||||
make ginkgo
|
||||
fi
|
||||
if ! { [ -f "${e2e_test}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
|
||||
hack/make-rules/build.sh test/e2e/e2e.test
|
||||
fi
|
||||
if ! { [ -f "${spec_summaries}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
|
||||
"${ginkgo}" --dryRun=true "${e2e_test}" -- --spec-dump "${spec_summaries}" > /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# evaluate ginkgo spec summaries against e2e test ownership polices
|
||||
# output to ${results_json}
|
||||
function generate_results_json() {
|
||||
readonly results_jq=${tmpdir}/results.jq
|
||||
cat >"${results_jq}" <<EOS
|
||||
[.[] | . as { ComponentTexts: \$text, ComponentCodeLocations: \$code } | {
|
||||
calls: [ \$text | range(1;length) as \$i | {
|
||||
sig: ((\$text[\$i] | match("\\\[(sig-[^\\\]]+)\\\]") | .captures[0].string) // "unknown"),
|
||||
text: \$text[\$i],
|
||||
# unused, but if we ever wanted to have policies based on other tags...
|
||||
# tags: \$text[\$i] | [match("(\\\[[^\\\]]+\\\])"; "g").string],
|
||||
line: \$code[\$i] | "\(.FileName):\(.LineNumber)",
|
||||
} + (\$code[\$i] | .FullStackTrace | match("^(.*)\\\.(.+)\\\(.*\\n") | .captures | {
|
||||
package: .[0].string,
|
||||
func: .[1].string
|
||||
})],
|
||||
} | {
|
||||
owner: .calls[0].sig,
|
||||
testname: .calls | map(.text) | join(" "),
|
||||
calls,
|
||||
policies: [(
|
||||
.calls[0] |
|
||||
{
|
||||
fail: (.sig == "unknown"),
|
||||
level: "FAIL",
|
||||
category: "unowned_test",
|
||||
reason: "must start with [sig-foo]",
|
||||
found: .,
|
||||
}, {
|
||||
fail: (.func != "SIGDescribe"),
|
||||
level: "FAIL",
|
||||
category: "no_sig_describe",
|
||||
reason: "must use top-level SIGDescribe",
|
||||
found: .,
|
||||
}
|
||||
), (
|
||||
.calls[1:] |
|
||||
(map(select(.sig != "unknown")) // [] | {
|
||||
fail: . | any,
|
||||
level: "WARN",
|
||||
category: "too_many_sigs",
|
||||
reason: "should not have multiple [sig-foo] tags",
|
||||
found: .,
|
||||
}),
|
||||
(map(select(.func == "SIGDescribe")) // [] | {
|
||||
fail: . | any,
|
||||
level: "FAIL",
|
||||
category: "nested_sig_describe",
|
||||
reason: "must not use nested SIGDescribe",
|
||||
found: .,
|
||||
})
|
||||
)
|
||||
]
|
||||
}]
|
||||
EOS
|
||||
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
||||
echo "about to ${results_jq}..."
|
||||
cat -n "${results_jq}"
|
||||
echo
|
||||
fi
|
||||
<"${spec_summaries}" jq --slurp --from-file "${results_jq}" > "${results_json}"
|
||||
}
|
||||
|
||||
# summarize e2e test policy results
|
||||
# output to ${summary_json}
|
||||
function generate_summary_json() {
|
||||
summary_jq=${tmpdir}/summary.jq
|
||||
cat >"${summary_jq}" <<EOS
|
||||
. as \$results |
|
||||
# for each policy category
|
||||
reduce \$results[0].policies[] as \$p ({}; . + {
|
||||
# add a convenience .policy field containing that policy's result
|
||||
(\$p.category): \$results | map(. + {policy: .policies[] | select(.category == \$p.category)}) | {
|
||||
level: \$p.level,
|
||||
reason: \$p.reason,
|
||||
passing: map(select(.policy.fail | not)) | length,
|
||||
failing: map(select(.policy.fail)) | length,
|
||||
testnames: map(select(.policy.fail) | .testname),
|
||||
}
|
||||
})
|
||||
# add a meta policy based on whether any policy failed
|
||||
+ {
|
||||
all_policies: \$results | {
|
||||
level: "WARN",
|
||||
reason: "should pass all policies",
|
||||
passing: map(select(.policies | map(.fail) | any | not)) | length,
|
||||
failing: map(select(.policies | map(.fail) | any)) | length,
|
||||
testnames: map(select(.policies | map(.fail) | any) | .testname),
|
||||
}
|
||||
}
|
||||
# if a policy has no failing tests, change its log output to PASS
|
||||
| with_entries(.value += { log: (if (.value.failing == 0) then "PASS" else .value.level end) })
|
||||
# sort by policies with the most failing tests first
|
||||
| to_entries | sort_by(.value.failing) | reverse | from_entries
|
||||
EOS
|
||||
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
||||
echo "about to run ${results_jq}..."
|
||||
cat -n "${summary_jq}"
|
||||
echo
|
||||
fi
|
||||
<"${results_json}" jq --from-file "${summary_jq}" > "${summary_json}"
|
||||
}
|
||||
|
||||
# filter e2e policy tests results to tests that failed, with the policies they failed
|
||||
# output to ${failures_json}
|
||||
function generate_failures_json() {
|
||||
local -r failures_jq="${tmpdir}/failures.jq"
|
||||
cat >"${failures_jq}" <<EOS
|
||||
.
|
||||
# for each test
|
||||
| map(
|
||||
# filter down to failing policies; trim category, .reason is more verbose
|
||||
.policies |= map(select(.fail) | del(.category))
|
||||
# trim the full callstack, .found will contain the relevant call
|
||||
| del(.calls)
|
||||
)
|
||||
# filter down to tests that have failed policies
|
||||
| map(select(.policies | map (.fail) | any))
|
||||
EOS
|
||||
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
||||
echo "about to run ${failures_jq}..."
|
||||
cat -n "${failures_jq}"
|
||||
echo
|
||||
fi
|
||||
<"${results_json}" jq --from-file "${failures_jq}" > "${failures_json}"
|
||||
}
|
||||
|
||||
function output_results_and_exit_if_failed() {
|
||||
local -r total_tests=$(<"${spec_summaries}" wc -l | awk '{print $1}')
|
||||
|
||||
# output results to console
|
||||
(
|
||||
echo "run at datetime: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "based on commit: $(git log -n1 --date=iso-strict --pretty='%h - %cd - %s')"
|
||||
echo
|
||||
<"${failures_json}" cat
|
||||
printf "%4s: e2e tests %-40s: %-4d\n" "INFO" "in total" "${total_tests}"
|
||||
<"${summary_json}" jq -r 'to_entries[].value |
|
||||
"printf \"%4s: ..failing %-40s: %-4d\\n\" \"\(.log)\" \"\(.reason)\" \"\(.failing)\""' | sh
|
||||
) | tee "${tmpdir}/output.txt"
|
||||
# if we said "FAIL" in that output, we should fail
|
||||
if <"${tmpdir}/output.txt" grep -q "^FAIL"; then
|
||||
echo "FAIL"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_dependencies
|
||||
generate_results_json
|
||||
generate_failures_json
|
||||
generate_summary_json
|
||||
output_results_and_exit_if_failed
|
||||
echo "PASS"
|
||||
@ -1 +1,80 @@
|
||||
See [e2e-tests](https://git.k8s.io/community/contributors/devel/sig-testing/e2e-tests.md)
|
||||
# test/e2e
|
||||
|
||||
This is home to e2e tests used for presubmit, periodic, and postsubmit jobs.
|
||||
|
||||
Some of these jobs are merge-blocking, some are release-blocking.
|
||||
|
||||
## e2e test ownership
|
||||
|
||||
All e2e tests must adhere to the following policies:
|
||||
- the test must be owned by one and only one SIG
|
||||
- the test must live in/underneath a sig-owned package matching pattern: `test/e2e/[{subpath}/]{sig}/...`, e.g.
|
||||
- `test/e2e/auth` - all tests owned by sig-`auth`
|
||||
- `test/e2e/common/storage` - all tests `common` to cluster-level and node-level e2e tests, owned by sig-`node`
|
||||
- `test/e2e/upgrade/apps` - all tests used in `upgrade` testing, owned by sig-`apps`
|
||||
- each sig-owned package should have an OWNERS file defining relevant approvers and labels for the owning sig, e.g.
|
||||
```yaml
|
||||
# test/e2e/node/OWNERS
|
||||
# See the OWNERS docs at https://go.k8s.io/owners
|
||||
|
||||
approvers:
|
||||
- alice
|
||||
- bob
|
||||
- cynthia
|
||||
emeritus_approvers:
|
||||
- dave
|
||||
reviewers:
|
||||
- sig-node-reviewers
|
||||
labels:
|
||||
- sig/node
|
||||
```
|
||||
- packages that use `{subpath}` should have an `imports.go` file importing sig-owned packages (for ginkgo's benefit), e.g.
|
||||
```golang
|
||||
// test/e2e/common/imports.go
|
||||
package common
|
||||
|
||||
import (
|
||||
// ensure these packages are scanned by ginkgo for e2e tests
|
||||
_ "k8s.io/kubernetes/test/e2e/common/network"
|
||||
_ "k8s.io/kubernetes/test/e2e/common/node"
|
||||
_ "k8s.io/kubernetes/test/e2e/common/storage"
|
||||
)
|
||||
```
|
||||
- test ownership must be declared via a top-level SIGDescribe call defined in the sig-owned package, e.g.
|
||||
```golang
|
||||
// test/e2e/lifecycle/framework.go
|
||||
package lifecycle
|
||||
|
||||
import "github.com/onsi/ginkgo"
|
||||
|
||||
// SIGDescribe annotates the test with the SIG label.
|
||||
func SIGDescribe(text string, body func()) bool {
|
||||
return ginkgo.Describe("[sig-cluster-lifecycle] "+text, body)
|
||||
}
|
||||
```
|
||||
```golang
|
||||
// test/e2e/lifecycle/bootstrap/bootstrap_signer.go
|
||||
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"github.com/onsi/ginkgo"
|
||||
"k8s.io/kubernetes/test/e2e/lifecycle"
|
||||
)
|
||||
var _ = lifecycle.SIGDescribe("[Feature:BootstrapTokens]", func() {
|
||||
/* ... */
|
||||
ginkgo.It("should sign the new added bootstrap tokens", func() {
|
||||
/* ... */
|
||||
})
|
||||
/* etc */
|
||||
})
|
||||
```
|
||||
|
||||
These polices are enforced:
|
||||
- via the merge-blocking presubmit job `pull-kubernetes-verify`
|
||||
- which ends up running `hack/verify-e2e-test-ownership.sh`
|
||||
- which can also be run via `make verify WHAT=e2e-test-ownership`
|
||||
|
||||
## more info
|
||||
|
||||
See [kubernetes/community/.../e2e-tests.md](https://git.k8s.io/community/contributors/devel/sig-testing/e2e-tests.md)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user