mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-27 21:47:07 +00:00
Merge pull request #99698 from spiffxp/verify-e2e-test-ownership
add verify-e2e-test-ownership.sh
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
f28cb4659d
223
hack/verify-e2e-test-ownership.sh
Executable file
223
hack/verify-e2e-test-ownership.sh
Executable file
@ -0,0 +1,223 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
# Copyright 2014 The Kubernetes Authors.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# This script verifies the following e2e test ownership policies
|
||||||
|
# - tests MUST start with [sig-foo]
|
||||||
|
# - tests MUST use top-level SIGDescribe
|
||||||
|
# - tests SHOULD NOT have multiple [sig-foo] tags
|
||||||
|
# - tests MUST NOT use nested SIGDescribe
|
||||||
|
# TODO: these two can be dropped if KubeDescribe is gone from codebase
|
||||||
|
# - tests MUST NOT have [k8s.io] in test names
|
||||||
|
# - tests MUST NOT use KubeDescribe
|
||||||
|
|
||||||
|
set -o errexit
|
||||||
|
set -o nounset
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
# This will canonicalize the path
|
||||||
|
KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P)
|
||||||
|
source "${KUBE_ROOT}/hack/lib/init.sh"
|
||||||
|
|
||||||
|
# Set REUSE_BUILD_OUTPUT=y to skip rebuilding dependencies if present
|
||||||
|
REUSE_BUILD_OUTPUT=${REUSE_BUILD_OUTPUT:-n}
|
||||||
|
# set VERBOSE_OUTPUT=y to output .jq files and shell commands
|
||||||
|
VERBOSE_OUTPUT=${VERBOSE_OUTPUT:-n}
|
||||||
|
|
||||||
|
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
||||||
|
set -x
|
||||||
|
fi
|
||||||
|
|
||||||
|
pushd "${KUBE_ROOT}" > /dev/null
|
||||||
|
|
||||||
|
# Setup a tmpdir to hold generated scripts and results
|
||||||
|
readonly tmpdir=$(mktemp -d -t verify-e2e-test-ownership.XXXX)
|
||||||
|
trap 'rm -rf ${tmpdir}' EXIT
|
||||||
|
|
||||||
|
# input
|
||||||
|
spec_summaries="${KUBE_ROOT}/_output/specsummaries.json"
|
||||||
|
# output
|
||||||
|
results_json="${tmpdir}/results.json"
|
||||||
|
summary_json="${tmpdir}/summary.json"
|
||||||
|
failures_json="${tmpdir}/failures.json"
|
||||||
|
|
||||||
|
# rebuild dependencies if necessary
|
||||||
|
function ensure_dependencies() {
|
||||||
|
local -r ginkgo="${KUBE_ROOT}/_output/bin/ginkgo"
|
||||||
|
local -r e2e_test="${KUBE_ROOT}/_output/bin/e2e.test"
|
||||||
|
if ! { [ -f "${ginkgo}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
|
||||||
|
make ginkgo
|
||||||
|
fi
|
||||||
|
if ! { [ -f "${e2e_test}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
|
||||||
|
hack/make-rules/build.sh test/e2e/e2e.test
|
||||||
|
fi
|
||||||
|
if ! { [ -f "${spec_summaries}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
|
||||||
|
"${ginkgo}" --dryRun=true "${e2e_test}" -- --spec-dump "${spec_summaries}" > /dev/null
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# evaluate ginkgo spec summaries against e2e test ownership polices
|
||||||
|
# output to ${results_json}
|
||||||
|
function generate_results_json() {
|
||||||
|
readonly results_jq=${tmpdir}/results.jq
|
||||||
|
cat >"${results_jq}" <<EOS
|
||||||
|
[.[] | . as { ComponentTexts: \$text, ComponentCodeLocations: \$code } | {
|
||||||
|
calls: [ \$text | range(1;length) as \$i | {
|
||||||
|
sig: ((\$text[\$i] | match("\\\[(sig-[^\\\]]+)\\\]") | .captures[0].string) // "unknown"),
|
||||||
|
text: \$text[\$i],
|
||||||
|
# unused, but if we ever wanted to have policies based on other tags...
|
||||||
|
# tags: \$text[\$i] | [match("(\\\[[^\\\]]+\\\])"; "g").string],
|
||||||
|
line: \$code[\$i] | "\(.FileName):\(.LineNumber)",
|
||||||
|
} + (\$code[\$i] | .FullStackTrace | match("^(.*)\\\.(.+)\\\(.*\\n") | .captures | {
|
||||||
|
package: .[0].string,
|
||||||
|
func: .[1].string
|
||||||
|
})],
|
||||||
|
} | {
|
||||||
|
owner: .calls[0].sig,
|
||||||
|
testname: .calls | map(.text) | join(" "),
|
||||||
|
calls,
|
||||||
|
policies: [(
|
||||||
|
.calls[0] |
|
||||||
|
{
|
||||||
|
fail: (.sig == "unknown"),
|
||||||
|
level: "FAIL",
|
||||||
|
category: "unowned_test",
|
||||||
|
reason: "must start with [sig-foo]",
|
||||||
|
found: .,
|
||||||
|
}, {
|
||||||
|
fail: (.func != "SIGDescribe"),
|
||||||
|
level: "FAIL",
|
||||||
|
category: "no_sig_describe",
|
||||||
|
reason: "must use top-level SIGDescribe",
|
||||||
|
found: .,
|
||||||
|
}
|
||||||
|
), (
|
||||||
|
.calls[1:] |
|
||||||
|
(map(select(.sig != "unknown")) // [] | {
|
||||||
|
fail: . | any,
|
||||||
|
level: "WARN",
|
||||||
|
category: "too_many_sigs",
|
||||||
|
reason: "should not have multiple [sig-foo] tags",
|
||||||
|
found: .,
|
||||||
|
}),
|
||||||
|
(map(select(.func == "SIGDescribe")) // [] | {
|
||||||
|
fail: . | any,
|
||||||
|
level: "FAIL",
|
||||||
|
category: "nested_sig_describe",
|
||||||
|
reason: "must not use nested SIGDescribe",
|
||||||
|
found: .,
|
||||||
|
})
|
||||||
|
)
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
EOS
|
||||||
|
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
||||||
|
echo "about to ${results_jq}..."
|
||||||
|
cat -n "${results_jq}"
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
<"${spec_summaries}" jq --slurp --from-file "${results_jq}" > "${results_json}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# summarize e2e test policy results
|
||||||
|
# output to ${summary_json}
|
||||||
|
function generate_summary_json() {
|
||||||
|
summary_jq=${tmpdir}/summary.jq
|
||||||
|
cat >"${summary_jq}" <<EOS
|
||||||
|
. as \$results |
|
||||||
|
# for each policy category
|
||||||
|
reduce \$results[0].policies[] as \$p ({}; . + {
|
||||||
|
# add a convenience .policy field containing that policy's result
|
||||||
|
(\$p.category): \$results | map(. + {policy: .policies[] | select(.category == \$p.category)}) | {
|
||||||
|
level: \$p.level,
|
||||||
|
reason: \$p.reason,
|
||||||
|
passing: map(select(.policy.fail | not)) | length,
|
||||||
|
failing: map(select(.policy.fail)) | length,
|
||||||
|
testnames: map(select(.policy.fail) | .testname),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
# add a meta policy based on whether any policy failed
|
||||||
|
+ {
|
||||||
|
all_policies: \$results | {
|
||||||
|
level: "WARN",
|
||||||
|
reason: "should pass all policies",
|
||||||
|
passing: map(select(.policies | map(.fail) | any | not)) | length,
|
||||||
|
failing: map(select(.policies | map(.fail) | any)) | length,
|
||||||
|
testnames: map(select(.policies | map(.fail) | any) | .testname),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
# if a policy has no failing tests, change its log output to PASS
|
||||||
|
| with_entries(.value += { log: (if (.value.failing == 0) then "PASS" else .value.level end) })
|
||||||
|
# sort by policies with the most failing tests first
|
||||||
|
| to_entries | sort_by(.value.failing) | reverse | from_entries
|
||||||
|
EOS
|
||||||
|
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
||||||
|
echo "about to run ${results_jq}..."
|
||||||
|
cat -n "${summary_jq}"
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
<"${results_json}" jq --from-file "${summary_jq}" > "${summary_json}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# filter e2e policy tests results to tests that failed, with the policies they failed
|
||||||
|
# output to ${failures_json}
|
||||||
|
function generate_failures_json() {
|
||||||
|
local -r failures_jq="${tmpdir}/failures.jq"
|
||||||
|
cat >"${failures_jq}" <<EOS
|
||||||
|
.
|
||||||
|
# for each test
|
||||||
|
| map(
|
||||||
|
# filter down to failing policies; trim category, .reason is more verbose
|
||||||
|
.policies |= map(select(.fail) | del(.category))
|
||||||
|
# trim the full callstack, .found will contain the relevant call
|
||||||
|
| del(.calls)
|
||||||
|
)
|
||||||
|
# filter down to tests that have failed policies
|
||||||
|
| map(select(.policies | map (.fail) | any))
|
||||||
|
EOS
|
||||||
|
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
||||||
|
echo "about to run ${failures_jq}..."
|
||||||
|
cat -n "${failures_jq}"
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
<"${results_json}" jq --from-file "${failures_jq}" > "${failures_json}"
|
||||||
|
}
|
||||||
|
|
||||||
|
function output_results_and_exit_if_failed() {
|
||||||
|
local -r total_tests=$(<"${spec_summaries}" wc -l | awk '{print $1}')
|
||||||
|
|
||||||
|
# output results to console
|
||||||
|
(
|
||||||
|
echo "run at datetime: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||||
|
echo "based on commit: $(git log -n1 --date=iso-strict --pretty='%h - %cd - %s')"
|
||||||
|
echo
|
||||||
|
<"${failures_json}" cat
|
||||||
|
printf "%4s: e2e tests %-40s: %-4d\n" "INFO" "in total" "${total_tests}"
|
||||||
|
<"${summary_json}" jq -r 'to_entries[].value |
|
||||||
|
"printf \"%4s: ..failing %-40s: %-4d\\n\" \"\(.log)\" \"\(.reason)\" \"\(.failing)\""' | sh
|
||||||
|
) | tee "${tmpdir}/output.txt"
|
||||||
|
# if we said "FAIL" in that output, we should fail
|
||||||
|
if <"${tmpdir}/output.txt" grep -q "^FAIL"; then
|
||||||
|
echo "FAIL"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_dependencies
|
||||||
|
generate_results_json
|
||||||
|
generate_failures_json
|
||||||
|
generate_summary_json
|
||||||
|
output_results_and_exit_if_failed
|
||||||
|
echo "PASS"
|
||||||
@ -1 +1,80 @@
|
|||||||
See [e2e-tests](https://git.k8s.io/community/contributors/devel/sig-testing/e2e-tests.md)
|
# test/e2e
|
||||||
|
|
||||||
|
This is home to e2e tests used for presubmit, periodic, and postsubmit jobs.
|
||||||
|
|
||||||
|
Some of these jobs are merge-blocking, some are release-blocking.
|
||||||
|
|
||||||
|
## e2e test ownership
|
||||||
|
|
||||||
|
All e2e tests must adhere to the following policies:
|
||||||
|
- the test must be owned by one and only one SIG
|
||||||
|
- the test must live in/underneath a sig-owned package matching pattern: `test/e2e/[{subpath}/]{sig}/...`, e.g.
|
||||||
|
- `test/e2e/auth` - all tests owned by sig-`auth`
|
||||||
|
- `test/e2e/common/storage` - all tests `common` to cluster-level and node-level e2e tests, owned by sig-`node`
|
||||||
|
- `test/e2e/upgrade/apps` - all tests used in `upgrade` testing, owned by sig-`apps`
|
||||||
|
- each sig-owned package should have an OWNERS file defining relevant approvers and labels for the owning sig, e.g.
|
||||||
|
```yaml
|
||||||
|
# test/e2e/node/OWNERS
|
||||||
|
# See the OWNERS docs at https://go.k8s.io/owners
|
||||||
|
|
||||||
|
approvers:
|
||||||
|
- alice
|
||||||
|
- bob
|
||||||
|
- cynthia
|
||||||
|
emeritus_approvers:
|
||||||
|
- dave
|
||||||
|
reviewers:
|
||||||
|
- sig-node-reviewers
|
||||||
|
labels:
|
||||||
|
- sig/node
|
||||||
|
```
|
||||||
|
- packages that use `{subpath}` should have an `imports.go` file importing sig-owned packages (for ginkgo's benefit), e.g.
|
||||||
|
```golang
|
||||||
|
// test/e2e/common/imports.go
|
||||||
|
package common
|
||||||
|
|
||||||
|
import (
|
||||||
|
// ensure these packages are scanned by ginkgo for e2e tests
|
||||||
|
_ "k8s.io/kubernetes/test/e2e/common/network"
|
||||||
|
_ "k8s.io/kubernetes/test/e2e/common/node"
|
||||||
|
_ "k8s.io/kubernetes/test/e2e/common/storage"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
- test ownership must be declared via a top-level SIGDescribe call defined in the sig-owned package, e.g.
|
||||||
|
```golang
|
||||||
|
// test/e2e/lifecycle/framework.go
|
||||||
|
package lifecycle
|
||||||
|
|
||||||
|
import "github.com/onsi/ginkgo"
|
||||||
|
|
||||||
|
// SIGDescribe annotates the test with the SIG label.
|
||||||
|
func SIGDescribe(text string, body func()) bool {
|
||||||
|
return ginkgo.Describe("[sig-cluster-lifecycle] "+text, body)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
```golang
|
||||||
|
// test/e2e/lifecycle/bootstrap/bootstrap_signer.go
|
||||||
|
|
||||||
|
package bootstrap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/onsi/ginkgo"
|
||||||
|
"k8s.io/kubernetes/test/e2e/lifecycle"
|
||||||
|
)
|
||||||
|
var _ = lifecycle.SIGDescribe("[Feature:BootstrapTokens]", func() {
|
||||||
|
/* ... */
|
||||||
|
ginkgo.It("should sign the new added bootstrap tokens", func() {
|
||||||
|
/* ... */
|
||||||
|
})
|
||||||
|
/* etc */
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
These polices are enforced:
|
||||||
|
- via the merge-blocking presubmit job `pull-kubernetes-verify`
|
||||||
|
- which ends up running `hack/verify-e2e-test-ownership.sh`
|
||||||
|
- which can also be run via `make verify WHAT=e2e-test-ownership`
|
||||||
|
|
||||||
|
## more info
|
||||||
|
|
||||||
|
See [kubernetes/community/.../e2e-tests.md](https://git.k8s.io/community/contributors/devel/sig-testing/e2e-tests.md)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user