From 2be0bd256905311f84940a85e0a0e688de787968 Mon Sep 17 00:00:00 2001
From: Aaron Crickenberger <spiffxp@google.com>
Date: Mon, 22 Feb 2021 16:35:24 -0500
Subject: [PATCH 1/2] hack/verify: add verify-e2e-test-ownership.sh

add a script to enforce that all e2e tests have one and only one sig
owner defined according to specific policies
---
 hack/verify-e2e-test-ownership.sh | 223 ++++++++++++++++++++++++++++++
 1 file changed, 223 insertions(+)
 create mode 100755 hack/verify-e2e-test-ownership.sh

diff --git a/hack/verify-e2e-test-ownership.sh b/hack/verify-e2e-test-ownership.sh
new file mode 100755
index 00000000000..fae2f3d2725
--- /dev/null
+++ b/hack/verify-e2e-test-ownership.sh
@@ -0,0 +1,223 @@
+#!/usr/bin/env bash
+
+# Copyright 2014 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script verifies the following e2e test ownership policies
+# - tests MUST start with [sig-foo]
+# - tests MUST use top-level SIGDescribe
+# - tests SHOULD NOT have multiple [sig-foo] tags
+# - tests MUST NOT use nested SIGDescribe
+# TODO: these two can be dropped if KubeDescribe is gone from codebase
+# - tests MUST NOT have [k8s.io] in test names
+# - tests MUST NOT use KubeDescribe
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# This will canonicalize the path
+KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P)
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+# Set REUSE_BUILD_OUTPUT=y to skip rebuilding dependencies if present
+REUSE_BUILD_OUTPUT=${REUSE_BUILD_OUTPUT:-n}
+# set VERBOSE_OUTPUT=y to output .jq files and shell commands
+VERBOSE_OUTPUT=${VERBOSE_OUTPUT:-n}
+
+if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
+  set -x
+fi
+
+pushd "${KUBE_ROOT}" > /dev/null
+
+# Setup a tmpdir to hold generated scripts and results
+readonly tmpdir=$(mktemp -d -t verify-e2e-test-ownership.XXXX)
+trap 'rm -rf ${tmpdir}' EXIT
+
+# input
+spec_summaries="${KUBE_ROOT}/_output/specsummaries.json"
+# output
+results_json="${tmpdir}/results.json"
+summary_json="${tmpdir}/summary.json"
+failures_json="${tmpdir}/failures.json"
+
+# rebuild dependencies if necessary
+function ensure_dependencies() {
+  local -r ginkgo="${KUBE_ROOT}/_output/bin/ginkgo"
+  local -r e2e_test="${KUBE_ROOT}/_output/bin/e2e.test"
+  if ! { [ -f "${ginkgo}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
+    make ginkgo
+  fi
+  if ! { [ -f "${e2e_test}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
+    hack/make-rules/build.sh test/e2e/e2e.test
+  fi
+  if ! { [ -f "${spec_summaries}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
+    "${ginkgo}" --dryRun=true "${e2e_test}" -- --spec-dump "${spec_summaries}" > /dev/null
+  fi
+}
+
+# evaluate ginkgo spec summaries against e2e test ownership polices
+# output to ${results_json}
+function generate_results_json() {
+  readonly results_jq=${tmpdir}/results.jq
+  cat >"${results_jq}" <<EOS
+  [.[] | . as { ComponentTexts: \$text, ComponentCodeLocations: \$code } | {
+      calls: [ \$text | range(1;length) as \$i | {
+        sig: ((\$text[\$i] | match("\\\[(sig-[^\\\]]+)\\\]") | .captures[0].string) // "unknown"),
+        text: \$text[\$i],
+        # unused, but if we ever wanted to have policies based on other tags...
+        # tags: \$text[\$i] | [match("(\\\[[^\\\]]+\\\])"; "g").string],
+        line: \$code[\$i] | "\(.FileName):\(.LineNumber)",
+      } + (\$code[\$i] | .FullStackTrace | match("^(.*)\\\.(.+)\\\(.*\\n") | .captures | {
+          package: .[0].string,
+          func: .[1].string
+      })],
+    } | {
+      owner: .calls[0].sig,
+      testname: .calls | map(.text) | join(" "),
+      calls,
+      policies: [(
+        .calls[0] |
+          {
+            fail: (.sig == "unknown"),
+            level: "FAIL",
+            category: "unowned_test",
+            reason: "must start with [sig-foo]",
+            found: .,
+          }, {
+            fail: (.func != "SIGDescribe"),
+            level: "FAIL",
+            category: "no_sig_describe",
+            reason: "must use top-level SIGDescribe",
+            found: .,
+          }
+        ), (
+        .calls[1:] |
+          (map(select(.sig != "unknown")) // [] | {
+            fail: . | any,
+            level: "WARN",
+            category: "too_many_sigs",
+            reason: "should not have multiple [sig-foo] tags",
+            found: .,
+          }),
+          (map(select(.func == "SIGDescribe")) // [] | {
+            fail: . | any,
+            level: "FAIL",
+            category: "nested_sig_describe",
+            reason: "must not use nested SIGDescribe",
+            found: .,
+          })
+        )
+      ]
+  }]
+EOS
+  if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
+    echo "about to  ${results_jq}..."
+    cat -n "${results_jq}"
+    echo
+  fi
+  <"${spec_summaries}" jq --slurp --from-file "${results_jq}" > "${results_json}"
+}
+
+# summarize e2e test policy results
+# output to ${summary_json}
+function generate_summary_json() {
+  summary_jq=${tmpdir}/summary.jq
+  cat >"${summary_jq}" <<EOS
+  . as \$results |
+  # for each policy category
+  reduce \$results[0].policies[] as \$p ({}; . + {
+    # add a convenience .policy field containing that policy's result
+    (\$p.category): \$results | map(. + {policy: .policies[] | select(.category == \$p.category)}) | {
+      level: \$p.level,
+      reason: \$p.reason,
+      passing: map(select(.policy.fail | not)) | length,
+      failing: map(select(.policy.fail)) | length,
+      testnames: map(select(.policy.fail) | .testname),
+    }
+  })
+  # add a meta policy based on whether any policy failed
+  + {
+    all_policies: \$results | {
+      level: "WARN",
+      reason: "should pass all policies",
+      passing: map(select(.policies | map(.fail) | any | not)) | length,
+      failing: map(select(.policies | map(.fail) | any)) | length,
+      testnames: map(select(.policies | map(.fail) | any) | .testname),
+    }
+  }
+  # if a policy has no failing tests, change its log output to PASS
+  | with_entries(.value += { log: (if (.value.failing == 0) then "PASS" else .value.level end) })
+  # sort by policies with the most failing tests first
+  | to_entries | sort_by(.value.failing) | reverse | from_entries
+EOS
+  if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
+    echo "about to run ${results_jq}..."
+    cat -n "${summary_jq}"
+    echo
+  fi
+  <"${results_json}" jq --from-file "${summary_jq}" > "${summary_json}"
+}
+
+# filter e2e policy tests results to tests that failed, with the policies they failed
+# output to ${failures_json}
+function generate_failures_json() {
+  local -r failures_jq="${tmpdir}/failures.jq"
+  cat >"${failures_jq}" <<EOS
+  .
+  # for each test
+  | map(
+    # filter down to failing policies; trim category, .reason is more verbose
+    .policies |= map(select(.fail) | del(.category))
+    # trim the full callstack, .found will contain the relevant call
+    | del(.calls)
+  )
+  # filter down to tests that have failed policies
+  | map(select(.policies | map (.fail) | any))
+EOS
+  if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
+    echo "about to run ${failures_jq}..."
+    cat -n "${failures_jq}"
+    echo
+  fi
+  <"${results_json}" jq --from-file "${failures_jq}" > "${failures_json}"
+}
+
+function output_results_and_exit_if_failed() {
+  local -r total_tests=$(<"${spec_summaries}" wc -l | awk '{print $1}')
+
+  # output results to console
+  (
+    echo "run at datetime: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+    echo "based on commit: $(git log -n1 --date=iso-strict --pretty='%h - %cd - %s')"
+    echo
+    <"${failures_json}" cat
+    printf "%4s: e2e tests %-40s: %-4d\n" "INFO" "in total" "${total_tests}"
+    <"${summary_json}" jq -r 'to_entries[].value |
+      "printf \"%4s: ..failing %-40s: %-4d\\n\" \"\(.log)\" \"\(.reason)\" \"\(.failing)\""' | sh
+  ) | tee "${tmpdir}/output.txt"
+  # if we said "FAIL" in that output, we should fail
+  if <"${tmpdir}/output.txt" grep -q "^FAIL"; then
+    echo "FAIL"
+    exit 1
+  fi
+}
+
+ensure_dependencies
+generate_results_json
+generate_failures_json
+generate_summary_json
+output_results_and_exit_if_failed
+echo "PASS"

From e12e5be329e0e5390dadcd4ee051bef8f035ff2b Mon Sep 17 00:00:00 2001
From: Aaron Crickenberger <spiffxp@google.com>
Date: Wed, 17 Mar 2021 16:47:05 -0400
Subject: [PATCH 2/2] test/e2e: Add ownership info to README

---
 test/e2e/README.md | 81 +++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 80 insertions(+), 1 deletion(-)

diff --git a/test/e2e/README.md b/test/e2e/README.md
index 71b00676181..3ff2135ea67 100644
--- a/test/e2e/README.md
+++ b/test/e2e/README.md
@@ -1 +1,80 @@
-See [e2e-tests](https://git.k8s.io/community/contributors/devel/sig-testing/e2e-tests.md)
+# test/e2e
+
+This is home to e2e tests used for presubmit, periodic, and postsubmit jobs.
+
+Some of these jobs are merge-blocking, some are release-blocking.
+
+## e2e test ownership
+
+All e2e tests must adhere to the following policies:
+- the test must be owned by one and only one SIG
+- the test must live in/underneath a sig-owned package matching pattern: `test/e2e/[{subpath}/]{sig}/...`, e.g.
+  - `test/e2e/auth` - all tests owned by sig-`auth`
+  - `test/e2e/common/storage` - all tests `common` to cluster-level and node-level e2e tests, owned by sig-`node`
+  - `test/e2e/upgrade/apps` - all tests used in `upgrade` testing, owned by sig-`apps`
+- each sig-owned package should have an OWNERS file defining relevant approvers and labels for the owning sig, e.g.
+```yaml
+# test/e2e/node/OWNERS
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- alice
+- bob
+- cynthia
+emeritus_approvers:
+- dave
+reviewers:
+- sig-node-reviewers
+labels:
+- sig/node
+```
+- packages that use `{subpath}` should have an `imports.go` file importing sig-owned packages (for ginkgo's benefit), e.g.
+```golang
+// test/e2e/common/imports.go
+package common
+
+import (
+	// ensure these packages are scanned by ginkgo for e2e tests
+	_ "k8s.io/kubernetes/test/e2e/common/network"
+	_ "k8s.io/kubernetes/test/e2e/common/node"
+	_ "k8s.io/kubernetes/test/e2e/common/storage"
+)
+```
+- test ownership must be declared via a top-level SIGDescribe call defined in the sig-owned package, e.g.
+```golang
+// test/e2e/lifecycle/framework.go
+package lifecycle
+
+import "github.com/onsi/ginkgo"
+
+// SIGDescribe annotates the test with the SIG label.
+func SIGDescribe(text string, body func()) bool {
+	return ginkgo.Describe("[sig-cluster-lifecycle] "+text, body)
+}
+```
+```golang
+// test/e2e/lifecycle/bootstrap/bootstrap_signer.go
+
+package bootstrap
+
+import (
+	"github.com/onsi/ginkgo"
+	"k8s.io/kubernetes/test/e2e/lifecycle"
+)
+var _ = lifecycle.SIGDescribe("[Feature:BootstrapTokens]", func() {
+  /* ... */
+  ginkgo.It("should sign the new added bootstrap tokens", func() {
+    /* ... */
+  })
+  /* etc */
+})
+```
+
+These polices are enforced:
+- via the merge-blocking presubmit job `pull-kubernetes-verify`
+- which ends up running `hack/verify-e2e-test-ownership.sh`
+- which can also be run via `make verify WHAT=e2e-test-ownership`
+
+## more info
+
+See [kubernetes/community/.../e2e-tests.md](https://git.k8s.io/community/contributors/devel/sig-testing/e2e-tests.md)