[KMSv2] update ci script to create cluster and gather metrics

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>

test/e2e/testing-manifests/auth/encrypt/encryption-config.yaml

@@ -1,10 +1,66 @@
 apiVersion: apiserver.config.k8s.io/v1
 kind: EncryptionConfiguration
 resources:
+  # The set of resources here are configured using output from "kubectl api-resources -o name" in a
+  # kind cluster running the latest built release.
   - resources:
+    - bindings
+    - componentstatuses
+    - configmaps
+    - endpoints
+    - events
+    - limitranges
+    - namespaces
+    - nodes
+    - persistentvolumeclaims
+    - persistentvolumes
+    - pods
+    - podtemplates
+    - replicationcontrollers
+    - resourcequotas
     - secrets
+    - serviceaccounts
+    - services
+    - mutatingwebhookconfigurations.admissionregistration.k8s.io
+    - validatingwebhookconfigurations.admissionregistration.k8s.io
+    - customresourcedefinitions.apiextensions.k8s.io
+    - apiservices.apiregistration.k8s.io
+    - controllerrevisions.apps
+    - daemonsets.apps
+    - deployments.apps
+    - replicasets.apps
+    - statefulsets.apps
+    - tokenreviews.authentication.k8s.io
+    - localsubjectaccessreviews.authorization.k8s.io
+    - selfsubjectaccessreviews.authorization.k8s.io
+    - selfsubjectrulesreviews.authorization.k8s.io
+    - subjectaccessreviews.authorization.k8s.io
+    - horizontalpodautoscalers.autoscaling
+    - cronjobs.batch
+    - jobs.batch
+    - certificatesigningrequests.certificates.k8s.io
+    - leases.coordination.k8s.io
+    - endpointslices.discovery.k8s.io
+    - events.events.k8s.io
+    - flowschemas.flowcontrol.apiserver.k8s.io
+    - prioritylevelconfigurations.flowcontrol.apiserver.k8s.io
+    - ingressclasses.networking.k8s.io
+    - ingresses.networking.k8s.io
+    - networkpolicies.networking.k8s.io
+    - runtimeclasses.node.k8s.io
+    - poddisruptionbudgets.policy
+    - clusterrolebindings.rbac.authorization.k8s.io
+    - clusterroles.rbac.authorization.k8s.io
+    - rolebindings.rbac.authorization.k8s.io
+    - roles.rbac.authorization.k8s.io
+    - priorityclasses.scheduling.k8s.io
+    - csidrivers.storage.k8s.io
+    - csinodes.storage.k8s.io
+    - csistoragecapacities.storage.k8s.io
+    - storageclasses.storage.k8s.io
+    - volumeattachments.storage.k8s.io
     providers:
     - kms:
         apiVersion: v2
-        name: kmsprovider
+        name: kmsv2provider
         endpoint: unix:///tmp/kms.socket

test/e2e/testing-manifests/auth/encrypt/run-e2e.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script does following:
+# 1. Creates local registry if not already present. This registry is used to push the kms mock plugin image.
+# 2. Build and push the kms mock plugin image to the local registry.
+# 3. Connect local registry to kind network so that kind cluster created using kubetest2 in prow CI job can pull the kms mock plugin image.
+# 4. Create kind cluster using kubetest2 and run e2e tests.
+# 5. Collect logs and metrics from kind cluster.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+readonly cluster_name="kms"
+readonly registry_name="kind-registry"
+readonly kind_network="kind"
+
+# build_and_push_mock_plugin builds and pushes the kms mock plugin image to the local registry.
+build_and_push_mock_plugin() {
+    docker buildx build \
+        --no-cache \
+        --platform linux/amd64 \
+        --output=type=docker \
+        -t localhost:5000/mock-kms-provider:e2e \
+        -f staging/src/k8s.io/kms/internal/plugins/mock/Dockerfile staging/src/k8s.io/ \
+        --progress=plain;
+
+    docker push localhost:5000/mock-kms-provider:e2e
+}
+
+# create_registry creates local registry if not already present.
+create_registry() {
+    running="$(docker inspect -f '{{.State.Running}}' "${registry_name}" 2>/dev/null || true)"
+    if [ "${running}" != 'true' ]; then
+        echo "Creating local registry"
+        docker run \
+            -d --restart=always -p "5000:5000" --name "${registry_name}" \
+            registry:2
+    else
+        echo "Local registry is already running"
+    fi
+}
+
+# connect_registry connects local registry to kind network.
+connect_registry(){
+    # wait for the kind network to exist
+    # infinite loop here is fine because kubetest2 will timeout if kind cluster creation fails and that will terminate the CI job
+    for ((; ;)); do
+        if docker network ls | grep "${kind_network}"; then
+            break
+        else
+            echo "'docker network ls' does not have '${kind_network}' network yet. Retrying in 1 second..."
+            sleep 1
+    fi
+    done
+
+    containers=$(docker network inspect "${kind_network}" -f "{{range .Containers}}{{.Name}} {{end}}")
+    needs_connect="true"
+    for c in $containers; do
+        if [ "$c" = "${registry_name}" ]; then
+            needs_connect="false"
+        fi
+    done
+
+    if [ "${needs_connect}" = "true" ]; then
+        echo "connecting kind network to local registry"
+        docker network connect "${kind_network}" "${registry_name}"
+    else
+        echo "'${kind_network}' network is already connected to local registry"
+    fi
+}
+
+# create_cluster_and_run_test creates a kind cluster using kubetest2 and runs e2e tests.
+create_cluster_and_run_test() {
+    CLUSTER_CREATE_ATTEMPTED=true
+
+    kubetest2 kind -v 5 \
+    --build \
+    --up \
+    --rundir-in-artifacts \
+    --config test/e2e/testing-manifests/auth/encrypt/kind.yaml \
+    --cluster-name "${cluster_name}" \
+    --test=ginkgo \
+    -- \
+    --v=5 \
+    --focus-regex='\[Conformance\]' \
+    --skip-regex='\[Serial\]' \
+    --parallel 20 \
+    --use-built-binaries # use the kubectl, e2e.test, and ginkgo binaries built during --build as opposed to from a GCS release tarball
+}
+
+cleanup() {
+    # CLUSTER_CREATE_ATTEMPTED is true once we run kubetest2 kind --up
+    if [ "${CLUSTER_CREATE_ATTEMPTED:-}" = true ]; then
+        # collect logs and metrics
+        echo "Collecting logs"
+        mkdir -p "${ARTIFACTS}/logs"
+        kind "export" logs "${ARTIFACTS}/logs" --name "${cluster_name}"
+
+        echo "Collecting metrics"
+        mkdir -p "${ARTIFACTS}/metrics"
+        kubectl get --raw /metrics > "${ARTIFACTS}/metrics/kube-apiserver-metrics.txt"
+
+        echo "Deleting kind cluster"
+        # delete cluster
+        kind delete cluster --name "${cluster_name}"
+    fi
+}
+
+main(){
+    # ensure artifacts (results) directory exists when not in CI
+    export ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
+    mkdir -p "${ARTIFACTS}"
+
+    export GO111MODULE=on;
+    go install sigs.k8s.io/kind@v0.17.0;
+    go install sigs.k8s.io/kubetest2@latest;
+    go install sigs.k8s.io/kubetest2/kubetest2-kind@latest;
+    go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest;
+
+    # The build e2e.test, ginkgo and kubectl binaries + copy to dockerized dir is
+    # because of https://github.com/kubernetes-sigs/kubetest2/issues/184
+    make all WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/v2/ginkgo cmd/kubectl";
+    mkdir -p _output/dockerized/bin/linux/amd64;
+    for binary in kubectl e2e.test ginkgo; do
+        cp -f _output/local/go/bin/${binary} _output/dockerized/bin/linux/amd64/${binary};
+    done;
+
+    create_registry
+    build_and_push_mock_plugin
+    connect_registry &
+    create_cluster_and_run_test
+    cleanup
+}
+
+trap cleanup INT TERM
+main "$@"

test/e2e/testing-manifests/auth/encrypt/setup-cluster-prereqs.sh

@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2023 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script does following:
-# 1. Creates local registry if not already present. This registry is used to push the kms mock plugin image.
-# 2. Build and push the kms mock plugin image to the local registry.
-# 3. Connect local registry to kind network so that kind cluster created using kubetest2 in prow CI job can pull the kms mock plugin image.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# build_and_push_mock_plugin builds and pushes the kms mock plugin image to the local registry.
-build_and_push_mock_plugin() {
-    docker buildx build \
-        --no-cache \
-        --platform linux/amd64 \
-        --output=type=docker \
-        -t localhost:5000/mock-kms-provider:e2e \
-        -f staging/src/k8s.io/kms/internal/plugins/mock/Dockerfile staging/src/k8s.io/ \
-        --progress=plain;
-
-    docker push localhost:5000/mock-kms-provider:e2e
-}
-
-# create_registry creates local registry if not already present.
-create_registry() {
-    running="$(docker inspect -f '{{.State.Running}}' "kind-registry" 2>/dev/null || true)"
-    if [ "${running}" != 'true' ]; then
-        echo "Creating local registry"
-        docker run \
-            -d --restart=always -p "5000:5000" --name "kind-registry" \
-            registry:2
-    else 
-        echo "Local registry is already running"
-    fi
-}
-
-# connect_registry connects local registry to kind network.
-connect_registry(){
-    # wait for the kind network to exist
-    # infinite loop here is fine because kubetest2 will timeout if kind cluster creation fails and that will terminate the CI job
-    for ((; ;)); do
-        if docker network ls | grep "kind"; then
-            break
-        else
-            echo "'docker network ls' does not have 'kind' network to connect registry"
-            sleep 1
-    fi
-    done
-
-    containers=$(docker network inspect "kind" -f "{{range .Containers}}{{.Name}} {{end}}")
-    needs_connect="true"
-    for c in $containers; do
-        if [ "$c" = "kind-registry" ]; then
-            needs_connect="false"
-        fi
-    done
-
-    if [ "${needs_connect}" = "true" ]; then
-        echo "connecting kind network to kind-registry"
-        docker network connect "kind" "kind-registry"
-    else
-        echo "'kind' network is already connected to 'kind-registry'"
-    fi
-}
-
-main(){
-    create_registry
-    build_and_push_mock_plugin
-    connect_registry &
-}
-
-main