upstream heptio/kube-conformance

Pick up some code from https://github.com/heptio/kube-conformance Fix up build scripts for the new conformance image Fix Header template and Copyright to make verify job go green update README and add execute permissions for script Change-Id: Ib6509acd816cc2fb3a516bfb8e0ff9e32bff8f79
2025-09-25 04:11:46 +00:00 · 2018-10-03 09:00:16 -04:00
parent 6b9de2bdc7
commit 6830bad2a6
12 changed files with 393 additions and 7 deletions
--- a/cluster/images/conformance/BUILD
+++ b/cluster/images/conformance/BUILD
@@ -0,0 +1,65 @@
+load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image", "container_layer")
+
+container_layer(
+    name = "cluster-srcs",
+    data_path = "/",
+    directory = "/kubernetes",
+    files = ["//cluster:all-srcs"],
+)
+
+container_layer(
+    name = "bins",
+    directory = "/usr/local/bin",
+    files = [
+        "//cmd/kubectl",
+        "//test/e2e:e2e.test",
+        "//vendor/github.com/onsi/ginkgo/ginkgo",
+    ],
+)
+
+container_image(
+    name = "conformance-internal",
+    base = "@debian-hyperkube-base-amd64//image",
+    cmd = [
+        "/bin/bash",
+        "-c",
+        "/run_e2e.sh",
+    ],
+    env = {
+        "E2E_FOCUS": "\[Conformance\]",
+        "E2E_SKIP": "",
+        "E2E_PARALLEL": "1",
+        "E2E_PROVIDER": "local",
+        "RESULTS_DIR": "/tmp/results",
+        "KUBECONFIG": "",
+    },
+    files = [
+        ":run_e2e.sh",
+    ],
+    layers = [
+        ":cluster-srcs",
+        ":bins",
+    ],
+    stamp = True,
+    workdir = "/usr/local/bin",
+)
+
+container_bundle(
+    name = "conformance",
+    images = {"k8s.gcr.io/conformance-amd64:{STABLE_DOCKER_TAG}": "conformance-internal"},
+    stamp = True,
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
--- a/cluster/images/conformance/Dockerfile
+++ b/cluster/images/conformance/Dockerfile
@@ -0,0 +1,31 @@
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM BASEIMAGE
+
+COPY ginkgo /usr/local/bin/
+COPY e2e.test /usr/local/bin/
+COPY kubectl /usr/local/bin/
+COPY run_e2e.sh /run_e2e.sh
+COPY cluster /kubernetes/cluster
+WORKDIR /usr/local/bin
+
+ENV E2E_FOCUS="\[Conformance\]"
+ENV E2E_SKIP=""
+ENV E2E_PROVIDER="local"
+ENV E2E_PARALLEL="1"
+ENV RESULTS_DIR="/tmp/results"
+ENV KUBECONFIG=""
+
+CMD [ "/bin/bash", "-c", "/run_e2e.sh" ]
--- a/cluster/images/conformance/Makefile
+++ b/cluster/images/conformance/Makefile
@@ -0,0 +1,66 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the conformance image.
+#
+# Usage:
+#   [ARCH=amd64] [REGISTRY="staging-k8s.gcr.io"] make (build|push) VERSION={some_released_version_of_kubernetes}
+
+REGISTRY?=staging-k8s.gcr.io
+ARCH?=amd64
+OUT_DIR?=_output
+GINKGO_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/ginkgo
+KUBECTL_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/kubectl
+E2E_TEST_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/e2e.test
+CLUSTER_DIR?=$(shell pwd)/../../../cluster/
+
+BASEIMAGE=k8s.gcr.io/debian-hyperkube-base-$(ARCH):0.10.2
+TEMP_DIR:=$(shell mktemp -d -t conformanceXXXXXX)
+
+all: build
+
+build:
+
+ifndef VERSION
+    $(error VERSION is undefined)
+endif
+	cp -r ./* ${TEMP_DIR}
+
+	cp ${GINKGO_BIN} ${TEMP_DIR}
+	cp ${KUBECTL_BIN} ${TEMP_DIR}
+	cp ${E2E_TEST_BIN} ${TEMP_DIR}
+	cp -r ${CLUSTER_DIR} ${TEMP_DIR}/cluster
+
+	chmod a+rx ${TEMP_DIR}/ginkgo
+	chmod a+rx ${TEMP_DIR}/kubectl
+	chmod a+rx ${TEMP_DIR}/e2e.test
+
+	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
+
+ifneq ($(ARCH),amd64)
+	# Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel
+	docker run --rm --privileged multiarch/qemu-user-static:register --reset
+endif
+	docker build --pull -t ${REGISTRY}/conformance-${ARCH}:${VERSION} ${TEMP_DIR}
+	rm -rf "${TEMP_DIR}"
+
+push: build
+	docker push ${REGISTRY}/conformance-${ARCH}:${VERSION}
+ifeq ($(ARCH),amd64)
+	docker rmi ${REGISTRY}/conformance:${VERSION} 2>/dev/null || true
+	docker tag ${REGISTRY}/conformance-${ARCH}:${VERSION} ${REGISTRY}/conformance:${VERSION}
+	docker push ${REGISTRY}/conformance:${VERSION}
+endif
+
+.PHONY: build push all
--- a/cluster/images/conformance/OWNERS
+++ b/cluster/images/conformance/OWNERS
@@ -0,0 +1,12 @@
+reviewers:
+  - timothysc
+  - dims
+  - ixdy
+  - spiffxp
+approvers:
+  - timothysc
+  - dims
+  - ixdy
+  - spiffxp
+labels:
+- sig/release
--- a/cluster/images/conformance/README.md
+++ b/cluster/images/conformance/README.md
@@ -0,0 +1,68 @@
+### conformance
+
+`conformance` is a standalone container to launch Kubernetes end-to-end tests, for the purposes of conformance testing.
+`conformance` is built for multiple architectures and _the image is pushed automatically on every release._
+
+#### How to release by hand
+
+```console
+# First, build the binaries
+$ build/run.sh make cross
+
+# Build for linux/amd64 (default)
+# export REGISTRY=$HOST/$ORG to switch from staging-k8s.gcr.io
+
+$ make push VERSION={target_version} ARCH=amd64
+# ---> staging-k8s.gcr.io/conformance-amd64:VERSION
+# ---> staging-k8s.gcr.io/conformance:VERSION (image with backwards-compatible naming)
+
+$ make push VERSION={target_version} ARCH=arm
+# ---> staging-k8s.gcr.io/conformance-arm:VERSION
+
+$ make push VERSION={target_version} ARCH=arm64
+# ---> staging-k8s.gcr.io/conformance-arm64:VERSION
+
+$ make push VERSION={target_version} ARCH=ppc64le
+# ---> staging-k8s.gcr.io/conformance-ppc64le:VERSION
+
+$ make push VERSION={target_version} ARCH=s390x
+# ---> staging-k8s.gcr.io/conformance-s390x:VERSION
+```
+
+If you don't want to push the images, run `make` or `make build` instead
+
+
+#### How to setup RBAC needed
+
+```
+kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=default:default
+```
+
+#### How to run a single test
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+  name: e2e-producer-consumer-test
+spec:
+  containers:
+  - name: conformance-container
+    image: gcr.io/heptio-images/kube-conformance:latest
+    image: staging-k8s.gcr.io/conformance-amd64:v1.12.1
+    imagePullPolicy: IfNotPresent
+    env:
+    - name: E2E_FOCUS
+      value: "Pods should be submitted and removed"
+    volumeMounts:
+    - name: output-volume
+      mountPath: /tmp/results
+  volumes:
+  - name: output-volume
+    hostPath:
+      path: /tmp/results
+  restartPolicy: Never
+```
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/images/conformance/README.md?pixel)]()
--- a/cluster/images/conformance/run_e2e.sh
+++ b/cluster/images/conformance/run_e2e.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Shutdown the tests gracefully then save the results
+shutdown () {
+    E2E_SUITE_PID=$(pgrep e2e.test)
+    echo "sending TERM to ${E2E_SUITE_PID}"
+    kill -s TERM "${E2E_SUITE_PID}"
+
+    # Kind of a hack to wait for this pid to finish.
+    # Since it's not a child of this shell we cannot use wait.
+    tail --pid ${E2E_SUITE_PID} -f /dev/null
+    saveResults
+}
+
+saveResults() {
+    cd "${RESULTS_DIR}" || exit
+    tar -czf e2e.tar.gz ./*
+    # mark the done file as a termination notice.
+    echo -n "${RESULTS_DIR}/e2e.tar.gz" > "${RESULTS_DIR}/done"
+}
+
+# We get the TERM from kubernetes and handle it gracefully
+trap shutdown TERM
+
+ginkgo_args=(
+    "--focus=${E2E_FOCUS}"
+    "--skip=${E2E_SKIP}"
+    "--noColor=true"
+)
+
+case ${E2E_PARALLEL} in
+    'y'|'Y')           ginkgo_args+=("--nodes=25") ;;
+    [1-9]|[1-9][0-9]*) ginkgo_args+=("--nodes=${E2E_PARALLEL}") ;;
+esac
+
+echo "/usr/local/bin/ginkgo ${ginkgo_args[@]} /usr/local/bin/e2e.test -- --disable-log-dump --repo-root=/kubernetes --provider=\"${E2E_PROVIDER}\" --report-dir=\"${RESULTS_DIR}\" --kubeconfig=\"${KUBECONFIG}\""
+/usr/local/bin/ginkgo "${ginkgo_args[@]}" /usr/local/bin/e2e.test -- --disable-log-dump --repo-root=/kubernetes --provider="${E2E_PROVIDER}" --report-dir="${RESULTS_DIR}" --kubeconfig="${KUBECONFIG}" | tee ${RESULTS_DIR}/e2e.log &
+# $! is the pid of tee, not ginkgo
+wait $(pgrep ginkgo)
+saveResults