Adds an optional golang runner to the conformance test image

Adds a go app which runs the e2e tests with ginkgo.

 - Supports all the existing env vars of the bash script
 - Improved flow control to avoid and better report issues
regarding the process PID
 - Adds flags for modifying where to find the test binary and
ginkgo binary so that you can run it locally
 - Adds 3 flags for specifying extra args before the double-dash,
extra args after the double-dash, and the seperator to use between
values in those env vars. This allows setting arbitrary, complex
values for use on the command such as flags which include spaces
or other characters.

build/BUILD

@@ -172,6 +172,7 @@ filegroup(
             "//cmd/linkcheck",
             "//test/e2e:e2e.test_binary",
             "//vendor/github.com/onsi/ginkgo/ginkgo",
+            "//cluster/images/conformance/go-runner",
         ],
     )),
 )

cluster/images/conformance/BUILD

@@ -13,6 +13,7 @@ container_layer(
     name = "bins",
     directory = "/usr/local/bin",
     files = [
+        "//cluster/images/conformance/go-runner",
         "//cmd/kubectl",
         "//test/e2e:e2e.test_binary",
         "//vendor/github.com/onsi/ginkgo/ginkgo",
@@ -62,7 +63,10 @@ filegroup(
 
 filegroup(
     name = "all-srcs",
-    srcs = [":package-srcs"],
+    srcs = [
+        ":package-srcs",
+        "//cluster/images/conformance/go-runner:all-srcs",
+    ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )

cluster/images/conformance/Dockerfile

@@ -18,6 +18,7 @@ COPY ginkgo /usr/local/bin/
 COPY e2e.test /usr/local/bin/
 COPY kubectl /usr/local/bin/
 COPY run_e2e.sh /run_e2e.sh
+COPY gorunner /gorunner
 COPY cluster /kubernetes/cluster
 WORKDIR /usr/local/bin
 

cluster/images/conformance/Makefile

@@ -27,6 +27,7 @@ DOCKERIZED_OUTPUT_PATH=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(A
 GINKGO_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/ginkgo && echo $(LOCAL_OUTPUT_PATH)/ginkgo || echo $(DOCKERIZED_OUTPUT_PATH)/ginkgo)
 KUBECTL_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/kubectl && echo $(LOCAL_OUTPUT_PATH)/kubectl || echo $(DOCKERIZED_OUTPUT_PATH)/kubectl)
 E2E_TEST_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/e2e.test && echo $(LOCAL_OUTPUT_PATH)/e2e.test || echo $(DOCKERIZED_OUTPUT_PATH)/e2e.test)
+E2E_GO_RUNNER_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/go-runner && echo $(LOCAL_OUTPUT_PATH)/go-runner || echo $(DOCKERIZED_OUTPUT_PATH)/go-runner)
 
 CLUSTER_DIR?=$(shell pwd)/../../../cluster/
 
@@ -45,11 +46,13 @@ endif
 	cp ${GINKGO_BIN} ${TEMP_DIR}
 	cp ${KUBECTL_BIN} ${TEMP_DIR}
 	cp ${E2E_TEST_BIN} ${TEMP_DIR}
+	cp ${E2E_GO_RUNNER_BIN} ${TEMP_DIR}/gorunner
 	cp -r ${CLUSTER_DIR} ${TEMP_DIR}/cluster
 
 	chmod a+rx ${TEMP_DIR}/ginkgo
 	chmod a+rx ${TEMP_DIR}/kubectl
 	chmod a+rx ${TEMP_DIR}/e2e.test
+	chmod a+rx ${TEMP_DIR}/gorunner
 
 	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
 

cluster/images/conformance/README.md

@@ -7,7 +7,7 @@
 
 ```console
 # First, build the binaries by running make from the root directory
-$ make WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo cmd/kubectl"
+$ make WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo cmd/kubectl cluster/images/conformance/go-runner"
 
 # Build for linux/amd64 (default)
 # export REGISTRY=$HOST/$ORG to switch from k8s.gcr.io

cluster/images/conformance/go-runner/BUILD

@@ -0,0 +1,47 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cmd.go",
+        "const.go",
+        "e2erunner.go",
+        "env.go",
+        "tar.go",
+    ],
+    importpath = "k8s.io/kubernetes/cluster/images/conformance/go-runner",
+    visibility = ["//visibility:private"],
+    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+)
+
+go_binary(
+    name = "go-runner",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "cmd_test.go",
+        "env_test.go",
+        "tar_test.go",
+    ],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+)

cluster/images/conformance/go-runner/Makefile

@@ -0,0 +1,27 @@
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HOST_GOOS ?= $(shell go env GOOS)
+HOST_GOARCH ?= $(shell go env GOARCH)
+GO_BUILD ?= go build
+
+.PHONY: all build clean
+
+all: build
+
+build:
+	$(GO_BUILD)
+
+clean:
+	rm done e2e.log e2e.tar.gz go-runner

cluster/images/conformance/go-runner/cmd.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+)
+
+// getCmd uses the given environment to form the ginkgo command to run tests. It will
+// set the stdout/stderr to the given writer.
+func getCmd(env Getenver, w io.Writer) *exec.Cmd {
+	ginkgoArgs := []string{}
+
+	// The logic of the parallel env var impacting the skip value necessitates it
+	// being placed before the rest of the flag resolution.
+	skip := env.Getenv(skipEnvKey)
+	switch env.Getenv(parallelEnvKey) {
+	case "y", "Y", "true":
+		ginkgoArgs = append(ginkgoArgs, "--p")
+		if len(skip) == 0 {
+			skip = serialTestsRegexp
+		}
+	}
+
+	ginkgoArgs = append(ginkgoArgs, []string{
+		"--focus=" + env.Getenv(focusEnvKey),
+		"--skip=" + skip,
+		"--noColor=true",
+	}...)
+
+	extraArgs := []string{
+		"--disable-log-dump",
+		"--repo-root=/kubernetes",
+		"--provider=" + env.Getenv(providerEnvKey),
+		"--report-dir=" + env.Getenv(resultsDirEnvKey),
+		"--kubeconfig=" + env.Getenv(kubeconfigEnvKey),
+	}
+
+	// Extra args handling
+	sep := " "
+	if len(env.Getenv(extraArgsSeparaterEnvKey)) > 0 {
+		sep = env.Getenv(extraArgsSeparaterEnvKey)
+	}
+
+	if len(env.Getenv(extraGinkgoArgsEnvKey)) > 0 {
+		ginkgoArgs = append(ginkgoArgs, strings.Split(env.Getenv(extraGinkgoArgsEnvKey), sep)...)
+	}
+
+	if len(env.Getenv(extraArgsEnvKey)) > 0 {
+		fmt.Printf("sep is %q args are %q", sep, env.Getenv(extraArgsEnvKey))
+		fmt.Println("split", strings.Split(env.Getenv(extraArgsEnvKey), sep))
+		extraArgs = append(extraArgs, strings.Split(env.Getenv(extraArgsEnvKey), sep)...)
+	}
+
+	if len(env.Getenv(dryRunEnvKey)) > 0 {
+		ginkgoArgs = append(ginkgoArgs, "--dryRun=true")
+	}
+
+	args := []string{}
+	args = append(args, ginkgoArgs...)
+	args = append(args, env.Getenv(testBinEnvKey))
+	args = append(args, "--")
+	args = append(args, extraArgs...)
+
+	cmd := exec.Command(env.Getenv(ginkgoEnvKey), args...)
+	cmd.Stdout = w
+	cmd.Stderr = w
+	return cmd
+}
+
+// cmdInfo generates a useful look at what the command is for printing/debug.
+func cmdInfo(cmd *exec.Cmd) string {
+	return fmt.Sprintf(
+		`Command env: %v
+Run from directory: %v
+Executable path: %v
+Args (comma-delimited): %v`, cmd.Env, cmd.Dir, cmd.Path, strings.Join(cmd.Args, ","),
+	)
+}

cluster/images/conformance/go-runner/cmd_test.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+	"reflect"
+	"testing"
+)
+
+func TestGetCmd(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		env        Getenver
+		expectArgs []string
+	}{
+		{
+			desc: "Default",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:  "ginkgobin",
+					testBinEnvKey: "testbin",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin",
+				"--focus=", "--skip=",
+				"--noColor=true", "testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=", "--report-dir=", "--kubeconfig=",
+			},
+		}, {
+			desc: "Filling in defaults",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:     "ginkgobin",
+					testBinEnvKey:    "testbin",
+					focusEnvKey:      "focus",
+					skipEnvKey:       "skip",
+					providerEnvKey:   "provider",
+					resultsDirEnvKey: "results",
+					kubeconfigEnvKey: "kubeconfig",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin",
+				"--focus=focus", "--skip=skip",
+				"--noColor=true", "testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=provider", "--report-dir=results", "--kubeconfig=kubeconfig",
+			},
+		}, {
+			desc: "Parallel gets set and skips serial",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:   "ginkgobin",
+					testBinEnvKey:  "testbin",
+					parallelEnvKey: "true",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin", "--p",
+				"--focus=", "--skip=[Serial]",
+				"--noColor=true", "testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=", "--report-dir=", "--kubeconfig=",
+			},
+		}, {
+			desc: "Arbitrary options before and after double dash split by space",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:          "ginkgobin",
+					testBinEnvKey:         "testbin",
+					extraArgsEnvKey:       "--extra=1 --extra=2",
+					extraGinkgoArgsEnvKey: "--ginkgo1 --ginkgo2",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin", "--focus=", "--skip=",
+				"--noColor=true", "--ginkgo1", "--ginkgo2",
+				"testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=", "--report-dir=", "--kubeconfig=",
+				"--extra=1", "--extra=2",
+			},
+		}, {
+			desc: "Arbitrary options can be split by other tokens",
+			env: &explicitEnv{
+				vals: map[string]string{
+					ginkgoEnvKey:             "ginkgobin",
+					testBinEnvKey:            "testbin",
+					extraArgsEnvKey:          "--extra=value with spaces:--extra=value with % anything!$$",
+					extraGinkgoArgsEnvKey:    `--ginkgo='with "quotes" and ':--ginkgo2=true$(foo)`,
+					extraArgsSeparaterEnvKey: ":",
+				},
+			},
+			expectArgs: []string{
+				"ginkgobin", "--focus=", "--skip=",
+				"--noColor=true", `--ginkgo='with "quotes" and '`, "--ginkgo2=true$(foo)",
+				"testbin", "--",
+				"--disable-log-dump", "--repo-root=/kubernetes",
+				"--provider=", "--report-dir=", "--kubeconfig=",
+				"--extra=value with spaces", "--extra=value with % anything!$$",
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			c := getCmd(tc.env, os.Stdout)
+			if !reflect.DeepEqual(c.Args, tc.expectArgs) {
+				t.Errorf("Expected args %q but got %q", tc.expectArgs, c.Args)
+			}
+		})
+	}
+}

cluster/images/conformance/go-runner/const.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+const (
+	// resultsTarballName is the name of the tarball we create with all the results.
+	resultsTarballName = "e2e.tar.gz"
+
+	// doneFileName is the name of the file that signals to the Sonobuoy worker we are
+	// done. The file should contain the path to the results file.
+	doneFileName = "done"
+
+	// resultsDirEnvKey is the env var which stores which directory to put the donefile
+	// and results into. It is a shared, mounted volume between the plugin and Sonobuoy.
+	resultsDirEnvKey = "RESULTS_DIR"
+
+	// logFileName is the name of the file which stdout is tee'd to.
+	logFileName = "e2e.log"
+
+	// Misc env vars which were explicitly supported prior to the go runner.
+	dryRunEnvKey     = "E2E_DRYRUN"
+	parallelEnvKey   = "E2E_PARALLEL"
+	focusEnvKey      = "E2E_FOCUS"
+	skipEnvKey       = "E2E_SKIP"
+	providerEnvKey   = "E2E_PROVIDER"
+	kubeconfigEnvKey = "KUBECONFIG"
+	ginkgoEnvKey     = "GINKGO_BIN"
+	testBinEnvKey    = "TEST_BIN"
+
+	// extraGinkgoArgsEnvKey, if set, will is a list of other arguments to pass to ginkgo.
+	// These are passed before the test binary and include things like `--afterSuiteHook`.
+	extraGinkgoArgsEnvKey = "E2E_EXTRA_GINKGO_ARGS"
+
+	// extraArgsEnvKey, if set, will is a list of other arguments to pass to the tests.
+	// These are passed after the `--` and include things like `--provider`.
+	extraArgsEnvKey = "E2E_EXTRA_ARGS"
+
+	// extraArgsSeparaterEnvKey specifies how to split the extra args values. If unset,
+	// it will default to splitting by spaces.
+	extraArgsSeparaterEnvKey = "E2E_EXTRA_ARGS_SEP"
+
+	defaultSkip         = ""
+	defaultFocus        = "[Conformance]"
+	defaultProvider     = "local"
+	defaultParallel     = "1"
+	defaultResultsDir   = "/tmp/results"
+	defaultGinkgoBinary = "/usr/local/bin/ginkgo"
+	defaultTestBinary   = "/usr/local/bin/e2e.test"
+
+	// serialTestsRegexp is the default skip value if running in parallel. Will not
+	// override an explicit E2E_SKIP value.
+	serialTestsRegexp = "[Serial]"
+)

cluster/images/conformance/go-runner/e2erunner.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/signal"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+func main() {
+	env := envWithDefaults(map[string]string{
+		resultsDirEnvKey: defaultResultsDir,
+		skipEnvKey:       defaultSkip,
+		focusEnvKey:      defaultFocus,
+		providerEnvKey:   defaultProvider,
+		parallelEnvKey:   defaultParallel,
+		ginkgoEnvKey:     defaultGinkgoBinary,
+		testBinEnvKey:    defaultTestBinary,
+	})
+
+	if err := configureAndRunWithEnv(env); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// configureAndRunWithEnv uses the given environment to configure and then start the test run.
+// It will handle TERM signals gracefully and kill the test process and will
+// save the logs/results to the location specified via the RESULTS_DIR environment
+// variable.
+func configureAndRunWithEnv(env Getenver) error {
+	// Ensure we save results regardless of other errors. This helps any
+	// consumer who may be polling for the results.
+	resultsDir := env.Getenv(resultsDirEnvKey)
+	defer saveResults(resultsDir)
+
+	// Print the output to stdout and a logfile which will be returned
+	// as part of the results tarball.
+	logFilePath := filepath.Join(resultsDir, logFileName)
+	logFile, err := os.Create(logFilePath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create log file %v", logFilePath)
+	}
+	mw := io.MultiWriter(os.Stdout, logFile)
+	cmd := getCmd(env, mw)
+
+	log.Printf("Running command:\n%v\n", cmdInfo(cmd))
+	err = cmd.Start()
+	if err != nil {
+		return errors.Wrap(err, "starting command")
+	}
+
+	// Handle signals and shutdown process gracefully.
+	go setupSigHandler(cmd.Process.Pid)
+	return errors.Wrap(cmd.Wait(), "running command")
+}
+
+// setupSigHandler will kill the process identified by the given PID if it
+// gets a TERM signal.
+func setupSigHandler(pid int) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+
+	// Block until a signal is received.
+	log.Println("Now listening for interrupts")
+	s := <-c
+	log.Printf("Got signal: %v. Shutting down test process (PID: %v)\n", s, pid)
+	p, err := os.FindProcess(pid)
+	if err != nil {
+		log.Printf("Could not find process %v to shut down.\n", pid)
+		return
+	}
+	if err := p.Signal(s); err != nil {
+		log.Printf("Failed to signal test process to terminate: %v\n", err)
+		return
+	}
+	log.Printf("Signalled process %v to terminate successfully.\n", pid)
+}
+
+// saveResults will tar the results directory and write the resulting tarball path
+// into the donefile.
+func saveResults(resultsDir string) error {
+	log.Printf("Saving results at %v\n", resultsDir)
+
+	err := tarDir(resultsDir, filepath.Join(resultsDir, resultsTarballName))
+	if err != nil {
+		return errors.Wrapf(err, "tar directory %v", resultsDir)
+	}
+
+	doneFile := filepath.Join(resultsDir, doneFileName)
+
+	resultsTarball := filepath.Join(resultsDir, resultsTarballName)
+	resultsTarball, err = filepath.Abs(resultsTarball)
+	if err != nil {
+		return errors.Wrapf(err, "failed to find absolute path for %v", resultsTarball)
+	}
+
+	return errors.Wrap(
+		ioutil.WriteFile(doneFile, []byte(resultsTarball), os.FileMode(0777)),
+		"writing donefile",
+	)
+}

cluster/images/conformance/go-runner/env.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+)
+
+// Getenver is the interface we use to mock out the env for easier testing. OS env
+// vars can't be as easily tested since internally it uses sync.Once.
+type Getenver interface {
+	Getenv(string) string
+}
+
+// osEnv uses the actual os.Getenv methods to lookup values.
+type osEnv struct{}
+
+// Getenv gets the value of the requested environment variable.
+func (*osEnv) Getenv(s string) string {
+	return os.Getenv(s)
+}
+
+// explicitEnv uses a map instead of os.Getenv methods to lookup values.
+type explicitEnv struct {
+	vals map[string]string
+}
+
+// Getenv returns the value of the requested environment variable (in this
+// implementation, really just a map lookup).
+func (e *explicitEnv) Getenv(s string) string {
+	return e.vals[s]
+}
+
+// defaultOSEnv uses a Getenver to lookup values but if it does
+// not have that value, it falls back to its internal set of defaults.
+type defaultEnver struct {
+	firstChoice Getenver
+	defaults    map[string]string
+}
+
+// Getenv returns the value of the environment variable or its default if unset.
+func (e *defaultEnver) Getenv(s string) string {
+	v := e.firstChoice.Getenv(s)
+	if len(v) == 0 {
+		return e.defaults[s]
+	}
+	return v
+}
+
+func envWithDefaults(defaults map[string]string) Getenver {
+	return &defaultEnver{firstChoice: &osEnv{}, defaults: defaults}
+}

cluster/images/conformance/go-runner/env_test.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+	"testing"
+)
+
+func TestEnv(t *testing.T) {
+	testCases := []struct {
+		desc    string
+		preHook func()
+		env     Getenver
+		expect  map[string]string
+	}{
+		{
+			desc: "OS env",
+			env:  &osEnv{},
+			preHook: func() {
+				os.Setenv("key1", "1")
+			},
+			expect: map[string]string{"key1": "1"},
+		}, {
+			desc: "OS env falls defaults to empty",
+			env:  &osEnv{},
+			preHook: func() {
+				os.Unsetenv("key1")
+			},
+			expect: map[string]string{"key1": ""},
+		}, {
+			desc: "First choice of env respected",
+			env: &defaultEnver{
+				firstChoice: &explicitEnv{
+					vals: map[string]string{
+						"key1": "1",
+					},
+				},
+				defaults: map[string]string{
+					"key1": "default1",
+					"key2": "default2",
+				},
+			},
+			expect: map[string]string{
+				"key1": "1",
+				"key2": "default2",
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			for k, expectVal := range tc.expect {
+				if tc.preHook != nil {
+					tc.preHook()
+				}
+				val := tc.env.Getenv(k)
+				if val != expectVal {
+					t.Errorf("Expected %q but got %q", expectVal, val)
+				}
+			}
+		})
+	}
+}

cluster/images/conformance/go-runner/tar.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// tarDir takes a source and variable writers and walks 'source' writing each file
+// found to the tar writer.
+func tarDir(dir, outpath string) error {
+	// ensure the src actually exists before trying to tar it
+	if _, err := os.Stat(dir); err != nil {
+		return errors.Wrapf(err, "tar unable to stat directory %v", dir)
+	}
+
+	outfile, err := os.Create(outpath)
+	if err != nil {
+		return errors.Wrapf(err, "creating tarball %v", outpath)
+	}
+	defer outfile.Close()
+
+	gzw := gzip.NewWriter(outfile)
+	defer gzw.Close()
+
+	tw := tar.NewWriter(gzw)
+	defer tw.Close()
+
+	return filepath.Walk(dir, func(file string, fi os.FileInfo, err error) error {
+		// Return on any error.
+		if err != nil {
+			return err
+		}
+
+		// Only write regular files and don't include the archive itself.
+		if !fi.Mode().IsRegular() || filepath.Join(dir, fi.Name()) == outpath {
+			return nil
+		}
+
+		// Create a new dir/file header.
+		header, err := tar.FileInfoHeader(fi, fi.Name())
+		if err != nil {
+			return errors.Wrapf(err, "creating file info header %v", fi.Name())
+		}
+
+		// Update the name to correctly reflect the desired destination when untaring.
+		header.Name = strings.TrimPrefix(strings.Replace(file, dir, "", -1), string(filepath.Separator))
+		if err := tw.WriteHeader(header); err != nil {
+			return errors.Wrapf(err, "writing header for tarball %v", header.Name)
+		}
+
+		// Open files, copy into tarfile, and close.
+		f, err := os.Open(file)
+		if err != nil {
+			return errors.Wrapf(err, "opening file %v for writing into tarball", file)
+		}
+		defer f.Close()
+
+		_, err = io.Copy(tw, f)
+		return errors.Wrapf(err, "creating file %v contents into tarball", file)
+	})
+}

cluster/images/conformance/go-runner/tar_test.go

@@ -0,0 +1,123 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/pkg/errors"
+)
+
+func TestTar(t *testing.T) {
+	testCases := []struct {
+		desc      string
+		dir       string
+		outpath   string
+		expectErr string
+		expect    map[string]string
+	}{
+		{
+			desc:    "Contents preserved and no self-reference",
+			dir:     "testdata/tartest",
+			outpath: "testdata/tartest/out.tar.gz",
+			expect: map[string]string{
+				"file1":        "file1 data",
+				"file2":        "file2 data",
+				"subdir/file4": "file4 data",
+			},
+		}, {
+			desc:      "Errors if directory does not exist",
+			dir:       "testdata/does-not-exist",
+			outpath:   "testdata/tartest/out.tar.gz",
+			expectErr: "tar unable to stat directory",
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			err := tarDir(tc.dir, tc.outpath)
+			switch {
+			case err != nil && len(tc.expectErr) == 0:
+				t.Fatalf("Expected nil error but got %q", err)
+			case err != nil && len(tc.expectErr) > 0:
+				if !strings.Contains(fmt.Sprint(err), tc.expectErr) {
+					t.Errorf("Expected error \n\t%q\nbut got\n\t%q", tc.expectErr, err)
+				}
+				return
+			case err == nil && len(tc.expectErr) > 0:
+				t.Fatalf("Expected error %q but got nil", tc.expectErr)
+			default:
+				// No error
+			}
+
+			data, err := readAllTar(tc.outpath)
+			if !reflect.DeepEqual(data, tc.expect) {
+				t.Errorf("Expected data %v but got %v", tc.expect, data)
+			}
+		})
+	}
+}
+
+// readAllTar walks all of the files in the archive. It returns a map
+// of filenames and their contents and any error encountered.
+func readAllTar(tarPath string) (map[string]string, error) {
+	tarPath, err := filepath.Abs(tarPath)
+	if err != nil {
+		return nil, err
+	}
+
+	fileReader, err := os.Open(tarPath)
+	if err != nil {
+		return nil, err
+	}
+	defer fileReader.Close()
+
+	gzStream, err := gzip.NewReader(fileReader)
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't uncompress reader")
+	}
+	defer gzStream.Close()
+
+	// Open and iterate through the files in the archive.
+	tr := tar.NewReader(gzStream)
+	fileData := map[string]string{}
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break // End of archive
+		}
+		if err != nil {
+
+			return nil, err
+		}
+
+		b, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return nil, err
+		}
+		fileData[filepath.ToSlash(hdr.Name)] = string(b)
+	}
+	return fileData, nil
+}

cluster/images/conformance/go-runner/testdata/tartest/file1

@@ -0,0 +1 @@
+file1 data

cluster/images/conformance/go-runner/testdata/tartest/file2

@@ -0,0 +1 @@
+file2 data

cluster/images/conformance/go-runner/testdata/tartest/subdir/file4

@@ -0,0 +1 @@
+file4 data

cluster/images/conformance/run_e2e.sh

@@ -36,6 +36,14 @@ saveResults() {
     echo -n "${RESULTS_DIR}/e2e.tar.gz" > "${RESULTS_DIR}/done"
 }
 
+# Optional Golang runner alternative to the bash script.
+# Entry provided via env var to simplify invocation. 
+if [[ -n ${E2E_USE_GO_RUNNER:-} ]]; then
+    set -x
+    /gorunner
+    exit $?
+fi
+
 # We get the TERM from kubernetes and handle it gracefully
 trap shutdown TERM
 

hack/dev-push-conformance.sh

@@ -42,7 +42,7 @@ IMAGE="${REGISTRY}/conformance-amd64:${VERSION}"
 
 kube::build::verify_prereqs
 kube::build::build_image
-kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl"
+kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl cluster/images/conformance/go-runner"
 kube::build::copy_output
 
 make -C "${KUBE_ROOT}/cluster/images/conformance" build

hack/lib/golang.sh

@@ -264,6 +264,7 @@ kube::golang::test_targets() {
     cmd/linkcheck
     vendor/github.com/onsi/ginkgo/ginkgo
     test/e2e/e2e.test
+    cluster/images/conformance/go-runner
   )
   echo "${targets[@]}"
 }