From b3f5a086ab40c5810995f5d8829ac68d8dd49ae6 Mon Sep 17 00:00:00 2001 From: John Schnake Date: Thu, 20 Jun 2019 22:34:10 -0500 Subject: [PATCH] Adds an optional golang runner to the conformance test image Adds a go app which runs the e2e tests with ginkgo. - Supports all the existing env vars of the bash script - Improved flow control to avoid and better report issues regarding the process PID - Adds flags for modifying where to find the test binary and ginkgo binary so that you can run it locally - Adds 3 flags for specifying extra args before the double-dash, extra args after the double-dash, and the seperator to use between values in those env vars. This allows setting arbitrary, complex values for use on the command such as flags which include spaces or other characters. --- build/BUILD | 1 + cluster/images/conformance/BUILD | 6 +- cluster/images/conformance/Dockerfile | 1 + cluster/images/conformance/Makefile | 3 + cluster/images/conformance/README.md | 2 +- cluster/images/conformance/go-runner/BUILD | 47 +++++++ cluster/images/conformance/go-runner/Makefile | 27 ++++ .../images/conformance/go-runner/README.md | 0 cluster/images/conformance/go-runner/cmd.go | 96 +++++++++++++ .../images/conformance/go-runner/cmd_test.go | 129 ++++++++++++++++++ cluster/images/conformance/go-runner/const.go | 67 +++++++++ .../images/conformance/go-runner/e2erunner.go | 121 ++++++++++++++++ cluster/images/conformance/go-runner/env.go | 66 +++++++++ .../images/conformance/go-runner/env_test.go | 77 +++++++++++ cluster/images/conformance/go-runner/tar.go | 83 +++++++++++ .../images/conformance/go-runner/tar_test.go | 123 +++++++++++++++++ .../go-runner/testdata/tartest/file1 | 1 + .../go-runner/testdata/tartest/file2 | 1 + .../go-runner/testdata/tartest/subdir/file4 | 1 + cluster/images/conformance/run_e2e.sh | 8 ++ hack/dev-push-conformance.sh | 2 +- hack/lib/golang.sh | 1 + 22 files changed, 860 insertions(+), 3 deletions(-) create mode 100644 cluster/images/conformance/go-runner/BUILD create mode 100644 cluster/images/conformance/go-runner/Makefile create mode 100644 cluster/images/conformance/go-runner/README.md create mode 100644 cluster/images/conformance/go-runner/cmd.go create mode 100644 cluster/images/conformance/go-runner/cmd_test.go create mode 100644 cluster/images/conformance/go-runner/const.go create mode 100644 cluster/images/conformance/go-runner/e2erunner.go create mode 100644 cluster/images/conformance/go-runner/env.go create mode 100644 cluster/images/conformance/go-runner/env_test.go create mode 100644 cluster/images/conformance/go-runner/tar.go create mode 100644 cluster/images/conformance/go-runner/tar_test.go create mode 100644 cluster/images/conformance/go-runner/testdata/tartest/file1 create mode 100644 cluster/images/conformance/go-runner/testdata/tartest/file2 create mode 100644 cluster/images/conformance/go-runner/testdata/tartest/subdir/file4 diff --git a/build/BUILD b/build/BUILD index 3ac15d07051..5086127c983 100644 --- a/build/BUILD +++ b/build/BUILD @@ -172,6 +172,7 @@ filegroup( "//cmd/linkcheck", "//test/e2e:e2e.test_binary", "//vendor/github.com/onsi/ginkgo/ginkgo", + "//cluster/images/conformance/go-runner", ], )), ) diff --git a/cluster/images/conformance/BUILD b/cluster/images/conformance/BUILD index 14fd8f98595..b7b328f1abe 100644 --- a/cluster/images/conformance/BUILD +++ b/cluster/images/conformance/BUILD @@ -13,6 +13,7 @@ container_layer( name = "bins", directory = "/usr/local/bin", files = [ + "//cluster/images/conformance/go-runner", "//cmd/kubectl", "//test/e2e:e2e.test_binary", "//vendor/github.com/onsi/ginkgo/ginkgo", @@ -62,7 +63,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//cluster/images/conformance/go-runner:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/cluster/images/conformance/Dockerfile b/cluster/images/conformance/Dockerfile index 5f12885fe7a..2466bbace45 100644 --- a/cluster/images/conformance/Dockerfile +++ b/cluster/images/conformance/Dockerfile @@ -18,6 +18,7 @@ COPY ginkgo /usr/local/bin/ COPY e2e.test /usr/local/bin/ COPY kubectl /usr/local/bin/ COPY run_e2e.sh /run_e2e.sh +COPY gorunner /gorunner COPY cluster /kubernetes/cluster WORKDIR /usr/local/bin diff --git a/cluster/images/conformance/Makefile b/cluster/images/conformance/Makefile index e75871e6f20..89335650cfa 100644 --- a/cluster/images/conformance/Makefile +++ b/cluster/images/conformance/Makefile @@ -27,6 +27,7 @@ DOCKERIZED_OUTPUT_PATH=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(A GINKGO_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/ginkgo && echo $(LOCAL_OUTPUT_PATH)/ginkgo || echo $(DOCKERIZED_OUTPUT_PATH)/ginkgo) KUBECTL_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/kubectl && echo $(LOCAL_OUTPUT_PATH)/kubectl || echo $(DOCKERIZED_OUTPUT_PATH)/kubectl) E2E_TEST_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/e2e.test && echo $(LOCAL_OUTPUT_PATH)/e2e.test || echo $(DOCKERIZED_OUTPUT_PATH)/e2e.test) +E2E_GO_RUNNER_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/go-runner && echo $(LOCAL_OUTPUT_PATH)/go-runner || echo $(DOCKERIZED_OUTPUT_PATH)/go-runner) CLUSTER_DIR?=$(shell pwd)/../../../cluster/ @@ -45,11 +46,13 @@ endif cp ${GINKGO_BIN} ${TEMP_DIR} cp ${KUBECTL_BIN} ${TEMP_DIR} cp ${E2E_TEST_BIN} ${TEMP_DIR} + cp ${E2E_GO_RUNNER_BIN} ${TEMP_DIR}/gorunner cp -r ${CLUSTER_DIR} ${TEMP_DIR}/cluster chmod a+rx ${TEMP_DIR}/ginkgo chmod a+rx ${TEMP_DIR}/kubectl chmod a+rx ${TEMP_DIR}/e2e.test + chmod a+rx ${TEMP_DIR}/gorunner cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile diff --git a/cluster/images/conformance/README.md b/cluster/images/conformance/README.md index b5df5e8c1db..34080fe3655 100644 --- a/cluster/images/conformance/README.md +++ b/cluster/images/conformance/README.md @@ -7,7 +7,7 @@ ```console # First, build the binaries by running make from the root directory -$ make WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo cmd/kubectl" +$ make WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo cmd/kubectl cluster/images/conformance/go-runner" # Build for linux/amd64 (default) # export REGISTRY=$HOST/$ORG to switch from k8s.gcr.io diff --git a/cluster/images/conformance/go-runner/BUILD b/cluster/images/conformance/go-runner/BUILD new file mode 100644 index 00000000000..55b76f5b871 --- /dev/null +++ b/cluster/images/conformance/go-runner/BUILD @@ -0,0 +1,47 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "cmd.go", + "const.go", + "e2erunner.go", + "env.go", + "tar.go", + ], + importpath = "k8s.io/kubernetes/cluster/images/conformance/go-runner", + visibility = ["//visibility:private"], + deps = ["//vendor/github.com/pkg/errors:go_default_library"], +) + +go_binary( + name = "go-runner", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = [ + "cmd_test.go", + "env_test.go", + "tar_test.go", + ], + data = glob(["testdata/**"]), + embed = [":go_default_library"], + deps = ["//vendor/github.com/pkg/errors:go_default_library"], +) diff --git a/cluster/images/conformance/go-runner/Makefile b/cluster/images/conformance/go-runner/Makefile new file mode 100644 index 00000000000..556fdc42080 --- /dev/null +++ b/cluster/images/conformance/go-runner/Makefile @@ -0,0 +1,27 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HOST_GOOS ?= $(shell go env GOOS) +HOST_GOARCH ?= $(shell go env GOARCH) +GO_BUILD ?= go build + +.PHONY: all build clean + +all: build + +build: + $(GO_BUILD) + +clean: + rm done e2e.log e2e.tar.gz go-runner diff --git a/cluster/images/conformance/go-runner/README.md b/cluster/images/conformance/go-runner/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cluster/images/conformance/go-runner/cmd.go b/cluster/images/conformance/go-runner/cmd.go new file mode 100644 index 00000000000..448a5055081 --- /dev/null +++ b/cluster/images/conformance/go-runner/cmd.go @@ -0,0 +1,96 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "io" + "os/exec" + "strings" +) + +// getCmd uses the given environment to form the ginkgo command to run tests. It will +// set the stdout/stderr to the given writer. +func getCmd(env Getenver, w io.Writer) *exec.Cmd { + ginkgoArgs := []string{} + + // The logic of the parallel env var impacting the skip value necessitates it + // being placed before the rest of the flag resolution. + skip := env.Getenv(skipEnvKey) + switch env.Getenv(parallelEnvKey) { + case "y", "Y", "true": + ginkgoArgs = append(ginkgoArgs, "--p") + if len(skip) == 0 { + skip = serialTestsRegexp + } + } + + ginkgoArgs = append(ginkgoArgs, []string{ + "--focus=" + env.Getenv(focusEnvKey), + "--skip=" + skip, + "--noColor=true", + }...) + + extraArgs := []string{ + "--disable-log-dump", + "--repo-root=/kubernetes", + "--provider=" + env.Getenv(providerEnvKey), + "--report-dir=" + env.Getenv(resultsDirEnvKey), + "--kubeconfig=" + env.Getenv(kubeconfigEnvKey), + } + + // Extra args handling + sep := " " + if len(env.Getenv(extraArgsSeparaterEnvKey)) > 0 { + sep = env.Getenv(extraArgsSeparaterEnvKey) + } + + if len(env.Getenv(extraGinkgoArgsEnvKey)) > 0 { + ginkgoArgs = append(ginkgoArgs, strings.Split(env.Getenv(extraGinkgoArgsEnvKey), sep)...) + } + + if len(env.Getenv(extraArgsEnvKey)) > 0 { + fmt.Printf("sep is %q args are %q", sep, env.Getenv(extraArgsEnvKey)) + fmt.Println("split", strings.Split(env.Getenv(extraArgsEnvKey), sep)) + extraArgs = append(extraArgs, strings.Split(env.Getenv(extraArgsEnvKey), sep)...) + } + + if len(env.Getenv(dryRunEnvKey)) > 0 { + ginkgoArgs = append(ginkgoArgs, "--dryRun=true") + } + + args := []string{} + args = append(args, ginkgoArgs...) + args = append(args, env.Getenv(testBinEnvKey)) + args = append(args, "--") + args = append(args, extraArgs...) + + cmd := exec.Command(env.Getenv(ginkgoEnvKey), args...) + cmd.Stdout = w + cmd.Stderr = w + return cmd +} + +// cmdInfo generates a useful look at what the command is for printing/debug. +func cmdInfo(cmd *exec.Cmd) string { + return fmt.Sprintf( + `Command env: %v +Run from directory: %v +Executable path: %v +Args (comma-delimited): %v`, cmd.Env, cmd.Dir, cmd.Path, strings.Join(cmd.Args, ","), + ) +} diff --git a/cluster/images/conformance/go-runner/cmd_test.go b/cluster/images/conformance/go-runner/cmd_test.go new file mode 100644 index 00000000000..0dc24671680 --- /dev/null +++ b/cluster/images/conformance/go-runner/cmd_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "reflect" + "testing" +) + +func TestGetCmd(t *testing.T) { + testCases := []struct { + desc string + env Getenver + expectArgs []string + }{ + { + desc: "Default", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + }, + }, + expectArgs: []string{ + "ginkgobin", + "--focus=", "--skip=", + "--noColor=true", "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=", "--report-dir=", "--kubeconfig=", + }, + }, { + desc: "Filling in defaults", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + focusEnvKey: "focus", + skipEnvKey: "skip", + providerEnvKey: "provider", + resultsDirEnvKey: "results", + kubeconfigEnvKey: "kubeconfig", + }, + }, + expectArgs: []string{ + "ginkgobin", + "--focus=focus", "--skip=skip", + "--noColor=true", "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=provider", "--report-dir=results", "--kubeconfig=kubeconfig", + }, + }, { + desc: "Parallel gets set and skips serial", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + parallelEnvKey: "true", + }, + }, + expectArgs: []string{ + "ginkgobin", "--p", + "--focus=", "--skip=[Serial]", + "--noColor=true", "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=", "--report-dir=", "--kubeconfig=", + }, + }, { + desc: "Arbitrary options before and after double dash split by space", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + extraArgsEnvKey: "--extra=1 --extra=2", + extraGinkgoArgsEnvKey: "--ginkgo1 --ginkgo2", + }, + }, + expectArgs: []string{ + "ginkgobin", "--focus=", "--skip=", + "--noColor=true", "--ginkgo1", "--ginkgo2", + "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=", "--report-dir=", "--kubeconfig=", + "--extra=1", "--extra=2", + }, + }, { + desc: "Arbitrary options can be split by other tokens", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + extraArgsEnvKey: "--extra=value with spaces:--extra=value with % anything!$$", + extraGinkgoArgsEnvKey: `--ginkgo='with "quotes" and ':--ginkgo2=true$(foo)`, + extraArgsSeparaterEnvKey: ":", + }, + }, + expectArgs: []string{ + "ginkgobin", "--focus=", "--skip=", + "--noColor=true", `--ginkgo='with "quotes" and '`, "--ginkgo2=true$(foo)", + "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=", "--report-dir=", "--kubeconfig=", + "--extra=value with spaces", "--extra=value with % anything!$$", + }, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := getCmd(tc.env, os.Stdout) + if !reflect.DeepEqual(c.Args, tc.expectArgs) { + t.Errorf("Expected args %q but got %q", tc.expectArgs, c.Args) + } + }) + } +} diff --git a/cluster/images/conformance/go-runner/const.go b/cluster/images/conformance/go-runner/const.go new file mode 100644 index 00000000000..161c498fb89 --- /dev/null +++ b/cluster/images/conformance/go-runner/const.go @@ -0,0 +1,67 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +const ( + // resultsTarballName is the name of the tarball we create with all the results. + resultsTarballName = "e2e.tar.gz" + + // doneFileName is the name of the file that signals to the Sonobuoy worker we are + // done. The file should contain the path to the results file. + doneFileName = "done" + + // resultsDirEnvKey is the env var which stores which directory to put the donefile + // and results into. It is a shared, mounted volume between the plugin and Sonobuoy. + resultsDirEnvKey = "RESULTS_DIR" + + // logFileName is the name of the file which stdout is tee'd to. + logFileName = "e2e.log" + + // Misc env vars which were explicitly supported prior to the go runner. + dryRunEnvKey = "E2E_DRYRUN" + parallelEnvKey = "E2E_PARALLEL" + focusEnvKey = "E2E_FOCUS" + skipEnvKey = "E2E_SKIP" + providerEnvKey = "E2E_PROVIDER" + kubeconfigEnvKey = "KUBECONFIG" + ginkgoEnvKey = "GINKGO_BIN" + testBinEnvKey = "TEST_BIN" + + // extraGinkgoArgsEnvKey, if set, will is a list of other arguments to pass to ginkgo. + // These are passed before the test binary and include things like `--afterSuiteHook`. + extraGinkgoArgsEnvKey = "E2E_EXTRA_GINKGO_ARGS" + + // extraArgsEnvKey, if set, will is a list of other arguments to pass to the tests. + // These are passed after the `--` and include things like `--provider`. + extraArgsEnvKey = "E2E_EXTRA_ARGS" + + // extraArgsSeparaterEnvKey specifies how to split the extra args values. If unset, + // it will default to splitting by spaces. + extraArgsSeparaterEnvKey = "E2E_EXTRA_ARGS_SEP" + + defaultSkip = "" + defaultFocus = "[Conformance]" + defaultProvider = "local" + defaultParallel = "1" + defaultResultsDir = "/tmp/results" + defaultGinkgoBinary = "/usr/local/bin/ginkgo" + defaultTestBinary = "/usr/local/bin/e2e.test" + + // serialTestsRegexp is the default skip value if running in parallel. Will not + // override an explicit E2E_SKIP value. + serialTestsRegexp = "[Serial]" +) diff --git a/cluster/images/conformance/go-runner/e2erunner.go b/cluster/images/conformance/go-runner/e2erunner.go new file mode 100644 index 00000000000..073b3159c0f --- /dev/null +++ b/cluster/images/conformance/go-runner/e2erunner.go @@ -0,0 +1,121 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "io" + "io/ioutil" + "log" + "os" + "os/signal" + "path/filepath" + + "github.com/pkg/errors" +) + +func main() { + env := envWithDefaults(map[string]string{ + resultsDirEnvKey: defaultResultsDir, + skipEnvKey: defaultSkip, + focusEnvKey: defaultFocus, + providerEnvKey: defaultProvider, + parallelEnvKey: defaultParallel, + ginkgoEnvKey: defaultGinkgoBinary, + testBinEnvKey: defaultTestBinary, + }) + + if err := configureAndRunWithEnv(env); err != nil { + log.Fatal(err) + } +} + +// configureAndRunWithEnv uses the given environment to configure and then start the test run. +// It will handle TERM signals gracefully and kill the test process and will +// save the logs/results to the location specified via the RESULTS_DIR environment +// variable. +func configureAndRunWithEnv(env Getenver) error { + // Ensure we save results regardless of other errors. This helps any + // consumer who may be polling for the results. + resultsDir := env.Getenv(resultsDirEnvKey) + defer saveResults(resultsDir) + + // Print the output to stdout and a logfile which will be returned + // as part of the results tarball. + logFilePath := filepath.Join(resultsDir, logFileName) + logFile, err := os.Create(logFilePath) + if err != nil { + return errors.Wrapf(err, "failed to create log file %v", logFilePath) + } + mw := io.MultiWriter(os.Stdout, logFile) + cmd := getCmd(env, mw) + + log.Printf("Running command:\n%v\n", cmdInfo(cmd)) + err = cmd.Start() + if err != nil { + return errors.Wrap(err, "starting command") + } + + // Handle signals and shutdown process gracefully. + go setupSigHandler(cmd.Process.Pid) + return errors.Wrap(cmd.Wait(), "running command") +} + +// setupSigHandler will kill the process identified by the given PID if it +// gets a TERM signal. +func setupSigHandler(pid int) { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + + // Block until a signal is received. + log.Println("Now listening for interrupts") + s := <-c + log.Printf("Got signal: %v. Shutting down test process (PID: %v)\n", s, pid) + p, err := os.FindProcess(pid) + if err != nil { + log.Printf("Could not find process %v to shut down.\n", pid) + return + } + if err := p.Signal(s); err != nil { + log.Printf("Failed to signal test process to terminate: %v\n", err) + return + } + log.Printf("Signalled process %v to terminate successfully.\n", pid) +} + +// saveResults will tar the results directory and write the resulting tarball path +// into the donefile. +func saveResults(resultsDir string) error { + log.Printf("Saving results at %v\n", resultsDir) + + err := tarDir(resultsDir, filepath.Join(resultsDir, resultsTarballName)) + if err != nil { + return errors.Wrapf(err, "tar directory %v", resultsDir) + } + + doneFile := filepath.Join(resultsDir, doneFileName) + + resultsTarball := filepath.Join(resultsDir, resultsTarballName) + resultsTarball, err = filepath.Abs(resultsTarball) + if err != nil { + return errors.Wrapf(err, "failed to find absolute path for %v", resultsTarball) + } + + return errors.Wrap( + ioutil.WriteFile(doneFile, []byte(resultsTarball), os.FileMode(0777)), + "writing donefile", + ) +} diff --git a/cluster/images/conformance/go-runner/env.go b/cluster/images/conformance/go-runner/env.go new file mode 100644 index 00000000000..c910d9184f9 --- /dev/null +++ b/cluster/images/conformance/go-runner/env.go @@ -0,0 +1,66 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" +) + +// Getenver is the interface we use to mock out the env for easier testing. OS env +// vars can't be as easily tested since internally it uses sync.Once. +type Getenver interface { + Getenv(string) string +} + +// osEnv uses the actual os.Getenv methods to lookup values. +type osEnv struct{} + +// Getenv gets the value of the requested environment variable. +func (*osEnv) Getenv(s string) string { + return os.Getenv(s) +} + +// explicitEnv uses a map instead of os.Getenv methods to lookup values. +type explicitEnv struct { + vals map[string]string +} + +// Getenv returns the value of the requested environment variable (in this +// implementation, really just a map lookup). +func (e *explicitEnv) Getenv(s string) string { + return e.vals[s] +} + +// defaultOSEnv uses a Getenver to lookup values but if it does +// not have that value, it falls back to its internal set of defaults. +type defaultEnver struct { + firstChoice Getenver + defaults map[string]string +} + +// Getenv returns the value of the environment variable or its default if unset. +func (e *defaultEnver) Getenv(s string) string { + v := e.firstChoice.Getenv(s) + if len(v) == 0 { + return e.defaults[s] + } + return v +} + +func envWithDefaults(defaults map[string]string) Getenver { + return &defaultEnver{firstChoice: &osEnv{}, defaults: defaults} +} diff --git a/cluster/images/conformance/go-runner/env_test.go b/cluster/images/conformance/go-runner/env_test.go new file mode 100644 index 00000000000..4229ebe3037 --- /dev/null +++ b/cluster/images/conformance/go-runner/env_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "testing" +) + +func TestEnv(t *testing.T) { + testCases := []struct { + desc string + preHook func() + env Getenver + expect map[string]string + }{ + { + desc: "OS env", + env: &osEnv{}, + preHook: func() { + os.Setenv("key1", "1") + }, + expect: map[string]string{"key1": "1"}, + }, { + desc: "OS env falls defaults to empty", + env: &osEnv{}, + preHook: func() { + os.Unsetenv("key1") + }, + expect: map[string]string{"key1": ""}, + }, { + desc: "First choice of env respected", + env: &defaultEnver{ + firstChoice: &explicitEnv{ + vals: map[string]string{ + "key1": "1", + }, + }, + defaults: map[string]string{ + "key1": "default1", + "key2": "default2", + }, + }, + expect: map[string]string{ + "key1": "1", + "key2": "default2", + }, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + for k, expectVal := range tc.expect { + if tc.preHook != nil { + tc.preHook() + } + val := tc.env.Getenv(k) + if val != expectVal { + t.Errorf("Expected %q but got %q", expectVal, val) + } + } + }) + } +} diff --git a/cluster/images/conformance/go-runner/tar.go b/cluster/images/conformance/go-runner/tar.go new file mode 100644 index 00000000000..2309387bfec --- /dev/null +++ b/cluster/images/conformance/go-runner/tar.go @@ -0,0 +1,83 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +// tarDir takes a source and variable writers and walks 'source' writing each file +// found to the tar writer. +func tarDir(dir, outpath string) error { + // ensure the src actually exists before trying to tar it + if _, err := os.Stat(dir); err != nil { + return errors.Wrapf(err, "tar unable to stat directory %v", dir) + } + + outfile, err := os.Create(outpath) + if err != nil { + return errors.Wrapf(err, "creating tarball %v", outpath) + } + defer outfile.Close() + + gzw := gzip.NewWriter(outfile) + defer gzw.Close() + + tw := tar.NewWriter(gzw) + defer tw.Close() + + return filepath.Walk(dir, func(file string, fi os.FileInfo, err error) error { + // Return on any error. + if err != nil { + return err + } + + // Only write regular files and don't include the archive itself. + if !fi.Mode().IsRegular() || filepath.Join(dir, fi.Name()) == outpath { + return nil + } + + // Create a new dir/file header. + header, err := tar.FileInfoHeader(fi, fi.Name()) + if err != nil { + return errors.Wrapf(err, "creating file info header %v", fi.Name()) + } + + // Update the name to correctly reflect the desired destination when untaring. + header.Name = strings.TrimPrefix(strings.Replace(file, dir, "", -1), string(filepath.Separator)) + if err := tw.WriteHeader(header); err != nil { + return errors.Wrapf(err, "writing header for tarball %v", header.Name) + } + + // Open files, copy into tarfile, and close. + f, err := os.Open(file) + if err != nil { + return errors.Wrapf(err, "opening file %v for writing into tarball", file) + } + defer f.Close() + + _, err = io.Copy(tw, f) + return errors.Wrapf(err, "creating file %v contents into tarball", file) + }) +} diff --git a/cluster/images/conformance/go-runner/tar_test.go b/cluster/images/conformance/go-runner/tar_test.go new file mode 100644 index 00000000000..7df8e0ad7b5 --- /dev/null +++ b/cluster/images/conformance/go-runner/tar_test.go @@ -0,0 +1,123 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/pkg/errors" +) + +func TestTar(t *testing.T) { + testCases := []struct { + desc string + dir string + outpath string + expectErr string + expect map[string]string + }{ + { + desc: "Contents preserved and no self-reference", + dir: "testdata/tartest", + outpath: "testdata/tartest/out.tar.gz", + expect: map[string]string{ + "file1": "file1 data", + "file2": "file2 data", + "subdir/file4": "file4 data", + }, + }, { + desc: "Errors if directory does not exist", + dir: "testdata/does-not-exist", + outpath: "testdata/tartest/out.tar.gz", + expectErr: "tar unable to stat directory", + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + err := tarDir(tc.dir, tc.outpath) + switch { + case err != nil && len(tc.expectErr) == 0: + t.Fatalf("Expected nil error but got %q", err) + case err != nil && len(tc.expectErr) > 0: + if !strings.Contains(fmt.Sprint(err), tc.expectErr) { + t.Errorf("Expected error \n\t%q\nbut got\n\t%q", tc.expectErr, err) + } + return + case err == nil && len(tc.expectErr) > 0: + t.Fatalf("Expected error %q but got nil", tc.expectErr) + default: + // No error + } + + data, err := readAllTar(tc.outpath) + if !reflect.DeepEqual(data, tc.expect) { + t.Errorf("Expected data %v but got %v", tc.expect, data) + } + }) + } +} + +// readAllTar walks all of the files in the archive. It returns a map +// of filenames and their contents and any error encountered. +func readAllTar(tarPath string) (map[string]string, error) { + tarPath, err := filepath.Abs(tarPath) + if err != nil { + return nil, err + } + + fileReader, err := os.Open(tarPath) + if err != nil { + return nil, err + } + defer fileReader.Close() + + gzStream, err := gzip.NewReader(fileReader) + if err != nil { + return nil, errors.Wrap(err, "couldn't uncompress reader") + } + defer gzStream.Close() + + // Open and iterate through the files in the archive. + tr := tar.NewReader(gzStream) + fileData := map[string]string{} + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + + return nil, err + } + + b, err := ioutil.ReadAll(tr) + if err != nil { + return nil, err + } + fileData[filepath.ToSlash(hdr.Name)] = string(b) + } + return fileData, nil +} diff --git a/cluster/images/conformance/go-runner/testdata/tartest/file1 b/cluster/images/conformance/go-runner/testdata/tartest/file1 new file mode 100644 index 00000000000..4365baaa1f1 --- /dev/null +++ b/cluster/images/conformance/go-runner/testdata/tartest/file1 @@ -0,0 +1 @@ +file1 data \ No newline at end of file diff --git a/cluster/images/conformance/go-runner/testdata/tartest/file2 b/cluster/images/conformance/go-runner/testdata/tartest/file2 new file mode 100644 index 00000000000..526a2a0b1d5 --- /dev/null +++ b/cluster/images/conformance/go-runner/testdata/tartest/file2 @@ -0,0 +1 @@ +file2 data \ No newline at end of file diff --git a/cluster/images/conformance/go-runner/testdata/tartest/subdir/file4 b/cluster/images/conformance/go-runner/testdata/tartest/subdir/file4 new file mode 100644 index 00000000000..e8ca25e8c71 --- /dev/null +++ b/cluster/images/conformance/go-runner/testdata/tartest/subdir/file4 @@ -0,0 +1 @@ +file4 data \ No newline at end of file diff --git a/cluster/images/conformance/run_e2e.sh b/cluster/images/conformance/run_e2e.sh index afe8d207a40..e24d8b52c2b 100755 --- a/cluster/images/conformance/run_e2e.sh +++ b/cluster/images/conformance/run_e2e.sh @@ -36,6 +36,14 @@ saveResults() { echo -n "${RESULTS_DIR}/e2e.tar.gz" > "${RESULTS_DIR}/done" } +# Optional Golang runner alternative to the bash script. +# Entry provided via env var to simplify invocation. +if [[ -n ${E2E_USE_GO_RUNNER:-} ]]; then + set -x + /gorunner + exit $? +fi + # We get the TERM from kubernetes and handle it gracefully trap shutdown TERM diff --git a/hack/dev-push-conformance.sh b/hack/dev-push-conformance.sh index 699d0686aea..7dee5a6a582 100755 --- a/hack/dev-push-conformance.sh +++ b/hack/dev-push-conformance.sh @@ -42,7 +42,7 @@ IMAGE="${REGISTRY}/conformance-amd64:${VERSION}" kube::build::verify_prereqs kube::build::build_image -kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl" +kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl cluster/images/conformance/go-runner" kube::build::copy_output make -C "${KUBE_ROOT}/cluster/images/conformance" build diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index e9c3b066925..96eb78b3389 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -264,6 +264,7 @@ kube::golang::test_targets() { cmd/linkcheck vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test + cluster/images/conformance/go-runner ) echo "${targets[@]}" }