diff --git a/build/BUILD b/build/BUILD index 3ac15d07051..5086127c983 100644 --- a/build/BUILD +++ b/build/BUILD @@ -172,6 +172,7 @@ filegroup( "//cmd/linkcheck", "//test/e2e:e2e.test_binary", "//vendor/github.com/onsi/ginkgo/ginkgo", + "//cluster/images/conformance/go-runner", ], )), ) diff --git a/cluster/images/conformance/BUILD b/cluster/images/conformance/BUILD index 14fd8f98595..b7b328f1abe 100644 --- a/cluster/images/conformance/BUILD +++ b/cluster/images/conformance/BUILD @@ -13,6 +13,7 @@ container_layer( name = "bins", directory = "/usr/local/bin", files = [ + "//cluster/images/conformance/go-runner", "//cmd/kubectl", "//test/e2e:e2e.test_binary", "//vendor/github.com/onsi/ginkgo/ginkgo", @@ -62,7 +63,10 @@ filegroup( filegroup( name = "all-srcs", - srcs = [":package-srcs"], + srcs = [ + ":package-srcs", + "//cluster/images/conformance/go-runner:all-srcs", + ], tags = ["automanaged"], visibility = ["//visibility:public"], ) diff --git a/cluster/images/conformance/Dockerfile b/cluster/images/conformance/Dockerfile index 5f12885fe7a..2466bbace45 100644 --- a/cluster/images/conformance/Dockerfile +++ b/cluster/images/conformance/Dockerfile @@ -18,6 +18,7 @@ COPY ginkgo /usr/local/bin/ COPY e2e.test /usr/local/bin/ COPY kubectl /usr/local/bin/ COPY run_e2e.sh /run_e2e.sh +COPY gorunner /gorunner COPY cluster /kubernetes/cluster WORKDIR /usr/local/bin diff --git a/cluster/images/conformance/Makefile b/cluster/images/conformance/Makefile index e75871e6f20..89335650cfa 100644 --- a/cluster/images/conformance/Makefile +++ b/cluster/images/conformance/Makefile @@ -27,6 +27,7 @@ DOCKERIZED_OUTPUT_PATH=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(A GINKGO_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/ginkgo && echo $(LOCAL_OUTPUT_PATH)/ginkgo || echo $(DOCKERIZED_OUTPUT_PATH)/ginkgo) KUBECTL_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/kubectl && echo $(LOCAL_OUTPUT_PATH)/kubectl || echo $(DOCKERIZED_OUTPUT_PATH)/kubectl) E2E_TEST_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/e2e.test && echo $(LOCAL_OUTPUT_PATH)/e2e.test || echo $(DOCKERIZED_OUTPUT_PATH)/e2e.test) +E2E_GO_RUNNER_BIN?=$(shell test -f $(LOCAL_OUTPUT_PATH)/go-runner && echo $(LOCAL_OUTPUT_PATH)/go-runner || echo $(DOCKERIZED_OUTPUT_PATH)/go-runner) CLUSTER_DIR?=$(shell pwd)/../../../cluster/ @@ -45,11 +46,13 @@ endif cp ${GINKGO_BIN} ${TEMP_DIR} cp ${KUBECTL_BIN} ${TEMP_DIR} cp ${E2E_TEST_BIN} ${TEMP_DIR} + cp ${E2E_GO_RUNNER_BIN} ${TEMP_DIR}/gorunner cp -r ${CLUSTER_DIR} ${TEMP_DIR}/cluster chmod a+rx ${TEMP_DIR}/ginkgo chmod a+rx ${TEMP_DIR}/kubectl chmod a+rx ${TEMP_DIR}/e2e.test + chmod a+rx ${TEMP_DIR}/gorunner cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile diff --git a/cluster/images/conformance/README.md b/cluster/images/conformance/README.md index b5df5e8c1db..34080fe3655 100644 --- a/cluster/images/conformance/README.md +++ b/cluster/images/conformance/README.md @@ -7,7 +7,7 @@ ```console # First, build the binaries by running make from the root directory -$ make WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo cmd/kubectl" +$ make WHAT="test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo cmd/kubectl cluster/images/conformance/go-runner" # Build for linux/amd64 (default) # export REGISTRY=$HOST/$ORG to switch from k8s.gcr.io diff --git a/cluster/images/conformance/go-runner/BUILD b/cluster/images/conformance/go-runner/BUILD new file mode 100644 index 00000000000..55b76f5b871 --- /dev/null +++ b/cluster/images/conformance/go-runner/BUILD @@ -0,0 +1,47 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "cmd.go", + "const.go", + "e2erunner.go", + "env.go", + "tar.go", + ], + importpath = "k8s.io/kubernetes/cluster/images/conformance/go-runner", + visibility = ["//visibility:private"], + deps = ["//vendor/github.com/pkg/errors:go_default_library"], +) + +go_binary( + name = "go-runner", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = [ + "cmd_test.go", + "env_test.go", + "tar_test.go", + ], + data = glob(["testdata/**"]), + embed = [":go_default_library"], + deps = ["//vendor/github.com/pkg/errors:go_default_library"], +) diff --git a/cluster/images/conformance/go-runner/Makefile b/cluster/images/conformance/go-runner/Makefile new file mode 100644 index 00000000000..556fdc42080 --- /dev/null +++ b/cluster/images/conformance/go-runner/Makefile @@ -0,0 +1,27 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HOST_GOOS ?= $(shell go env GOOS) +HOST_GOARCH ?= $(shell go env GOARCH) +GO_BUILD ?= go build + +.PHONY: all build clean + +all: build + +build: + $(GO_BUILD) + +clean: + rm done e2e.log e2e.tar.gz go-runner diff --git a/cluster/images/conformance/go-runner/README.md b/cluster/images/conformance/go-runner/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cluster/images/conformance/go-runner/cmd.go b/cluster/images/conformance/go-runner/cmd.go new file mode 100644 index 00000000000..448a5055081 --- /dev/null +++ b/cluster/images/conformance/go-runner/cmd.go @@ -0,0 +1,96 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "io" + "os/exec" + "strings" +) + +// getCmd uses the given environment to form the ginkgo command to run tests. It will +// set the stdout/stderr to the given writer. +func getCmd(env Getenver, w io.Writer) *exec.Cmd { + ginkgoArgs := []string{} + + // The logic of the parallel env var impacting the skip value necessitates it + // being placed before the rest of the flag resolution. + skip := env.Getenv(skipEnvKey) + switch env.Getenv(parallelEnvKey) { + case "y", "Y", "true": + ginkgoArgs = append(ginkgoArgs, "--p") + if len(skip) == 0 { + skip = serialTestsRegexp + } + } + + ginkgoArgs = append(ginkgoArgs, []string{ + "--focus=" + env.Getenv(focusEnvKey), + "--skip=" + skip, + "--noColor=true", + }...) + + extraArgs := []string{ + "--disable-log-dump", + "--repo-root=/kubernetes", + "--provider=" + env.Getenv(providerEnvKey), + "--report-dir=" + env.Getenv(resultsDirEnvKey), + "--kubeconfig=" + env.Getenv(kubeconfigEnvKey), + } + + // Extra args handling + sep := " " + if len(env.Getenv(extraArgsSeparaterEnvKey)) > 0 { + sep = env.Getenv(extraArgsSeparaterEnvKey) + } + + if len(env.Getenv(extraGinkgoArgsEnvKey)) > 0 { + ginkgoArgs = append(ginkgoArgs, strings.Split(env.Getenv(extraGinkgoArgsEnvKey), sep)...) + } + + if len(env.Getenv(extraArgsEnvKey)) > 0 { + fmt.Printf("sep is %q args are %q", sep, env.Getenv(extraArgsEnvKey)) + fmt.Println("split", strings.Split(env.Getenv(extraArgsEnvKey), sep)) + extraArgs = append(extraArgs, strings.Split(env.Getenv(extraArgsEnvKey), sep)...) + } + + if len(env.Getenv(dryRunEnvKey)) > 0 { + ginkgoArgs = append(ginkgoArgs, "--dryRun=true") + } + + args := []string{} + args = append(args, ginkgoArgs...) + args = append(args, env.Getenv(testBinEnvKey)) + args = append(args, "--") + args = append(args, extraArgs...) + + cmd := exec.Command(env.Getenv(ginkgoEnvKey), args...) + cmd.Stdout = w + cmd.Stderr = w + return cmd +} + +// cmdInfo generates a useful look at what the command is for printing/debug. +func cmdInfo(cmd *exec.Cmd) string { + return fmt.Sprintf( + `Command env: %v +Run from directory: %v +Executable path: %v +Args (comma-delimited): %v`, cmd.Env, cmd.Dir, cmd.Path, strings.Join(cmd.Args, ","), + ) +} diff --git a/cluster/images/conformance/go-runner/cmd_test.go b/cluster/images/conformance/go-runner/cmd_test.go new file mode 100644 index 00000000000..0dc24671680 --- /dev/null +++ b/cluster/images/conformance/go-runner/cmd_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "reflect" + "testing" +) + +func TestGetCmd(t *testing.T) { + testCases := []struct { + desc string + env Getenver + expectArgs []string + }{ + { + desc: "Default", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + }, + }, + expectArgs: []string{ + "ginkgobin", + "--focus=", "--skip=", + "--noColor=true", "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=", "--report-dir=", "--kubeconfig=", + }, + }, { + desc: "Filling in defaults", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + focusEnvKey: "focus", + skipEnvKey: "skip", + providerEnvKey: "provider", + resultsDirEnvKey: "results", + kubeconfigEnvKey: "kubeconfig", + }, + }, + expectArgs: []string{ + "ginkgobin", + "--focus=focus", "--skip=skip", + "--noColor=true", "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=provider", "--report-dir=results", "--kubeconfig=kubeconfig", + }, + }, { + desc: "Parallel gets set and skips serial", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + parallelEnvKey: "true", + }, + }, + expectArgs: []string{ + "ginkgobin", "--p", + "--focus=", "--skip=[Serial]", + "--noColor=true", "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=", "--report-dir=", "--kubeconfig=", + }, + }, { + desc: "Arbitrary options before and after double dash split by space", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + extraArgsEnvKey: "--extra=1 --extra=2", + extraGinkgoArgsEnvKey: "--ginkgo1 --ginkgo2", + }, + }, + expectArgs: []string{ + "ginkgobin", "--focus=", "--skip=", + "--noColor=true", "--ginkgo1", "--ginkgo2", + "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=", "--report-dir=", "--kubeconfig=", + "--extra=1", "--extra=2", + }, + }, { + desc: "Arbitrary options can be split by other tokens", + env: &explicitEnv{ + vals: map[string]string{ + ginkgoEnvKey: "ginkgobin", + testBinEnvKey: "testbin", + extraArgsEnvKey: "--extra=value with spaces:--extra=value with % anything!$$", + extraGinkgoArgsEnvKey: `--ginkgo='with "quotes" and ':--ginkgo2=true$(foo)`, + extraArgsSeparaterEnvKey: ":", + }, + }, + expectArgs: []string{ + "ginkgobin", "--focus=", "--skip=", + "--noColor=true", `--ginkgo='with "quotes" and '`, "--ginkgo2=true$(foo)", + "testbin", "--", + "--disable-log-dump", "--repo-root=/kubernetes", + "--provider=", "--report-dir=", "--kubeconfig=", + "--extra=value with spaces", "--extra=value with % anything!$$", + }, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := getCmd(tc.env, os.Stdout) + if !reflect.DeepEqual(c.Args, tc.expectArgs) { + t.Errorf("Expected args %q but got %q", tc.expectArgs, c.Args) + } + }) + } +} diff --git a/cluster/images/conformance/go-runner/const.go b/cluster/images/conformance/go-runner/const.go new file mode 100644 index 00000000000..161c498fb89 --- /dev/null +++ b/cluster/images/conformance/go-runner/const.go @@ -0,0 +1,67 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +const ( + // resultsTarballName is the name of the tarball we create with all the results. + resultsTarballName = "e2e.tar.gz" + + // doneFileName is the name of the file that signals to the Sonobuoy worker we are + // done. The file should contain the path to the results file. + doneFileName = "done" + + // resultsDirEnvKey is the env var which stores which directory to put the donefile + // and results into. It is a shared, mounted volume between the plugin and Sonobuoy. + resultsDirEnvKey = "RESULTS_DIR" + + // logFileName is the name of the file which stdout is tee'd to. + logFileName = "e2e.log" + + // Misc env vars which were explicitly supported prior to the go runner. + dryRunEnvKey = "E2E_DRYRUN" + parallelEnvKey = "E2E_PARALLEL" + focusEnvKey = "E2E_FOCUS" + skipEnvKey = "E2E_SKIP" + providerEnvKey = "E2E_PROVIDER" + kubeconfigEnvKey = "KUBECONFIG" + ginkgoEnvKey = "GINKGO_BIN" + testBinEnvKey = "TEST_BIN" + + // extraGinkgoArgsEnvKey, if set, will is a list of other arguments to pass to ginkgo. + // These are passed before the test binary and include things like `--afterSuiteHook`. + extraGinkgoArgsEnvKey = "E2E_EXTRA_GINKGO_ARGS" + + // extraArgsEnvKey, if set, will is a list of other arguments to pass to the tests. + // These are passed after the `--` and include things like `--provider`. + extraArgsEnvKey = "E2E_EXTRA_ARGS" + + // extraArgsSeparaterEnvKey specifies how to split the extra args values. If unset, + // it will default to splitting by spaces. + extraArgsSeparaterEnvKey = "E2E_EXTRA_ARGS_SEP" + + defaultSkip = "" + defaultFocus = "[Conformance]" + defaultProvider = "local" + defaultParallel = "1" + defaultResultsDir = "/tmp/results" + defaultGinkgoBinary = "/usr/local/bin/ginkgo" + defaultTestBinary = "/usr/local/bin/e2e.test" + + // serialTestsRegexp is the default skip value if running in parallel. Will not + // override an explicit E2E_SKIP value. + serialTestsRegexp = "[Serial]" +) diff --git a/cluster/images/conformance/go-runner/e2erunner.go b/cluster/images/conformance/go-runner/e2erunner.go new file mode 100644 index 00000000000..073b3159c0f --- /dev/null +++ b/cluster/images/conformance/go-runner/e2erunner.go @@ -0,0 +1,121 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "io" + "io/ioutil" + "log" + "os" + "os/signal" + "path/filepath" + + "github.com/pkg/errors" +) + +func main() { + env := envWithDefaults(map[string]string{ + resultsDirEnvKey: defaultResultsDir, + skipEnvKey: defaultSkip, + focusEnvKey: defaultFocus, + providerEnvKey: defaultProvider, + parallelEnvKey: defaultParallel, + ginkgoEnvKey: defaultGinkgoBinary, + testBinEnvKey: defaultTestBinary, + }) + + if err := configureAndRunWithEnv(env); err != nil { + log.Fatal(err) + } +} + +// configureAndRunWithEnv uses the given environment to configure and then start the test run. +// It will handle TERM signals gracefully and kill the test process and will +// save the logs/results to the location specified via the RESULTS_DIR environment +// variable. +func configureAndRunWithEnv(env Getenver) error { + // Ensure we save results regardless of other errors. This helps any + // consumer who may be polling for the results. + resultsDir := env.Getenv(resultsDirEnvKey) + defer saveResults(resultsDir) + + // Print the output to stdout and a logfile which will be returned + // as part of the results tarball. + logFilePath := filepath.Join(resultsDir, logFileName) + logFile, err := os.Create(logFilePath) + if err != nil { + return errors.Wrapf(err, "failed to create log file %v", logFilePath) + } + mw := io.MultiWriter(os.Stdout, logFile) + cmd := getCmd(env, mw) + + log.Printf("Running command:\n%v\n", cmdInfo(cmd)) + err = cmd.Start() + if err != nil { + return errors.Wrap(err, "starting command") + } + + // Handle signals and shutdown process gracefully. + go setupSigHandler(cmd.Process.Pid) + return errors.Wrap(cmd.Wait(), "running command") +} + +// setupSigHandler will kill the process identified by the given PID if it +// gets a TERM signal. +func setupSigHandler(pid int) { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + + // Block until a signal is received. + log.Println("Now listening for interrupts") + s := <-c + log.Printf("Got signal: %v. Shutting down test process (PID: %v)\n", s, pid) + p, err := os.FindProcess(pid) + if err != nil { + log.Printf("Could not find process %v to shut down.\n", pid) + return + } + if err := p.Signal(s); err != nil { + log.Printf("Failed to signal test process to terminate: %v\n", err) + return + } + log.Printf("Signalled process %v to terminate successfully.\n", pid) +} + +// saveResults will tar the results directory and write the resulting tarball path +// into the donefile. +func saveResults(resultsDir string) error { + log.Printf("Saving results at %v\n", resultsDir) + + err := tarDir(resultsDir, filepath.Join(resultsDir, resultsTarballName)) + if err != nil { + return errors.Wrapf(err, "tar directory %v", resultsDir) + } + + doneFile := filepath.Join(resultsDir, doneFileName) + + resultsTarball := filepath.Join(resultsDir, resultsTarballName) + resultsTarball, err = filepath.Abs(resultsTarball) + if err != nil { + return errors.Wrapf(err, "failed to find absolute path for %v", resultsTarball) + } + + return errors.Wrap( + ioutil.WriteFile(doneFile, []byte(resultsTarball), os.FileMode(0777)), + "writing donefile", + ) +} diff --git a/cluster/images/conformance/go-runner/env.go b/cluster/images/conformance/go-runner/env.go new file mode 100644 index 00000000000..c910d9184f9 --- /dev/null +++ b/cluster/images/conformance/go-runner/env.go @@ -0,0 +1,66 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" +) + +// Getenver is the interface we use to mock out the env for easier testing. OS env +// vars can't be as easily tested since internally it uses sync.Once. +type Getenver interface { + Getenv(string) string +} + +// osEnv uses the actual os.Getenv methods to lookup values. +type osEnv struct{} + +// Getenv gets the value of the requested environment variable. +func (*osEnv) Getenv(s string) string { + return os.Getenv(s) +} + +// explicitEnv uses a map instead of os.Getenv methods to lookup values. +type explicitEnv struct { + vals map[string]string +} + +// Getenv returns the value of the requested environment variable (in this +// implementation, really just a map lookup). +func (e *explicitEnv) Getenv(s string) string { + return e.vals[s] +} + +// defaultOSEnv uses a Getenver to lookup values but if it does +// not have that value, it falls back to its internal set of defaults. +type defaultEnver struct { + firstChoice Getenver + defaults map[string]string +} + +// Getenv returns the value of the environment variable or its default if unset. +func (e *defaultEnver) Getenv(s string) string { + v := e.firstChoice.Getenv(s) + if len(v) == 0 { + return e.defaults[s] + } + return v +} + +func envWithDefaults(defaults map[string]string) Getenver { + return &defaultEnver{firstChoice: &osEnv{}, defaults: defaults} +} diff --git a/cluster/images/conformance/go-runner/env_test.go b/cluster/images/conformance/go-runner/env_test.go new file mode 100644 index 00000000000..4229ebe3037 --- /dev/null +++ b/cluster/images/conformance/go-runner/env_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "testing" +) + +func TestEnv(t *testing.T) { + testCases := []struct { + desc string + preHook func() + env Getenver + expect map[string]string + }{ + { + desc: "OS env", + env: &osEnv{}, + preHook: func() { + os.Setenv("key1", "1") + }, + expect: map[string]string{"key1": "1"}, + }, { + desc: "OS env falls defaults to empty", + env: &osEnv{}, + preHook: func() { + os.Unsetenv("key1") + }, + expect: map[string]string{"key1": ""}, + }, { + desc: "First choice of env respected", + env: &defaultEnver{ + firstChoice: &explicitEnv{ + vals: map[string]string{ + "key1": "1", + }, + }, + defaults: map[string]string{ + "key1": "default1", + "key2": "default2", + }, + }, + expect: map[string]string{ + "key1": "1", + "key2": "default2", + }, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + for k, expectVal := range tc.expect { + if tc.preHook != nil { + tc.preHook() + } + val := tc.env.Getenv(k) + if val != expectVal { + t.Errorf("Expected %q but got %q", expectVal, val) + } + } + }) + } +} diff --git a/cluster/images/conformance/go-runner/tar.go b/cluster/images/conformance/go-runner/tar.go new file mode 100644 index 00000000000..2309387bfec --- /dev/null +++ b/cluster/images/conformance/go-runner/tar.go @@ -0,0 +1,83 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +// tarDir takes a source and variable writers and walks 'source' writing each file +// found to the tar writer. +func tarDir(dir, outpath string) error { + // ensure the src actually exists before trying to tar it + if _, err := os.Stat(dir); err != nil { + return errors.Wrapf(err, "tar unable to stat directory %v", dir) + } + + outfile, err := os.Create(outpath) + if err != nil { + return errors.Wrapf(err, "creating tarball %v", outpath) + } + defer outfile.Close() + + gzw := gzip.NewWriter(outfile) + defer gzw.Close() + + tw := tar.NewWriter(gzw) + defer tw.Close() + + return filepath.Walk(dir, func(file string, fi os.FileInfo, err error) error { + // Return on any error. + if err != nil { + return err + } + + // Only write regular files and don't include the archive itself. + if !fi.Mode().IsRegular() || filepath.Join(dir, fi.Name()) == outpath { + return nil + } + + // Create a new dir/file header. + header, err := tar.FileInfoHeader(fi, fi.Name()) + if err != nil { + return errors.Wrapf(err, "creating file info header %v", fi.Name()) + } + + // Update the name to correctly reflect the desired destination when untaring. + header.Name = strings.TrimPrefix(strings.Replace(file, dir, "", -1), string(filepath.Separator)) + if err := tw.WriteHeader(header); err != nil { + return errors.Wrapf(err, "writing header for tarball %v", header.Name) + } + + // Open files, copy into tarfile, and close. + f, err := os.Open(file) + if err != nil { + return errors.Wrapf(err, "opening file %v for writing into tarball", file) + } + defer f.Close() + + _, err = io.Copy(tw, f) + return errors.Wrapf(err, "creating file %v contents into tarball", file) + }) +} diff --git a/cluster/images/conformance/go-runner/tar_test.go b/cluster/images/conformance/go-runner/tar_test.go new file mode 100644 index 00000000000..7df8e0ad7b5 --- /dev/null +++ b/cluster/images/conformance/go-runner/tar_test.go @@ -0,0 +1,123 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/pkg/errors" +) + +func TestTar(t *testing.T) { + testCases := []struct { + desc string + dir string + outpath string + expectErr string + expect map[string]string + }{ + { + desc: "Contents preserved and no self-reference", + dir: "testdata/tartest", + outpath: "testdata/tartest/out.tar.gz", + expect: map[string]string{ + "file1": "file1 data", + "file2": "file2 data", + "subdir/file4": "file4 data", + }, + }, { + desc: "Errors if directory does not exist", + dir: "testdata/does-not-exist", + outpath: "testdata/tartest/out.tar.gz", + expectErr: "tar unable to stat directory", + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + err := tarDir(tc.dir, tc.outpath) + switch { + case err != nil && len(tc.expectErr) == 0: + t.Fatalf("Expected nil error but got %q", err) + case err != nil && len(tc.expectErr) > 0: + if !strings.Contains(fmt.Sprint(err), tc.expectErr) { + t.Errorf("Expected error \n\t%q\nbut got\n\t%q", tc.expectErr, err) + } + return + case err == nil && len(tc.expectErr) > 0: + t.Fatalf("Expected error %q but got nil", tc.expectErr) + default: + // No error + } + + data, err := readAllTar(tc.outpath) + if !reflect.DeepEqual(data, tc.expect) { + t.Errorf("Expected data %v but got %v", tc.expect, data) + } + }) + } +} + +// readAllTar walks all of the files in the archive. It returns a map +// of filenames and their contents and any error encountered. +func readAllTar(tarPath string) (map[string]string, error) { + tarPath, err := filepath.Abs(tarPath) + if err != nil { + return nil, err + } + + fileReader, err := os.Open(tarPath) + if err != nil { + return nil, err + } + defer fileReader.Close() + + gzStream, err := gzip.NewReader(fileReader) + if err != nil { + return nil, errors.Wrap(err, "couldn't uncompress reader") + } + defer gzStream.Close() + + // Open and iterate through the files in the archive. + tr := tar.NewReader(gzStream) + fileData := map[string]string{} + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + + return nil, err + } + + b, err := ioutil.ReadAll(tr) + if err != nil { + return nil, err + } + fileData[filepath.ToSlash(hdr.Name)] = string(b) + } + return fileData, nil +} diff --git a/cluster/images/conformance/go-runner/testdata/tartest/file1 b/cluster/images/conformance/go-runner/testdata/tartest/file1 new file mode 100644 index 00000000000..4365baaa1f1 --- /dev/null +++ b/cluster/images/conformance/go-runner/testdata/tartest/file1 @@ -0,0 +1 @@ +file1 data \ No newline at end of file diff --git a/cluster/images/conformance/go-runner/testdata/tartest/file2 b/cluster/images/conformance/go-runner/testdata/tartest/file2 new file mode 100644 index 00000000000..526a2a0b1d5 --- /dev/null +++ b/cluster/images/conformance/go-runner/testdata/tartest/file2 @@ -0,0 +1 @@ +file2 data \ No newline at end of file diff --git a/cluster/images/conformance/go-runner/testdata/tartest/subdir/file4 b/cluster/images/conformance/go-runner/testdata/tartest/subdir/file4 new file mode 100644 index 00000000000..e8ca25e8c71 --- /dev/null +++ b/cluster/images/conformance/go-runner/testdata/tartest/subdir/file4 @@ -0,0 +1 @@ +file4 data \ No newline at end of file diff --git a/cluster/images/conformance/run_e2e.sh b/cluster/images/conformance/run_e2e.sh index afe8d207a40..e24d8b52c2b 100755 --- a/cluster/images/conformance/run_e2e.sh +++ b/cluster/images/conformance/run_e2e.sh @@ -36,6 +36,14 @@ saveResults() { echo -n "${RESULTS_DIR}/e2e.tar.gz" > "${RESULTS_DIR}/done" } +# Optional Golang runner alternative to the bash script. +# Entry provided via env var to simplify invocation. +if [[ -n ${E2E_USE_GO_RUNNER:-} ]]; then + set -x + /gorunner + exit $? +fi + # We get the TERM from kubernetes and handle it gracefully trap shutdown TERM diff --git a/hack/dev-push-conformance.sh b/hack/dev-push-conformance.sh index 699d0686aea..7dee5a6a582 100755 --- a/hack/dev-push-conformance.sh +++ b/hack/dev-push-conformance.sh @@ -42,7 +42,7 @@ IMAGE="${REGISTRY}/conformance-amd64:${VERSION}" kube::build::verify_prereqs kube::build::build_image -kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl" +kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl cluster/images/conformance/go-runner" kube::build::copy_output make -C "${KUBE_ROOT}/cluster/images/conformance" build diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index e9c3b066925..96eb78b3389 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -264,6 +264,7 @@ kube::golang::test_targets() { cmd/linkcheck vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test + cluster/images/conformance/go-runner ) echo "${targets[@]}" }