From 0980f026c9d9b7dbe359437dd68a99dfe79dddd7 Mon Sep 17 00:00:00 2001 From: David Porter Date: Sat, 25 Feb 2023 17:28:53 -0800 Subject: [PATCH 1/3] test: Remove tests argument from node e2e image config This was never being used, the only config that used it was deleted in https://github.com/kubernetes/test-infra/pull/26017 so we don't need this anymore, so let's delete it. Signed-off-by: David Porter --- test/e2e_node/runner/remote/run_remote.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index 26872d5b4f1..5ce543b17f3 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -166,8 +166,6 @@ type GCEImage struct { Metadata string `json:"metadata"` Machine string `json:"machine,omitempty"` Resources Resources `json:"resources,omitempty"` - // This test is for benchmark (no limit verification, more result log, node name has format 'machine-image-uuid') if 'Tests' is non-empty. - Tests []string `json:"tests,omitempty"` } type internalImageConfig struct { @@ -185,7 +183,6 @@ type internalGCEImage struct { resources Resources metadata *compute.Metadata machine string - tests []string } func main() { @@ -359,7 +356,6 @@ func prepareGceImages() (*internalImageConfig, error) { metadata: getImageMetadata(metadata), kernelArguments: imageConfig.KernelArguments, machine: imageConfig.Machine, - tests: imageConfig.Tests, resources: imageConfig.Resources, } if gceImage.imageDesc == "" { @@ -963,16 +959,3 @@ func machineType(machine string) string { } return fmt.Sprintf("zones/%s/machineTypes/%s", *zone, machine) } - -// testsToGinkgoFocus converts the test string list to Ginkgo focus -func testsToGinkgoFocus(tests []string) string { - focus := "--focus=\"" - for i, test := range tests { - if i == 0 { - focus += test - } else { - focus += ("|" + test) - } - } - return focus + "\"" -} From e9ecdf3534e97751e6630ce4537b5907fa414031 Mon Sep 17 00:00:00 2001 From: David Porter Date: Sat, 25 Feb 2023 00:40:15 -0800 Subject: [PATCH 2/3] test: Emit ginkgo log for each node e2e When running multiple node e2e with multiple machine images, the tests are run separately for each node. The final build log has all of the results for each of the hosts combined together which make debugging the log difficult. To make it easier, emit a log for each host that was run. This log will be written to the results directory and uploaded as an artifact in prow jobs. Signed-off-by: David Porter --- test/e2e_node/remote/node_e2e.go | 9 ++++++--- test/e2e_node/runner/remote/run_remote.go | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go index afbcd18af35..cf5efaee111 100644 --- a/test/e2e_node/remote/node_e2e.go +++ b/test/e2e_node/remote/node_e2e.go @@ -196,12 +196,15 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePr systemSpecFile = systemSpecName + ".yaml" } + outputGinkgoFile := filepath.Join(results, fmt.Sprintf("%s-ginkgo.log", host)) + // Run the tests klog.V(2).Infof("Starting tests on %q", host) cmd := getSSHCommand(" && ", fmt.Sprintf("cd %s", workspace), - fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-name=%s --system-spec-file=%s --extra-envs=%s --runtime-config=%s --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=\"%s\" %s", - timeout.Seconds(), ginkgoArgs, systemSpecName, systemSpecFile, extraEnvs, runtimeConfig, host, results, junitFilePrefix, imageDesc, testArgs), + // Note, we need to have set -o pipefail here to ensure we return the appriorate exit code from ginkgo; not tee + fmt.Sprintf("set -o pipefail; timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-name=%s --system-spec-file=%s --extra-envs=%s --runtime-config=%s --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=\"%s\" %s 2>&1 | tee -i %s", + timeout.Seconds(), ginkgoArgs, systemSpecName, systemSpecFile, extraEnvs, runtimeConfig, host, results, junitFilePrefix, imageDesc, testArgs, outputGinkgoFile), ) - return SSH(host, "sh", "-c", cmd) + return SSH(host, "/bin/bash", "-c", cmd) } diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index 5ce543b17f3..26199fb5b1b 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -289,9 +289,9 @@ func main() { fmt.Printf("%s\n", tr.output) if tr.err != nil { errCount++ - fmt.Printf("Failure Finished Test Suite on Host %s\n%v\n", host, tr.err) + fmt.Printf("Failure Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n%v\n", host, tr.err) } else { - fmt.Printf("Success Finished Test Suite on Host %s\n", host) + fmt.Printf("Success Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n", host) } exitOk = exitOk && tr.exitOk fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour) From e00188459423a3b2a35a5a5fce7f28f178b63b97 Mon Sep 17 00:00:00 2001 From: David Porter Date: Sat, 25 Feb 2023 17:46:22 -0800 Subject: [PATCH 3/3] test: Add some default flags ginkgo flags for node e2e Add the following ginkgo flags for each node e2e similar to the existing hack/ginkgo-e2e.sh script. * --no-color, colors aren't rendered properly in prow and make examining the log in text editors more difficult, so let's disable them. `hack/ginkgo-e2e.sh` (used for kind e2e tests) also disables them already. * -v, enable verbose logs. This is needed so we get more detailed info even when the tests pass. This is useful so we can compare successful runs to failed runs. Signed-off-by: David Porter --- test/e2e_node/runner/remote/run_remote.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index 26199fb5b1b..58364b13fe0 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -222,6 +222,10 @@ func main() { return } + // Append some default ginkgo flags. We use similar defaults here as hack/ginkgo-e2e.sh + allGinkgoFlags := fmt.Sprintf("%s --no-color -v", *ginkgoFlags) + fmt.Printf("Will use ginkgo flags as: %s", allGinkgoFlags) + var gceImages *internalImageConfig if *mode == "gce" { if *hosts == "" && *imageConfigFile == "" && *images == "" { @@ -260,8 +264,9 @@ func main() { imageConfig := gceImages.images[shortName] fmt.Printf("Initializing e2e tests using image %s/%s/%s.\n", shortName, imageConfig.project, imageConfig.image) running++ + go func(image *internalGCEImage, junitFileName string) { - results <- testImage(image, junitFileName) + results <- testImage(image, junitFileName, allGinkgoFlags) }(&imageConfig, shortName) } } @@ -270,7 +275,7 @@ func main() { fmt.Printf("Initializing e2e tests using host %s.\n", host) running++ go func(host string, junitFileName string) { - results <- testHost(host, *cleanup, "", junitFileName, *ginkgoFlags) + results <- testHost(host, *cleanup, "", junitFileName, allGinkgoFlags) }(host, host) } } @@ -532,18 +537,7 @@ func getGCEImage(imageRegex, imageFamily string, project string) (string, error) // Provision a gce instance using image and run the tests in archive against the instance. // Delete the instance afterward. -func testImage(imageConfig *internalGCEImage, junitFileName string) *TestResult { - ginkgoFlagsStr := *ginkgoFlags - // Check whether the test is for benchmark. - if len(imageConfig.tests) > 0 { - // Benchmark needs machine type non-empty. - if imageConfig.machine == "" { - imageConfig.machine = defaultMachine - } - // Use the Ginkgo focus in benchmark config. - ginkgoFlagsStr += (" " + testsToGinkgoFocus(imageConfig.tests)) - } - +func testImage(imageConfig *internalGCEImage, junitFileName string, ginkgoFlagsStr string) *TestResult { host, err := createInstance(imageConfig) if *deleteInstances { defer deleteInstance(host)