diff --git a/build/root/Makefile b/build/root/Makefile index 1ed4bd1ce11..609f3a5e709 100644 --- a/build/root/Makefile +++ b/build/root/Makefile @@ -269,6 +269,9 @@ define TEST_E2E_NODE_HELP_INFO # INSTANCE_TYPE: For REMOTE=true only. Machine type to use. # NODE_ENV: For REMOTE=true only. Additional metadata keys to add the instance. # RUNTIME_CONFIG: The runtime configuration for the API server on the node e2e tests. +# E2E_TEST_DEBUG_TOOL: one of dlv/delve/gdb. Runs the test/e2e_node test binary +# interactively under the chosen debugger. Only works for REMOTE=false and +# in combination with DBG=1. # # Example: # make test-e2e-node FOCUS=Kubelet SKIP=container diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh index e791f845383..67798f605a3 100755 --- a/hack/make-rules/test-e2e-node.sh +++ b/hack/make-rules/test-e2e-node.sh @@ -53,6 +53,18 @@ ssh_key=${SSH_KEY:-} ssh_options=${SSH_OPTIONS:-} kubelet_config_file=${KUBELET_CONFIG_FILE:-"test/e2e_node/jenkins/default-kubelet-config.yaml"} +# If set, the command executed will be: +# - `dlv exec` if set to "delve" +# - `gdb` if set to "gdb" +# NOTE: for this to work the e2e_node.test binary has to be compiled with DBG=1. +# +# The name of this variable is the same as in ginkgo-e2e.sh. +debug_tool=${E2E_TEST_DEBUG_TOOL:-} +if [ "${remote}" = true ] && [ -n "${debug_tool}" ]; then + echo "Support for E2E_TEST_DEBUG_TOOL=${debug_tool} is only implemented for REMOTE=false." + exit 1 +fi + # Parse the flags to pass to ginkgo ginkgoflags="-timeout=24h" if [[ ${parallelism} -gt 1 ]]; then @@ -247,6 +259,7 @@ else # Test using the host the script was run on # Provided for backwards compatibility go run test/e2e_node/runner/local/run_local.go \ + --debug-tool="${debug_tool}" \ --system-spec-name="${system_spec_name}" --extra-envs="${extra_envs}" \ --ginkgo-flags="${ginkgoflags}" \ --test-flags="--v 4 --report-dir=${artifacts} --node-name $(hostname) ${test_args}" \ diff --git a/test/e2e_node/runner/local/run_local.go b/test/e2e_node/runner/local/run_local.go index a35ba1ef5b3..f908ea5bbeb 100644 --- a/test/e2e_node/runner/local/run_local.go +++ b/test/e2e_node/runner/local/run_local.go @@ -22,6 +22,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "k8s.io/kubernetes/test/e2e_node/builder" @@ -32,12 +33,13 @@ import ( ) var buildDependencies = flag.Bool("build-dependencies", true, "If true, build all dependencies.") -var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.") -var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.") +var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner, with shell-style quoting and escaping.") +var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test, with shell-style quoting and escaping.") var systemSpecName = flag.String("system-spec-name", "", fmt.Sprintf("The name of the system spec used for validating the image in the node conformance test. The specs are at %s. If unspecified, the default built-in spec (system.DefaultSpec) will be used.", system.SystemSpecPath)) var extraEnvs = flag.String("extra-envs", "", "The extra environment variables needed for node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2") var runtimeConfig = flag.String("runtime-config", "", "The runtime configuration for the API server on the node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2") var kubeletConfigFile = flag.String("kubelet-config-file", "", "The KubeletConfiguration file that should be applied to the kubelet") +var debugTool = flag.String("debug-tool", "", "'delve', 'dlv' or 'gdb': run e2e_node.test under that debugger") func main() { klog.InitFlags(nil) @@ -56,10 +58,34 @@ func main() { klog.Fatalf("Failed to get build output directory: %v", err) } klog.Infof("Got build output dir: %v", outputDir) - ginkgo := filepath.Join(outputDir, "ginkgo") test := filepath.Join(outputDir, "e2e_node.test") + interactive := false + var cmd string + var args []string + switch *debugTool { + case "": + // No debugger, run gingko directly. + cmd = filepath.Join(outputDir, "ginkgo") + args = []string{*ginkgoFlags, test, "--"} + case "delve", "dlv": + dlv, err := exec.LookPath("dlv") + if err != nil { + klog.Fatalf("'dlv' not found: %v", err) + } + interactive = true + cmd = dlv + args = []string{"exec", test, "--", addGinkgoArgPrefix(*ginkgoFlags)} + case "gdb": + gdb, err := exec.LookPath("gdb") + if err != nil { + klog.Fatalf("'gdb' not found: %v", err) + } + interactive = true + cmd = gdb + args = []string{test, "--", addGinkgoArgPrefix(*ginkgoFlags)} + } - args := []string{*ginkgoFlags, test, "--", *testFlags, fmt.Sprintf("--runtime-config=%s", *runtimeConfig)} + args = append(args, *testFlags, fmt.Sprintf("--runtime-config=%s", *runtimeConfig)) if *systemSpecName != "" { rootDir, err := utils.GetK8sRootDir() if err != nil { @@ -71,16 +97,40 @@ func main() { if *kubeletConfigFile != "" { args = append(args, fmt.Sprintf("--kubelet-config-file=\"%s\"", *kubeletConfigFile)) } - if err := runCommand(ginkgo, args...); err != nil { + if err := runCommand(interactive, cmd, args...); err != nil { klog.Exitf("Test failed: %v", err) } return } -func runCommand(name string, args ...string) error { +func addGinkgoArgPrefix(ginkgoFlags string) string { + // Warning, hack! This simplistic search/replace assumes that hyphens do not appear + // inside argument values. + // + // The right solution would be to use github.com/anmitsu/go-shlex to split + // the -ginkgo-flags and -test-flags strings into individual arguments, then invoke + // exec.Command with the resulting string slice instead of passing a single string + // to sh. But github.com/anmitsu/go-shlex is not a Kubernetes dependency and not + // worth adding. + + ginkgoFlags = regexp.MustCompile(`(^| )--?`).ReplaceAllString(ginkgoFlags, `$1--ginkgo.`) + return ginkgoFlags +} + +func runCommand(interactive bool, name string, args ...string) error { klog.Infof("Running command: %v %v", name, strings.Join(args, " ")) + // Using sh is necessary because the args are using POSIX quoting. + // sh has to parse that. cmd := exec.Command("sudo", "sh", "-c", strings.Join(append([]string{name}, args...), " ")) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + if interactive { + // stdin must be a console. + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdin + cmd.Stderr = os.Stdin + } else { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } + return cmd.Run() }