fix bug in DialFromNode where it fail the test when encountered a command execution failure

test/e2e/framework/exec_util.go

@@ -33,6 +33,14 @@ import (
 // ExecCommandInContainer execute a command in the specified container.
 // Pass in stdin, tty if needed in the future.
 func (f *Framework) ExecCommandInContainer(podName, containerName string, cmd ...string) string {
+	stdout, stderr, err := f.ExecCommandInContainerWithFullOutput(podName, containerName, cmd...)
+	Logf("Exec stderr: %q", stderr)
+	Expect(err).NotTo(HaveOccurred(), "fail to execute command")
+	return stdout
+}
+
+// ExecCommandInContainerWithFullOutput executes a command in the specified container and return stdout, stderr and error
+func (f *Framework) ExecCommandInContainerWithFullOutput(podName, containerName string, cmd ...string) (string, string, error) {
 	Logf("Exec running '%s'", strings.Join(cmd, " "))
 	config, err := LoadConfig()
 	Expect(err).NotTo(HaveOccurred(), "failed to load restclient config")
@@ -55,9 +63,7 @@ func (f *Framework) ExecCommandInContainer(podName, containerName string, cmd ..
 	}, api.ParameterCodec)
 
 	err = execute("POST", req.URL(), config, stdin, &stdout, &stderr, tty)
-	Logf("Exec stderr: %q", stderr.String())
-	Expect(err).NotTo(HaveOccurred(), "post request failed")
-	return strings.TrimSpace(stdout.String())
+	return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err
 }
 
 func (f *Framework) ExecShellInContainer(podName, containerName string, cmd string) string {
@@ -71,10 +77,21 @@ func (f *Framework) ExecCommandInPod(podName string, cmd ...string) string {
 	return f.ExecCommandInContainer(podName, pod.Spec.Containers[0].Name, cmd...)
 }
 
+func (f *Framework) ExecCommandInPodWithFullOutput(podName string, cmd ...string) (string, string, error) {
+	pod, err := f.PodClient().Get(podName)
+	Expect(err).NotTo(HaveOccurred(), "failed to get pod")
+	Expect(pod.Spec.Containers).NotTo(BeEmpty())
+	return f.ExecCommandInContainerWithFullOutput(podName, pod.Spec.Containers[0].Name, cmd...)
+}
+
 func (f *Framework) ExecShellInPod(podName string, cmd string) string {
 	return f.ExecCommandInPod(podName, "/bin/sh", "-c", cmd)
 }
 
+func (f *Framework) ExecShellInPodWithFullOutput(podName string, cmd string) (string, string, error) {
+	return f.ExecCommandInPodWithFullOutput(podName, "/bin/sh", "-c", cmd)
+}
+
 func execute(method string, url *url.URL, config *restclient.Config, stdin io.Reader, stdout, stderr io.Writer, tty bool) error {
 	exec, err := remotecommand.NewExecutor(config, method, url)
 	if err != nil {

test/utils/networking_utils.go

@@ -174,8 +174,13 @@ func (config *NetworkingTestConfig) DialFromContainer(protocol, containerIP, tar
 	eps := sets.NewString()
 
 	for i := 0; i < maxTries; i++ {
-		stdout := config.f.ExecShellInPod(config.HostTestContainerPod.Name, cmd)
-
+		stdout, stderr, err := config.f.ExecShellInPodWithFullOutput(config.HostTestContainerPod.Name, cmd)
+		if err != nil {
+			// A failure to kubectl exec counts as a try, not a hard fail.
+			// Also note that we will keep failing for maxTries in tests where
+			// we confirm unreachability.
+			framework.Logf("Failed to execute %q: %v, stdout: %q, stderr %q", cmd, err, stdout, stderr)
+		} else {
 			var output map[string][]string
 			if err := json.Unmarshal([]byte(stdout), &output); err != nil {
 				framework.Logf("WARNING: Failed to unmarshal curl response. Cmd %v run in %v, output: %s, err: %v",
@@ -184,9 +189,12 @@ func (config *NetworkingTestConfig) DialFromContainer(protocol, containerIP, tar
 			}
 
 			for _, hostName := range output["responses"] {
-			eps.Insert(hostName)
+				trimmed := strings.TrimSpace(hostName)
+				if trimmed != "" {
+					eps.Insert(trimmed)
+				}
+			}
 		}
-
 		framework.Logf("Waiting for endpoints: %v", expectedEps.Difference(eps))
 
 		// Check against i+1 so we exit if minTries == maxTries.
@@ -224,8 +232,18 @@ func (config *NetworkingTestConfig) DialFromNode(protocol, targetIP string, targ
 
 	filterCmd := fmt.Sprintf("%s | grep -v '^\\s*$'", cmd)
 	for i := 0; i < maxTries; i++ {
-		stdout := config.f.ExecShellInPod(config.HostTestContainerPod.Name, filterCmd)
-		eps.Insert(strings.TrimSpace(stdout))
+		stdout, stderr, err := config.f.ExecShellInPodWithFullOutput(config.HostTestContainerPod.Name, filterCmd)
+		if err != nil || len(stderr) > 0 {
+			// A failure to exec command counts as a try, not a hard fail.
+			// Also note that we will keep failing for maxTries in tests where
+			// we confirm unreachability.
+			framework.Logf("Failed to execute %q: %v, stdout: %q, stderr: %q", filterCmd, err, stdout, stderr)
+		} else {
+			trimmed := strings.TrimSpace(stdout)
+			if trimmed != "" {
+				eps.Insert(trimmed)
+			}
+		}
 		framework.Logf("Waiting for %+v endpoints, got endpoints %+v", expectedEps.Difference(eps), eps)
 
 		// Check against i+1 so we exit if minTries == maxTries.