From 2e3a2fa222e8142d1dc9c4d61abaeb7a812fc97d Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Sun, 25 Jan 2026 20:40:33 +0000 Subject: [PATCH 1/4] test: Read /proc/net/nf_conntrack instead of using conntrack binary The distroless-iptables image no longer includes the conntrack binary as of v0.8.7 (removed in kubernetes/release#4223 since kube-proxy no longer needs it after kubernetes#126847). Update the KubeProxy CLOSE_WAIT timeout test to read /proc/net/nf_conntrack directly instead of using the conntrack command. The file contains the same connection tracking data and is accessible from the privileged host-network pod. Signed-off-by: Davanum Srinivas --- test/e2e/network/kube_proxy.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/test/e2e/network/kube_proxy.go b/test/e2e/network/kube_proxy.go index 3e6573f1675..6cf2f1aaa58 100644 --- a/test/e2e/network/kube_proxy.go +++ b/test/e2e/network/kube_proxy.go @@ -216,11 +216,12 @@ var _ = common.SIGDescribe("KubeProxy", func() { if netutils.IsIPv6String(ip) { ipFamily = "ipv6" } - // Obtain the corresponding conntrack entry on the host checking - // the nf_conntrack file from the pod e2e-net-exec. + // Obtain the corresponding conntrack entry on the host by reading + // /proc/net/nf_conntrack directly from the pod e2e-net-exec. + // This avoids dependency on the conntrack binary. // It retries in a loop if the entry is not found. - cmd := fmt.Sprintf("conntrack -L -f %s -d %v "+ - "| grep -m 1 'CLOSE_WAIT.*dport=%v' ", + cmd := fmt.Sprintf("cat /proc/net/nf_conntrack "+ + "| grep -m 1 -E '%s.*CLOSE_WAIT.*dst=%v.*dport=%v'", ipFamily, ip, testDaemonTCPPort) if err := wait.PollImmediate(2*time.Second, epsilonSeconds*time.Second, func() (bool, error) { result, err := e2epodoutput.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", cmd) @@ -230,14 +231,15 @@ var _ = common.SIGDescribe("KubeProxy", func() { return false, nil } framework.Logf("conntrack entry for node %v and port %v: %v", serverNodeInfo.nodeIP, testDaemonTCPPort, result) - // Timeout in seconds is available as the third column of the matched entry + // /proc/net/nf_conntrack format: ipv4 2 tcp 6 src=... dst=... + // Timeout in seconds is available as the fifth column (index 4) line := strings.Fields(result) - if len(line) < 3 { + if len(line) < 5 { return false, fmt.Errorf("conntrack entry does not have a timeout field: %v", line) } - timeoutSeconds, err := strconv.Atoi(line[2]) + timeoutSeconds, err := strconv.Atoi(line[4]) if err != nil { - return false, fmt.Errorf("failed to convert matched timeout %s to integer: %w", line[2], err) + return false, fmt.Errorf("failed to convert matched timeout %s to integer: %w", line[4], err) } if math.Abs(float64(timeoutSeconds-expectedTimeoutSeconds)) < epsilonSeconds { return true, nil @@ -245,9 +247,9 @@ var _ = common.SIGDescribe("KubeProxy", func() { return false, fmt.Errorf("wrong TCP CLOSE_WAIT timeout: %v expected: %v", timeoutSeconds, expectedTimeoutSeconds) }); err != nil { // Dump all conntrack entries for debugging - result, err2 := e2epodoutput.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", "conntrack -L") + result, err2 := e2epodoutput.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", "cat /proc/net/nf_conntrack") if err2 != nil { - framework.Logf("failed to obtain conntrack entry: %v %v", result, err2) + framework.Logf("failed to read /proc/net/nf_conntrack: %v %v", result, err2) } framework.Logf("conntrack entries for node %v: %v", serverNodeInfo.nodeIP, result) framework.Failf("no valid conntrack entry for port %d on node %s: %v", testDaemonTCPPort, serverNodeInfo.nodeIP, err) From 727969bb687cee8f9b5096b8c8052630cf4e7693 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Mon, 26 Jan 2026 23:04:49 +0000 Subject: [PATCH 2/4] test: Fix KubeProxy CLOSE_WAIT test for IPv6 environments The /proc/net/nf_conntrack file uses fully expanded IPv6 addresses with leading zeros in each 16-bit group. For example: fc00:f853:ccd:e793::3 -> fc00:f853:0ccd:e793:0000:0000:0000:0003 Add expandIPv6ForConntrack() helper function to expand IPv6 addresses to the format used by /proc/net/nf_conntrack before using them in the grep pattern. Signed-off-by: Davanum Srinivas --- test/e2e/network/kube_proxy.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/e2e/network/kube_proxy.go b/test/e2e/network/kube_proxy.go index 6cf2f1aaa58..2632711c39f 100644 --- a/test/e2e/network/kube_proxy.go +++ b/test/e2e/network/kube_proxy.go @@ -46,6 +46,20 @@ import ( netutils "k8s.io/utils/net" ) +// expandIPv6ForConntrack expands an IPv6 address to the format used in /proc/net/nf_conntrack. +// The conntrack file uses fully expanded IPv6 addresses with leading zeros in each group. +// e.g., "fc00:f853:ccd:e793::3" -> "fc00:f853:0ccd:e793:0000:0000:0000:0003" +func expandIPv6ForConntrack(ipStr string) string { + ip := netutils.ParseIPSloppy(ipStr) + if ip == nil || ip.To4() != nil { + return ipStr // not IPv6 or invalid, return as-is + } + ip = ip.To16() + return fmt.Sprintf("%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x", + ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], + ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]) +} + var kubeProxyE2eImage = imageutils.GetE2EImage(imageutils.Agnhost) var _ = common.SIGDescribe("KubeProxy", func() { @@ -215,6 +229,7 @@ var _ = common.SIGDescribe("KubeProxy", func() { ipFamily := "ipv4" if netutils.IsIPv6String(ip) { ipFamily = "ipv6" + ip = expandIPv6ForConntrack(ip) } // Obtain the corresponding conntrack entry on the host by reading // /proc/net/nf_conntrack directly from the pod e2e-net-exec. From 1479c62a2054284039104be0d218f986d624502d Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 27 Jan 2026 09:30:49 +0000 Subject: [PATCH 3/4] test: cleanup from review - Use netutils.IsIPv6(ip) instead of manual nil/To4 check - Remove unnecessary ip.To16() call since IPv6 is already 16 bytes - Remove ipFamily from grep pattern since IP format ensures correctness Signed-off-by: Davanum Srinivas --- test/e2e/network/kube_proxy.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/e2e/network/kube_proxy.go b/test/e2e/network/kube_proxy.go index 2632711c39f..44d3d65a500 100644 --- a/test/e2e/network/kube_proxy.go +++ b/test/e2e/network/kube_proxy.go @@ -51,10 +51,9 @@ import ( // e.g., "fc00:f853:ccd:e793::3" -> "fc00:f853:0ccd:e793:0000:0000:0000:0003" func expandIPv6ForConntrack(ipStr string) string { ip := netutils.ParseIPSloppy(ipStr) - if ip == nil || ip.To4() != nil { + if !netutils.IsIPv6(ip) { return ipStr // not IPv6 or invalid, return as-is } - ip = ip.To16() return fmt.Sprintf("%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x", ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]) @@ -226,9 +225,7 @@ var _ = common.SIGDescribe("KubeProxy", func() { const expectedTimeoutSeconds = 60 * 60 // the conntrack file uses the IPv6 expanded format ip := serverNodeInfo.nodeIP - ipFamily := "ipv4" if netutils.IsIPv6String(ip) { - ipFamily = "ipv6" ip = expandIPv6ForConntrack(ip) } // Obtain the corresponding conntrack entry on the host by reading @@ -236,8 +233,8 @@ var _ = common.SIGDescribe("KubeProxy", func() { // This avoids dependency on the conntrack binary. // It retries in a loop if the entry is not found. cmd := fmt.Sprintf("cat /proc/net/nf_conntrack "+ - "| grep -m 1 -E '%s.*CLOSE_WAIT.*dst=%v.*dport=%v'", - ipFamily, ip, testDaemonTCPPort) + "| grep -m 1 -E 'CLOSE_WAIT.*dst=%v.*dport=%v'", + ip, testDaemonTCPPort) if err := wait.PollImmediate(2*time.Second, epsilonSeconds*time.Second, func() (bool, error) { result, err := e2epodoutput.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", cmd) // retry if we can't obtain the conntrack entry From 7be92699eae8f334e233b5987cf0b35efedc071a Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Tue, 27 Jan 2026 10:29:10 +0000 Subject: [PATCH 4/4] Apparently some EC2 images we use do not have /proc/net/nf_conntrack Signed-off-by: Davanum Srinivas --- test/e2e/network/kube_proxy.go | 63 ++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/test/e2e/network/kube_proxy.go b/test/e2e/network/kube_proxy.go index 44d3d65a500..04471002d34 100644 --- a/test/e2e/network/kube_proxy.go +++ b/test/e2e/network/kube_proxy.go @@ -47,12 +47,11 @@ import ( ) // expandIPv6ForConntrack expands an IPv6 address to the format used in /proc/net/nf_conntrack. -// The conntrack file uses fully expanded IPv6 addresses with leading zeros in each group. // e.g., "fc00:f853:ccd:e793::3" -> "fc00:f853:0ccd:e793:0000:0000:0000:0003" func expandIPv6ForConntrack(ipStr string) string { ip := netutils.ParseIPSloppy(ipStr) if !netutils.IsIPv6(ip) { - return ipStr // not IPv6 or invalid, return as-is + return ipStr } return fmt.Sprintf("%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x", ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], @@ -218,52 +217,56 @@ var _ = common.SIGDescribe("KubeProxy", func() { e2epod.NewPodClient(fr).CreateSync(ctx, clientPodSpec) ginkgo.By("Checking conntrack entries for the timeout") - // These must be synchronized from the default values set in - // pkg/apis/../defaults.go ConntrackTCPCloseWaitTimeout. The - // current defaults are hidden in the initialization code. const epsilonSeconds = 60 const expectedTimeoutSeconds = 60 * 60 - // the conntrack file uses the IPv6 expanded format + + // Detect conntrack method and build command ip := serverNodeInfo.nodeIP + ipFamily := "ipv4" if netutils.IsIPv6String(ip) { - ip = expandIPv6ForConntrack(ip) + ipFamily = "ipv6" } - // Obtain the corresponding conntrack entry on the host by reading - // /proc/net/nf_conntrack directly from the pod e2e-net-exec. - // This avoids dependency on the conntrack binary. - // It retries in a loop if the entry is not found. - cmd := fmt.Sprintf("cat /proc/net/nf_conntrack "+ - "| grep -m 1 -E 'CLOSE_WAIT.*dst=%v.*dport=%v'", - ip, testDaemonTCPPort) + + var cmd, dumpCmd string + var timeoutIdx int + if _, err := e2epodoutput.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", "test -f /proc/net/nf_conntrack"); err == nil { + procIP := ip + if ipFamily == "ipv6" { + procIP = expandIPv6ForConntrack(ip) + } + cmd = fmt.Sprintf("cat /proc/net/nf_conntrack | grep -m 1 -E 'CLOSE_WAIT.*dst=%s.*dport=%d'", procIP, testDaemonTCPPort) + dumpCmd = "cat /proc/net/nf_conntrack" + timeoutIdx = 4 // ipv4 2 tcp 6 CLOSE_WAIT ... + } else if _, err := e2epodoutput.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", "which conntrack"); err == nil { + cmd = fmt.Sprintf("conntrack -L -f %s -d %s 2>/dev/null | grep -m 1 'CLOSE_WAIT.*dport=%d'", ipFamily, ip, testDaemonTCPPort) + dumpCmd = "conntrack -L 2>/dev/null" + timeoutIdx = 2 // tcp 6 CLOSE_WAIT ... + } else { + e2eskipper.Skipf("Neither /proc/net/nf_conntrack nor conntrack binary available") + } + if err := wait.PollImmediate(2*time.Second, epsilonSeconds*time.Second, func() (bool, error) { result, err := e2epodoutput.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", cmd) - // retry if we can't obtain the conntrack entry if err != nil { - framework.Logf("failed to obtain conntrack entry: %v %v", result, err) + framework.Logf("failed to obtain conntrack entry: %v", err) return false, nil } - framework.Logf("conntrack entry for node %v and port %v: %v", serverNodeInfo.nodeIP, testDaemonTCPPort, result) - // /proc/net/nf_conntrack format: ipv4 2 tcp 6 src=... dst=... - // Timeout in seconds is available as the fifth column (index 4) - line := strings.Fields(result) - if len(line) < 5 { - return false, fmt.Errorf("conntrack entry does not have a timeout field: %v", line) + fields := strings.Fields(result) + if len(fields) <= timeoutIdx { + return false, nil } - timeoutSeconds, err := strconv.Atoi(line[4]) + timeoutSeconds, err := strconv.Atoi(fields[timeoutIdx]) if err != nil { - return false, fmt.Errorf("failed to convert matched timeout %s to integer: %w", line[4], err) + return false, nil } + framework.Logf("conntrack timeout for %v:%v = %v", serverNodeInfo.nodeIP, testDaemonTCPPort, timeoutSeconds) if math.Abs(float64(timeoutSeconds-expectedTimeoutSeconds)) < epsilonSeconds { return true, nil } return false, fmt.Errorf("wrong TCP CLOSE_WAIT timeout: %v expected: %v", timeoutSeconds, expectedTimeoutSeconds) }); err != nil { - // Dump all conntrack entries for debugging - result, err2 := e2epodoutput.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", "cat /proc/net/nf_conntrack") - if err2 != nil { - framework.Logf("failed to read /proc/net/nf_conntrack: %v %v", result, err2) - } - framework.Logf("conntrack entries for node %v: %v", serverNodeInfo.nodeIP, result) + result, _ := e2epodoutput.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", dumpCmd) + framework.Logf("conntrack entries: %v", result) framework.Failf("no valid conntrack entry for port %d on node %s: %v", testDaemonTCPPort, serverNodeInfo.nodeIP, err) } })