From fdf22533a8cdaae6e33d074c57f3a43e8fe2a460 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 18 Apr 2024 09:51:40 -0400 Subject: [PATCH 1/3] KEP-3866 kube-proxy nftables mode to beta --- pkg/features/kube_features.go | 3 ++- pkg/proxy/apis/config/validation/validation_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 07cc6e29097..fcc7cb5c93f 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -525,6 +525,7 @@ const ( // owner: @danwinship // kep: https://kep.k8s.io/3866 // alpha: v1.29 + // beta: v1.31 // // Allows running kube-proxy with `--mode nftables`. NFTablesProxyMode featuregate.Feature = "NFTablesProxyMode" @@ -1131,7 +1132,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS NewVolumeManagerReconstruction: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.32 - NFTablesProxyMode: {Default: false, PreRelease: featuregate.Alpha}, + NFTablesProxyMode: {Default: true, PreRelease: featuregate.Beta}, NodeLogQuery: {Default: false, PreRelease: featuregate.Beta}, diff --git a/pkg/proxy/apis/config/validation/validation_test.go b/pkg/proxy/apis/config/validation/validation_test.go index d9cc7283aa9..da56793df4a 100644 --- a/pkg/proxy/apis/config/validation/validation_test.go +++ b/pkg/proxy/apis/config/validation/validation_test.go @@ -827,7 +827,7 @@ func TestValidateKubeProxyConntrackConfiguration(t *testing.T) { func TestValidateProxyMode(t *testing.T) { newPath := field.NewPath("KubeProxyConfiguration") successCases := []kubeproxyconfig.ProxyMode{""} - expectedNonExistentErrorMsg := "must be iptables, ipvs or blank (blank means the best-available proxy [currently iptables])" + expectedNonExistentErrorMsg := "must be iptables, ipvs, nftables or blank (blank means the best-available proxy [currently iptables])" if runtime.GOOS == "windows" { successCases = append(successCases, kubeproxyconfig.ProxyModeKernelspace) From 2055a490e8780912ff7d73260400b62457c88eab Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 18 Apr 2024 10:02:00 -0400 Subject: [PATCH 2/3] Rework cluster/ kube-proxy setup for --proxy-mode --- cluster/gce/gci/configure-helper.sh | 39 ++++++++++++++++------------- 1 file changed, 22 insertions(+), 17 deletions(-) mode change 100644 => 100755 cluster/gce/gci/configure-helper.sh diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh old mode 100644 new mode 100755 index 5d1271d166a..7a938eafb5b --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1753,24 +1753,29 @@ function prepare-kube-proxy-manifest-variables { if [[ -n "${FEATURE_GATES:-}" ]]; then params+=" --feature-gates=${FEATURE_GATES}" fi - if [[ "${KUBE_PROXY_MODE:-}" == "ipvs" ]];then - # use 'nf_conntrack' instead of 'nf_conntrack_ipv4' for linux kernel >= 4.19 - # https://github.com/kubernetes/kubernetes/pull/70398 - local -r kernel_version=$(uname -r | cut -d\. -f1,2) - local conntrack_module="nf_conntrack" - if [[ $(printf '%s\n4.18\n' "${kernel_version}" | sort -V | tail -1) == "4.18" ]]; then - conntrack_module="nf_conntrack_ipv4" - fi - if sudo modprobe -a ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh ${conntrack_module}; then - params+=" --proxy-mode=ipvs" - else - # If IPVS modules are not present, make sure the node does not come up as - # healthy. - exit 1 - fi - fi - params+=" --iptables-sync-period=1m --iptables-min-sync-period=10s --ipvs-sync-period=1m --ipvs-min-sync-period=10s" + case "${KUBE_PROXY_MODE:-iptables}" in + iptables) + params+=" --proxy-mode=iptables --iptables-sync-period=1m --iptables-min-sync-period=10s" + ;; + ipvs) + # use 'nf_conntrack' instead of 'nf_conntrack_ipv4' for linux kernel >= 4.19 + # https://github.com/kubernetes/kubernetes/pull/70398 + local -r kernel_version=$(uname -r | cut -d\. -f1,2) + local conntrack_module="nf_conntrack" + if [[ $(printf '%s\n4.18\n' "${kernel_version}" | sort -V | tail -1) == "4.18" ]]; then + conntrack_module="nf_conntrack_ipv4" + fi + + if ! sudo modprobe -a ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh ${conntrack_module}; then + # If IPVS modules are not present, make sure the node does not come up as + # healthy. + exit 1 + fi + params+=" --proxy-mode=ipvs --ipvs-sync-period=1m --ipvs-min-sync-period=10s" + ;; + esac + if [[ -n "${KUBEPROXY_TEST_ARGS:-}" ]]; then params+=" ${KUBEPROXY_TEST_ARGS}" fi From 9f580afa66a039d9865f1024e04222849ebb5305 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 18 Apr 2024 10:02:00 -0400 Subject: [PATCH 3/3] cluster: configure nftables mode for kube-proxy (based on a patch from Antonio Ojea) --- cluster/gce/config-test.sh | 2 +- cluster/gce/gci/configure-helper.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 4db770315c4..b12fe3a73ba 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -532,7 +532,7 @@ KUBE_PROXY_DAEMONSET=${KUBE_PROXY_DAEMONSET:-false} # true, false # as an addon daemonset. KUBE_PROXY_DISABLE="${KUBE_PROXY_DISABLE:-false}" # true, false -# Optional: Change the kube-proxy implementation. Choices are [iptables, ipvs]. +# Optional: Change the kube-proxy implementation. Choices are [iptables, ipvs, nftables]. KUBE_PROXY_MODE=${KUBE_PROXY_MODE:-iptables} # Will be passed into the kube-proxy via `--detect-local-mode` diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 7a938eafb5b..168115a4424 100755 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1774,6 +1774,12 @@ function prepare-kube-proxy-manifest-variables { fi params+=" --proxy-mode=ipvs --ipvs-sync-period=1m --ipvs-min-sync-period=10s" ;; + nftables) + # Pass --conntrack-tcp-be-liberal so we can test that this makes the + # "proxy implementation should not be vulnerable to the invalid conntrack state bug" + # test pass. https://issues.k8s.io/122663#issuecomment-1885024015 + params+=" --proxy-mode=nftables --conntrack-tcp-be-liberal" + ;; esac if [[ -n "${KUBEPROXY_TEST_ARGS:-}" ]]; then