Add IPTablesOwnershipCleanup feature to disable kubelet iptables setup

pkg/features/kube_features.go

@@ -436,6 +436,13 @@ const (
 	// Allows Job controller to manage Pod completions per completion index.
 	IndexedJob featuregate.Feature = "IndexedJob"
 
+	// owner: @danwinship
+	// kep: http://kep.k8s.io/3178
+	// alpha: v1.25
+	//
+	// Causes kubelet to no longer create legacy IPTables rules
+	IPTablesOwnershipCleanup featuregate.Feature = "IPTablesOwnershipCleanup"
+
 	// owner: @ahg
 	// beta: v1.23
 	//
@@ -927,6 +934,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	IndexedJob: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.26
 
+	IPTablesOwnershipCleanup: {Default: false, PreRelease: featuregate.Alpha},
+
 	JobMutableNodeSchedulingDirectives: {Default: true, PreRelease: featuregate.Beta},
 
 	JobReadyPods: {Default: true, PreRelease: featuregate.Beta},

pkg/kubelet/kubelet_network_linux.go

@@ -24,7 +24,9 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/features"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
 	utilexec "k8s.io/utils/exec"
 )
@@ -57,12 +59,12 @@ func (kl *Kubelet) initNetworkUtil() {
 
 	for i := range iptClients {
 		iptClient := iptClients[i]
-		if kl.syncNetworkUtil(iptClient) {
+		if kl.syncIPTablesRules(iptClient) {
 			klog.InfoS("Initialized iptables rules.", "protocol", iptClient.Protocol())
 			go iptClient.Monitor(
 				utiliptables.Chain("KUBE-KUBELET-CANARY"),
 				[]utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter},
-				func() { kl.syncNetworkUtil(iptClient) },
+				func() { kl.syncIPTablesRules(iptClient) },
 				1*time.Minute, wait.NeverStop,
 			)
 		} else {
@@ -71,13 +73,68 @@ func (kl *Kubelet) initNetworkUtil() {
 	}
 }
 
-// syncNetworkUtil ensures the network utility are present on host.
+// syncIPTablesRules ensures the KUBE-IPTABLES-HINT chain exists, and the martian packet
-// Network util includes:
+// protection rule is installed. If the IPTablesOwnershipCleanup feature gate is disabled
+// it will also synchronize additional deprecated iptables rules.
+func (kl *Kubelet) syncIPTablesRules(iptClient utiliptables.Interface) bool {
+	// Create hint chain so other components can see whether we are using iptables-legacy
+	// or iptables-nft.
+	if _, err := iptClient.EnsureChain(utiliptables.TableMangle, KubeIPTablesHintChain); err != nil {
+		klog.ErrorS(err, "Failed to ensure that iptables hint chain exists")
+		return false
+	}
+
+	if !iptClient.IsIPv6() { // ipv6 doesn't have this issue
+		// Set up the KUBE-FIREWALL chain and martian packet protection rule.
+		// (See below.)
+		if _, err := iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil {
+			klog.ErrorS(err, "Failed to ensure that filter table KUBE-FIREWALL chain exists")
+			return false
+		}
+
+		if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil {
+			klog.ErrorS(err, "Failed to ensure that OUTPUT chain jumps to KUBE-FIREWALL")
+			return false
+		}
+		if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil {
+			klog.ErrorS(err, "Failed to ensure that INPUT chain jumps to KUBE-FIREWALL")
+			return false
+		}
+
+		// Kube-proxy's use of `route_localnet` to enable NodePorts on localhost
+		// creates a security hole (https://issue.k8s.io/90259) which this
+		// iptables rule mitigates. This rule should have been added to
+		// kube-proxy, but it mistakenly ended up in kubelet instead, and we are
+		// keeping it in kubelet for now in case other third-party components
+		// depend on it.
+		if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
+			"-m", "comment", "--comment", "block incoming localnet connections",
+			"--dst", "127.0.0.0/8",
+			"!", "--src", "127.0.0.0/8",
+			"-m", "conntrack",
+			"!", "--ctstate", "RELATED,ESTABLISHED,DNAT",
+			"-j", "DROP"); err != nil {
+			klog.ErrorS(err, "Failed to ensure rule to drop invalid localhost packets in filter table KUBE-FIREWALL chain")
+			return false
+		}
+	}
+
+	if !utilfeature.DefaultFeatureGate.Enabled(features.IPTablesOwnershipCleanup) {
+		ok := kl.syncIPTablesRulesDeprecated(iptClient)
+		if !ok {
+			return false
+		}
+	}
+
+	return true
+}
+
+// syncIPTablesRulesDeprecated ensures deprecated iptables rules are present:
 //  1. In nat table, KUBE-MARK-DROP rule to mark connections for dropping
 //     Marked connection will be drop on INPUT/OUTPUT Chain in filter table
 //  2. In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT
 //     Marked connection will get SNAT on POSTROUTING Chain in nat table
-func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) bool {
+func (kl *Kubelet) syncIPTablesRulesDeprecated(iptClient utiliptables.Interface) bool {
 	// Setup KUBE-MARK-DROP rules
 	dropMark := getIPTablesMark(kl.iptablesDropBit)
 	if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil {
@@ -100,30 +157,6 @@ func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) bool {
 		return false
 	}
 
-	// drop all non-local packets to localhost if they're not part of an existing
-	// forwarded connection. See #90259
-	if !iptClient.IsIPv6() { // ipv6 doesn't have this issue
-		if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
-			"-m", "comment", "--comment", "block incoming localnet connections",
-			"--dst", "127.0.0.0/8",
-			"!", "--src", "127.0.0.0/8",
-			"-m", "conntrack",
-			"!", "--ctstate", "RELATED,ESTABLISHED,DNAT",
-			"-j", "DROP"); err != nil {
-			klog.ErrorS(err, "Failed to ensure rule to drop invalid localhost packets exists")
-			return false
-		}
-	}
-
-	if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil {
-		klog.ErrorS(err, "Failed to ensure that OUTPUT chain jumps to KUBE-FIREWALL")
-		return false
-	}
-	if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil {
-		klog.ErrorS(err, "Failed to ensure that INPUT chain jumps to KUBE-FIREWALL")
-		return false
-	}
-
 	// Setup KUBE-MARK-MASQ rules
 	masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit)
 	if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil {
@@ -173,13 +206,6 @@ func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) bool {
 		return false
 	}
 
-	// Create hint chain so other components can see whether we are using iptables-legacy
-	// or iptables-nft.
-	if _, err := iptClient.EnsureChain(utiliptables.TableMangle, KubeIPTablesHintChain); err != nil {
-		klog.ErrorS(err, "Failed to ensure that iptables hint chain exists")
-		return false
-	}
-
 	return true
 }