From 3fdece285b651a853a5b35a22d8f8b4ed5bbdd5a Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 18 Feb 2022 12:59:05 -0500 Subject: [PATCH] Add IPTablesOwnershipCleanup feature to disable kubelet iptables setup --- pkg/features/kube_features.go | 9 +++ pkg/kubelet/kubelet_network_linux.go | 98 ++++++++++++++++++---------- 2 files changed, 71 insertions(+), 36 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index c2a4709c3aa..f6eae9234ba 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -436,6 +436,13 @@ const ( // Allows Job controller to manage Pod completions per completion index. IndexedJob featuregate.Feature = "IndexedJob" + // owner: @danwinship + // kep: http://kep.k8s.io/3178 + // alpha: v1.25 + // + // Causes kubelet to no longer create legacy IPTables rules + IPTablesOwnershipCleanup featuregate.Feature = "IPTablesOwnershipCleanup" + // owner: @ahg // beta: v1.23 // @@ -927,6 +934,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS IndexedJob: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.26 + IPTablesOwnershipCleanup: {Default: false, PreRelease: featuregate.Alpha}, + JobMutableNodeSchedulingDirectives: {Default: true, PreRelease: featuregate.Beta}, JobReadyPods: {Default: true, PreRelease: featuregate.Beta}, diff --git a/pkg/kubelet/kubelet_network_linux.go b/pkg/kubelet/kubelet_network_linux.go index 362a247dbdf..8f767817a0f 100644 --- a/pkg/kubelet/kubelet_network_linux.go +++ b/pkg/kubelet/kubelet_network_linux.go @@ -24,7 +24,9 @@ import ( "time" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" utiliptables "k8s.io/kubernetes/pkg/util/iptables" utilexec "k8s.io/utils/exec" ) @@ -57,12 +59,12 @@ func (kl *Kubelet) initNetworkUtil() { for i := range iptClients { iptClient := iptClients[i] - if kl.syncNetworkUtil(iptClient) { + if kl.syncIPTablesRules(iptClient) { klog.InfoS("Initialized iptables rules.", "protocol", iptClient.Protocol()) go iptClient.Monitor( utiliptables.Chain("KUBE-KUBELET-CANARY"), []utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter}, - func() { kl.syncNetworkUtil(iptClient) }, + func() { kl.syncIPTablesRules(iptClient) }, 1*time.Minute, wait.NeverStop, ) } else { @@ -71,13 +73,68 @@ func (kl *Kubelet) initNetworkUtil() { } } -// syncNetworkUtil ensures the network utility are present on host. -// Network util includes: +// syncIPTablesRules ensures the KUBE-IPTABLES-HINT chain exists, and the martian packet +// protection rule is installed. If the IPTablesOwnershipCleanup feature gate is disabled +// it will also synchronize additional deprecated iptables rules. +func (kl *Kubelet) syncIPTablesRules(iptClient utiliptables.Interface) bool { + // Create hint chain so other components can see whether we are using iptables-legacy + // or iptables-nft. + if _, err := iptClient.EnsureChain(utiliptables.TableMangle, KubeIPTablesHintChain); err != nil { + klog.ErrorS(err, "Failed to ensure that iptables hint chain exists") + return false + } + + if !iptClient.IsIPv6() { // ipv6 doesn't have this issue + // Set up the KUBE-FIREWALL chain and martian packet protection rule. + // (See below.) + if _, err := iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil { + klog.ErrorS(err, "Failed to ensure that filter table KUBE-FIREWALL chain exists") + return false + } + + if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil { + klog.ErrorS(err, "Failed to ensure that OUTPUT chain jumps to KUBE-FIREWALL") + return false + } + if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil { + klog.ErrorS(err, "Failed to ensure that INPUT chain jumps to KUBE-FIREWALL") + return false + } + + // Kube-proxy's use of `route_localnet` to enable NodePorts on localhost + // creates a security hole (https://issue.k8s.io/90259) which this + // iptables rule mitigates. This rule should have been added to + // kube-proxy, but it mistakenly ended up in kubelet instead, and we are + // keeping it in kubelet for now in case other third-party components + // depend on it. + if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, + "-m", "comment", "--comment", "block incoming localnet connections", + "--dst", "127.0.0.0/8", + "!", "--src", "127.0.0.0/8", + "-m", "conntrack", + "!", "--ctstate", "RELATED,ESTABLISHED,DNAT", + "-j", "DROP"); err != nil { + klog.ErrorS(err, "Failed to ensure rule to drop invalid localhost packets in filter table KUBE-FIREWALL chain") + return false + } + } + + if !utilfeature.DefaultFeatureGate.Enabled(features.IPTablesOwnershipCleanup) { + ok := kl.syncIPTablesRulesDeprecated(iptClient) + if !ok { + return false + } + } + + return true +} + +// syncIPTablesRulesDeprecated ensures deprecated iptables rules are present: // 1. In nat table, KUBE-MARK-DROP rule to mark connections for dropping // Marked connection will be drop on INPUT/OUTPUT Chain in filter table // 2. In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT // Marked connection will get SNAT on POSTROUTING Chain in nat table -func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) bool { +func (kl *Kubelet) syncIPTablesRulesDeprecated(iptClient utiliptables.Interface) bool { // Setup KUBE-MARK-DROP rules dropMark := getIPTablesMark(kl.iptablesDropBit) if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil { @@ -100,30 +157,6 @@ func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) bool { return false } - // drop all non-local packets to localhost if they're not part of an existing - // forwarded connection. See #90259 - if !iptClient.IsIPv6() { // ipv6 doesn't have this issue - if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, - "-m", "comment", "--comment", "block incoming localnet connections", - "--dst", "127.0.0.0/8", - "!", "--src", "127.0.0.0/8", - "-m", "conntrack", - "!", "--ctstate", "RELATED,ESTABLISHED,DNAT", - "-j", "DROP"); err != nil { - klog.ErrorS(err, "Failed to ensure rule to drop invalid localhost packets exists") - return false - } - } - - if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil { - klog.ErrorS(err, "Failed to ensure that OUTPUT chain jumps to KUBE-FIREWALL") - return false - } - if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil { - klog.ErrorS(err, "Failed to ensure that INPUT chain jumps to KUBE-FIREWALL") - return false - } - // Setup KUBE-MARK-MASQ rules masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit) if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil { @@ -173,13 +206,6 @@ func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) bool { return false } - // Create hint chain so other components can see whether we are using iptables-legacy - // or iptables-nft. - if _, err := iptClient.EnsureChain(utiliptables.TableMangle, KubeIPTablesHintChain); err != nil { - klog.ErrorS(err, "Failed to ensure that iptables hint chain exists") - return false - } - return true }