From 9a7afa69ef5f46339507525452430c87dabee8f2 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Wed, 3 Jun 2020 14:21:14 -0400 Subject: [PATCH] kubelet: do dual-stack iptables rules When the dual-stack feature gate is enabled, set up dual-stack iptables rules. (When it is not enabled, preserve the traditional behavior of setting up IPv4 rules only, unless the user explicitly passed an IPv6 --node-ip.) --- pkg/kubelet/BUILD | 7 ++- pkg/kubelet/kubelet.go | 10 ----- pkg/kubelet/kubelet_network_linux.go | 66 +++++++++++++++++++--------- 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 0fd455c450b..9873957f246 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -153,10 +153,15 @@ go_library( "//vendor/k8s.io/klog/v2:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/integer:go_default_library", - "//vendor/k8s.io/utils/net:go_default_library", "//vendor/k8s.io/utils/path:go_default_library", "//vendor/k8s.io/utils/strings:go_default_library", ] + select({ + "@io_bazel_rules_go//go/platform:android": [ + "//vendor/k8s.io/utils/net:go_default_library", + ], + "@io_bazel_rules_go//go/platform:linux": [ + "//vendor/k8s.io/utils/net:go_default_library", + ], "@io_bazel_rules_go//go/platform:windows": [ "//pkg/kubelet/winstats:go_default_library", ], diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 81f2542d2dc..6d478de0c85 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -35,9 +35,7 @@ import ( cadvisorapi "github.com/google/cadvisor/info/v1" "k8s.io/mount-utils" - utilexec "k8s.io/utils/exec" "k8s.io/utils/integer" - utilnet "k8s.io/utils/net" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -111,7 +109,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/volumemanager" "k8s.io/kubernetes/pkg/security/apparmor" sysctlwhitelist "k8s.io/kubernetes/pkg/security/podsecuritypolicy/sysctl" - utilipt "k8s.io/kubernetes/pkg/util/iptables" "k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/util/selinux" "k8s.io/kubernetes/pkg/volume" @@ -466,11 +463,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, } httpClient := &http.Client{} parsedNodeIP := net.ParseIP(nodeIP) - protocol := utilipt.ProtocolIPv4 - if utilnet.IsIPv6(parsedNodeIP) { - klog.V(0).Infof("IPv6 node IP (%s), assume IPv6 operation", nodeIP) - protocol = utilipt.ProtocolIPv6 - } klet := &Kubelet{ hostname: hostname, @@ -518,7 +510,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, nodeIPValidator: validateNodeIP, clock: clock.RealClock{}, enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach, - iptClient: utilipt.New(utilexec.New(), protocol), makeIPTablesUtilChains: kubeCfg.MakeIPTablesUtilChains, iptablesMasqueradeBit: int(kubeCfg.IPTablesMasqueradeBit), iptablesDropBit: int(kubeCfg.IPTablesDropBit), @@ -819,7 +810,6 @@ type Kubelet struct { runtimeCache kubecontainer.RuntimeCache kubeClient clientset.Interface heartbeatClient clientset.Interface - iptClient utilipt.Interface rootDirectory string lastObservedNodeAddressesMux sync.RWMutex diff --git a/pkg/kubelet/kubelet_network_linux.go b/pkg/kubelet/kubelet_network_linux.go index f7fa4dba681..e7646e590e7 100644 --- a/pkg/kubelet/kubelet_network_linux.go +++ b/pkg/kubelet/kubelet_network_linux.go @@ -23,15 +23,39 @@ import ( "time" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/features" utiliptables "k8s.io/kubernetes/pkg/util/iptables" + utilexec "k8s.io/utils/exec" + utilnet "k8s.io/utils/net" ) func (kl *Kubelet) initNetworkUtil() { - kl.syncNetworkUtil() - go kl.iptClient.Monitor(utiliptables.Chain("KUBE-KUBELET-CANARY"), - []utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter}, - kl.syncNetworkUtil, 1*time.Minute, wait.NeverStop) + exec := utilexec.New() + + // At this point in startup we don't know the actual node IPs, so we configure dual stack iptables + // rules if the node _might_ be dual-stack, and single-stack based on requested nodeIP otherwise. + maybeDualStack := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) + ipv6Primary := kl.nodeIP != nil && utilnet.IsIPv6(kl.nodeIP) + + var iptClients []utiliptables.Interface + if maybeDualStack || !ipv6Primary { + iptClients = append(iptClients, utiliptables.New(exec, utiliptables.ProtocolIPv4)) + } + if maybeDualStack || ipv6Primary { + iptClients = append(iptClients, utiliptables.New(exec, utiliptables.ProtocolIPv6)) + } + + for _, iptClient := range iptClients { + kl.syncNetworkUtil(iptClient) + go iptClient.Monitor( + utiliptables.Chain("KUBE-KUBELET-CANARY"), + []utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter}, + func() { kl.syncNetworkUtil(iptClient) }, + 1*time.Minute, wait.NeverStop, + ) + } } // syncNetworkUtil ensures the network utility are present on host. @@ -40,22 +64,22 @@ func (kl *Kubelet) initNetworkUtil() { // Marked connection will be drop on INPUT/OUTPUT Chain in filter table // 2. In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT // Marked connection will get SNAT on POSTROUTING Chain in nat table -func (kl *Kubelet) syncNetworkUtil() { +func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) { // Setup KUBE-MARK-DROP rules dropMark := getIPTablesMark(kl.iptablesDropBit) - if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil { + if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil { klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err) return } - if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--or-mark", dropMark); err != nil { + if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--or-mark", dropMark); err != nil { klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err) return } - if _, err := kl.iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil { + if _, err := iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil { klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableFilter, KubeFirewallChain, err) return } - if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, + if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, "-m", "comment", "--comment", "kubernetes firewall for dropping marked packets", "-m", "mark", "--mark", fmt.Sprintf("%s/%s", dropMark, dropMark), "-j", "DROP"); err != nil { @@ -65,8 +89,8 @@ func (kl *Kubelet) syncNetworkUtil() { // drop all non-local packets to localhost if they're not part of an existing // forwarded connection. See #90259 - if !kl.iptClient.IsIPv6() { // ipv6 doesn't have this issue - if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, + if !iptClient.IsIPv6() { // ipv6 doesn't have this issue + if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, "-m", "comment", "--comment", "block incoming localnet connections", "--dst", "127.0.0.0/8", "!", "--src", "127.0.0.0/8", @@ -78,30 +102,30 @@ func (kl *Kubelet) syncNetworkUtil() { } } - if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil { + if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil { klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainOutput, KubeFirewallChain, err) return } - if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil { + if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil { klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainInput, KubeFirewallChain, err) return } // Setup KUBE-MARK-MASQ rules masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit) - if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil { + if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil { klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkMasqChain, err) return } - if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil { + if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil { klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err) return } - if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--or-mark", masqueradeMark); err != nil { + if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--or-mark", masqueradeMark); err != nil { klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err) return } - if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting, + if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting, "-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil { klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err) return @@ -110,7 +134,7 @@ func (kl *Kubelet) syncNetworkUtil() { // Set up KUBE-POSTROUTING to unmark and masquerade marked packets // NB: THIS MUST MATCH the corresponding code in the iptables and ipvs // modes of kube-proxy - if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, + if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, "-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", masqueradeMark, masqueradeMark), "-j", "RETURN"); err != nil { klog.Errorf("Failed to ensure filtering rule for %v: %v", KubePostroutingChain, err) @@ -119,7 +143,7 @@ func (kl *Kubelet) syncNetworkUtil() { // Clear the mark to avoid re-masquerading if the packet re-traverses the network stack. // We know the mark bit is currently set so we can use --xor-mark to clear it (without needing // to Sprintf another bitmask). - if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, + if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, "-j", "MARK", "--xor-mark", masqueradeMark); err != nil { klog.Errorf("Failed to ensure unmarking rule for %v: %v", KubePostroutingChain, err) return @@ -128,10 +152,10 @@ func (kl *Kubelet) syncNetworkUtil() { "-m", "comment", "--comment", "kubernetes service traffic requiring SNAT", "-j", "MASQUERADE", } - if kl.iptClient.HasRandomFully() { + if iptClient.HasRandomFully() { masqRule = append(masqRule, "--random-fully") } - if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, masqRule...); err != nil { + if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, masqRule...); err != nil { klog.Errorf("Failed to ensure SNAT rule for packets marked by %v in %v chain %v: %v", KubeMarkMasqChain, utiliptables.TableNAT, KubePostroutingChain, err) return }