github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 19:31:44 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #94474 from danwinship/dual-stack-kubelet-iptables

Browse Source 
DualStack: kubelet: do dual-stack iptables rules
This commit is contained in:
Kubernetes Prow Robot 2020-10-07 01:20:42 -07:00 committed by GitHub
commit 13be1cb571
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 51 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/kubelet/BUILD
View File

@ -153,10 +153,15 @@ go_library(
"//vendor/k8s.io/klog/v2:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library",
"//vendor/k8s.io/utils/integer:go_default_library", "//vendor/k8s.io/utils/integer:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
"//vendor/k8s.io/utils/path:go_default_library", "//vendor/k8s.io/utils/path:go_default_library",
"//vendor/k8s.io/utils/strings:go_default_library", "//vendor/k8s.io/utils/strings:go_default_library",
] + select({ ] + select({
"@io_bazel_rules_go//go/platform:android": [
"//vendor/k8s.io/utils/net:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/k8s.io/utils/net:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [ "@io_bazel_rules_go//go/platform:windows": [
"//pkg/kubelet/winstats:go_default_library", "//pkg/kubelet/winstats:go_default_library",
], ],

10
pkg/kubelet/kubelet.go
View File

@ -35,9 +35,7 @@ import (
cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapi "github.com/google/cadvisor/info/v1"
"k8s.io/mount-utils" "k8s.io/mount-utils"
utilexec "k8s.io/utils/exec"
"k8s.io/utils/integer" "k8s.io/utils/integer"
utilnet "k8s.io/utils/net"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -111,7 +109,6 @@ import (
"k8s.io/kubernetes/pkg/kubelet/volumemanager" "k8s.io/kubernetes/pkg/kubelet/volumemanager"
"k8s.io/kubernetes/pkg/security/apparmor" "k8s.io/kubernetes/pkg/security/apparmor"
sysctlwhitelist "k8s.io/kubernetes/pkg/security/podsecuritypolicy/sysctl" sysctlwhitelist "k8s.io/kubernetes/pkg/security/podsecuritypolicy/sysctl"
utilipt "k8s.io/kubernetes/pkg/util/iptables"
"k8s.io/kubernetes/pkg/util/oom" "k8s.io/kubernetes/pkg/util/oom"
"k8s.io/kubernetes/pkg/util/selinux" "k8s.io/kubernetes/pkg/util/selinux"
"k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume"
@ -466,11 +463,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
} }
httpClient := &http.Client{} httpClient := &http.Client{}
parsedNodeIP := net.ParseIP(nodeIP) parsedNodeIP := net.ParseIP(nodeIP)
protocol := utilipt.ProtocolIPv4
if utilnet.IsIPv6(parsedNodeIP) {
klog.V(0).Infof("IPv6 node IP (%s), assume IPv6 operation", nodeIP)
protocol = utilipt.ProtocolIPv6
}
klet := &Kubelet{ klet := &Kubelet{
hostname: hostname, hostname: hostname,
@ -518,7 +510,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
nodeIPValidator: validateNodeIP, nodeIPValidator: validateNodeIP,
clock: clock.RealClock{}, clock: clock.RealClock{},
enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach, enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach,
iptClient: utilipt.New(utilexec.New(), protocol),
makeIPTablesUtilChains: kubeCfg.MakeIPTablesUtilChains, makeIPTablesUtilChains: kubeCfg.MakeIPTablesUtilChains,
iptablesMasqueradeBit: int(kubeCfg.IPTablesMasqueradeBit), iptablesMasqueradeBit: int(kubeCfg.IPTablesMasqueradeBit),
iptablesDropBit: int(kubeCfg.IPTablesDropBit), iptablesDropBit: int(kubeCfg.IPTablesDropBit),
@ -819,7 +810,6 @@ type Kubelet struct {
runtimeCache kubecontainer.RuntimeCache runtimeCache kubecontainer.RuntimeCache
kubeClient clientset.Interface kubeClient clientset.Interface
heartbeatClient clientset.Interface heartbeatClient clientset.Interface
iptClient utilipt.Interface
rootDirectory string rootDirectory string
lastObservedNodeAddressesMux sync.RWMutex lastObservedNodeAddressesMux sync.RWMutex

81
pkg/kubelet/kubelet_network_linux.go
View File

@ -23,15 +23,39 @@ import (
"time" "time"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2" "k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
utiliptables "k8s.io/kubernetes/pkg/util/iptables" utiliptables "k8s.io/kubernetes/pkg/util/iptables"
utilexec "k8s.io/utils/exec"
utilnet "k8s.io/utils/net"
) )
func (kl *Kubelet) initNetworkUtil() { func (kl *Kubelet) initNetworkUtil() {
kl.syncNetworkUtil() exec := utilexec.New()
go kl.iptClient.Monitor(utiliptables.Chain("KUBE-KUBELET-CANARY"),
[]utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter}, // At this point in startup we don't know the actual node IPs, so we configure dual stack iptables
kl.syncNetworkUtil, 1*time.Minute, wait.NeverStop) // rules if the node _might_ be dual-stack, and single-stack based on requested nodeIP otherwise.
maybeDualStack := utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack)
ipv6Primary := kl.nodeIP != nil && utilnet.IsIPv6(kl.nodeIP)
var iptClients []utiliptables.Interface
if maybeDualStack || !ipv6Primary {
iptClients = append(iptClients, utiliptables.New(exec, utiliptables.ProtocolIPv4))
}
if maybeDualStack || ipv6Primary {
iptClients = append(iptClients, utiliptables.New(exec, utiliptables.ProtocolIPv6))
}
for _, iptClient := range iptClients {
kl.syncNetworkUtil(iptClient)
go iptClient.Monitor(
utiliptables.Chain("KUBE-KUBELET-CANARY"),
[]utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter},
func() { kl.syncNetworkUtil(iptClient) },
1*time.Minute, wait.NeverStop,
)
}
} }
// syncNetworkUtil ensures the network utility are present on host. // syncNetworkUtil ensures the network utility are present on host.
@ -40,37 +64,22 @@ func (kl *Kubelet) initNetworkUtil() {
// Marked connection will be drop on INPUT/OUTPUT Chain in filter table // Marked connection will be drop on INPUT/OUTPUT Chain in filter table
// 2. In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT // 2. In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT
// Marked connection will get SNAT on POSTROUTING Chain in nat table // Marked connection will get SNAT on POSTROUTING Chain in nat table
func (kl *Kubelet) syncNetworkUtil() { func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) {
if kl.iptablesMasqueradeBit < 0 || kl.iptablesMasqueradeBit > 31 {
klog.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", kl.iptablesMasqueradeBit)
return
}
if kl.iptablesDropBit < 0 || kl.iptablesDropBit > 31 {
klog.Errorf("invalid iptables-drop-bit %v not in [0, 31]", kl.iptablesDropBit)
return
}
if kl.iptablesDropBit == kl.iptablesMasqueradeBit {
klog.Errorf("iptables-masquerade-bit %v and iptables-drop-bit %v must be different", kl.iptablesMasqueradeBit, kl.iptablesDropBit)
return
}
// Setup KUBE-MARK-DROP rules // Setup KUBE-MARK-DROP rules
dropMark := getIPTablesMark(kl.iptablesDropBit) dropMark := getIPTablesMark(kl.iptablesDropBit)
if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil { if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil {
klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err) klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err)
return return
} }
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--or-mark", dropMark); err != nil { if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--or-mark", dropMark); err != nil {
klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err) klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err)
return return
} }
if _, err := kl.iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil { if _, err := iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil {
klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableFilter, KubeFirewallChain, err) klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableFilter, KubeFirewallChain, err)
return return
} }
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
"-m", "comment", "--comment", "kubernetes firewall for dropping marked packets", "-m", "comment", "--comment", "kubernetes firewall for dropping marked packets",
"-m", "mark", "--mark", fmt.Sprintf("%s/%s", dropMark, dropMark), "-m", "mark", "--mark", fmt.Sprintf("%s/%s", dropMark, dropMark),
"-j", "DROP"); err != nil { "-j", "DROP"); err != nil {
@ -80,8 +89,8 @@ func (kl *Kubelet) syncNetworkUtil() {
// drop all non-local packets to localhost if they're not part of an existing // drop all non-local packets to localhost if they're not part of an existing
// forwarded connection. See #90259 // forwarded connection. See #90259
if !kl.iptClient.IsIPv6() { // ipv6 doesn't have this issue if !iptClient.IsIPv6() { // ipv6 doesn't have this issue
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain, if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
"-m", "comment", "--comment", "block incoming localnet connections", "-m", "comment", "--comment", "block incoming localnet connections",
"--dst", "127.0.0.0/8", "--dst", "127.0.0.0/8",
"!", "--src", "127.0.0.0/8", "!", "--src", "127.0.0.0/8",
@ -93,30 +102,30 @@ func (kl *Kubelet) syncNetworkUtil() {
} }
} }
if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil { if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil {
klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainOutput, KubeFirewallChain, err) klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainOutput, KubeFirewallChain, err)
return return
} }
if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil { if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil {
klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainInput, KubeFirewallChain, err) klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainInput, KubeFirewallChain, err)
return return
} }
// Setup KUBE-MARK-MASQ rules // Setup KUBE-MARK-MASQ rules
masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit) masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit)
if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil { if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil {
klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkMasqChain, err) klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkMasqChain, err)
return return
} }
if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil { if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil {
klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err) klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err)
return return
} }
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--or-mark", masqueradeMark); err != nil { if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--or-mark", masqueradeMark); err != nil {
klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err) klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err)
return return
} }
if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting, if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting,
"-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil { "-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil {
klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err) klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err)
return return
@ -125,7 +134,7 @@ func (kl *Kubelet) syncNetworkUtil() {
// Set up KUBE-POSTROUTING to unmark and masquerade marked packets // Set up KUBE-POSTROUTING to unmark and masquerade marked packets
// NB: THIS MUST MATCH the corresponding code in the iptables and ipvs // NB: THIS MUST MATCH the corresponding code in the iptables and ipvs
// modes of kube-proxy // modes of kube-proxy
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
"-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", masqueradeMark, masqueradeMark), "-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", masqueradeMark, masqueradeMark),
"-j", "RETURN"); err != nil { "-j", "RETURN"); err != nil {
klog.Errorf("Failed to ensure filtering rule for %v: %v", KubePostroutingChain, err) klog.Errorf("Failed to ensure filtering rule for %v: %v", KubePostroutingChain, err)
@ -134,7 +143,7 @@ func (kl *Kubelet) syncNetworkUtil() {
// Clear the mark to avoid re-masquerading if the packet re-traverses the network stack. // Clear the mark to avoid re-masquerading if the packet re-traverses the network stack.
// We know the mark bit is currently set so we can use --xor-mark to clear it (without needing // We know the mark bit is currently set so we can use --xor-mark to clear it (without needing
// to Sprintf another bitmask). // to Sprintf another bitmask).
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
"-j", "MARK", "--xor-mark", masqueradeMark); err != nil { "-j", "MARK", "--xor-mark", masqueradeMark); err != nil {
klog.Errorf("Failed to ensure unmarking rule for %v: %v", KubePostroutingChain, err) klog.Errorf("Failed to ensure unmarking rule for %v: %v", KubePostroutingChain, err)
return return
@ -143,10 +152,10 @@ func (kl *Kubelet) syncNetworkUtil() {
"-m", "comment", "--comment", "kubernetes service traffic requiring SNAT", "-m", "comment", "--comment", "kubernetes service traffic requiring SNAT",
"-j", "MASQUERADE", "-j", "MASQUERADE",
} }
if kl.iptClient.HasRandomFully() { if iptClient.HasRandomFully() {
masqRule = append(masqRule, "--random-fully") masqRule = append(masqRule, "--random-fully")
} }
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, masqRule...); err != nil { if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, masqRule...); err != nil {
klog.Errorf("Failed to ensure SNAT rule for packets marked by %v in %v chain %v: %v", KubeMarkMasqChain, utiliptables.TableNAT, KubePostroutingChain, err) klog.Errorf("Failed to ensure SNAT rule for packets marked by %v in %v chain %v: %v", KubeMarkMasqChain, utiliptables.TableNAT, KubePostroutingChain, err)
return return
} }