Revert "syncNetworkUtil in kubelet and fix loadbalancerSourceRange on GCE"

This commit is contained in:
Daniel Smith 2016-08-16 18:12:28 -07:00 committed by GitHub
parent d412d5721d
commit 2aa0bb2dfc
15 changed files with 2553 additions and 3008 deletions

View File

@ -204,7 +204,8 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err
// IPTablesMasqueradeBit must be specified or defaulted. // IPTablesMasqueradeBit must be specified or defaulted.
return nil, fmt.Errorf("Unable to read IPTablesMasqueradeBit from config") return nil, fmt.Errorf("Unable to read IPTablesMasqueradeBit from config")
} }
proxierIptables, err := iptables.NewProxier(iptInterface, execer, config.IPTablesSyncPeriod.Duration, config.MasqueradeAll, int(*config.IPTablesMasqueradeBit), config.ClusterCIDR, hostname, getNodeIP(client, hostname))
proxierIptables, err := iptables.NewProxier(iptInterface, execer, config.IPTablesSyncPeriod.Duration, config.MasqueradeAll, int(*config.IPTablesMasqueradeBit), config.ClusterCIDR, hostname)
if err != nil { if err != nil {
glog.Fatalf("Unable to create proxier: %v", err) glog.Fatalf("Unable to create proxier: %v", err)
} }
@ -408,18 +409,3 @@ func tryIptablesProxy(iptver iptables.IptablesVersioner, kcompat iptables.Kernel
func (s *ProxyServer) birthCry() { func (s *ProxyServer) birthCry() {
s.Recorder.Eventf(s.Config.NodeRef, api.EventTypeNormal, "Starting", "Starting kube-proxy.") s.Recorder.Eventf(s.Config.NodeRef, api.EventTypeNormal, "Starting", "Starting kube-proxy.")
} }
func getNodeIP(client *kubeclient.Client, hostname string) net.IP {
var nodeIP net.IP
node, err := client.Nodes().Get(hostname)
if err != nil {
glog.Warningf("Failed to retrieve node info: %v", err)
return nil
}
nodeIP, err = nodeutil.GetNodeHostIP(node)
if err != nil {
glog.Warningf("Failed to retrieve node IP: %v", err)
return nil
}
return nodeIP
}

View File

@ -164,9 +164,6 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.ResolverConfig, "resolv-conf", s.ResolverConfig, "Resolver configuration file used as the basis for the container DNS resolution configuration.") fs.StringVar(&s.ResolverConfig, "resolv-conf", s.ResolverConfig, "Resolver configuration file used as the basis for the container DNS resolution configuration.")
fs.BoolVar(&s.CPUCFSQuota, "cpu-cfs-quota", s.CPUCFSQuota, "Enable CPU CFS quota enforcement for containers that specify CPU limits") fs.BoolVar(&s.CPUCFSQuota, "cpu-cfs-quota", s.CPUCFSQuota, "Enable CPU CFS quota enforcement for containers that specify CPU limits")
fs.BoolVar(&s.EnableControllerAttachDetach, "enable-controller-attach-detach", s.EnableControllerAttachDetach, "Enables the Attach/Detach controller to manage attachment/detachment of volumes scheduled to this node, and disables kubelet from executing any attach/detach operations") fs.BoolVar(&s.EnableControllerAttachDetach, "enable-controller-attach-detach", s.EnableControllerAttachDetach, "Enables the Attach/Detach controller to manage attachment/detachment of volumes scheduled to this node, and disables kubelet from executing any attach/detach operations")
fs.BoolVar(&s.MakeIPTablesUtilChains, "make-iptables-util-chains", s.MakeIPTablesUtilChains, "If true, kubelet will ensure iptables utility rules are present on host.")
fs.Int32Var(&s.IPTablesMasqueradeBit, "iptables-masquerade-bit", s.IPTablesMasqueradeBit, "The bit of the fwmark space to mark packets for SNAT. Must be within the range [0, 31]. Please match this parameter with corresponding parameter in kube-proxy.")
fs.Int32Var(&s.IPTablesDropBit, "iptables-drop-bit", s.IPTablesDropBit, "The bit of the fwmark space to mark packets for dropping. Must be within the range [0, 31].")
// Flags intended for testing, not recommended used in production environments. // Flags intended for testing, not recommended used in production environments.
fs.BoolVar(&s.ReallyCrashForTesting, "really-crash-for-testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.") fs.BoolVar(&s.ReallyCrashForTesting, "really-crash-for-testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.")

View File

@ -199,18 +199,6 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
Thresholds: thresholds, Thresholds: thresholds,
} }
if s.MakeIPTablesUtilChains {
if s.IPTablesMasqueradeBit > 31 || s.IPTablesMasqueradeBit < 0 {
return nil, fmt.Errorf("iptables-masquerade-bit is not valid. Must be within [0, 31]")
}
if s.IPTablesDropBit > 31 || s.IPTablesDropBit < 0 {
return nil, fmt.Errorf("iptables-drop-bit is not valid. Must be within [0, 31]")
}
if s.IPTablesDropBit == s.IPTablesMasqueradeBit {
return nil, fmt.Errorf("iptables-masquerade-bit and iptables-drop-bit must be different")
}
}
return &KubeletConfig{ return &KubeletConfig{
Address: net.ParseIP(s.Address), Address: net.ParseIP(s.Address),
AllowPrivileged: s.AllowPrivileged, AllowPrivileged: s.AllowPrivileged,
@ -293,13 +281,10 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
HairpinMode: s.HairpinMode, HairpinMode: s.HairpinMode,
BabysitDaemons: s.BabysitDaemons, BabysitDaemons: s.BabysitDaemons,
ExperimentalFlannelOverlay: s.ExperimentalFlannelOverlay, ExperimentalFlannelOverlay: s.ExperimentalFlannelOverlay,
NodeIP: net.ParseIP(s.NodeIP), NodeIP: net.ParseIP(s.NodeIP),
EvictionConfig: evictionConfig, EvictionConfig: evictionConfig,
PodsPerCore: int(s.PodsPerCore), PodsPerCore: int(s.PodsPerCore),
ProtectKernelDefaults: s.ProtectKernelDefaults, ProtectKernelDefaults: s.ProtectKernelDefaults,
MakeIPTablesUtilChains: s.MakeIPTablesUtilChains,
iptablesMasqueradeBit: int(s.IPTablesMasqueradeBit),
iptablesDropBit: int(s.IPTablesDropBit),
}, nil }, nil
} }
@ -908,10 +893,8 @@ type KubeletConfig struct {
HairpinMode string HairpinMode string
BabysitDaemons bool BabysitDaemons bool
Options []kubelet.Option Options []kubelet.Option
ProtectKernelDefaults bool
MakeIPTablesUtilChains bool ProtectKernelDefaults bool
iptablesMasqueradeBit int
iptablesDropBit int
} }
func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) { func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) {
@ -1009,9 +992,6 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
kc.EvictionConfig, kc.EvictionConfig,
kc.Options, kc.Options,
kc.EnableControllerAttachDetach, kc.EnableControllerAttachDetach,
kc.MakeIPTablesUtilChains,
kc.iptablesMasqueradeBit,
kc.iptablesDropBit,
) )
if err != nil { if err != nil {

View File

@ -230,7 +230,6 @@ insecure-port
insecure-skip-tls-verify insecure-skip-tls-verify
instance-metadata instance-metadata
instance-name-prefix instance-name-prefix
iptables-drop-bit
iptables-masquerade-bit iptables-masquerade-bit
iptables-sync-period iptables-sync-period
ir-data-source ir-data-source
@ -289,7 +288,6 @@ lock-file
log-flush-frequency log-flush-frequency
long-running-request-regexp long-running-request-regexp
low-diskspace-threshold-mb low-diskspace-threshold-mb
make-iptables-util-chains
make-symlinks make-symlinks
manifest-url manifest-url
manifest-url-header manifest-url-header

File diff suppressed because it is too large Load Diff

View File

@ -399,18 +399,6 @@ type KubeletConfiguration struct {
KubeReserved utilconfig.ConfigurationMap `json:"kubeReserved"` KubeReserved utilconfig.ConfigurationMap `json:"kubeReserved"`
// Default behaviour for kernel tuning // Default behaviour for kernel tuning
ProtectKernelDefaults bool `json:"protectKernelDefaults"` ProtectKernelDefaults bool `json:"protectKernelDefaults"`
// If true, Kubelet ensures a set of iptables rules are present on host.
// These rules will serve as utility for various components, e.g. kube-proxy.
// The rules will be created based on IPTablesMasqueradeBit and IPTablesDropBit.
MakeIPTablesUtilChains bool `json:"makeIPTablesUtilChains"`
// iptablesMasqueradeBit is the bit of the iptables fwmark space to use for SNAT
// Values must be within the range [0, 31].
// Warning: Please match the value of corresponding parameter in kube-proxy
// TODO: clean up IPTablesMasqueradeBit in kube-proxy
IPTablesMasqueradeBit int32 `json:"iptablesMasqueradeBit"`
// iptablesDropBit is the bit of the iptables fwmark space to use for dropping packets. Kubelet will ensure iptables mark and drop rules.
// Values must be within the range [0, 31]. Must be different from IPTablesMasqueradeBit
IPTablesDropBit int32 `json:"iptablesDropBit"`
} }
type KubeSchedulerConfiguration struct { type KubeSchedulerConfiguration struct {

View File

@ -43,9 +43,6 @@ const (
defaultRktAPIServiceEndpoint = "localhost:15441" defaultRktAPIServiceEndpoint = "localhost:15441"
AutoDetectCloudProvider = "auto-detect" AutoDetectCloudProvider = "auto-detect"
defaultIPTablesMasqueradeBit = 14
defaultIPTablesDropBit = 15
) )
var zeroDuration = unversioned.Duration{} var zeroDuration = unversioned.Duration{}
@ -340,17 +337,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
if obj.KubeReserved == nil { if obj.KubeReserved == nil {
obj.KubeReserved = make(map[string]string) obj.KubeReserved = make(map[string]string)
} }
if obj.MakeIPTablesUtilChains == nil {
obj.MakeIPTablesUtilChains = boolVar(true)
}
if obj.IPTablesMasqueradeBit == nil {
temp := int32(defaultIPTablesMasqueradeBit)
obj.IPTablesMasqueradeBit = &temp
}
if obj.IPTablesDropBit == nil {
temp := int32(defaultIPTablesDropBit)
obj.IPTablesDropBit = &temp
}
} }
func boolVar(b bool) *bool { func boolVar(b bool) *bool {

View File

@ -454,16 +454,4 @@ type KubeletConfiguration struct {
KubeReserved map[string]string `json:"kubeReserved"` KubeReserved map[string]string `json:"kubeReserved"`
// Default behaviour for kernel tuning // Default behaviour for kernel tuning
ProtectKernelDefaults bool `json:"protectKernelDefaults"` ProtectKernelDefaults bool `json:"protectKernelDefaults"`
// If true, Kubelet ensures a set of iptables rules are present on host.
// These rules will serve as utility rules for various components, e.g. KubeProxy.
// The rules will be created based on IPTablesMasqueradeBit and IPTablesDropBit.
MakeIPTablesUtilChains *bool `json:"makeIPTablesUtilChains"`
// iptablesMasqueradeBit is the bit of the iptables fwmark space to mark for SNAT
// Values must be within the range [0, 31]. Must be different from other mark bits.
// Warning: Please match the value of corresponding parameter in kube-proxy
// TODO: clean up IPTablesMasqueradeBit in kube-proxy
IPTablesMasqueradeBit *int32 `json:"iptablesMasqueradeBit"`
// iptablesDropBit is the bit of the iptables fwmark space to mark for dropping packets.
// Values must be within the range [0, 31]. Must be different from other mark bits.
IPTablesDropBit *int32 `json:"iptablesDropBit"`
} }

View File

@ -320,15 +320,6 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
out.KubeReserved = nil out.KubeReserved = nil
} }
out.ProtectKernelDefaults = in.ProtectKernelDefaults out.ProtectKernelDefaults = in.ProtectKernelDefaults
if err := api.Convert_Pointer_bool_To_bool(&in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains, s); err != nil {
return err
}
if err := api.Convert_Pointer_int32_To_int32(&in.IPTablesMasqueradeBit, &out.IPTablesMasqueradeBit, s); err != nil {
return err
}
if err := api.Convert_Pointer_int32_To_int32(&in.IPTablesDropBit, &out.IPTablesDropBit, s); err != nil {
return err
}
return nil return nil
} }
@ -494,15 +485,6 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
out.KubeReserved = nil out.KubeReserved = nil
} }
out.ProtectKernelDefaults = in.ProtectKernelDefaults out.ProtectKernelDefaults = in.ProtectKernelDefaults
if err := api.Convert_bool_To_Pointer_bool(&in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains, s); err != nil {
return err
}
if err := api.Convert_int32_To_Pointer_int32(&in.IPTablesMasqueradeBit, &out.IPTablesMasqueradeBit, s); err != nil {
return err
}
if err := api.Convert_int32_To_Pointer_int32(&in.IPTablesDropBit, &out.IPTablesDropBit, s); err != nil {
return err
}
return nil return nil
} }

View File

@ -378,27 +378,6 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
out.KubeReserved = nil out.KubeReserved = nil
} }
out.ProtectKernelDefaults = in.ProtectKernelDefaults out.ProtectKernelDefaults = in.ProtectKernelDefaults
if in.MakeIPTablesUtilChains != nil {
in, out := &in.MakeIPTablesUtilChains, &out.MakeIPTablesUtilChains
*out = new(bool)
**out = **in
} else {
out.MakeIPTablesUtilChains = nil
}
if in.IPTablesMasqueradeBit != nil {
in, out := &in.IPTablesMasqueradeBit, &out.IPTablesMasqueradeBit
*out = new(int32)
**out = **in
} else {
out.IPTablesMasqueradeBit = nil
}
if in.IPTablesDropBit != nil {
in, out := &in.IPTablesDropBit, &out.IPTablesDropBit
*out = new(int32)
**out = **in
} else {
out.IPTablesDropBit = nil
}
return nil return nil
} }
} }

View File

@ -326,9 +326,6 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface
out.KubeReserved = nil out.KubeReserved = nil
} }
out.ProtectKernelDefaults = in.ProtectKernelDefaults out.ProtectKernelDefaults = in.ProtectKernelDefaults
out.MakeIPTablesUtilChains = in.MakeIPTablesUtilChains
out.IPTablesMasqueradeBit = in.IPTablesMasqueradeBit
out.IPTablesDropBit = in.IPTablesDropBit
return nil return nil
} }
} }

View File

@ -244,9 +244,6 @@ func NewMainKubelet(
evictionConfig eviction.Config, evictionConfig eviction.Config,
kubeOptions []Option, kubeOptions []Option,
enableControllerAttachDetach bool, enableControllerAttachDetach bool,
makeIPTablesUtilChains bool,
iptablesMasqueradeBit int,
iptablesDropBit int,
) (*Kubelet, error) { ) (*Kubelet, error) {
if rootDirectory == "" { if rootDirectory == "" {
return nil, fmt.Errorf("invalid root directory %q", rootDirectory) return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
@ -371,9 +368,6 @@ func NewMainKubelet(
babysitDaemons: babysitDaemons, babysitDaemons: babysitDaemons,
enableControllerAttachDetach: enableControllerAttachDetach, enableControllerAttachDetach: enableControllerAttachDetach,
iptClient: utilipt.New(utilexec.New(), utildbus.New(), utilipt.ProtocolIpv4), iptClient: utilipt.New(utilexec.New(), utildbus.New(), utilipt.ProtocolIpv4),
makeIPTablesUtilChains: makeIPTablesUtilChains,
iptablesMasqueradeBit: iptablesMasqueradeBit,
iptablesDropBit: iptablesDropBit,
} }
if klet.flannelExperimentalOverlay { if klet.flannelExperimentalOverlay {
@ -851,15 +845,6 @@ type Kubelet struct {
// trigger deleting containers in a pod // trigger deleting containers in a pod
containerDeletor *podContainerDeletor containerDeletor *podContainerDeletor
// config iptables util rules
makeIPTablesUtilChains bool
// The bit of the fwmark space to mark packets for SNAT.
iptablesMasqueradeBit int
// The bit of the fwmark space to mark packets for dropping.
iptablesDropBit int
} }
// setupDataDirs creates: // setupDataDirs creates:
@ -986,11 +971,6 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
go wait.Until(kl.syncNetworkStatus, 30*time.Second, wait.NeverStop) go wait.Until(kl.syncNetworkStatus, 30*time.Second, wait.NeverStop)
go wait.Until(kl.updateRuntimeUp, 5*time.Second, wait.NeverStop) go wait.Until(kl.updateRuntimeUp, 5*time.Second, wait.NeverStop)
// Start loop to sync iptables util rules
if kl.makeIPTablesUtilChains {
go wait.Until(kl.syncNetworkUtil, 1*time.Minute, wait.NeverStop)
}
// Start a goroutine responsible for killing pods (that are not properly // Start a goroutine responsible for killing pods (that are not properly
// handled by pod workers). // handled by pod workers).
go wait.Until(kl.podKiller, 1*time.Second, wait.NeverStop) go wait.Until(kl.podKiller, 1*time.Second, wait.NeverStop)

View File

@ -28,25 +28,9 @@ import (
"k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/kubelet/network" "k8s.io/kubernetes/pkg/kubelet/network"
"k8s.io/kubernetes/pkg/util/bandwidth" "k8s.io/kubernetes/pkg/util/bandwidth"
utiliptables "k8s.io/kubernetes/pkg/util/iptables"
"k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/sets"
) )
const (
// the mark-for-masquerade chain
// TODO: clean up this logic in kube-proxy
KubeMarkMasqChain utiliptables.Chain = "KUBE-MARK-MASQ"
// the mark-for-drop chain
KubeMarkDropChain utiliptables.Chain = "KUBE-MARK-DROP"
// kubernetes postrouting rules
KubePostroutingChain utiliptables.Chain = "KUBE-POSTROUTING"
// kubernetes postrouting rules
KubeFirewallChain utiliptables.Chain = "KUBE-FIREWALL"
)
// effectiveHairpinMode determines the effective hairpin mode given the // effectiveHairpinMode determines the effective hairpin mode given the
// configured mode, container runtime, and whether cbr0 should be configured. // configured mode, container runtime, and whether cbr0 should be configured.
func effectiveHairpinMode(hairpinMode componentconfig.HairpinMode, containerRuntime string, configureCBR0 bool, networkPlugin string) (componentconfig.HairpinMode, error) { func effectiveHairpinMode(hairpinMode componentconfig.HairpinMode, containerRuntime string, configureCBR0 bool, networkPlugin string) (componentconfig.HairpinMode, error) {
@ -319,88 +303,3 @@ func (kl *Kubelet) shapingEnabled() bool {
} }
return true return true
} }
// syncNetworkUtil ensures the network utility are present on host.
// Network util includes:
// 1. In nat table, KUBE-MARK-DROP rule to mark connections for dropping
// Marked connection will be drop on INPUT/OUTPUT Chain in filter table
// 2. In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT
// Marked connection will get SNAT on POSTROUTING Chain in nat table
func (kl *Kubelet) syncNetworkUtil() {
if kl.iptablesMasqueradeBit < 0 || kl.iptablesMasqueradeBit > 31 {
glog.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", kl.iptablesMasqueradeBit)
return
}
if kl.iptablesDropBit < 0 || kl.iptablesDropBit > 31 {
glog.Errorf("invalid iptables-drop-bit %v not in [0, 31]", kl.iptablesDropBit)
return
}
if kl.iptablesDropBit == kl.iptablesMasqueradeBit {
glog.Errorf("iptables-masquerade-bit %v and iptables-drop-bit %v must be different", kl.iptablesMasqueradeBit, kl.iptablesDropBit)
return
}
// Setup KUBE-MARK-DROP rules
dropMark := getIPTablesMark(kl.iptablesDropBit)
if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil {
glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err)
return
}
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--set-xmark", dropMark); err != nil {
glog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err)
return
}
if _, err := kl.iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil {
glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableFilter, KubeFirewallChain, err)
return
}
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
"-m", "comment", "--comment", "kubernetes firewall for dropping marked packets",
"-m", "mark", "--mark", dropMark,
"-j", "DROP"); err != nil {
glog.Errorf("Failed to ensure rule to drop packet marked by %v in %v chain %v: %v", KubeMarkDropChain, utiliptables.TableFilter, KubeFirewallChain, err)
return
}
if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil {
glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainOutput, KubeFirewallChain, err)
return
}
if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil {
glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableFilter, utiliptables.ChainInput, KubeFirewallChain, err)
return
}
// Setup KUBE-MARK-MASQ rules
masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit)
if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil {
glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkMasqChain, err)
return
}
if _, err := kl.iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil {
glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err)
return
}
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--set-xmark", masqueradeMark); err != nil {
glog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err)
return
}
if _, err := kl.iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting,
"-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil {
glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err)
return
}
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
"-m", "comment", "--comment", "kubernetes service traffic requiring SNAT",
"-m", "mark", "--mark", masqueradeMark, "-j", "MASQUERADE"); err != nil {
glog.Errorf("Failed to ensure SNAT rule for packets marked by %v in %v chain %v: %v", KubeMarkMasqChain, utiliptables.TableNAT, KubePostroutingChain, err)
return
}
}
// getIPTablesMark returns the fwmark given the bit
func getIPTablesMark(bit int) string {
value := 1 << uint(bit)
return fmt.Sprintf("%#08x/%#08x", value, value)
}

View File

@ -217,25 +217,3 @@ func TestCleanupBandwidthLimits(t *testing.T) {
} }
} }
} }
func TestGetIPTablesMark(t *testing.T) {
tests := []struct {
bit int
expect string
}{
{
14,
"0x00004000/0x00004000",
},
{
15,
"0x00008000/0x00008000",
},
}
for _, tc := range tests {
res := getIPTablesMark(tc.bit)
if res != tc.expect {
t.Errorf("getIPTablesMark output unexpected result: %v when input bit is %d. Expect result: %v", res, tc.bit, tc.expect)
}
}
}

View File

@ -45,35 +45,31 @@ import (
utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
) )
const ( // iptablesMinVersion is the minimum version of iptables for which we will use the Proxier
// iptablesMinVersion is the minimum version of iptables for which we will use the Proxier // from this package instead of the userspace Proxier. While most of the
// from this package instead of the userspace Proxier. While most of the // features we need were available earlier, the '-C' flag was added more
// features we need were available earlier, the '-C' flag was added more // recently. We use that indirectly in Ensure* functions, and if we don't
// recently. We use that indirectly in Ensure* functions, and if we don't // have it, we have to be extra careful about the exact args we feed in being
// have it, we have to be extra careful about the exact args we feed in being // the same as the args we read back (iptables itself normalizes some args).
// the same as the args we read back (iptables itself normalizes some args). // This is the "new" Proxier, so we require "new" versions of tools.
// This is the "new" Proxier, so we require "new" versions of tools. const iptablesMinVersion = utiliptables.MinCheckVersion
iptablesMinVersion = utiliptables.MinCheckVersion
// the services chain // the services chain
kubeServicesChain utiliptables.Chain = "KUBE-SERVICES" const kubeServicesChain utiliptables.Chain = "KUBE-SERVICES"
// the nodeports chain // the nodeports chain
kubeNodePortsChain utiliptables.Chain = "KUBE-NODEPORTS" const kubeNodePortsChain utiliptables.Chain = "KUBE-NODEPORTS"
// the kubernetes postrouting chain // the kubernetes postrouting chain
kubePostroutingChain utiliptables.Chain = "KUBE-POSTROUTING" const kubePostroutingChain utiliptables.Chain = "KUBE-POSTROUTING"
// the mark-for-masquerade chain // the mark-for-masquerade chain
KubeMarkMasqChain utiliptables.Chain = "KUBE-MARK-MASQ" // TODO: let kubelet manage this chain. Other component should just assume it exists and use it.
const KubeMarkMasqChain utiliptables.Chain = "KUBE-MARK-MASQ"
// the mark we apply to traffic needing SNAT // the mark we apply to traffic needing SNAT
// TODO(thockin): Remove this for v1.3 or v1.4. // TODO(thockin): Remove this for v1.3 or v1.4.
oldIptablesMasqueradeMark = "0x4d415351" const oldIptablesMasqueradeMark = "0x4d415351"
// the mark-for-drop chain
KubeMarkDropChain utiliptables.Chain = "KUBE-MARK-DROP"
)
// IptablesVersioner can query the current iptables version. // IptablesVersioner can query the current iptables version.
type IptablesVersioner interface { type IptablesVersioner interface {
@ -132,15 +128,14 @@ const sysctlBridgeCallIptables = "net/bridge/bridge-nf-call-iptables"
// internal struct for string service information // internal struct for string service information
type serviceInfo struct { type serviceInfo struct {
clusterIP net.IP clusterIP net.IP
port int port int
protocol api.Protocol protocol api.Protocol
nodePort int nodePort int
loadBalancerStatus api.LoadBalancerStatus loadBalancerStatus api.LoadBalancerStatus
sessionAffinityType api.ServiceAffinity sessionAffinityType api.ServiceAffinity
stickyMaxAgeSeconds int stickyMaxAgeSeconds int
externalIPs []string externalIPs []string
loadBalancerSourceRanges []string
} }
// returns a new serviceInfo struct // returns a new serviceInfo struct
@ -169,7 +164,6 @@ type Proxier struct {
exec utilexec.Interface exec utilexec.Interface
clusterCIDR string clusterCIDR string
hostname string hostname string
nodeIP net.IP
} }
type localPort struct { type localPort struct {
@ -195,7 +189,7 @@ var _ proxy.ProxyProvider = &Proxier{}
// An error will be returned if iptables fails to update or acquire the initial lock. // An error will be returned if iptables fails to update or acquire the initial lock.
// Once a proxier is created, it will keep iptables up to date in the background and // Once a proxier is created, it will keep iptables up to date in the background and
// will not terminate if a particular iptables call fails. // will not terminate if a particular iptables call fails.
func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod time.Duration, masqueradeAll bool, masqueradeBit int, clusterCIDR string, hostname string, nodeIP net.IP) (*Proxier, error) { func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod time.Duration, masqueradeAll bool, masqueradeBit int, clusterCIDR string, hostname string) (*Proxier, error) {
// Set the route_localnet sysctl we need for // Set the route_localnet sysctl we need for
if err := utilsysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil { if err := utilsysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil {
return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlRouteLocalnet, err) return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlRouteLocalnet, err)
@ -215,10 +209,6 @@ func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod
masqueradeValue := 1 << uint(masqueradeBit) masqueradeValue := 1 << uint(masqueradeBit)
masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue) masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue)
if nodeIP == nil {
glog.Warningf("invalid nodeIP, initialize kube-proxy with 127.0.0.1 as nodeIP")
nodeIP = net.ParseIP("127.0.0.1")
}
return &Proxier{ return &Proxier{
serviceMap: make(map[proxy.ServicePortName]*serviceInfo), serviceMap: make(map[proxy.ServicePortName]*serviceInfo),
endpointsMap: make(map[proxy.ServicePortName][]string), endpointsMap: make(map[proxy.ServicePortName][]string),
@ -230,7 +220,6 @@ func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod
exec: exec, exec: exec,
clusterCIDR: clusterCIDR, clusterCIDR: clusterCIDR,
hostname: hostname, hostname: hostname,
nodeIP: nodeIP,
}, nil }, nil
} }
@ -439,7 +428,6 @@ func (proxier *Proxier) OnServiceUpdate(allServices []api.Service) {
// Deep-copy in case the service instance changes // Deep-copy in case the service instance changes
info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer) info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer)
info.sessionAffinityType = service.Spec.SessionAffinity info.sessionAffinityType = service.Spec.SessionAffinity
info.loadBalancerSourceRanges = service.Spec.LoadBalancerSourceRanges
proxier.serviceMap[serviceName] = info proxier.serviceMap[serviceName] = info
glog.V(4).Infof("added serviceInfo(%s): %s", serviceName, spew.Sdump(info)) glog.V(4).Infof("added serviceInfo(%s): %s", serviceName, spew.Sdump(info))
@ -564,28 +552,15 @@ func flattenValidEndpoints(endpoints []hostPortPair) []string {
return result return result
} }
// portProtoHash takes the ServicePortName and protocol for a service // servicePortChainName takes the ServicePortName for a service and
// returns the associated 16 character hash. This is computed by hashing (sha256) // returns the associated iptables chain. This is computed by hashing (sha256)
// then encoding to base32 and truncating to 16 chars. We do this because Iptables // then encoding to base32 and truncating with the prefix "KUBE-SVC-". We do
// Chain Names must be <= 28 chars long, and the longer they are the harder they are to read. // this because Iptables Chain Names must be <= 28 chars long, and the longer
func portProtoHash(s proxy.ServicePortName, protocol string) string { // they are the harder they are to read.
func servicePortChainName(s proxy.ServicePortName, protocol string) utiliptables.Chain {
hash := sha256.Sum256([]byte(s.String() + protocol)) hash := sha256.Sum256([]byte(s.String() + protocol))
encoded := base32.StdEncoding.EncodeToString(hash[:]) encoded := base32.StdEncoding.EncodeToString(hash[:])
return encoded[:16] return utiliptables.Chain("KUBE-SVC-" + encoded[:16])
}
// servicePortChainName takes the ServicePortName for a service and
// returns the associated iptables chain. This is computed by hashing (sha256)
// then encoding to base32 and truncating with the prefix "KUBE-SVC-".
func servicePortChainName(s proxy.ServicePortName, protocol string) utiliptables.Chain {
return utiliptables.Chain("KUBE-SVC-" + portProtoHash(s, protocol))
}
// servicePortChainName takes the ServicePortName for a service and
// returns the associated iptables chain. This is computed by hashing (sha256)
// then encoding to base32 and truncating with the prefix "KUBE-FW-".
func serviceFirewallChainName(s proxy.ServicePortName, protocol string) utiliptables.Chain {
return utiliptables.Chain("KUBE-FW-" + portProtoHash(s, protocol))
} }
// This is the same as servicePortChainName but with the endpoint included. // This is the same as servicePortChainName but with the endpoint included.
@ -879,54 +854,9 @@ func (proxier *Proxier) syncProxyRules() {
"-d", fmt.Sprintf("%s/32", ingress.IP), "-d", fmt.Sprintf("%s/32", ingress.IP),
"--dport", fmt.Sprintf("%d", svcInfo.port), "--dport", fmt.Sprintf("%d", svcInfo.port),
} }
// create service firewall chain
fwChain := serviceFirewallChainName(svcName, protocol)
if chain, ok := existingNATChains[fwChain]; ok {
writeLine(natChains, chain)
} else {
writeLine(natChains, utiliptables.MakeChainLine(fwChain))
}
// jump to service firewall chain
// The service firewall rules are created based on ServiceSpec.loadBalancerSourceRanges field.
// This currently works for loadbalancers that preserves source ips.
// For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply.
writeLine(natRules, append(args, "-j", string(fwChain))...)
args = []string{
"-A", string(fwChain),
"-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcName.String()),
"-m", protocol, "-p", protocol,
"-d", fmt.Sprintf("%s/32", ingress.IP),
"--dport", fmt.Sprintf("%d", svcInfo.port),
}
// We have to SNAT packets from external IPs. // We have to SNAT packets from external IPs.
writeLine(natRules, append(args, "-j", string(KubeMarkMasqChain))...) writeLine(natRules, append(args, "-j", string(KubeMarkMasqChain))...)
writeLine(natRules, append(args, "-j", string(svcChain))...)
if len(svcInfo.loadBalancerSourceRanges) == 0 {
// allow all sources, so jump directly to KUBE-SVC chain
writeLine(natRules, append(args, "-j", string(svcChain))...)
} else {
// firewall filter based on each source range
allowFromNode := false
for _, src := range svcInfo.loadBalancerSourceRanges {
writeLine(natRules, append(args, "-s", src, "-j", string(svcChain))...)
// ignore error because it has been validated
_, cidr, _ := net.ParseCIDR(src)
if cidr.Contains(proxier.nodeIP) {
allowFromNode = true
}
}
// generally, ip route rule was added to intercept request to loadbalancer vip from the
// loadbalancer's backend hosts. In this case, request will not hit the loadbalancer but loop back directly.
// Need to add the following rule to allow request on host.
if allowFromNode {
writeLine(natRules, append(args, "-s", fmt.Sprintf("%s/32", ingress.IP), "-j", string(svcChain))...)
}
}
// If the packet was able to reach the end of firewall chain, then it did not get DNATed.
// It means the packet cannot go thru the firewall, then mark it for DROP
writeLine(natRules, append(args, "-j", string(KubeMarkDropChain))...)
} }
} }