Masquerade traffic from off-cluster going through kube-proxy.

This commit is contained in:
CJ Cullen 2016-04-18 14:45:35 -07:00
parent 3b2aae809f
commit 760568796f
12 changed files with 350 additions and 285 deletions

View File

@ -14,6 +14,10 @@
{% if pillar['kubeproxy_test_args'] is defined -%} {% if pillar['kubeproxy_test_args'] is defined -%}
{% set test_args=pillar['kubeproxy_test_args'] %} {% set test_args=pillar['kubeproxy_test_args'] %}
{% endif -%} {% endif -%}
{% set cluster_cidr = "" -%}
{% if pillar['cluster_cidr'] is defined -%}
{% set cluster_cidr=" --cluster-cidr=" + pillar['cluster_cidr'] %}
{% endif -%}
{% set log_level = pillar['log_level'] -%} {% set log_level = pillar['log_level'] -%}
{% if pillar['kubeproxy_test_log_level'] is defined -%} {% if pillar['kubeproxy_test_log_level'] is defined -%}
@ -40,7 +44,7 @@ spec:
command: command:
- /bin/sh - /bin/sh
- -c - -c
- kube-proxy {{api_servers_with_port}} {{kubeconfig}} --resource-container="" {{log_level}} {{test_args}} 1>>/var/log/kube-proxy.log 2>&1 - kube-proxy {{api_servers_with_port}} {{kubeconfig}} {{cluster_cidr}} --resource-container="" {{log_level}} {{test_args}} 1>>/var/log/kube-proxy.log 2>&1
securityContext: securityContext:
privileged: true privileged: true
volumeMounts: volumeMounts:

View File

@ -75,6 +75,7 @@ func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) {
fs.DurationVar(&s.IPTablesSyncPeriod.Duration, "iptables-sync-period", s.IPTablesSyncPeriod.Duration, "How often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") fs.DurationVar(&s.IPTablesSyncPeriod.Duration, "iptables-sync-period", s.IPTablesSyncPeriod.Duration, "How often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.")
fs.DurationVar(&s.ConfigSyncPeriod, "config-sync-period", s.ConfigSyncPeriod, "How often configuration from the apiserver is refreshed. Must be greater than 0.") fs.DurationVar(&s.ConfigSyncPeriod, "config-sync-period", s.ConfigSyncPeriod, "How often configuration from the apiserver is refreshed. Must be greater than 0.")
fs.BoolVar(&s.MasqueradeAll, "masquerade-all", s.MasqueradeAll, "If using the pure iptables proxy, SNAT everything") fs.BoolVar(&s.MasqueradeAll, "masquerade-all", s.MasqueradeAll, "If using the pure iptables proxy, SNAT everything")
fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "The CIDR range of pods in the cluster. It is used to bridge traffic coming from outside of the cluster. If not provided, no off-cluster bridging will be performed.")
fs.BoolVar(&s.CleanupAndExit, "cleanup-iptables", s.CleanupAndExit, "If true cleanup iptables rules and exit.") fs.BoolVar(&s.CleanupAndExit, "cleanup-iptables", s.CleanupAndExit, "If true cleanup iptables rules and exit.")
fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver") fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
fs.IntVar(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver") fs.IntVar(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")

View File

@ -203,7 +203,7 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err
return nil, fmt.Errorf("Unable to read IPTablesMasqueradeBit from config") return nil, fmt.Errorf("Unable to read IPTablesMasqueradeBit from config")
} }
proxierIptables, err := iptables.NewProxier(iptInterface, execer, config.IPTablesSyncPeriod.Duration, config.MasqueradeAll, *config.IPTablesMasqueradeBit) proxierIptables, err := iptables.NewProxier(iptInterface, execer, config.IPTablesSyncPeriod.Duration, config.MasqueradeAll, *config.IPTablesMasqueradeBit, config.ClusterCIDR)
if err != nil { if err != nil {
glog.Fatalf("Unable to create proxier: %v", err) glog.Fatalf("Unable to create proxier: %v", err)
} }

View File

@ -56,6 +56,7 @@ kube-proxy
``` ```
--bind-address=0.0.0.0: The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces) --bind-address=0.0.0.0: The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)
--cleanup-iptables[=false]: If true cleanup iptables rules and exit. --cleanup-iptables[=false]: If true cleanup iptables rules and exit.
--cluster-cidr="": The CIDR range of pods in the cluster. It is used to bridge traffic coming from outside of the cluster. If not provided, no off-cluster bridging will be performed.
--config-sync-period=15m0s: How often configuration from the apiserver is refreshed. Must be greater than 0. --config-sync-period=15m0s: How often configuration from the apiserver is refreshed. Must be greater than 0.
--conntrack-max=262144: Maximum number of NAT connections to track (0 to leave as-is) --conntrack-max=262144: Maximum number of NAT connections to track (0 to leave as-is)
--conntrack-tcp-timeout-established=24h0m0s: Idle timeout for established TCP connections (0 to leave as-is) --conntrack-tcp-timeout-established=24h0m0s: Idle timeout for established TCP connections (0 to leave as-is)
@ -77,7 +78,7 @@ kube-proxy
--udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace --udp-timeout=250ms: How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace
``` ```
###### Auto generated by spf13/cobra on 7-Feb-2016 ###### Auto generated by spf13/cobra on 18-Apr-2016
<!-- BEGIN MUNGE: GENERATED_ANALYTICS --> <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

View File

@ -33,6 +33,7 @@ cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = add
cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + terminated_pod_gc + " " + cloud_provider + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%} cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + terminated_pod_gc + " " + cloud_provider + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%}
cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set api_servers_with_port = api_servers + ":6443" -%}
cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set api_servers_with_port = api_servers -%} cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set api_servers_with_port = api_servers -%}
cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set cluster_cidr=" --cluster-cidr=" + pillar['cluster_cidr'] %}
cluster/saltbase/salt/kubelet/default: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kubelet/default: {% set api_servers_with_port = api_servers + ":6443" -%}
cluster/saltbase/salt/kubelet/default: {% set api_servers_with_port = api_servers -%} cluster/saltbase/salt/kubelet/default: {% set api_servers_with_port = api_servers -%}
cluster/saltbase/salt/kubelet/default: {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %} cluster/saltbase/salt/kubelet/default: {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %}

View File

@ -138,6 +138,7 @@ func DeepCopy_componentconfig_KubeProxyConfiguration(in KubeProxyConfiguration,
return err return err
} }
out.BindAddress = in.BindAddress out.BindAddress = in.BindAddress
out.ClusterCIDR = in.ClusterCIDR
out.HealthzBindAddress = in.HealthzBindAddress out.HealthzBindAddress = in.HealthzBindAddress
out.HealthzPort = in.HealthzPort out.HealthzPort = in.HealthzPort
out.HostnameOverride = in.HostnameOverride out.HostnameOverride = in.HostnameOverride

File diff suppressed because it is too large Load Diff

View File

@ -24,6 +24,10 @@ type KubeProxyConfiguration struct {
// bindAddress is the IP address for the proxy server to serve on (set to 0.0.0.0 // bindAddress is the IP address for the proxy server to serve on (set to 0.0.0.0
// for all interfaces) // for all interfaces)
BindAddress string `json:"bindAddress"` BindAddress string `json:"bindAddress"`
// clusterCIDR is the CIDR range of the pods in the cluster. It is used to
// bridge traffic coming from outside of the cluster. If not provided,
// no off-cluster bridging will be performed.
ClusterCIDR string `json:"clusterCIDR"`
// healthzBindAddress is the IP address for the health check server to serve on, // healthzBindAddress is the IP address for the health check server to serve on,
// defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces) // defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
HealthzBindAddress string `json:"healthzBindAddress"` HealthzBindAddress string `json:"healthzBindAddress"`

View File

@ -49,6 +49,7 @@ func autoConvert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyCon
return err return err
} }
out.BindAddress = in.BindAddress out.BindAddress = in.BindAddress
out.ClusterCIDR = in.ClusterCIDR
out.HealthzBindAddress = in.HealthzBindAddress out.HealthzBindAddress = in.HealthzBindAddress
out.HealthzPort = int(in.HealthzPort) out.HealthzPort = int(in.HealthzPort)
out.HostnameOverride = in.HostnameOverride out.HostnameOverride = in.HostnameOverride
@ -100,6 +101,7 @@ func autoConvert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyCon
return err return err
} }
out.BindAddress = in.BindAddress out.BindAddress = in.BindAddress
out.ClusterCIDR = in.ClusterCIDR
out.HealthzBindAddress = in.HealthzBindAddress out.HealthzBindAddress = in.HealthzBindAddress
out.HealthzPort = int32(in.HealthzPort) out.HealthzPort = int32(in.HealthzPort)
out.HostnameOverride = in.HostnameOverride out.HostnameOverride = in.HostnameOverride

View File

@ -42,6 +42,7 @@ func DeepCopy_v1alpha1_KubeProxyConfiguration(in KubeProxyConfiguration, out *Ku
return err return err
} }
out.BindAddress = in.BindAddress out.BindAddress = in.BindAddress
out.ClusterCIDR = in.ClusterCIDR
out.HealthzBindAddress = in.HealthzBindAddress out.HealthzBindAddress = in.HealthzBindAddress
out.HealthzPort = in.HealthzPort out.HealthzPort = in.HealthzPort
out.HostnameOverride = in.HostnameOverride out.HostnameOverride = in.HostnameOverride

View File

@ -24,6 +24,10 @@ type KubeProxyConfiguration struct {
// bindAddress is the IP address for the proxy server to serve on (set to 0.0.0.0 // bindAddress is the IP address for the proxy server to serve on (set to 0.0.0.0
// for all interfaces) // for all interfaces)
BindAddress string `json:"bindAddress"` BindAddress string `json:"bindAddress"`
// clusterCIDR is the CIDR range of the pods in the cluster. It is used to
// bridge traffic coming from outside of the cluster. If not provided,
// no off-cluster bridging will be performed.
ClusterCIDR string `json:"clusterCIDR"`
// healthzBindAddress is the IP address for the health check server to serve on, // healthzBindAddress is the IP address for the health check server to serve on,
// defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces) // defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
HealthzBindAddress string `json:"healthzBindAddress"` HealthzBindAddress string `json:"healthzBindAddress"`

View File

@ -162,6 +162,7 @@ type Proxier struct {
masqueradeAll bool masqueradeAll bool
masqueradeMark string masqueradeMark string
exec utilexec.Interface exec utilexec.Interface
clusterCIDR string
} }
type localPort struct { type localPort struct {
@ -187,7 +188,7 @@ var _ proxy.ProxyProvider = &Proxier{}
// An error will be returned if iptables fails to update or acquire the initial lock. // An error will be returned if iptables fails to update or acquire the initial lock.
// Once a proxier is created, it will keep iptables up to date in the background and // Once a proxier is created, it will keep iptables up to date in the background and
// will not terminate if a particular iptables call fails. // will not terminate if a particular iptables call fails.
func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod time.Duration, masqueradeAll bool, masqueradeBit int) (*Proxier, error) { func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod time.Duration, masqueradeAll bool, masqueradeBit int, clusterCIDR string) (*Proxier, error) {
// Set the route_localnet sysctl we need for // Set the route_localnet sysctl we need for
if err := utilsysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil { if err := utilsysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil {
return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlRouteLocalnet, err) return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlRouteLocalnet, err)
@ -223,6 +224,7 @@ func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod
masqueradeAll: masqueradeAll, masqueradeAll: masqueradeAll,
masqueradeMark: masqueradeMark, masqueradeMark: masqueradeMark,
exec: exec, exec: exec,
clusterCIDR: clusterCIDR,
}, nil }, nil
} }
@ -792,6 +794,9 @@ func (proxier *Proxier) syncProxyRules() {
if proxier.masqueradeAll { if proxier.masqueradeAll {
writeLine(natRules, append(args, "-j", string(kubeMarkMasqChain))...) writeLine(natRules, append(args, "-j", string(kubeMarkMasqChain))...)
} }
if len(proxier.clusterCIDR) > 0 {
writeLine(natRules, append(args, "! -s", proxier.clusterCIDR, "-j", string(kubeMarkMasqChain))...)
}
writeLine(natRules, append(args, "-j", string(svcChain))...) writeLine(natRules, append(args, "-j", string(svcChain))...)
// Capture externalIPs. // Capture externalIPs.