Merge pull request #19389 from justinsb/internal_cidr

Auto commit by PR queue bot

cluster/aws/config-default.sh

@@ -74,7 +74,8 @@ MASTER_TAG="${INSTANCE_PREFIX}-master"
 NODE_TAG="${INSTANCE_PREFIX}-minion"
 NODE_SCOPES=""
 POLL_SLEEP_INTERVAL=3
-SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16"  # formerly PORTAL_NET
+NON_MASQUERADE_CIDR="${NON_MASQUERADE_CIDR:-10.0.0.0/8}" # Traffic to IPs outside this range will use IP masquerade
+SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/16}"  # formerly PORTAL_NET
 CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}"
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 # If set to Elastic IP, master instance will be associated with this IP.
@@ -104,12 +105,12 @@ ELASTICSEARCH_LOGGING_REPLICAS=1
 
 # Optional: Don't require https for registries in our local RFC1918 network
 if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then
-  EXTRA_DOCKER_OPTS="--insecure-registry 10.0.0.0/8"
+  EXTRA_DOCKER_OPTS="--insecure-registry ${NON_MASQUERADE_CIDR}"
 fi
 
 # Optional: Install cluster DNS.
 ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"
-DNS_SERVER_IP="10.0.0.10"
+DNS_SERVER_IP="${DNS_SERVER_IP:-10.0.0.10}"
 DNS_DOMAIN="cluster.local"
 DNS_REPLICAS=1
 

cluster/aws/config-test.sh

@@ -72,7 +72,8 @@ MASTER_TAG="${INSTANCE_PREFIX}-master"
 NODE_TAG="${INSTANCE_PREFIX}-minion"
 NODE_SCOPES=""
 POLL_SLEEP_INTERVAL=3
-SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16"  # formerly PORTAL_NET
+NON_MASQUERADE_CIDR="${NON_MASQUERADE_CIDR:-10.0.0.0/8}" # Traffic to IPs outside this range will use IP masquerade
+SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/16}"  # formerly PORTAL_NET
 CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.245.0.0/16}"
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 # If set to Elastic IP, master instance will be associated with this IP.
@@ -100,12 +101,12 @@ ELASTICSEARCH_LOGGING_REPLICAS=1
 
 # Optional: Don't require https for registries in our local RFC1918 network
 if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then
-  EXTRA_DOCKER_OPTS="--insecure-registry 10.0.0.0/8"
+  EXTRA_DOCKER_OPTS="--insecure-registry ${NON_MASQUERADE_CIDR}"
 fi
 
 # Optional: Install cluster DNS.
 ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"
-DNS_SERVER_IP="10.0.0.10"
+DNS_SERVER_IP="${DNS_SERVER_IP:-10.0.0.10}"
 DNS_DOMAIN="cluster.local"
 DNS_REPLICAS=1
 

cluster/aws/options.md

@@ -88,4 +88,47 @@ Defaults to vivid (Ubuntu Vivid Vervet), which has a modern kernel and does not
 Other options may require reboots, updates or configuration, and should be used only if you have a compelling
 requirement to do so.
 
+**NON_MASQUERADE_CIDR**
+
+The 'internal' IP range which Kuberenetes will use, which will therefore not
+use IP masquerade.  By default kubernetes runs an internal network for traffic
+between pods (and between pods and services), and by default this uses the
+`10.0.0.0/8` range.  However, this sometimes overlaps with a range that you may
+want to use; in particular the range cannot be used with EC2 ClassicLink.  You
+may also want to run kubernetes in an existing VPC where you have chosen a CIDR
+in the `10.0.0.0/8` range.
+
+Setting this flag allows you to change this internal network CIDR.  Note that
+you must set other values consistently within the CIDR that you choose.
+
+For example, you might choose `172.16.0.0/14`; and you could then choose to
+configure like this:
+
+```
+export NON_MASQUERADE_CIDR="172.16.0.0/14"
+export SERVICE_CLUSTER_IP_RANGE="172.16.0.0/16"
+export DNS_SERVER_IP="172.16.0.10"
+export MASTER_IP_RANGE="172.17.0.0/24"
+export CLUSTER_IP_RANGE="172.18.0.0/16"
+```
+
+When choosing a CIDR in the 172.20/12 reserved range you should be careful not
+to choose a CIDR that overlaps your VPC CIDR (the kube-up script sets the VPC
+CIDR to 172.20.0.0/16 by default, so you should not overlap that).  If you want
+to allow inter-VPC traffic you should be careful to avoid your other VPCs as
+well.
+
+There is also a 100.64/10 address block which is reserved for "Carrier Grade
+NAT", and which some users have reported success using.  While we haven't seen
+any problems, or conflicts with any AWS networks, we can't guarantee it.  If you
+decide you are comfortable using 100.64, you might use:
+
+```
+export NON_MASQUERADE_CIDR="100.64.0.0/10"
+export SERVICE_CLUSTER_IP_RANGE="100.64.0.0/16"
+export DNS_SERVER_IP="100.64.0.10"
+export MASTER_IP_RANGE="100.65.0.0/24"
+export CLUSTER_IP_RANGE="100.66.0.0/16"
+```
+
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/aws/options.md?pixel)]()

cluster/aws/templates/salt-master.sh

@@ -44,6 +44,7 @@ env_to_salt docker_root
 env_to_salt kubelet_root
 env_to_salt master_extra_sans
 env_to_salt runtime_config
+env_to_salt non_masquerade_cidr
 
 # Auto accept all keys from minions that try to join
 mkdir -p /etc/salt/master.d

cluster/aws/templates/salt-minion.sh

@@ -37,29 +37,24 @@ if [[ -z "${HOSTNAME_OVERRIDE}" ]]; then
   HOSTNAME_OVERRIDE=`curl --silent curl http://169.254.169.254/2007-01-19/meta-data/local-hostname`
 fi
 
-if [[ -n "${HOSTNAME_OVERRIDE}" ]]; then
+# Helper that sets a salt grain in grains.conf, if the upper-cased key is a non-empty env
-  cat <<EOF >>/etc/salt/minion.d/grains.conf
+function env_to_salt {
-  hostname_override: "${HOSTNAME_OVERRIDE}"
+  local key=$1
+  local env_key=`echo $key | tr '[:lower:]' '[:upper:]'`
+  local value=${!env_key}
+  if [[ -n "${value}" ]]; then
+    # Note this is yaml, so indentation matters
+    cat <<EOF >>/etc/salt/minion.d/grains.conf
+  ${key}: '$(echo "${value}" | sed -e "s/'/''/g")'
 EOF
-fi
+  fi
+}
 
-if [[ -n "${DOCKER_OPTS}" ]]; then
+env_to_salt hostname_override
-  cat <<EOF >>/etc/salt/minion.d/grains.conf
+env_to_salt docker_opts
-  docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")'
+env_to_salt docker_root
-EOF
+env_to_salt kubelet_root
-fi
+env_to_salt non_masquerade_cidr
-
-if [[ -n "${DOCKER_ROOT}" ]]; then
-  cat <<EOF >>/etc/salt/minion.d/grains.conf
-  docker_root: '$(echo "$DOCKER_ROOT" | sed -e "s/'/''/g")'
-EOF
-fi
-
-if [[ -n "${KUBELET_ROOT}" ]]; then
-  cat <<EOF >>/etc/salt/minion.d/grains.conf
-  kubelet_root: '$(echo "$KUBELET_ROOT" | sed -e "s/'/''/g")'
-EOF
-fi
 
 install-salt
 

cluster/aws/trusty/common.sh

@@ -29,6 +29,7 @@ function generate-minion-user-data {
   echo "#! /bin/bash"
   echo "SALT_MASTER='${MASTER_INTERNAL_IP}'"
   echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS:-}'"
+  echo "readonly NON_MASQUERADE_CIDR='${NON_MASQUERADE_CIDR:-}'"
   echo "readonly DOCKER_STORAGE='${DOCKER_STORAGE:-}'"
   grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
   grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"

cluster/aws/util.sh

@@ -893,6 +893,7 @@ function start-master() {
     echo "readonly SALT_MASTER='${MASTER_INTERNAL_IP}'"
     echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
     echo "readonly NODE_INSTANCE_PREFIX='${NODE_INSTANCE_PREFIX}'"
+    echo "readonly NON_MASQUERADE_CIDR='${NON_MASQUERADE_CIDR:-}'"
     echo "readonly CLUSTER_IP_RANGE='${CLUSTER_IP_RANGE}'"
     echo "readonly ALLOCATE_NODE_CIDRS='${ALLOCATE_NODE_CIDRS}'"
     echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"

cluster/saltbase/salt/kubelet/default

@@ -85,6 +85,11 @@
   {% set configure_cbr0 = "--configure-cbr0=" + pillar['allocate_node_cidrs'] -%}
 {% endif -%}
 
+{% set non_masquerade_cidr = "" -%}
+{% if grains.non_masquerade_cidr is defined -%}
+  {% set non_masquerade_cidr = "--non-masquerade-cidr=" + grains.non_masquerade_cidr -%}
+{% endif -%}
+
 # The master kubelet cannot wait for the flannel daemon because it is responsible
 # for starting up the flannel server in a static pod. So even though the flannel
 # daemon runs on the master, it doesn't hold up cluster bootstrap. All the pods
@@ -137,4 +142,4 @@
 {% endif -%}
 
 # test_args has to be kept at the end, so they'll overwrite any prior configuration
-DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{test_args}}"
+DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{test_args}}"

cmd/kubelet/app/options/options.go

@@ -97,6 +97,7 @@ func NewKubeletServer() *KubeletServer {
 			MinimumGCAge:                unversioned.Duration{1 * time.Minute},
 			NetworkPluginDir:            "/usr/libexec/kubernetes/kubelet-plugins/net/exec/",
 			NetworkPluginName:           "",
+			NonMasqueradeCIDR:           "10.0.0.0/8",
 			VolumePluginDir:             "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
 			NodeStatusUpdateFrequency:   unversioned.Duration{10 * time.Second},
 			NodeLabels:                  make(map[string]string),
@@ -194,6 +195,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.ConfigureCBR0, "configure-cbr0", s.ConfigureCBR0, "If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.")
 	fs.IntVar(&s.MaxPods, "max-pods", s.MaxPods, "Number of Pods that can run on this Kubelet.")
 	fs.StringVar(&s.DockerExecHandlerName, "docker-exec-handler", s.DockerExecHandlerName, "Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.")
+	fs.StringVar(&s.NonMasqueradeCIDR, "non-masquerade-cidr", s.NonMasqueradeCIDR, "Traffic to IPs outside this range will use IP masquerade.")
 	fs.StringVar(&s.PodCIDR, "pod-cidr", "", "The CIDR to use for pod IP addresses, only used in standalone mode.  In cluster mode, this is obtained from the master.")
 	fs.StringVar(&s.ResolverConfig, "resolv-conf", kubetypes.ResolvConfDefault, "Resolver configuration file used as the basis for the container DNS resolution configuration.")
 	fs.BoolVar(&s.CPUCFSQuota, "cpu-cfs-quota", s.CPUCFSQuota, "Enable CPU CFS quota enforcement for containers that specify CPU limits")

cmd/kubelet/app/server.go

@@ -222,6 +222,7 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 		NetworkPlugins:            ProbeNetworkPlugins(s.NetworkPluginDir),
 		NodeLabels:                s.NodeLabels,
 		NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency.Duration,
+		NonMasqueradeCIDR:         s.NonMasqueradeCIDR,
 		OOMAdjuster:               oom.NewOOMAdjuster(),
 		OSInterface:               kubecontainer.RealOS{},
 		PodCIDR:                   s.PodCIDR,
@@ -700,6 +701,7 @@ type KubeletConfig struct {
 	NodeName                       string
 	NodeLabels                     map[string]string
 	NodeStatusUpdateFrequency      time.Duration
+	NonMasqueradeCIDR              string
 	OOMAdjuster                    *oom.OOMAdjuster
 	OSInterface                    kubecontainer.OSInterface
 	PodCIDR                        string
@@ -803,6 +805,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.DockerDaemonContainer,
 		kc.SystemContainer,
 		kc.ConfigureCBR0,
+		kc.NonMasqueradeCIDR,
 		kc.PodCIDR,
 		kc.ReconcileCIDR,
 		kc.MaxPods,

docs/admin/kubelet.md

@@ -117,6 +117,7 @@ kubelet
       --node-ip="": IP address of the node. If set, kubelet will use this IP address for the node
       --node-labels=: <Warning: Alpha feature> Labels to add when registering the node in the cluster.  Labels must are key=value pairs seperated by ','.
       --node-status-update-frequency=10s: Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s
+      --non-masquerade-cidr="10.0.0.0/8": Traffic to IPs outside this range will use IP masquerade.
       --oom-score-adj=-999: The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]
       --outofdisk-transition-frequency=5m0s: Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status. Default: 5m0s
       --pod-cidr="": The CIDR to use for pod IP addresses, only used in standalone mode.  In cluster mode, this is obtained from the master.

hack/verify-flags/known-flags.txt

@@ -237,6 +237,7 @@ node-name
 node-startup-grace-period
 node-status-update-frequency
 node-sync-period
+non-masquerade-cidr
 num-nodes
 oidc-ca-file
 oidc-client-id

pkg/apis/componentconfig/types.go

@@ -274,6 +274,8 @@ type KubeletConfiguration struct {
 	NodeIP string `json:"nodeIP,omitempty"`
 	// nodeLabels to add when registering the node in the cluster.
 	NodeLabels map[string]string `json:"nodeLabels"`
+	// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
+	NonMasqueradeCIDR string `json:"nonMasqueradeCIDR"`
 }
 
 // LeaderElectionConfiguration defines the configuration of leader election

pkg/kubelet/container_bridge.go

@@ -119,23 +119,24 @@ func cbr0CidrCorrect(wantCIDR *net.IPNet) bool {
 }
 
 // TODO(dawnchen): Using pkg/util/iptables
-func ensureIPTablesMasqRule() error {
+// nonMasqueradeCIDR is the CIDR for our internal IP range; traffic to IPs outside this range will use IP masquerade.
+func ensureIPTablesMasqRule(nonMasqueradeCIDR string) error {
 	// Check if the MASQUERADE rule exist or not
 	if err := exec.Command("iptables",
 		"-t", "nat",
 		"-C", "POSTROUTING",
-		"!", "-d", "10.0.0.0/8",
+		"!", "-d", nonMasqueradeCIDR,
 		"-m", "addrtype", "!", "--dst-type", "LOCAL",
 		"-j", "MASQUERADE").Run(); err == nil {
 		// The MASQUERADE rule exists
 		return nil
 	}
 
-	glog.Infof("MASQUERADE rule doesn't exist, recreate it")
+	glog.Infof("MASQUERADE rule doesn't exist, recreate it (with nonMasqueradeCIDR %s)", nonMasqueradeCIDR)
 	if err := exec.Command("iptables",
 		"-t", "nat",
 		"-A", "POSTROUTING",
-		"!", "-d", "10.0.0.0/8",
+		"!", "-d", nonMasqueradeCIDR,
 		"-m", "addrtype", "!", "--dst-type", "LOCAL",
 		"-j", "MASQUERADE").Run(); err != nil {
 		return err

pkg/kubelet/kubelet.go

@@ -183,6 +183,7 @@ func NewMainKubelet(
 	dockerDaemonContainer string,
 	systemContainer string,
 	configureCBR0 bool,
+	nonMasqueradeCIDR string,
 	podCIDR string,
 	reconcileCIDR bool,
 	maxPods int,
@@ -302,6 +303,7 @@ func NewMainKubelet(
 		chownRunner:                    chownRunner,
 		writer:                         writer,
 		configureCBR0:                  configureCBR0,
+		nonMasqueradeCIDR:              nonMasqueradeCIDR,
 		reconcileCIDR:                  reconcileCIDR,
 		maxPods:                        maxPods,
 		syncLoopMonitor:                atomic.Value{},
@@ -610,6 +612,9 @@ type Kubelet struct {
 	configureCBR0 bool
 	reconcileCIDR bool
 
+	// Traffic to IPs outside this range will use IP masquerade.
+	nonMasqueradeCIDR string
+
 	// Maximum Number of Pods which can be run by this Kubelet
 	maxPods int
 
@@ -2637,7 +2642,7 @@ func (kl *Kubelet) syncNetworkStatus() {
 				kl.runtimeState.podCIDR(), podCIDR)
 			kl.runtimeState.setPodCIDR(podCIDR)
 		}
-		if err := ensureIPTablesMasqRule(); err != nil {
+		if err := ensureIPTablesMasqRule(kl.nonMasqueradeCIDR); err != nil {
 			err = fmt.Errorf("Error on adding ip table rules: %v", err)
 			glog.Error(err)
 			kl.runtimeState.setNetworkState(err)