mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-30 15:05:27 +00:00
Merge pull request #19389 from justinsb/internal_cidr
Auto commit by PR queue bot
This commit is contained in:
k8s-merge-robot
commit
8c821700f3
@ -74,7 +74,8 @@ MASTER_TAG="${INSTANCE_PREFIX}-master"
|
||||
NODE_TAG="${INSTANCE_PREFIX}-minion"
|
||||
NODE_SCOPES=""
|
||||
POLL_SLEEP_INTERVAL=3
|
||||
SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET
|
||||
NON_MASQUERADE_CIDR="${NON_MASQUERADE_CIDR:-10.0.0.0/8}" # Traffic to IPs outside this range will use IP masquerade
|
||||
SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/16}" # formerly PORTAL_NET
|
||||
CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}"
|
||||
MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
|
||||
# If set to Elastic IP, master instance will be associated with this IP.
|
||||
@ -104,12 +105,12 @@ ELASTICSEARCH_LOGGING_REPLICAS=1
|
||||
|
||||
# Optional: Don't require https for registries in our local RFC1918 network
|
||||
if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then
|
||||
EXTRA_DOCKER_OPTS="--insecure-registry 10.0.0.0/8"
|
||||
EXTRA_DOCKER_OPTS="--insecure-registry ${NON_MASQUERADE_CIDR}"
|
||||
fi
|
||||
|
||||
# Optional: Install cluster DNS.
|
||||
ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"
|
||||
DNS_SERVER_IP="10.0.0.10"
|
||||
DNS_SERVER_IP="${DNS_SERVER_IP:-10.0.0.10}"
|
||||
DNS_DOMAIN="cluster.local"
|
||||
DNS_REPLICAS=1
|
||||
|
||||
|
||||
@ -72,7 +72,8 @@ MASTER_TAG="${INSTANCE_PREFIX}-master"
|
||||
NODE_TAG="${INSTANCE_PREFIX}-minion"
|
||||
NODE_SCOPES=""
|
||||
POLL_SLEEP_INTERVAL=3
|
||||
SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET
|
||||
NON_MASQUERADE_CIDR="${NON_MASQUERADE_CIDR:-10.0.0.0/8}" # Traffic to IPs outside this range will use IP masquerade
|
||||
SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/16}" # formerly PORTAL_NET
|
||||
CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.245.0.0/16}"
|
||||
MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
|
||||
# If set to Elastic IP, master instance will be associated with this IP.
|
||||
@ -100,12 +101,12 @@ ELASTICSEARCH_LOGGING_REPLICAS=1
|
||||
|
||||
# Optional: Don't require https for registries in our local RFC1918 network
|
||||
if [[ ${KUBE_ENABLE_INSECURE_REGISTRY:-false} == "true" ]]; then
|
||||
EXTRA_DOCKER_OPTS="--insecure-registry 10.0.0.0/8"
|
||||
EXTRA_DOCKER_OPTS="--insecure-registry ${NON_MASQUERADE_CIDR}"
|
||||
fi
|
||||
|
||||
# Optional: Install cluster DNS.
|
||||
ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"
|
||||
DNS_SERVER_IP="10.0.0.10"
|
||||
DNS_SERVER_IP="${DNS_SERVER_IP:-10.0.0.10}"
|
||||
DNS_DOMAIN="cluster.local"
|
||||
DNS_REPLICAS=1
|
||||
|
||||
|
||||
@ -88,4 +88,47 @@ Defaults to vivid (Ubuntu Vivid Vervet), which has a modern kernel and does not
|
||||
Other options may require reboots, updates or configuration, and should be used only if you have a compelling
|
||||
requirement to do so.
|
||||
|
||||
**NON_MASQUERADE_CIDR**
|
||||
|
||||
The 'internal' IP range which Kuberenetes will use, which will therefore not
|
||||
use IP masquerade. By default kubernetes runs an internal network for traffic
|
||||
between pods (and between pods and services), and by default this uses the
|
||||
`10.0.0.0/8` range. However, this sometimes overlaps with a range that you may
|
||||
want to use; in particular the range cannot be used with EC2 ClassicLink. You
|
||||
may also want to run kubernetes in an existing VPC where you have chosen a CIDR
|
||||
in the `10.0.0.0/8` range.
|
||||
|
||||
Setting this flag allows you to change this internal network CIDR. Note that
|
||||
you must set other values consistently within the CIDR that you choose.
|
||||
|
||||
For example, you might choose `172.16.0.0/14`; and you could then choose to
|
||||
configure like this:
|
||||
|
||||
```
|
||||
export NON_MASQUERADE_CIDR="172.16.0.0/14"
|
||||
export SERVICE_CLUSTER_IP_RANGE="172.16.0.0/16"
|
||||
export DNS_SERVER_IP="172.16.0.10"
|
||||
export MASTER_IP_RANGE="172.17.0.0/24"
|
||||
export CLUSTER_IP_RANGE="172.18.0.0/16"
|
||||
```
|
||||
|
||||
When choosing a CIDR in the 172.20/12 reserved range you should be careful not
|
||||
to choose a CIDR that overlaps your VPC CIDR (the kube-up script sets the VPC
|
||||
CIDR to 172.20.0.0/16 by default, so you should not overlap that). If you want
|
||||
to allow inter-VPC traffic you should be careful to avoid your other VPCs as
|
||||
well.
|
||||
|
||||
There is also a 100.64/10 address block which is reserved for "Carrier Grade
|
||||
NAT", and which some users have reported success using. While we haven't seen
|
||||
any problems, or conflicts with any AWS networks, we can't guarantee it. If you
|
||||
decide you are comfortable using 100.64, you might use:
|
||||
|
||||
```
|
||||
export NON_MASQUERADE_CIDR="100.64.0.0/10"
|
||||
export SERVICE_CLUSTER_IP_RANGE="100.64.0.0/16"
|
||||
export DNS_SERVER_IP="100.64.0.10"
|
||||
export MASTER_IP_RANGE="100.65.0.0/24"
|
||||
export CLUSTER_IP_RANGE="100.66.0.0/16"
|
||||
```
|
||||
|
||||
[]()
|
||||
|
||||
@ -44,6 +44,7 @@ env_to_salt docker_root
|
||||
env_to_salt kubelet_root
|
||||
env_to_salt master_extra_sans
|
||||
env_to_salt runtime_config
|
||||
env_to_salt non_masquerade_cidr
|
||||
|
||||
# Auto accept all keys from minions that try to join
|
||||
mkdir -p /etc/salt/master.d
|
||||
|
||||
@ -37,29 +37,24 @@ if [[ -z "${HOSTNAME_OVERRIDE}" ]]; then
|
||||
HOSTNAME_OVERRIDE=`curl --silent curl http://169.254.169.254/2007-01-19/meta-data/local-hostname`
|
||||
fi
|
||||
|
||||
if [[ -n "${HOSTNAME_OVERRIDE}" ]]; then
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
hostname_override: "${HOSTNAME_OVERRIDE}"
|
||||
# Helper that sets a salt grain in grains.conf, if the upper-cased key is a non-empty env
|
||||
function env_to_salt {
|
||||
local key=$1
|
||||
local env_key=`echo $key | tr '[:lower:]' '[:upper:]'`
|
||||
local value=${!env_key}
|
||||
if [[ -n "${value}" ]]; then
|
||||
# Note this is yaml, so indentation matters
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
${key}: '$(echo "${value}" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ -n "${DOCKER_OPTS}" ]]; then
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
|
||||
if [[ -n "${DOCKER_ROOT}" ]]; then
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
docker_root: '$(echo "$DOCKER_ROOT" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
|
||||
if [[ -n "${KUBELET_ROOT}" ]]; then
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
kubelet_root: '$(echo "$KUBELET_ROOT" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
env_to_salt hostname_override
|
||||
env_to_salt docker_opts
|
||||
env_to_salt docker_root
|
||||
env_to_salt kubelet_root
|
||||
env_to_salt non_masquerade_cidr
|
||||
|
||||
install-salt
|
||||
|
||||
|
||||
@ -29,6 +29,7 @@ function generate-minion-user-data {
|
||||
echo "#! /bin/bash"
|
||||
echo "SALT_MASTER='${MASTER_INTERNAL_IP}'"
|
||||
echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS:-}'"
|
||||
echo "readonly NON_MASQUERADE_CIDR='${NON_MASQUERADE_CIDR:-}'"
|
||||
echo "readonly DOCKER_STORAGE='${DOCKER_STORAGE:-}'"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
|
||||
|
||||
@ -893,6 +893,7 @@ function start-master() {
|
||||
echo "readonly SALT_MASTER='${MASTER_INTERNAL_IP}'"
|
||||
echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
|
||||
echo "readonly NODE_INSTANCE_PREFIX='${NODE_INSTANCE_PREFIX}'"
|
||||
echo "readonly NON_MASQUERADE_CIDR='${NON_MASQUERADE_CIDR:-}'"
|
||||
echo "readonly CLUSTER_IP_RANGE='${CLUSTER_IP_RANGE}'"
|
||||
echo "readonly ALLOCATE_NODE_CIDRS='${ALLOCATE_NODE_CIDRS}'"
|
||||
echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
|
||||
|
||||
@ -85,6 +85,11 @@
|
||||
{% set configure_cbr0 = "--configure-cbr0=" + pillar['allocate_node_cidrs'] -%}
|
||||
{% endif -%}
|
||||
|
||||
{% set non_masquerade_cidr = "" -%}
|
||||
{% if grains.non_masquerade_cidr is defined -%}
|
||||
{% set non_masquerade_cidr = "--non-masquerade-cidr=" + grains.non_masquerade_cidr -%}
|
||||
{% endif -%}
|
||||
|
||||
# The master kubelet cannot wait for the flannel daemon because it is responsible
|
||||
# for starting up the flannel server in a static pod. So even though the flannel
|
||||
# daemon runs on the master, it doesn't hold up cluster bootstrap. All the pods
|
||||
@ -137,4 +142,4 @@
|
||||
{% endif -%}
|
||||
|
||||
# test_args has to be kept at the end, so they'll overwrite any prior configuration
|
||||
DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{test_args}}"
|
||||
DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{test_args}}"
|
||||
|
||||
@ -97,6 +97,7 @@ func NewKubeletServer() *KubeletServer {
|
||||
MinimumGCAge: unversioned.Duration{1 * time.Minute},
|
||||
NetworkPluginDir: "/usr/libexec/kubernetes/kubelet-plugins/net/exec/",
|
||||
NetworkPluginName: "",
|
||||
NonMasqueradeCIDR: "10.0.0.0/8",
|
||||
VolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
|
||||
NodeStatusUpdateFrequency: unversioned.Duration{10 * time.Second},
|
||||
NodeLabels: make(map[string]string),
|
||||
@ -194,6 +195,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
|
||||
fs.BoolVar(&s.ConfigureCBR0, "configure-cbr0", s.ConfigureCBR0, "If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.")
|
||||
fs.IntVar(&s.MaxPods, "max-pods", s.MaxPods, "Number of Pods that can run on this Kubelet.")
|
||||
fs.StringVar(&s.DockerExecHandlerName, "docker-exec-handler", s.DockerExecHandlerName, "Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.")
|
||||
fs.StringVar(&s.NonMasqueradeCIDR, "non-masquerade-cidr", s.NonMasqueradeCIDR, "Traffic to IPs outside this range will use IP masquerade.")
|
||||
fs.StringVar(&s.PodCIDR, "pod-cidr", "", "The CIDR to use for pod IP addresses, only used in standalone mode. In cluster mode, this is obtained from the master.")
|
||||
fs.StringVar(&s.ResolverConfig, "resolv-conf", kubetypes.ResolvConfDefault, "Resolver configuration file used as the basis for the container DNS resolution configuration.")
|
||||
fs.BoolVar(&s.CPUCFSQuota, "cpu-cfs-quota", s.CPUCFSQuota, "Enable CPU CFS quota enforcement for containers that specify CPU limits")
|
||||
|
||||
@ -222,6 +222,7 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
|
||||
NetworkPlugins: ProbeNetworkPlugins(s.NetworkPluginDir),
|
||||
NodeLabels: s.NodeLabels,
|
||||
NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency.Duration,
|
||||
NonMasqueradeCIDR: s.NonMasqueradeCIDR,
|
||||
OOMAdjuster: oom.NewOOMAdjuster(),
|
||||
OSInterface: kubecontainer.RealOS{},
|
||||
PodCIDR: s.PodCIDR,
|
||||
@ -700,6 +701,7 @@ type KubeletConfig struct {
|
||||
NodeName string
|
||||
NodeLabels map[string]string
|
||||
NodeStatusUpdateFrequency time.Duration
|
||||
NonMasqueradeCIDR string
|
||||
OOMAdjuster *oom.OOMAdjuster
|
||||
OSInterface kubecontainer.OSInterface
|
||||
PodCIDR string
|
||||
@ -803,6 +805,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
|
||||
kc.DockerDaemonContainer,
|
||||
kc.SystemContainer,
|
||||
kc.ConfigureCBR0,
|
||||
kc.NonMasqueradeCIDR,
|
||||
kc.PodCIDR,
|
||||
kc.ReconcileCIDR,
|
||||
kc.MaxPods,
|
||||
|
||||
@ -117,6 +117,7 @@ kubelet
|
||||
--node-ip="": IP address of the node. If set, kubelet will use this IP address for the node
|
||||
--node-labels=: <Warning: Alpha feature> Labels to add when registering the node in the cluster. Labels must are key=value pairs seperated by ','.
|
||||
--node-status-update-frequency=10s: Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s
|
||||
--non-masquerade-cidr="10.0.0.0/8": Traffic to IPs outside this range will use IP masquerade.
|
||||
--oom-score-adj=-999: The oom-score-adj value for kubelet process. Values must be within the range [-1000, 1000]
|
||||
--outofdisk-transition-frequency=5m0s: Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status. Default: 5m0s
|
||||
--pod-cidr="": The CIDR to use for pod IP addresses, only used in standalone mode. In cluster mode, this is obtained from the master.
|
||||
|
||||
@ -237,6 +237,7 @@ node-name
|
||||
node-startup-grace-period
|
||||
node-status-update-frequency
|
||||
node-sync-period
|
||||
non-masquerade-cidr
|
||||
num-nodes
|
||||
oidc-ca-file
|
||||
oidc-client-id
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -274,6 +274,8 @@ type KubeletConfiguration struct {
|
||||
NodeIP string `json:"nodeIP,omitempty"`
|
||||
// nodeLabels to add when registering the node in the cluster.
|
||||
NodeLabels map[string]string `json:"nodeLabels"`
|
||||
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
|
||||
NonMasqueradeCIDR string `json:"nonMasqueradeCIDR"`
|
||||
}
|
||||
|
||||
// LeaderElectionConfiguration defines the configuration of leader election
|
||||
|
||||
@ -119,23 +119,24 @@ func cbr0CidrCorrect(wantCIDR *net.IPNet) bool {
|
||||
}
|
||||
|
||||
// TODO(dawnchen): Using pkg/util/iptables
|
||||
func ensureIPTablesMasqRule() error {
|
||||
// nonMasqueradeCIDR is the CIDR for our internal IP range; traffic to IPs outside this range will use IP masquerade.
|
||||
func ensureIPTablesMasqRule(nonMasqueradeCIDR string) error {
|
||||
// Check if the MASQUERADE rule exist or not
|
||||
if err := exec.Command("iptables",
|
||||
"-t", "nat",
|
||||
"-C", "POSTROUTING",
|
||||
"!", "-d", "10.0.0.0/8",
|
||||
"!", "-d", nonMasqueradeCIDR,
|
||||
"-m", "addrtype", "!", "--dst-type", "LOCAL",
|
||||
"-j", "MASQUERADE").Run(); err == nil {
|
||||
// The MASQUERADE rule exists
|
||||
return nil
|
||||
}
|
||||
|
||||
glog.Infof("MASQUERADE rule doesn't exist, recreate it")
|
||||
glog.Infof("MASQUERADE rule doesn't exist, recreate it (with nonMasqueradeCIDR %s)", nonMasqueradeCIDR)
|
||||
if err := exec.Command("iptables",
|
||||
"-t", "nat",
|
||||
"-A", "POSTROUTING",
|
||||
"!", "-d", "10.0.0.0/8",
|
||||
"!", "-d", nonMasqueradeCIDR,
|
||||
"-m", "addrtype", "!", "--dst-type", "LOCAL",
|
||||
"-j", "MASQUERADE").Run(); err != nil {
|
||||
return err
|
||||
|
||||
@ -183,6 +183,7 @@ func NewMainKubelet(
|
||||
dockerDaemonContainer string,
|
||||
systemContainer string,
|
||||
configureCBR0 bool,
|
||||
nonMasqueradeCIDR string,
|
||||
podCIDR string,
|
||||
reconcileCIDR bool,
|
||||
maxPods int,
|
||||
@ -302,6 +303,7 @@ func NewMainKubelet(
|
||||
chownRunner: chownRunner,
|
||||
writer: writer,
|
||||
configureCBR0: configureCBR0,
|
||||
nonMasqueradeCIDR: nonMasqueradeCIDR,
|
||||
reconcileCIDR: reconcileCIDR,
|
||||
maxPods: maxPods,
|
||||
syncLoopMonitor: atomic.Value{},
|
||||
@ -610,6 +612,9 @@ type Kubelet struct {
|
||||
configureCBR0 bool
|
||||
reconcileCIDR bool
|
||||
|
||||
// Traffic to IPs outside this range will use IP masquerade.
|
||||
nonMasqueradeCIDR string
|
||||
|
||||
// Maximum Number of Pods which can be run by this Kubelet
|
||||
maxPods int
|
||||
|
||||
@ -2637,7 +2642,7 @@ func (kl *Kubelet) syncNetworkStatus() {
|
||||
kl.runtimeState.podCIDR(), podCIDR)
|
||||
kl.runtimeState.setPodCIDR(podCIDR)
|
||||
}
|
||||
if err := ensureIPTablesMasqRule(); err != nil {
|
||||
if err := ensureIPTablesMasqRule(kl.nonMasqueradeCIDR); err != nil {
|
||||
err = fmt.Errorf("Error on adding ip table rules: %v", err)
|
||||
glog.Error(err)
|
||||
kl.runtimeState.setNetworkState(err)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user