Merge pull request #20883 from bprashanth/kubelet_flag

Kubelet flag for disabling hairpin mode
This commit is contained in:
Prashanth B 2016-02-09 17:31:30 -08:00
commit f0620a5efd
15 changed files with 1015 additions and 934 deletions

View File

@ -499,6 +499,7 @@ CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-})
KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-})
KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-})
NETWORK_PROVIDER: $(yaml-quote ${NETWORK_PROVIDER:-})
HAIRPIN_MODE: $(yaml-quote ${HAIRPIN_MODE:-})
OPENCONTRAIL_TAG: $(yaml-quote ${OPENCONTRAIL_TAG:-})
OPENCONTRAIL_KUBERNETES_TAG: $(yaml-quote ${OPENCONTRAIL_KUBERNETES_TAG:-})
OPENCONTRAIL_PUBLIC_SUBNET: $(yaml-quote ${OPENCONTRAIL_PUBLIC_SUBNET:-})

View File

@ -118,5 +118,8 @@ OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
# Should the kubelet configure hairpin mode on the bridge?
HAIRPIN_MODE="${HAIRPIN_MODE:-true}" # true, false
# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}

View File

@ -138,5 +138,8 @@ OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
# Should the kubelet configure hairpin mode on the bridge?
HAIRPIN_MODE="${HAIRPIN_MODE:-true}" # true, false
# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}

View File

@ -316,6 +316,7 @@ dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")'
dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")'
admission_control: '$(echo "$ADMISSION_CONTROL" | sed -e "s/'/''/g")'
network_provider: '$(echo "$NETWORK_PROVIDER" | sed -e "s/'/''/g")'
hairpin_mode: '$(echo "$HAIRPIN_MODE" | sed -e "s/'/''/g")'
opencontrail_tag: '$(echo "$OPENCONTRAIL_TAG" | sed -e "s/'/''/g")'
opencontrail_kubernetes_tag: '$(echo "$OPENCONTRAIL_KUBERNETES_TAG")'
opencontrail_public_subnet: '$(echo "$OPENCONTRAIL_PUBLIC_SUBNET")'

View File

@ -147,6 +147,12 @@
{% endif -%}
{% endif -%}
{% set configure_hairpin_mode = "--configure-hairpin-mode=true" -%}
{% if pillar.get('hairpin_mode', '').lower() == 'false' %}
{% set configure_hairpin_mode = "--configure-hairpin-mode=false" %}
{% endif -%}
{% set kubelet_port = "" -%}
{% if pillar['kubelet_port'] is defined -%}
{% set kubelet_port="--port=" + pillar['kubelet_port'] %}
@ -158,4 +164,4 @@
{% endif -%}
# test_args has to be kept at the end, so they'll overwrite any prior configuration
DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{test_args}}"
DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ configure_hairpin_mode }} {{test_args}}"

View File

@ -35,6 +35,7 @@ import (
const (
defaultRootDir = "/var/lib/kubelet"
experimentalFlannelOverlay = false
hairpinMode = true
)
// KubeletServer encapsulates all of the parameters necessary for starting up
@ -124,6 +125,7 @@ func NewKubeletServer() *KubeletServer {
KubeAPIBurst: 10,
ExperimentalFlannelOverlay: experimentalFlannelOverlay,
OutOfDiskTransitionFrequency: unversioned.Duration{5 * time.Minute},
HairpinMode: hairpinMode,
},
}
}
@ -196,6 +198,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.RktStage1Image, "rkt-stage1-image", s.RktStage1Image, "image to use as stage1. Local paths and http/https URLs are supported. If empty, the 'stage1.aci' in the same directory as '--rkt-path' will be used")
fs.StringVar(&s.SystemContainer, "system-container", s.SystemContainer, "Optional resource-only container in which to place all non-kernel processes that are not already in a container. Empty for no container. Rolling back the flag requires a reboot. (Default: \"\").")
fs.BoolVar(&s.ConfigureCBR0, "configure-cbr0", s.ConfigureCBR0, "If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.")
fs.BoolVar(&s.HairpinMode, "configure-hairpin-mode", s.HairpinMode, "If true, kubelet will set the hairpin mode flag on container interfaces. This allows endpoints of a Service to loadbalance back to themselves if they should try to access their own Service.")
fs.IntVar(&s.MaxPods, "max-pods", s.MaxPods, "Number of Pods that can run on this Kubelet.")
fs.StringVar(&s.DockerExecHandlerName, "docker-exec-handler", s.DockerExecHandlerName, "Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.")
fs.StringVar(&s.NonMasqueradeCIDR, "non-masquerade-cidr", s.NonMasqueradeCIDR, "Traffic to IPs outside this range will use IP masquerade.")

View File

@ -249,6 +249,7 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
Writer: writer,
VolumePlugins: ProbeVolumePlugins(s.VolumePluginDir),
OutOfDiskTransitionFrequency: s.OutOfDiskTransitionFrequency.Duration,
HairpinMode: s.HairpinMode,
ExperimentalFlannelOverlay: s.ExperimentalFlannelOverlay,
NodeIP: net.ParseIP(s.NodeIP),
@ -734,6 +735,7 @@ type KubeletConfig struct {
ExperimentalFlannelOverlay bool
NodeIP net.IP
ContainerRuntimeOptions []kubecontainer.Option
HairpinMode bool
}
func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.PodConfig, err error) {
@ -822,6 +824,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
kc.EnableCustomMetrics,
kc.VolumeStatsAggPeriod,
kc.ContainerRuntimeOptions,
kc.HairpinMode,
)
if err != nil {

View File

@ -77,6 +77,7 @@ kubelet
--cluster-domain="": Domain for this cluster. If set, kubelet will configure all containers to search this domain in addition to the host's search domains
--config="": Path to the config file or directory of files
--configure-cbr0[=false]: If true, kubelet will configure cbr0 based on Node.Spec.PodCIDR.
--configure-hairpin-mode[=true]: If true, kubelet will set the hairpin mode flag on container interfaces. This allows endpoints of a Service to loadbalance back to themselves if they should try to access their own Service.
--container-runtime="docker": The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.
--containerized[=false]: Experimental support for running kubelet in a container. Intended for testing. [default=false]
--cpu-cfs-quota[=true]: Enable CPU CFS quota enforcement for containers that specify CPU limits
@ -148,7 +149,7 @@ kubelet
--volume-stats-agg-period=1m0s: Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes. To disable volume calculations, set to 0. Default: '1m'
```
###### Auto generated by spf13/cobra on 5-Feb-2016
###### Auto generated by spf13/cobra on 9-Feb-2016
<!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

View File

@ -609,6 +609,7 @@ case ${JOB_NAME} in
: ${E2E_TEST:="false"}
: ${E2E_UP:="true"}
: ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly"}
: ${HAIRPIN_MODE:="false"}
: ${PROJECT:="kubernetes-jenkins"}
;;
@ -627,6 +628,7 @@ case ${JOB_NAME} in
# TODO(ihmccreery) remove [Skipped] once tests are relabeled
: ${GINKGO_TEST_ARGS:="--ginkgo.skip=\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[Skipped\]"}
: ${KUBE_GCE_INSTANCE_PREFIX:="gce-soak-weekly"}
: ${HAIRPIN_MODE:="false"}
: ${PROJECT:="kubernetes-jenkins"}
;;

View File

@ -381,3 +381,4 @@ leader-elect-lease-duration
leader-elect-renew-deadline
leader-elect-retry-period
watch-cache-sizes
configure-hairpin-mode

File diff suppressed because it is too large Load Diff

View File

@ -244,6 +244,11 @@ type KubeletConfiguration struct {
// configureCBR0 enables the kublet to configure cbr0 based on
// Node.Spec.PodCIDR.
ConfigureCBR0 bool `json:"configureCbr0"`
// Should the kubelet set the hairpin flag on veth interfaces for containers
// it creates? Setting this flag allows endpoints in a Service to
// loadbalance back to themselves if they should try to access their own
// Service.
HairpinMode bool `json:"configureHairpinMode"`
// maxPods is the number of pods that can run on this Kubelet.
MaxPods int `json:"maxPods"`
// dockerExecHandlerName is the handler to use when executing a command

View File

@ -47,7 +47,7 @@ func NewFakeDockerManager(
fakeProcFs := procfs.NewFakeProcFS()
dm := NewDockerManager(client, recorder, livenessManager, containerRefManager, machineInfo, podInfraContainerImage, qps,
burst, containerLogsDir, osInterface, networkPlugin, runtimeHelper, httpClient, &NativeExecHandler{},
fakeOOMAdjuster, fakeProcFs, false, imageBackOff, true, false)
fakeOOMAdjuster, fakeProcFs, false, imageBackOff, true, false, true)
dm.dockerPuller = &FakeDockerPuller{}
return dm
}

View File

@ -141,6 +141,11 @@ type DockerManager struct {
// Support for gathering custom metrics.
enableCustomMetrics bool
// If true, the "hairpin mode" flag is set on container interfaces.
// A false value means the kubelet just backs off from setting it,
// it might already be true.
configureHairpinMode bool
}
func PodInfraContainerEnv(env map[string]string) kubecontainer.Option {
@ -176,6 +181,7 @@ func NewDockerManager(
imageBackOff *util.Backoff,
serializeImagePulls bool,
enableCustomMetrics bool,
hairpinMode bool,
options ...kubecontainer.Option) *DockerManager {
// Work out the location of the Docker runtime, defaulting to /var/lib/docker
@ -208,6 +214,7 @@ func NewDockerManager(
procFs: procFs,
cpuCFSQuota: cpuCFSQuota,
enableCustomMetrics: enableCustomMetrics,
configureHairpinMode: hairpinMode,
}
dm.runner = lifecycle.NewHandlerRunner(httpClient, dm, dm)
if serializeImagePulls {
@ -1790,7 +1797,7 @@ func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubec
result.Fail(err)
return
}
if !usesHostNetwork(pod) {
if !usesHostNetwork(pod) && dm.configureHairpinMode {
if err = hairpin.SetUpContainer(podInfraContainer.State.Pid, "eth0"); err != nil {
glog.Warningf("Hairpin setup failed for pod %q: %v", format.Pod(pod), err)
}

View File

@ -207,6 +207,7 @@ func NewMainKubelet(
enableCustomMetrics bool,
volumeStatsAggPeriod time.Duration,
containerRuntimeOptions []kubecontainer.Option,
hairpinMode bool,
) (*Kubelet, error) {
if rootDirectory == "" {
return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
@ -366,6 +367,7 @@ func NewMainKubelet(
// Initialize the runtime.
switch containerRuntime {
case "docker":
glog.Infof("Hairpin mode set to %v", hairpinMode)
// Only supported one for now, continue.
klet.containerRuntime = dockertools.NewDockerManager(
dockerClient,
@ -388,9 +390,11 @@ func NewMainKubelet(
imageBackOff,
serializeImagePulls,
enableCustomMetrics,
hairpinMode,
containerRuntimeOptions...,
)
case "rkt":
// TODO: Include hairpin mode settings in rkt?
conf := &rkt.Config{
Path: rktPath,
Stage1Image: rktStage1Image,