From b22a913079aa5501e3fd2c1536da4161d61f8aae Mon Sep 17 00:00:00 2001 From: Vishnu kannan Date: Fri, 3 Jun 2016 16:06:28 -0700 Subject: [PATCH 1/4] Enable configuration of hard eviction thresholds in kubelet. This will allow for enabling evictions whenever memory availability is too low. Signed-off-by: Vishnu kannan --- cluster/common.sh | 5 +++++ cluster/gce/configure-vm.sh | 5 +++++ cluster/gce/gci/configure-helper.sh | 3 +++ cluster/saltbase/salt/kubelet/default | 7 ++++++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/cluster/common.sh b/cluster/common.sh index 00a4a8e9377..ac9b2ac364c 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -647,6 +647,11 @@ EOF if [ -n "${NODE_LABELS:-}" ]; then cat >>$file <>$file <>/srv/salt-overlay/pillar/cluster-params.sls node_labels: '$(echo "${NODE_LABELS}" | sed -e "s/'/''/g")' +EOF + fi + if [ -n "${EVICTION_HARD:-}" ]; then + cat <>/srv/salt-overlay/pillar/cluster-params.sls +eviction_hard: '$(echo "${EVICTION_HARD}" | sed -e "s/'/''/g")' EOF fi if [[ "${ENABLE_NODE_AUTOSCALER:-false}" == "true" ]]; then diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index a090d8bc579..2fb1d4f527d 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -351,6 +351,9 @@ function start-kubelet { if [[ -n "${NODE_LABELS:-}" ]]; then flags+=" --node-labels=${NODE_LABELS}" fi + if [[ -n "${EVICTION_HARD:-}" ]]; then + flags+=" --eviction-hard=${EVICTION_HARD}" + fi if [[ "${ALLOCATE_NODE_CIDRS:-}" == "true" ]]; then flags+=" --configure-cbr0=${ALLOCATE_NODE_CIDRS}" fi diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 255faeb59df..b72e1a6e984 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -196,5 +196,10 @@ {% set node_labels="--node-labels=" + pillar['node_labels'] %} {% endif -%} +{% set eviction_hard = "" %} +{% if pillar['eviction_hard'] is defined -%} + {% set eviction_hard="--eviction-hard=" + pillar['eviction_hard'] %} +{% endif -%} + # test_args has to be kept at the end, so they'll overwrite any prior configuration -DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{test_args}}" +DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{cloud_config}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{log_level}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{non_masquerade_cidr}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{kubelet_port}} {{experimental_flannel_overlay}} {{ reconcile_cidr_args }} {{ hairpin_mode }} {{enable_custom_metrics}} {{runtime_container}} {{kubelet_container}} {{node_labels}} {{babysit_daemons}} {{eviction_hard}} {{test_args}}" From b120ce98f129580fb5c8bfc1bb52a98f6fa1b3b3 Mon Sep 17 00:00:00 2001 From: Vishnu kannan Date: Fri, 3 Jun 2016 16:12:37 -0700 Subject: [PATCH 2/4] Set hard memory eviction thresholds for GCE clusters. Signed-off-by: Vishnu kannan --- cluster/gce/config-default.sh | 3 +++ cluster/gce/config-test.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 3a043612b5a..c8bb0ac97c2 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -138,3 +138,6 @@ OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}" HAIRPIN_MODE="${HAIRPIN_MODE:-promiscuous-bridge}" # promiscuous-bridge, hairpin-veth, none # Optional: if set to true, kube-up will configure the cluster to run e2e tests. E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false} + +# Evict pods whenever compute resource availability on the nodes gets below a threshold. +EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 8d15479c536..f729f3d84d6 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -170,3 +170,6 @@ E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false} # Optional: if set to true, a image puller is deployed. Only for use in e2e clusters. # TODO: Pipe this through GKE e2e clusters once we know it helps. PREPULL_E2E_IMAGES="${PREPULL_E2E_IMAGES:-true}" + +# Evict pods whenever compute resource availability on the nodes gets below a threshold. +EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi}" From ca657cecff582cc8275ee5eb0a9850cede9c0ce0 Mon Sep 17 00:00:00 2001 From: Vishnu kannan Date: Mon, 6 Jun 2016 17:25:56 -0700 Subject: [PATCH 3/4] make flags script happy Signed-off-by: Vishnu kannan --- hack/verify-flags/exceptions.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 88d60a67ec6..91435dfaf79 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -8,8 +8,6 @@ cluster/aws/templates/configure-vm-aws.sh: env-to-grains "hostname_override" cluster/aws/templates/configure-vm-aws.sh: env-to-grains "runtime_config" cluster/aws/templates/configure-vm-aws.sh: kubelet_api_servers: '${KUBELET_APISERVER}' cluster/centos/util.sh: local node_ip=${node#*@} -cluster/gce/config-default.sh:# reloads /helper.sh in the gap between when the master is created -cluster/gce/config-test.sh:# reloads /helper.sh in the gap between when the master is created cluster/gce/configure-vm.sh: advertise_address: '${EXTERNAL_IP}' cluster/gce/configure-vm.sh: api_servers: '${KUBERNETES_MASTER_NAME}' cluster/gce/configure-vm.sh: cloud_config: ${CLOUD_CONFIG} @@ -46,6 +44,7 @@ cluster/saltbase/salt/kube-proxy/kube-proxy.manifest: {% set cluster_cidr=" --c cluster/saltbase/salt/kubelet/default: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kubelet/default: {% set api_servers_with_port = api_servers -%} cluster/saltbase/salt/kubelet/default: {% set enable_custom_metrics="--enable-custom-metrics=" + pillar['enable_custom_metrics'] %} +cluster/saltbase/salt/kubelet/default: {% set eviction_hard="--eviction-hard=" + pillar['eviction_hard'] %} cluster/saltbase/salt/kubelet/default: {% set kubelet_port="--port=" + pillar['kubelet_port'] %} cluster/saltbase/salt/kubelet/default: {% set node_labels="--node-labels=" + pillar['node_labels'] %} cluster/saltbase/salt/kubelet/default:{% if pillar.get('non_masquerade_cidr','') -%} From 7e731dac60deac7072b0e93b4b06961fea88fb81 Mon Sep 17 00:00:00 2001 From: Vishnu kannan Date: Mon, 6 Jun 2016 17:28:55 -0700 Subject: [PATCH 4/4] ignore _output directory in verify flags script Signed-off-by: Vishnu kannan --- hack/verify-flags-underscore.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/verify-flags-underscore.py b/hack/verify-flags-underscore.py index 1fc4ed35a4e..555c3d264a5 100755 --- a/hack/verify-flags-underscore.py +++ b/hack/verify-flags-underscore.py @@ -55,6 +55,8 @@ def get_all_files(rootdir): # don't visit certain dirs if 'vendor' in dirs: dirs.remove('vendor') + if '_output' in dirs: + dirs.remove('_output') if '_gopath' in dirs: dirs.remove('_gopath') if 'third_party' in dirs: