From 2bce0cb16410f7f0e45fba55a74f88950cc55c9d Mon Sep 17 00:00:00 2001 From: Van Tu Date: Wed, 11 Jul 2018 22:34:48 -0400 Subject: [PATCH 1/4] Update configure-helper.sh to support heapster resource optimizations In addition to the shell script changes the heapster yaml has been updated to use addon resizer 1.8.3 for the heapster-nanny. Addon resizer 1.8.3 is being used to take advantage of the new minClusterSize flag. Note this is a no-op change. The values specified for heapster-nanny reflect the current configuration used with version 1.8.2. --- .../google/heapster-controller.yaml | 3 ++- .../googleinfluxdb/heapster-controller-combined.yaml | 3 ++- .../influxdb/heapster-controller.yaml | 3 ++- .../stackdriver/heapster-controller.yaml | 3 ++- .../standalone/heapster-controller.yaml | 3 ++- cluster/gce/gci/configure-helper.sh | 11 ++++++++++- 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml index 1a472965217..cea80f0dfad 100644 --- a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml @@ -79,7 +79,7 @@ spec: - /eventer - --source=kubernetes:'' - --sink=gcl - - image: k8s.gcr.io/addon-resizer:1.8.2 + - image: k8s.gcr.io/addon-resizer:1.8.3 name: heapster-nanny resources: limits: @@ -112,6 +112,7 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + - --minClusterSize={{ heapster_min_cluster_size }} - image: k8s.gcr.io/addon-resizer:1.8.2 name: eventer-nanny resources: diff --git a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml index df5d617fdec..62c38bd2098 100644 --- a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml +++ b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml @@ -80,7 +80,7 @@ spec: - /eventer - --source=kubernetes:'' - --sink=gcl - - image: k8s.gcr.io/addon-resizer:1.8.2 + - image: k8s.gcr.io/addon-resizer:1.8.3 name: heapster-nanny resources: limits: @@ -113,6 +113,7 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + - --minClusterSize={{ heapster_min_cluster_size }} - image: k8s.gcr.io/addon-resizer:1.8.2 name: eventer-nanny resources: diff --git a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml index 10abb448794..06fc3a4f6c5 100644 --- a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml @@ -79,7 +79,7 @@ spec: - /eventer - --source=kubernetes:'' - --sink=influxdb:http://monitoring-influxdb:8086 - - image: k8s.gcr.io/addon-resizer:1.8.2 + - image: k8s.gcr.io/addon-resizer:1.8.3 name: heapster-nanny resources: limits: @@ -112,6 +112,7 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + - --minClusterSize={{ heapster_min_cluster_size }} - image: k8s.gcr.io/addon-resizer:1.8.2 name: eventer-nanny resources: diff --git a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml index c6a548081ab..6198f3c9902 100644 --- a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml @@ -80,7 +80,7 @@ spec: fieldRef: fieldPath: metadata.namespace # END_PROMETHEUS_TO_SD - - image: k8s.gcr.io/addon-resizer:1.8.2 + - image: k8s.gcr.io/addon-resizer:1.8.3 name: heapster-nanny resources: limits: @@ -113,6 +113,7 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + - --minClusterSize={{ heapster_min_cluster_size }} volumes: - name: heapster-config-volume configMap: diff --git a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml index 2e0c8460ecf..1883099781f 100644 --- a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml @@ -59,7 +59,7 @@ spec: command: - /heapster - --source=kubernetes.summary_api:'' - - image: k8s.gcr.io/addon-resizer:1.8.2 + - image: k8s.gcr.io/addon-resizer:1.8.3 name: heapster-nanny resources: limits: @@ -92,6 +92,7 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + - --minClusterSize={{ heapster_min_cluster_size }} volumes: - name: heapster-config-volume configMap: diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 1330b3edfdd..ba597983738 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2348,10 +2348,18 @@ EOF base_eventer_memory="190Mi" base_metrics_cpu="${HEAPSTER_GCP_BASE_CPU:-80m}" nanny_memory="90Mi" - local -r metrics_memory_per_node="${HEAPSTER_GCP_MEMORY_PER_NODE:-4}" + local heapster_min_cluster_size="16" + local metrics_memory_per_node="${HEAPSTER_GCP_MEMORY_PER_NODE:-4}" local -r metrics_cpu_per_node="${HEAPSTER_GCP_CPU_PER_NODE:-0.5}" local -r eventer_memory_per_node="500" local -r nanny_memory_per_node="200" + if [[ "${ENABLE_SYSTEM_ADDON_RESOURCE_OPTIMIZATIONS:-}" == "true" ]]; then + # TODO: Set each of the following variables to optimized values. + base_metrics_memory="140Mi" + base_metrics_cpu="80m" + metrics_memory_per_node="4" + heapster_min_cluster_size="16" + fi if [[ -n "${NUM_NODES:-}" && "${NUM_NODES}" -ge 1 ]]; then num_kube_nodes="$((${NUM_NODES}+1))" nanny_memory="$((${num_kube_nodes} * ${nanny_memory_per_node} + 90 * 1024))Ki" @@ -2372,6 +2380,7 @@ EOF sed -i -e "s@{{ *eventer_memory_per_node *}}@${eventer_memory_per_node}@g" "${controller_yaml}" sed -i -e "s@{{ *nanny_memory *}}@${nanny_memory}@g" "${controller_yaml}" sed -i -e "s@{{ *metrics_cpu_per_node *}}@${metrics_cpu_per_node}@g" "${controller_yaml}" + sed -i -e "s@{{ *heapster_min_cluster_size *}}@${heapster_min_cluster_size}@g" "${controller_yaml}" update-prometheus-to-sd-parameters ${controller_yaml} if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]]; then From 5e6a884dc60bd221e56b2f7d92b24c514b6463a3 Mon Sep 17 00:00:00 2001 From: Van Tu Date: Wed, 8 Aug 2018 13:59:34 -0400 Subject: [PATCH 2/4] Revise the setting of variables to leverage environment variables. --- cluster/gce/gci/configure-helper.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index ba597983738..798a3b298c8 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2355,9 +2355,9 @@ EOF local -r nanny_memory_per_node="200" if [[ "${ENABLE_SYSTEM_ADDON_RESOURCE_OPTIMIZATIONS:-}" == "true" ]]; then # TODO: Set each of the following variables to optimized values. - base_metrics_memory="140Mi" - base_metrics_cpu="80m" - metrics_memory_per_node="4" + base_metrics_memory="${HEAPSTER_GCP_BASE_MEMORY:-140Mi}" + base_metrics_cpu="${HEAPSTER_GCP_BASE_CPU:-80m}" + metrics_memory_per_node="${HEAPSTER_GCP_MEMORY_PER_NODE:-4}" heapster_min_cluster_size="16" fi if [[ -n "${NUM_NODES:-}" && "${NUM_NODES}" -ge 1 ]]; then From d347d283bf22f7a2f47d3ae31a377f1cf42cc80a Mon Sep 17 00:00:00 2001 From: Van Tu Date: Wed, 8 Aug 2018 15:27:50 -0400 Subject: [PATCH 3/4] Add comments clarifying the use of the minClusterSize flag. --- .../addons/cluster-monitoring/google/heapster-controller.yaml | 2 ++ .../googleinfluxdb/heapster-controller-combined.yaml | 2 ++ .../addons/cluster-monitoring/influxdb/heapster-controller.yaml | 2 ++ .../cluster-monitoring/stackdriver/heapster-controller.yaml | 2 ++ .../cluster-monitoring/standalone/heapster-controller.yaml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml index cea80f0dfad..f936d5d2980 100644 --- a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml @@ -112,6 +112,8 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + # Specifies the smallest cluster (defined in number of nodes) + # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} - image: k8s.gcr.io/addon-resizer:1.8.2 name: eventer-nanny diff --git a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml index 62c38bd2098..9d6bde7f0b1 100644 --- a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml +++ b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml @@ -113,6 +113,8 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + # Specifies the smallest cluster (defined in number of nodes) + # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} - image: k8s.gcr.io/addon-resizer:1.8.2 name: eventer-nanny diff --git a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml index 06fc3a4f6c5..d86739a7a9b 100644 --- a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml @@ -112,6 +112,8 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + # Specifies the smallest cluster (defined in number of nodes) + # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} - image: k8s.gcr.io/addon-resizer:1.8.2 name: eventer-nanny diff --git a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml index 6198f3c9902..a69ab9026b7 100644 --- a/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/stackdriver/heapster-controller.yaml @@ -113,6 +113,8 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + # Specifies the smallest cluster (defined in number of nodes) + # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} volumes: - name: heapster-config-volume diff --git a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml index 1883099781f..18bfe64c373 100644 --- a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml @@ -92,6 +92,8 @@ spec: - --container=heapster - --poll-period=300000 - --estimator=exponential + # Specifies the smallest cluster (defined in number of nodes) + # resources will be scaled to. - --minClusterSize={{ heapster_min_cluster_size }} volumes: - name: heapster-config-volume From 5d191d877a51b10edd989880a227c4d5473aff44 Mon Sep 17 00:00:00 2001 From: Van Tu Date: Fri, 10 Aug 2018 10:14:28 -0400 Subject: [PATCH 4/4] Populate final scaling policy values for system addon optimizations --- cluster/gce/gci/configure-helper.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 798a3b298c8..3e74d2f2a91 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -2354,11 +2354,10 @@ EOF local -r eventer_memory_per_node="500" local -r nanny_memory_per_node="200" if [[ "${ENABLE_SYSTEM_ADDON_RESOURCE_OPTIMIZATIONS:-}" == "true" ]]; then - # TODO: Set each of the following variables to optimized values. - base_metrics_memory="${HEAPSTER_GCP_BASE_MEMORY:-140Mi}" - base_metrics_cpu="${HEAPSTER_GCP_BASE_CPU:-80m}" + base_metrics_memory="${HEAPSTER_GCP_BASE_MEMORY:-100Mi}" + base_metrics_cpu="${HEAPSTER_GCP_BASE_CPU:-10m}" metrics_memory_per_node="${HEAPSTER_GCP_MEMORY_PER_NODE:-4}" - heapster_min_cluster_size="16" + heapster_min_cluster_size="5" fi if [[ -n "${NUM_NODES:-}" && "${NUM_NODES}" -ge 1 ]]; then num_kube_nodes="$((${NUM_NODES}+1))"