From 4f2d22263ed8204b54b88c7a540f2cf4e3ac965b Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Wed, 1 Jul 2015 14:39:10 -0700 Subject: [PATCH 1/4] Set resource limit for both heapster and influxdb container based on data collected by #10335. Please noted that both influxdb and heapster could be oom-killed due to memory leakage here. --- .../cluster-monitoring/google/heapster-controller.yaml | 4 ++++ .../googleinfluxdb/heapster-controller-combined.yaml | 4 ++++ .../cluster-monitoring/influxdb/heapster-controller.yaml | 4 ++++ .../influxdb/influxdb-grafana-controller.yaml | 8 ++++++++ .../standalone/heapster-controller.yaml | 4 ++++ 5 files changed, 24 insertions(+) diff --git a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml index 42a55530026..78c2d1a1769 100644 --- a/cluster/addons/cluster-monitoring/google/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/google/heapster-controller.yaml @@ -22,6 +22,10 @@ spec: containers: - image: gcr.io/google_containers/heapster:v0.15.0 name: heapster + resources: + limits: + cpu: 100m + memory: 200Mi command: - /heapster - --source=kubernetes:'' diff --git a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml index b05eb098b5d..a283f2eb0d2 100644 --- a/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml +++ b/cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml @@ -22,6 +22,10 @@ spec: containers: - image: gcr.io/google_containers/heapster:v0.15.0 name: heapster + resources: + limits: + cpu: 100m + memory: 200Mi command: - /heapster - --source=kubernetes:'' diff --git a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml index 348db83b9ac..030c34dccc3 100644 --- a/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml @@ -22,6 +22,10 @@ spec: containers: - image: gcr.io/google_containers/heapster:v0.15.0 name: heapster + resources: + limits: + cpu: 100m + memory: 200Mi command: - /heapster - --source=kubernetes:'' diff --git a/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml b/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml index 94d3dc06b2b..1f7cbcd79cc 100644 --- a/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml +++ b/cluster/addons/cluster-monitoring/influxdb/influxdb-grafana-controller.yaml @@ -22,6 +22,10 @@ spec: containers: - image: gcr.io/google_containers/heapster_influxdb:v0.3 name: influxdb + resources: + limits: + cpu: 100m + memory: 200Mi ports: - containerPort: 8083 hostPort: 8083 @@ -29,6 +33,10 @@ spec: hostPort: 8086 - image: gcr.io/google_containers/heapster_grafana:v0.7 name: grafana + resources: + limits: + cpu: 100m + memory: 100Mi env: - name: INFLUXDB_EXTERNAL_URL value: /api/v1beta3/proxy/namespaces/default/services/monitoring-influxdb:api/db/ diff --git a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml index 130ee8827e2..ee765b1091c 100644 --- a/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml +++ b/cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml @@ -22,6 +22,10 @@ spec: containers: - image: gcr.io/google_containers/heapster:v0.15.0 name: heapster + resources: + limits: + cpu: 100m + memory: 200Mi command: - /heapster - --source=kubernetes:'' From ce520dd6e800ff1604f3a69c83585eec14d8393f Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Wed, 1 Jul 2015 14:47:29 -0700 Subject: [PATCH 2/4] Set resource limit for skydns containers based on data collected by #10335 --- cluster/addons/dns/skydns-rc.yaml.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cluster/addons/dns/skydns-rc.yaml.in b/cluster/addons/dns/skydns-rc.yaml.in index 3f658f45f5b..fea4f41cc02 100644 --- a/cluster/addons/dns/skydns-rc.yaml.in +++ b/cluster/addons/dns/skydns-rc.yaml.in @@ -22,6 +22,10 @@ spec: containers: - name: etcd image: gcr.io/google_containers/etcd:2.0.9 + resources: + limits: + cpu: 100m + memory: 50Mi command: - /usr/local/bin/etcd - -listen-client-urls @@ -32,11 +36,19 @@ spec: - skydns-etcd - name: kube2sky image: gcr.io/google_containers/kube2sky:1.10 + resources: + limits: + cpu: 100m + memory: 50Mi args: # command = "/kube2sky" - -domain={{ pillar['dns_domain'] }} - name: skydns image: gcr.io/google_containers/skydns:2015-03-11-001 + resources: + limits: + cpu: 100m + memory: 50Mi args: # command = "/skydns" - -machines=http://localhost:4001 From 6b61918b678199bbb421eef4233d6270f7bf9184 Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Wed, 1 Jul 2015 15:08:35 -0700 Subject: [PATCH 3/4] Set resource limit for both elasticsearch and kibana containers based on data collected by #10335 --- cluster/addons/fluentd-elasticsearch/es-controller.yaml | 3 +++ cluster/addons/fluentd-elasticsearch/kibana-controller.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/cluster/addons/fluentd-elasticsearch/es-controller.yaml b/cluster/addons/fluentd-elasticsearch/es-controller.yaml index 9ef3adc93c8..f2f4bedf5cd 100644 --- a/cluster/addons/fluentd-elasticsearch/es-controller.yaml +++ b/cluster/addons/fluentd-elasticsearch/es-controller.yaml @@ -22,6 +22,9 @@ spec: containers: - image: gcr.io/google_containers/elasticsearch:1.4 name: elasticsearch-logging + resources: + limits: + cpu: 100m ports: - containerPort: 9200 name: es-port diff --git a/cluster/addons/fluentd-elasticsearch/kibana-controller.yaml b/cluster/addons/fluentd-elasticsearch/kibana-controller.yaml index 677bc5f664a..f28060b3bdb 100644 --- a/cluster/addons/fluentd-elasticsearch/kibana-controller.yaml +++ b/cluster/addons/fluentd-elasticsearch/kibana-controller.yaml @@ -22,6 +22,9 @@ spec: containers: - name: kibana-logging image: gcr.io/google_containers/kibana:1.3 + resources: + limits: + cpu: 100m env: - name: "ELASTICSEARCH_URL" value: "http://elasticsearch-logging:9200" From 54531d9b55b86b3c6354f34f7ebc41b2cb39e2d7 Mon Sep 17 00:00:00 2001 From: Dawn Chen Date: Wed, 1 Jul 2015 17:00:06 -0700 Subject: [PATCH 4/4] Set memory limit (200mi) to fluentd static pod --- cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml index 7433852be58..fbf9fcdfef5 100644 --- a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml +++ b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml @@ -9,6 +9,7 @@ spec: resources: limits: cpu: 100m + memory: 200Mi env: - name: "FLUENTD_ARGS" value: "-qq"