From 38bb110905e2a6aaaaed37d393bf52d2373bdcac Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Thu, 17 Dec 2015 12:10:57 +0100 Subject: [PATCH 1/2] Enable etcd for events only in large clusters --- cluster/saltbase/salt/etcd/init.sls | 3 +++ .../saltbase/salt/kube-apiserver/kube-apiserver.manifest | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cluster/saltbase/salt/etcd/init.sls b/cluster/saltbase/salt/etcd/init.sls index edd9f652f4b..3aa10e57ac4 100644 --- a/cluster/saltbase/salt/etcd/init.sls +++ b/cluster/saltbase/salt/etcd/init.sls @@ -67,6 +67,8 @@ touch /var/log/etcd-events.log: server_port: 2380 cpulimit: '"200m"' +# Switch on second etcd instance if there are more than 50 nodes. +{% if pillar['num_nodes'] is defined and pillar['num_nodes'] > 50 -%} /etc/kubernetes/manifests/etcd-events.manifest: file.managed: - source: salt://etcd/etcd.manifest @@ -81,3 +83,4 @@ touch /var/log/etcd-events.log: port: 4002 server_port: 2381 cpulimit: '"100m"' +{% endif -%} diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index f43deb9f9b0..fbc000209c4 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -41,7 +41,11 @@ {% endif -%} {% set etcd_servers = "--etcd-servers=http://127.0.0.1:4001" -%} -{% set etcd_servers_overrides = "--etcd-servers-overrides=/events#http://127.0.0.1:4002" -%} +{% set etcd_servers_overrides = "" -%} +# If there are more than 50 nodes, there is a dedicated etcd instance for events. +{% if pillar['num_nodes'] is defined and pillar['num_nodes'] > 50 -%} + {% set etcd_servers_overrides = "--etcd-servers-overrides=/events#http://127.0.0.1:4002" -%} +{% endif -%} {% set service_cluster_ip_range = "" -%} {% if pillar['service_cluster_ip_range'] is defined -%} From 0209792f7b932a1e05fd07cdc24d1c8d73ab7720 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Fri, 18 Dec 2015 12:11:42 +0100 Subject: [PATCH 2/2] Update docs about etcd for events --- docs/admin/cluster-large.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/admin/cluster-large.md b/docs/admin/cluster-large.md index ceb51ff8dfe..391aec60140 100644 --- a/docs/admin/cluster-large.md +++ b/docs/admin/cluster-large.md @@ -62,6 +62,16 @@ To avoid running into cloud provider quota issues, when creating a cluster with * Target pools * Gating the setup script so that it brings up new node VMs in smaller batches with waits in between, because some cloud providers rate limit the creation of VMs. +### Etcd storage + +To improve performance of large clusters, we store events in a separate dedicated etcd instance. + +When creating a cluster, existing salt scripts: +* start and configure additional etcd instance +* configure api-server to use it for storing events + +However, this is done only for clusters having more than 50 nodes. + ### Addon Resources To prevent memory leaks or other resource issues in [cluster addons](../../cluster/addons/) from consuming all the resources available on a node, Kubernetes sets resource limits on addon containers to limit the CPU and Memory resources they can consume (See PR [#10653](http://pr.k8s.io/10653/files) and [#10778](http://pr.k8s.io/10778/files)).