From 4bfbd5f55328f9db73211f2ead5e84e2aa4af587 Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Fri, 3 Jul 2015 23:24:08 +0000 Subject: [PATCH] Add limits to the fluentd buffer size. --- .../fluentd-elasticsearch/fluentd-es-image/td-agent.conf | 4 ++++ .../addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf index f34a79b0544..cd07b235150 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf @@ -133,6 +133,10 @@ host elasticsearch-logging port 9200 logstash_format true + # Set the chunk limit the same as for fluentd-gcp. + buffer_chunk_limit 512K + # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB + buffer_queue_limit 128 flush_interval 5s # Never wait longer than 5 minutes between retries. max_retry_wait 300 diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf index 02615d1db79..4fb6cc30034 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf @@ -113,6 +113,11 @@ type google_cloud + # Set the chunk limit conservatively to avoid exceeding the GCL limit + # of 2MB per write request. + buffer_chunk_limit 512K + # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB + buffer_queue_limit 128 flush_interval 5s # Never wait longer than 5 minutes between retries. max_retry_wait 300