github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-23 19:56:01 +00:00
Code Issues Projects Releases Wiki Activity

Increase the fluentd buffer chunk size to improve write throughput.

Browse Source 
Also reduce the max wait between retries, 30 seconds should be more than
enough backoff.
This commit is contained in:
Alex Robinson 2016-01-26 23:57:02 +00:00
parent 2a83f5d5c7
commit 95a4ea3fc8
2 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf
View File

@ -186,12 +186,12 @@
port 9200
logstash_format true
# Set the chunk limit the same as for fluentd-gcp.
buffer_chunk_limit 512K
# Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB
buffer_queue_limit 128
buffer_chunk_limit 2M
# Cap buffer memory usage to 2MiB/chunk * 32 chunks = 64 MiB
buffer_queue_limit 32
flush_interval 5s
# Never wait longer than 5 minutes between retries.
max_retry_wait 300
max_retry_wait 30
# Disable the limit on the number of retries (retry forever).
disable_retry_limit
</match>

18
cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
View File

@ -130,15 +130,15 @@
<match kubernetes.**>
type google_cloud
# Set the chunk limit conservatively to avoid exceeding the GCL limit
# of 2MB per write request.
buffer_chunk_limit 512K
# of 10MiB per write request.
buffer_chunk_limit 2M
# Cap the combined memory usage of this buffer and the one below to
# 512KB/chunk * (96 + 32) chunks = 65 MB
buffer_queue_limit 96
# 2MiB/chunk * (24 + 8) chunks = 64 MiB
buffer_queue_limit 24
# Never wait more than 5 seconds before flushing logs in the non-error case.
flush_interval 5s
# Never wait longer than 5 minutes between retries.
max_retry_wait 300
# Never wait longer than 30 seconds between retries.
max_retry_wait 30
# Disable the limit on the number of retries (retry forever).
disable_retry_limit
</match>
@ -148,9 +148,9 @@
<match **>
type google_cloud
detect_subservice false
buffer_chunk_limit 512K
buffer_queue_limit 32
buffer_chunk_limit 2M
buffer_queue_limit 8
flush_interval 5s
max_retry_wait 300
max_retry_wait 30
disable_retry_limit
</match>