Increase the fluentd buffer chunk size to improve write throughput.

Also reduce the max wait between retries, 30 seconds should be more than
enough backoff.
This commit is contained in:
Alex Robinson 2016-01-26 23:57:02 +00:00
parent 2a83f5d5c7
commit 95a4ea3fc8
2 changed files with 13 additions and 13 deletions

View File

@ -186,12 +186,12 @@
port 9200
logstash_format true
# Set the chunk limit the same as for fluentd-gcp.
buffer_chunk_limit 512K
# Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB
buffer_queue_limit 128
buffer_chunk_limit 2M
# Cap buffer memory usage to 2MiB/chunk * 32 chunks = 64 MiB
buffer_queue_limit 32
flush_interval 5s
# Never wait longer than 5 minutes between retries.
max_retry_wait 300
max_retry_wait 30
# Disable the limit on the number of retries (retry forever).
disable_retry_limit
</match>

View File

@ -130,15 +130,15 @@
<match kubernetes.**>
type google_cloud
# Set the chunk limit conservatively to avoid exceeding the GCL limit
# of 2MB per write request.
buffer_chunk_limit 512K
# of 10MiB per write request.
buffer_chunk_limit 2M
# Cap the combined memory usage of this buffer and the one below to
# 512KB/chunk * (96 + 32) chunks = 65 MB
buffer_queue_limit 96
# 2MiB/chunk * (24 + 8) chunks = 64 MiB
buffer_queue_limit 24
# Never wait more than 5 seconds before flushing logs in the non-error case.
flush_interval 5s
# Never wait longer than 5 minutes between retries.
max_retry_wait 300
# Never wait longer than 30 seconds between retries.
max_retry_wait 30
# Disable the limit on the number of retries (retry forever).
disable_retry_limit
</match>
@ -148,9 +148,9 @@
<match **>
type google_cloud
detect_subservice false
buffer_chunk_limit 512K
buffer_queue_limit 32
buffer_chunk_limit 2M
buffer_queue_limit 8
flush_interval 5s
max_retry_wait 300
max_retry_wait 30
disable_retry_limit
</match>