diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile index fdae1c06f3c..f20c9c99e2c 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile @@ -1,7 +1,7 @@ .PHONY: build push IMAGE = fluentd-elasticsearch -TAG = 1.12 +TAG = 1.13 build: docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) . diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf index 4a155ac4296..bbe39854181 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf @@ -100,6 +100,11 @@ # problem yet to be solved as secrets are not usable in static pods which the fluentd # pod must be until a per-node controller is available in Kubernetes. +# Do not directly collect fluentd's own logs to avoid infinite loops. + + type null + + type tail path /var/log/containers/*.log @@ -186,12 +191,12 @@ port 9200 logstash_format true # Set the chunk limit the same as for fluentd-gcp. - buffer_chunk_limit 512K - # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB - buffer_queue_limit 128 + buffer_chunk_limit 2M + # Cap buffer memory usage to 2MiB/chunk * 32 chunks = 64 MiB + buffer_queue_limit 32 flush_interval 5s # Never wait longer than 5 minutes between retries. - max_retry_wait 300 + max_retry_wait 30 # Disable the limit on the number of retries (retry forever). disable_retry_limit diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile index ff9a86c4c96..e7a506f3aa6 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile @@ -14,7 +14,7 @@ .PHONY: kbuild kpush -TAG = 1.14 +TAG = 1.15 # Rules for building the test image for deployment to Dockerhub with user kubernetes. diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf index 25a49850f01..98caf02fb60 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf @@ -42,6 +42,11 @@ # the name of the Kubernetes container regardless of how many times the # Kubernetes pod has been restarted (resulting in a several Docker container IDs). +# Do not directly collect fluentd's own logs to avoid infinite loops. + + type null + + type tail format json @@ -130,15 +135,15 @@ type google_cloud # Set the chunk limit conservatively to avoid exceeding the GCL limit - # of 2MB per write request. - buffer_chunk_limit 512K + # of 10MiB per write request. + buffer_chunk_limit 2M # Cap the combined memory usage of this buffer and the one below to - # 512KB/chunk * (96 + 32) chunks = 65 MB - buffer_queue_limit 96 + # 2MiB/chunk * (24 + 8) chunks = 64 MiB + buffer_queue_limit 24 # Never wait more than 5 seconds before flushing logs in the non-error case. flush_interval 5s - # Never wait longer than 5 minutes between retries. - max_retry_wait 300 + # Never wait longer than 30 seconds between retries. + max_retry_wait 30 # Disable the limit on the number of retries (retry forever). disable_retry_limit @@ -148,9 +153,9 @@ type google_cloud detect_subservice false - buffer_chunk_limit 512K - buffer_queue_limit 32 + buffer_chunk_limit 2M + buffer_queue_limit 8 flush_interval 5s - max_retry_wait 300 + max_retry_wait 30 disable_retry_limit diff --git a/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml b/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml index 6784dd13b33..af662a74811 100644 --- a/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml +++ b/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml @@ -8,7 +8,7 @@ metadata: spec: containers: - name: fluentd-elasticsearch - image: gcr.io/google_containers/fluentd-elasticsearch:1.12 + image: gcr.io/google_containers/fluentd-elasticsearch:1.13 resources: limits: cpu: 100m diff --git a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml index 90a7d86a019..61869685fbd 100644 --- a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml +++ b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml @@ -8,7 +8,7 @@ metadata: spec: containers: - name: fluentd-cloud-logging - image: gcr.io/google_containers/fluentd-gcp:1.14 + image: gcr.io/google_containers/fluentd-gcp:1.15 resources: limits: cpu: 100m diff --git a/docs/getting-started-guides/logging.md b/docs/getting-started-guides/logging.md index c0632b8f220..273065f6764 100644 --- a/docs/getting-started-guides/logging.md +++ b/docs/getting-started-guides/logging.md @@ -172,7 +172,7 @@ metadata: spec: containers: - name: fluentd-cloud-logging - image: gcr.io/google_containers/fluentd-gcp:1.14 + image: gcr.io/google_containers/fluentd-gcp:1.15 resources: limits: cpu: 100m