diff --git a/contrib/logging/fluentd-sidecar-es/Makefile b/contrib/logging/fluentd-sidecar-es/Makefile index 29a94eedb99..b289b47fc00 100644 --- a/contrib/logging/fluentd-sidecar-es/Makefile +++ b/contrib/logging/fluentd-sidecar-es/Makefile @@ -1,6 +1,6 @@ .PHONY: build push -TAG = 1.1 +TAG = 1.2 build: docker build -t gcr.io/google_containers/fluentd-sidecar-es:$(TAG) . diff --git a/contrib/logging/fluentd-sidecar-es/README.md b/contrib/logging/fluentd-sidecar-es/README.md index 3802355e0ee..bd6f6c28a2e 100644 --- a/contrib/logging/fluentd-sidecar-es/README.md +++ b/contrib/logging/fluentd-sidecar-es/README.md @@ -9,7 +9,7 @@ This shouldn't be necessary if your container writes its logs to stdout or stder In order to make this work, you have to add a few things to your pod config: -1. A second container, using the `gcr.io/google_containers/fluentd-sidecar-es:1.1` image to send the logs to Elasticsearch. +1. A second container, using the `gcr.io/google_containers/fluentd-sidecar-es:1.2` image to send the logs to Elasticsearch. 2. A volume for the two containers to share. The emptyDir volume type is a good choice for this because we only want the volume to exist for the lifetime of the pod. 3. Mount paths for the volume in each container. In your primary container, this should be the path that the applications log files are written to. In the secondary container, this can be just about anything, so we put it under /mnt/log to keep it out of the way of the rest of the filesystem. 4. The `FILES_TO_COLLECT` environment variable in the sidecar container, telling it which files to collect logs from. These paths should always be in the mounted volume. diff --git a/contrib/logging/fluentd-sidecar-es/td-agent.conf b/contrib/logging/fluentd-sidecar-es/td-agent.conf index cb9fd14ae13..f532a95d910 100644 --- a/contrib/logging/fluentd-sidecar-es/td-agent.conf +++ b/contrib/logging/fluentd-sidecar-es/td-agent.conf @@ -19,6 +19,10 @@ host elasticsearch-logging.kube-system port 9200 logstash_format true + # Set the chunk limit the same as for fluentd-gcp. + buffer_chunk_limit 512K + # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB + buffer_queue_limit 128 flush_interval 5s # Never wait longer than 5 minutes between retries. max_retry_wait 300 diff --git a/contrib/logging/fluentd-sidecar-gcp/Makefile b/contrib/logging/fluentd-sidecar-gcp/Makefile index e7730a9cb9c..7b70e96e20b 100644 --- a/contrib/logging/fluentd-sidecar-gcp/Makefile +++ b/contrib/logging/fluentd-sidecar-gcp/Makefile @@ -1,6 +1,6 @@ .PHONY: build push -TAG = 1.1 +TAG = 1.2 build: docker build -t gcr.io/google_containers/fluentd-sidecar-gcp:$(TAG) . diff --git a/contrib/logging/fluentd-sidecar-gcp/README.md b/contrib/logging/fluentd-sidecar-gcp/README.md index 0c500a7003c..57b7d66a001 100644 --- a/contrib/logging/fluentd-sidecar-gcp/README.md +++ b/contrib/logging/fluentd-sidecar-gcp/README.md @@ -9,7 +9,7 @@ This shouldn't be necessary if your container writes its logs to stdout or stder In order to make this work, you have to add a few things to your pod config: -1. A second container, using the `gcr.io/google_containers/fluentd-sidecar-gcp:1.1` image to send the logs to Google Cloud Logging. +1. A second container, using the `gcr.io/google_containers/fluentd-sidecar-gcp:1.2` image to send the logs to Google Cloud Logging. 2. A volume for the two containers to share. The emptyDir volume type is a good choice for this because we only want the volume to exist for the lifetime of the pod. 3. Mount paths for the volume in each container. In your primary container, this should be the path that the applications log files are written to. In the secondary container, this can be just about anything, so we put it under /mnt/log to keep it out of the way of the rest of the filesystem. 4. The `FILES_TO_COLLECT` environment variable in the sidecar container, telling it which files to collect logs from. These paths should always be in the mounted volume. diff --git a/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf b/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf index 1bbbfe73a63..ed113c7a1e1 100644 --- a/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf +++ b/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf @@ -18,6 +18,11 @@ # All the auto-generated files should use the tag "file.". type google_cloud + # Set the chunk limit conservatively to avoid exceeding the GCL limit + # of 2MB per write request. + buffer_chunk_limit 512K + # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB + buffer_queue_limit 128 flush_interval 5s # Never wait longer than 5 minutes between retries. max_retry_wait 300