diff --git a/contrib/logging/fluentd-sidecar-es/Makefile b/contrib/logging/fluentd-sidecar-es/Makefile index 29a94eedb99..b289b47fc00 100644 --- a/contrib/logging/fluentd-sidecar-es/Makefile +++ b/contrib/logging/fluentd-sidecar-es/Makefile @@ -1,6 +1,6 @@ .PHONY: build push -TAG = 1.1 +TAG = 1.2 build: docker build -t gcr.io/google_containers/fluentd-sidecar-es:$(TAG) . diff --git a/contrib/logging/fluentd-sidecar-es/README.md b/contrib/logging/fluentd-sidecar-es/README.md index 3802355e0ee..73ad9eb00e4 100644 --- a/contrib/logging/fluentd-sidecar-es/README.md +++ b/contrib/logging/fluentd-sidecar-es/README.md @@ -9,7 +9,7 @@ This shouldn't be necessary if your container writes its logs to stdout or stder In order to make this work, you have to add a few things to your pod config: -1. A second container, using the `gcr.io/google_containers/fluentd-sidecar-es:1.1` image to send the logs to Elasticsearch. +1. A second container, using the `gcr.io/google_containers/fluentd-sidecar-es:1.2` image to send the logs to Elasticsearch. We recommend attaching resource constraints of 100m CPU and 200Mi memory to this container, as in the example. 2. A volume for the two containers to share. The emptyDir volume type is a good choice for this because we only want the volume to exist for the lifetime of the pod. 3. Mount paths for the volume in each container. In your primary container, this should be the path that the applications log files are written to. In the secondary container, this can be just about anything, so we put it under /mnt/log to keep it out of the way of the rest of the filesystem. 4. The `FILES_TO_COLLECT` environment variable in the sidecar container, telling it which files to collect logs from. These paths should always be in the mounted volume. diff --git a/contrib/logging/fluentd-sidecar-es/logging-sidecar-example.yaml b/contrib/logging/fluentd-sidecar-es/logging-sidecar-example.yaml index bf70515e813..b2fa8b7b075 100644 --- a/contrib/logging/fluentd-sidecar-es/logging-sidecar-example.yaml +++ b/contrib/logging/fluentd-sidecar-es/logging-sidecar-example.yaml @@ -14,6 +14,10 @@ spec: mountPath: /var/log - name: sidecar-log-collector image: gcr.io/google_containers/fluentd-sidecar-es:1.1 + resources: + limits: + cpu: 100m + memory: 200Mi env: - name: FILES_TO_COLLECT value: "/mnt/log/synthetic-count.log /mnt/log/synthetic-dates.log" diff --git a/contrib/logging/fluentd-sidecar-es/td-agent.conf b/contrib/logging/fluentd-sidecar-es/td-agent.conf index cb9fd14ae13..f532a95d910 100644 --- a/contrib/logging/fluentd-sidecar-es/td-agent.conf +++ b/contrib/logging/fluentd-sidecar-es/td-agent.conf @@ -19,6 +19,10 @@ host elasticsearch-logging.kube-system port 9200 logstash_format true + # Set the chunk limit the same as for fluentd-gcp. + buffer_chunk_limit 512K + # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB + buffer_queue_limit 128 flush_interval 5s # Never wait longer than 5 minutes between retries. max_retry_wait 300 diff --git a/contrib/logging/fluentd-sidecar-gcp/Makefile b/contrib/logging/fluentd-sidecar-gcp/Makefile index e7730a9cb9c..7b70e96e20b 100644 --- a/contrib/logging/fluentd-sidecar-gcp/Makefile +++ b/contrib/logging/fluentd-sidecar-gcp/Makefile @@ -1,6 +1,6 @@ .PHONY: build push -TAG = 1.1 +TAG = 1.2 build: docker build -t gcr.io/google_containers/fluentd-sidecar-gcp:$(TAG) . diff --git a/contrib/logging/fluentd-sidecar-gcp/README.md b/contrib/logging/fluentd-sidecar-gcp/README.md index 0c500a7003c..88dfbc6f0e0 100644 --- a/contrib/logging/fluentd-sidecar-gcp/README.md +++ b/contrib/logging/fluentd-sidecar-gcp/README.md @@ -9,7 +9,7 @@ This shouldn't be necessary if your container writes its logs to stdout or stder In order to make this work, you have to add a few things to your pod config: -1. A second container, using the `gcr.io/google_containers/fluentd-sidecar-gcp:1.1` image to send the logs to Google Cloud Logging. +1. A second container, using the `gcr.io/google_containers/fluentd-sidecar-gcp:1.2` image to send the logs to Google Cloud Logging. We recommend attaching resource constraints of `100m` CPU and `200Mi` memory to this container, as in the example. 2. A volume for the two containers to share. The emptyDir volume type is a good choice for this because we only want the volume to exist for the lifetime of the pod. 3. Mount paths for the volume in each container. In your primary container, this should be the path that the applications log files are written to. In the secondary container, this can be just about anything, so we put it under /mnt/log to keep it out of the way of the rest of the filesystem. 4. The `FILES_TO_COLLECT` environment variable in the sidecar container, telling it which files to collect logs from. These paths should always be in the mounted volume. diff --git a/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf b/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf index 1bbbfe73a63..ed113c7a1e1 100644 --- a/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf +++ b/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf @@ -18,6 +18,11 @@ # All the auto-generated files should use the tag "file.". type google_cloud + # Set the chunk limit conservatively to avoid exceeding the GCL limit + # of 2MB per write request. + buffer_chunk_limit 512K + # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB + buffer_queue_limit 128 flush_interval 5s # Never wait longer than 5 minutes between retries. max_retry_wait 300 diff --git a/contrib/logging/fluentd-sidecar-gcp/logging-sidecar-example.yaml b/contrib/logging/fluentd-sidecar-gcp/logging-sidecar-example.yaml index 86d486f8d75..d8470236085 100644 --- a/contrib/logging/fluentd-sidecar-gcp/logging-sidecar-example.yaml +++ b/contrib/logging/fluentd-sidecar-gcp/logging-sidecar-example.yaml @@ -14,6 +14,10 @@ spec: mountPath: /var/log - name: sidecar-log-collector image: gcr.io/google_containers/fluentd-sidecar-gcp:1.1 + resources: + limits: + cpu: 100m + memory: 200Mi env: - name: FILES_TO_COLLECT value: "/mnt/log/synthetic-count.log /mnt/log/synthetic-dates.log"