Merge pull request #39656 from thomasschickinger/master

Automatic merge from submit-queue

Add rule for detecting exceptions to fluentd config for GKE logging (#1)

**What this PR does / why we need it**:
Add the [fluent-detect-exceptions-plugin](https://rubygems.org/gems/fluent-plugin-detect-exceptions) version 0.0.4 to the fluentd config for Kubernetes clusters running on Google Cloud. This plugin detects exception stacks in the stdout/stderr log streams of the containers that run in the cluster and makes sure that they are forwarded as a single log entry to Stackdriver Logging.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
fluentd config for GKE clusters updated: detect exceptions in container log streams and forward them as one log entry.
```

cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml

@@ -2,23 +2,23 @@
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
-  name: fluentd-gcp-v1.33
+  name: fluentd-gcp-v1.34
   namespace: kube-system
   labels:
     k8s-app: fluentd-gcp
     kubernetes.io/cluster-service: "true"
-    version: v1.33
+    version: v1.34
 spec:
   template:
     metadata:
       labels:
         k8s-app: fluentd-gcp
         kubernetes.io/cluster-service: "true"
-        version: v1.33
+        version: v1.34
     spec:
       containers:
       - name: fluentd-gcp
-        image: gcr.io/google_containers/fluentd-gcp:1.33
+        image: gcr.io/google_containers/fluentd-gcp:1.34
         # If fluentd consumes its own logs, the following situation may happen:
         # fluentd fails to send a chunk to the server => writes it to the log =>
         # tries to send this message to the server => fails to send a chunk and so on.

cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile

@@ -36,6 +36,7 @@ RUN apt-get -qq update && \
     td-agent-gem install --no-document fluent-plugin-record-reformer -v 0.8.2 && \
     td-agent-gem install --no-document fluent-plugin-systemd -v 0.0.5 && \
     td-agent-gem install --no-document fluent-plugin-google-cloud -v 0.5.2 && \
+    td-agent-gem install --no-document fluent-plugin-detect-exceptions -v 0.0.4 && \
     # Remove build tools
     apt-get remove -y -qq gcc make && \
     apt-get autoremove -y -qq && \

cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile

@@ -26,7 +26,7 @@
 .PHONY:	build push
 
 PREFIX=gcr.io/google_containers
-TAG = 1.33
+TAG = 1.34
 
 build:
 	docker build --pull -t $(PREFIX)/fluentd-gcp:$(TAG) .

cluster/addons/fluentd-gcp/fluentd-gcp-image/fluent.conf

@@ -70,7 +70,18 @@
 <match reform.**>
   type record_reformer
   enable_ruby true
-  tag kubernetes.${tag_suffix[4].split('-')[0..-2].join('-')}
+  tag raw.kubernetes.${tag_suffix[4].split('-')[0..-2].join('-')}
+</match>
+
+# Detect exceptions in the log output and forward them as one log entry.
+<match raw.kubernetes.**>
+  type detect_exceptions
+  remove_tag_prefix raw
+  message log
+  stream stream
+  multiline_flush_interval 5
+  max_bytes 500000
+  max_lines 1000
 </match>
 
 # Example:

cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml

@@ -10,7 +10,7 @@ spec:
   dnsPolicy: Default
   containers:
   - name: fluentd-cloud-logging
-    image: gcr.io/google_containers/fluentd-gcp:1.33
+    image: gcr.io/google_containers/fluentd-gcp:1.34
     # If fluentd consumes its own logs, the following situation may happen:
     # fluentd fails to send a chunk to the server => writes it to the log =>
     # tries to send this message to the server => fails to send a chunk and so on.