Merge pull request #58063 from monotek/detect-exceptions

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

added fluentd detect-exceptions plugin to fluentd-es-image for multiline logging

**What this PR does / why we need it**:
This PR adds proper multiline logging (java stacktraces for example) to fluentd-es docker image by adding the fluentd detect-exceptions plugin.

Plugin: https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions

Its also used in GCP and it also works outside of Kubernetes, so it should be safe to use it here too.


**Release note**:
```release-note
fluentd-es addon: multiline stacktraces are now grouped into one entry automatically
```

cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml

@@ -105,7 +105,7 @@ data:
       path /var/log/containers/*.log
       pos_file /var/log/es-containers.log.pos
       time_format %Y-%m-%dT%H:%M:%S.%NZ
-      tag kubernetes.*
+      tag raw.kubernetes.*
       read_from_head true
       format multi_format
       <pattern>
@@ -118,6 +118,16 @@ data:
         time_format %Y-%m-%dT%H:%M:%S.%N%:z
       </pattern>
     </source>
+    # Detect exceptions in the log output and forward them as one log entry.
+    <match raw.kubernetes.**>
+      @type detect_exceptions
+      remove_tag_prefix raw
+      message log
+      stream stream
+      multiline_flush_interval 5
+      max_bytes 500000
+      max_lines 1000
+    </match>
   system.input.conf: |-
     # Example:
     # 2015-12-21 23:17:22,066 [salt.state       ][INFO    ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081
@@ -367,7 +377,7 @@ data:
        num_threads 2
     </match>
 metadata:
-  name: fluentd-es-config-v0.1.1
+  name: fluentd-es-config-v0.1.2
   namespace: kube-system
   labels:
     addonmanager.kubernetes.io/mode: Reconcile

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -48,24 +48,24 @@ roleRef:
 apiVersion: apps/v1beta2
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v2.0.2
+  name: fluentd-es-v2.0.3
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
-    version: v2.0.2
+    version: v2.0.3
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
       k8s-app: fluentd-es
-      version: v2.0.2
+      version: v2.0.3
   template:
     metadata:
       labels:
         k8s-app: fluentd-es
         kubernetes.io/cluster-service: "true"
-        version: v2.0.2
+        version: v2.0.3
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -75,7 +75,7 @@ spec:
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es
-        image: gcr.io/google-containers/fluentd-elasticsearch:v2.0.2
+        image: gcr.io/google-containers/fluentd-elasticsearch:v2.0.3
         env:
         - name: FLUENTD_ARGS
           value: --no-supervisor -q
@@ -112,4 +112,4 @@ spec:
           path: /usr/lib64
       - name: config-volume
         configMap:
-          name: fluentd-es-config-v0.1.1
+          name: fluentd-es-config-v0.1.2

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile

@@ -5,6 +5,7 @@ gem 'activesupport', '~>4.2.6'
 gem 'fluent-plugin-kubernetes_metadata_filter', '~>0.27.0'
 gem 'fluent-plugin-elasticsearch', '~>1.9.5'
 gem 'fluent-plugin-systemd', '~>0.0.8'
+gem 'fluent-plugin-detect-exceptions', '~>0.0.8'
 gem 'fluent-plugin-prometheus', '~>0.3.0'
 gem 'fluent-plugin-multi-format-parser', '~>0.1.1'
 gem 'oj', '~>2.18.1'

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = gcr.io/google-containers
 IMAGE = fluentd-elasticsearch
-TAG = v2.0.2
+TAG = v2.0.3
 
 build:
 	docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .