Merge pull request #68012 from wiremind/fluentd-concat

Fluentd: concatenate long logs

cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml

@@ -1,7 +1,7 @@
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: fluentd-es-config-v0.1.5
+  name: fluentd-es-config-v0.1.6
   namespace: kube-system
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
@@ -420,6 +420,14 @@ data:
       @type kubernetes_metadata
     </filter>
 
+    # Concatenate multi-line logs
+    <filter **>
+      @type concat
+      key message
+      multiline_end_regexp /\n$/
+      separator ""
+    </filter>
+
     <match **>
       @id elasticsearch
       @type elasticsearch

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -48,24 +48,24 @@ roleRef:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v2.2.0
+  name: fluentd-es-v2.2.1
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
-    version: v2.2.0
+    version: v2.2.1
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
       k8s-app: fluentd-es
-      version: v2.2.0
+      version: v2.2.1
   template:
     metadata:
       labels:
         k8s-app: fluentd-es
         kubernetes.io/cluster-service: "true"
-        version: v2.2.0
+        version: v2.2.1
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -107,4 +107,4 @@ spec:
           path: /var/lib/docker/containers
       - name: config-volume
         configMap:
-          name: fluentd-es-config-v0.1.5
+          name: fluentd-es-config-v0.1.6

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile

@@ -2,10 +2,11 @@ source 'https://rubygems.org'
 
 gem 'fluentd', '<=1.2.4'
 gem 'activesupport', '~>5.2.1'
-gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0'
-gem 'fluent-plugin-elasticsearch', '~>2.11.5'
-gem 'fluent-plugin-systemd', '~>1.0.1'
+gem 'fluent-plugin-concat', '~>2.3.0'
 gem 'fluent-plugin-detect-exceptions', '~>0.0.11'
-gem 'fluent-plugin-prometheus', '~>1.0.1'
+gem 'fluent-plugin-elasticsearch', '~>2.11.5'
+gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0'
 gem 'fluent-plugin-multi-format-parser', '~>1.0.0'
+gem 'fluent-plugin-prometheus', '~>1.0.1'
+gem 'fluent-plugin-systemd', '~>1.0.1'
 gem 'oj', '~>3.6.5'