Merge pull request #54777 from Random-Liu/add-cri-log-support

Automatic merge from submit-queue (batch tested with PRs 54533, 54777, 54763, 54806, 54703). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add CRI log format support in fluentd.

Without this fluentd will log a lot of errors for each line, because it doesn't recognize the CRI log format.
With this PR, it could support CRI log format now. I've tried with cri-containerd.

The PR is using https://github.com/repeatedly/fluent-plugin-multi-format-parser.
This PR depends on https://github.com/GoogleCloudPlatform/k8s-stackdriver/pull/62.

@crassirostris Should I build/push image before the PR is merged? What is our process?

**Release note**:

```release-note
fluentd now supports CRI log format.
```

cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml

@@ -96,16 +96,27 @@ data:
     # the name of the Kubernetes container regardless of how many times the
     # Kubernetes pod has been restarted (resulting in a several Docker container IDs).
 
-    # Example:
+    # Json Log Example:
     # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
+    # CRI Log Example:
+    # 2016-02-17T00:04:05.931087621Z stdout [info:2016-02-16T16:04:05.930-08:00] Some log text here
     <source>
       type tail
       path /var/log/containers/*.log
       pos_file /var/log/es-containers.log.pos
       time_format %Y-%m-%dT%H:%M:%S.%NZ
       tag kubernetes.*
-      format json
       read_from_head true
+      format multi_format
+      <pattern>
+        format json
+        time_key time
+        time_format %Y-%m-%dT%H:%M:%S.%NZ
+      </pattern>
+      <pattern>
+        format /^(?<time>.+)\b(?<stream>stdout|stderr)\b(?<log>.*)$/
+        time_format %Y-%m-%dT%H:%M:%S.%N%:z
+      </pattern>
     </source>
   system.input.conf: |-
     # Example:

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -48,11 +48,11 @@ roleRef:
 apiVersion: apps/v1beta2
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v2.0.1
+  name: fluentd-es-v2.0.2
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
-    version: v2.0.1
+    version: v2.0.2
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
@@ -61,7 +61,7 @@ spec:
       labels:
         k8s-app: fluentd-es
         kubernetes.io/cluster-service: "true"
-        version: v2.0.1
+        version: v2.0.2
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -71,7 +71,7 @@ spec:
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es
-        image: gcr.io/google-containers/fluentd-elasticsearch:v2.0.1
+        image: gcr.io/google-containers/fluentd-elasticsearch:v2.0.2
         env:
         - name: FLUENTD_ARGS
           value: --no-supervisor -q

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile

@@ -6,4 +6,5 @@ gem 'fluent-plugin-kubernetes_metadata_filter', '~>0.27.0'
 gem 'fluent-plugin-elasticsearch', '~>1.9.5'
 gem 'fluent-plugin-systemd', '~>0.0.8'
 gem 'fluent-plugin-prometheus', '~>0.3.0'
+gem 'fluent-plugin-multi-format-parser', '~>0.1.1'
 gem 'oj', '~>2.18.1'

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = gcr.io/google-containers
 IMAGE = fluentd-elasticsearch
-TAG = v2.0.1
+TAG = v2.0.2
 
 build:
 	docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .

cluster/addons/fluentd-gcp/fluentd-gcp-configmap.yaml

@@ -41,17 +41,26 @@ data:
     # Tag is then parsed by google_cloud plugin and translated to the metadata,
     # visible in the log viewer
 
-    # Example:
+    # Json Log Example:
     # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
+    # CRI Log Example:
+    # 2016-02-17T00:04:05.931087621Z stdout [info:2016-02-16T16:04:05.930-08:00] Some log text here
     <source>
       type tail
-      format json
-      time_key time
       path /var/log/containers/*.log
       pos_file /var/log/gcp-containers.log.pos
-      time_format %Y-%m-%dT%H:%M:%S.%N%Z
       tag reform.*
       read_from_head true
+      format multi_format
+      <pattern>
+        format json
+        time_key time
+        time_format %Y-%m-%dT%H:%M:%S.%NZ
+      </pattern>
+      <pattern>
+        format /^(?<time>.+)\b(?<stream>stdout|stderr)\b(?<log>.*)$/
+        time_format %Y-%m-%dT%H:%M:%S.%N%:z
+      </pattern>
     </source>
 
     <filter reform.**>

cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml

@@ -10,13 +10,13 @@ metadata:
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
-  name: fluentd-gcp-v2.0.9
+  name: fluentd-gcp-v2.0.10
   namespace: kube-system
   labels:
     k8s-app: fluentd-gcp
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-    version: v2.0.9
+    version: v2.0.10
 spec:
   updateStrategy:
     type: RollingUpdate
@@ -25,7 +25,7 @@ spec:
       labels:
         k8s-app: fluentd-gcp
         kubernetes.io/cluster-service: "true"
-        version: v2.0.9
+        version: v2.0.10
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -36,7 +36,7 @@ spec:
       dnsPolicy: Default
       containers:
       - name: fluentd-gcp
-        image: gcr.io/google-containers/fluentd-gcp:2.0.9
+        image: gcr.io/google-containers/fluentd-gcp:2.0.10
         env:
         - name: FLUENTD_ARGS
           value: --no-supervisor -q