- updated fluentd to 1.3.0

- added filter_parse_json_field to restore removed functionality of kubernetes_metadata (https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/145)
- updated all gems to newest version

cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml

@@ -1,7 +1,7 @@
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: fluentd-es-config-v0.1.6
+  name: fluentd-es-config-v0.1.7
   namespace: kube-system
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
@@ -420,6 +420,11 @@ data:
       @type kubernetes_metadata
     </filter>
 
+    # Adds the json parsing which was removed in kubernetes_metadata filter > 2.1.x
+    <filter kubernetes.**>
+      @type parse_json_field
+    </filter>
+
     # Concatenate multi-line logs
     <filter **>
       @type concat

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -48,24 +48,24 @@ roleRef:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v2.2.1
+  name: fluentd-es-v2.4.0
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
-    version: v2.2.1
+    version: v2.4.0
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
       k8s-app: fluentd-es
-      version: v2.2.1
+      version: v2.4.0
   template:
     metadata:
       labels:
         k8s-app: fluentd-es
         kubernetes.io/cluster-service: "true"
-        version: v2.2.1
+        version: v2.4.0
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -77,7 +77,7 @@ spec:
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es
-        image: k8s.gcr.io/fluentd-elasticsearch:v2.2.0
+        image: k8s.gcr.io/fluentd-elasticsearch:v2.4.0
         env:
         - name: FLUENTD_ARGS
           value: --no-supervisor -q
@@ -107,4 +107,4 @@ spec:
           path: /var/lib/docker/containers
       - name: config-volume
         configMap:
-          name: fluentd-es-config-v0.1.6
+          name: fluentd-es-config-v0.1.7

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile

@@ -31,13 +31,15 @@ COPY Gemfile /Gemfile
 # 2. Install fluentd via ruby.
 # 3. Remove build dependencies.
 # 4. Cleanup leftover caches & files.
-RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev" \
+RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev curl" \
     && clean-install $BUILD_DEPS \
                      ca-certificates \
                      libjemalloc1 \
                      ruby \
     && echo 'gem: --no-document' >> /etc/gemrc \
     && gem install --file Gemfile \
+    && mkdir -p /etc/fluent/plugin \
+    && curl -s https://raw.githubusercontent.com/openshift/origin-aggregated-logging/master/fluentd/lib/filter_parse_json_field/lib/filter_parse_json_field.rb -o /etc/fluent/plugin/filter_parse_json_field.rb \
     && apt-get purge -y --auto-remove \
                      -o APT::AutoRemove::RecommendsImportant=false \
                      $BUILD_DEPS \

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile

@@ -1,12 +1,12 @@
 source 'https://rubygems.org'
 
-gem 'fluentd', '<=1.2.4'
 gem 'activesupport', '~>5.2.1'
+gem 'fluentd', '<=1.3.0'
 gem 'fluent-plugin-concat', '~>2.3.0'
 gem 'fluent-plugin-detect-exceptions', '~>0.0.11'
-gem 'fluent-plugin-elasticsearch', '~>2.11.5'
-gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0'
+gem 'fluent-plugin-elasticsearch', '~>2.12.0'
+gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.1.4'
 gem 'fluent-plugin-multi-format-parser', '~>1.0.0'
-gem 'fluent-plugin-prometheus', '~>1.0.1'
+gem 'fluent-plugin-prometheus', '~>1.2.1'
 gem 'fluent-plugin-systemd', '~>1.0.1'
-gem 'oj', '~>3.6.5'
+gem 'oj', '~>3.7.1'

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = staging-k8s.gcr.io
 IMAGE = fluentd-elasticsearch
-TAG = v2.3.1
+TAG = v2.4.0
 
 build:
 	docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .