updated fluentd to 1.9.2 & use docker builder pattern in dockerfile

Signed-off-by: André Bauer <monotek23@gmail.com>

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -45,22 +45,22 @@ roleRef:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v2.8.0
+  name: fluentd-es-v3.0.0
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
-    version: v2.8.0
+    version: v3.0.0
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
       k8s-app: fluentd-es
-      version: v2.8.0
+      version: v3.0.0
   template:
     metadata:
       labels:
         k8s-app: fluentd-es
-        version: v2.8.0
+        version: v3.0.0
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -71,7 +71,7 @@ spec:
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es
-        image: quay.io/fluentd_elasticsearch/fluentd:v2.8.0
+        image: quay.io/fluentd_elasticsearch/fluentd:v3.0.0
         env:
         - name: FLUENTD_ARGS
           value: --no-supervisor -q

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile

@@ -19,20 +19,46 @@
 # Note that fluentd is run with root permssion to allow access to
 # log files with root only access under /var/log/containers/*
 
-FROM debian:buster-slim
+# 1. Install & configure dependencies.
+# 2. Install fluentd via ruby.
+# 3. Remove build dependencies.
+# 4. Cleanup leftover caches & files.
+
+FROM ruby:2.7-slim-buster as builder
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-COPY install.sh /tmp/install.sh
 COPY Gemfile /Gemfile
 
-RUN chmod +x /tmp/install.sh && \
+SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
-    /bin/bash -l -c /tmp/install.sh && \
+
-    rm /tmp/*
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends g++ gcc make && \
+    echo 'gem: --no-document' >> /etc/gemrc && \
+    gem install --file Gemfile
+
+
+FROM ruby:2.7-slim-buster
+
+ARG DEBIAN_FRONTEND=noninteractive
 
 # Copy the Fluentd configuration file for logging Docker container logs.
 COPY fluent.conf /etc/fluent/fluent.conf
-COPY run.sh /run.sh
+COPY entrypoint.sh /entrypoint.sh
+COPY --from=builder /usr/local/bundle/ /usr/local/bundle 
+
+SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends libjemalloc2 && \
+    apt-get clean -y && \
+    ulimit -n 65536 && \    
+    rm -rf \
+        /var/cache/debconf/* \
+        /var/lib/apt/lists/* \
+        /var/log/* \
+        /var/tmp/* \
+        rm -rf /tmp/*
 
 # Expose prometheus metrics.
 EXPOSE 80
@@ -40,4 +66,4 @@ EXPOSE 80
 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
 
 # Start Fluentd to pick up our config that watches Docker container logs.
-CMD ["/run.sh"]
+CMD ["/entrypoint.sh"]

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile

@@ -1,12 +1,12 @@
 source 'https://rubygems.org'
 
 gem 'activesupport', '6.0.2.1'
-gem 'fluentd', '1.8.0'
+gem 'fluentd', '1.9.2'
 gem 'fluent-plugin-concat', '2.4.0'
 gem 'fluent-plugin-detect-exceptions', '0.0.13'
-gem 'fluent-plugin-elasticsearch', '3.8.0'
+gem 'fluent-plugin-elasticsearch', '4.0.5'
-gem 'fluent-plugin-kubernetes_metadata_filter', '2.4.1'
+gem 'fluent-plugin-kubernetes_metadata_filter', '2.4.2'
 gem 'fluent-plugin-multi-format-parser', '1.0.0'
-gem 'fluent-plugin-prometheus', '1.7.0'
+gem 'fluent-plugin-prometheus', '1.7.3'
 gem 'fluent-plugin-systemd', '1.0.2'
-gem 'oj', '3.10.0'
+gem 'oj', '3.10.5'

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = quay.io/fluentd_elasticsearch
 IMAGE = fluentd
-TAG = v2.9.0
+TAG = v3.0.0
 
 build:
 	docker build --tag ${PREFIX}/${IMAGE}:${TAG} .

cluster/addons/fluentd-elasticsearch/fluentd-es-image/entrypoint.sh

@@ -24,4 +24,4 @@ mkdir -p /var/log/journal
 # A non-quoted string and add the comment to prevent shellcheck failures on this line.
 # See https://github.com/koalaman/shellcheck/wiki/SC2086
 # shellcheck disable=SC2086
-exec /usr/local/bin/fluentd $FLUENTD_ARGS
+exec /usr/local/bundle/bin/fluentd $FLUENTD_ARGS

cluster/addons/fluentd-elasticsearch/fluentd-es-image/fluent.conf

@@ -1,8 +1,10 @@
 # This is the root config file, which only includes components of the actual configuration
 
 # Do not collect fluentd's own logs to avoid infinite loops.
-<match fluent.**>
+<label @FLUENT_LOG>
-  @type null
+  <match fluent.*>
-</match>
+    @type stdout
+  </match>
+</label>
 
 @include /etc/fluent/config.d/*.conf

cluster/addons/fluentd-elasticsearch/fluentd-es-image/install.sh

@@ -1,47 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A script encapsulating a common Dockerimage pattern for installing packages
-# and then cleaning up the unnecessary install artifacts.
-# e.g. clean-install iptables ebtables conntrack
-
-set -o errexit
-
-# 1. Install & configure dependencies.
-# 2. Install fluentd via ruby.
-# 3. Remove build dependencies.
-# 4. Cleanup leftover caches & files.
-BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev"
-
-# apt install
-apt-get update
-echo "${BUILD_DEPS} ca-certificates libjemalloc2 ruby" | xargs apt-get install -y --no-install-recommends
-
-# ruby install
-echo 'gem: --no-document' >> /etc/gemrc 
-gem install --file Gemfile
-
-# cleanup
-echo "${BUILD_DEPS}" | xargs apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false
-apt-get clean -y
-rm -rf \
-   /var/cache/debconf/* \
-   /var/lib/apt/lists/* \
-   /var/log/* \
-   /var/tmp/*
-
-# Ensure fluent has enough file descriptors
-ulimit -n 65536