Merge pull request #86039 from monotek/fluentd-elasticsearch

updated fluentd to 1.7.4 + plugin updates and switch to debian buster

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -45,22 +45,22 @@ roleRef:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v2.7.0
+  name: fluentd-es-v2.8.0
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
-    version: v2.7.0
+    version: v2.8.0
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
       k8s-app: fluentd-es
-      version: v2.7.0
+      version: v2.8.0
   template:
     metadata:
       labels:
         k8s-app: fluentd-es
-        version: v2.7.0
+        version: v2.8.0
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -71,7 +71,7 @@ spec:
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es
-        image: quay.io/fluentd_elasticsearch/fluentd:v2.7.0
+        image: quay.io/fluentd_elasticsearch/fluentd:v2.8.0
         env:
         - name: FLUENTD_ARGS
           value: --no-supervisor -q

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile

@@ -19,31 +19,16 @@
 # Note that fluentd is run with root permssion to allow access to
 # log files with root only access under /var/log/containers/*
 
-FROM debian:stretch-slim
+FROM debian:buster-slim
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-COPY clean-apt /usr/bin
-COPY clean-install /usr/bin
+COPY install.sh /tmp/install.sh
 COPY Gemfile /Gemfile
 
-# 1. Install & configure dependencies.
-# 2. Install fluentd via ruby.
-# 3. Remove build dependencies.
-# 4. Cleanup leftover caches & files.
-RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev" \
-    && clean-install $BUILD_DEPS \
-                     ca-certificates \
-                     libjemalloc1 \
-                     ruby \
-    && echo 'gem: --no-document' >> /etc/gemrc \
-    && gem install --file Gemfile \
-    && apt-get purge -y --auto-remove \
-                     -o APT::AutoRemove::RecommendsImportant=false \
-                     $BUILD_DEPS \
-    && clean-apt \
-    # Ensure fluent has enough file descriptors
-    && ulimit -n 65536
+RUN chmod +x /tmp/install.sh && \
+    /bin/bash -l -c /tmp/install.sh && \
+    rm /tmp/*
 
 # Copy the Fluentd configuration file for logging Docker container logs.
 COPY fluent.conf /etc/fluent/fluent.conf
@@ -52,7 +37,7 @@ COPY run.sh /run.sh
 # Expose prometheus metrics.
 EXPOSE 80
 
-ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
+ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2
 
 # Start Fluentd to pick up our config that watches Docker container logs.
 CMD ["/run.sh"]

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile

@@ -1,12 +1,12 @@
 source 'https://rubygems.org'
 
-gem 'activesupport', '~>5.2.3'
-gem 'fluentd', '<=1.6.3'
+gem 'activesupport', '~>6.0.1'
+gem 'fluentd', '<=1.7.4'
 gem 'fluent-plugin-concat', '~>2.4.0'
-gem 'fluent-plugin-detect-exceptions', '~>0.0.12'
-gem 'fluent-plugin-elasticsearch', '~>3.5.3'
-gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.2.0'
+gem 'fluent-plugin-detect-exceptions', '~>0.0.13'
+gem 'fluent-plugin-elasticsearch', '~>3.7.1'
+gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.4.0'
 gem 'fluent-plugin-multi-format-parser', '~>1.0.0'
-gem 'fluent-plugin-prometheus', '~>1.4.0'
+gem 'fluent-plugin-prometheus', '~>1.7.0'
 gem 'fluent-plugin-systemd', '~>1.0.2'
-gem 'oj', '~>3.8.1'
+gem 'oj', '~>3.10.0'

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = quay.io/fluentd_elasticsearch
 IMAGE = fluentd
-TAG = v2.7.0
+TAG = v2.8.0
 
 build:
 	docker build --tag ${PREFIX}/${IMAGE}:${TAG} .

cluster/addons/fluentd-elasticsearch/fluentd-es-image/clean-install

@@ -1,30 +0,0 @@
-#!/bin/sh
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A script encapsulating a common Dockerimage pattern for installing packages
-# and then cleaning up the unnecessary install artifacts.
-# e.g. clean-install iptables ebtables conntrack
-
-set -o errexit
-
-if [ $# = 0 ]; then
-  echo >&2 "No packages specified"
-  exit 1
-fi
-
-apt-get update
-apt-get install -y --no-install-recommends $@
-clean-apt

cluster/addons/fluentd-elasticsearch/fluentd-es-image/install.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Copyright 2017 The Kubernetes Authors.
 #
@@ -20,10 +20,28 @@
 
 set -o errexit
 
+# 1. Install & configure dependencies.
+# 2. Install fluentd via ruby.
+# 3. Remove build dependencies.
+# 4. Cleanup leftover caches & files.
+BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev"
+
+# apt install
+apt-get update
+apt-get install -y --no-install-recommends "${BUILD_DEPS}" ca-certificates libjemalloc2 ruby
+
+# ruby install
+echo 'gem: --no-document' >> /etc/gemrc 
+gem install --file Gemfile
+
+# cleanup
+apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false "${BUILD_DEPS}"
 apt-get clean -y
 rm -rf \
    /var/cache/debconf/* \
    /var/lib/apt/lists/* \
    /var/log/* \
-   /tmp/* \
    /var/tmp/*
+
+# Ensure fluent has enough file descriptors
+ulimit -n 65536