From 31c302994ab4a59e59d4eb7146e2ff83275d6cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Sun, 8 Dec 2019 16:27:02 +0100 Subject: [PATCH] updated fluentd to 1.7.4 + plugin updates and switch to debian buster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: André Bauer --- .../fluentd-elasticsearch/fluentd-es-ds.yaml | 10 +++---- .../fluentd-es-image/Dockerfile | 27 ++++------------- .../fluentd-es-image/Gemfile | 14 ++++----- .../fluentd-es-image/Makefile | 2 +- .../fluentd-es-image/clean-install | 30 ------------------- .../{clean-apt => install.sh} | 22 ++++++++++++-- 6 files changed, 39 insertions(+), 66 deletions(-) delete mode 100755 cluster/addons/fluentd-elasticsearch/fluentd-es-image/clean-install rename cluster/addons/fluentd-elasticsearch/fluentd-es-image/{clean-apt => install.sh} (60%) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index c6bb79136c4..41808cb3932 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -45,22 +45,22 @@ roleRef: apiVersion: apps/v1 kind: DaemonSet metadata: - name: fluentd-es-v2.7.0 + name: fluentd-es-v2.8.0 namespace: kube-system labels: k8s-app: fluentd-es - version: v2.7.0 + version: v2.8.0 addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: fluentd-es - version: v2.7.0 + version: v2.8.0 template: metadata: labels: k8s-app: fluentd-es - version: v2.7.0 + version: v2.8.0 # This annotation ensures that fluentd does not get evicted if the node # supports critical pod annotation based priority scheme. # Note that this does not guarantee admission on the nodes (#40573). @@ -71,7 +71,7 @@ spec: serviceAccountName: fluentd-es containers: - name: fluentd-es - image: quay.io/fluentd_elasticsearch/fluentd:v2.7.0 + image: quay.io/fluentd_elasticsearch/fluentd:v2.8.0 env: - name: FLUENTD_ARGS value: --no-supervisor -q diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile index 7f84870bd5b..905a01d633c 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile @@ -19,31 +19,16 @@ # Note that fluentd is run with root permssion to allow access to # log files with root only access under /var/log/containers/* -FROM debian:stretch-slim +FROM debian:buster-slim ARG DEBIAN_FRONTEND=noninteractive -COPY clean-apt /usr/bin -COPY clean-install /usr/bin +COPY install.sh /tmp/install.sh COPY Gemfile /Gemfile -# 1. Install & configure dependencies. -# 2. Install fluentd via ruby. -# 3. Remove build dependencies. -# 4. Cleanup leftover caches & files. -RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev" \ - && clean-install $BUILD_DEPS \ - ca-certificates \ - libjemalloc1 \ - ruby \ - && echo 'gem: --no-document' >> /etc/gemrc \ - && gem install --file Gemfile \ - && apt-get purge -y --auto-remove \ - -o APT::AutoRemove::RecommendsImportant=false \ - $BUILD_DEPS \ - && clean-apt \ - # Ensure fluent has enough file descriptors - && ulimit -n 65536 +RUN chmod +x /tmp/install.sh && \ + /bin/bash -l -c /tmp/install.sh && \ + rm /tmp/* # Copy the Fluentd configuration file for logging Docker container logs. COPY fluent.conf /etc/fluent/fluent.conf @@ -52,7 +37,7 @@ COPY run.sh /run.sh # Expose prometheus metrics. EXPOSE 80 -ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 +ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 # Start Fluentd to pick up our config that watches Docker container logs. CMD ["/run.sh"] diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index c813ffa2d6f..b9b79ebdee2 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -1,12 +1,12 @@ source 'https://rubygems.org' -gem 'activesupport', '~>5.2.3' -gem 'fluentd', '<=1.6.3' +gem 'activesupport', '~>6.0.1' +gem 'fluentd', '<=1.7.4' gem 'fluent-plugin-concat', '~>2.4.0' -gem 'fluent-plugin-detect-exceptions', '~>0.0.12' -gem 'fluent-plugin-elasticsearch', '~>3.5.3' -gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.2.0' +gem 'fluent-plugin-detect-exceptions', '~>0.0.13' +gem 'fluent-plugin-elasticsearch', '~>3.7.1' +gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.4.0' gem 'fluent-plugin-multi-format-parser', '~>1.0.0' -gem 'fluent-plugin-prometheus', '~>1.4.0' +gem 'fluent-plugin-prometheus', '~>1.7.0' gem 'fluent-plugin-systemd', '~>1.0.2' -gem 'oj', '~>3.8.1' +gem 'oj', '~>3.10.0' diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile index 74d9bcedb56..dc8da73b946 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile @@ -16,7 +16,7 @@ PREFIX = quay.io/fluentd_elasticsearch IMAGE = fluentd -TAG = v2.7.0 +TAG = v2.8.0 build: docker build --tag ${PREFIX}/${IMAGE}:${TAG} . diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/clean-install b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/clean-install deleted file mode 100755 index 4dbc85cfa31..00000000000 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/clean-install +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A script encapsulating a common Dockerimage pattern for installing packages -# and then cleaning up the unnecessary install artifacts. -# e.g. clean-install iptables ebtables conntrack - -set -o errexit - -if [ $# = 0 ]; then - echo >&2 "No packages specified" - exit 1 -fi - -apt-get update -apt-get install -y --no-install-recommends $@ -clean-apt diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/clean-apt b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/install.sh similarity index 60% rename from cluster/addons/fluentd-elasticsearch/fluentd-es-image/clean-apt rename to cluster/addons/fluentd-elasticsearch/fluentd-es-image/install.sh index 4b02a061f96..0d4baa5b20d 100755 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/clean-apt +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/install.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2017 The Kubernetes Authors. # @@ -20,10 +20,28 @@ set -o errexit +# 1. Install & configure dependencies. +# 2. Install fluentd via ruby. +# 3. Remove build dependencies. +# 4. Cleanup leftover caches & files. +BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev" + +# apt install +apt-get update +apt-get install -y --no-install-recommends "${BUILD_DEPS}" ca-certificates libjemalloc2 ruby + +# ruby install +echo 'gem: --no-document' >> /etc/gemrc +gem install --file Gemfile + +# cleanup +apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false "${BUILD_DEPS}" apt-get clean -y rm -rf \ /var/cache/debconf/* \ /var/lib/apt/lists/* \ /var/log/* \ - /tmp/* \ /var/tmp/* + +# Ensure fluent has enough file descriptors +ulimit -n 65536