diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index bd703b07174..f83260d2c83 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -45,22 +45,22 @@ roleRef: apiVersion: apps/v1 kind: DaemonSet metadata: - name: fluentd-es-v2.8.0 + name: fluentd-es-v3.0.0 namespace: kube-system labels: k8s-app: fluentd-es - version: v2.8.0 + version: v3.0.0 addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: fluentd-es - version: v2.8.0 + version: v3.0.0 template: metadata: labels: k8s-app: fluentd-es - version: v2.8.0 + version: v3.0.0 # This annotation ensures that fluentd does not get evicted if the node # supports critical pod annotation based priority scheme. # Note that this does not guarantee admission on the nodes (#40573). @@ -71,7 +71,7 @@ spec: serviceAccountName: fluentd-es containers: - name: fluentd-es - image: quay.io/fluentd_elasticsearch/fluentd:v2.8.0 + image: quay.io/fluentd_elasticsearch/fluentd:v3.0.0 env: - name: FLUENTD_ARGS value: --no-supervisor -q diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile index 905a01d633c..45ba669e9ae 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile @@ -19,20 +19,46 @@ # Note that fluentd is run with root permssion to allow access to # log files with root only access under /var/log/containers/* -FROM debian:buster-slim +# 1. Install & configure dependencies. +# 2. Install fluentd via ruby. +# 3. Remove build dependencies. +# 4. Cleanup leftover caches & files. + +FROM ruby:2.7-slim-buster as builder ARG DEBIAN_FRONTEND=noninteractive -COPY install.sh /tmp/install.sh COPY Gemfile /Gemfile -RUN chmod +x /tmp/install.sh && \ - /bin/bash -l -c /tmp/install.sh && \ - rm /tmp/* +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] + +RUN apt-get update && \ + apt-get install -y --no-install-recommends g++ gcc make && \ + echo 'gem: --no-document' >> /etc/gemrc && \ + gem install --file Gemfile + + +FROM ruby:2.7-slim-buster + +ARG DEBIAN_FRONTEND=noninteractive # Copy the Fluentd configuration file for logging Docker container logs. COPY fluent.conf /etc/fluent/fluent.conf -COPY run.sh /run.sh +COPY entrypoint.sh /entrypoint.sh +COPY --from=builder /usr/local/bundle/ /usr/local/bundle + +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] + +RUN apt-get update && \ + apt-get install -y --no-install-recommends libjemalloc2 && \ + apt-get clean -y && \ + ulimit -n 65536 && \ + rm -rf \ + /var/cache/debconf/* \ + /var/lib/apt/lists/* \ + /var/log/* \ + /var/tmp/* \ + rm -rf /tmp/* # Expose prometheus metrics. EXPOSE 80 @@ -40,4 +66,4 @@ EXPOSE 80 ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 # Start Fluentd to pick up our config that watches Docker container logs. -CMD ["/run.sh"] +CMD ["/entrypoint.sh"] diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index d12cfdffbdc..33f567997ad 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -1,12 +1,12 @@ source 'https://rubygems.org' gem 'activesupport', '6.0.2.1' -gem 'fluentd', '1.8.0' +gem 'fluentd', '1.9.2' gem 'fluent-plugin-concat', '2.4.0' gem 'fluent-plugin-detect-exceptions', '0.0.13' -gem 'fluent-plugin-elasticsearch', '3.8.0' -gem 'fluent-plugin-kubernetes_metadata_filter', '2.4.1' +gem 'fluent-plugin-elasticsearch', '4.0.5' +gem 'fluent-plugin-kubernetes_metadata_filter', '2.4.2' gem 'fluent-plugin-multi-format-parser', '1.0.0' -gem 'fluent-plugin-prometheus', '1.7.0' +gem 'fluent-plugin-prometheus', '1.7.3' gem 'fluent-plugin-systemd', '1.0.2' -gem 'oj', '3.10.0' +gem 'oj', '3.10.5' diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile index 897b516c402..aad4b402daa 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile @@ -16,7 +16,7 @@ PREFIX = quay.io/fluentd_elasticsearch IMAGE = fluentd -TAG = v2.9.0 +TAG = v3.0.0 build: docker build --tag ${PREFIX}/${IMAGE}:${TAG} . diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/run.sh b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/entrypoint.sh similarity index 95% rename from cluster/addons/fluentd-elasticsearch/fluentd-es-image/run.sh rename to cluster/addons/fluentd-elasticsearch/fluentd-es-image/entrypoint.sh index 18670a3bc14..d501cb8f9ff 100755 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/run.sh +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/entrypoint.sh @@ -24,4 +24,4 @@ mkdir -p /var/log/journal # A non-quoted string and add the comment to prevent shellcheck failures on this line. # See https://github.com/koalaman/shellcheck/wiki/SC2086 # shellcheck disable=SC2086 -exec /usr/local/bin/fluentd $FLUENTD_ARGS +exec /usr/local/bundle/bin/fluentd $FLUENTD_ARGS diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/fluent.conf b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/fluent.conf index 646d5288c4c..43a70ae0b72 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/fluent.conf +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/fluent.conf @@ -1,8 +1,10 @@ # This is the root config file, which only includes components of the actual configuration # Do not collect fluentd's own logs to avoid infinite loops. - - @type null - + @include /etc/fluent/config.d/*.conf diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/install.sh b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/install.sh deleted file mode 100755 index 3a4bed796a7..00000000000 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/install.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A script encapsulating a common Dockerimage pattern for installing packages -# and then cleaning up the unnecessary install artifacts. -# e.g. clean-install iptables ebtables conntrack - -set -o errexit - -# 1. Install & configure dependencies. -# 2. Install fluentd via ruby. -# 3. Remove build dependencies. -# 4. Cleanup leftover caches & files. -BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev" - -# apt install -apt-get update -echo "${BUILD_DEPS} ca-certificates libjemalloc2 ruby" | xargs apt-get install -y --no-install-recommends - -# ruby install -echo 'gem: --no-document' >> /etc/gemrc -gem install --file Gemfile - -# cleanup -echo "${BUILD_DEPS}" | xargs apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false -apt-get clean -y -rm -rf \ - /var/cache/debconf/* \ - /var/lib/apt/lists/* \ - /var/log/* \ - /var/tmp/* - -# Ensure fluent has enough file descriptors -ulimit -n 65536