From 303c6ce6386ba029cc1692dde727d8a727833e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Bauer?= Date: Sun, 18 Nov 2018 11:37:54 +0100 Subject: [PATCH] - updated fluentd to 1.3.0 - added filter_parse_json_field to restore removed functionality of kubernetes_metadata (https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/issues/145) - updated all gems to newest version --- .../fluentd-elasticsearch/fluentd-es-configmap.yaml | 7 ++++++- .../addons/fluentd-elasticsearch/fluentd-es-ds.yaml | 12 ++++++------ .../fluentd-es-image/Dockerfile | 4 +++- .../fluentd-elasticsearch/fluentd-es-image/Gemfile | 10 +++++----- .../fluentd-elasticsearch/fluentd-es-image/Makefile | 2 +- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml index db3f5af254d..74bb65b5a1b 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml @@ -1,7 +1,7 @@ kind: ConfigMap apiVersion: v1 metadata: - name: fluentd-es-config-v0.1.6 + name: fluentd-es-config-v0.1.7 namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile @@ -420,6 +420,11 @@ data: @type kubernetes_metadata + # Adds the json parsing which was removed in kubernetes_metadata filter > 2.1.x + + @type parse_json_field + + # Concatenate multi-line logs @type concat diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index d804801b669..dbf9b92081d 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -48,24 +48,24 @@ roleRef: apiVersion: apps/v1 kind: DaemonSet metadata: - name: fluentd-es-v2.2.1 + name: fluentd-es-v2.4.0 namespace: kube-system labels: k8s-app: fluentd-es - version: v2.2.1 + version: v2.4.0 kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: fluentd-es - version: v2.2.1 + version: v2.4.0 template: metadata: labels: k8s-app: fluentd-es kubernetes.io/cluster-service: "true" - version: v2.2.1 + version: v2.4.0 # This annotation ensures that fluentd does not get evicted if the node # supports critical pod annotation based priority scheme. # Note that this does not guarantee admission on the nodes (#40573). @@ -77,7 +77,7 @@ spec: serviceAccountName: fluentd-es containers: - name: fluentd-es - image: k8s.gcr.io/fluentd-elasticsearch:v2.2.0 + image: k8s.gcr.io/fluentd-elasticsearch:v2.4.0 env: - name: FLUENTD_ARGS value: --no-supervisor -q @@ -107,4 +107,4 @@ spec: path: /var/lib/docker/containers - name: config-volume configMap: - name: fluentd-es-config-v0.1.6 + name: fluentd-es-config-v0.1.7 diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile index 7f84870bd5b..408e4962824 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Dockerfile @@ -31,13 +31,15 @@ COPY Gemfile /Gemfile # 2. Install fluentd via ruby. # 3. Remove build dependencies. # 4. Cleanup leftover caches & files. -RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev" \ +RUN BUILD_DEPS="make gcc g++ libc6-dev ruby-dev libffi-dev curl" \ && clean-install $BUILD_DEPS \ ca-certificates \ libjemalloc1 \ ruby \ && echo 'gem: --no-document' >> /etc/gemrc \ && gem install --file Gemfile \ + && mkdir -p /etc/fluent/plugin \ + && curl -s https://raw.githubusercontent.com/openshift/origin-aggregated-logging/master/fluentd/lib/filter_parse_json_field/lib/filter_parse_json_field.rb -o /etc/fluent/plugin/filter_parse_json_field.rb \ && apt-get purge -y --auto-remove \ -o APT::AutoRemove::RecommendsImportant=false \ $BUILD_DEPS \ diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index 145b15fc737..1f8ddff200c 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -1,12 +1,12 @@ source 'https://rubygems.org' -gem 'fluentd', '<=1.2.4' gem 'activesupport', '~>5.2.1' +gem 'fluentd', '<=1.3.0' gem 'fluent-plugin-concat', '~>2.3.0' gem 'fluent-plugin-detect-exceptions', '~>0.0.11' -gem 'fluent-plugin-elasticsearch', '~>2.11.5' -gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0' +gem 'fluent-plugin-elasticsearch', '~>2.12.0' +gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.1.4' gem 'fluent-plugin-multi-format-parser', '~>1.0.0' -gem 'fluent-plugin-prometheus', '~>1.0.1' +gem 'fluent-plugin-prometheus', '~>1.2.1' gem 'fluent-plugin-systemd', '~>1.0.1' -gem 'oj', '~>3.6.5' +gem 'oj', '~>3.7.1' diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile index 064c5260dd6..b3b8c4c8961 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile @@ -16,7 +16,7 @@ PREFIX = staging-k8s.gcr.io IMAGE = fluentd-elasticsearch -TAG = v2.3.1 +TAG = v2.4.0 build: docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .