From 079136fd5964502e6d3b43a6bdbabe24d078767d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20de=20Saint=20Martin?= Date: Mon, 27 Aug 2018 16:09:03 +0200 Subject: [PATCH 1/3] fluent-es-image: add fluentd-plugin-concat to gemfile. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédric de Saint Martin --- cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index 52b668b6ac4..dd617eafc3f 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -8,4 +8,5 @@ gem 'fluent-plugin-systemd', '~>1.0.1' gem 'fluent-plugin-detect-exceptions', '~>0.0.11' gem 'fluent-plugin-prometheus', '~>1.0.1' gem 'fluent-plugin-multi-format-parser', '~>1.0.0' +gem 'fluent-plugin-concat', '~>2.3.0' gem 'oj', '~>3.6.5' From 46c5c8e20a3fe903e8a2a4723ac5ed6b9bbbc1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20de=20Saint=20Martin?= Date: Mon, 27 Aug 2018 16:09:33 +0200 Subject: [PATCH 2/3] fluent-es-image: sort plugins in Gemfile. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédric de Saint Martin --- .../fluentd-elasticsearch/fluentd-es-image/Gemfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index dd617eafc3f..145b15fc737 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -2,11 +2,11 @@ source 'https://rubygems.org' gem 'fluentd', '<=1.2.4' gem 'activesupport', '~>5.2.1' -gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0' -gem 'fluent-plugin-elasticsearch', '~>2.11.5' -gem 'fluent-plugin-systemd', '~>1.0.1' -gem 'fluent-plugin-detect-exceptions', '~>0.0.11' -gem 'fluent-plugin-prometheus', '~>1.0.1' -gem 'fluent-plugin-multi-format-parser', '~>1.0.0' gem 'fluent-plugin-concat', '~>2.3.0' +gem 'fluent-plugin-detect-exceptions', '~>0.0.11' +gem 'fluent-plugin-elasticsearch', '~>2.11.5' +gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0' +gem 'fluent-plugin-multi-format-parser', '~>1.0.0' +gem 'fluent-plugin-prometheus', '~>1.0.1' +gem 'fluent-plugin-systemd', '~>1.0.1' gem 'oj', '~>3.6.5' From def00298c3c47b514895d3978f47f2fdc2733a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20de=20Saint=20Martin?= Date: Wed, 29 Aug 2018 15:50:38 +0200 Subject: [PATCH 3/3] Fluentd: concatenate long logs (>16KB) which have been splitted by Docker into several lines. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/kubernetes/kubernetes/issues/52444. Signed-off-by: Cédric de Saint Martin --- .../fluentd-elasticsearch/fluentd-es-configmap.yaml | 10 +++++++++- .../addons/fluentd-elasticsearch/fluentd-es-ds.yaml | 10 +++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml index 369a468ec30..db3f5af254d 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-configmap.yaml @@ -1,7 +1,7 @@ kind: ConfigMap apiVersion: v1 metadata: - name: fluentd-es-config-v0.1.5 + name: fluentd-es-config-v0.1.6 namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile @@ -420,6 +420,14 @@ data: @type kubernetes_metadata + # Concatenate multi-line logs + + @type concat + key message + multiline_end_regexp /\n$/ + separator "" + + @id elasticsearch @type elasticsearch diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index 4d00803b002..d804801b669 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -48,24 +48,24 @@ roleRef: apiVersion: apps/v1 kind: DaemonSet metadata: - name: fluentd-es-v2.2.0 + name: fluentd-es-v2.2.1 namespace: kube-system labels: k8s-app: fluentd-es - version: v2.2.0 + version: v2.2.1 kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: fluentd-es - version: v2.2.0 + version: v2.2.1 template: metadata: labels: k8s-app: fluentd-es kubernetes.io/cluster-service: "true" - version: v2.2.0 + version: v2.2.1 # This annotation ensures that fluentd does not get evicted if the node # supports critical pod annotation based priority scheme. # Note that this does not guarantee admission on the nodes (#40573). @@ -107,4 +107,4 @@ spec: path: /var/lib/docker/containers - name: config-volume configMap: - name: fluentd-es-config-v0.1.5 + name: fluentd-es-config-v0.1.6