Merge pull request #48566 from vasartori/fluentd-prometheus

Automatic merge from submit-queue (batch tested with PRs 48279, 48566, 48319, 48794, 47952)

Add prometheus plugin on fluentd image.

**What this PR does / why we need it**:
This PR adds the prometheus plugin on Fluentd.

**Special notes for your reviewer**:
The plugin used was: https://github.com/kazegusuri/fluent-plugin-prometheus, on the latest stable version.
All configs used are default.

**Release note**:

```release-note
Fluentd-es addon now exposes a /metrics endpoint for monitoring on port 24231. 
```

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -1,20 +1,20 @@
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v1.22
+  name: fluentd-es-v1.24
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
-    version: v1.22
+    version: v1.24
 spec:
   template:
     metadata:
       labels:
         k8s-app: fluentd-es
         kubernetes.io/cluster-service: "true"
-        version: v1.22
+        version: v1.24
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -24,7 +24,7 @@ spec:
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es
-        image: gcr.io/google_containers/fluentd-elasticsearch:1.23
+        image: gcr.io/google_containers/fluentd-elasticsearch:1.24
         command:
           - '/bin/sh'
           - '-c'

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = gcr.io/google_containers
 IMAGE = fluentd-elasticsearch
-TAG = 1.23
+TAG = 1.24
 
 build:
 	docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .

cluster/addons/fluentd-elasticsearch/fluentd-es-image/build.sh

@@ -32,6 +32,7 @@ sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/in
 # http://docs.fluentd.org/articles/plugin-management
 td-agent-gem install --no-document fluent-plugin-kubernetes_metadata_filter -v 0.27.0
 td-agent-gem install --no-document fluent-plugin-elasticsearch -v 1.9.5
+td-agent-gem install --no-document fluent-plugin-prometheus -v 0.3.0
 
 # Remove docs and postgres references
 rm -rf /opt/td-agent/embedded/share/doc \

cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf

@@ -283,6 +283,44 @@
   type kubernetes_metadata
 </filter>
 
+# Prometheus Exporter Plugin
+# input plugin that exports metrics
+<source>
+  type prometheus
+</source>
+
+<source>
+  type monitor_agent
+</source>
+
+<source>
+  type forward
+</source>
+
+# input plugin that collects metrics from MonitorAgent
+<source>
+  @type prometheus_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+# input plugin that collects metrics for output plugin
+<source>
+  @type prometheus_output_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+# input plugin that collects metrics for in_tail plugin
+<source>
+  @type prometheus_tail_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
 <match **>
    type elasticsearch
    log_level info