From bb51c94172c168151167c71d45b35209f1541300 Mon Sep 17 00:00:00 2001
From: Alex Robinson <arob@google.com>
Date: Fri, 3 Jun 2016 11:42:40 -0400
Subject: [PATCH] Add collection of the new glbc and cluster-autoscaler logs

---
 .../fluentd-es-image/Makefile                 |  2 +-
 .../fluentd-es-image/td-agent.conf            | 30 +++++++++++++++++-
 .../fluentd-gcp/fluentd-gcp-image/Makefile    |  2 +-
 .../fluentd-gcp-image/google-fluentd.conf     | 31 +++++++++++++++++--
 .../saltbase/salt/fluentd-es/fluentd-es.yaml  |  2 +-
 .../salt/fluentd-gcp/fluentd-gcp.yaml         |  2 +-
 6 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile
index 0dbf820d976..9b2a3fc4914 100644
--- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile
+++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile
@@ -15,7 +15,7 @@
 .PHONY:	build push
 
 IMAGE = fluentd-elasticsearch
-TAG = 1.15
+TAG = 1.17
 
 build:	
 	docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) .
diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf
index 269f0abeb75..0b5ed420ce6 100644
--- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf
+++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf
@@ -124,7 +124,7 @@
   format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
   time_format %Y-%m-%d %H:%M:%S
   path /var/log/salt/minion
-  pos_file /var/log/gcp-salt.pos
+  pos_file /var/log/es-salt.pos
   tag salt
 </source>
 
@@ -222,6 +222,34 @@
   type kubernetes_metadata
 </filter>
 
+# Example:
+# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
+<source>
+  type tail
+  format multiline
+  multiline_flush_interval 5s
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+  time_format %m%d %H:%M:%S.%N
+  path /var/log/glbc.log
+  pos_file /var/log/es-glbc.log.pos
+  tag glbc
+</source>
+
+# Example:
+# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
+<source>
+  type tail
+  format multiline
+  multiline_flush_interval 5s
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+  time_format %m%d %H:%M:%S.%N
+  path /var/log/cluster-autoscaler.log
+  pos_file /var/log/es-cluster-autoscaler.log.pos
+  tag cluster-autoscaler
+</source>
+
 <match **>
    type elasticsearch
    log_level info
diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile
index b0ee2667547..95b77f33c62 100644
--- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile
+++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile
@@ -28,7 +28,7 @@
 
 .PHONY:	kbuild kpush
 
-TAG = 1.18
+TAG = 1.20
 
 # Rules for building the test image for deployment to Dockerhub with user kubernetes.
 
diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
index 20ecb844b6c..eaf6e2b6b69 100644
--- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
+++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
@@ -13,9 +13,6 @@
 #  "stream":"stderr",
 #   "time":"2014-09-25T21:15:03.499185026Z"}
 #
-# Currently, the log information is ingested as plain text rather than JSON.
-# TODO: When Cloud Logging supports it, ingest as JSON.
-#
 # The record reformer is used to write the tag to focus on the pod name
 # and the Kubernetes container name. For example a Docker container's logs
 # might be in the directory:
@@ -171,6 +168,34 @@
   tag kube-scheduler
 </source>
 
+# Example:
+# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
+<source>
+  type tail
+  format multiline
+  multiline_flush_interval 5s
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+  time_format %m%d %H:%M:%S.%N
+  path /var/log/glbc.log
+  pos_file /var/log/gcp-glbc.log.pos
+  tag glbc
+</source>
+
+# Example:
+# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
+<source>
+  type tail
+  format multiline
+  multiline_flush_interval 5s
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+  time_format %m%d %H:%M:%S.%N
+  path /var/log/cluster-autoscaler.log
+  pos_file /var/log/gcp-cluster-autoscaler.log.pos
+  tag cluster-autoscaler
+</source>
+
 # We use 2 output stanzas - one to handle the container logs and one to handle
 # the node daemon logs, the latter of which explicitly sends its logs to the
 # compute.googleapis.com service rather than container.googleapis.com to keep
diff --git a/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml b/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml
index aea68123d45..5f72b7bad23 100644
--- a/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml
+++ b/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml
@@ -8,7 +8,7 @@ metadata:
 spec:
   containers:
   - name: fluentd-elasticsearch
-    image: gcr.io/google_containers/fluentd-elasticsearch:1.15
+    image: gcr.io/google_containers/fluentd-elasticsearch:1.17
     resources:
       limits:
         memory: 200Mi
diff --git a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml
index 9f34c2f3edf..ef573f5b55d 100644
--- a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml
+++ b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml
@@ -9,7 +9,7 @@ spec:
   dnsPolicy: Default
   containers:
   - name: fluentd-cloud-logging
-    image: gcr.io/google_containers/fluentd-gcp:1.18
+    image: gcr.io/google_containers/fluentd-gcp:1.20
     resources:
       limits:
         memory: 200Mi