From f8965a6f41d20029e0209873012e4309f93e60c0 Mon Sep 17 00:00:00 2001
From: Alex Robinson <arob@google.com>
Date: Fri, 26 Feb 2016 22:58:46 +0000
Subject: [PATCH] Fix parsing of kube logs to handle logs split across lines.

I didn't expect glog to split single log statements onto multiple lines,
but apparently it does if they're long enough. This groups them back
together appropriately.
---
 .../fluentd-es-image/td-agent.conf            | 20 +++++++++++++++----
 .../fluentd-gcp-image/google-fluentd.conf     | 20 +++++++++++++++----
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf
index d76522680aa..269f0abeb75 100644
--- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf
+++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf
@@ -162,11 +162,17 @@
   tag etcd
 </source>
 
+# Multi-line parsing is required for all the kube logs because very large log
+# statements, such as those that include entire object bodies, get split into
+# multiple lines by glog.
+
 # Example:
 # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kubelet.log
   pos_file /var/log/es-kubelet.log.pos
@@ -177,7 +183,9 @@
 # I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-apiserver.log
   pos_file /var/log/es-kube-apiserver.log.pos
@@ -188,7 +196,9 @@
 # I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-controller-manager.log
   pos_file /var/log/es-kube-controller-manager.log.pos
@@ -199,7 +209,9 @@
 # W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-scheduler.log
   pos_file /var/log/es-kube-scheduler.log.pos
diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
index f479c6293c1..b387f327856 100644
--- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
+++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf
@@ -111,11 +111,17 @@
   tag etcd
 </source>
 
+# Multi-line parsing is required for all the kube logs because very large log
+# statements, such as those that include entire object bodies, get split into
+# multiple lines by glog.
+
 # Example:
 # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kubelet.log
   pos_file /var/log/gcp-kubelet.log.pos
@@ -126,7 +132,9 @@
 # I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-apiserver.log
   pos_file /var/log/gcp-kube-apiserver.log.pos
@@ -137,7 +145,9 @@
 # I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-controller-manager.log
   pos_file /var/log/gcp-kube-controller-manager.log.pos
@@ -148,7 +158,9 @@
 # W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
 <source>
   type tail
-  format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)$/
+  format multiline
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
   time_format %m%d %H:%M:%S.%N
   path /var/log/kube-scheduler.log
   pos_file /var/log/gcp-kube-scheduler.log.pos