Merge pull request #26783 from a-robinson/newlogs

Automatic merge from submit-queue

Add collection of the new glbc and cluster-autoscaler logs

I've incremented the version numbers by 2 to avoid conflicting with #26652. I'll make sure the potential conflict between the images gets resolved reasonably.

cc @piosz @bprashanth @aledbf

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -15,7 +15,7 @@
 .PHONY:	build push
 
 IMAGE = fluentd-elasticsearch
-TAG = 1.15
+TAG = 1.17
 
 build:	
 	docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) .

cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf

@@ -124,7 +124,7 @@
   format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
   time_format %Y-%m-%d %H:%M:%S
   path /var/log/salt/minion
-  pos_file /var/log/gcp-salt.pos
+  pos_file /var/log/es-salt.pos
   tag salt
 </source>
 
@@ -222,6 +222,34 @@
   type kubernetes_metadata
 </filter>
 
+# Example:
+# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
+<source>
+  type tail
+  format multiline
+  multiline_flush_interval 5s
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+  time_format %m%d %H:%M:%S.%N
+  path /var/log/glbc.log
+  pos_file /var/log/es-glbc.log.pos
+  tag glbc
+</source>
+
+# Example:
+# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
+<source>
+  type tail
+  format multiline
+  multiline_flush_interval 5s
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+  time_format %m%d %H:%M:%S.%N
+  path /var/log/cluster-autoscaler.log
+  pos_file /var/log/es-cluster-autoscaler.log.pos
+  tag cluster-autoscaler
+</source>
+
 <match **>
    type elasticsearch
    log_level info

cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile

@@ -28,7 +28,7 @@
 
 .PHONY:	kbuild kpush
 
-TAG = 1.18
+TAG = 1.20
 
 # Rules for building the test image for deployment to Dockerhub with user kubernetes.
 

cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf

@@ -13,9 +13,6 @@
 #  "stream":"stderr",
 #   "time":"2014-09-25T21:15:03.499185026Z"}
 #
-# Currently, the log information is ingested as plain text rather than JSON.
-# TODO: When Cloud Logging supports it, ingest as JSON.
-#
 # The record reformer is used to write the tag to focus on the pod name
 # and the Kubernetes container name. For example a Docker container's logs
 # might be in the directory:
@@ -171,6 +168,34 @@
   tag kube-scheduler
 </source>
 
+# Example:
+# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
+<source>
+  type tail
+  format multiline
+  multiline_flush_interval 5s
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+  time_format %m%d %H:%M:%S.%N
+  path /var/log/glbc.log
+  pos_file /var/log/gcp-glbc.log.pos
+  tag glbc
+</source>
+
+# Example:
+# I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
+<source>
+  type tail
+  format multiline
+  multiline_flush_interval 5s
+  format_firstline /^\w\d{4}/
+  format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/
+  time_format %m%d %H:%M:%S.%N
+  path /var/log/cluster-autoscaler.log
+  pos_file /var/log/gcp-cluster-autoscaler.log.pos
+  tag cluster-autoscaler
+</source>
+
 # We use 2 output stanzas - one to handle the container logs and one to handle
 # the node daemon logs, the latter of which explicitly sends its logs to the
 # compute.googleapis.com service rather than container.googleapis.com to keep

cluster/saltbase/salt/fluentd-es/fluentd-es.yaml

@@ -8,7 +8,7 @@ metadata:
 spec:
   containers:
   - name: fluentd-elasticsearch
-    image: gcr.io/google_containers/fluentd-elasticsearch:1.15
+    image: gcr.io/google_containers/fluentd-elasticsearch:1.17
     resources:
       limits:
         memory: 200Mi

cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml

@@ -9,7 +9,7 @@ spec:
   dnsPolicy: Default
   containers:
   - name: fluentd-cloud-logging
-    image: gcr.io/google_containers/fluentd-gcp:1.18
+    image: gcr.io/google_containers/fluentd-gcp:1.20
     resources:
       limits:
         memory: 200Mi