Merge pull request #10597 from a-robinson/fluentd

Run fluentd on the master to collect the core master logs

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -1,7 +1,7 @@
 .PHONY:	build push
 
 IMAGE = fluentd-elasticsearch
-TAG = 1.5
+TAG = 1.6
 
 build:	
 	docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) .

cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf

@@ -78,19 +78,21 @@
   tag kubernetes.${tag_suffix[3].split('-')[0..-2].join('-')}
 </match>
 
-<match kubernetes.**>
-   type elasticsearch
-   log_level info
-   include_tag_key true
-   host elasticsearch-logging.default
-   port 9200
-   logstash_format true
-   flush_interval 5s
-   # Never wait longer than 5 minutes between retries.
-   max_retry_wait 300
-   # Disable the limit on the number of retries (retry forever).
-   disable_retry_limit
-</match>
+<source>
+  type tail
+  format none
+  path /varlog/startupscript.log
+  pos_file /varlog/es-startupscript.log.pos
+  tag startupscript
+</source>
+
+<source>
+  type tail
+  format none
+  path /varlog/docker.log
+  pos_file /varlog/es-docker.log.pos
+  tag docker
+</source>
 
 <source>
   type tail
@@ -100,13 +102,41 @@
   tag kubelet
 </source>
 
-<match kubelet>
+<source>
+  type tail
+  format none
+  path /varlog/kube-apiserver.log
+  pos_file /varlog/es-kube-apiserver.log.pos
+  tag kube-apiserver
+</source>
+
+<source>
+  type tail
+  format none
+  path /varlog/kube-controller-manager.log
+  pos_file /varlog/es-kube-controller-manager.log.pos
+  tag kube-controller-manager
+</source>
+
+<source>
+  type tail
+  format none
+  path /varlog/kube-scheduler.log
+  pos_file /varlog/es-kube-scheduler.log.pos
+  tag kube-scheduler
+</source>
+
+<match **>
    type elasticsearch
    log_level info
    include_tag_key true
-   host elasticsearch-logging.default
+   host elasticsearch-logging
    port 9200
    logstash_format true
+   # Set the chunk limit the same as for fluentd-gcp.
+   buffer_chunk_limit 512K
+   # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB
+   buffer_queue_limit 128
    flush_interval 5s
    # Never wait longer than 5 minutes between retries.
    max_retry_wait 300