Merge pull request #10597 from a-robinson/fluentd

Run fluentd on the master to collect the core master logs

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -1,7 +1,7 @@
 .PHONY:	build push
 
 IMAGE = fluentd-elasticsearch
-TAG = 1.5
+TAG = 1.6
 
 build:	
 	docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) .

cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf

@@ -78,19 +78,21 @@
   tag kubernetes.${tag_suffix[3].split('-')[0..-2].join('-')}
 </match>
 
-<match kubernetes.**>
-   type elasticsearch
-   log_level info
-   include_tag_key true
-   host elasticsearch-logging.default
-   port 9200
-   logstash_format true
-   flush_interval 5s
-   # Never wait longer than 5 minutes between retries.
-   max_retry_wait 300
-   # Disable the limit on the number of retries (retry forever).
-   disable_retry_limit
-</match>
+<source>
+  type tail
+  format none
+  path /varlog/startupscript.log
+  pos_file /varlog/es-startupscript.log.pos
+  tag startupscript
+</source>
+
+<source>
+  type tail
+  format none
+  path /varlog/docker.log
+  pos_file /varlog/es-docker.log.pos
+  tag docker
+</source>
 
 <source>
   type tail
@@ -100,13 +102,41 @@
   tag kubelet
 </source>
 
-<match kubelet>
+<source>
+  type tail
+  format none
+  path /varlog/kube-apiserver.log
+  pos_file /varlog/es-kube-apiserver.log.pos
+  tag kube-apiserver
+</source>
+
+<source>
+  type tail
+  format none
+  path /varlog/kube-controller-manager.log
+  pos_file /varlog/es-kube-controller-manager.log.pos
+  tag kube-controller-manager
+</source>
+
+<source>
+  type tail
+  format none
+  path /varlog/kube-scheduler.log
+  pos_file /varlog/es-kube-scheduler.log.pos
+  tag kube-scheduler
+</source>
+
+<match **>
    type elasticsearch
    log_level info
    include_tag_key true
-   host elasticsearch-logging.default
+   host elasticsearch-logging
    port 9200
    logstash_format true
+   # Set the chunk limit the same as for fluentd-gcp.
+   buffer_chunk_limit 512K
+   # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB
+   buffer_queue_limit 128
    flush_interval 5s
    # Never wait longer than 5 minutes between retries.
    max_retry_wait 300

cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile

@@ -15,7 +15,7 @@
 
 .PHONY:	kbuild kpush
 
-TAG = 1.7
+TAG = 1.8
 
 # Rules for building the test image for deployment to Dockerhub with user kubernetes.
 

cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf

@@ -63,26 +63,61 @@
   tag kubernetes.${tag_suffix[3].split('-')[0..-2].join('-')}
 </match>
 
-<match kubernetes.**>
-  type google_cloud
-  flush_interval 5s
-  # Never wait longer than 5 minutes between retries.
-  max_retry_wait 300
-  # Disable the limit on the number of retries (retry forever).
-  disable_retry_limit
-</match>
+<source>
+  type tail
+  format none
+  path /varlog/startupscript.log
+  pos_file /varlog/gcp-startupscript.log.pos
+  tag startupscript
+</source>
+
+<source>
+  type tail
+  format none
+  path /varlog/docker.log
+  pos_file /varlog/gcp-docker.log.pos
+  tag docker
+</source>
 
 <source>
   type tail
   format none
-  time_key time
   path /varlog/kubelet.log
   pos_file /varlog/gcp-kubelet.log.pos
   tag kubelet
 </source>
 
-<match kubelet>
+<source>
+  type tail
+  format none
+  path /varlog/kube-apiserver.log
+  pos_file /varlog/gcp-kube-apiserver.log.pos
+  tag kube-apiserver
+</source>
+
+<source>
+  type tail
+  format none
+  path /varlog/kube-controller-manager.log
+  pos_file /varlog/gcp-kube-controller-manager.log.pos
+  tag kube-controller-manager
+</source>
+
+<source>
+  type tail
+  format none
+  path /varlog/kube-scheduler.log
+  pos_file /varlog/gcp-kube-scheduler.log.pos
+  tag kube-scheduler
+</source>
+
+<match **>
   type google_cloud
+  # Set the chunk limit conservatively to avoid exceeding the GCL limit
+  # of 2MB per write request.
+  buffer_chunk_limit 512K
+  # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB
+  buffer_queue_limit 128
   flush_interval 5s
   # Never wait longer than 5 minutes between retries.
   max_retry_wait 300

cluster/gce/coreos/helper.sh

@@ -123,7 +123,7 @@ function create-master-instance {
     --image "${MASTER_IMAGE}" \
     --tags "${MASTER_TAG}" \
     --network "${NETWORK}" \
-    --scopes "storage-ro,compute-rw" \
+    --scopes "storage-ro,compute-rw,logging-write" \
     --can-ip-forward \
     --metadata-from-file \
       "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh,kube-env=${KUBE_TEMP}/master-kube-env.yaml" \

cluster/gce/debian/helper.sh

@@ -105,7 +105,7 @@ function create-master-instance {
     --image "${MASTER_IMAGE}" \
     --tags "${MASTER_TAG}" \
     --network "${NETWORK}" \
-    --scopes "storage-ro,compute-rw" \
+    --scopes "storage-ro,compute-rw,logging-write" \
     --can-ip-forward \
     --metadata-from-file \
       "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh,kube-env=${KUBE_TEMP}/master-kube-env.yaml" \

cluster/saltbase/salt/fluentd-es/fluentd-es.yaml

@@ -5,7 +5,7 @@ metadata:
 spec:
   containers:
   - name: fluentd-elasticsearch
-    image: gcr.io/google_containers/fluentd-elasticsearch:1.5
+    image: gcr.io/google_containers/fluentd-elasticsearch:1.6
     resources:
       limits:
         cpu: 100m

cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml

@@ -5,7 +5,7 @@ metadata:
 spec:
   containers:
   - name: fluentd-cloud-logging
-    image: gcr.io/google_containers/fluentd-gcp:1.7
+    image: gcr.io/google_containers/fluentd-gcp:1.8
     resources:
       limits:
         cpu: 100m

cluster/saltbase/salt/top.sls

@@ -38,6 +38,13 @@ base:
     - kube-client-tools
     - kube-master-addons
     - kube-admission-controls
+{% if pillar.get('enable_node_logging', '').lower() == 'true' and pillar['logging_destination'] is defined %}
+  {% if pillar['logging_destination'] == 'elasticsearch' %}
+    - fluentd-es
+  {% elif pillar['logging_destination'] == 'gcp' %}
+    - fluentd-gcp
+  {% endif %}
+{% endif %}
 {% if grains['cloud'] is defined and grains['cloud'] != 'vagrant' %}
     - logrotate
 {% endif %}