diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile index e654cdcdcca..213381e5db3 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile @@ -1,7 +1,7 @@ .PHONY: build push IMAGE = fluentd-elasticsearch -TAG = 1.5 +TAG = 1.6 build: docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) . diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf index 908655e8a73..cd07b235150 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf @@ -78,19 +78,21 @@ tag kubernetes.${tag_suffix[3].split('-')[0..-2].join('-')} - - type elasticsearch - log_level info - include_tag_key true - host elasticsearch-logging.default - port 9200 - logstash_format true - flush_interval 5s - # Never wait longer than 5 minutes between retries. - max_retry_wait 300 - # Disable the limit on the number of retries (retry forever). - disable_retry_limit - + + type tail + format none + path /varlog/startupscript.log + pos_file /varlog/es-startupscript.log.pos + tag startupscript + + + + type tail + format none + path /varlog/docker.log + pos_file /varlog/es-docker.log.pos + tag docker + type tail @@ -100,13 +102,41 @@ tag kubelet - + + type tail + format none + path /varlog/kube-apiserver.log + pos_file /varlog/es-kube-apiserver.log.pos + tag kube-apiserver + + + + type tail + format none + path /varlog/kube-controller-manager.log + pos_file /varlog/es-kube-controller-manager.log.pos + tag kube-controller-manager + + + + type tail + format none + path /varlog/kube-scheduler.log + pos_file /varlog/es-kube-scheduler.log.pos + tag kube-scheduler + + + type elasticsearch log_level info include_tag_key true - host elasticsearch-logging.default + host elasticsearch-logging port 9200 logstash_format true + # Set the chunk limit the same as for fluentd-gcp. + buffer_chunk_limit 512K + # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB + buffer_queue_limit 128 flush_interval 5s # Never wait longer than 5 minutes between retries. max_retry_wait 300 diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile index 19d5ea704ae..d0ba196a67b 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile @@ -15,7 +15,7 @@ .PHONY: kbuild kpush -TAG = 1.7 +TAG = 1.8 # Rules for building the test image for deployment to Dockerhub with user kubernetes. diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf index 5aa7b6d450a..4fb6cc30034 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf @@ -63,26 +63,61 @@ tag kubernetes.${tag_suffix[3].split('-')[0..-2].join('-')} - - type google_cloud - flush_interval 5s - # Never wait longer than 5 minutes between retries. - max_retry_wait 300 - # Disable the limit on the number of retries (retry forever). - disable_retry_limit - + + type tail + format none + path /varlog/startupscript.log + pos_file /varlog/gcp-startupscript.log.pos + tag startupscript + + + + type tail + format none + path /varlog/docker.log + pos_file /varlog/gcp-docker.log.pos + tag docker + type tail format none - time_key time path /varlog/kubelet.log pos_file /varlog/gcp-kubelet.log.pos tag kubelet - + + type tail + format none + path /varlog/kube-apiserver.log + pos_file /varlog/gcp-kube-apiserver.log.pos + tag kube-apiserver + + + + type tail + format none + path /varlog/kube-controller-manager.log + pos_file /varlog/gcp-kube-controller-manager.log.pos + tag kube-controller-manager + + + + type tail + format none + path /varlog/kube-scheduler.log + pos_file /varlog/gcp-kube-scheduler.log.pos + tag kube-scheduler + + + type google_cloud + # Set the chunk limit conservatively to avoid exceeding the GCL limit + # of 2MB per write request. + buffer_chunk_limit 512K + # Cap buffer memory usage to 512KB/chunk * 128 chunks = 65 MB + buffer_queue_limit 128 flush_interval 5s # Never wait longer than 5 minutes between retries. max_retry_wait 300 diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index 2fc38e1d790..bddf826926a 100644 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -123,7 +123,7 @@ function create-master-instance { --image "${MASTER_IMAGE}" \ --tags "${MASTER_TAG}" \ --network "${NETWORK}" \ - --scopes "storage-ro,compute-rw" \ + --scopes "storage-ro,compute-rw,logging-write" \ --can-ip-forward \ --metadata-from-file \ "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh,kube-env=${KUBE_TEMP}/master-kube-env.yaml" \ diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index 142cab11787..eaa056ba370 100644 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -105,7 +105,7 @@ function create-master-instance { --image "${MASTER_IMAGE}" \ --tags "${MASTER_TAG}" \ --network "${NETWORK}" \ - --scopes "storage-ro,compute-rw" \ + --scopes "storage-ro,compute-rw,logging-write" \ --can-ip-forward \ --metadata-from-file \ "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh,kube-env=${KUBE_TEMP}/master-kube-env.yaml" \ diff --git a/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml b/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml index ef19173c430..1d173d7742f 100644 --- a/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml +++ b/cluster/saltbase/salt/fluentd-es/fluentd-es.yaml @@ -5,7 +5,7 @@ metadata: spec: containers: - name: fluentd-elasticsearch - image: gcr.io/google_containers/fluentd-elasticsearch:1.5 + image: gcr.io/google_containers/fluentd-elasticsearch:1.6 resources: limits: cpu: 100m diff --git a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml index 7790e3dd546..c699d4d8aa6 100644 --- a/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml +++ b/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.yaml @@ -5,7 +5,7 @@ metadata: spec: containers: - name: fluentd-cloud-logging - image: gcr.io/google_containers/fluentd-gcp:1.7 + image: gcr.io/google_containers/fluentd-gcp:1.8 resources: limits: cpu: 100m diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 642461b7302..30c42dd4538 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -38,6 +38,13 @@ base: - kube-client-tools - kube-master-addons - kube-admission-controls +{% if pillar.get('enable_node_logging', '').lower() == 'true' and pillar['logging_destination'] is defined %} + {% if pillar['logging_destination'] == 'elasticsearch' %} + - fluentd-es + {% elif pillar['logging_destination'] == 'gcp' %} + - fluentd-gcp + {% endif %} +{% endif %} {% if grains['cloud'] is defined and grains['cloud'] != 'vagrant' %} - logrotate {% endif %}