Merge pull request #1610 from satnam6502/salt

Salt configuration for fluentd to Elasticsearch

cluster/gce/templates/create-dynamic-salt-files.sh

@@ -22,6 +22,7 @@ mkdir -p /srv/salt-overlay/pillar
 cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
 node_instance_prefix: $NODE_INSTANCE_PREFIX
 portal_net: $PORTAL_NET
+use-fluentd-es: $FLUENTD_ELASTICSEARCH
 EOF
 
 mkdir -p /srv/salt-overlay/salt/nginx

cluster/gce/util.sh

@@ -264,6 +264,7 @@ function kube-up {
     echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
     echo "readonly MASTER_HTPASSWD='${htpasswd}'"
     echo "readonly PORTAL_NET='${PORTAL_NET}'"
+    echo "readonly FLUENTD_ELASTICSEARCH='${FLUENTD_ELASTICSEARCH:-false}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh"

cluster/saltbase/salt/fluentd-es/fluentd-es.manifest

@@ -0,0 +1,21 @@
+version: v1beta2
+id: fluentd-to-elasticsearch
+containers:
+  - name: fluentd-es
+    image: kubernetes/fluentd-elasticsearch
+    volumeMounts:
+      - name: containers
+        mountPath: /var/lib/docker/containers
+        readOnly: true
+      - name: hosts
+        mountPath: /outerhost
+        readOnly: true
+volumes:
+  - name: containers
+    source:
+      hostDir:
+        path: /var/lib/docker/containers
+  - name: hosts
+    source:
+      hostDir:
+        path: /etc/hosts

cluster/saltbase/salt/fluentd-es/init.sls

@@ -0,0 +1,8 @@
+/etc/kubernetes/manifests/fluentd-es.manifest:
+  file.managed:
+    - source: salt://fluentd-es/fluentd-es.manifest
+    - user: root
+    - group: root
+    - mode: 644
+    - makedirs: true
+    - dir_mode: 755

cluster/saltbase/salt/top.sls

@@ -8,6 +8,9 @@ base:
     - kubelet
     - kube-proxy
     - cadvisor
+{% if pillar['use-fluentd-es'] is defined and pillar['use-fluentd-es'] %}
+    - fluentd-es
+{% endif %}
     # We need a binary release of nsinit
     # - nsinit
     - logrotate
@@ -36,4 +39,3 @@ base:
   'roles:kubernetes-pool-vagrant':
     - match: grain
     - vagrant
-

docs/getting-started-guides/logging.md

@@ -0,0 +1,18 @@
+## Logging
+
+**Experimental work in progress.**
+
+### Logging with Fluentd and Elastiscsearch
+
+To enable logging of the stdout and stderr output of every Docker container in
+a Kubernetes cluster set the shell environment
+variable ``FLUENTD_ELASTICSEARCH`` to ``true`` e.g. in bash:
+```
+export FLUENTD_ELASTICSEARCH=true
+```
+This will instantiate a [Fluentd](http://www.fluentd.org/) instance on each node which will
+collect all the Dcoker container log files. The collected logs will
+be targetted at an [Elasticsearch](http://www.elasticsearch.org/) instance assumed to be running on the
+local node and accepting log information on port 9200. This can be accomplished
+by writing a pod specification and service sepecificaiton to define an
+Elasticsearch service (more informaiton to follow shortly in the contrib directory).