diff --git a/cluster/aws/config-default.sh b/cluster/aws/config-default.sh index fb395da1bba..7e8b1e789e5 100644 --- a/cluster/aws/config-default.sh +++ b/cluster/aws/config-default.sh @@ -35,3 +35,10 @@ MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24")) MINION_SCOPES="" POLL_SLEEP_INTERVAL=3 PORTAL_NET="10.0.0.0/16" + +# Optional: Install node monitoring. +ENABLE_NODE_MONITORING=true + +# Optional: Install node logging +ENABLE_NODE_LOGGING=true +LOGGING_DESTINATION=elasticsearch # options: elasticsearch, gcp diff --git a/cluster/aws/templates/create-dynamic-salt-files.sh b/cluster/aws/templates/create-dynamic-salt-files.sh index 0183e286007..0a06dd73983 100644 --- a/cluster/aws/templates/create-dynamic-salt-files.sh +++ b/cluster/aws/templates/create-dynamic-salt-files.sh @@ -22,8 +22,9 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls node_instance_prefix: $NODE_INSTANCE_PREFIX portal_net: $PORTAL_NET -use-fluentd-es: $FLUENTD_ELASTICSEARCH -use-fluentd-gcp: $FLUENTD_GCP +enable_node_monitoring: $ENABLE_NODE_MONITORING +enable_node_logging: $ENABLE_NODE_LOGGING +logging_destination: $LOGGING_DESTINATION EOF mkdir -p /srv/salt-overlay/salt/nginx diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 7cdd30fa10e..c69a967a4fc 100644 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -106,14 +106,14 @@ function ensure-temp-dir { } function setup-monitoring { - if [ ${MONITORING-} ]; then + if [[ "${ENABLE_CLUSTER_MONITORING}" == "true" ]]; then # TODO: Implement this. echo "Monitoring not currently supported on AWS" fi } function teardown-monitoring { - if [ ${MONITORING-} ]; then + if [[ "${ENABLE_CLUSTER_MONITORING}" == "true" ]]; then # TODO: Implement this. echo "Monitoring not currently supported on AWS" fi @@ -255,8 +255,9 @@ function kube-up { echo "readonly AWS_ZONE='${ZONE}'" echo "readonly MASTER_HTPASSWD='${htpasswd}'" echo "readonly PORTAL_NET='${PORTAL_NET}'" - echo "readonly FLUENTD_ELASTICSEARCH='${FLUENTD_ELASTICSEARCH:-false}'" - echo "readonly FLUENTD_GCP='false'" + echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'" + echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'" + echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/create-dynamic-salt-files.sh" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/download-release.sh" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/salt-master.sh" diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 6ee7574b7f7..296d9abe959 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -35,14 +35,16 @@ MINION_SCOPES=("storage-ro" "compute-rw") # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 PORTAL_NET="10.0.0.0/16" -# When set to true, heapster will be setup as part of the cluster bring up. -MONITORING=true -# Turn on Elasticsearch logging unless Google Cloud Logging has been selected or -# if Elasticsearch logging has been specifically turned off. -if [[ "${FLUENTD_GCP-}" != "true" ]]; then - if [[ "${FLUENTD_ELASTICSEARCH-}" != "false" ]]; then - FLUENTD_ELASTICSEARCH="true" - fi -fi + +# Optional: Install node monitoring. +ENABLE_NODE_MONITORING=true + +# Optional: When set to true, heapster will be setup as part of the cluster bring up. +ENABLE_CLUSTER_MONITORING=true + # When set to true, Docker Cache is enabled by default as part of the cluster bring up. ENABLE_DOCKER_REGISTRY_CACHE=true + +# Optional: Enable node logging. +ENABLE_NODE_LOGGING=true +LOGGING_DESTINATION=elasticsearch # options: elasticsearch, gcp diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index c2939d8f45b..6bad3a681c5 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -35,7 +35,13 @@ MINION_SCOPES=("storage-ro" "compute-rw") # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 PORTAL_NET="10.0.0.0/16" -MONITORING=false # When set to true, Docker Cache is enabled by default as part of the cluster bring up. ENABLE_DOCKER_REGISTRY_CACHE=true + +ENABLE_NODE_MONITORING=true + +ENABLE_NODE_LOGGING=true +LOGGING_DESTINATION=elasticsearch # options: elasticsearch, gcp + +ENABLE_CLUSTER_MONITORING=false diff --git a/cluster/gce/templates/create-dynamic-salt-files.sh b/cluster/gce/templates/create-dynamic-salt-files.sh index c02ff5d7e63..6cde56a7948 100644 --- a/cluster/gce/templates/create-dynamic-salt-files.sh +++ b/cluster/gce/templates/create-dynamic-salt-files.sh @@ -22,8 +22,9 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls node_instance_prefix: $NODE_INSTANCE_PREFIX portal_net: $PORTAL_NET -use-fluentd-es: $FLUENTD_ELASTICSEARCH -use-fluentd-gcp: $FLUENTD_GCP +enable_node_monitoring: $ENABLE_NODE_MONITORING +enable_node_logging: $ENABLE_NODE_LOGGING +logging_destination: $LOGGING_DESTINATION EOF mkdir -p /srv/salt-overlay/salt/nginx diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 2f0c3bba935..54147ef9898 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -283,8 +283,9 @@ function kube-up { echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" echo "readonly MASTER_HTPASSWD='${htpasswd}'" echo "readonly PORTAL_NET='${PORTAL_NET}'" - echo "readonly FLUENTD_ELASTICSEARCH='${FLUENTD_ELASTICSEARCH:-false}'" - echo "readonly FLUENTD_GCP='${FLUENTD_GCP:-false}'" + echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'" + echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'" + echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" @@ -292,16 +293,12 @@ function kube-up { ) > "${KUBE_TEMP}/master-start.sh" # Report logging choice (if any). - if [[ "${FLUENTD_ELASTICSEARCH-}" == "true" ]]; then - echo "+++ Logging using Fluentd to Elasticsearch" - fi - if [[ "${FLUENTD_GCP-}" == "true" ]]; then - echo "+++ Logging using Fluentd to Google Cloud Logging" - fi - - # For logging to GCP we need to enable some minion scopes. - if [[ "${FLUENTD_GCP-}" == "true" ]]; then - MINION_SCOPES="${MINION_SCOPES}, https://www.googleapis.com/auth/logging.write" + if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]]; then + echo "+++ Logging using Fluentd to ${LOGGING_DESTINATION:-unknown}" + # For logging to GCP we need to enable some minion scopes. + if [[ "${LOGGING_DESTINATION-}" == "gcp" ]]; then + MINION_SCOPES="${MINION_SCOPES}, https://www.googleapis.com/auth/logging.write" + fi fi gcloud compute instances create "${MASTER_NAME}" \ @@ -566,7 +563,7 @@ function restart-kube-proxy { # Setup monitoring using heapster and InfluxDB function setup-monitoring { - if [[ "${MONITORING}" == "true" ]]; then + if [[ "${ENABLE_CLUSTER_MONITORING}" == "true" ]]; then echo "Setting up cluster monitoring using Heapster." if ! gcloud compute firewall-rules describe monitoring-heapster &>/dev/null; then @@ -584,9 +581,9 @@ function setup-monitoring { cp "${KUBE_ROOT}/examples/monitoring/influx-grafana-pod.json" "${KUBE_TEMP}/influx-grafana-pod.0.json" sed "s/HTTP_USER, \"value\": \"[^\"]*\"/HTTP_USER, \"value\": \"$KUBE_USER\"/g" \ - "${KUBE_TEMP}/influx-grafana-pod.0.json" > "${KUBE_TEMP}/influx-grafana-pod.1.json" + "${KUBE_TEMP}/influx-grafana-pod.0.json" > "${KUBE_TEMP}/influx-grafana-pod.1.json" sed "s/HTTP_PASS, \"value\": \"[^\"]*\"/HTTP_PASS, \"value\": \"$KUBE_PASSWORD\"/g" \ - "${KUBE_TEMP}/influx-grafana-pod.1.json" > "${KUBE_TEMP}/influx-grafana-pod.2.json" + "${KUBE_TEMP}/influx-grafana-pod.1.json" > "${KUBE_TEMP}/influx-grafana-pod.2.json" local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" if "${kubectl}" create -f "${KUBE_TEMP}/influx-grafana-pod.2.json" &> /dev/null \ && "${kubectl}" create -f "${KUBE_ROOT}/examples/monitoring/influx-grafana-service.json" &> /dev/null \ @@ -604,7 +601,7 @@ function setup-monitoring { } function teardown-monitoring { - if [[ "${MONITORING}" == "true" ]]; then + if [[ "${ENABLE_CLUSTER_MONITORING}" == "true" ]]; then detect-project local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" @@ -613,9 +610,9 @@ function teardown-monitoring { "${kubectl}" delete services influx-master &> /dev/null || true if gcloud compute firewall-rules describe monitoring-heapster &> /dev/null; then gcloud compute firewall-rules delete \ - --project "${PROJECT}" \ - --quiet \ - monitoring-heapster &> /dev/null || true + --project "${PROJECT}" \ + --quiet \ + monitoring-heapster &> /dev/null || true fi fi } diff --git a/cluster/rackspace/config-default.sh b/cluster/rackspace/config-default.sh index 7fb84a37a46..5133e23b7f0 100644 --- a/cluster/rackspace/config-default.sh +++ b/cluster/rackspace/config-default.sh @@ -37,3 +37,10 @@ MINION_TAG="tags=${INSTANCE_PREFIX}-minion" MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${RAX_NUM_MINIONS}})) KUBE_NETWORK=($(eval echo "10.240.{1..${RAX_NUM_MINIONS}}.0/24")) PORTAL_NET="10.0.0.0/16" + +# Optional: Install node monitoring. +ENABLE_NODE_MONITORING=true + +# Optional: Enable node logging. +ENABLE_NODE_LOGGING=true +LOGGING_DESTINATION=elasticsearch diff --git a/cluster/rackspace/util.sh b/cluster/rackspace/util.sh index ae32a4db6e2..57519a8a842 100644 --- a/cluster/rackspace/util.sh +++ b/cluster/rackspace/util.sh @@ -184,6 +184,9 @@ rax-boot-minions() { sed -e "s|DISCOVERY_ID|${DISCOVERY_ID}|" \ -e "s|INDEX|$((i + 1))|g" \ -e "s|CLOUD_FILES_URL|${RELEASE_TMP_URL//&/\&}|" \ + -e "s|ENABLE_NODE_MONITORING|${ENABLE_NODE_MONITORING:-false}|" \ + -e "s|ENABLE_NODE_LOGGING|${ENABLE_NODE_LOGGING:-false}|" \ + -e "s|LOGGING_DESTINATION|${LOGGING_DESTINATION:-}|" \ $(dirname $0)/rackspace/cloud-config/minion-cloud-config.yaml > $KUBE_TEMP/minion-cloud-config-$(($i + 1)).yaml diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index d817b5d2c78..e4ec1660a70 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -8,12 +8,16 @@ base: - docker - kubelet - kube-proxy +{% if pillar['enable_node_monitoring'] is defined and pillar['enable_node_monitoring'] %} - cadvisor -{% if pillar['use-fluentd-es'] is defined and pillar['use-fluentd-es'] %} - - fluentd-es {% endif %} -{% if pillar['use-fluentd-gcp'] is defined and pillar['use-fluentd-gcp'] %} +{% if pillar['enable_node_logging'] is defined and pillar['enable_node_logging'] %} + {% if pillar['logging_destination'] is defined and pillar['logging_destination'] == 'elasticsearch' %} + - fluentd-es + {% endif %} + {% if pillar['logging_destination'] is defined and pillar['logging_destination'] == 'gcp' %} - fluentd-gcp + {% endif %} {% endif %} - logrotate {% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} diff --git a/cluster/vagrant/config-default.sh b/cluster/vagrant/config-default.sh index 79171655fe6..af1264c1deb 100755 --- a/cluster/vagrant/config-default.sh +++ b/cluster/vagrant/config-default.sh @@ -37,3 +37,10 @@ for ((i=0; i < NUM_MINIONS; i++)) do MINION_NAMES[$i]="${MINION_IP[$i]}" VAGRANT_MINION_NAMES[$i]="minion-$[$i+1]" done + +# Optional: Install node monitoring. +ENABLE_NODE_MONITORING=true + +# Optional: Enable node logging. +ENABLE_NODE_LOGGING=true +LOGGING_DESTINATION=elasticsearch diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh index c4605e799ff..71b404ffdf3 100755 --- a/cluster/vagrant/provision-master.sh +++ b/cluster/vagrant/provision-master.sh @@ -82,6 +82,9 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls portal_net: $PORTAL_NET cert_ip: $MASTER_IP + enable_node_monitoring: $ENABLE_NODE_MONITORING + enable_node_logging: $ENABLE_NODE_LOGGING + logging_destination: $LOGGING_DESTINATION EOF # Configure the salt-master diff --git a/cluster/vsphere/config-default.sh b/cluster/vsphere/config-default.sh index e8d9a637c9e..9b30ad1f4b2 100755 --- a/cluster/vsphere/config-default.sh +++ b/cluster/vsphere/config-default.sh @@ -32,3 +32,10 @@ MINION_MEMORY_MB=2048 MINION_CPU=1 PORTAL_NET="10.244.240.0/20" + +# Optional: Install node monitoring. +ENABLE_NODE_MONITORING=true + +# Optional: Enable node logging. +ENABLE_NODE_LOGGING=true +LOGGING_DESTINATION=elasticsearch diff --git a/cluster/vsphere/templates/create-dynamic-salt-files.sh b/cluster/vsphere/templates/create-dynamic-salt-files.sh index afea82b88c8..0a06dd73983 100755 --- a/cluster/vsphere/templates/create-dynamic-salt-files.sh +++ b/cluster/vsphere/templates/create-dynamic-salt-files.sh @@ -22,6 +22,9 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls node_instance_prefix: $NODE_INSTANCE_PREFIX portal_net: $PORTAL_NET +enable_node_monitoring: $ENABLE_NODE_MONITORING +enable_node_logging: $ENABLE_NODE_LOGGING +logging_destination: $LOGGING_DESTINATION EOF mkdir -p /srv/salt-overlay/salt/nginx diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh index e665d559dcf..7aacf264a1a 100755 --- a/cluster/vsphere/util.sh +++ b/cluster/vsphere/util.sh @@ -291,6 +291,9 @@ function kube-up { echo "readonly MASTER_NAME='${MASTER_NAME}'" echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'" echo "readonly PORTAL_NET='${PORTAL_NET}'" + echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'" + echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'" + echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'" echo "readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'" echo "readonly SALT_TAR='${SALT_TAR##*/}'" echo "readonly MASTER_HTPASSWD='${htpasswd}'" diff --git a/contrib/logging/fluentd-gcp/Makefile b/contrib/logging/fluentd-gcp/Makefile index 2f7aaab3302..843817cacd7 100644 --- a/contrib/logging/fluentd-gcp/Makefile +++ b/contrib/logging/fluentd-gcp/Makefile @@ -1,6 +1,6 @@ # Makefile for a synthetic logger to be logged # by GCP. The cluster must have been created with -# the environment variable FLUENTD_GCP set to 'true'. +# the variable LOGGING_DESTINATION=GCP. .PHONY: up down logger-up logger-down get diff --git a/docs/getting-started-guides/logging.md b/docs/getting-started-guides/logging.md index 585075f4b8e..335d4e49ed1 100644 --- a/docs/getting-started-guides/logging.md +++ b/docs/getting-started-guides/logging.md @@ -5,11 +5,15 @@ ### Logging with Fluentd and Elastiscsearch To enable logging of the stdout and stderr output of every Docker container in -a Kubernetes cluster set the shell environment -variable ``FLUENTD_ELASTICSEARCH`` to ``true`` e.g. in bash: +a Kubernetes cluster set the shell environment variables +``ENABLE_NODE_LOGGING`` to ``true`` and ``LOGGING_DESTINATION`` to ``elasticsearch``. + +e.g. in bash: ``` -export FLUENTD_ELASTICSEARCH=true +export ENABLE_NODE_LOGGING=true +export LOGGING_DESTINATION=elasticsearch ``` + This will instantiate a [Fluentd](http://www.fluentd.org/) instance on each node which will collect all the Dcoker container log files. The collected logs will be targetted at an [Elasticsearch](http://www.elasticsearch.org/) instance assumed to be running on the @@ -20,4 +24,5 @@ Elasticsearch service (more informaiton to follow shortly in the contrib directo ### Logging with Fluentd and Google Compute Platform To enable logging of Docker contains in a cluster using Google Compute -Platform set the shell environment variable ``FLUENTD_GCP`` to ``true``. +Platform set the config flags ``ENABLE_NODE_LOGGING`` to ``true`` and +``LOGGING_DESTINATION`` to ``gcp``.