From 178a901db77a3ab05bfe539e04c1766f50e8ee31 Mon Sep 17 00:00:00 2001 From: Satnam Singh Date: Mon, 13 Oct 2014 11:06:01 -0700 Subject: [PATCH 1/6] Source files for the fluentd-elasticsearch image. --- contrib/logging/fluentd-es-image/Dockerfile | 43 +++++++++++++++ contrib/logging/fluentd-es-image/build.sh | 18 +++++++ contrib/logging/fluentd-es-image/run.sh | 20 +++++++ .../logging/fluentd-es-image/td-agent.conf | 53 +++++++++++++++++++ 4 files changed, 134 insertions(+) create mode 100644 contrib/logging/fluentd-es-image/Dockerfile create mode 100755 contrib/logging/fluentd-es-image/build.sh create mode 100755 contrib/logging/fluentd-es-image/run.sh create mode 100644 contrib/logging/fluentd-es-image/td-agent.conf diff --git a/contrib/logging/fluentd-es-image/Dockerfile b/contrib/logging/fluentd-es-image/Dockerfile new file mode 100644 index 00000000000..ff6b9a1e344 --- /dev/null +++ b/contrib/logging/fluentd-es-image/Dockerfile @@ -0,0 +1,43 @@ +# This Dockerfile will build an image that is configured +# to run Fluentd with an Elasticsearch plug-in and the +# provided configuration file. +# TODO(satnam): Use a lighter base image, e.g. some form of busybox. +# The image acts as an executable for the binary /usr/sbin/td-agent +# which runs fluentd with the default flag -v (which can be over-ridden). +# Note that fluentd is run with root permssion to allow access to +# log files with root only access under /var/lib/docker/containers/* +# Please see http://docs.fluentd.org/articles/install-by-deb for more +# information about installing fluentd using deb package. + +FROM ubuntu:14.04 +MAINTAINER Satnam Singh "satnam@google.com" + +# Ensure there are enough file descriptors for running Fluentd. +RUN ulimit -n 65536 + +# Install prerequisites. +RUN apt-get update && \ + apt-get install -y curl && \ + apt-get install -y -q libcurl4-openssl-dev make && \ + apt-get clean + +# Install Fluentd. +RUN /usr/bin/curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sh + +# Change the default user and group to root. +# Needed to allow access to /var/log/docker/... files. +RUN sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent + +# Install the Elasticsearch Fluentd plug0in. +RUN /usr/sbin/td-agent-gem install fluent-plugin-elasticsearch + +# Copy the Fluentd configuration file. +COPY td-agent.conf /etc/td-agent/td-agent.conf + +# Copy a script that determines the name of the host machine +# and then patches the Fluentd configuration files and then +# runs Fluentd in the foreground. +ADD run.sh /run.sh + +# Always run the this setup script. +ENTRYPOINT ["/run.sh"] diff --git a/contrib/logging/fluentd-es-image/build.sh b/contrib/logging/fluentd-es-image/build.sh new file mode 100755 index 00000000000..6810cba13bd --- /dev/null +++ b/contrib/logging/fluentd-es-image/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker build -t google/fluentd-elasticsearch . +docker push google/lambda-fluentd \ No newline at end of file diff --git a/contrib/logging/fluentd-es-image/run.sh b/contrib/logging/fluentd-es-image/run.sh new file mode 100755 index 00000000000..4b8e4e8f5e8 --- /dev/null +++ b/contrib/logging/fluentd-es-image/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OUTER_HOST=`tail -n 1 /outerhost | awk '{print $3}'` +cp td-agent.conf /etc/td-agent +sed -i -e "s/ES_HOST/${OUTER_HOST}/" /etc/td-agent/td-agent.conf +/usr/sbin/td-agent diff --git a/contrib/logging/fluentd-es-image/td-agent.conf b/contrib/logging/fluentd-es-image/td-agent.conf new file mode 100644 index 00000000000..08bb16b9308 --- /dev/null +++ b/contrib/logging/fluentd-es-image/td-agent.conf @@ -0,0 +1,53 @@ +# This configuration file for Fluentd / td-agent is used +# to watch changes to Docker log files that live in the +# directory /var/lib/docker/containers/ which are then submitted to +# Elasticsearch (running on the machine ES_HOST:9200) which +# assumes the installation of the fluentd-elasticsearch plug-in. +# See https://github.com/uken/fluent-plugin-elasticsearch for +# more information about the plug-in. This file needs to be +# patched to replace ES_HOST with the name of the actual +# machine running Elasticsearch. +# Maintainer: Satnam Singh (satnam@google.com) +# +# Exampe +# ====== +# A line in the Docker log file might like like this JSON: +# +# {"log":"2014/09/25 21:15:03 Got request with path wombat\n", +# "stream":"stderr", +# "time":"2014-09-25T21:15:03.499185026Z"} +# +# The time_format specification below makes sure we properly +# parse the time format produced by Docker. This will be +# submitted to Elasticsearch and should appear like: +# $ curl 'http://elasticsearch:9200/_search?pretty' +# ... +# { +# "_index" : "logstash-2014.09.25", +# "_type" : "fluentd", +# "_id" : "VBrbor2QTuGpsQyTCdfzqA", +# "_score" : 1.0, +# "_source":{"log":"2014/09/25 22:45:50 Got request with path wombat\n", +# "stream":"stderr","tag":"docker.container.all", +# "@timestamp":"2014-09-25T22:45:50+00:00"} +# }, +# ... + + + type tail + format json + time_key time + path /var/lib/docker/containers/*/*-json.log + time_format %Y-%m-%dT%H:%M:%S + tag docker.container.all + + + + type elasticsearch + log_level info + include_tag_key true + host ES_HOST + port 9200 + logstash_format true + flush_interval 5s + From 303da8b036065a1cdc1ca57508f09aab065181f0 Mon Sep 17 00:00:00 2001 From: Satnam Singh Date: Mon, 13 Oct 2014 18:27:39 +0000 Subject: [PATCH 2/6] Dockerfile and build script for fluentd-elasticsearch image --- contrib/logging/fluentd-es-image/Dockerfile | 5 ++--- contrib/logging/fluentd-es-image/build.sh | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/contrib/logging/fluentd-es-image/Dockerfile b/contrib/logging/fluentd-es-image/Dockerfile index ff6b9a1e344..dc649a7c839 100644 --- a/contrib/logging/fluentd-es-image/Dockerfile +++ b/contrib/logging/fluentd-es-image/Dockerfile @@ -1,9 +1,8 @@ # This Dockerfile will build an image that is configured # to run Fluentd with an Elasticsearch plug-in and the # provided configuration file. -# TODO(satnam): Use a lighter base image, e.g. some form of busybox. -# The image acts as an executable for the binary /usr/sbin/td-agent -# which runs fluentd with the default flag -v (which can be over-ridden). +# TODO(satnam6502): Use a lighter base image, e.g. some form of busybox. +# The image acts as an executable for the binary /usr/sbin/td-agent. # Note that fluentd is run with root permssion to allow access to # log files with root only access under /var/lib/docker/containers/* # Please see http://docs.fluentd.org/articles/install-by-deb for more diff --git a/contrib/logging/fluentd-es-image/build.sh b/contrib/logging/fluentd-es-image/build.sh index 6810cba13bd..5ada3b7feda 100755 --- a/contrib/logging/fluentd-es-image/build.sh +++ b/contrib/logging/fluentd-es-image/build.sh @@ -14,5 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -docker build -t google/fluentd-elasticsearch . -docker push google/lambda-fluentd \ No newline at end of file + +# Build the fluentd-elasticsearch image and push +# to google/fluentd-elasticsearch. + +sudo docker build -t google/fluentd-elasticsearch . +sudo docker push google/fluentd-elasticsearch From d7b1550d3d1187ff18a15c843b480b085f9552db Mon Sep 17 00:00:00 2001 From: Satnam Singh Date: Mon, 13 Oct 2014 18:40:56 +0000 Subject: [PATCH 3/6] Change target image to kubernetes/elasticsearch --- contrib/logging/fluentd-es-image/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/logging/fluentd-es-image/build.sh b/contrib/logging/fluentd-es-image/build.sh index 5ada3b7feda..ae19afa0a0d 100755 --- a/contrib/logging/fluentd-es-image/build.sh +++ b/contrib/logging/fluentd-es-image/build.sh @@ -18,5 +18,5 @@ # Build the fluentd-elasticsearch image and push # to google/fluentd-elasticsearch. -sudo docker build -t google/fluentd-elasticsearch . -sudo docker push google/fluentd-elasticsearch +sudo docker build -t kubernetes/fluentd-elasticsearch . +sudo docker push kubernetes/fluentd-elasticsearch From a92dc444e6e50e727c229d5c1ff60568e71a7137 Mon Sep 17 00:00:00 2001 From: Satnam Singh Date: Mon, 13 Oct 2014 18:46:14 +0000 Subject: [PATCH 4/6] Fix typo in plug-in --- contrib/logging/fluentd-es-image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/logging/fluentd-es-image/Dockerfile b/contrib/logging/fluentd-es-image/Dockerfile index dc649a7c839..e802604d8dd 100644 --- a/contrib/logging/fluentd-es-image/Dockerfile +++ b/contrib/logging/fluentd-es-image/Dockerfile @@ -27,7 +27,7 @@ RUN /usr/bin/curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-t # Needed to allow access to /var/log/docker/... files. RUN sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent -# Install the Elasticsearch Fluentd plug0in. +# Install the Elasticsearch Fluentd plug-in. RUN /usr/sbin/td-agent-gem install fluent-plugin-elasticsearch # Copy the Fluentd configuration file. From c7447a719f1bd1723c35fe813cbc37fe704824da Mon Sep 17 00:00:00 2001 From: Satnam Singh Date: Mon, 13 Oct 2014 18:48:07 +0000 Subject: [PATCH 5/6] Fix tense for run.sh comment --- contrib/logging/fluentd-es-image/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/logging/fluentd-es-image/Dockerfile b/contrib/logging/fluentd-es-image/Dockerfile index e802604d8dd..e4481a95c22 100644 --- a/contrib/logging/fluentd-es-image/Dockerfile +++ b/contrib/logging/fluentd-es-image/Dockerfile @@ -34,8 +34,8 @@ RUN /usr/sbin/td-agent-gem install fluent-plugin-elasticsearch COPY td-agent.conf /etc/td-agent/td-agent.conf # Copy a script that determines the name of the host machine -# and then patches the Fluentd configuration files and then -# runs Fluentd in the foreground. +# and then patch the Fluentd configuration files and then +# run Fluentd in the foreground. ADD run.sh /run.sh # Always run the this setup script. From d140c71a0d967382ee880b4f4f7f982082b976cf Mon Sep 17 00:00:00 2001 From: Satnam Singh Date: Mon, 13 Oct 2014 18:55:58 +0000 Subject: [PATCH 6/6] Confess the horrible hack for outer host --- contrib/logging/fluentd-es-image/run.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/contrib/logging/fluentd-es-image/run.sh b/contrib/logging/fluentd-es-image/run.sh index 4b8e4e8f5e8..17e680afa41 100755 --- a/contrib/logging/fluentd-es-image/run.sh +++ b/contrib/logging/fluentd-es-image/run.sh @@ -14,7 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +# WARNING! HORRIBLE HACK! We expect /outerhost to be mapped to +# the enclosing /etc/host file so we can determine the name of +# the host machine (super fragile). This is a temporary hack until +# service IPs are done. OUTER_HOST=`tail -n 1 /outerhost | awk '{print $3}'` + +# Copy the Fluentd config file and patch it to refer to the +# name of the host machine for ES_HOST. HACK! cp td-agent.conf /etc/td-agent sed -i -e "s/ES_HOST/${OUTER_HOST}/" /etc/td-agent/td-agent.conf /usr/sbin/td-agent