diff --git a/contrib/logging/fluentd-es-image/Dockerfile b/contrib/logging/fluentd-es-image/Dockerfile new file mode 100644 index 00000000000..ff6b9a1e344 --- /dev/null +++ b/contrib/logging/fluentd-es-image/Dockerfile @@ -0,0 +1,43 @@ +# This Dockerfile will build an image that is configured +# to run Fluentd with an Elasticsearch plug-in and the +# provided configuration file. +# TODO(satnam): Use a lighter base image, e.g. some form of busybox. +# The image acts as an executable for the binary /usr/sbin/td-agent +# which runs fluentd with the default flag -v (which can be over-ridden). +# Note that fluentd is run with root permssion to allow access to +# log files with root only access under /var/lib/docker/containers/* +# Please see http://docs.fluentd.org/articles/install-by-deb for more +# information about installing fluentd using deb package. + +FROM ubuntu:14.04 +MAINTAINER Satnam Singh "satnam@google.com" + +# Ensure there are enough file descriptors for running Fluentd. +RUN ulimit -n 65536 + +# Install prerequisites. +RUN apt-get update && \ + apt-get install -y curl && \ + apt-get install -y -q libcurl4-openssl-dev make && \ + apt-get clean + +# Install Fluentd. +RUN /usr/bin/curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sh + +# Change the default user and group to root. +# Needed to allow access to /var/log/docker/... files. +RUN sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent + +# Install the Elasticsearch Fluentd plug0in. +RUN /usr/sbin/td-agent-gem install fluent-plugin-elasticsearch + +# Copy the Fluentd configuration file. +COPY td-agent.conf /etc/td-agent/td-agent.conf + +# Copy a script that determines the name of the host machine +# and then patches the Fluentd configuration files and then +# runs Fluentd in the foreground. +ADD run.sh /run.sh + +# Always run the this setup script. +ENTRYPOINT ["/run.sh"] diff --git a/contrib/logging/fluentd-es-image/build.sh b/contrib/logging/fluentd-es-image/build.sh new file mode 100755 index 00000000000..6810cba13bd --- /dev/null +++ b/contrib/logging/fluentd-es-image/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker build -t google/fluentd-elasticsearch . +docker push google/lambda-fluentd \ No newline at end of file diff --git a/contrib/logging/fluentd-es-image/run.sh b/contrib/logging/fluentd-es-image/run.sh new file mode 100755 index 00000000000..4b8e4e8f5e8 --- /dev/null +++ b/contrib/logging/fluentd-es-image/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OUTER_HOST=`tail -n 1 /outerhost | awk '{print $3}'` +cp td-agent.conf /etc/td-agent +sed -i -e "s/ES_HOST/${OUTER_HOST}/" /etc/td-agent/td-agent.conf +/usr/sbin/td-agent diff --git a/contrib/logging/fluentd-es-image/td-agent.conf b/contrib/logging/fluentd-es-image/td-agent.conf new file mode 100644 index 00000000000..08bb16b9308 --- /dev/null +++ b/contrib/logging/fluentd-es-image/td-agent.conf @@ -0,0 +1,53 @@ +# This configuration file for Fluentd / td-agent is used +# to watch changes to Docker log files that live in the +# directory /var/lib/docker/containers/ which are then submitted to +# Elasticsearch (running on the machine ES_HOST:9200) which +# assumes the installation of the fluentd-elasticsearch plug-in. +# See https://github.com/uken/fluent-plugin-elasticsearch for +# more information about the plug-in. This file needs to be +# patched to replace ES_HOST with the name of the actual +# machine running Elasticsearch. +# Maintainer: Satnam Singh (satnam@google.com) +# +# Exampe +# ====== +# A line in the Docker log file might like like this JSON: +# +# {"log":"2014/09/25 21:15:03 Got request with path wombat\n", +# "stream":"stderr", +# "time":"2014-09-25T21:15:03.499185026Z"} +# +# The time_format specification below makes sure we properly +# parse the time format produced by Docker. This will be +# submitted to Elasticsearch and should appear like: +# $ curl 'http://elasticsearch:9200/_search?pretty' +# ... +# { +# "_index" : "logstash-2014.09.25", +# "_type" : "fluentd", +# "_id" : "VBrbor2QTuGpsQyTCdfzqA", +# "_score" : 1.0, +# "_source":{"log":"2014/09/25 22:45:50 Got request with path wombat\n", +# "stream":"stderr","tag":"docker.container.all", +# "@timestamp":"2014-09-25T22:45:50+00:00"} +# }, +# ... + + + type tail + format json + time_key time + path /var/lib/docker/containers/*/*-json.log + time_format %Y-%m-%dT%H:%M:%S + tag docker.container.all + + + + type elasticsearch + log_level info + include_tag_key true + host ES_HOST + port 9200 + logstash_format true + flush_interval 5s +