diff --git a/contrib/logging/fluentd-gcp-image/Dockerfile b/contrib/logging/fluentd-gcp-image/Dockerfile new file mode 100644 index 00000000000..c7339a16dc4 --- /dev/null +++ b/contrib/logging/fluentd-gcp-image/Dockerfile @@ -0,0 +1,36 @@ +# This Dockerfile will build an image that is configured +# to use Fluentd to collect all Docker container log files +# and then cause them to be ingested using the Google Cloud +# Logging API. This configuration assumes that the host performning +# the collection is a VM that has been created with a logging.write +# scope. + +FROM ubuntu:14.04 +MAINTAINER Satnam Singh "satnam@google.com" + +# Ensure there are enough file descriptors for running Fluentd. +RUN ulimit -n 65536 + +# Install prerequisites. +RUN apt-get update && \ + apt-get install -y curl && \ + apt-get install -y -q libcurl4-openssl-dev make && \ + apt-get clean + +# Install Fluentd. +RUN /usr/bin/curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sh + +# Change the default user and group to root. +# Needed to allow access to /var/log/docker/... files. +RUN sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent +# Supress use of V1 config. +RUN sed -i~ -e 's/ --use-v1-config//' /etc/init.d/td-agent + +# Install GCP logging plug-in for Fluentd. +RUN gsutil cp gs://signals-agents/out_google_cloud.rb /etc/td-agent/plugin + +# Copy the Fluentd configuration file. +COPY td-agent.conf /etc/td-agent/td-agent.conf + +# Run Fluentd in the foreground. +ENTRYPOINT ["/usr/sbin/td-agent"] diff --git a/contrib/logging/fluentd-gcp-image/README.md b/contrib/logging/fluentd-gcp-image/README.md new file mode 100644 index 00000000000..6153b489a43 --- /dev/null +++ b/contrib/logging/fluentd-gcp-image/README.md @@ -0,0 +1,8 @@ +# Collecting Docker Log Files with Fluentd and sending to GCP. +This directory contains the source files needed to make a Docker image +that collects Docker container log files using [Fluentd](http://www.fluentd.org/) +and sends them to GCP. +This image is designed to be used as part of the [Kubernetes](https://github.com/GoogleCloudPlatform/kubernetes) +cluster bring up process. The image resides at DockerHub under the name +[kubernetes/fluentd-gcp](https://registry.hub.docker.com/u/kubernetes/fluentd-gcp/). + diff --git a/contrib/logging/fluentd-gcp-image/build.sh b/contrib/logging/fluentd-gcp-image/build.sh new file mode 100644 index 00000000000..7d162d3adc3 --- /dev/null +++ b/contrib/logging/fluentd-gcp-image/build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Build the fluentd-gcp image. +sudo docker build -t kubernetes/fluentd-gcp . diff --git a/contrib/logging/fluentd-gcp-image/push.sh b/contrib/logging/fluentd-gcp-image/push.sh new file mode 100755 index 00000000000..611c2781b09 --- /dev/null +++ b/contrib/logging/fluentd-gcp-image/push.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sudo docker push kubernetes/fluentd-gcp diff --git a/contrib/logging/fluentd-gcp-image/td-agent.conf b/contrib/logging/fluentd-gcp-image/td-agent.conf new file mode 100644 index 00000000000..765e6bac3ab --- /dev/null +++ b/contrib/logging/fluentd-gcp-image/td-agent.conf @@ -0,0 +1,31 @@ +# This Fluentd configuration file specifies the colleciton +# of all Docker container log files under /var/lib/docker/containers/... +# followed by ingestion using the Google Cloud Logging API. +# This configuration assumes the correct installation of the the +# Google fluentd plug-in. Currently the collector uses a text format +# rather than JOSN (which is the format used to store the Docker +# log files). When the fluentd plug-in can accept JSON this +# configuraiton file should be changed by specifying: +# format json +# in the source section. +# This configuration file assumes that the VM host running +# this configuraiton has been created with a logging.write scope. +# Maintainer: Satnam Singh (satnam@google.com) + + + type tail + format none + time_key time + path /var/lib/docker/containers/*/*-json.log + time_format %Y-%m-%dT%H:%M:%S + tag docker.container.* + + + + type google_cloud + flush_interval 5s + # Never wait longer than 5 minutes between retries. + max_retry_wait 300 + # Disable the limit on the number of retries (retry forever). + disable_retry_limit +