diff --git a/contrib/logging/fluentd-sidecar-gcp/Dockerfile b/contrib/logging/fluentd-sidecar-gcp/Dockerfile new file mode 100644 index 00000000000..dbd276c3f59 --- /dev/null +++ b/contrib/logging/fluentd-sidecar-gcp/Dockerfile @@ -0,0 +1,35 @@ +# This Dockerfile will build an image that is configured to use Fluentd to +# collect container log files from the specified paths and send them to the +# Google Cloud Logging API. +# The environment variable that controls which log files are collected is +# FILES_TO_COLLECT. Files specified in the environment variable should be +# separated by whitespace, as in "/var/log/syslog /var/log/nginx/access.log". +# This configuration assumes that the host performning the collection is a VM +# that has been created with a logging.write scope and that the Logging API +# has been enabled for the project in the Google Developer Console. + +FROM ubuntu:14.04 +MAINTAINER Alex Robinson "arob@google.com" + +# Disable prompts from apt. +ENV DEBIAN_FRONTEND noninteractive +ENV OPTS_APT -y --force-yes --no-install-recommends + +# Install the Fluentd agent that knows how to send logs to Google Cloud Logging. +RUN apt-get -q update && \ + apt-get -y install curl && \ + apt-get clean && \ + curl -s https://storage.googleapis.com/signals-agents/logging/google-fluentd-install.sh | sudo bash + +# Copy the configuration file generator for creating input configurations for +# each file specified in the FILES_TO_COLLECT environment variable. +COPY config_generator.sh /usr/local/sbin/config_generator.sh + +# Copy the Fluentd configuration file for collecting from all the inputs +# generated by the config generator and sending them to Google Cloud Logging. +COPY google-fluentd.conf /etc/google-fluentd/google-fluentd.conf + +# Run the config generator to get the config files in place and start Fluentd. +# We have to run the config generator at runtime rather than now so that it can +# incorporate the files provided in the environment variable in its config. +CMD /usr/local/sbin/config_generator.sh && /usr/sbin/google-fluentd -qq --use-v1-config --suppress-repeated-stacktrace > /var/log/google-fluentd/google-fluentd.log diff --git a/contrib/logging/fluentd-sidecar-gcp/Makefile b/contrib/logging/fluentd-sidecar-gcp/Makefile new file mode 100644 index 00000000000..3b6dd3be938 --- /dev/null +++ b/contrib/logging/fluentd-sidecar-gcp/Makefile @@ -0,0 +1,9 @@ +.PHONY: build push + +TAG = 1.0 + +build: + docker build -t gcr.io/google_containers/fluentd-sidecar-gcp:$(TAG) . + +push: + gcloud preview docker push gcr.io/google_containers/fluentd-sidecar-gcp:$(TAG) diff --git a/contrib/logging/fluentd-sidecar-gcp/README.md b/contrib/logging/fluentd-sidecar-gcp/README.md new file mode 100644 index 00000000000..fb891b19886 --- /dev/null +++ b/contrib/logging/fluentd-sidecar-gcp/README.md @@ -0,0 +1,8 @@ +# Collecting log files from within containers with Fluentd and sending to the Google Cloud Logging service. +This directory contains the source files needed to make a Docker image that collects log files from arbitrary files within a container using [Fluentd](http://www.fluentd.org/) and sends them to GCP. +This image is designed to be used as a sidecar container as part of a [Kubernetes](https://github.com/GoogleCloudPlatform/kubernetes) pod. +The image resides at DockerHub under the name +[kubernetes/fluentd-sidecar-gcp](https://registry.hub.docker.com/u/kubernetes/fluentd-sidecar-gcp/). + +# TODO: Add example pod config. +# TODO: say that it resides at gcr.io instead? diff --git a/contrib/logging/fluentd-sidecar-gcp/config_generator.sh b/contrib/logging/fluentd-sidecar-gcp/config_generator.sh new file mode 100755 index 00000000000..a39ab636fa6 --- /dev/null +++ b/contrib/logging/fluentd-sidecar-gcp/config_generator.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mkdir -p /etc/google-fluentd/files +if [ -z "$FILES_TO_COLLECT" ]; then + exit 0 +fi + +for filepath in $FILES_TO_COLLECT +do + filename=$(basename $filepath) + cat > "/etc/google-fluentd/files/${filename}" << EndOfMessage + + type tail + format none + time_key time + path ${filepath} + pos_file /etc/google-fluentd/fluentd-gcp.log.pos + time_format %Y-%m-%dT%H:%M:%S + tag file.${filename} + read_from_head true + +EndOfMessage +done diff --git a/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf b/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf new file mode 100644 index 00000000000..1bbbfe73a63 --- /dev/null +++ b/contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf @@ -0,0 +1,26 @@ +# This Fluentd configuration file enables the collection of log files +# that can be specified at the time of its creation in an environment +# variable, assuming that the config_generator.sh script runs to generate +# a configuration file for each log file to collect. +# Logs collected will be sent to the Google Cloud Logging API. +# +# Currently the collector uses a text format rather than allowing the user +# to specify how to parse each file. +# +# This configuration assumes the correct installation of the the Google +# Fluentd plug-in, and that the VM host running this configuration is on +# Google Compute Engine and has been created with the logging.write scope. + +# Pick up all the auto-generated input config files, one for each file +# specified in the FILES_TO_COLLECT environment variable. +@include files/* + +# All the auto-generated files should use the tag "file.". + + type google_cloud + flush_interval 5s + # Never wait longer than 5 minutes between retries. + max_retry_wait 300 + # Disable the limit on the number of retries (retry forever). + disable_retry_limit +