diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile index 731af6845b6..6d74ff28f1b 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile @@ -18,9 +18,9 @@ # Logging API. This configuration assumes that the host performning # the collection is a VM that has been created with a logging.write # scope and that the Logging API has been enabled for the project -# in the Google Developer Console. +# in the Google Developer Console. -FROM ubuntu:14.04 +FROM ubuntu:16.04 MAINTAINER Alex Robinson "arob@google.com" # Disable prompts from apt. @@ -30,17 +30,24 @@ ENV DO_NOT_INSTALL_CATCH_ALL_CONFIG true RUN apt-get -q update && \ apt-get install -y curl && \ + apt-get install -y gcc && \ + apt-get install -y make && \ apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ curl -s https://dl.google.com/cloudagents/install-logging-agent.sh | bash -# Install the record reformer plugin. -RUN /usr/sbin/google-fluentd-gem install fluent-plugin-record-reformer +# Install the record reformer and systemd plugins. +RUN /usr/sbin/google-fluentd-gem install fluent-plugin-record-reformer -v 0.8.1 +RUN /usr/sbin/google-fluentd-gem install fluent-plugin-systemd -v 0.0.3 # Remove the misleading log file that gets generated when the agent is installed RUN rm -rf /var/log/google-fluentd -# Copy the Fluentd configuration file for logging Docker container logs. +# Copy the Fluentd configuration files for logging Docker container logs. +# Either configuration file can be used by specifying `-c ` as a command +# line argument. COPY google-fluentd.conf /etc/google-fluentd/google-fluentd.conf +COPY google-fluentd-journal.conf /etc/google-fluentd/google-fluentd-journal.conf # Start Fluentd to pick up our config that watches Docker container logs. CMD /usr/sbin/google-fluentd "$FLUENTD_ARGS" diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile index 95b77f33c62..755139838bd 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile @@ -28,7 +28,7 @@ .PHONY: kbuild kpush -TAG = 1.20 +TAG = 1.21 # Rules for building the test image for deployment to Dockerhub with user kubernetes. diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd-journal.conf b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd-journal.conf new file mode 100644 index 00000000000..b0a4fd401ab --- /dev/null +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd-journal.conf @@ -0,0 +1,249 @@ +# This configuration file for Fluentd / td-agent is used +# to watch changes to Docker log files that live in the +# directory /var/lib/docker/containers/ and are symbolically +# linked to from the /var/log directory using names that capture the +# pod name and container name. These logs are then submitted to +# Google Cloud Logging which assumes the installation of the cloud-logging plug-in. +# +# This configuration is almost identical to google-fluentd.conf, with the one +# difference being that this collects systemd journal logs. +# +# Example +# ======= +# A line in the Docker log file might like like this JSON: +# +# {"log":"2014/09/25 21:15:03 Got request with path wombat\n", +# "stream":"stderr", +# "time":"2014-09-25T21:15:03.499185026Z"} +# +# The record reformer is used to write the tag to focus on the pod name +# and the Kubernetes container name. For example a Docker container's logs +# might be in the directory: +# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b +# and in the file: +# 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log +# where 997599971ee6... is the Docker ID of the running container. +# The Kubernetes kubelet makes a symbolic link to this file on the host machine +# in the /var/log/containers directory which includes the pod name and the Kubernetes +# container name: +# synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log +# -> +# /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log +# The /var/log directory on the host is mapped to the /var/log directory in the container +# running this instance of Fluentd and we end up collecting the file: +# /var/log/containers/synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log +# This results in the tag: +# var.log.containers.synthetic-logger-0.25lps-pod_default-synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log +# The record reformer is used is discard the var.log.containers prefix and +# the Docker container ID suffix and "kubernetes." is pre-pended giving the +# final tag which is ingested into Elasticsearch: +# kubernetes.synthetic-logger-0.25lps-pod_default-synth-lgr +# This makes it easier for users to search for logs by pod name or by +# the name of the Kubernetes container regardless of how many times the +# Kubernetes pod has been restarted (resulting in a several Docker container IDs). + +# Do not directly collect fluentd's own logs to avoid infinite loops. + + type null + + +# Example: +# {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} + + type tail + format json + time_key time + path /var/log/containers/*.log + pos_file /var/log/gcp-containers.log.pos + time_format %Y-%m-%dT%H:%M:%S.%NZ + tag reform.* + read_from_head true + + + + type record_reformer + enable_ruby true + tag kubernetes.${tag_suffix[4].split('-')[0..-2].join('-')} + + +# Example: +# 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081 + + type tail + format /^(?