diff --git a/cluster/addons/fluentd-elasticsearch/README.md b/cluster/addons/fluentd-elasticsearch/README.md index b1f27d41cb4..065bc6dd750 100644 --- a/cluster/addons/fluentd-elasticsearch/README.md +++ b/cluster/addons/fluentd-elasticsearch/README.md @@ -1,11 +1,45 @@ -# Logging Agent For Elasticsearch -============== +# Elasticsearch Add-On +This add-on consists of a combination of +[Elasticsearch](https://www.elastic.co/products/elasticsearch), [Fluentd](http://www.fluentd.org/) +and [Kibana](https://www.elastic.co/products/elasticsearch). Elasticsearch is a search engine +that is responsible for storing our logs and allowing for them to be queried. Fluentd sends +log messages from Kubernetes to Elasticsearch, whereas Kibana is a graphical interface for +viewing and querying the logs stored in Elasticsearch. -Logging Agent For Elasticsearch is a DaemonSet which spawns a pod on each node -that reads logs, generated by kubelet, container runtime and containers -and sends them to Elasticsearch, deployed in the cluster. Later logs can be -accessed either by querying Elasticsearch directly or by using Kibana. +## Elasticsearch +Elasticsearch is deployed as a +[StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/), which +is like a Deployment, but allows for maintaining state on storage volumes. + +### Authentication +Elasticsearch has basic authentication enabled by default, in our configuration the credentials +are at their default values, i.e. username 'elastic' and password 'changeme'. In order to change +them, please read up on [the official documentation](https://www.elastic.co/guide/en/x-pack/current/setting-up-authentication.html#reset-built-in-user-passwords). + +### Initialization +The Elasticsearch Statefulset manifest specifies that there shall be an +[init container](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) executing +before Elasticsearch containers themselves, in order to ensure that the kernel state variable +`vm.max_map_count` is at least 262144, since this is a requirement of Elasticsearch. +You may remove the init container if you know that your host OS meets this requirement. + +### Storage +The Elasticsearch StatefulSet will claim a storage volume 'elasticsearch-logging', +of the standard +[StorageClass](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#storageclasses), +that by default will be 100 Gi per replica. Please adjust this to your needs (including +possibly choosing a more suitable StorageClass). + +## Fluentd +Fluentd is deployed as a +[DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) which spawns a +pod on each node that reads logs, generated by kubelet, container runtime and containers and +sends them to Elasticsearch. + +*Please note that for Fluentd to work, every Kubernetes node must be labeled* +`beta.kubernetes.io/fluentd-ds-ready=true`, as otherwise Fluentd will ignore them. Learn more at: https://kubernetes.io/docs/tasks/debug-application-cluster/logging-elasticsearch-kibana [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/fluentd-elasticsearch/README.md?pixel)]() + diff --git a/cluster/addons/fluentd-elasticsearch/env-configmap.yaml b/cluster/addons/fluentd-elasticsearch/env-configmap.yaml new file mode 100644 index 00000000000..0ab1075fc85 --- /dev/null +++ b/cluster/addons/fluentd-elasticsearch/env-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: environment + namespace: kube-system +data: + elasticsearch-user: elastic diff --git a/cluster/addons/fluentd-elasticsearch/env-secret.yaml b/cluster/addons/fluentd-elasticsearch/env-secret.yaml new file mode 100644 index 00000000000..9e045135463 --- /dev/null +++ b/cluster/addons/fluentd-elasticsearch/env-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: environment + namespace: kube-system +type: Opaque +data: + elasticsearch-password: Y2hhbmdlbWU= diff --git a/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile index 553ccfdc6c1..a9f2c56cffc 100644 --- a/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile @@ -12,43 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A Dockerfile for creating an Elasticsearch instance that is designed -# to work with Kubernetes logging. Inspired by the Dockerfile -# dockerfile/elasticsearch +FROM docker.elastic.co/elasticsearch/elasticsearch:5.5.1 -FROM java:openjdk-8-jre-alpine +USER root +RUN mkdir /data +RUN chown -R elasticsearch:elasticsearch /data -ENV DEBIAN_FRONTEND noninteractive -ENV ELASTICSEARCH_VERSION 5.4.0 - -RUN apk update && \ - apk --no-cache add \ - --repository https://dl-3.alpinelinux.org/alpine/edge/testing \ - --repository https://dl-3.alpinelinux.org/alpine/edge/community \ - curl \ - shadow \ - tar \ - gosu - -RUN set -x \ - && cd / \ - && mkdir /elasticsearch \ - && curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.tar.gz \ - && tar xf elasticsearch-$ELASTICSEARCH_VERSION.tar.gz -C /elasticsearch --strip-components=1 \ - && rm elasticsearch-$ELASTICSEARCH_VERSION.tar.gz - -COPY config /elasticsearch/config - -COPY run.sh / -COPY elasticsearch_logging_discovery / - -RUN useradd --no-create-home --user-group elasticsearch \ - && mkdir /data \ - && chown -R elasticsearch:elasticsearch /elasticsearch - +WORKDIR /usr/share/elasticsearch VOLUME ["/data"] EXPOSE 9200 9300 -CMD ["/run.sh"] +USER elasticsearch +COPY elasticsearch_logging_discovery bin/ +COPY config/elasticsearch.yml config/ +COPY config/log4j2.properties config/ +COPY run.sh bin/ + +USER root +RUN chown -R elasticsearch:elasticsearch config +CMD ["bin/run.sh"] diff --git a/cluster/addons/fluentd-elasticsearch/es-image/Makefile b/cluster/addons/fluentd-elasticsearch/es-image/Makefile index 5c3b8380a6d..753d09f6cd0 100755 --- a/cluster/addons/fluentd-elasticsearch/es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/es-image/Makefile @@ -16,7 +16,7 @@ # The current value of the tag to be used for building and # pushing an image to gcr.io -TAG = v5.4.0-1 +TAG = v5.5.1 build: elasticsearch_logging_discovery docker build --pull -t gcr.io/google_containers/elasticsearch:$(TAG) . diff --git a/cluster/addons/fluentd-elasticsearch/es-image/config/log4j2.properties b/cluster/addons/fluentd-elasticsearch/es-image/config/log4j2.properties index 23b0fa33c16..f3650c1be63 100644 --- a/cluster/addons/fluentd-elasticsearch/es-image/config/log4j2.properties +++ b/cluster/addons/fluentd-elasticsearch/es-image/config/log4j2.properties @@ -1,4 +1,3 @@ -# you can override this using by setting a system property, for example -Des.logger.level=DEBUG status = error appender.console.type = Console diff --git a/cluster/addons/fluentd-elasticsearch/es-image/run.sh b/cluster/addons/fluentd-elasticsearch/es-image/run.sh index f2e64129e04..eb1f2f70f8f 100755 --- a/cluster/addons/fluentd-elasticsearch/es-image/run.sh +++ b/cluster/addons/fluentd-elasticsearch/es-image/run.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -sysctl -w vm.max_map_count=262144 +set -ex export NODE_NAME=${NODE_NAME:-${HOSTNAME}} export NODE_MASTER=${NODE_MASTER:-true} @@ -23,8 +23,7 @@ export HTTP_PORT=${HTTP_PORT:-9200} export TRANSPORT_PORT=${TRANSPORT_PORT:-9300} export MINIMUM_MASTER_NODES=${MINIMUM_MASTER_NODES:-2} -/elasticsearch_logging_discovery >> /elasticsearch/config/elasticsearch.yml - chown -R elasticsearch:elasticsearch /data -exec gosu elasticsearch sh /elasticsearch/bin/elasticsearch +./bin/elasticsearch_logging_discovery >> ./config/elasticsearch.yml +exec su elasticsearch -c ./bin/es-docker diff --git a/cluster/addons/fluentd-elasticsearch/es-controller.yaml b/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml similarity index 61% rename from cluster/addons/fluentd-elasticsearch/es-controller.yaml rename to cluster/addons/fluentd-elasticsearch/es-statefulset.yaml index 37aadaf8e23..85256106794 100644 --- a/cluster/addons/fluentd-elasticsearch/es-controller.yaml +++ b/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml @@ -1,5 +1,5 @@ -apiVersion: v1 -kind: ReplicationController +apiVersion: apps/v1beta1 +kind: StatefulSet metadata: name: elasticsearch-logging-v1 namespace: kube-system @@ -9,10 +9,12 @@ metadata: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: + serviceName: elasticsearch-logging replicas: 2 selector: - k8s-app: elasticsearch-logging - version: v1 + matchLabels: + k8s-app: elasticsearch-logging + version: v1 template: metadata: labels: @@ -22,7 +24,7 @@ spec: spec: serviceAccountName: elasticsearch-logging containers: - - image: gcr.io/google_containers/elasticsearch:v2.4.1-2 + - image: gcr.io/google_containers/elasticsearch:v5.5.1 name: elasticsearch-logging resources: # need more cpu upon initialization, therefore burstable class @@ -38,13 +40,24 @@ spec: name: transport protocol: TCP volumeMounts: - - name: es-persistent-storage + - name: elasticsearch-logging mountPath: /data env: - name: "NAMESPACE" valueFrom: fieldRef: fieldPath: metadata.namespace - volumes: - - name: es-persistent-storage - emptyDir: {} + initContainers: + - image: alpine:3.6 + command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"] + name: elasticsearch-logging-init + securityContext: + privileged: true + volumeClaimTemplates: + - metadata: + name: elasticsearch-logging + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 100Gi diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index c9587bfcdd8..035297f4b5d 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -32,6 +32,16 @@ spec: env: - name: FLUENTD_ARGS value: -q + - name: FLUENT_ELASTICSEARCH_USER + valueFrom: + configMapKeyRef: + name: environment + key: elasticsearch-user + - name: FLUENT_ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: environment + key: elasticsearch-password resources: limits: memory: 200Mi diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/README.md b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/README.md index f15dfa872fd..1942b77996e 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/README.md +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/README.md @@ -7,4 +7,4 @@ cluster bring up process. The image resides at DockerHub under the name [kubernetes/fluentd-elasticsearch](https://registry.hub.docker.com/u/kubernetes/fluentd-elasticsearch/). -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/fluentd-elasticsearch/fluentd-es-image/README.md?pixel)]() +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/fluentd-elasticsearch/fluentd-es-image/README.md?pixel)]() \ No newline at end of file diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/build.sh b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/build.sh index 924b7c83a4d..4c0339ca75f 100755 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/build.sh +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/build.sh @@ -42,7 +42,7 @@ rm -rf /opt/td-agent/embedded/share/doc \ /opt/td-agent/embedded/share/postgresql apt-get remove -y make g++ -apt-get autoremove -y -apt-get clean -y +apt-get autoremove -y +apt-get clean -y rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf index 0b42639f752..bf2c0821554 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf @@ -53,7 +53,7 @@ # in the /var/log/containers directory which includes the pod name and the Kubernetes # container name: # -# synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log +# synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log # -> # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log # @@ -323,6 +323,8 @@ type elasticsearch + user "#{ENV['FLUENT_ELASTICSEARCH_USER']}" + password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}" log_level info include_tag_key true host elasticsearch-logging diff --git a/cluster/addons/fluentd-elasticsearch/kibana-controller.yaml b/cluster/addons/fluentd-elasticsearch/kibana-deployment.yaml similarity index 90% rename from cluster/addons/fluentd-elasticsearch/kibana-controller.yaml rename to cluster/addons/fluentd-elasticsearch/kibana-deployment.yaml index bc7725686d0..fcd74865c55 100644 --- a/cluster/addons/fluentd-elasticsearch/kibana-controller.yaml +++ b/cluster/addons/fluentd-elasticsearch/kibana-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: kibana-logging - image: gcr.io/google_containers/kibana:v4.6.1-1 + image: docker.elastic.co/kibana/kibana:5.5.1 resources: # keep request = limit to keep this container in guaranteed class limits: @@ -29,7 +29,7 @@ spec: env: - name: "ELASTICSEARCH_URL" value: "http://elasticsearch-logging:9200" - - name: "KIBANA_BASE_URL" + - name: "SERVER_BASEPATH" value: "/api/v1/proxy/namespaces/kube-system/services/kibana-logging" ports: - containerPort: 5601 diff --git a/cluster/addons/fluentd-elasticsearch/kibana-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/kibana-image/Dockerfile deleted file mode 100644 index 9a82060ee51..00000000000 --- a/cluster/addons/fluentd-elasticsearch/kibana-image/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A Dockerfile for creating a Kibana container that is designed -# to work with Kubernetes logging. - -FROM gcr.io/google_containers/ubuntu-slim:0.6 - - -ENV DEBIAN_FRONTEND noninteractive -ENV KIBANA_VERSION 5.4.0 - -RUN apt-get update \ - && apt-get install -y curl \ - && apt-get clean - -RUN set -x \ - && cd / \ - && mkdir /kibana \ - && curl -O https://artifacts.elastic.co/downloads/kibana/kibana-$KIBANA_VERSION-linux-x86_64.tar.gz \ - && tar xf kibana-$KIBANA_VERSION-linux-x86_64.tar.gz -C /kibana --strip-components=1 \ - && rm kibana-$KIBANA_VERSION-linux-x86_64.tar.gz - -COPY run.sh /run.sh - -EXPOSE 5601 - -CMD ["/run.sh"] diff --git a/cluster/addons/fluentd-elasticsearch/kibana-image/Makefile b/cluster/addons/fluentd-elasticsearch/kibana-image/Makefile deleted file mode 100755 index cdd1fc85e20..00000000000 --- a/cluster/addons/fluentd-elasticsearch/kibana-image/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -.PHONY: build push - -TAG = v5.4.0 -PREFIX = gcr.io/google_containers - -build: - docker build --pull -t $(PREFIX)/kibana:$(TAG) . - -push: - gcloud docker -- push $(PREFIX)/kibana:$(TAG) diff --git a/cluster/addons/fluentd-elasticsearch/kibana-image/run.sh b/cluster/addons/fluentd-elasticsearch/kibana-image/run.sh deleted file mode 100755 index 2662ca92683..00000000000 --- a/cluster/addons/fluentd-elasticsearch/kibana-image/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -export ELASTICSEARCH_URL=${ELASTICSEARCH_URL:-"http://localhost:9200"} -echo ELASTICSEARCH_URL=${ELASTICSEARCH_URL} - -export KIBANA_BASE_URL=${KIBANA_BASE_URL:-"''"} -echo "server.basePath: ${KIBANA_BASE_URL}" -echo "server.basePath: ${KIBANA_BASE_URL}" >> /kibana/config/kibana.yml -echo "server.host: ${KIBANA_HOST}" -echo "server.host: ${KIBANA_HOST}" >> /kibana/config/kibana.yml - -/kibana/bin/kibana -e ${ELASTICSEARCH_URL}