From a7ed23be8e95a8db59e6298d63cdc8cdd9d481b2 Mon Sep 17 00:00:00 2001 From: Jan Klare Date: Fri, 20 Apr 2018 15:44:19 +0200 Subject: [PATCH] update fluentd-elasticsearch addon * elastic now provides a fully opensource version for their prebuild docker images (elasticsearch, kibana and so on). To avoid running into licensing conflicts for this addon example, we should rather use these images instead of the premium ones (were we also have to disable premium features manually right now) * remove disable flags for xpack, since *-oss images do not include this anymore * bump elasticsearch and kibana version from 5.6.4 to 6.2.4 * use oss version from elastic as baseimg for kibana and elasticsearch * bump fluentd version to ~>1.1.3 * bump gem 'fluent-plugin-elasticsearch' to '~>2.9.1' to allow usage of elasticsearch 6.x * bump fluentd-es-image to v2.1.0 * fix elasticserach run.sh to align with new elasticsearch upstream container structure --- .../addons/fluentd-elasticsearch/README.md | 19 ++++++++++--------- .../fluentd-elasticsearch/es-image/Dockerfile | 2 +- .../fluentd-elasticsearch/es-image/Makefile | 2 +- .../es-image/config/elasticsearch.yml | 3 --- .../fluentd-elasticsearch/es-image/run.sh | 2 +- .../fluentd-elasticsearch/es-statefulset.yaml | 8 ++++---- .../fluentd-elasticsearch/fluentd-es-ds.yaml | 10 +++++----- .../fluentd-es-image/Gemfile | 4 ++-- .../fluentd-es-image/Makefile | 2 +- .../kibana-deployment.yaml | 6 +----- 10 files changed, 26 insertions(+), 32 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/README.md b/cluster/addons/fluentd-elasticsearch/README.md index d51b3b142d2..82f50df5911 100644 --- a/cluster/addons/fluentd-elasticsearch/README.md +++ b/cluster/addons/fluentd-elasticsearch/README.md @@ -19,15 +19,16 @@ a Deployment, but allows for maintaining state on storage volumes. ### Security -Elasticsearch has capabilities to enable authorization using the -[X-Pack plugin][xPack]. See configuration parameter `xpack.security.enabled` -in Elasticsearch and Kibana configurations. It can also be set via the -`XPACK_SECURITY_ENABLED` env variable. After enabling the feature, -follow [official documentation][setupCreds] to set up credentials in -Elasticsearch and Kibana. Don't forget to propagate those credentials also to -Fluentd in its [configuration][fluentdCreds], using for example -[environment variables][fluentdEnvVar]. You can utilize [ConfigMaps][configMap] -and [Secrets][secret] to store credentials in the Kubernetes apiserver. +Elasticsearch has capabilities to enable authorization using the [X-Pack +plugin][xPack]. For the sake of simplicity this example uses the fully open +source prebuild images from elastic that do not contain the X-Pack plugin. If +you need these features, please consider building the images from either the +"basic" or "platinum" version. After enabling these features, follow [official +documentation][setupCreds] to set up credentials in Elasticsearch and Kibana. +Don't forget to propagate those credentials also to Fluentd in its +[configuration][fluentdCreds], using for example [environment +variables][fluentdEnvVar]. You can utilize [ConfigMaps][configMap] and +[Secrets][secret] to store credentials in the Kubernetes apiserver. ### Initialization diff --git a/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile index 8e971d78349..de69bcf9f4d 100644 --- a/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.4 +FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4 VOLUME ["/data"] EXPOSE 9200 9300 diff --git a/cluster/addons/fluentd-elasticsearch/es-image/Makefile b/cluster/addons/fluentd-elasticsearch/es-image/Makefile index be7ba47df18..3067c746121 100755 --- a/cluster/addons/fluentd-elasticsearch/es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/es-image/Makefile @@ -16,7 +16,7 @@ PREFIX = staging-k8s.gcr.io IMAGE = elasticsearch -TAG = v5.6.4 +TAG = v6.2.4 build: docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) . diff --git a/cluster/addons/fluentd-elasticsearch/es-image/config/elasticsearch.yml b/cluster/addons/fluentd-elasticsearch/es-image/config/elasticsearch.yml index 23c59c53b79..f4ffee74a7f 100644 --- a/cluster/addons/fluentd-elasticsearch/es-image/config/elasticsearch.yml +++ b/cluster/addons/fluentd-elasticsearch/es-image/config/elasticsearch.yml @@ -12,6 +12,3 @@ path.data: /data network.host: 0.0.0.0 discovery.zen.minimum_master_nodes: ${MINIMUM_MASTER_NODES} - -xpack.security.enabled: false -xpack.monitoring.enabled: false diff --git a/cluster/addons/fluentd-elasticsearch/es-image/run.sh b/cluster/addons/fluentd-elasticsearch/es-image/run.sh index eb1f2f70f8f..e6d3d24c6c7 100755 --- a/cluster/addons/fluentd-elasticsearch/es-image/run.sh +++ b/cluster/addons/fluentd-elasticsearch/es-image/run.sh @@ -26,4 +26,4 @@ export MINIMUM_MASTER_NODES=${MINIMUM_MASTER_NODES:-2} chown -R elasticsearch:elasticsearch /data ./bin/elasticsearch_logging_discovery >> ./config/elasticsearch.yml -exec su elasticsearch -c ./bin/es-docker +exec su elasticsearch -c /usr/local/bin/docker-entrypoint.sh diff --git a/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml b/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml index 36adf388313..a6178c8a5e8 100644 --- a/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml +++ b/cluster/addons/fluentd-elasticsearch/es-statefulset.yaml @@ -54,7 +54,7 @@ metadata: namespace: kube-system labels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.2.4 kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: @@ -63,17 +63,17 @@ spec: selector: matchLabels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.2.4 template: metadata: labels: k8s-app: elasticsearch-logging - version: v5.6.4 + version: v6.2.4 kubernetes.io/cluster-service: "true" spec: serviceAccountName: elasticsearch-logging containers: - - image: k8s.gcr.io/elasticsearch:v5.6.4 + - image: k8s.gcr.io/elasticsearch:v6.2.4 name: elasticsearch-logging resources: # need more cpu upon initialization, therefore burstable class diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index eecd6b4801b..c39cb5958d7 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -48,24 +48,24 @@ roleRef: apiVersion: apps/v1 kind: DaemonSet metadata: - name: fluentd-es-v2.0.4 + name: fluentd-es-v2.1.0 namespace: kube-system labels: k8s-app: fluentd-es - version: v2.0.4 + version: v2.1.0 kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: fluentd-es - version: v2.0.4 + version: v2.1.0 template: metadata: labels: k8s-app: fluentd-es kubernetes.io/cluster-service: "true" - version: v2.0.4 + version: v2.1.0 # This annotation ensures that fluentd does not get evicted if the node # supports critical pod annotation based priority scheme. # Note that this does not guarantee admission on the nodes (#40573). @@ -76,7 +76,7 @@ spec: serviceAccountName: fluentd-es containers: - name: fluentd-es - image: k8s.gcr.io/fluentd-elasticsearch:v2.0.4 + image: k8s.gcr.io/fluentd-elasticsearch:v2.1.0 env: - name: FLUENTD_ARGS value: --no-supervisor -q diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile index 93155aa4fc6..f97ad5fadb4 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile @@ -1,9 +1,9 @@ source 'https://rubygems.org' -gem 'fluentd', '<=1.1.0' +gem 'fluentd', '~>1.1.3' gem 'activesupport', '~>5.1.4' gem 'fluent-plugin-kubernetes_metadata_filter', '~>1.0.0' -gem 'fluent-plugin-elasticsearch', '~>2.4.1' +gem 'fluent-plugin-elasticsearch', '~>2.9.1' gem 'fluent-plugin-systemd', '~>0.3.1' gem 'fluent-plugin-detect-exceptions', '~>0.0.9' gem 'fluent-plugin-prometheus', '~>0.3.0' diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile index 59b9f155c2b..bb0680cc4f1 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile @@ -16,7 +16,7 @@ PREFIX = staging-k8s.gcr.io IMAGE = fluentd-elasticsearch -TAG = v2.0.4 +TAG = v2.1.0 build: docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) . diff --git a/cluster/addons/fluentd-elasticsearch/kibana-deployment.yaml b/cluster/addons/fluentd-elasticsearch/kibana-deployment.yaml index 8cd00fe52e5..c775cd9c68e 100644 --- a/cluster/addons/fluentd-elasticsearch/kibana-deployment.yaml +++ b/cluster/addons/fluentd-elasticsearch/kibana-deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: kibana-logging - image: docker.elastic.co/kibana/kibana:5.6.4 + image: docker.elastic.co/kibana/kibana-oss:6.2.4 resources: # need more cpu upon initialization, therefore burstable class limits: @@ -31,10 +31,6 @@ spec: value: http://elasticsearch-logging:9200 - name: SERVER_BASEPATH value: /api/v1/namespaces/kube-system/services/kibana-logging/proxy - - name: XPACK_MONITORING_ENABLED - value: "false" - - name: XPACK_SECURITY_ENABLED - value: "false" ports: - containerPort: 5601 name: ui