update fluentd-elasticsearch addon

* elastic now provides a fully opensource version for their prebuild
docker images (elasticsearch, kibana and so on). To avoid running into
licensing conflicts for this addon example, we should rather use these
images instead of the premium ones (were we also have to disable premium
features manually right now)
* remove disable flags for xpack, since *-oss images do not include this anymore
* bump elasticsearch and kibana version from 5.6.4 to 6.2.4
* use oss version from elastic as baseimg for kibana and elasticsearch
* bump fluentd version to ~>1.1.3
* bump gem 'fluent-plugin-elasticsearch' to '~>2.9.1' to allow usage of elasticsearch 6.x
* bump fluentd-es-image to v2.1.0
* fix elasticserach run.sh to align with new elasticsearch upstream container structure

cluster/addons/fluentd-elasticsearch/README.md

@@ -19,15 +19,16 @@ a Deployment, but allows for maintaining state on storage volumes.
 
 ### Security
 
-Elasticsearch has capabilities to enable authorization using the
+Elasticsearch has capabilities to enable authorization using the [X-Pack
-[X-Pack plugin][xPack]. See configuration parameter `xpack.security.enabled`
+plugin][xPack]. For the sake of simplicity this example uses the fully open
-in Elasticsearch and Kibana configurations. It can also be set via the
+source prebuild images from elastic that do not contain the X-Pack plugin. If
-`XPACK_SECURITY_ENABLED` env variable. After enabling the feature,
+you need these features, please consider building the images from either the
-follow [official documentation][setupCreds] to set up credentials in
+"basic" or "platinum" version. After enabling these features, follow [official
-Elasticsearch and Kibana. Don't forget to propagate those credentials also to
+documentation][setupCreds] to set up credentials in Elasticsearch and Kibana.
-Fluentd in its [configuration][fluentdCreds], using for example
+Don't forget to propagate those credentials also to Fluentd in its
-[environment variables][fluentdEnvVar]. You can utilize [ConfigMaps][configMap]
+[configuration][fluentdCreds], using for example [environment
-and [Secrets][secret] to store credentials in the Kubernetes apiserver.
+variables][fluentdEnvVar]. You can utilize [ConfigMaps][configMap] and
+[Secrets][secret] to store credentials in the Kubernetes apiserver.
 
 ### Initialization
 

cluster/addons/fluentd-elasticsearch/es-image/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.4
+FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4
 
 VOLUME ["/data"]
 EXPOSE 9200 9300

cluster/addons/fluentd-elasticsearch/es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = staging-k8s.gcr.io
 IMAGE = elasticsearch
-TAG = v5.6.4
+TAG = v6.2.4
 
 build:
 	docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .

cluster/addons/fluentd-elasticsearch/es-image/config/elasticsearch.yml

@@ -12,6 +12,3 @@ path.data: /data
 network.host: 0.0.0.0
 
 discovery.zen.minimum_master_nodes: ${MINIMUM_MASTER_NODES}
-
-xpack.security.enabled: false
-xpack.monitoring.enabled: false

cluster/addons/fluentd-elasticsearch/es-image/run.sh

@@ -26,4 +26,4 @@ export MINIMUM_MASTER_NODES=${MINIMUM_MASTER_NODES:-2}
 chown -R elasticsearch:elasticsearch /data
 
 ./bin/elasticsearch_logging_discovery >> ./config/elasticsearch.yml
-exec su elasticsearch -c ./bin/es-docker
+exec su elasticsearch -c /usr/local/bin/docker-entrypoint.sh

cluster/addons/fluentd-elasticsearch/es-statefulset.yaml

@@ -54,7 +54,7 @@ metadata:
   namespace: kube-system
   labels:
     k8s-app: elasticsearch-logging
-    version: v5.6.4
+    version: v6.2.4
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
@@ -63,17 +63,17 @@ spec:
   selector:
     matchLabels:
       k8s-app: elasticsearch-logging
-      version: v5.6.4
+      version: v6.2.4
   template:
     metadata:
       labels:
         k8s-app: elasticsearch-logging
-        version: v5.6.4
+        version: v6.2.4
         kubernetes.io/cluster-service: "true"
     spec:
       serviceAccountName: elasticsearch-logging
       containers:
-      - image: k8s.gcr.io/elasticsearch:v5.6.4
+      - image: k8s.gcr.io/elasticsearch:v6.2.4
         name: elasticsearch-logging
         resources:
           # need more cpu upon initialization, therefore burstable class

cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml

@@ -48,24 +48,24 @@ roleRef:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: fluentd-es-v2.0.4
+  name: fluentd-es-v2.1.0
   namespace: kube-system
   labels:
     k8s-app: fluentd-es
-    version: v2.0.4
+    version: v2.1.0
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
       k8s-app: fluentd-es
-      version: v2.0.4
+      version: v2.1.0
   template:
     metadata:
       labels:
         k8s-app: fluentd-es
         kubernetes.io/cluster-service: "true"
-        version: v2.0.4
+        version: v2.1.0
       # This annotation ensures that fluentd does not get evicted if the node
       # supports critical pod annotation based priority scheme.
       # Note that this does not guarantee admission on the nodes (#40573).
@@ -76,7 +76,7 @@ spec:
       serviceAccountName: fluentd-es
       containers:
       - name: fluentd-es
-        image: k8s.gcr.io/fluentd-elasticsearch:v2.0.4
+        image: k8s.gcr.io/fluentd-elasticsearch:v2.1.0
         env:
         - name: FLUENTD_ARGS
           value: --no-supervisor -q

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Gemfile

@@ -1,9 +1,9 @@
 source 'https://rubygems.org'
 
-gem 'fluentd', '<=1.1.0'
+gem 'fluentd', '~>1.1.3'
 gem 'activesupport', '~>5.1.4'
 gem 'fluent-plugin-kubernetes_metadata_filter', '~>1.0.0'
-gem 'fluent-plugin-elasticsearch', '~>2.4.1'
+gem 'fluent-plugin-elasticsearch', '~>2.9.1'
 gem 'fluent-plugin-systemd', '~>0.3.1'
 gem 'fluent-plugin-detect-exceptions', '~>0.0.9'
 gem 'fluent-plugin-prometheus', '~>0.3.0'

cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

@@ -16,7 +16,7 @@
 
 PREFIX = staging-k8s.gcr.io
 IMAGE = fluentd-elasticsearch
-TAG = v2.0.4
+TAG = v2.1.0
 
 build:
 	docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) .

cluster/addons/fluentd-elasticsearch/kibana-deployment.yaml

@@ -19,7 +19,7 @@ spec:
     spec:
       containers:
       - name: kibana-logging
-        image: docker.elastic.co/kibana/kibana:5.6.4
+        image: docker.elastic.co/kibana/kibana-oss:6.2.4
         resources:
           # need more cpu upon initialization, therefore burstable class
           limits:
@@ -31,10 +31,6 @@ spec:
             value: http://elasticsearch-logging:9200
           - name: SERVER_BASEPATH
             value: /api/v1/namespaces/kube-system/services/kibana-logging/proxy
-          - name: XPACK_MONITORING_ENABLED
-            value: "false"
-          - name: XPACK_SECURITY_ENABLED
-            value: "false"
         ports:
         - containerPort: 5601
           name: ui