From 64918cc2b4a48d2740f5e41e37dd2d9dde17cd2b Mon Sep 17 00:00:00 2001 From: patc Date: Mon, 28 Jan 2019 22:44:44 -0800 Subject: [PATCH] update makefiles to use cloud build and update README --- .../addons/fluentd-elasticsearch/README.md | 174 +++++++++--------- .../fluentd-elasticsearch/es-image/Makefile | 15 +- .../fluentd-es-image/Makefile | 11 +- 3 files changed, 98 insertions(+), 102 deletions(-) diff --git a/cluster/addons/fluentd-elasticsearch/README.md b/cluster/addons/fluentd-elasticsearch/README.md index 82f50df5911..0763b127656 100644 --- a/cluster/addons/fluentd-elasticsearch/README.md +++ b/cluster/addons/fluentd-elasticsearch/README.md @@ -1,83 +1,91 @@ -# Elasticsearch Add-On - -This add-on consists of a combination of [Elasticsearch][elasticsearch], -[Fluentd][fluentd] and [Kibana][kibana]. Elasticsearch is a search engine -that is responsible for storing our logs and allowing for them to be queried. -Fluentd sends log messages from Kubernetes to Elasticsearch, whereas Kibana -is a graphical interface for viewing and querying the logs stored in -Elasticsearch. - -**Note:** this addon should **not** be used as-is in production. This is -an example and you should treat it as such. Please see at least the -[Security](#security) and the [Storage](#storage) sections for more -information. - -## Elasticsearch - -Elasticsearch is deployed as a [StatefulSet][statefulSet], which is like -a Deployment, but allows for maintaining state on storage volumes. - -### Security - -Elasticsearch has capabilities to enable authorization using the [X-Pack -plugin][xPack]. For the sake of simplicity this example uses the fully open -source prebuild images from elastic that do not contain the X-Pack plugin. If -you need these features, please consider building the images from either the -"basic" or "platinum" version. After enabling these features, follow [official -documentation][setupCreds] to set up credentials in Elasticsearch and Kibana. -Don't forget to propagate those credentials also to Fluentd in its -[configuration][fluentdCreds], using for example [environment -variables][fluentdEnvVar]. You can utilize [ConfigMaps][configMap] and -[Secrets][secret] to store credentials in the Kubernetes apiserver. - -### Initialization - -The Elasticsearch StatefulSet manifest specifies that there shall be an -[init container][initContainer] executing before Elasticsearch containers -themselves, in order to ensure that the kernel state variable -`vm.max_map_count` is at least 262144, since this is a requirement of -Elasticsearch. You may remove the init container if you know that your host -OS meets this requirement. - -### Storage - -The Elasticsearch StatefulSet will use the [EmptyDir][emptyDir] volume to -store data. EmptyDir is erased when the pod terminates, here it is used only -for testing purposes. **Important:** please change the storage to persistent -volume claim before actually using this StatefulSet in your setup! - -## Fluentd - -Fluentd is deployed as a [DaemonSet][daemonSet] which spawns a pod on each -node that reads logs, generated by kubelet, container runtime and containers -and sends them to Elasticsearch. - -**Note:** in order for Fluentd to work, every Kubernetes node must be labeled -with `beta.kubernetes.io/fluentd-ds-ready=true`, as otherwise the Fluentd -DaemonSet will ignore them. - -Learn more in the [official Kubernetes documentation][k8sElasticsearchDocs]. - -### Known problems - -Since Fluentd talks to the Elasticsearch service inside the cluster, instances -on masters won't work, because masters have no kube-proxy. Don't mark masters -with the label mentioned in the previous paragraph or add a taint on them to -avoid Fluentd pods scheduling there. - -[fluentd]: http://www.fluentd.org/ -[elasticsearch]: https://www.elastic.co/products/elasticsearch -[kibana]: https://www.elastic.co/products/kibana -[xPack]: https://www.elastic.co/products/x-pack -[setupCreds]: https://www.elastic.co/guide/en/x-pack/current/setting-up-authentication.html#reset-built-in-user-passwords -[fluentdCreds]: https://github.com/uken/fluent-plugin-elasticsearch#user-password-path-scheme-ssl_verify -[fluentdEnvVar]: https://docs.fluentd.org/v0.12/articles/faq#how-can-i-use-environment-variables-to-configure-parameters-dynamically -[configMap]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/ -[secret]: https://kubernetes.io/docs/concepts/configuration/secret/ -[statefulSet]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset -[initContainer]: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ -[emptyDir]: https://kubernetes.io/docs/concepts/storage/volumes#emptydir -[daemonSet]: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/ -[k8sElasticsearchDocs]: https://kubernetes.io/docs/tasks/debug-application-cluster/logging-elasticsearch-kibana - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/fluentd-elasticsearch/README.md?pixel)]() +# Elasticsearch Add-On + +This add-on consists of a combination of [Elasticsearch][elasticsearch], +[Fluentd][fluentd] and [Kibana][kibana]. Elasticsearch is a search engine +that is responsible for storing our logs and allowing for them to be queried. +Fluentd sends log messages from Kubernetes to Elasticsearch, whereas Kibana +is a graphical interface for viewing and querying the logs stored in +Elasticsearch. + +**Note:** this addon should **not** be used as-is in production. This is +an example and you should treat it as such. Please see at least the +[Security](#security) and the [Storage](#storage) sections for more +information. + +## Elasticsearch + +Elasticsearch is deployed as a [StatefulSet][statefulSet], which is like +a Deployment, but allows for maintaining state on storage volumes. + +### Security + +Elasticsearch has capabilities to enable authorization using the [X-Pack +plugin][xPack]. For the sake of simplicity this example uses the fully open +source prebuild images from elastic that do not contain the X-Pack plugin. If +you need these features, please consider building the images from either the +"basic" or "platinum" version. After enabling these features, follow [official +documentation][setupCreds] to set up credentials in Elasticsearch and Kibana. +Don't forget to propagate those credentials also to Fluentd in its +[configuration][fluentdCreds], using for example [environment +variables][fluentdEnvVar]. You can utilize [ConfigMaps][configMap] and +[Secrets][secret] to store credentials in the Kubernetes apiserver. + +### Initialization + +The Elasticsearch StatefulSet manifest specifies that there shall be an +[init container][initContainer] executing before Elasticsearch containers +themselves, in order to ensure that the kernel state variable +`vm.max_map_count` is at least 262144, since this is a requirement of +Elasticsearch. You may remove the init container if you know that your host +OS meets this requirement. + +### Storage + +The Elasticsearch StatefulSet will use the [EmptyDir][emptyDir] volume to +store data. EmptyDir is erased when the pod terminates, here it is used only +for testing purposes. **Important:** please change the storage to persistent +volume claim before actually using this StatefulSet in your setup! + +## Fluentd + +Fluentd is deployed as a [DaemonSet][daemonSet] which spawns a pod on each +node that reads logs, generated by kubelet, container runtime and containers +and sends them to Elasticsearch. + +**Note:** in order for Fluentd to work, every Kubernetes node must be labeled +with `beta.kubernetes.io/fluentd-ds-ready=true`, as otherwise the Fluentd +DaemonSet will ignore them. + +Learn more in the [official Kubernetes documentation][k8sElasticsearchDocs]. + +## Building + +Both images are now being hosted in google cloud and are built via the +[cloud build](https://cloud.google.com/cloud-build/) product. To build these +images yourself you will need to have the [gcloud sdk](https://cloud.google.com/sdk/install) +installed and you will need to login. You can then run `make` in either +image directory to trigger a container build. + +### Known problems + +Since Fluentd talks to the Elasticsearch service inside the cluster, instances +on masters won't work, because masters have no kube-proxy. Don't mark masters +with the label mentioned in the previous paragraph or add a taint on them to +avoid Fluentd pods scheduling there. + +[fluentd]: http://www.fluentd.org/ +[elasticsearch]: https://www.elastic.co/products/elasticsearch +[kibana]: https://www.elastic.co/products/kibana +[xPack]: https://www.elastic.co/products/x-pack +[setupCreds]: https://www.elastic.co/guide/en/x-pack/current/setting-up-authentication.html#reset-built-in-user-passwords +[fluentdCreds]: https://github.com/uken/fluent-plugin-elasticsearch#user-password-path-scheme-ssl_verify +[fluentdEnvVar]: https://docs.fluentd.org/v0.12/articles/faq#how-can-i-use-environment-variables-to-configure-parameters-dynamically +[configMap]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/ +[secret]: https://kubernetes.io/docs/concepts/configuration/secret/ +[statefulSet]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset +[initContainer]: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +[emptyDir]: https://kubernetes.io/docs/concepts/storage/volumes#emptydir +[daemonSet]: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/ +[k8sElasticsearchDocs]: https://kubernetes.io/docs/tasks/debug-application-cluster/logging-elasticsearch-kibana + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/fluentd-elasticsearch/README.md?pixel)]() diff --git a/cluster/addons/fluentd-elasticsearch/es-image/Makefile b/cluster/addons/fluentd-elasticsearch/es-image/Makefile index e17d51ac2e7..e2b8e68757d 100755 --- a/cluster/addons/fluentd-elasticsearch/es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/es-image/Makefile @@ -12,20 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: binary build push +.PHONY: build -PREFIX = staging-k8s.gcr.io +PREFIX = gcr.io/fluentd-elasticsearch IMAGE = elasticsearch TAG = v6.3.0 build: - docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) . - -push: - docker push $(PREFIX)/$(IMAGE):$(TAG) - -binary: - CGO_ENABLED=0 GOOS=linux go build -a -ldflags "-w" elasticsearch_logging_discovery.go - -clean: - rm elasticsearch_logging_discovery + gcloud builds submit --tag ${PREFIX}/${IMAGE}:${TAG} \ No newline at end of file diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile index b3b8c4c8961..02b4be3199a 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile @@ -12,14 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: build push +.PHONY: build -PREFIX = staging-k8s.gcr.io -IMAGE = fluentd-elasticsearch +PREFIX = gcr.io/fluentd-elasticsearch +IMAGE = fluentd TAG = v2.4.0 build: - docker build --pull -t $(PREFIX)/$(IMAGE):$(TAG) . - -push: - docker push $(PREFIX)/$(IMAGE):$(TAG) + gcloud builds submit --tag $(PREFIX)/$(IMAGE):$(TAG)