mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-09-05 19:21:37 +00:00
Launch Elasticsearch and Kibana automatically
This commit is contained in:
Satnam Singh
@@ -1,62 +0,0 @@
|
||||
# Makefile for Fluentd to Elastiscsearch and Kibana configured
|
||||
# in separate pods.
|
||||
|
||||
|
||||
.PHONY: up dow es-up kibana-up es-down kibana-down update \
|
||||
logger-up logger-down get net firewall rmfirewall
|
||||
|
||||
KUBECTL=kubectl.sh
|
||||
|
||||
up: logger-up es-up kibana-up
|
||||
|
||||
down: logger-down es-down kibana-down
|
||||
|
||||
|
||||
es-up:
|
||||
-${KUBECTL} create -f es-pod.yml
|
||||
-${KUBECTL} create -f es-service.yml
|
||||
|
||||
kibana-up:
|
||||
-${KUBECTL} create -f kibana-pod.yml
|
||||
-${KUBECTL} create -f kibana-service.yml
|
||||
|
||||
es-down:
|
||||
-${KUBECTL} delete pods elasticsearch-pod
|
||||
-${KUBECTL} delete service elasticsearch
|
||||
|
||||
kibana-down:
|
||||
-${KUBECTL} delete pods kibana-pod
|
||||
-${KUBECTL} delete service kibana
|
||||
|
||||
|
||||
update:
|
||||
-${KUBECTL} delete pods kibana-pod
|
||||
-${KUBECTL} create -f kibana-pod.yml
|
||||
|
||||
logger-up:
|
||||
-${KUBECTL} create -f synthetic_0_25lps.yml
|
||||
|
||||
logger-down:
|
||||
-${KUBECTL} delete pods synthetic-logger-0.25lps-pod
|
||||
|
||||
logger10-up:
|
||||
-${KUBECTL} create -f synthetic_10lps.yml
|
||||
|
||||
logger10-down:
|
||||
-${KUBECTL} delete pods synthetic-logger-10lps-pod
|
||||
|
||||
get:
|
||||
${KUBECTL} get pods
|
||||
${KUBECTL} get services
|
||||
|
||||
net:
|
||||
gcutil getforwardingrule elasticsearch
|
||||
gcutil getforwardingrule kibana
|
||||
|
||||
firewall:
|
||||
gcutil addfirewall --allowed=tcp:5601,tcp:9200,tcp:9300 --target_tags=kubernetes-minion kubernetes-elk-example
|
||||
|
||||
|
||||
rmfirewall:
|
||||
gcutil deletefirewall -f kubernetes-elk-example
|
||||
|
||||
@@ -1,333 +0,0 @@
|
||||
# Logging Pods in a Kubernetes Cluster using Fluentd, Elasticsearch and Kibana
|
||||
|
||||
When a GCE Kubernetes cluster is created a [pod](../../../docs/pods.md) will be placed on each node which uses the [Fluentd](http://www.fluentd.org/) log collector to collect all the Docker container logs and send them to an instance of [Elasticsearch](http://www.elasticsearch.org/) (or anything else that will listen to [Logstash](http://logstash.net/docs/1.4.2/tutorials/getting-started-with-logstash) format JSON on port 9200).
|
||||
|
||||
We can verify that a Fluentd collector is running by ssh-ing into one of the nodes and looking at the running containers.
|
||||
|
||||
|
||||
|
||||
Let's take a look in node 1.
|
||||
|
||||
```console
|
||||
$ gcloud compute --project "kubernetes-elk" ssh --zone "us-central1-b" "kubernetes-minion-1"
|
||||
...
|
||||
$ sudo -s
|
||||
$ docker ps
|
||||
satnam@kubernetes-minion-1:~$ sudo -s
|
||||
root@kubernetes-minion-1:/home/satnam# docker ps
|
||||
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
|
||||
a309846b005c ubuntu:14.04 "\"bash -c 'i=\"0\"; 3 days ago Up 3 days k8s_synth-lgr.ede24f12_synthetic-logger-10lps-pod.default.etcd_35c7b808-6c45-11e4-a194-42010af05d02_2abc4cd9
|
||||
d8d60784806b kubernetes/pause:latest "/pause" 3 days ago Up 3 days k8s_net.dbcb7509_synthetic-logger-10lps-pod.default.etcd_35c7b808-6c45-11e4-a194-42010af05d02_be1026dd
|
||||
2f47a6219e82 kubernetes/heapster:0.2 "/run.sh /bin/bash" 3 days ago Up 3 days k8s_heapster.24e32151_heapster.default.etcd_511e5a9d-6c39-11e4-a194-42010af05d02_b5ed97c1
|
||||
7dfd030bab93 kubernetes/fluentd-elasticsearch:latest "/run.sh" 3 days ago Up 3 days k8s_fluentd-es.f0eebcdc_fluentdesmanife2u464h05heqcpotoddodpnehjaqsde.default.file_fluentdesmanife2u464h05heqcpotoddodpnehjaqsde_90bbba27
|
||||
9a869d00c17b ubuntu:14.04 "\"bash -c 'i=\"0\"; 3 days ago Up 3 days k8s_synth-lgr.f0d3e2b_synthetic-logger-0.25lps-pod.default.etcd_7c7d3b8d-6c39-11e4-a194-42010af05d02_d3c519d5
|
||||
6abc80cadf3f kubernetes/pause:latest "/pause" 3 days ago Up 3 days k8s_net.dbcb7509_synthetic-logger-0.25lps-pod.default.etcd_7c7d3b8d-6c39-11e4-a194-42010af05d02_a8e3b763
|
||||
9b2787803043 kubernetes/pause:latest "/pause" 3 days ago Up 3 days k8s_net.dbcb7509_heapster.default.etcd_511e5a9d-6c39-11e4-a194-42010af05d02_f3fac3cc
|
||||
fda05d821371 kubernetes/pause:latest "/pause" 3 days ago Up 3 days k8s_net.dbcb7509_fluentdesmanife2u464h05heqcpotoddodpnehjaqsde.default.file_fluentdesmanife2u464h05heqcpotoddodpnehjaqsde_936da1a7
|
||||
04b1225d0ed3 google/cadvisor:0.5.0 "/usr/bin/cadvisor" 3 days ago Up 3 days k8s_cadvisor.b0dae998_cadvisormanifes12uqn2ohido76855gdecd9roadm7l0.default.file_cadvisormanifes12uqn2ohido76855gdecd9roadm7l0_70af8640
|
||||
ecf63dd4aa43 kubernetes/pause:latest "/pause" 3 days ago Up 3 days 0.0.0.0:4194->8080/tcp k8s_net.a0f18f6e_cadvisormanifes12uqn2ohido76855gdecd9roadm7l0.default.file_cadvisormanifes12uqn2ohido76855gdecd9roadm7l0_9e43beba
|
||||
```
|
||||
|
||||
There are several containers running running on this node. The containers `google/cadvisor` and `kubernetes/heapster` provide monitoring and profiling support. The container `kubernetes/fluentd-elasticsearch` is constantly looking at the logs files of Docker containers in the directories `/var/lib/docker/containers/*` and sending (tailing)
|
||||
this information in Logstash format to port 9200 on the local node.
|
||||
|
||||
We can look at the pod specification used to launch the Fluentd Elasticsearch container which is stored as a manifest file on the node.
|
||||
|
||||
```console
|
||||
$ cd /etc/kubernetes/manifests/
|
||||
$ ls
|
||||
cadvisor.manifest fluentd-es.manifest
|
||||
$ cat fluentd-es.manifest
|
||||
version: v1beta2
|
||||
id: fluentd-to-elasticsearch
|
||||
containers:
|
||||
- name: fluentd-es
|
||||
image: kubernetes/fluentd-elasticsearch
|
||||
volumeMounts:
|
||||
- name: containers
|
||||
mountPath: /var/lib/docker/containers
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: containers
|
||||
source:
|
||||
hostDir:
|
||||
path: /var/lib/docker/containers
|
||||
```
|
||||
|
||||
This is just a regular pod specification which you could have run using `kubectl.sh`. However, what you could not have done yourself is run an instance of this pod specification on each node which is what is accomplished with the manifest file at cluster creation time. Notice that CAdvisor also has a manifest pod specification.
|
||||
|
||||
We can connect to a running Fluentd Elasticsearch container to inspect the Fluentd configuration.
|
||||
|
||||
```console
|
||||
$ docker exec -ti 3c3816c0cfc6 bash
|
||||
$ cat /etc/td-agent/td-agent.conf
|
||||
...
|
||||
<source>
|
||||
type tail
|
||||
format json
|
||||
time_key time
|
||||
path /var/lib/docker/containers/*/*-json.log
|
||||
time_format %Y-%m-%dT%H:%M:%S
|
||||
tag docker.container.*
|
||||
</source>
|
||||
|
||||
<match docker.container.**>
|
||||
type elasticsearch
|
||||
log_level info
|
||||
include_tag_key true
|
||||
host kubernetes-minion-2
|
||||
port 9200
|
||||
logstash_format true
|
||||
flush_interval 5s
|
||||
</match>
|
||||
```
|
||||
|
||||
This configures Fluentd to gather all the Docker container log files and send them in Logstash format to port 9200.
|
||||
|
||||
Once you have turned up a Kubernetes cluster you can use the `Makefile` in this GitHub directory to try out some logging experiments.
|
||||
|
||||
We need to create an instance of Elasticsearch which will run on the cluster (this is not done automatically as part of the manifest pod creation step). We only want to run one instance of Elasticsearch on the cluster but we want it to appear as if it is running on every node. We can accomplish this by writing a suitable pod specification and service specification since this "appear to run everywhere on the cluster" abstraction is one of the things that Kubernetes provides.
|
||||
|
||||
First, here is the pod specification for Elasticsearch [es-pod.yml](es.pod.yml):
|
||||
|
||||
```
|
||||
apiVersion: v1beta1
|
||||
kind: Pod
|
||||
id: elasticsearch-pod
|
||||
desiredState:
|
||||
manifest:
|
||||
version: v1beta1
|
||||
id: es
|
||||
containers:
|
||||
- name: elasticsearch
|
||||
image: dockerfile/elasticsearch
|
||||
ports:
|
||||
- name: es-port
|
||||
containerPort: 9200
|
||||
- name: es-transport-port
|
||||
containerPort: 9300
|
||||
volumeMounts:
|
||||
- name: es-persistent-storage
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: es-persistent-storage
|
||||
source:
|
||||
emptyDir: {}
|
||||
labels:
|
||||
app: elasticsearch
|
||||
```
|
||||
|
||||
This runs the official Docker image for Elasticsearch and wires up ports 9200 (for submitting and querying information) and 9300 (a port used to compose multiple instances of Elasticsearch -- more about this elsewhere). Kubernetes may have to restart an Elasticsearch container if something goes wrong and it would be shame to loose all the information (logs) that has been gathered when the original container dies and takes down all its information with it. To avoid this problem we wire up some persistent storage for Elasticsearch so the gathered data persists between one invocation of the Elasticsearch container and another.
|
||||
|
||||
To allow us to query Elasticsearch from outside the cluster (e.g. from our laptop) and to allow other Kubernetes pods access to the Elasticsearch web interface we define a Kubernetes Elasticsearch service [es-service.yml](es-service.yml):
|
||||
|
||||
```
|
||||
apiVersion: v1beta1
|
||||
kind: Service
|
||||
id: elasticsearch
|
||||
containerPort: es-port
|
||||
port: 9200
|
||||
selector:
|
||||
app: elasticsearch
|
||||
createExternalLoadBalancer: true
|
||||
```
|
||||
|
||||
The service specification will group together all containers that have the label `app=elasticsearch` (we will only use one) and for these containers it will map their internal port (9200) to port 9200 for a service which will act as a proxy for all the identified containers. Furthermore, an external load balancer is created to allow external access to the pods that are encapsulated by this service. The container ports identified by the service description are proxied by a single IP address scoped within the cluster.
|
||||
|
||||
```console
|
||||
$ kubectl.sh get services
|
||||
NAME LABELS SELECTOR IP PORT
|
||||
elasticsearch app=elasticsearch 10.0.0.1 9200
|
||||
```
|
||||
|
||||
Inside the cluster, the Elasticsearch service is reached at http://10.0.0.1:9200 which is its service address.
|
||||
|
||||
We can see which node the Elasticsearch instance is actually running one e.g. in the example below it is running on node 3.
|
||||
|
||||
```console
|
||||
$ kubectl.sh get pods
|
||||
NAME IMAGE(S) HOST LABELS STATUS
|
||||
elasticsearch-pod dockerfile/elasticsearch kubernetes-minion-3.c.kubernetes-elk.internal/146.148.59.62 app=elasticsearch Running
|
||||
```
|
||||
|
||||
You can see that Elasticsearch can be reached on port 9200 on node 1:
|
||||
|
||||
```console
|
||||
$ curl localhost:9200
|
||||
{
|
||||
"status" : 200,
|
||||
"name" : "Elsie-Dee",
|
||||
"version" : {
|
||||
"number" : "1.3.2",
|
||||
"build_hash" : "dee175dbe2f254f3f26992f5d7591939aaefd12f",
|
||||
"build_timestamp" : "2014-08-13T14:29:30Z",
|
||||
"build_snapshot" : false,
|
||||
"lucene_version" : "4.9"
|
||||
},
|
||||
"tagline" : "You Know, for Search"
|
||||
}
|
||||
```
|
||||
|
||||
If we ran the same curl command on node 2, node 3, or node 4 we would have got a response from the same instance of Elasticsearch. The actual instance is running on node 3, but it appears to run on every node.
|
||||
|
||||
We can also contact the Elasticsearch instance from outside the cluster by finding its external IP address and port number.
|
||||
|
||||
```console
|
||||
$ gcutil getforwardingrule elasticsearch
|
||||
+---------------+---------------------------------------+
|
||||
| name | elasticsearch |
|
||||
| description | |
|
||||
| creation-time | 2014-10-27T22:07:39.585-07:00 |
|
||||
| region | us-central1 |
|
||||
| ip | 130.211.122.249 |
|
||||
| protocol | TCP |
|
||||
| port-range | 9200-9200 |
|
||||
| target | us-central1/targetPools/elasticsearch |
|
||||
+---------------+---------------------------------------+
|
||||
$ curl http://130.211.122.249:9200
|
||||
{
|
||||
"status" : 200,
|
||||
"name" : "Elsie-Dee",
|
||||
"version" : {
|
||||
"number" : "1.3.2",
|
||||
"build_hash" : "dee175dbe2f254f3f26992f5d7591939aaefd12f",
|
||||
"build_timestamp" : "2014-08-13T14:29:30Z",
|
||||
"build_snapshot" : false,
|
||||
"lucene_version" : "4.9"
|
||||
},
|
||||
"tagline" : "You Know, for Search"
|
||||
}
|
||||
```
|
||||
|
||||
A nice aspect of this architecture is that all the Docker container log files from all the nodes get automatically interleaved into the same Elasticsearch datastore. Each node thinks it is talking directly to Elasticsearch but in reality only one node has the instance and requests to Elasticsearch on other nodes are proxies to the actual instance. All of this is transparent to the Fluentd configuration.
|
||||
|
||||
To view the log information gathered inside Elasticsearch we can use the [Kibana](http://www.elasticsearch.org/overview/kibana/) viewer. Again, we will create one instance of this and run it on the cluster somewhere (Kubernetes will decide where) and this will be done with a Docker container. Here is the pod specification [kibana-pod.yml](kibana-pod.yml):
|
||||
|
||||
```
|
||||
apiVersion: v1beta1
|
||||
kind: Pod
|
||||
id: kibana-pod
|
||||
desiredState:
|
||||
manifest:
|
||||
version: v1beta1
|
||||
id: kibana-server
|
||||
containers:
|
||||
- name: kibana-image
|
||||
image: kubernetes/kibana:latest
|
||||
ports:
|
||||
- name: kibana-port
|
||||
containerPort: 80
|
||||
labels:
|
||||
app: kibana-viewer
|
||||
```
|
||||
|
||||
This runs a specially made Kibana Docker image which is tailored for use with Kubernetes. One reason for this is that this image needs to know how to contact the Elasticsearch server which it should do by contacting the internal cluster IP and port number for the service. This information is made available with environment variable. For a service called `elasticsearch` the environment variables `ELASTICSEARCH_SERVICE_HOST` and `ELASTICSEARCH_SERVICE_PORT` define the internal cluster IP address and port of the Elasticsearch service. This capability allows us to compose Kubernetes services. This pod wires up port 80 of the container which serves the Kibana dashboard web page.
|
||||
|
||||
The Kibana service is defined as follows [kibana-service.yml](kibana-service.yml):
|
||||
|
||||
```
|
||||
apiVersion: v1beta1
|
||||
kind: Service
|
||||
id: kibana
|
||||
containerPort: kibana-port
|
||||
port: 5601
|
||||
selector:
|
||||
app: kibana-viewer
|
||||
createExternalLoadBalancer: true
|
||||
```
|
||||
|
||||
This maps the internal container port 80 to an external port 5601 for the Kibana viewer service.
|
||||
|
||||
Finally, we need some pod that will produce some output which can be logged. We use a synthetic logger which periodically writes out the name of the pod that is is running in, a count and the date at a rate of 0.25 lines per second [synthetic_0_25lps.yml](synthetic_0_25lps.yml):
|
||||
|
||||
```
|
||||
apiVersion: v1beta1
|
||||
kind: Pod
|
||||
id: synthetic-logger-0.25lps-pod
|
||||
desiredState:
|
||||
manifest:
|
||||
version: v1beta1
|
||||
id: synth-logger-0.25lps
|
||||
containers:
|
||||
- name: synth-lgr
|
||||
image: ubuntu:14.04
|
||||
command: ["bash", "-c", "i=\"0\"; while true; do echo -n \"`hostname`: $i: \"; date --rfc-3339 ns; sleep 4; i=$[$i+1]; done"]
|
||||
labels:
|
||||
name: synth-logging-source
|
||||
```
|
||||
|
||||
Once Elasticsearch, Kibana and the synthetic logger are running we should see something like:
|
||||
|
||||
```console
|
||||
$ kubectl.sh get pods
|
||||
NAME IMAGE(S) HOST LABELS STATUS
|
||||
synthetic-logger-0.25lps-pod ubuntu:14.04 kubernetes-minion-2.c.kubernetes-elk.internal/146.148.37.102 name=synth-logging-source Running
|
||||
elasticsearch-pod dockerfile/elasticsearch kubernetes-minion-3.c.kubernetes-elk.internal/146.148.59.62 app=elasticsearch Running
|
||||
kibana-pod kubernetes/kibana:latest kubernetes-minion-2.c.kubernetes-elk.internal/146.148.37.102 app=kibana-viewer Running
|
||||
kubectl.sh get services
|
||||
NAME LABELS SELECTOR IP PORT
|
||||
elasticsearch app=elasticsearch 10.0.0.1 9200
|
||||
kibana app=kibana-viewer 10.0.0.2 5601
|
||||
$ gcutil getforwardingrule elasticsearch
|
||||
+---------------+---------------------------------------+
|
||||
| name | elasticsearch |
|
||||
| description | |
|
||||
| creation-time | 2014-10-27T22:07:39.585-07:00 |
|
||||
| region | us-central1 |
|
||||
| ip | 130.211.122.249 |
|
||||
| protocol | TCP |
|
||||
| port-range | 9200-9200 |
|
||||
| target | us-central1/targetPools/elasticsearch |
|
||||
+---------------+---------------------------------------+
|
||||
$ gcutil getforwardingrule kibana
|
||||
+---------------+--------------------------------+
|
||||
| name | kibana |
|
||||
| description | |
|
||||
| creation-time | 2014-10-27T16:26:57.432-07:00 |
|
||||
| region | us-central1 |
|
||||
| ip | 23.236.59.213 |
|
||||
| protocol | TCP |
|
||||
| port-range | 5601-5601 |
|
||||
| target | us-central1/targetPools/kibana |
|
||||
+---------------+--------------------------------+
|
||||
```
|
||||
|
||||
This tells us that inside the cluster the Elasticsearch service is known as 10.0.0.1:9200 and outside the cluster it is known as 130.211.122.249:9200. Inside the cluster the Kibana service is known as 10.0.0.1:5601 and outside the cluster it is known as 23.236.59.213:5601. Let's visit this web site and check that we can see some logs.
|
||||
|
||||
|
||||
|
||||
Note that in this example Kibana is running on node 2. We can ssh into this machine and look at its log files to make sure it got the correct information about where to find Elasticsearch.
|
||||
|
||||
```console
|
||||
$ gcloud compute --project "kubernetes-elk" ssh --zone "us-central1-a"
|
||||
$ sudo -s
|
||||
$ docker ps
|
||||
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
|
||||
bae7cf093eba kubernetes/kibana:latest "/usr/local/bin/run. 4 hours ago Up 4 hours k8s_kibana-image.7ece93f5_kibana-pod.default.etcd_1414472864_0d7d25bd
|
||||
47cb11bf0f8f kubernetes/pause:go "/pause" 4 hours ago Up 4 hours k8s_net.d5468756_kibana-pod.default.etcd_1414472864_8f3effbe
|
||||
e98d629ca5f0 google/cadvisor:0.4.1 "/usr/bin/cadvisor / 8 hours ago Up 8 hours k8s_cadvisor.417cd83c_cadvisormanifes12uqn2ohido76855gdecd9roadm7l0.default.file_cadvisormanifes12uqn2ohido76855gdecd9roadm7l0_daa00a70
|
||||
3c3816c0cfc6 kubernetes/fluentd-elasticsearch:latest "/run.sh" 8 hours ago Up 8 hours k8s_fluentd-es.1b9eab35_fluentdesmanife2u464h05heqcpotoddodpnehjaqsde.default.file_fluentdesmanife2u464h05heqcpotoddodpnehjaqsde_5a344730
|
||||
bad9cbdb464c ubuntu:14.04 "\"bash -c 'i=\"0\"; 8 hours ago Up 8 hours k8s_synth-lgr.c1f588c9_synthetic-logger-0.25lps-pod.default.etcd_1414458076_08a6b51a
|
||||
4eff7c5e2c15 kubernetes/pause:go "/pause" 8 hours ago Up 8 hours k8s_net.fadb6b63_synthetic-logger-0.25lps-pod.default.etcd_1414458076_92f74236
|
||||
25e6677155b0 kubernetes/pause:go "/pause" 8 hours ago Up 8 hours k8s_net.fadb6b63_fluentdesmanife2u464h05heqcpotoddodpnehjaqsde.default.file_fluentdesmanife2u464h05heqcpotoddodpnehjaqsde_bf6ed0e9
|
||||
44a7db3c8e82 kubernetes/pause:go "/pause" 8 hours ago Up 8 hours 0.0.0.0:4194->8080/tcp k8s_net.f72d85c8_cadvisormanifes12uqn2ohido76855gdecd9roadm7l0.default.file_cadvisormanifes12uqn2ohido76855gdecd9roadm7l0_fc3e7475
|
||||
$ docker logs bae7cf093eba
|
||||
ELASTICSEARCH_PORT=tcp://10.0.0.1:9200
|
||||
ELASTICSEARCH_PORT_9200_TCP=tcp://10.0.0.1:9200
|
||||
ELASTICSEARCH_PORT_9200_TCP_ADDR=10.0.0.1
|
||||
ELASTICSEARCH_PORT_9200_TCP_PORT=9200
|
||||
ELASTICSEARCH_PORT_9200_TCP_PROTO=tcp
|
||||
ELASTICSEARCH_SERVICE_HOST=10.0.0.1
|
||||
ELASTICSEARCH_SERVICE_PORT=9200
|
||||
```
|
||||
|
||||
As expected we see that `ELASTICSEARCH_SERVICE_HOST` has the value 10.0.0.1 and that `ELASTICSEARCH_SERVICE_PORT` has the value 9200.
|
||||
|
||||
## Summary and Other Things
|
||||
* Kubernetes provides intrinsic support for various logging options including the collection of Docker log files using Fluentd.
|
||||
* The storage of log files (using Elasticsearch) and the viewing of log files (using Kibana) can be performed by writing regular pod and service specifications.
|
||||
* This example could be adapted to use multiple Elasticsearch instances. In this case the service address 10.0.0.1:9200 within the cluster (or the corresponding external address) will load balance requests automatically amongst several instances.
|
||||
* Likewise, the number of Kibana instances may also be scaled up if required.
|
||||
* The number of Elasticsearch instances and Kibana instances can be scaled up independently.
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 102 KiB |
@@ -1,25 +0,0 @@
|
||||
apiVersion: v1beta1
|
||||
kind: Pod
|
||||
id: elasticsearch-pod
|
||||
desiredState:
|
||||
manifest:
|
||||
version: v1beta1
|
||||
id: es
|
||||
containers:
|
||||
- name: elasticsearch
|
||||
image: dockerfile/elasticsearch
|
||||
ports:
|
||||
- name: es-port
|
||||
containerPort: 9200
|
||||
- name: es-transport-port
|
||||
containerPort: 9300
|
||||
volumeMounts:
|
||||
- name: es-persistent-storage
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: es-persistent-storage
|
||||
source:
|
||||
emptyDir: {}
|
||||
labels:
|
||||
app: elasticsearch
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
apiVersion: v1beta1
|
||||
kind: Service
|
||||
id: elasticsearch
|
||||
containerPort: es-port
|
||||
port: 9200
|
||||
selector:
|
||||
app: elasticsearch
|
||||
createExternalLoadBalancer: true
|
||||
@@ -1,15 +0,0 @@
|
||||
apiVersion: v1beta1
|
||||
kind: Pod
|
||||
id: kibana-pod
|
||||
desiredState:
|
||||
manifest:
|
||||
version: v1beta1
|
||||
id: kibana-server
|
||||
containers:
|
||||
- name: kibana-image
|
||||
image: kubernetes/kibana:latest
|
||||
ports:
|
||||
- name: kibana-port
|
||||
containerPort: 80
|
||||
labels:
|
||||
app: kibana-viewer
|
||||
@@ -1,9 +0,0 @@
|
||||
apiVersion: v1beta1
|
||||
kind: Service
|
||||
id: kibana
|
||||
containerPort: kibana-port
|
||||
port: 5601
|
||||
selector:
|
||||
app: kibana-viewer
|
||||
createExternalLoadBalancer: true
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 461 KiB |
File diff suppressed because one or more lines are too long
Binary file not shown.
|
Before Width: | Height: | Size: 209 KiB |
File diff suppressed because one or more lines are too long
Binary file not shown.
|
Before Width: | Height: | Size: 43 KiB |
@@ -1,29 +0,0 @@
|
||||
# This pod specification creates an instance of a synthetic logger. The logger
|
||||
# is simply a program that writes out the hostname of the pod, a count which increments
|
||||
# by one on each iteration (to help notice missing log enteries) and the date using
|
||||
# a long format (RFC-3339) to nano-second precision. This program logs at a frequency
|
||||
# of 0.25 lines per second. The shellscript program is given directly to bash as -c argument
|
||||
# and could have been written out as:
|
||||
# i="0"
|
||||
# while true
|
||||
# do
|
||||
# echo -n "`hostname`: $i: "
|
||||
# date --rfc-3339 ns
|
||||
# sleep 4
|
||||
# i=$[$i+1]
|
||||
# done
|
||||
|
||||
apiVersion: v1beta1
|
||||
kind: Pod
|
||||
id: synthetic-logger-0.25lps-pod
|
||||
desiredState:
|
||||
manifest:
|
||||
version: v1beta1
|
||||
id: synth-logger-0.25lps
|
||||
containers:
|
||||
- name: synth-lgr
|
||||
image: ubuntu:14.04
|
||||
command: ["bash", "-c", "i=\"0\"; while true; do echo -n \"`hostname`: $i: \"; date --rfc-3339 ns; sleep 4; i=$[$i+1]; done"]
|
||||
labels:
|
||||
name: synth-logging-source
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
# This pod specification creates an instance of a synthetic logger. The logger
|
||||
# is simply a program that writes out the hostname of the pod, a count which increments
|
||||
# by one on each iteration (to help notice missing log enteries) and the date using
|
||||
# a long format (RFC-3339) to nano-second precision. This program logs at a frequency
|
||||
# of 0.25 lines per second. The shellscript program is given directly to bash as -c argument
|
||||
# and could have been written out as:
|
||||
# i="0"
|
||||
# while true
|
||||
# do
|
||||
# echo -n "`hostname`: $i: "
|
||||
# date --rfc-3339 ns
|
||||
# sleep 4
|
||||
# i=$[$i+1]
|
||||
# done
|
||||
|
||||
apiVersion: v1beta1
|
||||
kind: Pod
|
||||
id: synthetic-logger-10lps-pod
|
||||
desiredState:
|
||||
manifest:
|
||||
version: v1beta1
|
||||
id: synth-logger-10lps
|
||||
containers:
|
||||
- name: synth-lgr
|
||||
image: ubuntu:14.04
|
||||
command: ["bash", "-c", "i=\"0\"; while true; do echo -n \"`hostname`: $i: \"; date --rfc-3339 ns; sleep 0.1; i=$[$i+1]; done"]
|
||||
labels:
|
||||
name: synth-logging-source
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
# This Dockerfile will build an image that is configured
|
||||
# to run Fluentd with an Elasticsearch plug-in and the
|
||||
# provided configuration file.
|
||||
# TODO(satnam6502): Use a lighter base image, e.g. some form of busybox.
|
||||
# The image acts as an executable for the binary /usr/sbin/td-agent.
|
||||
# Note that fluentd is run with root permssion to allow access to
|
||||
# log files with root only access under /var/lib/docker/containers/*
|
||||
# Please see http://docs.fluentd.org/articles/install-by-deb for more
|
||||
# information about installing fluentd using deb package.
|
||||
|
||||
FROM ubuntu:14.04
|
||||
MAINTAINER Satnam Singh "satnam@google.com"
|
||||
|
||||
# Ensure there are enough file descriptors for running Fluentd.
|
||||
RUN ulimit -n 65536
|
||||
|
||||
# Install prerequisites.
|
||||
RUN apt-get update && \
|
||||
apt-get install -y curl && \
|
||||
apt-get install -y -q libcurl4-openssl-dev make && \
|
||||
apt-get clean
|
||||
|
||||
# Install Fluentd.
|
||||
RUN /usr/bin/curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sh
|
||||
|
||||
# Change the default user and group to root.
|
||||
# Needed to allow access to /var/log/docker/... files.
|
||||
RUN sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent
|
||||
|
||||
# Install the Elasticsearch Fluentd plug-in.
|
||||
RUN /usr/sbin/td-agent-gem install fluent-plugin-elasticsearch
|
||||
|
||||
# Copy the Fluentd configuration file.
|
||||
COPY td-agent.conf /etc/td-agent/td-agent.conf
|
||||
|
||||
# Run the Fluentd service.
|
||||
CMD /usr/sbin/td-agent -qq > /var/log/td-agent/td-agent.log
|
||||
@@ -1,9 +0,0 @@
|
||||
.PHONY: build push
|
||||
|
||||
TAG = latest
|
||||
|
||||
build:
|
||||
sudo docker build -t kubernetes/fluentd-elasticsearch:$(TAG) .
|
||||
|
||||
push:
|
||||
sudo docker push kubernetes/fluentd-elasticsearch:$(TAG)
|
||||
@@ -1,7 +0,0 @@
|
||||
# Collecting Docker Log Files with Fluentd and Elasticsearch
|
||||
This directory contains the source files needed to make a Docker image
|
||||
that collects Docker container log files using [Fluentd](http://www.fluentd.org/)
|
||||
and sends them to an instance of [Elasticsearch](http://www.elasticsearch.org/).
|
||||
This image is designed to be used as part of the [Kubernetes](https://github.com/GoogleCloudPlatform/kubernetes)
|
||||
cluster bring up process. The image resides at DockerHub under the name
|
||||
[kubernetes/fluentd-eslasticsearch](https://registry.hub.docker.com/u/kubernetes/fluentd-elasticsearch/).
|
||||
@@ -1,80 +0,0 @@
|
||||
# This configuration file for Fluentd / td-agent is used
|
||||
# to watch changes to Docker log files that live in the
|
||||
# directory /var/lib/docker/containers/ which are then submitted to
|
||||
# Elasticsearch (running on the machine %ES_HOST%:9200) which
|
||||
# assumes the installation of the fluentd-elasticsearch plug-in.
|
||||
# See https://github.com/uken/fluent-plugin-elasticsearch for
|
||||
# more information about the plug-in. This file needs to be
|
||||
# patched to replace ES_HOST with the name of the actual
|
||||
# machine running Elasticsearch.
|
||||
# Maintainer: Satnam Singh (satnam@google.com)
|
||||
#
|
||||
# Exampe
|
||||
# ======
|
||||
# A line in the Docker log file might like like this JSON:
|
||||
#
|
||||
# {"log":"2014/09/25 21:15:03 Got request with path wombat\n",
|
||||
# "stream":"stderr",
|
||||
# "time":"2014-09-25T21:15:03.499185026Z"}
|
||||
#
|
||||
# The time_format specification below makes sure we properly
|
||||
# parse the time format produced by Docker. This will be
|
||||
# submitted to Elasticsearch and should appear like:
|
||||
# $ curl 'http://elasticsearch:9200/_search?pretty'
|
||||
# ...
|
||||
# {
|
||||
# "_index" : "logstash-2014.09.25",
|
||||
# "_type" : "fluentd",
|
||||
# "_id" : "VBrbor2QTuGpsQyTCdfzqA",
|
||||
# "_score" : 1.0,
|
||||
# "_source":{"log":"2014/09/25 22:45:50 Got request with path wombat\n",
|
||||
# "stream":"stderr","tag":"docker.container.all",
|
||||
# "@timestamp":"2014-09-25T22:45:50+00:00"}
|
||||
# },
|
||||
# ...
|
||||
|
||||
<source>
|
||||
type tail
|
||||
format json
|
||||
time_key time
|
||||
path /var/lib/docker/containers/*/*-json.log
|
||||
pos_file /var/lib/docker/containers/containers.log.pos
|
||||
time_format %Y-%m-%dT%H:%M:%S
|
||||
tag docker.container.*
|
||||
</source>
|
||||
|
||||
<match docker.container.**>
|
||||
type elasticsearch
|
||||
log_level info
|
||||
include_tag_key true
|
||||
host elasticsearch.default
|
||||
port 9200
|
||||
logstash_format true
|
||||
flush_interval 5s
|
||||
# Never wait longer than 5 minutes between retries.
|
||||
max_retry_wait 300
|
||||
# Disable the limit on the number of retries (retry forever).
|
||||
disable_retry_limit
|
||||
</match>
|
||||
|
||||
<source>
|
||||
type tail
|
||||
format none
|
||||
path /varlog/kubelet.log
|
||||
pos_file /varlog/kubelet.log.pos
|
||||
tag kubelet
|
||||
</source>
|
||||
|
||||
<match kubelet>
|
||||
type elasticsearch
|
||||
log_level info
|
||||
include_tag_key true
|
||||
host elasticsearch.default
|
||||
port 9200
|
||||
logstash_format true
|
||||
flush_interval 5s
|
||||
# Never wait longer than 5 minutes between retries.
|
||||
max_retry_wait 300
|
||||
# Disable the limit on the number of retries (retry forever).
|
||||
disable_retry_limit
|
||||
</match>
|
||||
@@ -1,24 +0,0 @@
|
||||
# A Dockerfile for creating a Kibana container that is designed
|
||||
# to work with Kubernetes logging.
|
||||
|
||||
FROM ubuntu:14.04
|
||||
MAINTAINER Satnam Singh "satnam@google.com"
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y nginx-full curl && \
|
||||
apt-get clean
|
||||
|
||||
RUN curl -O http://download.elasticsearch.org/kibana/kibana/kibana-3.1.1.tar.gz && \
|
||||
tar xf kibana-3.1.1.tar.gz && \
|
||||
mv kibana-3.1.1/* /usr/share/nginx/html && \
|
||||
rm kibana-3.1.1.tar.gz
|
||||
|
||||
RUN echo "daemon off;" >> /etc/nginx/nginx.conf
|
||||
|
||||
# ADD default /etc/nginx/sites-available/default
|
||||
ADD run_kibana_nginx.sh /usr/local/bin/run_kibana_nginx.sh
|
||||
|
||||
EXPOSE 80
|
||||
CMD ["/usr/local/bin/run_kibana_nginx.sh"]
|
||||
@@ -1,5 +0,0 @@
|
||||
build:
|
||||
docker build -t kubernetes/kibana .
|
||||
|
||||
push:
|
||||
docker push kubernetes/kibana
|
||||
@@ -1,183 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This code below is designed to support two specific scenarios for
|
||||
# using Elasticsearch and Kibana with Kubernetes. In both cases the
|
||||
# environment variables PROXY_HOST and PROXY_PORT identify the instance
|
||||
# of Elasticsearch to be used by Kibana. The default value for ES_HOST
|
||||
# identifies the location that served the Javascript for Kibana and
|
||||
# the default value of ES_PORT 5601 is the port to be used for connecting
|
||||
# to Kibana. Both of these may be overriden if required. The two scenarios are:
|
||||
# 1. Elasticsearch and Kibana containers running in a single pod. In this
|
||||
# case PROXY_HOST is set to the local host i.e. 127.0.0.1 and the
|
||||
# PROXY_PORT is set to 9200 because Elasticsearch is running on the
|
||||
# same name as Kibana. If KIBANA_IP is the external IP address of
|
||||
# the Kubernetes Kibna service then all requests to:
|
||||
# KIBANA_SERVICE:$ES_PORT/elasticsearch/XXX
|
||||
# are proxied to:
|
||||
# http://127.0.0.1:9200/XXX
|
||||
# 2. Elasticsearch and Kibana are run in separate pods and Elasticsearch
|
||||
# has an IP and port exposed via a Kubernetes service. In this case
|
||||
# the Elasticsearch service *must* be called 'elasticsearch' and then
|
||||
# all requests sent to:
|
||||
# KIBANA_SERVICE:$ES_PORT/elasticsearch/XXX
|
||||
# are proxied to:
|
||||
# http://$ELASTICSEARCH_SERVICE_HOST:$ELASTICSEARCH_SERVICE_PORT:9200/XXX
|
||||
# The proxy configuration occurs in a location block of the nginx configuration
|
||||
# file /etc/nginx/sites-available/default.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# Report all environment variables containing 'elasticsearch'
|
||||
set | grep -i elasticsearch
|
||||
# Set the default value for the Elasticsearch host as seen by the client
|
||||
# Javascript code for Kibana.
|
||||
: ${ES_HOST:='"+window.location.hostname+"'}
|
||||
echo ES_HOST=$ES_HOST
|
||||
# Set the default port for Elasticsearch host as seen by the client
|
||||
# Javascript for Kibana.
|
||||
: ${ES_PORT:=5601}
|
||||
echo ES_PORT=$ES_PORT
|
||||
# Set the default host IP and port for Elasticsearch as seen by the proxy
|
||||
# code in the configuration for nginx. If a Kubernetes Elasticsearch
|
||||
# service called 'elasticsearch' is defined, use that. Otherwise, use
|
||||
# a local instance of Elasticsearch on port 9200.
|
||||
PROXY_HOST=${ELASTICSEARCH_SERVICE_HOST:-127.0.0.1}
|
||||
echo PROXY_HOST=${PROXY_HOST}
|
||||
PROXY_PORT=${ELASTICSEARCH_SERVICE_PORT:-9200}
|
||||
echo PROXY_PORT=${PROXY_PORT}
|
||||
# Test the connection to Elasticsearch
|
||||
echo "Running curl http://${PROXY_HOST}:${PROXY_PORT}"
|
||||
curl http://${PROXY_HOST}:${PROXY_PORT}
|
||||
|
||||
# Create a config.hs that defines the Elasticsearch server to be
|
||||
# at http://${ES_HOST}:${ES_PORT}/elasticsearch from the perspective of
|
||||
# the client Javascript code.
|
||||
cat << EOF > /usr/share/nginx/html/config.js
|
||||
/** @scratch /configuration/config.js/1
|
||||
*
|
||||
* == Configuration
|
||||
* config.js is where you will find the core Kibana configuration. This file contains parameter that
|
||||
* must be set before kibana is run for the first time.
|
||||
*/
|
||||
define(['settings'],
|
||||
function (Settings) {
|
||||
|
||||
|
||||
/** @scratch /configuration/config.js/2
|
||||
*
|
||||
* === Parameters
|
||||
*/
|
||||
return new Settings({
|
||||
|
||||
/** @scratch /configuration/config.js/5
|
||||
*
|
||||
* ==== elasticsearch
|
||||
*
|
||||
* The URL to your elasticsearch server. You almost certainly don't
|
||||
* want +http://localhost:9200+ here. Even if Kibana and Elasticsearch are on
|
||||
* the same host. By default this will attempt to reach ES at the same host you have
|
||||
* kibana installed on. You probably want to set it to the FQDN of your
|
||||
* elasticsearch host
|
||||
*
|
||||
* Note: this can also be an object if you want to pass options to the http client. For example:
|
||||
*
|
||||
* +elasticsearch: {server: "http://localhost:9200", withCredentials: true}+
|
||||
*
|
||||
*/
|
||||
elasticsearch: "http://${ES_HOST}:${ES_PORT}/elasticsearch",
|
||||
|
||||
/** @scratch /configuration/config.js/5
|
||||
*
|
||||
* ==== default_route
|
||||
*
|
||||
* This is the default landing page when you don't specify a dashboard to load. You can specify
|
||||
* files, scripts or saved dashboards here. For example, if you had saved a dashboard called
|
||||
* WebLogs to elasticsearch you might use:
|
||||
*
|
||||
* default_route: '/dashboard/elasticsearch/WebLogs',
|
||||
*/
|
||||
default_route : '/dashboard/file/logstash.json',
|
||||
|
||||
/** @scratch /configuration/config.js/5
|
||||
*
|
||||
* ==== kibana-int
|
||||
*
|
||||
* The default ES index to use for storing Kibana specific object
|
||||
* such as stored dashboards
|
||||
*/
|
||||
kibana_index: "kibana-int",
|
||||
|
||||
/** @scratch /configuration/config.js/5
|
||||
*
|
||||
* ==== panel_name
|
||||
*
|
||||
* An array of panel modules available. Panels will only be loaded when they are defined in the
|
||||
* dashboard, but this list is used in the "add panel" interface.
|
||||
*/
|
||||
panel_names: [
|
||||
'histogram',
|
||||
'map',
|
||||
'goal',
|
||||
'table',
|
||||
'filtering',
|
||||
'timepicker',
|
||||
'text',
|
||||
'hits',
|
||||
'column',
|
||||
'trends',
|
||||
'bettermap',
|
||||
'query',
|
||||
'terms',
|
||||
'stats',
|
||||
'sparklines'
|
||||
]
|
||||
});
|
||||
});
|
||||
EOF
|
||||
|
||||
# Proxy all calls to ...:80/elasticsearch to the location
|
||||
# defined by http://${PROXY_HOST}:${PROXY_PORT}
|
||||
cat <<EOF > /etc/nginx/sites-available/default
|
||||
server {
|
||||
listen 80 default_server;
|
||||
listen [::]:80 default_server ipv6only=on;
|
||||
|
||||
root /usr/share/nginx/html;
|
||||
index index.html index.htm;
|
||||
|
||||
# Make site accessible from http://localhost/
|
||||
server_name localhost;
|
||||
|
||||
location ~ /elasticsearch/?(.*)$ {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade \$http_upgrade;
|
||||
proxy_read_timeout 1d;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_pass http://${PROXY_HOST}:${PROXY_PORT}/\$1;
|
||||
}
|
||||
|
||||
location / {
|
||||
# First attempt to serve request as file, then
|
||||
# as directory, then fall back to displaying a 404.
|
||||
try_files \$uri \$uri/ =404;
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
exec nginx -c /etc/nginx/nginx.conf "$@"
|
||||
Reference in New Issue
Block a user