diff --git a/contrib/prometheus/Dockerfile b/contrib/prometheus/Dockerfile new file mode 100644 index 00000000000..2757433c921 --- /dev/null +++ b/contrib/prometheus/Dockerfile @@ -0,0 +1,7 @@ +FROM prom/prometheus +MAINTAINER Marcin Wielgus + +COPY ./run_prometheus.sh /prometheus/run_prometheus.sh + +ENTRYPOINT ["/bin/sh", "/prometheus/run_prometheus.sh"] +CMD [] diff --git a/contrib/prometheus/README.md b/contrib/prometheus/README.md new file mode 100644 index 00000000000..39d206aeb9e --- /dev/null +++ b/contrib/prometheus/README.md @@ -0,0 +1,71 @@ +# Prometheus in Kubernetes + +This is an experimental [Prometheus](http://prometheus.io/) setup for monitoring +Kubernetes services that expose prometheus-friendly metrics through address +http://service_address:service_port/metrics. + +# Purpose +The purpose of the setup is to gather performance-related metrics during load +tests and analyze them to find and fix bottlenecks. + +# Quick start + +## Promdash/Prometheus + +1. Pick a local directory for promdash. It can be any directory, preferably one which is stable and which you don't mind keeping around. Then (in our case, we use */mnt/promdash*, just run this docker command `docker run -v /mnt/promdash:/mnt/promdash -e DATABASE_URL=sqlite3:/mnt/promdash/file.sqlite3 prom/promdash ./bin/rake db:migrate`. In the future, we might use mysql as the promdash database, however, in any case, this 1 time db setup step is required. + +Now quickly confirm that /mnt/promdash/file.sqlite3 exists, and has a non-zero size, and make sure its permissions are open so that containers can read from it. For example: +``` + [jay@rhbd kubernetes]$ ls -altrh /mnt/promdash/ + total 20K + drwxr-xr-x. 6 root root 4.0K May 6 23:12 .. + -rwxrwxrwx 1 root root 12K May 6 23:33 file.sqlite3 +``` +Looks open enough :). + +1. Now, you can start this pod, like so `kubectl create -f cluster/add-ons/prometheus/prometheusB3.yaml`. This pod will start both prometheus, the server, as well as promdash, the visualization tool. You can then configure promdash, and next time you restart the pod - you're configuration will be remain (since the promdash directory was mounted as a local docker volume). + +1. Finally, you can simply access localhost:3000, which will have promdash running. Then, add the prometheus server (locahost:9090)to as a promdash server, and create a dashboard according to the promdash directions. + +## Prometheus + +You can launch prometheus easily, by simply running. + +`kubectl create -f cluster/addons/prometheus/prometheus.yaml` + +This will bind to port 9090 locally. You can see the prometheus database at that URL. + +# How it works + +This is a v1beta1 based, containerized prometheus pod, which scrapes endpoints which are readable on the KUBERNETES_RO service (the internal kubernetes service running in the default namespace, which is visible to all pods). + +1. The KUBERNETES_RO service is already running : providing read access to the API metrics. + +1. The list of services to be monitored is passed as a command line aguments in +the yaml file. + +1. The startup scripts assumes that each service T will have +2 environment variables set ```T_SERVICE_HOST``` and ```T_SERVICE_PORT``` + +1. Each can be configured manually in yaml file if you want to monitor something +that is not a regular Kubernetes service. For example, you can add comma delimted +endpoints which can be scraped like so... +``` +- -t +- KUBERNETES_RO,MY_OTHER_METRIC_SERVICE +``` + +# Other notes + +For regular Kubernetes services the env variables are set up automatically and injected at runtime. + +By default the metrics are written to a temporary location (that can be changed +in the the volumes section of the yaml file). Prometheus' UI is available +at port 9090. + +# TODO + +- We should publish this image into the kube/ namespace. +- Possibly use postgre or mysql as a promdash database. +- push gateway (https://github.com/prometheus/pushgateway) setup. +- Setup high availability via NFS diff --git a/contrib/prometheus/prometheusB3.yaml b/contrib/prometheus/prometheusB3.yaml new file mode 100644 index 00000000000..74a341202c5 --- /dev/null +++ b/contrib/prometheus/prometheusB3.yaml @@ -0,0 +1,83 @@ +apiVersion: v1beta3 +kind: Pod +metadata: + creationTimestamp: null + labels: + name: kube-prometheus + name: kube-prometheus +spec: + containers: + - capabilities: {} + env: + - name: DATABASE_URL + value: sqlite3:/promdash/file.sqlite3 #see volume comment below. + image: prom/promdash + imagePullPolicy: IfNotPresent + name: kube-promdash + ports: + - containerPort: 3000 + hostPort: 3000 + protocol: TCP + resources: {} + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /promdash + name: promdashdb + - args: + - -t + - KUBERNETES_RO + - -d + - /var/prometheus/ + capabilities: {} + image: jayunit100/kube-prometheus + imagePullPolicy: IfNotPresent + name: kube-prometheus + ports: + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + resources: {} + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /var/prometheus/ + name: data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + # There are many ways to create these volumes. + # for example, gcePersistentDisk:, glusterfs:, and so on... + # for the shared data volume (which we may not need going forward) + # we just use the undefined shared volume type. + - awsElasticBlockStore: null + emptyDir: + medium: "" + gcePersistentDisk: null + gitRepo: null + glusterfs: null + hostPath: null + iscsi: null + name: data + nfs: null + secret: null + # Again, many ways to create the promdash mount. We are just using local + # disk for now. Later maybe just replace with pure RDBMS rather than file + # based sqlite db. The reason we have a volume is so that its persistent between + # pod restarts. + - awsElasticBlockStore: null + emptyDir: null + gcePersistentDisk: null + gitRepo: null + glusterfs: null + hostPath: + path: /mnt/promdash + iscsi: null + name: promdashdb + nfs: null + secret: null +status: {} diff --git a/contrib/prometheus/run_prometheus.sh b/contrib/prometheus/run_prometheus.sh new file mode 100755 index 00000000000..c28d8c9711a --- /dev/null +++ b/contrib/prometheus/run_prometheus.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# Copyright 2014 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This script builds a configuration for Prometheus based on command line +# arguments and environment variables and starts the Prometheus server. +# +# Sample usage (to be run inside Kubernetes-created docker container). +# ./run_prometheus -t KUBERNETES_RO -d /tmp/prometheus +# + +show_usage() { + echo "usage: ./run_prometheus -t TARGET_1,TARGET_2 -d data_directory" + echo "where" + echo " -t List of services to be monitored. Each service T should be described by" + echo " the T_SERVICE_HOST and T_SERVICE_PORT env variables." + echo "-d Prometheus' root directory (i.e. where config file/metrics data will be stored)." +} + +build_config() { + echo >$1 'global: { scrape_interval: "10s" evaluation_interval: "10s"}' + local target + + ### Allow prometheus to scrape multiple targets + ### i.e. -t KUBERNETES_RO,MY_METRICS_ABC + for target in ${2//,/ }; do + local host_variable=$target"_SERVICE_HOST" + local port_variable=$target"_SERVICE_PORT" + local host=`eval echo '$'$host_variable` + local port=`eval echo '$'$port_variable` + echo "Checking $target" + if [ -z $host ]; then + echo "No env variable for $host_variable." + exit 3 + fi + if [ -z $port ]; then + echo "No env variable for $port_variable." + exit 3 + fi + local target_address="http://"$host":"$port"/metrics" + echo >>$1 "job: { name: \"${target}\" target_group: { target: \"${target_address}\" } }" + done +} + +while getopts :t:d: flag; do + case $flag in + t) # targets. + targets=$OPTARG + ;; + d) # data location + location=$OPTARG + ;; + \?) + echo "Unknown parameter: $flag" + show_usage + exit 2 + ;; + esac +done + +if [ -z $targets ] || [ -z $location ]; then + echo "Missing parameters." + show_usage + exit 2 +fi + +echo "------------------" +echo "Using $location as the root for prometheus configs and data." +mkdir -p $location +config="$location/config.pb" +storage="$location/storage" + +echo "-------------------" +echo "Starting Prometheus with:" +echo "targets: $targets" +echo "config: $config" +echo "storage: $storage" + +build_config $config $targets +echo "-------------------" +echo "config file:" +cat $config +echo "-------------------" + +exec /bin/prometheus \ + "-logtostderr" \ + "-config.file=$config" \ + "-storage.local.path=$storage" \ + "-web.console.libraries=/go/src/github.com/prometheus/prometheus/console_libraries" \ + "-web.console.templates=/go/src/github.com/prometheus/prometheus/consoles"