Experimental config for Prometheus

cluster/addons/prometheus/Dockerfile

@@ -0,0 +1,7 @@
+FROM       prom/prometheus
+MAINTAINER Marcin Wielgus <mwielgus@google.com>
+
+COPY ./run_prometheus.sh /prometheus/run_prometheus.sh
+
+ENTRYPOINT ["/bin/sh", "/prometheus/run_prometheus.sh"]
+CMD []

cluster/addons/prometheus/README.md

@@ -0,0 +1,19 @@
+# Prometheus in Kubernetes
+
+This is an experimental [Prometheus](http://prometheus.io/) setup for monitoring
+Kubernetes services that expose prometheus-friendly metrics through address
+http://service_address:service_port/metrics.
+
+The purpose of the setup is to gather performance-related metrics during load
+tests and analyze them to find and fix bottlenecks.
+
+The list of services to be monitored is passed as a command line aguments in
+the yaml file. The startup scripts assumes that each service T will have
+2 environment variables set ```T_SERVICE_HOST``` and ```T_SERVICE_PORT``` which
+can be configured manually in yaml file if you want to monitor something
+that is not a regular Kubernetes service. For regular Kubernetes services
+the env variables are set up automatically.
+
+By default the metrics are written to a temporary location (that can be changed
+in the the volumes section of the yaml file). Prometheus' UI is available 
+at port 9090.

cluster/addons/prometheus/prometheus.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1beta1
+kind: Pod
+id: kube-prometheus
+desiredState:
+  manifest:
+    version: v1beta1
+    containers:
+      - name: kube-prometheus
+        image: kube/prometheus
+        command:
+          - -t
+          - KUBERNETES_RO
+          - -d
+          - /var/prometheus/
+        ports:
+          - containerPort: 9090
+            hostPort: 9090
+        volumeMounts:
+          - name: data
+            mountPath: /var/prometheus/
+    volumes:
+      - name: data
+        source:
+          emptyDir:
+labels:
+  name: kube-prometheus

cluster/addons/prometheus/run_prometheus.sh

@@ -0,0 +1,97 @@
+#!/bin/sh
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This script builds a configuration for Prometheus based on command line
+# arguments and environment variables and starts the Prometheus server.
+#
+# Sample usage (to be run inside Kubernetes-created docker container).
+#  ./run_prometheus -t KUBERNETES_RO -d /tmp/prometheus
+#
+
+show_usage() {
+  echo "usage: ./run_prometheus -t TARGET_1,TARGET_2 -d data_directory"
+  echo "where"
+  echo " -t List of services to be monitored. Each service T should be described by"
+  echo "    the T_SERVICE_HOST and T_SERVICE_PORT env variables."
+  echo " -d Location where the config file and metrics will be written."
+}
+
+build_config() {
+  echo >$1 'global: { scrape_interval: "10s" evaluation_interval: "10s"}'
+  local target
+  for target in ${2//,/ }; do
+    local host_variable=$target"_SERVICE_HOST"
+    local port_variable=$target"_SERVICE_PORT"
+    local host=`eval echo '$'$host_variable`
+    local port=`eval echo '$'$port_variable`
+    echo "Checking $target"
+    if [ -z $host ]; then
+      echo "No env variable for $host_variable."
+      exit 3
+    fi
+    if [ -z $port ]; then
+      echo "No env variable for $port_variable."
+      exit 3
+    fi
+    local target_address="http://"$host":"$port"/metrics"
+    echo >>$1 "job: { name: \"${target}\" target_group: { target: \"${target_address}\" } }"
+  done
+}
+
+while getopts :t:d: flag; do
+  case $flag in
+  t) # targets.
+    targets=$OPTARG
+    ;;
+  d) # data location
+    location=$OPTARG
+    ;;
+  \?)
+    echo "Unknown parameter: $flag"
+    show_usage
+    exit 2
+    ;;
+  esac
+done
+
+if [ -z $targets ] || [ -z $location ]; then
+  echo "Missing parameters."
+  show_usage
+  exit 2
+fi
+
+mkdir -p $location
+config="$location/config.pb"
+storage="$location/storage"
+
+echo "-------------------"
+echo "Starting Prometheus with:"
+echo "targets: $targets"
+echo "config: $config"
+echo "storage: $storage"
+
+build_config $config $targets
+echo "-------------------"
+echo "config file:"
+cat $config
+echo "-------------------"
+
+exec /bin/prometheus \
+  "-logtostderr" \
+  "-config.file=$config" \
+  "-storage.local.path=$storage" \
+  "-web.console.libraries=/go/src/github.com/prometheus/prometheus/console_libraries" \
+  "-web.console.templates=/go/src/github.com/prometheus/prometheus/consoles"