diff --git a/cluster/addons/cluster-monitoring/grafana-service.yaml b/cluster/addons/cluster-monitoring/grafana-service.yaml new file mode 100644 index 00000000000..f791e7512e4 --- /dev/null +++ b/cluster/addons/cluster-monitoring/grafana-service.yaml @@ -0,0 +1,7 @@ +apiVersion: "v1beta1" +kind: "Service" +id: "monitoring-grafana" +port: 8087 +containerPort: 80 +selector: + name: "influxGrafana" diff --git a/cluster/addons/cluster-monitoring/heapster-controller.yaml b/cluster/addons/cluster-monitoring/heapster-controller.yaml new file mode 100644 index 00000000000..f44f9660398 --- /dev/null +++ b/cluster/addons/cluster-monitoring/heapster-controller.yaml @@ -0,0 +1,20 @@ +apiVersion: "v1beta1" +id: "monitoring-heapsterController" +kind: "ReplicationController" +desiredState: + replicas: 1 + replicaSelector: + name: "heapster" + podTemplate: + desiredState: + manifest: + version: "v1beta1" + id: "monitoring-heapsterController" + containers: + - name: "heapster" + image: "kubernetes/heapster:v0.5" + labels: + name: "heapster" + uses: "monitoring-influxdb" +labels: + name: "heapster" diff --git a/cluster/addons/cluster-monitoring/heapster-pod.json b/cluster/addons/cluster-monitoring/heapster-pod.json deleted file mode 100644 index 61a3631a013..00000000000 --- a/cluster/addons/cluster-monitoring/heapster-pod.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "id": "heapster", - "kind": "Pod", - "apiVersion": "v1beta1", - "desiredState": { - "manifest": { - "version": "v1beta1", - "id": "heapster", - "containers": [ - { - "name": "heapster", - "image": "kubernetes/heapster" - } - ] - } - }, - "labels": { - "name": "heapster" - } -} diff --git a/cluster/addons/cluster-monitoring/influx-grafana-pod.json b/cluster/addons/cluster-monitoring/influx-grafana-pod.json deleted file mode 100644 index 1a104680ca2..00000000000 --- a/cluster/addons/cluster-monitoring/influx-grafana-pod.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "id": "influx-grafana", - "kind": "Pod", - "apiVersion": "v1beta1", - "desiredState": { - "manifest": { - "version": "v1beta1", - "id": "influx-grafana", - "containers": [ - { - "name": "influxdb", - "image": "kubernetes/heapster_influxdb", - "ports": [ - { - "containerPort": 8083, - "hostPort": 8083 - }, - { - "containerPort": 8086, - "hostPort": 8086 - }, - { - "containerPort": 8090, - "hostPort": 8090 - }, - { - "containerPort": 8099, - "hostPort": 8099 - } - ] - }, - { - "name": "grafana", - "image": "kubernetes/heapster_grafana", - "ports": [ - { - "containerPort": 80, - "hostPort": 80 - } - ], - "env": [ - { - "name": "HTTP_USER", - "value": "{KUBE_USER}" - }, - { - "name": "HTTP_PASS", - "value": "{KUBE_PASSWORD}" - } - ] - }, - { - "name": "elasticsearch", - "image": "dockerfile/elasticsearch", - "ports": [ - { - "containerPort": 9200, - "hostPort": 9200 - }, - { - "containerPort": 9300 - } - ] - } - ] - } - }, - "labels": { - "name": "influxdb" - } -} diff --git a/cluster/addons/cluster-monitoring/influx-grafana-service.json b/cluster/addons/cluster-monitoring/influx-grafana-service.json deleted file mode 100644 index 3d879dc9f38..00000000000 --- a/cluster/addons/cluster-monitoring/influx-grafana-service.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "id": "influx-master", - "kind": "Service", - "apiVersion": "v1beta1", - "port": 8085, - "containerPort": 8086, - "provider": "kubernetes-default", - "component": "influxdb", - "selector": { - "name": "influxdb" - } -} diff --git a/cluster/addons/cluster-monitoring/influxdb-grafana-controller.yaml b/cluster/addons/cluster-monitoring/influxdb-grafana-controller.yaml new file mode 100644 index 00000000000..c1855390da7 --- /dev/null +++ b/cluster/addons/cluster-monitoring/influxdb-grafana-controller.yaml @@ -0,0 +1,34 @@ +apiVersion: "v1beta1" +kind: "ReplicationController" +id: "monitoring-influxGrafanaController" +desiredState: + replicas: 1 + replicaSelector: + name: "influxGrafana" + podTemplate: + labels: + name: "influxGrafana" + desiredState: + manifest: + version: "v1beta1" + id: "monitoring-influxGrafanaController" + containers: + - name: "influxdb" + image: "kubernetes/heapster_influxdb:v0.2" + ports: + - containerPort: 8083 + hostPort: 8083 + - containerPort: 8086 + hostPort: 8086 + - name: "grafana" + image: "kubernetes/heapster_grafana:v0.2" + ports: + - containerPort: 80 + hostPort: 80 + env: + - name: "HTTP_USER" + value: "admin" + - name: "HTTP_PASS" + value: "**None**" +labels: + name: "influxGrafana" diff --git a/cluster/addons/cluster-monitoring/influxdb-service.yaml b/cluster/addons/cluster-monitoring/influxdb-service.yaml new file mode 100644 index 00000000000..ae51c3c2b24 --- /dev/null +++ b/cluster/addons/cluster-monitoring/influxdb-service.yaml @@ -0,0 +1,7 @@ +apiVersion: "v1beta1" +kind: "Service" +id: "monitoring-influxdb" +port: 8085 +containerPort: 8086 +selector: + name: "influxGrafana" diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 4c42240de07..391e8ca542a 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -737,35 +737,37 @@ function setup-monitoring { echo "Setting up cluster monitoring using Heapster." detect-project - if ! gcloud compute firewall-rules --project "{$PROJECT}" describe monitoring-heapster &>/dev/null; then + if ! gcloud compute firewall-rules --project "${PROJECT}" describe monitoring-heapster &> /dev/null; then if ! gcloud compute firewall-rules create monitoring-heapster \ --project "${PROJECT}" \ --target-tags="${MINION_TAG}" \ --network="${NETWORK}" \ - --allow tcp:80 tcp:8083 tcp:8086 tcp:9200; then - echo "Failed to set up firewall for monitoring" && false + --allow tcp:80 tcp:8083 tcp:8086; then + echo -e "${color_red}Failed to set up firewall for monitoring ${color_norm}" && false fi fi - # Re-use master auth for Grafana - get-password - ensure-temp-dir - - sed -e "s/{KUBE_USER}/$KUBE_USER/g" \ - -e "s/{KUBE_PASSWORD}/$KUBE_PASSWORD/g" \ - "${KUBE_ROOT}/cluster/addons/cluster-monitoring/influx-grafana-pod.json" \ - > "${KUBE_TEMP}/influx-grafana-pod.json" local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - if "${kubectl}" create -f "${KUBE_TEMP}/influx-grafana-pod.json" &> /dev/null \ - && "${kubectl}" create -f "${KUBE_ROOT}/cluster/addons/cluster-monitoring/influx-grafana-service.json" &> /dev/null \ - && "${kubectl}" create -f "${KUBE_ROOT}/cluster/addons/cluster-monitoring/heapster-pod.json" &> /dev/null; then - local dashboard_url="http://$(${kubectl} get -o json pod influx-grafana | grep hostIP | awk '{print $2}' | sed 's/[,|\"]//g')" - echo - echo "Grafana dashboard will be available at $dashboard_url. Wait for the monitoring dashboard to be online." - echo "Use the master user name and password for the dashboard." - echo + local grafana_host="" + if "${kubectl}" create -f "${KUBE_ROOT}/cluster/addons/cluster-monitoring/" &> /dev/null; then + # wait for pods to be scheduled on a node. + echo "waiting for monitoring pods to be scheduled." + for i in `seq 1 10`; do + grafana_host=$("${kubectl}" get pods -l name=influxGrafana -o template -t {{range.items}}{{.currentState.hostIP}}:{{end}} | sed s/://g) + if [[ $grafana_host != *"<"* ]]; then + break + fi + sleep 10 + done + if [[ $grafana_host != *"<"* ]]; then + echo + echo -e "${color_green}Grafana dashboard will be available at ${color_yellow}http://$grafana_host${color_green}. Wait for the monitoring dashboard to be online.${color_norm}" + echo + else + echo -e "${color_red}monitoring pods failed to be scheduled.${color_norm}" + fi else - echo "Failed to Setup Monitoring" + echo -e "${color_red}Failed to Setup Monitoring ${color_norm}" teardown-monitoring fi fi @@ -776,9 +778,10 @@ function teardown-monitoring { detect-project local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - "${kubectl}" delete pods heapster &> /dev/null || true - "${kubectl}" delete pods influx-grafana &> /dev/null || true - "${kubectl}" delete services influx-master &> /dev/null || true + local kubecfg="${KUBE_ROOT}/cluster/kubecfg.sh" + "${kubecfg}" resize monitoring-influxGrafanaController 0 &> /dev/null || true + "${kubecfg}" resize monitoring-heapsterController 0 &> /dev/null || true + "${kubectl}" delete -f "${KUBE_ROOT}/cluster/addons/cluster-monitoring/" &> /dev/null || true if gcloud compute firewall-rules describe --project "${PROJECT}" monitoring-heapster &> /dev/null; then gcloud compute firewall-rules delete \ --project "${PROJECT}" \ diff --git a/hack/e2e-suite/monitoring.sh b/hack/e2e-suite/monitoring.sh index e027d7dd32a..5820ec463d9 100755 --- a/hack/e2e-suite/monitoring.sh +++ b/hack/e2e-suite/monitoring.sh @@ -28,6 +28,7 @@ source "${KUBE_ROOT}/cluster/$KUBERNETES_PROVIDER/util.sh" MONITORING="${KUBE_ROOT}/cluster/addons/cluster-monitoring" KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" +KUBECFG="${KUBE_ROOT}/cluster/kubecfg.sh" BIGRAND=$(printf "%x\n" $(( $RANDOM << 16 | $RANDOM ))) # random 2^32 in hex MONITORING_FIREWALL_RULE="monitoring-test-${BIGRAND}" @@ -44,15 +45,13 @@ function setup { fi fi - "${KUBECTL}" create -f "${MONITORING}/influx-grafana-pod.json" - "${KUBECTL}" create -f "${MONITORING}/influx-grafana-service.json" - "${KUBECTL}" create -f "${MONITORING}/heapster-pod.json" + "${KUBECTL}" create -f "${MONITORING}/" } function cleanup { - "${KUBECTL}" delete -f "${MONITORING}/influx-grafana-pod.json" || true - "${KUBECTL}" delete -f "${MONITORING}/influx-grafana-service.json" || true - "${KUBECTL}" delete -f "${MONITORING}/heapster-pod.json" || true + "${KUBECFG}" resize monitoring-influxGrafanaController 0 &> /dev/null || true + "${KUBECFG}" resize monitoring-heapsterController 0 &> /dev/null || true + "${KUBECTL}" delete -f "${MONITORING}/" &> /dev/null || true # This only has work to do on gce and gke if [[ "${KUBERNETES_PROVIDER}" == "gce" ]] || [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then @@ -69,7 +68,7 @@ function cleanup { function influx-data-exists { local max_retries=10 local retry_delay=30 #seconds - local influx_ip=$("${KUBECTL}" get -o json pods influx-grafana | grep hostIP | awk '{print $2}' | sed 's/["|,]//g') + local influx_ip=$("${KUBECTL}" get pods -l name=influxGrafana -o template -t {{range.items}}{{.currentState.hostIP}}:{{end}} | sed s/://g) local influx_url="http://$influx_ip:8086/db/k8s/series?u=root&p=root" local ok="false" for i in `seq 1 10`; do @@ -91,8 +90,8 @@ function wait-for-pods { local running=false for i in `seq 1 20`; do sleep 20 - if "${KUBECTL}" get pods influx-grafana | grep Running &> /dev/null \ - && "${KUBECTL}" get pods heapster | grep Running &> /dev/null; then + if "${KUBECTL}" get pods -l name=influxGrafana -o template -t {{range.items}}{{.currentState.status}}:{{end}} | grep Running &> /dev/null \ + && "${KUBECTL}" get pods -l name=heapster -o template -t {{range.items}}{{.currentState.status}}:{{end}} | grep Running &> /dev/null; then running=true break fi