From fe262c0d98dca5eb050e7ce3d393cefa13c83f03 Mon Sep 17 00:00:00 2001 From: Alex Robinson Date: Thu, 16 Apr 2015 22:19:44 +0000 Subject: [PATCH] Add an integration test that checks for the metrics we expect to be exported from the master. --- test/integration/client_test.go | 34 +-------- test/integration/metrics_test.go | 126 +++++++++++++++++++++++++++++++ test/integration/utils.go | 33 ++++++++ 3 files changed, 161 insertions(+), 32 deletions(-) create mode 100644 test/integration/metrics_test.go diff --git a/test/integration/client_test.go b/test/integration/client_test.go index cf8e7b18c7e..ad43765d027 100644 --- a/test/integration/client_test.go +++ b/test/integration/client_test.go @@ -21,8 +21,6 @@ package integration import ( "fmt" "log" - "net/http" - "net/http/httptest" "reflect" "runtime" "sync" @@ -31,47 +29,19 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi" - "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/fields" "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" - "github.com/GoogleCloudPlatform/kubernetes/pkg/master" "github.com/GoogleCloudPlatform/kubernetes/pkg/version" "github.com/GoogleCloudPlatform/kubernetes/pkg/watch" - "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit" ) func init() { requireEtcd() } -func RunAMaster(t *testing.T) (*master.Master, *httptest.Server) { - helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version()) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var m *master.Master - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - m.Handler.ServeHTTP(w, req) - })) - - m = master.New(&master.Config{ - EtcdHelper: helper, - KubeletClient: client.FakeKubeletClient{}, - EnableLogsSupport: false, - EnableProfiling: true, - EnableUISupport: false, - APIPrefix: "/api", - Authorizer: apiserver.NewAlwaysAllowAuthorizer(), - AdmissionControl: admit.NewAlwaysAdmit(), - }) - - return m, s -} - func TestClient(t *testing.T) { - _, s := RunAMaster(t) + _, s := runAMaster(t) defer s.Close() ns := api.NamespaceDefault @@ -149,7 +119,7 @@ func TestMultiWatch(t *testing.T) { deleteAllEtcdKeys() defer deleteAllEtcdKeys() - _, s := RunAMaster(t) + _, s := runAMaster(t) defer s.Close() ns := api.NamespaceDefault diff --git a/test/integration/metrics_test.go b/test/integration/metrics_test.go new file mode 100644 index 00000000000..0e331861a91 --- /dev/null +++ b/test/integration/metrics_test.go @@ -0,0 +1,126 @@ +// +build integration,!no-etcd + +/* +Copyright 2015 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "bufio" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" + "github.com/golang/glog" + "github.com/golang/protobuf/proto" + prometheuspb "github.com/prometheus/client_model/go" +) + +const scrapeRequestHeader = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=compact-text" + +func init() { + requireEtcd() +} + +func scrapeMetrics(s *httptest.Server) ([]*prometheuspb.MetricFamily, error) { + req, err := http.NewRequest("GET", s.URL+"/metrics", nil) + if err != nil { + return nil, fmt.Errorf("Unable to create http request: %v", err) + } + // Ask the prometheus exporter for its text protocol buffer format, since it's + // much easier to parse than its plain-text format. Don't use the serialized + // proto representation since it uses a non-standard varint delimiter between + // metric families. + req.Header.Add("Accept", scrapeRequestHeader) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("Unable to contact metrics endpoint of master: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("Non-200 response trying to scrape metrics from master: %v", resp) + } + + // Each line in the response body should contain all the data for a single metric. + var metrics []*prometheuspb.MetricFamily + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + var metric prometheuspb.MetricFamily + if err := proto.UnmarshalText(scanner.Text(), &metric); err != nil { + return nil, fmt.Errorf("Failed to unmarshal line of metrics response: %v", err) + } + glog.Infof("Got metric %q", metric.GetName()) + metrics = append(metrics, &metric) + } + return metrics, nil +} + +func checkForExpectedMetrics(t *testing.T, metrics []*prometheuspb.MetricFamily, expectedMetrics []string) { + foundMetrics := make(map[string]bool) + for _, metric := range metrics { + foundMetrics[metric.GetName()] = true + } + for _, expected := range expectedMetrics { + if _, found := foundMetrics[expected]; !found { + t.Errorf("Master metrics did not include expected metric %q", expected) + } + } +} + +func TestMasterProcessMetrics(t *testing.T) { + _, s := runAMaster(t) + defer s.Close() + + metrics, err := scrapeMetrics(s) + if err != nil { + t.Fatal(err) + } + checkForExpectedMetrics(t, metrics, []string{ + "process_start_time_seconds", + "process_cpu_seconds_total", + "process_goroutines", + "process_open_fds", + "process_resident_memory_bytes", + }) +} + +func TestApiserverMetrics(t *testing.T) { + _, s := runAMaster(t) + defer s.Close() + + // Make a request to the apiserver to ensure there's at least one data point + // for the metrics we're expecting -- otherwise, they won't be exported. + client := client.NewOrDie(&client.Config{Host: s.URL, Version: testapi.Version()}) + if _, err := client.Pods(api.NamespaceDefault).List(labels.Everything()); err != nil { + t.Fatalf("unexpected error getting pods: %v", err) + } + + metrics, err := scrapeMetrics(s) + if err != nil { + t.Fatal(err) + } + checkForExpectedMetrics(t, metrics, []string{ + "apiserver_request_count", + "apiserver_request_latencies", + }) +} diff --git a/test/integration/utils.go b/test/integration/utils.go index a1b442067bc..24b80703cf3 100644 --- a/test/integration/utils.go +++ b/test/integration/utils.go @@ -21,6 +21,15 @@ package integration import ( "fmt" "math/rand" + "net/http" + "net/http/httptest" + "testing" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi" + "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/master" + "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit" "github.com/coreos/go-etcd/etcd" "github.com/golang/glog" @@ -55,3 +64,27 @@ func deleteAllEtcdKeys() { } } + +func runAMaster(t *testing.T) (*master.Master, *httptest.Server) { + helper, err := master.NewEtcdHelper(newEtcdClient(), testapi.Version()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + m := master.New(&master.Config{ + EtcdHelper: helper, + KubeletClient: client.FakeKubeletClient{}, + EnableLogsSupport: false, + EnableProfiling: true, + EnableUISupport: false, + APIPrefix: "/api", + Authorizer: apiserver.NewAlwaysAllowAuthorizer(), + AdmissionControl: admit.NewAlwaysAdmit(), + }) + + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + m.Handler.ServeHTTP(w, req) + })) + + return m, s +}