From 4cff1311c5543af30a3cbb3fa873b3fadf9ccf7a Mon Sep 17 00:00:00 2001 From: Leila Jalali Date: Fri, 30 Oct 2020 11:15:50 +0000 Subject: [PATCH 1/3] Added tests to check metrics labels --- test/integration/metrics/metrics_test.go | 254 ++++++++++++++++++++++- 1 file changed, 253 insertions(+), 1 deletion(-) diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go index 9dd08bc8bb6..b04cd7e0669 100644 --- a/test/integration/metrics/metrics_test.go +++ b/test/integration/metrics/metrics_test.go @@ -18,6 +18,7 @@ package metrics import ( "context" + "errors" "fmt" "io/ioutil" "net/http" @@ -26,7 +27,6 @@ import ( "testing" "github.com/prometheus/common/model" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -241,3 +241,255 @@ func TestApiserverMetricsLabels(t *testing.T) { } } } + +func TestApiserverMetricsPods(t *testing.T) { + callOrDie := func(_ interface{}, err error) { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + + makePod := func(labelValue string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{"foo": labelValue}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container", + Image: "image", + }, + }, + }, + } + } + + _, s, closeFn := framework.RunAMaster(framework.NewMasterConfig()) + defer closeFn() + + client, err := clientset.NewForConfig(&restclient.Config{Host: s.URL, QPS: -1}) + if err != nil { + t.Fatalf("apiserver_request_total metric not exposed") + } + + c := client.CoreV1().Pods(metav1.NamespaceDefault) + + for _, tc := range []struct { + name string + executor func() + + want string + }{ + { + name: "create pod", + executor: func() { + callOrDie(c.Create(context.TODO(), makePod("foo"), metav1.CreateOptions{})) + }, + want: `apiserver_request_total{code="201", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="POST", version="v1"} => 1 @[0]`, + }, + { + name: "update pod", + executor: func() { + callOrDie(c.Update(context.TODO(), makePod("bar"), metav1.UpdateOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="PUT", version="v1"} => 1 @[0]`, + }, + { + name: "update pod status", + executor: func() { + callOrDie(c.UpdateStatus(context.TODO(), makePod("bar"), metav1.UpdateOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="status", verb="PUT", version="v1"} => 1 @[0]`, + }, + { + name: "get pod", + executor: func() { + callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="GET", version="v1"} => 1 @[0]`, + }, + { + name: "list pod", + executor: func() { + callOrDie(c.List(context.TODO(), metav1.ListOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="namespace", subresource="", verb="LIST", version="v1"} => 1 @[0]`, + }, + { + name: "delete pod", + executor: func() { + callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="DELETE", version="v1"} => 1 @[0]`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + + baseSamples, err := getSamples(s) + if err != nil { + t.Fatal(err) + } + + tc.executor() + + updatedSamples, err := getSamples(s) + if err != nil { + t.Fatal(err) + } + + newSamples := diffMetrics(updatedSamples, baseSamples) + found := false + + for _, s := range newSamples { + if s.String() == tc.want { + found = true; + break + } + } + + if !found { + t.Fatalf("could not find metric for API call >%s<", tc.name) + } + }) + } +} + +func TestApiserverMetricsNamespaces(t *testing.T) { + callOrDie := func(_ interface{}, err error) { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + + makeNamespace := func(labelValue string) *v1.Namespace { + return &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{"foo": labelValue}, + }, + } + } + + _, s, closeFn := framework.RunAMaster(framework.NewMasterConfig()) + defer closeFn() + + client, err := clientset.NewForConfig(&restclient.Config{Host: s.URL, QPS: -1}) + if err != nil { + t.Fatalf("apiserver_request_total metric not exposed") + } + + c := client.CoreV1().Namespaces() + + for _, tc := range []struct { + name string + executor func() + + want string + }{ + { + name: "create namespace", + executor: func() { + callOrDie(c.Create(context.TODO(), makeNamespace("foo"), metav1.CreateOptions{})) + }, + want: `apiserver_request_total{code="201", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="POST", version="v1"} => 5 @[0]`, + }, + { + name: "update namespace", + executor: func() { + callOrDie(c.Update(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="PUT", version="v1"} => 1 @[0]`, + }, + { + name: "update namespace status", + executor: func() { + callOrDie(c.UpdateStatus(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="status", verb="PUT", version="v1"} => 1 @[0]`, + }, + { + name: "get namespace", + executor: func() { + callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="GET", version="v1"} => 15 @[0]`, + }, + { + name: "list namespace", + executor: func() { + callOrDie(c.List(context.TODO(), metav1.ListOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="cluster", subresource="", verb="LIST", version="v1"} => 1 @[0]`, + }, + { + name: "delete namespace", + executor: func() { + callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{})) + }, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="DELETE", version="v1"} => 1 @[0]`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + + baseSamples, err := getSamples(s) + if err != nil { + t.Fatal(err) + } + + tc.executor() + + updatedSamples, err := getSamples(s) + if err != nil { + t.Fatal(err) + } + + newSamples := diffMetrics(updatedSamples, baseSamples) + found := false + + for _, s := range newSamples { + if s.String() == tc.want { + found = true; + break + } + } + + if !found { + t.Fatalf("could not find metric for API call >%s<", tc.name) + } + }) + } +} + +func getSamples(s *httptest.Server) (model.Samples, error) { + metrics, err := scrapeMetrics(s) + if err != nil { + return nil, err + } + + samples, ok := metrics["apiserver_request_total"] + if !ok { + return nil, errors.New("apiserver_request_total doesn't exist") + } + return samples, nil +} + +func diffMetrics(newSamples model.Samples, oldSamples model.Samples) model.Samples { + samplesDiff := model.Samples{} + for _, sample := range newSamples { + if !sampleExistsInSamples(sample, oldSamples) { + samplesDiff = append(samplesDiff, sample) + } + } + return samplesDiff +} + +func sampleExistsInSamples(s *model.Sample, samples model.Samples) bool { + for _, sample := range samples { + if sample.Equal(s) { + return true + } + } + return false +} From 9de601910f2a83d6663cb25c60911d2d08a0c3fe Mon Sep 17 00:00:00 2001 From: Leila Jalali Date: Wed, 4 Nov 2020 21:23:25 +0000 Subject: [PATCH 2/3] minor changes to tests for checking metrics labels based on review comments --- test/integration/metrics/metrics_test.go | 36 ++++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go index b04cd7e0669..aa67057cfdf 100644 --- a/test/integration/metrics/metrics_test.go +++ b/test/integration/metrics/metrics_test.go @@ -266,12 +266,12 @@ func TestApiserverMetricsPods(t *testing.T) { } } - _, s, closeFn := framework.RunAMaster(framework.NewMasterConfig()) + _, server, closeFn := framework.RunAMaster(framework.NewMasterConfig()) defer closeFn() - client, err := clientset.NewForConfig(&restclient.Config{Host: s.URL, QPS: -1}) + client, err := clientset.NewForConfig(&restclient.Config{Host: server.URL, QPS: -1}) if err != nil { - t.Fatalf("apiserver_request_total metric not exposed") + t.Fatalf("Error in create clientset: %v", err) } c := client.CoreV1().Pods(metav1.NamespaceDefault) @@ -327,14 +327,14 @@ func TestApiserverMetricsPods(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { - baseSamples, err := getSamples(s) + baseSamples, err := getSamples(server) if err != nil { t.Fatal(err) } tc.executor() - updatedSamples, err := getSamples(s) + updatedSamples, err := getSamples(server) if err != nil { t.Fatal(err) } @@ -342,15 +342,15 @@ func TestApiserverMetricsPods(t *testing.T) { newSamples := diffMetrics(updatedSamples, baseSamples) found := false - for _, s := range newSamples { - if s.String() == tc.want { - found = true; + for _, sample := range newSamples { + if sample.String() == tc.want { + found = true break } } if !found { - t.Fatalf("could not find metric for API call >%s<", tc.name) + t.Fatalf("could not find metric for API call >%s< among samples >%+v<", tc.name, newSamples) } }) } @@ -372,12 +372,12 @@ func TestApiserverMetricsNamespaces(t *testing.T) { } } - _, s, closeFn := framework.RunAMaster(framework.NewMasterConfig()) + _, server, closeFn := framework.RunAMaster(framework.NewMasterConfig()) defer closeFn() - client, err := clientset.NewForConfig(&restclient.Config{Host: s.URL, QPS: -1}) + client, err := clientset.NewForConfig(&restclient.Config{Host: server.URL, QPS: -1}) if err != nil { - t.Fatalf("apiserver_request_total metric not exposed") + t.Fatalf("Error in create clientset: %v", err) } c := client.CoreV1().Namespaces() @@ -433,14 +433,14 @@ func TestApiserverMetricsNamespaces(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { - baseSamples, err := getSamples(s) + baseSamples, err := getSamples(server) if err != nil { t.Fatal(err) } tc.executor() - updatedSamples, err := getSamples(s) + updatedSamples, err := getSamples(server) if err != nil { t.Fatal(err) } @@ -448,15 +448,15 @@ func TestApiserverMetricsNamespaces(t *testing.T) { newSamples := diffMetrics(updatedSamples, baseSamples) found := false - for _, s := range newSamples { - if s.String() == tc.want { - found = true; + for _, sample := range newSamples { + if sample.String() == tc.want { + found = true break } } if !found { - t.Fatalf("could not find metric for API call >%s<", tc.name) + t.Fatalf("could not find metric for API call >%s< among samples >%+v<", tc.name, newSamples) } }) } From 79e1ba1ae6584c53e4f2547f80086b0ab77ac1ba Mon Sep 17 00:00:00 2001 From: Leila Jalali Date: Thu, 5 Nov 2020 18:42:28 +0000 Subject: [PATCH 3/3] minor changes in adding tests for checking metrics labels --- test/integration/metrics/metrics_test.go | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go index aa67057cfdf..bdda9c0ce96 100644 --- a/test/integration/metrics/metrics_test.go +++ b/test/integration/metrics/metrics_test.go @@ -287,42 +287,42 @@ func TestApiserverMetricsPods(t *testing.T) { executor: func() { callOrDie(c.Create(context.TODO(), makePod("foo"), metav1.CreateOptions{})) }, - want: `apiserver_request_total{code="201", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="POST", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="201", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="POST", version="v1"}`, }, { name: "update pod", executor: func() { callOrDie(c.Update(context.TODO(), makePod("bar"), metav1.UpdateOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="PUT", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="PUT", version="v1"}`, }, { name: "update pod status", executor: func() { callOrDie(c.UpdateStatus(context.TODO(), makePod("bar"), metav1.UpdateOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="status", verb="PUT", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="status", verb="PUT", version="v1"}`, }, { name: "get pod", executor: func() { callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="GET", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="GET", version="v1"}`, }, { name: "list pod", executor: func() { callOrDie(c.List(context.TODO(), metav1.ListOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="namespace", subresource="", verb="LIST", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="namespace", subresource="", verb="LIST", version="v1"}`, }, { name: "delete pod", executor: func() { callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="DELETE", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="DELETE", version="v1"}`, }, } { t.Run(tc.name, func(t *testing.T) { @@ -343,7 +343,7 @@ func TestApiserverMetricsPods(t *testing.T) { found := false for _, sample := range newSamples { - if sample.String() == tc.want { + if sample.Metric.String() == tc.want { found = true break } @@ -393,42 +393,42 @@ func TestApiserverMetricsNamespaces(t *testing.T) { executor: func() { callOrDie(c.Create(context.TODO(), makeNamespace("foo"), metav1.CreateOptions{})) }, - want: `apiserver_request_total{code="201", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="POST", version="v1"} => 5 @[0]`, + want: `apiserver_request_total{code="201", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="POST", version="v1"}`, }, { name: "update namespace", executor: func() { callOrDie(c.Update(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="PUT", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="PUT", version="v1"}`, }, { name: "update namespace status", executor: func() { callOrDie(c.UpdateStatus(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="status", verb="PUT", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="status", verb="PUT", version="v1"}`, }, { name: "get namespace", executor: func() { callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="GET", version="v1"} => 15 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="GET", version="v1"}`, }, { name: "list namespace", executor: func() { callOrDie(c.List(context.TODO(), metav1.ListOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="cluster", subresource="", verb="LIST", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="cluster", subresource="", verb="LIST", version="v1"}`, }, { name: "delete namespace", executor: func() { callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{})) }, - want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="DELETE", version="v1"} => 1 @[0]`, + want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="DELETE", version="v1"}`, }, } { t.Run(tc.name, func(t *testing.T) { @@ -449,7 +449,7 @@ func TestApiserverMetricsNamespaces(t *testing.T) { found := false for _, sample := range newSamples { - if sample.String() == tc.want { + if sample.Metric.String() == tc.want { found = true break }