From c4c25747789d7a394ae31501f7c0b5beb92e711d Mon Sep 17 00:00:00 2001 From: Jian Zeng Date: Mon, 3 May 2021 00:12:06 +0800 Subject: [PATCH] refactor(e2e): grab metrics from controller-manager via nginx Signed-off-by: Jian Zeng --- test/e2e/e2e.go | 6 + test/e2e/framework/metrics/metrics_grabber.go | 25 +-- test/e2e/framework/metrics/metrics_proxy.go | 199 ++++++++++++++++++ test/e2e/framework/pod/resource.go | 2 +- test/e2e/framework/ports.go | 4 - test/e2e/framework/providers/gce/firewall.go | 2 +- .../monitoring/metrics_grabber.go | 6 +- 7 files changed, 221 insertions(+), 23 deletions(-) create mode 100644 test/e2e/framework/metrics/metrics_proxy.go diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 9b826b06112..28f72057122 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -46,6 +46,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" e2emanifest "k8s.io/kubernetes/test/e2e/framework/manifest" + "k8s.io/kubernetes/test/e2e/framework/metrics" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2ereporters "k8s.io/kubernetes/test/e2e/reporters" @@ -307,6 +308,11 @@ func setupSuite() { nodeKiller := framework.NewNodeKiller(framework.TestContext.NodeKiller, c, framework.TestContext.Provider) go nodeKiller.Run(framework.TestContext.NodeKiller.NodeKillerStopCh) } + + err = metrics.SetupMetricsProxy(c) + if err != nil { + framework.Logf("Fail to setup metrics proxy: %v", err) + } } // logClusterImageSources writes out cluster image sources. diff --git a/test/e2e/framework/metrics/metrics_grabber.go b/test/e2e/framework/metrics/metrics_grabber.go index 1d341cc9360..24d97a7c0fd 100644 --- a/test/e2e/framework/metrics/metrics_grabber.go +++ b/test/e2e/framework/metrics/metrics_grabber.go @@ -27,20 +27,17 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" - e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - "k8s.io/klog/v2" + + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" ) const ( - // insecureSchedulerPort is the default port for the scheduler status server. - // May be overridden by a flag at startup. - // Deprecated: use the secure KubeSchedulerPort instead. - insecureSchedulerPort = 10251 - // insecureKubeControllerManagerPort is the default port for the controller manager status server. - // May be overridden by a flag at startup. - // Deprecated: use the secure KubeControllerManagerPort instead. - insecureKubeControllerManagerPort = 10252 + // kubeSchedulerPort is the default port for the scheduler status server. + kubeSchedulerPort = 10259 + // kubeControllerManagerPort is the default port for the controller manager status server. + kubeControllerManagerPort = 10257 + metricsProxyPod = "metrics-proxy" ) // Collection is metrics collection of components @@ -152,7 +149,7 @@ func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) { if g.kubeScheduler == "" { return SchedulerMetrics{}, fmt.Errorf("kube-scheduler pod is not registered. Skipping Scheduler's metrics gathering") } - output, err := g.getMetricsFromPod(g.client, g.kubeScheduler, metav1.NamespaceSystem, insecureSchedulerPort) + output, err := g.getMetricsFromPod(g.client, metricsProxyPod, metav1.NamespaceSystem, kubeSchedulerPort) if err != nil { return SchedulerMetrics{}, err } @@ -196,7 +193,7 @@ func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) var lastMetricsFetchErr error if metricsWaitErr := wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { - _, lastMetricsFetchErr = g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, insecureKubeControllerManagerPort) + _, lastMetricsFetchErr = g.getMetricsFromPod(g.client, metricsProxyPod, metav1.NamespaceSystem, kubeControllerManagerPort) return lastMetricsFetchErr == nil, nil }); metricsWaitErr != nil { err = fmt.Errorf("error waiting for controller manager pod to expose metrics: %v; %v", metricsWaitErr, lastMetricsFetchErr) @@ -207,7 +204,7 @@ func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) return ControllerManagerMetrics{}, err } - output, err := g.getMetricsFromPod(g.client, podName, metav1.NamespaceSystem, insecureKubeControllerManagerPort) + output, err := g.getMetricsFromPod(g.client, metricsProxyPod, metav1.NamespaceSystem, kubeControllerManagerPort) if err != nil { return ControllerManagerMetrics{}, err } @@ -286,7 +283,7 @@ func (g *Grabber) getMetricsFromPod(client clientset.Interface, podName string, Namespace(namespace). Resource("pods"). SubResource("proxy"). - Name(fmt.Sprintf("%v:%v", podName, port)). + Name(fmt.Sprintf("%s:%d", podName, port)). Suffix("metrics"). Do(context.TODO()).Raw() if err != nil { diff --git a/test/e2e/framework/metrics/metrics_proxy.go b/test/e2e/framework/metrics/metrics_proxy.go new file mode 100644 index 00000000000..f096238cf8f --- /dev/null +++ b/test/e2e/framework/metrics/metrics_proxy.go @@ -0,0 +1,199 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "fmt" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + imageutils "k8s.io/kubernetes/test/utils/image" +) + +type componentInfo struct { + Port int + IP string +} + +// SetupMetricsProxy creates a nginx Pod to expose metrics from the secure port of kube-scheduler and kube-controller-manager in tests. +func SetupMetricsProxy(c clientset.Interface) error { + podList, err := c.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + var infos []componentInfo + for _, pod := range podList.Items { + switch { + case strings.HasPrefix(pod.Name, "kube-scheduler-"): + infos = append(infos, componentInfo{ + Port: kubeSchedulerPort, + IP: pod.Status.PodIP, + }) + case strings.HasPrefix(pod.Name, "kube-controller-manager-"): + infos = append(infos, componentInfo{ + Port: kubeControllerManagerPort, + IP: pod.Status.PodIP, + }) + } + if len(infos) == 2 { + break + } + } + if len(infos) == 0 { + klog.Warningf("Can't find any pods in namespace %s to grab metrics from", metav1.NamespaceSystem) + return nil + } + + const name = metricsProxyPod + _, err = c.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Create(context.TODO(), &v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + }, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create serviceAccount: %w", err) + } + _, err = c.RbacV1().ClusterRoles().Create(context.TODO(), &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Rules: []rbacv1.PolicyRule{ + { + NonResourceURLs: []string{"/metrics"}, + Verbs: []string{"get"}, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create clusterRole: %w", err) + } + _, err = c.RbacV1().ClusterRoleBindings().Create(context.TODO(), &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: name, + Namespace: metav1.NamespaceSystem, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: name, + }, + }, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create clusterRoleBinding: %w", err) + } + + var token string + err = wait.PollImmediate(time.Second*5, time.Minute*5, func() (done bool, err error) { + sa, err := c.CoreV1().ServiceAccounts(metav1.NamespaceSystem).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + klog.Warningf("Fail to get serviceAccount %s: %v", name, err) + return false, nil + } + if len(sa.Secrets) < 1 { + klog.Warningf("No secret found in serviceAccount %s", name) + return false, nil + } + secretRef := sa.Secrets[0] + secret, err := c.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), secretRef.Name, metav1.GetOptions{}) + if err != nil { + klog.Warningf("Fail to get secret %s", secretRef.Name) + return false, nil + } + token = string(secret.Data["token"]) + if len(token) == 0 { + klog.Warningf("Token in secret %s is empty", secretRef.Name) + return false, nil + } + return true, nil + }) + if err != nil { + return err + } + + var nginxConfig string + for _, info := range infos { + nginxConfig += fmt.Sprintf(` +server { + listen %d; + server_name _; + proxy_set_header Authorization "Bearer %s"; + proxy_ssl_verify off; + location /metrics { + proxy_pass https://%s:%d; + } +} +`, info.Port, token, info.IP, info.Port) + } + _, err = c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Create(context.TODO(), &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{ + "metrics.conf": nginxConfig, + }, + }, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create nginx configmap: %w", err) + } + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: metav1.NamespaceSystem, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: "nginx", + Image: imageutils.GetE2EImage(imageutils.Nginx), + VolumeMounts: []v1.VolumeMount{{ + Name: "config", + MountPath: "/etc/nginx/conf.d", + ReadOnly: true, + }}, + }}, + Volumes: []v1.Volume{{ + Name: "config", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: name, + }, + }, + }, + }}, + }, + } + _, err = c.CoreV1().Pods(metav1.NamespaceSystem).Create(context.TODO(), pod, metav1.CreateOptions{}) + if err != nil { + return err + } + err = e2epod.WaitForPodNameRunningInNamespace(c, name, metav1.NamespaceSystem) + if err != nil { + return err + } + klog.Info("Successfully setup metrics-proxy") + return nil +} diff --git a/test/e2e/framework/pod/resource.go b/test/e2e/framework/pod/resource.go index 668cf263ad7..9c34125b0d1 100644 --- a/test/e2e/framework/pod/resource.go +++ b/test/e2e/framework/pod/resource.go @@ -25,7 +25,6 @@ import ( "github.com/onsi/ginkgo" "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -34,6 +33,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "k8s.io/kubectl/pkg/util/podutils" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" testutils "k8s.io/kubernetes/test/utils" imageutils "k8s.io/kubernetes/test/utils/image" diff --git a/test/e2e/framework/ports.go b/test/e2e/framework/ports.go index 5eb08c6a8fb..38326a12234 100644 --- a/test/e2e/framework/ports.go +++ b/test/e2e/framework/ports.go @@ -22,10 +22,6 @@ const ( // KubeletPort is the default port for the kubelet server on each host machine. // May be overridden by a flag at startup. KubeletPort = 10250 - // InsecureKubeControllerManagerPort is the default port for the controller manager status server. - // May be overridden by a flag at startup. - // Deprecated: use the secure KubeControllerManagerPort instead. - InsecureKubeControllerManagerPort = 10252 // KubeControllerManagerPort is the default port for the controller manager status server. // May be overridden by a flag at startup. KubeControllerManagerPort = 10257 diff --git a/test/e2e/framework/providers/gce/firewall.go b/test/e2e/framework/providers/gce/firewall.go index 6126a9a6238..76478598f9e 100644 --- a/test/e2e/framework/providers/gce/firewall.go +++ b/test/e2e/framework/providers/gce/firewall.go @@ -25,7 +25,7 @@ import ( compute "google.golang.org/api/compute/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" cloudprovider "k8s.io/cloud-provider" diff --git a/test/e2e/instrumentation/monitoring/metrics_grabber.go b/test/e2e/instrumentation/monitoring/metrics_grabber.go index 4ddede2d5d1..b4f2fbaab1b 100644 --- a/test/e2e/instrumentation/monitoring/metrics_grabber.go +++ b/test/e2e/instrumentation/monitoring/metrics_grabber.go @@ -22,15 +22,15 @@ import ( "strings" "time" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" e2emetrics "k8s.io/kubernetes/test/e2e/framework/metrics" e2enode "k8s.io/kubernetes/test/e2e/framework/node" instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common" - - "github.com/onsi/ginkgo" - "github.com/onsi/gomega" ) var _ = instrumentation.SIGDescribe("MetricsGrabber", func() {