From ccd87fca3f855b4de14a8852193f884ba36e91b7 Mon Sep 17 00:00:00 2001 From: Seth Jennings Date: Mon, 27 Feb 2017 15:13:31 -0600 Subject: [PATCH] kubelet: add cgroup manager metrics --- pkg/kubelet/cm/BUILD | 1 + pkg/kubelet/cm/cgroup_manager_linux.go | 16 ++++++++++++++++ pkg/kubelet/container/BUILD | 1 - pkg/kubelet/container/helpers.go | 3 +-- pkg/kubelet/container/testing/BUILD | 1 - .../container/testing/fake_runtime_helper.go | 5 ++--- pkg/kubelet/kubelet_pods.go | 7 ++++--- pkg/kubelet/kuberuntime/kuberuntime_sandbox.go | 2 +- pkg/kubelet/metrics/metrics.go | 10 ++++++++++ test/e2e/framework/kubelet_stats.go | 1 + 10 files changed, 36 insertions(+), 11 deletions(-) diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index acf1ed53af2..200b2ade671 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -30,6 +30,7 @@ go_library( "//pkg/kubelet/cm/util:go_default_library", "//pkg/kubelet/events:go_default_library", "//pkg/kubelet/eviction/api:go_default_library", + "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/qos:go_default_library", "//pkg/util:go_default_library", "//pkg/util/mount:go_default_library", diff --git a/pkg/kubelet/cm/cgroup_manager_linux.go b/pkg/kubelet/cm/cgroup_manager_linux.go index e8fcca3ec36..9590b26d146 100644 --- a/pkg/kubelet/cm/cgroup_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_manager_linux.go @@ -22,6 +22,7 @@ import ( "path" "path/filepath" "strings" + "time" "github.com/golang/glog" libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups" @@ -29,6 +30,7 @@ import ( cgroupsystemd "github.com/opencontainers/runc/libcontainer/cgroups/systemd" libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/pkg/kubelet/metrics" ) // libcontainerCgroupManagerType defines how to interface with libcontainer @@ -237,6 +239,11 @@ func (m *cgroupManagerImpl) Exists(name CgroupName) bool { // Destroy destroys the specified cgroup func (m *cgroupManagerImpl) Destroy(cgroupConfig *CgroupConfig) error { + start := time.Now() + defer func() { + metrics.CgroupManagerLatency.WithLabelValues("destroy").Observe(metrics.SinceInMicroseconds(start)) + }() + cgroupPaths := m.buildCgroupPaths(cgroupConfig.Name) // we take the location in traditional cgroupfs format. @@ -329,6 +336,11 @@ func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcont // Update updates the cgroup with the specified Cgroup Configuration func (m *cgroupManagerImpl) Update(cgroupConfig *CgroupConfig) error { + start := time.Now() + defer func() { + metrics.CgroupManagerLatency.WithLabelValues("update").Observe(metrics.SinceInMicroseconds(start)) + }() + // Extract the cgroup resource parameters resourceConfig := cgroupConfig.ResourceParameters resources := m.toResources(resourceConfig) @@ -364,6 +376,10 @@ func (m *cgroupManagerImpl) Update(cgroupConfig *CgroupConfig) error { // Create creates the specified cgroup func (m *cgroupManagerImpl) Create(cgroupConfig *CgroupConfig) error { + start := time.Now() + defer func() { + metrics.CgroupManagerLatency.WithLabelValues("create").Observe(metrics.SinceInMicroseconds(start)) + }() // we take the location in traditional cgroupfs format. abstractCgroupFsName := string(cgroupConfig.Name) diff --git a/pkg/kubelet/container/BUILD b/pkg/kubelet/container/BUILD index 94375356ce2..ffac0682657 100644 --- a/pkg/kubelet/container/BUILD +++ b/pkg/kubelet/container/BUILD @@ -29,7 +29,6 @@ go_library( "//pkg/api:go_default_library", "//pkg/api/v1:go_default_library", "//pkg/kubelet/api/v1alpha1/runtime:go_default_library", - "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/events:go_default_library", "//pkg/kubelet/util/format:go_default_library", "//pkg/kubelet/util/ioutils:go_default_library", diff --git a/pkg/kubelet/container/helpers.go b/pkg/kubelet/container/helpers.go index 2c1be7660ca..4c5344be990 100644 --- a/pkg/kubelet/container/helpers.go +++ b/pkg/kubelet/container/helpers.go @@ -33,7 +33,6 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api/v1" runtimeapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" - "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/kubelet/util/ioutils" @@ -53,7 +52,7 @@ type RuntimeHelper interface { GetClusterDNS(pod *v1.Pod) (dnsServers []string, dnsSearches []string, useClusterFirstPolicy bool, err error) // GetPodCgroupParent returns the the CgroupName identifer, and its literal cgroupfs form on the host // of a pod. - GetPodCgroupParent(pod *v1.Pod) (cm.CgroupName, string) + GetPodCgroupParent(pod *v1.Pod) string GetPodDir(podUID types.UID) string GeneratePodHostNameAndDomain(pod *v1.Pod) (hostname string, hostDomain string, err error) // GetExtraSupplementalGroupsForPod returns a list of the extra diff --git a/pkg/kubelet/container/testing/BUILD b/pkg/kubelet/container/testing/BUILD index 570acd8e23d..39d10eae9fb 100644 --- a/pkg/kubelet/container/testing/BUILD +++ b/pkg/kubelet/container/testing/BUILD @@ -20,7 +20,6 @@ go_library( tags = ["automanaged"], deps = [ "//pkg/api/v1:go_default_library", - "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/util/term:go_default_library", "//pkg/volume:go_default_library", diff --git a/pkg/kubelet/container/testing/fake_runtime_helper.go b/pkg/kubelet/container/testing/fake_runtime_helper.go index c81c21de2df..76bbd0a4bc3 100644 --- a/pkg/kubelet/container/testing/fake_runtime_helper.go +++ b/pkg/kubelet/container/testing/fake_runtime_helper.go @@ -19,7 +19,6 @@ package testing import ( kubetypes "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/api/v1" - "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -41,8 +40,8 @@ func (f *FakeRuntimeHelper) GenerateRunContainerOptions(pod *v1.Pod, container * return &opts, false, nil } -func (f *FakeRuntimeHelper) GetPodCgroupParent(pod *v1.Pod) (cm.CgroupName, string) { - return "", "" +func (f *FakeRuntimeHelper) GetPodCgroupParent(pod *v1.Pod) string { + return "" } func (f *FakeRuntimeHelper) GetClusterDNS(pod *v1.Pod) ([]string, []string, bool, error) { diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 675c75e1f0e..659e7ff8196 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -267,9 +267,10 @@ func (kl *Kubelet) GeneratePodHostNameAndDomain(pod *v1.Pod) (string, string, er } // GetPodCgroupParent gets pod cgroup parent from container manager. -func (kl *Kubelet) GetPodCgroupParent(pod *v1.Pod) (cm.CgroupName, string) { +func (kl *Kubelet) GetPodCgroupParent(pod *v1.Pod) string { pcm := kl.containerManager.NewPodContainerManager() - return pcm.GetPodContainerName(pod) + _, cgroupParent := pcm.GetPodContainerName(pod) + return cgroupParent } // GenerateRunContainerOptions generates the RunContainerOptions, which can be used by @@ -277,7 +278,7 @@ func (kl *Kubelet) GetPodCgroupParent(pod *v1.Pod) (cm.CgroupName, string) { func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, bool, error) { var err error useClusterFirstPolicy := false - _, cgroupParent := kl.GetPodCgroupParent(pod) + cgroupParent := kl.GetPodCgroupParent(pod) opts := &kubecontainer.RunContainerOptions{CgroupParent: cgroupParent} hostname, hostDomainName, err := kl.GeneratePodHostNameAndDomain(pod) if err != nil { diff --git a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go index 88e20f98c0d..9089d5f9b15 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go @@ -114,7 +114,7 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxConfig(pod *v1.Pod, attemp } - _, cgroupParent := m.runtimeHelper.GetPodCgroupParent(pod) + cgroupParent := m.runtimeHelper.GetPodCgroupParent(pod) podSandboxConfig.Linux = m.generatePodSandboxLinuxConfig(pod, cgroupParent) if len(portMappings) > 0 { podSandboxConfig.PortMappings = portMappings diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 433f57715f3..682e6a94610 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -32,6 +32,7 @@ const ( PodStartLatencyKey = "pod_start_latency_microseconds" PodStatusLatencyKey = "generate_pod_status_latency_microseconds" ContainerManagerOperationsKey = "container_manager_latency_microseconds" + CgroupManagerOperationsKey = "cgroup_manager_latency_microseconds" DockerOperationsLatencyKey = "docker_operations_latency_microseconds" DockerOperationsKey = "docker_operations" DockerOperationsErrorsKey = "docker_operations_errors" @@ -90,6 +91,14 @@ var ( }, []string{"operation_type"}, ) + CgroupManagerLatency = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Subsystem: KubeletSubsystem, + Name: CgroupManagerOperationsKey, + Help: "Latency in microseconds for cgroup manager operations. Broken down by method.", + }, + []string{"operation_type"}, + ) PodWorkerStartLatency = prometheus.NewSummary( prometheus.SummaryOpts{ Subsystem: KubeletSubsystem, @@ -182,6 +191,7 @@ func Register(containerCache kubecontainer.RuntimeCache) { prometheus.MustRegister(PodStatusLatency) prometheus.MustRegister(DockerOperationsLatency) prometheus.MustRegister(ContainerManagerLatency) + prometheus.MustRegister(CgroupManagerLatency) prometheus.MustRegister(SyncPodsLatency) prometheus.MustRegister(PodWorkerStartLatency) prometheus.MustRegister(ContainersPerPodCount) diff --git a/test/e2e/framework/kubelet_stats.go b/test/e2e/framework/kubelet_stats.go index ba5810b9231..7b9f889c71c 100644 --- a/test/e2e/framework/kubelet_stats.go +++ b/test/e2e/framework/kubelet_stats.go @@ -106,6 +106,7 @@ func GetKubeletLatencyMetrics(ms metrics.KubeletMetrics) KubeletLatencyMetrics { kubeletmetrics.PodStartLatencyKey, kubeletmetrics.PodStatusLatencyKey, kubeletmetrics.ContainerManagerOperationsKey, + kubeletmetrics.CgroupManagerOperationsKey, kubeletmetrics.DockerOperationsLatencyKey, kubeletmetrics.PodWorkerStartLatencyKey, kubeletmetrics.PLEGRelistLatencyKey,