diff --git a/pkg/kubelet/metrics/metrics.go b/pkg/kubelet/metrics/metrics.go index 1b73082c098..86b84c244fd 100644 --- a/pkg/kubelet/metrics/metrics.go +++ b/pkg/kubelet/metrics/metrics.go @@ -26,18 +26,24 @@ import ( ) const ( - KubeletSubsystem = "kubelet" - PodWorkerLatencyKey = "pod_worker_latency_microseconds" - PodStartLatencyKey = "pod_start_latency_microseconds" - CgroupManagerOperationsKey = "cgroup_manager_latency_microseconds" - DockerOperationsLatencyKey = "docker_operations_latency_microseconds" - DockerOperationsKey = "docker_operations" - DockerOperationsErrorsKey = "docker_operations_errors" - DockerOperationsTimeoutKey = "docker_operations_timeout" - PodWorkerStartLatencyKey = "pod_worker_start_latency_microseconds" - PLEGRelistLatencyKey = "pleg_relist_latency_microseconds" - PLEGRelistIntervalKey = "pleg_relist_interval_microseconds" - EvictionStatsAgeKey = "eviction_stats_age_microseconds" + KubeletSubsystem = "kubelet" + PodWorkerLatencyKey = "pod_worker_latency_microseconds" + PodStartLatencyKey = "pod_start_latency_microseconds" + CgroupManagerOperationsKey = "cgroup_manager_latency_microseconds" + DockerOperationsLatencyKey = "docker_operations_latency_microseconds" + DockerOperationsKey = "docker_operations" + DockerOperationsErrorsKey = "docker_operations_errors" + DockerOperationsTimeoutKey = "docker_operations_timeout" + PodWorkerStartLatencyKey = "pod_worker_start_latency_microseconds" + PLEGRelistLatencyKey = "pleg_relist_latency_microseconds" + PLEGRelistIntervalKey = "pleg_relist_interval_microseconds" + EvictionStatsAgeKey = "eviction_stats_age_microseconds" + VolumeStatsCapacityBytesKey = "volume_stats_capacity_bytes" + VolumeStatsAvailableBytesKey = "volume_stats_available_bytes" + VolumeStatsUsedBytesKey = "volume_stats_used_bytes" + VolumeStatsInodesKey = "volume_stats_inodes" + VolumeStatsInodesFreeKey = "volume_stats_inodes_free" + VolumeStatsInodesUsedKey = "volume_stats_inodes_used" // Metrics keys of remote runtime operations RuntimeOperationsKey = "runtime_operations" RuntimeOperationsLatencyKey = "runtime_operations_latency_microseconds" @@ -162,6 +168,54 @@ var ( }, []string{"eviction_signal"}, ) + VolumeStatsCapacityBytes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsCapacityBytesKey, + Help: "Capacity in bytes of the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsAvailableBytes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsAvailableBytesKey, + Help: "Number of available bytes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsUsedBytes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsUsedBytesKey, + Help: "Number of used bytes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsInodes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsInodesKey, + Help: "Maximum number of inodes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsInodesFree = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsInodesFreeKey, + Help: "Number of free inodes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) + VolumeStatsInodesUsed = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: KubeletSubsystem, + Name: VolumeStatsInodesUsedKey, + Help: "Number of used inodes in the volume", + }, + []string{"namespace", "persistentvolumeclaim"}, + ) ) var registerMetrics sync.Once @@ -186,6 +240,12 @@ func Register(containerCache kubecontainer.RuntimeCache) { prometheus.MustRegister(RuntimeOperationsLatency) prometheus.MustRegister(RuntimeOperationsErrors) prometheus.MustRegister(EvictionStatsAge) + prometheus.MustRegister(VolumeStatsCapacityBytes) + prometheus.MustRegister(VolumeStatsAvailableBytes) + prometheus.MustRegister(VolumeStatsUsedBytes) + prometheus.MustRegister(VolumeStatsInodes) + prometheus.MustRegister(VolumeStatsInodesFree) + prometheus.MustRegister(VolumeStatsInodesUsed) }) } diff --git a/pkg/kubelet/server/stats/BUILD b/pkg/kubelet/server/stats/BUILD index 56b5c1c3fda..c4f93551e2b 100644 --- a/pkg/kubelet/server/stats/BUILD +++ b/pkg/kubelet/server/stats/BUILD @@ -15,6 +15,7 @@ go_library( "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container:go_default_library", + "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/util/format:go_default_library", "//pkg/volume:go_default_library", "//vendor/github.com/emicklei/go-restful:go_default_library", diff --git a/pkg/kubelet/server/stats/volume_stat_calculator.go b/pkg/kubelet/server/stats/volume_stat_calculator.go index e3c186c75a9..e151ab83d2d 100644 --- a/pkg/kubelet/server/stats/volume_stat_calculator.go +++ b/pkg/kubelet/server/stats/volume_stat_calculator.go @@ -24,6 +24,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/volume" @@ -85,6 +86,7 @@ func (s *volumeStatCalculator) GetLatest() (PodVolumeStats, bool) { } // calcAndStoreStats calculates PodVolumeStats for a given pod and writes the result to the s.latest cache. +// If the pod references PVCs, the prometheus metrics for those are updated with the result. func (s *volumeStatCalculator) calcAndStoreStats() { // Find all Volumes for the Pod volumes, found := s.statsProvider.ListVolumesForPod(s.pod.UID) @@ -117,6 +119,8 @@ func (s *volumeStatCalculator) calcAndStoreStats() { Namespace: s.pod.GetNamespace(), } fsStats = append(fsStats, s.parsePodVolumeStats(name, &pvcRef, metric)) + // Set the PVC's prometheus metrics + s.setPVCMetrics(&pvcRef, metric) } else { fsStats = append(fsStats, s.parsePodVolumeStats(name, nil, metric)) } @@ -141,3 +145,13 @@ func (s *volumeStatCalculator) parsePodVolumeStats(podName string, pvcRef *stats UsedBytes: &used, Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed}, } } + +// setPVCMetrics sets the given PVC's prometheus metrics to match the given volume.Metrics +func (s *volumeStatCalculator) setPVCMetrics(pvcRef *stats.PVCReference, metric *volume.Metrics) { + metrics.VolumeStatsAvailableBytes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Available.Value())) + metrics.VolumeStatsCapacityBytes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Capacity.Value())) + metrics.VolumeStatsUsedBytes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Used.Value())) + metrics.VolumeStatsInodes.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.Inodes.Value())) + metrics.VolumeStatsInodesFree.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.InodesFree.Value())) + metrics.VolumeStatsInodesUsed.WithLabelValues(pvcRef.Namespace, pvcRef.Name).Set(float64(metric.InodesUsed.Value())) +}