From ce20c3b1006bbbcff8a7708dce1b3ea52ad220b0 Mon Sep 17 00:00:00 2001 From: "Tim St. Clair" Date: Thu, 10 Dec 2015 15:39:23 -0800 Subject: [PATCH 1/3] Add new Stats API for serving pod level stats This API has been discussed ad nauseam across several forums, and this API represents the latest conclusion. In summary, we will provide this API as temporary solution for providing the new stats required for 1.2. In the longterm this API will be split into "essential" stats, which will be provided by a first-party API served through the kubelet, and "non-essential" (monitoring) stats, which will be provided by a 3rd party API served from a pod. --- pkg/kubelet/server/stats/doc.go | 20 ++++ pkg/kubelet/server/stats/types.go | 158 ++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 pkg/kubelet/server/stats/doc.go create mode 100644 pkg/kubelet/server/stats/types.go diff --git a/pkg/kubelet/server/stats/doc.go b/pkg/kubelet/server/stats/doc.go new file mode 100644 index 00000000000..289fdae7064 --- /dev/null +++ b/pkg/kubelet/server/stats/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package stats handles exporting Kubelet and container stats. +// NOTE: We intend to move this functionality into a standalone pod, so this package should be very +// loosely coupled to the rest of the Kubelet. +package stats diff --git a/pkg/kubelet/server/stats/types.go b/pkg/kubelet/server/stats/types.go new file mode 100644 index 00000000000..af056d99c73 --- /dev/null +++ b/pkg/kubelet/server/stats/types.go @@ -0,0 +1,158 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/api/unversioned" +) + +// Summary is a top-level container for holding NodeStats and PodStats. +type Summary struct { + // Overall node stats. + Node NodeStats `json:"node"` + // Per-pod stats. + Pods []PodStats `json:"pods"` +} + +// NodeStats holds node-level unprocessed sample stats. +type NodeStats struct { + // Reference to the measured Node. + NodeName string `json:"nodeName"` + // Overall node stats. + Total []NodeSample `json:"total,omitempty" patchStrategy:"merge" patchMergeKey:"sampleTime"` + // Stats of system daemons tracked as raw containers, which may include: + // "/kubelet", "/docker-daemon", "kube-proxy" - Tracks respective component stats + // "/system" - Tracks stats of non-kubernetes and non-kernel processes (grouped together) + SystemContainers []ContainerStats `json:"systemContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name"` +} + +// PodStats holds pod-level unprocessed sample stats. +type PodStats struct { + // Reference to the measured Pod. + PodRef NonLocalObjectReference `json:"podRef"` + // Stats of containers in the measured pod. + Containers []ContainerStats `json:"containers" patchStrategy:"merge" patchMergeKey:"name"` + // Historical stat samples of pod-level resources. + Samples []PodSample `json:"samples" patchStrategy:"merge" patchMergeKey:"sampleTime"` +} + +// ContainerStats holds container-level unprocessed sample stats. +type ContainerStats struct { + // Reference to the measured container. + Name string `json:"name"` + // Historical stat samples gathered from the container. + Samples []ContainerSample `json:"samples" patchStrategy:"merge" patchMergeKey:"sampleTime"` +} + +// NonLocalObjectReference contains enough information to locate the referenced object. +type NonLocalObjectReference struct { + Name string `json:"name"` + Namespace string `json:"namespace"` +} + +// Sample defines metadata common to all sample types. +// Samples may not be nested within other samples. +type Sample struct { + // The time this data point was collected at. + SampleTime unversioned.Time `json:"sampleTime"` +} + +// NodeSample contains a sample point of data aggregated over a node. +type NodeSample struct { + Sample `json:",inline"` + // Stats pertaining to CPU resources. + CPU *CPUStats `json:"cpu,omitempty"` + // Stats pertaining to memory (RAM) resources. + Memory *MemoryStats `json:"memory,omitempty"` + // Stats pertaining to network resources. + Network *NetworkStats `json:"network,omitempty"` + // Stats pertaining to filesystem resources. Reported per-device. + Filesystem []FilesystemStats `json:"filesystem,omitempty" patchStrategy:"merge" patchMergeKey:"device"` +} + +// PodSample contains a sample point of pod-level resources. +type PodSample struct { + Sample `json:",inline"` + // Stats pertaining to network resources. + Network *NetworkStats `json:"network,omitempty"` +} + +// ContainerSample contains a sample point of container-level resources. +type ContainerSample struct { + Sample `json:",inline"` + // Stats pertaining to CPU resources. + CPU *CPUStats `json:"cpu,omitempty"` + // Stats pertaining to memory (RAM) resources. + Memory *MemoryStats `json:"memory,omitempty"` + // Stats pertaining to filesystem resources. Reported per-device. + Filesystem []FilesystemStats `json:"filesystem,omitempty" patchStrategy:"merge" patchMergeKey:"device"` +} + +// NetworkStats contains data about network resources. +type NetworkStats struct { + // Cumulative count of bytes received. + RxBytes *resource.Quantity `json:"rxBytes,omitempty"` + // Cumulative count of receive errors encountered. + RxErrors *int64 `json:"rxErrors,omitempty"` + // Cumulative count of bytes transmitted. + TxBytes *resource.Quantity `json:"txBytes,omitempty"` + // Cumulative count of transmit errors encountered. + TxErrors *int64 `json:"txErrors,omitempty"` +} + +// CPUStats contains data about CPU usage. +type CPUStats struct { + // Total CPU usage (sum of all cores) averaged over the sample window. + // The "core" unit can be interpreted as CPU core-seconds per second. + UsageCores *resource.Quantity `json:"totalCores,omitempty"` +} + +// MemoryStats contains data about memory usage. +type MemoryStats struct { + // Total memory in use. This includes all memory regardless of when it was accessed. + UsageBytes *resource.Quantity `json:"totalBytes,omitempty"` + // The amount of working set memory. This includes recently accessed memory, + // dirty memory, and kernel memory. UsageBytes is <= TotalBytes. + WorkingSetBytes *resource.Quantity `json:"usageBytes,omitempty"` + // Cumulative number of minor page faults. + PageFaults *int64 `json:"pageFaults,omitempty"` + // Cumulative number of major page faults. + MajorPageFaults *int64 `json:"majorPageFaults,omitempty"` +} + +// FilesystemStats contains data about filesystem usage. +type FilesystemStats struct { + // The block device name associated with the filesystem. + Device string `json:"device"` + // Number of bytes that is consumed by the container on this filesystem. + UsageBytes *resource.Quantity `json:"usageBytes,omitempty"` + // Number of bytes that can be consumed by the container on this filesystem. + LimitBytes *resource.Quantity `json:"limitBytes,omitempty"` +} + +// StatsOptions are the query options for raw stats endpoints. +type StatsOptions struct { + // Only include samples with sampleTime equal to or more recent than this time. + // This does not affect cumulative values, which are cumulative from object creation. + SinceTime *unversioned.Time `json:"sinceTime,omitempty"` + // Only include samples with sampleTime less recent than this time. + UntilTime *unversioned.Time `json:"untilTime,omitempty"` + // Specifies the maximum number of elements in any list of samples. + // When the total number of samples exceeds the maximum the most recent samples are returned. + MaxSamples int `json:"maxSamples,omitempty"` +} From e1fc2c11521de26d4a9cfbb3c80dfa80a3df3d6f Mon Sep 17 00:00:00 2001 From: "Tim St. Clair" Date: Wed, 16 Dec 2015 15:44:07 -0800 Subject: [PATCH 2/3] @vishh PR feedback --- pkg/kubelet/server/stats/types.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pkg/kubelet/server/stats/types.go b/pkg/kubelet/server/stats/types.go index af056d99c73..7a02957f156 100644 --- a/pkg/kubelet/server/stats/types.go +++ b/pkg/kubelet/server/stats/types.go @@ -35,12 +35,20 @@ type NodeStats struct { NodeName string `json:"nodeName"` // Overall node stats. Total []NodeSample `json:"total,omitempty" patchStrategy:"merge" patchMergeKey:"sampleTime"` - // Stats of system daemons tracked as raw containers, which may include: - // "/kubelet", "/docker-daemon", "kube-proxy" - Tracks respective component stats - // "/system" - Tracks stats of non-kubernetes and non-kernel processes (grouped together) + // Stats of system daemons tracked as raw containers. + // The system containers are named according to the SystemContainer* constants. SystemContainers []ContainerStats `json:"systemContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name"` } +const ( + // Container name for the system container tracking Kubelet usage. + SystemContainerKubelet = "/kubelet" + // Container name for the system container tracking the runtime (e.g. docker or rkt) usage. + SystemContainerRuntime = "/runtime" + // Container name for the system container tracking non-kubernetes processes. + SystemContainerMisc = "/misc" +) + // PodStats holds pod-level unprocessed sample stats. type PodStats struct { // Reference to the measured Pod. @@ -119,16 +127,18 @@ type NetworkStats struct { type CPUStats struct { // Total CPU usage (sum of all cores) averaged over the sample window. // The "core" unit can be interpreted as CPU core-seconds per second. - UsageCores *resource.Quantity `json:"totalCores,omitempty"` + UsageCores *resource.Quantity `json:"usageCores,omitempty"` + // Cumulative CPU usage (sum of all cores) since object creation. + UsageCoreSeconds *resource.Quantity `json:"usageCoreSeconds,omitempty"` } // MemoryStats contains data about memory usage. type MemoryStats struct { // Total memory in use. This includes all memory regardless of when it was accessed. - UsageBytes *resource.Quantity `json:"totalBytes,omitempty"` + UsageBytes *resource.Quantity `json:"usageBytes,omitempty"` // The amount of working set memory. This includes recently accessed memory, // dirty memory, and kernel memory. UsageBytes is <= TotalBytes. - WorkingSetBytes *resource.Quantity `json:"usageBytes,omitempty"` + WorkingSetBytes *resource.Quantity `json:"workingSetBytes,omitempty"` // Cumulative number of minor page faults. PageFaults *int64 `json:"pageFaults,omitempty"` // Cumulative number of major page faults. @@ -153,6 +163,7 @@ type StatsOptions struct { // Only include samples with sampleTime less recent than this time. UntilTime *unversioned.Time `json:"untilTime,omitempty"` // Specifies the maximum number of elements in any list of samples. - // When the total number of samples exceeds the maximum the most recent samples are returned. + // When the total number of samples exceeds the maximum the most recent MaxSamples samples are + // returned. MaxSamples int `json:"maxSamples,omitempty"` } From c357b91d2c3da7707e5854c693b109cf4ccecb9a Mon Sep 17 00:00:00 2001 From: "Tim St. Clair" Date: Thu, 17 Dec 2015 15:45:14 -0800 Subject: [PATCH 3/3] Update from offline discussions - Sample is now the toplevel struct, so all child structs have the same timestamp - Removed FilesystemStats. There are more discussions needed wrt. volumes and disk accounting, so this will be added in a follow up PR - Removed Options. The most recent sample will be returned. --- pkg/kubelet/server/stats/types.go | 88 +++++++------------------------ 1 file changed, 18 insertions(+), 70 deletions(-) diff --git a/pkg/kubelet/server/stats/types.go b/pkg/kubelet/server/stats/types.go index 7a02957f156..ebc36d15b91 100644 --- a/pkg/kubelet/server/stats/types.go +++ b/pkg/kubelet/server/stats/types.go @@ -23,6 +23,9 @@ import ( // Summary is a top-level container for holding NodeStats and PodStats. type Summary struct { + // The time the most recent data included in this summary was collect at, rounded to the nearest + // second. + Time unversioned.Time `json:"time"` // Overall node stats. Node NodeStats `json:"node"` // Per-pod stats. @@ -33,20 +36,24 @@ type Summary struct { type NodeStats struct { // Reference to the measured Node. NodeName string `json:"nodeName"` - // Overall node stats. - Total []NodeSample `json:"total,omitempty" patchStrategy:"merge" patchMergeKey:"sampleTime"` // Stats of system daemons tracked as raw containers. // The system containers are named according to the SystemContainer* constants. SystemContainers []ContainerStats `json:"systemContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name"` + // Stats pertaining to CPU resources. + CPU *CPUStats `json:"cpu,omitempty"` + // Stats pertaining to memory (RAM) resources. + Memory *MemoryStats `json:"memory,omitempty"` + // Stats pertaining to network resources. + Network *NetworkStats `json:"network,omitempty"` } const ( // Container name for the system container tracking Kubelet usage. - SystemContainerKubelet = "/kubelet" + SystemContainerKubelet = "kubelet" // Container name for the system container tracking the runtime (e.g. docker or rkt) usage. - SystemContainerRuntime = "/runtime" + SystemContainerRuntime = "runtime" // Container name for the system container tracking non-kubernetes processes. - SystemContainerMisc = "/misc" + SystemContainerMisc = "misc" ) // PodStats holds pod-level unprocessed sample stats. @@ -55,16 +62,18 @@ type PodStats struct { PodRef NonLocalObjectReference `json:"podRef"` // Stats of containers in the measured pod. Containers []ContainerStats `json:"containers" patchStrategy:"merge" patchMergeKey:"name"` - // Historical stat samples of pod-level resources. - Samples []PodSample `json:"samples" patchStrategy:"merge" patchMergeKey:"sampleTime"` + // Stats pertaining to network resources. + Network *NetworkStats `json:"network,omitempty"` } // ContainerStats holds container-level unprocessed sample stats. type ContainerStats struct { // Reference to the measured container. Name string `json:"name"` - // Historical stat samples gathered from the container. - Samples []ContainerSample `json:"samples" patchStrategy:"merge" patchMergeKey:"sampleTime"` + // Stats pertaining to CPU resources. + CPU *CPUStats `json:"cpu,omitempty"` + // Stats pertaining to memory (RAM) resources. + Memory *MemoryStats `json:"memory,omitempty"` } // NonLocalObjectReference contains enough information to locate the referenced object. @@ -73,44 +82,6 @@ type NonLocalObjectReference struct { Namespace string `json:"namespace"` } -// Sample defines metadata common to all sample types. -// Samples may not be nested within other samples. -type Sample struct { - // The time this data point was collected at. - SampleTime unversioned.Time `json:"sampleTime"` -} - -// NodeSample contains a sample point of data aggregated over a node. -type NodeSample struct { - Sample `json:",inline"` - // Stats pertaining to CPU resources. - CPU *CPUStats `json:"cpu,omitempty"` - // Stats pertaining to memory (RAM) resources. - Memory *MemoryStats `json:"memory,omitempty"` - // Stats pertaining to network resources. - Network *NetworkStats `json:"network,omitempty"` - // Stats pertaining to filesystem resources. Reported per-device. - Filesystem []FilesystemStats `json:"filesystem,omitempty" patchStrategy:"merge" patchMergeKey:"device"` -} - -// PodSample contains a sample point of pod-level resources. -type PodSample struct { - Sample `json:",inline"` - // Stats pertaining to network resources. - Network *NetworkStats `json:"network,omitempty"` -} - -// ContainerSample contains a sample point of container-level resources. -type ContainerSample struct { - Sample `json:",inline"` - // Stats pertaining to CPU resources. - CPU *CPUStats `json:"cpu,omitempty"` - // Stats pertaining to memory (RAM) resources. - Memory *MemoryStats `json:"memory,omitempty"` - // Stats pertaining to filesystem resources. Reported per-device. - Filesystem []FilesystemStats `json:"filesystem,omitempty" patchStrategy:"merge" patchMergeKey:"device"` -} - // NetworkStats contains data about network resources. type NetworkStats struct { // Cumulative count of bytes received. @@ -144,26 +115,3 @@ type MemoryStats struct { // Cumulative number of major page faults. MajorPageFaults *int64 `json:"majorPageFaults,omitempty"` } - -// FilesystemStats contains data about filesystem usage. -type FilesystemStats struct { - // The block device name associated with the filesystem. - Device string `json:"device"` - // Number of bytes that is consumed by the container on this filesystem. - UsageBytes *resource.Quantity `json:"usageBytes,omitempty"` - // Number of bytes that can be consumed by the container on this filesystem. - LimitBytes *resource.Quantity `json:"limitBytes,omitempty"` -} - -// StatsOptions are the query options for raw stats endpoints. -type StatsOptions struct { - // Only include samples with sampleTime equal to or more recent than this time. - // This does not affect cumulative values, which are cumulative from object creation. - SinceTime *unversioned.Time `json:"sinceTime,omitempty"` - // Only include samples with sampleTime less recent than this time. - UntilTime *unversioned.Time `json:"untilTime,omitempty"` - // Specifies the maximum number of elements in any list of samples. - // When the total number of samples exceeds the maximum the most recent MaxSamples samples are - // returned. - MaxSamples int `json:"maxSamples,omitempty"` -}