Merge pull request #30801 from soundcloud/grobie/update-cadvisor

Automatic merge from submit-queue

Update cAdvisor to 2ed7198

**What this PR does / why we need it**:

Update cAdvisor to 2ed7198 so that we get Prometheus metrics on CPU throttling when pod resource limits are configured. We're flying blind right now.

**Changes**:

* Add container_cpu_cfs_* metrics (CPU throttling due to limits)
* Add container_memory_swap metric
* Ensure minimum kernel version for thin_ls

Diff: c6c06d4...2ed7198
This commit is contained in:
Kubernetes Submit Queue 2016-08-20 07:27:15 -07:00 committed by GitHub
commit ecad028aaa
6 changed files with 225 additions and 99 deletions

164
Godeps/Godeps.json generated
View File

@ -1098,208 +1098,208 @@
},
{
"ImportPath": "github.com/google/cadvisor/api",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/cache/memory",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/client/v2",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/collector",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/container",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/container/common",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/container/docker",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/container/libcontainer",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/container/raw",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/container/rkt",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/container/systemd",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/devicemapper",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/events",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/fs",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/healthz",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/http",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/http/mux",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/info/v1",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/info/v1/test",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/info/v2",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/machine",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/manager",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/manager/watcher",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/manager/watcher/raw",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/manager/watcher/rkt",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/metrics",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/pages",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/pages/static",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/storage",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/summary",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/utils",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/utils/cloudinfo",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/utils/cpuload",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/utils/cpuload/netlink",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/utils/docker",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/utils/oomparser",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/utils/sysfs",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/utils/sysinfo",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/utils/tail",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/validate",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/cadvisor/version",
"Comment": "v0.23.2-79-gc6c06d4",
"Rev": "c6c06d440ab2fcaae9211dda6dcbbaa1e98a054b"
"Comment": "v0.23.2-89-g2ed7198",
"Rev": "2ed7198f77395ee9a172878a0a7ab92ab59a2cfd"
},
{
"ImportPath": "github.com/google/certificate-transparency/go",

View File

@ -110,6 +110,7 @@ func GetSpec(cgroupPaths map[string]string, machineInfoFactory info.MachineInfoF
spec.HasMemory = true
spec.Memory.Limit = readUInt64(memoryRoot, "memory.limit_in_bytes")
spec.Memory.SwapLimit = readUInt64(memoryRoot, "memory.memsw.limit_in_bytes")
spec.Memory.Reservation = readUInt64(memoryRoot, "memory.soft_limit_in_bytes")
}
}

View File

@ -19,15 +19,18 @@ import (
"fmt"
"path"
"regexp"
"strconv"
"strings"
"sync"
"github.com/blang/semver"
dockertypes "github.com/docker/engine-api/types"
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/devicemapper"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
"github.com/google/cadvisor/machine"
"github.com/google/cadvisor/manager/watcher"
dockerutil "github.com/google/cadvisor/utils/docker"
@ -178,6 +181,10 @@ func startThinPoolWatcher(dockerInfo *dockertypes.Info) (*devicemapper.ThinPoolW
return nil, err
}
if err := ensureThinLsKernelVersion(machine.KernelVersion()); err != nil {
return nil, err
}
dockerThinPoolName, err := dockerutil.DockerThinPoolName(*dockerInfo)
if err != nil {
return nil, err
@ -197,6 +204,66 @@ func startThinPoolWatcher(dockerInfo *dockertypes.Info) (*devicemapper.ThinPoolW
return thinPoolWatcher, nil
}
func ensureThinLsKernelVersion(kernelVersion string) error {
// kernel 4.4.0 has the proper bug fixes to allow thin_ls to work without corrupting the thin pool
minKernelVersion := semver.MustParse("4.4.0")
// RHEL 7 kernel 3.10.0 release >= 366 has the proper bug fixes backported from 4.4.0 to allow
// thin_ls to work without corrupting the thin pool
minRhel7KernelVersion := semver.MustParse("3.10.0")
matches := version_re.FindStringSubmatch(kernelVersion)
if len(matches) < 4 {
return fmt.Errorf("error parsing kernel version: %q is not a semver", kernelVersion)
}
sem, err := semver.Make(matches[0])
if err != nil {
return err
}
if sem.GTE(minKernelVersion) {
// kernel 4.4+ - good
return nil
}
// Certain RHEL/Centos 7.x kernels have a backport to fix the corruption bug
if !strings.Contains(kernelVersion, ".el7") {
// not a RHEL 7.x kernel - won't work
return fmt.Errorf("kernel version 4.4.0 or later is required to use thin_ls - you have %q", kernelVersion)
}
// RHEL/Centos 7.x from here on
if sem.Major != 3 {
// only 3.x kernels *may* work correctly
return fmt.Errorf("RHEL/Centos 7.x kernel version 3.10.0-366 or later is required to use thin_ls - you have %q", kernelVersion)
}
if sem.GT(minRhel7KernelVersion) {
// 3.10.1+ - good
return nil
}
if sem.EQ(minRhel7KernelVersion) {
// need to check release
releaseRE := regexp.MustCompile(`^[^-]+-([0-9]+)\.`)
releaseMatches := releaseRE.FindStringSubmatch(kernelVersion)
if len(releaseMatches) != 2 {
return fmt.Errorf("unable to determine RHEL/Centos 7.x kernel release from %q", kernelVersion)
}
release, err := strconv.Atoi(releaseMatches[1])
if err != nil {
return fmt.Errorf("error parsing release %q: %v", releaseMatches[1], err)
}
if release >= 366 {
return nil
}
}
return fmt.Errorf("RHEL/Centos 7.x kernel version 3.10.0-366 or later is required to use thin_ls - you have %q", kernelVersion)
}
// Register root container before running this function!
func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, ignoreMetrics container.MetricSet) error {
client, err := Client()

View File

@ -89,7 +89,7 @@ func GetStats(cgroupManager cgroups.Manager, rootFs string, pid int, ignoreMetri
libcontainerStats := &libcontainer.Stats{
CgroupStats: cgroupStats,
}
stats := toContainerStats(libcontainerStats)
stats := newContainerStats(libcontainerStats)
// If we know the pid then get network stats from /proc/<pid>/net/dev
if pid == 0 {
@ -350,7 +350,7 @@ func DiskStatsCopy(blkio_stats []cgroups.BlkioStatEntry) (stat []info.PerDiskSta
}
// Convert libcontainer stats to info.ContainerStats.
func toContainerStats0(s *cgroups.Stats, ret *info.ContainerStats) {
func setCpuStats(s *cgroups.Stats, ret *info.ContainerStats) {
ret.Cpu.Usage.User = s.CpuStats.CpuUsage.UsageInUsermode
ret.Cpu.Usage.System = s.CpuStats.CpuUsage.UsageInKernelmode
n := len(s.CpuStats.CpuUsage.PercpuUsage)
@ -361,9 +361,13 @@ func toContainerStats0(s *cgroups.Stats, ret *info.ContainerStats) {
ret.Cpu.Usage.PerCpu[i] = s.CpuStats.CpuUsage.PercpuUsage[i]
ret.Cpu.Usage.Total += s.CpuStats.CpuUsage.PercpuUsage[i]
}
ret.Cpu.CFS.Periods = s.CpuStats.ThrottlingData.Periods
ret.Cpu.CFS.ThrottledPeriods = s.CpuStats.ThrottlingData.ThrottledPeriods
ret.Cpu.CFS.ThrottledTime = s.CpuStats.ThrottlingData.ThrottledTime
}
func toContainerStats1(s *cgroups.Stats, ret *info.ContainerStats) {
func setDiskIoStats(s *cgroups.Stats, ret *info.ContainerStats) {
ret.DiskIo.IoServiceBytes = DiskStatsCopy(s.BlkioStats.IoServiceBytesRecursive)
ret.DiskIo.IoServiced = DiskStatsCopy(s.BlkioStats.IoServicedRecursive)
ret.DiskIo.IoQueued = DiskStatsCopy(s.BlkioStats.IoQueuedRecursive)
@ -374,11 +378,12 @@ func toContainerStats1(s *cgroups.Stats, ret *info.ContainerStats) {
ret.DiskIo.IoTime = DiskStatsCopy(s.BlkioStats.IoTimeRecursive)
}
func toContainerStats2(s *cgroups.Stats, ret *info.ContainerStats) {
func setMemoryStats(s *cgroups.Stats, ret *info.ContainerStats) {
ret.Memory.Usage = s.MemoryStats.Usage.Usage
ret.Memory.Failcnt = s.MemoryStats.Usage.Failcnt
ret.Memory.Cache = s.MemoryStats.Stats["cache"]
ret.Memory.RSS = s.MemoryStats.Stats["rss"]
ret.Memory.Swap = s.MemoryStats.Stats["swap"]
if v, ok := s.MemoryStats.Stats["pgfault"]; ok {
ret.Memory.ContainerData.Pgfault = v
ret.Memory.HierarchicalData.Pgfault = v
@ -399,7 +404,7 @@ func toContainerStats2(s *cgroups.Stats, ret *info.ContainerStats) {
ret.Memory.WorkingSet = workingSet
}
func toContainerStats3(libcontainerStats *libcontainer.Stats, ret *info.ContainerStats) {
func setNetworkStats(libcontainerStats *libcontainer.Stats, ret *info.ContainerStats) {
ret.Network.Interfaces = make([]info.InterfaceStats, len(libcontainerStats.Interfaces))
for i := range libcontainerStats.Interfaces {
ret.Network.Interfaces[i] = info.InterfaceStats{
@ -421,18 +426,18 @@ func toContainerStats3(libcontainerStats *libcontainer.Stats, ret *info.Containe
}
}
func toContainerStats(libcontainerStats *libcontainer.Stats) *info.ContainerStats {
s := libcontainerStats.CgroupStats
ret := new(info.ContainerStats)
ret.Timestamp = time.Now()
func newContainerStats(libcontainerStats *libcontainer.Stats) *info.ContainerStats {
ret := &info.ContainerStats{
Timestamp: time.Now(),
}
if s != nil {
toContainerStats0(s, ret)
toContainerStats1(s, ret)
toContainerStats2(s, ret)
if s := libcontainerStats.CgroupStats; s != nil {
setCpuStats(s, ret)
setDiskIoStats(s, ret)
setMemoryStats(s, ret)
}
if len(libcontainerStats.Interfaces) > 0 {
toContainerStats3(libcontainerStats, ret)
setNetworkStats(libcontainerStats, ret)
}
return ret
}

View File

@ -266,7 +266,7 @@ type LoadStats struct {
// CPU usage time statistics.
type CpuUsage struct {
// Total CPU usage.
// Units: nanoseconds
// Unit: nanoseconds.
Total uint64 `json:"total"`
// Per CPU/core usage of the container.
@ -274,17 +274,31 @@ type CpuUsage struct {
PerCpu []uint64 `json:"per_cpu_usage,omitempty"`
// Time spent in user space.
// Unit: nanoseconds
// Unit: nanoseconds.
User uint64 `json:"user"`
// Time spent in kernel space.
// Unit: nanoseconds
// Unit: nanoseconds.
System uint64 `json:"system"`
}
// Cpu Completely Fair Scheduler statistics.
type CpuCFS struct {
// Total number of elapsed enforcement intervals.
Periods uint64 `json:"periods"`
// Total number of times tasks in the cgroup have been throttled.
ThrottledPeriods uint64 `json:"throttled_periods"`
// Total time duration for which tasks in the cgroup have been throttled.
// Unit: nanoseconds.
ThrottledTime uint64 `json:"throttled_time"`
}
// All CPU usage metrics are cumulative from the creation of the container
type CpuStats struct {
Usage CpuUsage `json:"usage"`
CFS CpuCFS `json:"cfs"`
// Smoothed average of number of runnable threads x 1000.
// We multiply by thousand to avoid using floats, but preserving precision.
// Load is smoothed over the last 10 seconds. Instantaneous value can be read
@ -324,6 +338,10 @@ type MemoryStats struct {
// Units: Bytes.
RSS uint64 `json:"rss"`
// The amount of swap currently used by the processes in this cgroup
// Units: Bytes.
Swap uint64 `json:"swap"`
// The amount of working set memory, this includes recently accessed memory,
// dirty memory, and kernel memory. Working set is <= "usage".
// Units: Bytes.

View File

@ -63,6 +63,7 @@ type containerMetric struct {
help string
valueType prometheus.ValueType
extraLabels []string
condition func(s info.ContainerSpec) bool
getValues func(s *info.ContainerStats) metricValues
}
@ -127,6 +128,30 @@ func NewPrometheusCollector(infoProvider infoProvider, f ContainerNameToLabelsFu
}
return values
},
}, {
name: "container_cpu_cfs_periods_total",
help: "Number of elapsed enforcement period intervals.",
valueType: prometheus.CounterValue,
condition: func(s info.ContainerSpec) bool { return s.Cpu.Quota != 0 },
getValues: func(s *info.ContainerStats) metricValues {
return metricValues{{value: float64(s.Cpu.CFS.Periods)}}
},
}, {
name: "container_cpu_cfs_throttled_periods_total",
help: "Number of throttled period intervals.",
valueType: prometheus.CounterValue,
condition: func(s info.ContainerSpec) bool { return s.Cpu.Quota != 0 },
getValues: func(s *info.ContainerStats) metricValues {
return metricValues{{value: float64(s.Cpu.CFS.ThrottledPeriods)}}
},
}, {
name: "container_cpu_cfs_throttled_seconds_total",
help: "Total time duration the container has been throttled.",
valueType: prometheus.CounterValue,
condition: func(s info.ContainerSpec) bool { return s.Cpu.Quota != 0 },
getValues: func(s *info.ContainerStats) metricValues {
return metricValues{{value: float64(s.Cpu.CFS.ThrottledTime) / float64(time.Second)}}
},
}, {
name: "container_memory_cache",
help: "Number of bytes of page cache memory.",
@ -141,6 +166,13 @@ func NewPrometheusCollector(infoProvider infoProvider, f ContainerNameToLabelsFu
getValues: func(s *info.ContainerStats) metricValues {
return metricValues{{value: float64(s.Memory.RSS)}}
},
}, {
name: "container_memory_swap",
help: "Container swap usage in bytes.",
valueType: prometheus.GaugeValue,
getValues: func(s *info.ContainerStats) metricValues {
return metricValues{{value: float64(s.Memory.Swap)}}
},
}, {
name: "container_memory_failcnt",
help: "Number of memory usage hits limits",
@ -568,6 +600,9 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric)
// Now for the actual metrics
stats := container.Stats[0]
for _, cm := range c.containerMetrics {
if cm.condition != nil && !cm.condition(container.Spec) {
continue
}
desc := cm.desc(baseLabels)
for _, metricValue := range cm.getValues(stats) {
ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(baseLabelValues, metricValue.labels...)...)