From 7a54f94cf4a9ba7f15326afe474711ad8ba8fa44 Mon Sep 17 00:00:00 2001 From: "Tim St. Clair" Date: Wed, 10 Feb 2016 15:41:57 -0800 Subject: [PATCH] Make summary timestamps more granular --- pkg/kubelet/server/stats/summary.go | 14 ++++----- pkg/kubelet/server/stats/summary_test.go | 36 ++++++++++++++++++++---- pkg/kubelet/server/stats/types.go | 11 ++++++-- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/pkg/kubelet/server/stats/summary.go b/pkg/kubelet/server/stats/summary.go index 4f5fba9a17b..ea5356bd5fa 100644 --- a/pkg/kubelet/server/stats/summary.go +++ b/pkg/kubelet/server/stats/summary.go @@ -100,10 +100,6 @@ func (sb *summaryBuilder) build() (*Summary, error) { if !found { return nil, fmt.Errorf("Missing stats for root container") } - cstat, found := sb.latestContainerStats(&rootInfo) - if !found { - return nil, fmt.Errorf("Missing stats for root container") - } rootStats := sb.containerInfoV2ToStats("", &rootInfo) nodeStats := NodeStats{ @@ -130,7 +126,6 @@ func (sb *summaryBuilder) build() (*Summary, error) { } summary := Summary{ - Time: unversioned.NewTime(cstat.Timestamp), Node: nodeStats, Pods: sb.buildSummaryPods(), } @@ -250,15 +245,17 @@ func (sb *summaryBuilder) containerInfoV2ToStats( name string, info *cadvisorapiv2.ContainerInfo) ContainerStats { stats := ContainerStats{ - Name: name, StartTime: unversioned.NewTime(info.Spec.CreationTime), + Name: name, } cstat, found := sb.latestContainerStats(info) if !found { return stats } if info.Spec.HasCpu { - cpuStats := CPUStats{} + cpuStats := CPUStats{ + Time: unversioned.NewTime(cstat.Timestamp), + } if cstat.CpuInst != nil { cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total } @@ -271,6 +268,7 @@ func (sb *summaryBuilder) containerInfoV2ToStats( pageFaults := cstat.Memory.ContainerData.Pgfault majorPageFaults := cstat.Memory.ContainerData.Pgmajfault stats.Memory = &MemoryStats{ + Time: unversioned.NewTime(cstat.Timestamp), UsageBytes: &cstat.Memory.Usage, WorkingSetBytes: &cstat.Memory.WorkingSet, PageFaults: &pageFaults, @@ -304,6 +302,7 @@ func (sb *summaryBuilder) containerInfoV2ToNetworkStats(info *cadvisorapiv2.Cont txErrors += inter.TxErrors } return &NetworkStats{ + Time: unversioned.NewTime(cstat.Timestamp), RxBytes: &rxBytes, RxErrors: &rxErrors, TxBytes: &txBytes, @@ -353,6 +352,7 @@ func (sb *summaryBuilder) containerInfoV2ToUserDefinedMetrics(info *cadvisorapiv for _, specVal := range udmMap { udm = append(udm, UserDefinedMetric{ UserDefinedMetricDescriptor: specVal.ref, + Time: unversioned.NewTime(specVal.time), Value: specVal.value, }) } diff --git a/pkg/kubelet/server/stats/summary_test.go b/pkg/kubelet/server/stats/summary_test.go index 1a8c4955bda..6811428c8ab 100644 --- a/pkg/kubelet/server/stats/summary_test.go +++ b/pkg/kubelet/server/stats/summary_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/leaky" ) @@ -44,6 +45,11 @@ const ( offsetNetTxErrors ) +var ( + timestamp = time.Now() + creationTime = timestamp.Add(-5 * time.Minute) +) + func TestBuildSummary(t *testing.T) { node := api.Node{} node.Name = "FooNode" @@ -111,6 +117,7 @@ func TestBuildSummary(t *testing.T) { assert.NoError(t, err) nodeStats := summary.Node assert.Equal(t, "FooNode", nodeStats.NodeName) + assert.EqualValues(t, testTime(creationTime, seedRoot).Unix(), nodeStats.StartTime.Time.Unix()) checkCPUStats(t, "Node", seedRoot, nodeStats.CPU) checkMemoryStats(t, "Node", seedRoot, nodeStats.Memory) checkNetworkStats(t, "Node", seedRoot, nodeStats.Network) @@ -126,6 +133,7 @@ func TestBuildSummary(t *testing.T) { if !found { t.Errorf("Unknown SystemContainer: %q", name) } + assert.EqualValues(t, testTime(creationTime, seed).Unix(), sys.StartTime.Time.Unix(), name+".StartTime") checkCPUStats(t, name, seed, sys.CPU) checkMemoryStats(t, name, seed, sys.Memory) } @@ -145,13 +153,16 @@ func TestBuildSummary(t *testing.T) { indexCon[con.Name] = con } con := indexCon[cName00] + assert.EqualValues(t, testTime(creationTime, seedPod0Container0).Unix(), con.StartTime.Time.Unix()) checkCPUStats(t, "container", seedPod0Container0, con.CPU) checkMemoryStats(t, "container", seedPod0Container0, con.Memory) con = indexCon[cName01] + assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix()) checkCPUStats(t, "container", seedPod0Container1, con.CPU) checkMemoryStats(t, "container", seedPod0Container1, con.Memory) + assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix()) checkNetworkStats(t, "Pod", seedPod0Infra, ps.Network) // Validate Pod1 Results @@ -231,6 +242,7 @@ func summaryTestContainerInfo(seed int, podName string, podNamespace string, con } } spec := v2.ContainerSpec{ + CreationTime: testTime(creationTime, seed), HasCpu: true, HasMemory: true, HasNetwork: true, @@ -239,8 +251,9 @@ func summaryTestContainerInfo(seed int, podName string, podNamespace string, con } stats := v2.ContainerStats{ - Cpu: &v1.CpuStats{}, - CpuInst: &v2.CpuInstStats{}, + Timestamp: testTime(timestamp, seed), + Cpu: &v1.CpuStats{}, + CpuInst: &v2.CpuInstStats{}, Memory: &v1.MemoryStats{ Usage: uint64(seed + offsetMemUsageBytes), WorkingSet: uint64(seed + offsetMemWorkingSetBytes), @@ -267,7 +280,12 @@ func summaryTestContainerInfo(seed int, podName string, podNamespace string, con } } +func testTime(base time.Time, seed int) time.Time { + return base.Add(time.Duration(seed) * time.Second) +} + func checkNetworkStats(t *testing.T, label string, seed int, stats *NetworkStats) { + assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Net.Time") assert.EqualValues(t, seed+offsetNetRxBytes, *stats.RxBytes, label+".Net.RxBytes") assert.EqualValues(t, seed+offsetNetRxErrors, *stats.RxErrors, label+".Net.RxErrors") assert.EqualValues(t, seed+offsetNetTxBytes, *stats.TxBytes, label+".Net.TxBytes") @@ -275,11 +293,13 @@ func checkNetworkStats(t *testing.T, label string, seed int, stats *NetworkStats } func checkCPUStats(t *testing.T, label string, seed int, stats *CPUStats) { + assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".CPU.Time") assert.EqualValues(t, seed+offsetCPUUsageCores, *stats.UsageNanoCores, label+".CPU.UsageCores") assert.EqualValues(t, seed+offsetCPUUsageCoreSeconds, *stats.UsageCoreNanoSeconds, label+".CPU.UsageCoreSeconds") } func checkMemoryStats(t *testing.T, label string, seed int, stats *MemoryStats) { + assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Mem.Time") assert.EqualValues(t, seed+offsetMemUsageBytes, *stats.UsageBytes, label+".Mem.UsageBytes") assert.EqualValues(t, seed+offsetMemWorkingSetBytes, *stats.WorkingSetBytes, label+".Mem.WorkingSetBytes") assert.EqualValues(t, seed+offsetMemPageFaults, *stats.PageFaults, label+".Mem.PageFaults") @@ -301,24 +321,26 @@ func TestCustomMetrics(t *testing.T) { Units: "count", }, } + timestamp1 := time.Now() + timestamp2 := time.Now().Add(time.Minute) metrics := map[string][]v1.MetricVal{ "qos": { { - Timestamp: time.Now(), + Timestamp: timestamp1, IntValue: 10, }, { - Timestamp: time.Now().Add(time.Minute), + Timestamp: timestamp2, IntValue: 100, }, }, "cpuLoad": { { - Timestamp: time.Now(), + Timestamp: timestamp1, FloatValue: 1.2, }, { - Timestamp: time.Now().Add(time.Minute), + Timestamp: timestamp2, FloatValue: 2.1, }, }, @@ -341,6 +363,7 @@ func TestCustomMetrics(t *testing.T) { Type: MetricGauge, Units: "per second", }, + Time: unversioned.NewTime(timestamp2), Value: 100, }, UserDefinedMetric{ @@ -349,6 +372,7 @@ func TestCustomMetrics(t *testing.T) { Type: MetricCumulative, Units: "count", }, + Time: unversioned.NewTime(timestamp2), Value: 2.1, }) } diff --git a/pkg/kubelet/server/stats/types.go b/pkg/kubelet/server/stats/types.go index 017cc2936cb..91229286070 100644 --- a/pkg/kubelet/server/stats/types.go +++ b/pkg/kubelet/server/stats/types.go @@ -22,9 +22,6 @@ import ( // Summary is a top-level container for holding NodeStats and PodStats. type Summary struct { - // The time the most recent data included in this summary was collect at, rounded to the nearest - // second. - Time unversioned.Time `json:"time"` // Overall node stats. Node NodeStats `json:"node"` // Per-pod stats. @@ -104,6 +101,8 @@ type PodReference struct { // NetworkStats contains data about network resources. type NetworkStats struct { + // The time at which these stats were updated. + Time unversioned.Time `json:"time"` // Cumulative count of bytes received. RxBytes *uint64 `json:"rxBytes,omitempty"` // Cumulative count of receive errors encountered. @@ -116,6 +115,8 @@ type NetworkStats struct { // CPUStats contains data about CPU usage. type CPUStats struct { + // The time at which these stats were updated. + Time unversioned.Time `json:"time"` // Total CPU usage (sum of all cores) averaged over the sample window. // The "core" unit can be interpreted as CPU core-nanoseconds per second. UsageNanoCores *uint64 `json:"usageNanoCores,omitempty"` @@ -125,6 +126,8 @@ type CPUStats struct { // MemoryStats contains data about memory usage. type MemoryStats struct { + // The time at which these stats were updated. + Time unversioned.Time `json:"time"` // Total memory in use. This includes all memory regardless of when it was accessed. UsageBytes *uint64 `json:"usageBytes,omitempty"` // The amount of working set memory. This includes recently accessed memory, @@ -188,6 +191,8 @@ type UserDefinedMetricDescriptor struct { // UserDefinedMetric represents a metric defined and generate by users. type UserDefinedMetric struct { UserDefinedMetricDescriptor `json:",inline"` + // The time at which these stats were updated. + Time unversioned.Time `json:"time"` // Value of the metric. Float64s have 53 bit precision. // We do not forsee any metrics exceeding that value. Value float64 `json:"value"`