diff --git a/test/e2e/kubelet_perf.go b/test/e2e/kubelet_perf.go index 5fca65b7068..15dcd7f57a7 100644 --- a/test/e2e/kubelet_perf.go +++ b/test/e2e/kubelet_perf.go @@ -99,7 +99,7 @@ func runResourceTrackingTest(framework *Framework, podsPerNode int, nodeNames se usageSummary, err := rm.GetLatest() Expect(err).NotTo(HaveOccurred()) Logf("%s", rm.FormatResourceUsage(usageSummary)) - verifyMemoryLimits(expectedMemory, usageSummary) + verifyMemoryLimits(framework.Client, expectedMemory, usageSummary) cpuSummary := rm.GetCPUSummary() Logf("%s", rm.FormatCPUSummary(cpuSummary)) @@ -109,7 +109,7 @@ func runResourceTrackingTest(framework *Framework, podsPerNode int, nodeNames se DeleteRC(framework.Client, framework.Namespace.Name, rcName) } -func verifyMemoryLimits(expected resourceUsagePerContainer, actual resourceUsagePerNode) { +func verifyMemoryLimits(c *client.Client, expected resourceUsagePerContainer, actual resourceUsagePerNode) { if expected == nil { return } @@ -132,6 +132,12 @@ func verifyMemoryLimits(expected resourceUsagePerContainer, actual resourceUsage } if len(nodeErrs) > 0 { errList = append(errList, fmt.Sprintf("node %v:\n %s", nodeName, strings.Join(nodeErrs, ", "))) + heapStats, err := getKubeletHeapStats(c, nodeName) + if err != nil { + Logf("Unable to get heap stats from %q", nodeName) + } else { + Logf("Heap stats on %q\n:%v", nodeName, heapStats) + } } } if len(errList) > 0 { diff --git a/test/e2e/kubelet_stats.go b/test/e2e/kubelet_stats.go index dd68c1721f1..f34415e9545 100644 --- a/test/e2e/kubelet_stats.go +++ b/test/e2e/kubelet_stats.go @@ -407,6 +407,22 @@ func getKubeletMetricsThroughNode(nodeName string) (string, error) { return string(body), nil } +func getKubeletHeapStats(c *client.Client, nodeName string) (string, error) { + client, err := nodeProxyRequest(c, nodeName, "debug/pprof/heap") + if err != nil { + return "", err + } + raw, errRaw := client.Raw() + if errRaw != nil { + return "", err + } + stats := string(raw) + // Only dumping the runtime.MemStats numbers to avoid polluting the log. + numLines := 23 + lines := strings.Split(stats, "\n") + return strings.Join(lines[len(lines)-numLines:], "\n"), nil +} + // GetKubeletPods retrieves the list of running pods on the kubelet. The pods // includes necessary information (e.g., UID, name, namespace for // pods/containers), but do not contain the full spec.