From 52d75d2188d0d66bfff3a67cc17ec6f929fb86f8 Mon Sep 17 00:00:00 2001
From: Yu-Ju Hong <yjhong@google.com>
Date: Wed, 2 Mar 2016 11:33:46 -0800
Subject: [PATCH] e2e: print runtime MemStats when memory usage exceeds
 expectation

---
 test/e2e/kubelet_perf.go  | 12 +++++++++---
 test/e2e/kubelet_stats.go | 16 ++++++++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/test/e2e/kubelet_perf.go b/test/e2e/kubelet_perf.go
index c3dc73e4bb9..1d5b080b9aa 100644
--- a/test/e2e/kubelet_perf.go
+++ b/test/e2e/kubelet_perf.go
@@ -99,7 +99,7 @@ func runResourceTrackingTest(framework *Framework, podsPerNode int, nodeNames se
 	usageSummary, err := rm.GetLatest()
 	Expect(err).NotTo(HaveOccurred())
 	Logf("%s", rm.FormatResourceUsage(usageSummary))
-	verifyMemoryLimits(expectedMemory, usageSummary)
+	verifyMemoryLimits(framework.Client, expectedMemory, usageSummary)
 
 	cpuSummary := rm.GetCPUSummary()
 	Logf("%s", rm.FormatCPUSummary(cpuSummary))
@@ -109,7 +109,7 @@ func runResourceTrackingTest(framework *Framework, podsPerNode int, nodeNames se
 	DeleteRC(framework.Client, framework.Namespace.Name, rcName)
 }
 
-func verifyMemoryLimits(expected resourceUsagePerContainer, actual resourceUsagePerNode) {
+func verifyMemoryLimits(c *client.Client, expected resourceUsagePerContainer, actual resourceUsagePerNode) {
 	if expected == nil {
 		return
 	}
@@ -132,10 +132,16 @@ func verifyMemoryLimits(expected resourceUsagePerContainer, actual resourceUsage
 		}
 		if len(nodeErrs) > 0 {
 			errList = append(errList, fmt.Sprintf("node %v:\n %s", nodeName, strings.Join(nodeErrs, ", ")))
+			heapStats, err := getKubeletHeapStats(c, nodeName)
+			if err != nil {
+				Logf("Unable to get heap stats from %q", nodeName)
+			} else {
+				Logf("Heap stats on %q\n:%v", nodeName, heapStats)
+			}
 		}
 	}
 	if len(errList) > 0 {
-		Failf("CPU usage exceeding limits:\n %s", strings.Join(errList, "\n"))
+		Failf("Memory usage exceeding limits:\n %s", strings.Join(errList, "\n"))
 	}
 }
 
diff --git a/test/e2e/kubelet_stats.go b/test/e2e/kubelet_stats.go
index ee00560bb84..a0d1b44edc3 100644
--- a/test/e2e/kubelet_stats.go
+++ b/test/e2e/kubelet_stats.go
@@ -391,6 +391,22 @@ func getKubeletMetricsThroughNode(nodeName string) (string, error) {
 	return string(body), nil
 }
 
+func getKubeletHeapStats(c *client.Client, nodeName string) (string, error) {
+	client, err := nodeProxyRequest(c, nodeName, "debug/pprof/heap")
+	if err != nil {
+		return "", err
+	}
+	raw, errRaw := client.Raw()
+	if errRaw != nil {
+		return "", err
+	}
+	stats := string(raw)
+	// Only dumping the runtime.MemStats numbers to avoid polluting the log.
+	numLines := 23
+	lines := strings.Split(stats, "\n")
+	return strings.Join(lines[len(lines)-numLines:], "\n"), nil
+}
+
 // GetKubeletPods retrieves the list of running pods on the kubelet. The pods
 // includes necessary information (e.g., UID, name, namespace for
 // pods/containers), but do not contain the full spec.