Use gcloud to list nodes so we know their ips

test/e2e/density.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"math"
 	"os"
+	"os/exec"
 	"sort"
 	"strconv"
 	"sync"
@@ -66,6 +67,20 @@ func printLatencies(latencies []podLatencyData, header string) {
 	Logf("perc50: %v, perc90: %v, perc99: %v", perc50, perc90, perc99)
 }
 
+// List nodes via gcloud. We don't rely on the apiserver because we really want the node ips
+// and sometimes the node controller is slow to populate them.
+func gcloudListNodes() {
+	Logf("Listing nodes via gcloud:")
+	output, err := exec.Command("gcloud", "compute", "instances", "list",
+		"--project="+testContext.CloudConfig.ProjectID, "--zone="+testContext.CloudConfig.Zone).CombinedOutput()
+	if err != nil {
+		Logf("Failed to list nodes: %v, %v", err)
+		return
+	}
+	Logf(string(output))
+	return
+}
+
 // This test suite can take a long time to run, so by default it is added to
 // the ginkgo.skip list (see driver.go).
 // To run this suite you must explicitly ask for it by setting the
@@ -101,6 +116,7 @@ var _ = Describe("Density", func() {
 		expectNoError(resetMetrics(c))
 		expectNoError(os.Mkdir(fmt.Sprintf(testContext.OutputDir+"/%s", uuid), 0777))
 		expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "before"))
+		gcloudListNodes()
 	})
 
 	AfterEach(func() {

test/e2e/kubelet_stats.go

@@ -183,11 +183,7 @@ func nodeProxyRequest(c *client.Client, node, endpoint string) client.Result {
 }
 
 // Retrieve metrics from the kubelet server of the given node.
-<<<<<<< HEAD
 func getKubeletMetricsThroughProxy(c *client.Client, node string) (string, error) {
-=======
-func getKubeletMetrics(c *client.Client, node string) (string, error) {
->>>>>>> Density logging
 	metric, err := nodeProxyRequest(c, node, "metrics").Raw()
 	if err != nil {
 		return "", err

test/e2e/util.go

@@ -1133,7 +1133,6 @@ func dumpPodDebugInfo(c *client.Client, pods []*api.Pod) {
 }
 
 func dumpNodeDebugInfo(c *client.Client, nodeNames []string) {
-	// TODO: Actually log running pods instead of the pods in the pod manager.
 	for _, n := range nodeNames {
 		Logf("\nLogging pods the kubelet thinks is on node %v", n)
 		podList, err := GetKubeletPods(c, n)
@@ -1142,9 +1141,7 @@ func dumpNodeDebugInfo(c *client.Client, nodeNames []string) {
 			continue
 		}
 		for _, p := range podList.Items {
-			// If the pod is in pending it's probably not going to have a starttime or container statuses, since
-			// we're only checking the pod manager. This should change.
-			Logf("%v started at %v (%d containers)", p.Name, p.Status.StartTime, len(p.Status.ContainerStatuses))
+			Logf("%v started at %v (%d container statuses recorded)", p.Name, p.Status.StartTime, len(p.Status.ContainerStatuses))
 			for _, c := range p.Status.ContainerStatuses {
 				Logf("\tContainer %v ready: %v, restart count %v",
 					c.Name, c.Ready, c.RestartCount)
@@ -1153,7 +1150,6 @@ func dumpNodeDebugInfo(c *client.Client, nodeNames []string) {
 		HighLatencyKubeletOperations(c, 10*time.Second, n)
 		// TODO: Log node resource info
 	}
-
 }
 
 func ScaleRC(c *client.Client, ns, name string, size uint) error {