diff --git a/test/e2e/kubelet_stats.go b/test/e2e/kubelet_stats.go index a973540f475..d254e8c2a6d 100644 --- a/test/e2e/kubelet_stats.go +++ b/test/e2e/kubelet_stats.go @@ -183,7 +183,11 @@ func nodeProxyRequest(c *client.Client, node, endpoint string) client.Result { } // Retrieve metrics from the kubelet server of the given node. +<<<<<<< HEAD func getKubeletMetricsThroughProxy(c *client.Client, node string) (string, error) { +======= +func getKubeletMetrics(c *client.Client, node string) (string, error) { +>>>>>>> Density logging metric, err := nodeProxyRequest(c, node, "metrics").Raw() if err != nil { return "", err diff --git a/test/e2e/util.go b/test/e2e/util.go index d4ac486dc96..bb5caf41e52 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1046,7 +1046,7 @@ func RunRC(config RCConfig) error { oldPods := make([]*api.Pod, 0) oldRunning := 0 lastChange := time.Now() - for oldRunning != config.Replicas && time.Since(lastChange) < timeout { + for oldRunning != config.Replicas { time.Sleep(interval) running := 0 @@ -1104,6 +1104,11 @@ func RunRC(config RCConfig) error { } oldPods = pods oldRunning = running + + if time.Since(lastChange) > timeout { + dumpPodDebugInfo(config.Client, pods) + break + } } if oldRunning != config.Replicas { @@ -1112,6 +1117,45 @@ func RunRC(config RCConfig) error { return nil } +func dumpPodDebugInfo(c *client.Client, pods []*api.Pod) { + badNodes := util.NewStringSet() + for _, p := range pods { + if p.Status.Phase != api.PodRunning { + if p.Spec.NodeName != "" { + Logf("Pod %v assigned to host %v (IP: %v) in %v", p.Name, p.Spec.NodeName, p.Status.HostIP, p.Status.Phase) + badNodes.Insert(p.Spec.NodeName) + } else { + Logf("Pod %v still unassigned", p.Name) + } + } + } + dumpNodeDebugInfo(c, badNodes.List()) +} + +func dumpNodeDebugInfo(c *client.Client, nodeNames []string) { + // TODO: Actually log running pods instead of the pods in the pod manager. + for _, n := range nodeNames { + Logf("\nLogging pods the kubelet thinks is on node %v", n) + podList, err := GetKubeletPods(c, n) + if err != nil { + Logf("Unable to retrieve kubelet pods for node %v", n) + continue + } + for _, p := range podList.Items { + // If the pod is in pending it's probably not going to have a starttime or container statuses, since + // we're only checking the pod manager. This should change. + Logf("%v started at %v (%d containers)", p.Name, p.Status.StartTime, len(p.Status.ContainerStatuses)) + for _, c := range p.Status.ContainerStatuses { + Logf("\tContainer %v ready: %v, restart count %v", + c.Name, c.Ready, c.RestartCount) + } + } + HighLatencyKubeletOperations(c, 10*time.Second, n) + // TODO: Log node resource info + } + +} + func ScaleRC(c *client.Client, ns, name string, size uint) error { By(fmt.Sprintf("%v Scaling replication controller %s in namespace %s to %d", time.Now(), name, ns, size)) scaler, err := kubectl.ScalerFor("ReplicationController", kubectl.NewScalerClient(c))