diff --git a/pkg/metrics/kubelet_metrics.go b/pkg/metrics/kubelet_metrics.go index 4cacfd09e1b..2c779655ae7 100644 --- a/pkg/metrics/kubelet_metrics.go +++ b/pkg/metrics/kubelet_metrics.go @@ -18,6 +18,7 @@ package metrics import ( "fmt" + "time" "k8s.io/kubernetes/pkg/util/sets" @@ -133,14 +134,27 @@ func parseKubeletMetrics(data string) (KubeletMetrics, error) { } func (g *MetricsGrabber) getMetricsFromNode(nodeName string, kubeletPort int) (string, error) { - rawOutput, err := g.client.Get(). - Prefix("proxy"). - Resource("nodes"). - Name(fmt.Sprintf("%v:%v", nodeName, kubeletPort)). - Suffix("metrics"). - Do().Raw() - if err != nil { - return "", err + // There's a problem with timing out during proxy. Wrapping this in a goroutine to prevent deadlock. + // Hanging goroutine will be leaked. + finished := make(chan struct{}) + var err error + var rawOutput []byte + go func() { + rawOutput, err = g.client.Get(). + Prefix("proxy"). + Resource("nodes"). + Name(fmt.Sprintf("%v:%v", nodeName, kubeletPort)). + Suffix("metrics"). + Do().Raw() + finished <- struct{}{} + }() + select { + case <-time.After(ProxyTimeout): + return "", fmt.Errorf("Timed out when waiting for proxy to gather metrics from %v", nodeName) + case <-finished: + if err != nil { + return "", err + } + return string(rawOutput), nil } - return string(rawOutput), nil } diff --git a/pkg/metrics/metrics_grabber.go b/pkg/metrics/metrics_grabber.go index 62eabfffb85..ac95999ec5f 100644 --- a/pkg/metrics/metrics_grabber.go +++ b/pkg/metrics/metrics_grabber.go @@ -19,6 +19,7 @@ package metrics import ( "fmt" "strings" + "time" "k8s.io/kubernetes/pkg/api" client "k8s.io/kubernetes/pkg/client/unversioned" @@ -29,6 +30,10 @@ import ( "github.com/golang/glog" ) +const ( + ProxyTimeout = 2 * time.Minute +) + type MetricsCollection struct { ApiServerMetrics ApiServerMetrics ControllerManagerMetrics ControllerManagerMetrics