Better kubelet logging for probes

Log when we actually run probes and event when they fail.  Print the output of
a probe, too.

pkg/probe/http/http.go

@@ -17,6 +17,7 @@ limitations under the License.
 package http
 
 import (
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
@@ -34,7 +35,7 @@ func New() HTTPProber {
 }
 
 type HTTPProber interface {
-	Probe(host string, port int, path string, timeout time.Duration) (probe.Result, error)
+	Probe(host string, port int, path string, timeout time.Duration) (probe.Result, string, error)
 }
 
 type httpProber struct {
@@ -42,7 +43,7 @@ type httpProber struct {
 }
 
 // Probe returns a ProbeRunner capable of running an http check.
-func (pr httpProber) Probe(host string, port int, path string, timeout time.Duration) (probe.Result, error) {
+func (pr httpProber) Probe(host string, port int, path string, timeout time.Duration) (probe.Result, string, error) {
 	return DoHTTPProbe(formatURL(host, port, path), &http.Client{Timeout: timeout, Transport: pr.transport})
 }
 
@@ -54,18 +55,23 @@ type HTTPGetInterface interface {
 // If the HTTP response code is successful (i.e. 400 > code >= 200), it returns Success.
 // If the HTTP response code is unsuccessful or HTTP communication fails, it returns Failure.
 // This is exported because some other packages may want to do direct HTTP probes.
-func DoHTTPProbe(url string, client HTTPGetInterface) (probe.Result, error) {
+func DoHTTPProbe(url string, client HTTPGetInterface) (probe.Result, string, error) {
 	res, err := client.Get(url)
 	if err != nil {
-		glog.V(1).Infof("HTTP probe error: %v", err)
-		return probe.Failure, nil
+		// Convert errors into failures to catch timeouts.
+		return probe.Failure, err.Error(), nil
 	}
 	defer res.Body.Close()
-	if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
-		return probe.Success, nil
+	b, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return probe.Failure, "", err
 	}
-	glog.V(1).Infof("Health check failed for %s, Response: %v", url, *res)
-	return probe.Failure, nil
+	body := string(b)
+	if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
+		return probe.Success, body, nil
+	}
+	glog.V(4).Infof("Probe failed for %s, Response: %v", url, *res)
+	return probe.Failure, body, nil
 }
 
 // formatURL formats a URL from args.  For testability.

pkg/probe/http/http_test.go

@@ -22,6 +22,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -47,19 +48,23 @@ func TestFormatURL(t *testing.T) {
 }
 
 func TestHTTPProbeChecker(t *testing.T) {
-	handleReq := func(s int) func(w http.ResponseWriter) {
-		return func(w http.ResponseWriter) { w.WriteHeader(s) }
+	handleReq := func(s int, body string) func(w http.ResponseWriter) {
+		return func(w http.ResponseWriter) {
+			w.WriteHeader(s)
+			w.Write([]byte(body))
+		}
 	}
 
 	prober := New()
 	testCases := []struct {
 		handler func(w http.ResponseWriter)
 		health  probe.Result
+		body    string
 	}{
 		// The probe will be filled in below.  This is primarily testing that an HTTP GET happens.
-		{handleReq(http.StatusOK), probe.Success},
-		{handleReq(-1), probe.Failure},
-		{func(w http.ResponseWriter) { time.Sleep(3 * time.Second) }, probe.Failure},
+		{handleReq(http.StatusOK, "ok body"), probe.Success, "ok body"},
+		{handleReq(-1, "fail body"), probe.Failure, "fail body"},
+		{func(w http.ResponseWriter) { time.Sleep(3 * time.Second) }, probe.Failure, "use of closed network connection"},
 	}
 	for _, test := range testCases {
 		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -77,7 +82,7 @@ func TestHTTPProbeChecker(t *testing.T) {
 		if err != nil {
 			t.Errorf("Unexpected error: %v", err)
 		}
-		health, err := prober.Probe(host, p, "", 1*time.Second)
+		health, output, err := prober.Probe(host, p, "", 1*time.Second)
 		if test.health == probe.Unknown && err == nil {
 			t.Errorf("Expected error")
 		}
@@ -87,5 +92,8 @@ func TestHTTPProbeChecker(t *testing.T) {
 		if health != test.health {
 			t.Errorf("Expected %v, got %v", test.health, health)
 		}
+		if !strings.Contains(output, test.body) {
+			t.Errorf("Expected %v, got %v", test.body, output)
+		}
 	}
 }