Add timeouts to HealthChecks and retry checks

Fixes issue #3532. Added timeouts for HTTP and TCP checks
and enabled kubelet/probe to kubelet#maxRetries times
before declaring Failure.

Added Probe.TimeoutSecs to API

Probe variants now check container.LivenessProbe.TimeoutSeconds
Also added a test for timeouts in http_test.go.

pkg/probe/tcp/tcp.go

@@ -19,6 +19,7 @@ package tcp
 import (
 	"net"
 	"strconv"
+	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/probe"
 
@@ -31,16 +32,16 @@ func New() TCPProber {
 
 type TCPProber struct{}
 
-func (pr TCPProber) Probe(host string, port int) (probe.Status, error) {
-	return DoTCPProbe(net.JoinHostPort(host, strconv.Itoa(port)))
+func (pr TCPProber) Probe(host string, port int, timeout time.Duration) (probe.Status, error) {
+	return DoTCPProbe(net.JoinHostPort(host, strconv.Itoa(port)), timeout)
 }
 
 // DoTCPProbe checks that a TCP socket to the address can be opened.
 // If the socket can be opened, it returns Success
 // If the socket fails to open, it returns Failure.
 // This is exported because some other packages may want to do direct TCP probes.
-func DoTCPProbe(addr string) (probe.Status, error) {
-	conn, err := net.Dial("tcp", addr)
+func DoTCPProbe(addr string, timeout time.Duration) (probe.Status, error) {
+	conn, err := net.DialTimeout("tcp", addr, timeout)
 	if err != nil {
 		return probe.Failure, nil
 	}

pkg/probe/tcp/tcp_test.go

@@ -23,6 +23,7 @@ import (
 	"net/url"
 	"strconv"
 	"testing"
+	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/probe"
 )
@@ -59,7 +60,7 @@ func TestTcpHealthChecker(t *testing.T) {
 		if !test.usePort {
 			p = -1
 		}
-		status, err := prober.Probe(host, p)
+		status, err := prober.Probe(host, p, 1*time.Second)
 		if status != test.expectedStatus {
 			t.Errorf("expected: %v, got: %v", test.expectedStatus, status)
 		}