Configure the kubelet to bind a simple healthz server to a localhost

port for monitoring by monit. This is in preparation for the standard
kubelet port to switch to SSL only (and eventually to only accepting
connections on the SSL port that present a proper client SSL cert).

Also standardize the formatting of the monit config files a bit.

cluster/saltbase/salt/monit/docker

@@ -1,8 +1,9 @@
 check process docker with pidfile /var/run/docker.pid
-group docker 
+group docker
 start program = "/etc/init.d/docker start"
 stop program = "/etc/init.d/docker stop"
 if does not exist then restart
-if failed unixsocket /var/run/docker.sock
+if failed
+  unixsocket /var/run/docker.sock
   protocol HTTP request "/version"
-  then restart
+then restart

cluster/saltbase/salt/monit/etcd

@@ -1,5 +1,5 @@
 check process etcd with pidfile /var/run/etcd.pid
-group etcd 
+group etcd
 start program = "/etc/init.d/etcd start"
 stop program = "/etc/init.d/etcd stop"
 if failed

cluster/saltbase/salt/monit/kube-proxy

@@ -3,7 +3,9 @@ group kube-proxy
 start program = "/etc/init.d/kube-proxy start"
 stop program = "/etc/init.d/kube-proxy stop"
 if does not exist then restart
-if failed port 10249
-  protocol HTTP request "/healthz"
-  with timeout 10 seconds
-  then restart
+if failed
+  host 127.0.0.1
+  port 10249
+  protocol HTTP
+  request "/healthz"
+then restart

cluster/saltbase/salt/monit/kubelet

@@ -1,9 +1,11 @@
 check process kubelet with pidfile /var/run/kubelet.pid
-group kubelet 
+group kubelet
 start program = "/etc/init.d/kubelet start"
 stop program = "/etc/init.d/kubelet stop"
 if does not exist then restart
-if failed port 10250
-  protocol HTTP request "/healthz"
-  with timeout 10 seconds
-  then restart
+if failed
+  host 127.0.0.1
+  port 10248
+  protocol HTTP
+  request "/healthz"
+then restart

cmd/kubelet/app/server.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
+	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -29,7 +31,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/credentialprovider"
-	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
@@ -72,6 +74,8 @@ type KubeletServer struct {
 	MaxContainerCount              int
 	AuthPath                       string
 	CadvisorPort                   uint
+	HealthzPort                    int
+	HealthzBindAddress             util.IP
 	OOMScoreAdj                    int
 	APIServerList                  util.StringList
 	ClusterDomain                  string
@@ -103,6 +107,8 @@ func NewKubeletServer() *KubeletServer {
 		MaxPerPodContainerCount:     5,
 		MaxContainerCount:           100,
 		CadvisorPort:                4194,
+		HealthzPort:                 10248,
+		HealthzBindAddress:          util.IP(net.ParseIP("127.0.0.1")),
 		OOMScoreAdj:                 -900,
 		MasterServiceNamespace:      api.NamespaceDefault,
 		ImageGCHighThresholdPercent: 90,
@@ -137,6 +143,8 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&s.MaxContainerCount, "maximum_dead_containers", s.MaxContainerCount, "Maximum number of old instances of a containers to retain globally.  Each container takes up some disk space.  Default: 100.")
 	fs.StringVar(&s.AuthPath, "auth_path", s.AuthPath, "Path to .kubernetes_auth file, specifying how to authenticate to API server.")
 	fs.UintVar(&s.CadvisorPort, "cadvisor_port", s.CadvisorPort, "The port of the localhost cAdvisor endpoint")
+	fs.IntVar(&s.HealthzPort, "healthz_port", s.HealthzPort, "The port of the localhost healthz endpoint")
+	fs.Var(&s.HealthzBindAddress, "healthz_bind_address", "The IP address for the healthz server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)")
 	fs.IntVar(&s.OOMScoreAdj, "oom_score_adj", s.OOMScoreAdj, "The oom_score_adj value for kubelet process. Values must be within the range [-1000, 1000]")
 	fs.Var(&s.APIServerList, "api_servers", "List of Kubernetes API servers for publishing events, and reading pods and services. (ip:port), comma separated.")
 	fs.StringVar(&s.ClusterDomain, "cluster_domain", s.ClusterDomain, "Domain for this cluster.  If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
@@ -223,6 +231,16 @@ func (s *KubeletServer) Run(_ []string) error {
 
 	RunKubelet(&kcfg)
 
+	if s.HealthzPort > 0 {
+		healthz.DefaultHealthz()
+		go util.Forever(func() {
+			err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress.String(), strconv.Itoa(s.HealthzPort)), nil)
+			if err != nil {
+				glog.Errorf("Starting health server failed: %v", err)
+			}
+		}, 5*time.Second)
+	}
+
 	// runs forever
 	select {}
 

pkg/master/ports/ports.go

@@ -17,7 +17,13 @@ limitations under the License.
 package ports
 
 const (
-	// KubeletPort is the default port for the kubelet status server on each host machine.
+	// KubeletStatusPort is the default port for the kubelet healthz server.
+	// May be overridden by a flag at startup.
+	KubeletStatusPort = 10248
+	// ProxyPort is the default port for the proxy healthz server.
+	// May be overriden by a flag at startup.
+	ProxyStatusPort = 10249
+	// KubeletPort is the default port for the kubelet server on each host machine.
 	// May be overridden by a flag at startup.
 	KubeletPort = 10250
 	// SchedulerPort is the default port for the scheduler status server.
@@ -26,7 +32,4 @@ const (
 	// ControllerManagerPort is the default port for the controller manager status server.
 	// May be overridden by a flag at startup.
 	ControllerManagerPort = 10252
-	// ProxyPort is the default port for the proxy status server.
-	// May be overriden by a flag at startup.
-	ProxyPort = 10249
 )