Merge pull request #388 from brendandburns/health

Address some comments from thockin@
This commit is contained in:
Tim Hockin 2014-07-11 11:27:32 -07:00
commit 9861f0d806
5 changed files with 41 additions and 32 deletions

View File

@ -116,11 +116,10 @@ type HTTPGetProbe struct {
// LivenessProbe describes a liveness probe to be examined to the container.
type LivenessProbe struct {
Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"`
// Type of liveness probe. Current legal values "http"
Type string `yaml:"type,omitempty" json:"type,omitempty"`
// HTTPGetProbe parameters, required if Type == 'http'
HTTPGet HTTPGetProbe `yaml:"httpGet,omitempty" json:"httpGet,omitempty"`
HTTPGet *HTTPGetProbe `yaml:"httpGet,omitempty" json:"httpGet,omitempty"`
// Length of time before health checking is activated. In seconds.
InitialDelaySeconds int64 `yaml:"initialDelaySeconds,omitempty" json:"initialDelaySeconds,omitempty"`
}
@ -141,9 +140,9 @@ type Container struct {
// Optional: Defaults to unlimited.
Memory int `yaml:"memory,omitempty" json:"memory,omitempty"`
// Optional: Defaults to unlimited.
CPU int `yaml:"cpu,omitempty" json:"cpu,omitempty"`
VolumeMounts []VolumeMount `yaml:"volumeMounts,omitempty" json:"volumeMounts,omitempty"`
LivenessProbe LivenessProbe `yaml:"livenessProbe,omitempty" json:"livenessProbe,omitempty"`
CPU int `yaml:"cpu,omitempty" json:"cpu,omitempty"`
VolumeMounts []VolumeMount `yaml:"volumeMounts,omitempty" json:"volumeMounts,omitempty"`
LivenessProbe *LivenessProbe `yaml:"livenessProbe,omitempty" json:"livenessProbe,omitempty"`
}
// Percentile represents a pair which contains a percentage from 0 to 100 and

View File

@ -25,10 +25,16 @@ import (
"github.com/golang/glog"
)
// HealthChecker is an abstract interface for container health checker.
type HealthCheckStatus int
const (
CheckHealthy HealthCheckStatus = 0
CheckUnhealthy HealthCheckStatus = 1
CheckUnknown HealthCheckStatus = 2
)
type HealthChecker interface {
// IsHealthy checks if the container is healthy.
IsHealthy(container api.Container) (bool, error)
HealthCheck(container api.Container) (HealthCheckStatus, error)
}
type httpDoInterface interface {
@ -51,14 +57,13 @@ type MuxHealthChecker struct {
checkers map[string]HealthChecker
}
// IsHealthy checks the health of the container by delegating to an appropriate HealthChecker according to container.LivenessProbe.Type.
func (m *MuxHealthChecker) IsHealthy(container api.Container) (bool, error) {
func (m *MuxHealthChecker) HealthCheck(container api.Container) (HealthCheckStatus, error) {
checker, ok := m.checkers[container.LivenessProbe.Type]
if !ok || checker == nil {
glog.Warningf("Failed to find health checker for %s %s", container.Name, container.LivenessProbe.Type)
return true, nil
return CheckUnknown, nil
}
return checker.IsHealthy(container)
return checker.HealthCheck(container)
}
// HTTPHealthChecker is an implementation of HealthChecker which checks container health by sending HTTP Get requests.
@ -76,15 +81,17 @@ func (h *HTTPHealthChecker) findPort(container api.Container, portName string) i
return -1
}
// IsHealthy checks if the container is healthy by trying sending HTTP Get requests to the container.
func (h *HTTPHealthChecker) IsHealthy(container api.Container) (bool, error) {
func (h *HTTPHealthChecker) HealthCheck(container api.Container) (HealthCheckStatus, error) {
params := container.LivenessProbe.HTTPGet
if params == nil {
return CheckUnknown, fmt.Errorf("Error, no HTTP parameters specified: %v", container)
}
port := h.findPort(container, params.Port)
if port == -1 {
var err error
port, err = strconv.ParseInt(params.Port, 10, 0)
if err != nil {
return true, err
return CheckUnknown, err
}
}
var host string
@ -100,7 +107,11 @@ func (h *HTTPHealthChecker) IsHealthy(container api.Container) (bool, error) {
}
if err != nil {
// At this point, if it fails, its either a policy (unlikely) or HTTP protocol (likely) error.
return false, nil
return CheckUnhealthy, nil
}
return res.StatusCode == http.StatusOK, nil
if res.StatusCode == http.StatusOK {
return CheckHealthy, nil
}
glog.V(1).Infof("Health check failed for %v, Response: %v", container, *res)
return CheckUnhealthy, nil
}

View File

@ -46,8 +46,8 @@ func TestHttpHealth(t *testing.T) {
}
container := api.Container{
LivenessProbe: api.LivenessProbe{
HTTPGet: api.HTTPGetProbe{
LivenessProbe: &api.LivenessProbe{
HTTPGet: &api.HTTPGetProbe{
Port: "8080",
Path: "/foo/bar",
},
@ -55,8 +55,8 @@ func TestHttpHealth(t *testing.T) {
},
}
ok, err := check.IsHealthy(container)
if !ok {
ok, err := check.HealthCheck(container)
if ok != CheckHealthy {
t.Error("Unexpected unhealthy")
}
if err != nil {

View File

@ -732,7 +732,7 @@ func (kl *Kubelet) syncManifest(manifest *api.ContainerManifest, keepChannel cha
glog.V(1).Infof("health check errored: %v", err)
continue
}
if !healthy {
if healthy != CheckHealthy {
glog.V(1).Infof("manifest %s container %s is unhealthy.", manifest.ID, container.Name)
if err != nil {
glog.V(1).Infof("Failed to get container info %v, for %s", err, containerID)
@ -989,16 +989,16 @@ func (kl *Kubelet) GetMachineStats() (*api.ContainerStats, error) {
return kl.statsFromContainerPath("/")
}
func (kl *Kubelet) healthy(container api.Container, dockerContainer *docker.APIContainers) (bool, error) {
func (kl *Kubelet) healthy(container api.Container, dockerContainer *docker.APIContainers) (HealthCheckStatus, error) {
// Give the container 60 seconds to start up.
if !container.LivenessProbe.Enabled {
return true, nil
if container.LivenessProbe == nil {
return CheckHealthy, nil
}
if time.Now().Unix()-dockerContainer.Created < container.LivenessProbe.InitialDelaySeconds {
return true, nil
return CheckHealthy, nil
}
if kl.HealthChecker == nil {
return true, nil
return CheckHealthy, nil
}
return kl.HealthChecker.IsHealthy(container)
return kl.HealthChecker.HealthCheck(container)
}

View File

@ -424,8 +424,8 @@ func TestSyncManifestsDeletes(t *testing.T) {
type FalseHealthChecker struct{}
func (f *FalseHealthChecker) IsHealthy(container api.Container) (bool, error) {
return false, nil
func (f *FalseHealthChecker) HealthCheck(container api.Container) (HealthCheckStatus, error) {
return CheckUnhealthy, nil
}
func TestSyncManifestsUnhealthy(t *testing.T) {
@ -448,8 +448,7 @@ func TestSyncManifestsUnhealthy(t *testing.T) {
ID: "foo",
Containers: []api.Container{
{Name: "bar",
LivenessProbe: api.LivenessProbe{
Enabled: true,
LivenessProbe: &api.LivenessProbe{
// Always returns healthy == false
Type: "false",
},