seperate readiness and liveness in the code

This commit is contained in:
Mike Danese 2015-06-11 10:47:25 -07:00
parent 5a02fc07d8
commit 35771acdfa
2 changed files with 87 additions and 86 deletions

View File

@ -583,6 +583,8 @@ func replaceProber(dm *DockerManager, result probe.Result, err error) {
// delay has passed, whether the probe handler will return Success, Failure, // delay has passed, whether the probe handler will return Success, Failure,
// Unknown or error. // Unknown or error.
// //
// PLEASE READ THE PROBE DOCS BEFORE CHANGING THIS TEST IF YOU ARE UNSURE HOW PROBES ARE SUPPOSED TO WORK:
// (See https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/pod-states.md#pod-conditions)
func TestProbeContainer(t *testing.T) { func TestProbeContainer(t *testing.T) {
manager, _ := newTestDockerManager() manager, _ := newTestDockerManager()
dc := &docker.APIContainers{ dc := &docker.APIContainers{
@ -601,7 +603,7 @@ func TestProbeContainer(t *testing.T) {
expectedResult: probe.Success, expectedResult: probe.Success,
expectedReadiness: true, expectedReadiness: true,
}, },
// Only LivenessProbe. // Only LivenessProbe. expectedReadiness should always be true here.
{ {
testContainer: api.Container{ testContainer: api.Container{
LivenessProbe: &api.Probe{InitialDelaySeconds: 100}, LivenessProbe: &api.Probe{InitialDelaySeconds: 100},
@ -614,7 +616,7 @@ func TestProbeContainer(t *testing.T) {
LivenessProbe: &api.Probe{InitialDelaySeconds: -100}, LivenessProbe: &api.Probe{InitialDelaySeconds: -100},
}, },
expectedResult: probe.Unknown, expectedResult: probe.Unknown,
expectedReadiness: false, expectedReadiness: true,
}, },
{ {
testContainer: api.Container{ testContainer: api.Container{
@ -626,7 +628,7 @@ func TestProbeContainer(t *testing.T) {
}, },
}, },
expectedResult: probe.Failure, expectedResult: probe.Failure,
expectedReadiness: false, expectedReadiness: true,
}, },
{ {
testContainer: api.Container{ testContainer: api.Container{
@ -650,7 +652,7 @@ func TestProbeContainer(t *testing.T) {
}, },
}, },
expectedResult: probe.Unknown, expectedResult: probe.Unknown,
expectedReadiness: false, expectedReadiness: true,
}, },
{ {
testContainer: api.Container{ testContainer: api.Container{
@ -663,21 +665,21 @@ func TestProbeContainer(t *testing.T) {
}, },
expectError: true, expectError: true,
expectedResult: probe.Unknown, expectedResult: probe.Unknown,
expectedReadiness: false, expectedReadiness: true,
}, },
// Only ReadinessProbe. // // Only ReadinessProbe. expectedResult should always be probe.Success here.
{ {
testContainer: api.Container{ testContainer: api.Container{
ReadinessProbe: &api.Probe{InitialDelaySeconds: 100}, ReadinessProbe: &api.Probe{InitialDelaySeconds: 100},
}, },
expectedResult: probe.Failure, expectedResult: probe.Success,
expectedReadiness: false, expectedReadiness: false,
}, },
{ {
testContainer: api.Container{ testContainer: api.Container{
ReadinessProbe: &api.Probe{InitialDelaySeconds: -100}, ReadinessProbe: &api.Probe{InitialDelaySeconds: -100},
}, },
expectedResult: probe.Unknown, expectedResult: probe.Success,
expectedReadiness: false, expectedReadiness: false,
}, },
{ {
@ -735,7 +737,7 @@ func TestProbeContainer(t *testing.T) {
LivenessProbe: &api.Probe{InitialDelaySeconds: 100}, LivenessProbe: &api.Probe{InitialDelaySeconds: 100},
ReadinessProbe: &api.Probe{InitialDelaySeconds: 100}, ReadinessProbe: &api.Probe{InitialDelaySeconds: 100},
}, },
expectedResult: probe.Failure, expectedResult: probe.Success,
expectedReadiness: false, expectedReadiness: false,
}, },
{ {
@ -743,7 +745,7 @@ func TestProbeContainer(t *testing.T) {
LivenessProbe: &api.Probe{InitialDelaySeconds: 100}, LivenessProbe: &api.Probe{InitialDelaySeconds: 100},
ReadinessProbe: &api.Probe{InitialDelaySeconds: -100}, ReadinessProbe: &api.Probe{InitialDelaySeconds: -100},
}, },
expectedResult: probe.Unknown, expectedResult: probe.Success,
expectedReadiness: false, expectedReadiness: false,
}, },
{ {
@ -816,16 +818,16 @@ func TestProbeContainer(t *testing.T) {
} }
result, err := manager.prober.Probe(&api.Pod{}, api.PodStatus{}, test.testContainer, dc.ID, dc.Created) result, err := manager.prober.Probe(&api.Pod{}, api.PodStatus{}, test.testContainer, dc.ID, dc.Created)
if test.expectError && err == nil { if test.expectError && err == nil {
t.Error("[%d] Expected error but did no error was returned.", i) t.Error("[%d] Expected error but no error was returned.", i)
} }
if !test.expectError && err != nil { if !test.expectError && err != nil {
t.Errorf("[%d] Expected error but got: %v", i, err) t.Errorf("[%d] Didn't expect error but got: %v", i, err)
} }
if test.expectedResult != result { if test.expectedResult != result {
t.Errorf("[%d] Expected result was %v but probeContainer() returned %v", i, test.expectedResult, result) t.Errorf("[%d] Expected result to be %v but was %v", i, test.expectedResult, result)
} }
if test.expectedReadiness != manager.readinessManager.GetReadiness(dc.ID) { if test.expectedReadiness != manager.readinessManager.GetReadiness(dc.ID) {
t.Errorf("[%d] Expected readiness was %v but probeContainer() set %v", i, test.expectedReadiness, manager.readinessManager.GetReadiness(dc.ID)) t.Errorf("[%d] Expected readiness to be %v but was %v", i, test.expectedReadiness, manager.readinessManager.GetReadiness(dc.ID))
} }
} }
} }

View File

@ -89,18 +89,29 @@ func NewTestProber(
} }
// Probe checks the liveness/readiness of the given container. // Probe checks the liveness/readiness of the given container.
// If the container's liveness probe is unsuccessful, set readiness to false.
// If liveness is successful, do a readiness check and set readiness accordingly.
func (pb *prober) Probe(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) { func (pb *prober) Probe(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) {
// Build a name string for logs. pb.probeReadiness(pod, status, container, containerID, createdAt)
ctrName := fmt.Sprintf("%s:%s", kubecontainer.GetPodFullName(pod), container.Name) return pb.probeLiveness(pod, status, container, containerID, createdAt)
}
// Probe liveness. // probeLiveness probes the liveness of a container.
if container.LivenessProbe != nil { // If the initalDelay since container creation on liveness probe has not passed the probe will return probe.Success.
live, output, err := pb.probeLiveness(pod, status, container, containerID, createdAt) func (pb *prober) probeLiveness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) {
var live probe.Result
var output string
var err error
p := container.LivenessProbe
if p == nil {
return probe.Success, nil
}
if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
return probe.Success, nil
} else {
live, output, err = pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
}
ctrName := fmt.Sprintf("%s:%s", kubecontainer.GetPodFullName(pod), container.Name)
if err != nil || live != probe.Success { if err != nil || live != probe.Success {
// Liveness failed in one way or another. // Liveness failed in one way or another.
pb.readinessManager.SetReadiness(containerID, false)
ref, ok := pb.refManager.GetRef(containerID) ref, ok := pb.refManager.GetRef(containerID)
if !ok { if !ok {
glog.Warningf("No ref for pod %q - '%v'", containerID, container.Name) glog.Warningf("No ref for pod %q - '%v'", containerID, container.Name)
@ -120,12 +131,25 @@ func (pb *prober) Probe(pod *api.Pod, status api.PodStatus, container api.Contai
} }
} }
glog.V(3).Infof("Liveness probe for %q succeeded", ctrName) glog.V(3).Infof("Liveness probe for %q succeeded", ctrName)
} return probe.Success, nil
}
// Probe readiness. // probeReadiness probes and sets the readiness of a container.
if container.ReadinessProbe != nil { // If the initial delay on the readiness probe has not passed, we set readiness to false.
ready, output, err := pb.probeReadiness(pod, status, container, containerID, createdAt) func (pb *prober) probeReadiness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) {
if err != nil || ready != probe.Success { var ready probe.Result
var output string
var err error
p := container.ReadinessProbe
if p == nil {
ready = probe.Success
} else if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
ready = probe.Failure
} else {
ready, output, err = pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
}
ctrName := fmt.Sprintf("%s:%s", kubecontainer.GetPodFullName(pod), container.Name)
if err != nil || ready == probe.Failure {
// Readiness failed in one way or another. // Readiness failed in one way or another.
pb.readinessManager.SetReadiness(containerID, false) pb.readinessManager.SetReadiness(containerID, false)
ref, ok := pb.refManager.GetRef(containerID) ref, ok := pb.refManager.GetRef(containerID)
@ -137,46 +161,21 @@ func (pb *prober) Probe(pod *api.Pod, status api.PodStatus, container api.Contai
if ok { if ok {
pb.recorder.Eventf(ref, "unhealthy", "Readiness probe errored: %v", err) pb.recorder.Eventf(ref, "unhealthy", "Readiness probe errored: %v", err)
} }
return probe.Unknown, err return
} else { // ready != probe.Success } else { // ready != probe.Success
glog.V(1).Infof("Readiness probe for %q failed (%v): %s", ctrName, ready, output) glog.V(1).Infof("Readiness probe for %q failed (%v): %s", ctrName, ready, output)
if ok { if ok {
pb.recorder.Eventf(ref, "unhealthy", "Readiness probe failed: %s", output) pb.recorder.Eventf(ref, "unhealthy", "Readiness probe failed: %s", output)
} }
return ready, nil return
} }
} }
glog.V(3).Infof("Readiness probe for %q succeeded", ctrName) if ready == probe.Success {
}
pb.readinessManager.SetReadiness(containerID, true) pb.readinessManager.SetReadiness(containerID, true)
return probe.Success, nil }
}
// probeLiveness probes the liveness of a container. glog.V(3).Infof("Readiness probe for %q succeeded", ctrName)
// If the initalDelay since container creation on liveness probe has not passed the probe will return probe.Success.
func (pb *prober) probeLiveness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, string, error) {
p := container.LivenessProbe
if p == nil {
return probe.Success, "", nil
}
if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
return probe.Success, "", nil
}
return pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
}
// probeReadiness probes the readiness of a container.
// If the initial delay on the readiness probe has not passed the probe will return probe.Failure.
func (pb *prober) probeReadiness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, string, error) {
p := container.ReadinessProbe
if p == nil {
return probe.Success, "", nil
}
if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
return probe.Failure, "", nil
}
return pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
} }
// runProbeWithRetries tries to probe the container in a finite loop, it returns the last result // runProbeWithRetries tries to probe the container in a finite loop, it returns the last result