diff --git a/pkg/kubelet/prober/worker.go b/pkg/kubelet/prober/worker.go index 1efffe5888f..cac63117b4c 100644 --- a/pkg/kubelet/prober/worker.go +++ b/pkg/kubelet/prober/worker.go @@ -222,6 +222,20 @@ func (w *worker) doProbe() (keepGoing bool) { w.pod.Spec.RestartPolicy != v1.RestartPolicyNever } + // Graceful shutdown of the pod. + if w.pod.ObjectMeta.DeletionTimestamp != nil && (w.probeType == liveness || w.probeType == startup) { + klog.V(3).Infof("Pod deletion requested, setting %v probe result to success: %v - %v", + w.probeType.String(), format.Pod(w.pod), w.container.Name) + if w.probeType == startup { + klog.Warningf("Pod deletion requested before container has fully started: %v - %v", + format.Pod(w.pod), w.container.Name) + } + // Set a last result to ensure quiet shutdown. + w.resultsManager.Set(w.containerID, results.Success, w.pod) + // Stop probing at this point. + return false + } + // Probe disabled for InitialDelaySeconds. if int32(time.Since(c.State.Running.StartedAt.Time).Seconds()) < w.spec.InitialDelaySeconds { return true @@ -230,7 +244,7 @@ func (w *worker) doProbe() (keepGoing bool) { if c.Started != nil && *c.Started { // Stop probing for startup once container has started. if w.probeType == startup { - return true + return false } } else { // Disable other probes until container has started. diff --git a/pkg/kubelet/prober/worker_test.go b/pkg/kubelet/prober/worker_test.go index c2e3cba971c..e8d9e115029 100644 --- a/pkg/kubelet/prober/worker_test.go +++ b/pkg/kubelet/prober/worker_test.go @@ -58,25 +58,47 @@ func TestDoProbe(t *testing.T) { failedStatus.Phase = v1.PodFailed tests := []struct { - probe v1.Probe - podStatus *v1.PodStatus - expectContinue bool - expectSet bool - expectedResult results.Result + probe v1.Probe + podStatus *v1.PodStatus + expectContinue map[string]bool + expectSet bool + expectedResult results.Result + setDeletionTimestamp bool }{ { // No status. - expectContinue: true, + expectContinue: map[string]bool{ + liveness.String(): true, + readiness.String(): true, + startup.String(): true, + }, }, { // Pod failed podStatus: &failedStatus, }, + { // Pod deletion + podStatus: &runningStatus, + setDeletionTimestamp: true, + expectSet: true, + expectContinue: map[string]bool{ + readiness.String(): true, + }, + expectedResult: results.Success, + }, { // No container status - podStatus: &otherStatus, - expectContinue: true, + podStatus: &otherStatus, + expectContinue: map[string]bool{ + liveness.String(): true, + readiness.String(): true, + startup.String(): true, + }, }, { // Container waiting - podStatus: &pendingStatus, - expectContinue: true, + podStatus: &pendingStatus, + expectContinue: map[string]bool{ + liveness.String(): true, + readiness.String(): true, + startup.String(): true, + }, expectSet: true, expectedResult: results.Failure, }, @@ -86,8 +108,12 @@ func TestDoProbe(t *testing.T) { expectedResult: results.Failure, }, { // Probe successful. - podStatus: &runningStatus, - expectContinue: true, + podStatus: &runningStatus, + expectContinue: map[string]bool{ + liveness.String(): true, + readiness.String(): true, + startup.String(): true, + }, expectSet: true, expectedResult: results.Success, }, @@ -96,7 +122,11 @@ func TestDoProbe(t *testing.T) { probe: v1.Probe{ InitialDelaySeconds: -100, }, - expectContinue: true, + expectContinue: map[string]bool{ + liveness.String(): true, + readiness.String(): true, + startup.String(): true, + }, expectSet: true, expectedResult: results.Success, }, @@ -107,8 +137,12 @@ func TestDoProbe(t *testing.T) { if test.podStatus != nil { m.statusManager.SetPodStatus(w.pod, *test.podStatus) } - if c := w.doProbe(); c != test.expectContinue { - t.Errorf("[%s-%d] Expected continue to be %v but got %v", probeType, i, test.expectContinue, c) + if test.setDeletionTimestamp { + now := metav1.Now() + w.pod.ObjectMeta.DeletionTimestamp = &now + } + if c := w.doProbe(); c != test.expectContinue[probeType.String()] { + t.Errorf("[%s-%d] Expected continue to be %v but got %v", probeType, i, test.expectContinue[probeType.String()], c) } result, ok := resultsManager(m, probeType).Get(testContainerID) if ok != test.expectSet { @@ -299,6 +333,12 @@ func expectContinue(t *testing.T, w *worker, c bool, msg string) { } } +func expectStop(t *testing.T, w *worker, c bool, msg string) { + if c { + t.Errorf("[%s - %s] Expected to stop, but did not", w.probeType, msg) + } +} + func resultsManager(m *manager, probeType probeType) results.Manager { switch probeType { case readiness: @@ -468,6 +508,6 @@ func TestStartupProbeDisabledByStarted(t *testing.T) { // startupProbe fails, but is disabled m.prober.exec = fakeExecProber{probe.Failure, nil} msg = "Started, probe failure, result success" - expectContinue(t, w, w.doProbe(), msg) + expectStop(t, w, w.doProbe(), msg) expectResult(t, w, results.Success, msg) }