From e2e938066de97345e0c440b469ead42cbe0e7984 Mon Sep 17 00:00:00 2001 From: Ryan Phillips Date: Wed, 22 Sep 2021 14:13:25 -0500 Subject: [PATCH] kubelet: add probe termination to graceful shutdowns --- pkg/kubelet/kubelet.go | 2 +- pkg/kubelet/kubelet_test.go | 2 +- .../nodeshutdown/nodeshutdown_manager_linux.go | 12 +++++++++--- .../nodeshutdown/nodeshutdown_manager_linux_test.go | 10 +++++++--- .../nodeshutdown/nodeshutdown_manager_others.go | 3 ++- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 00a2d1575fe..34396c82b4c 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -867,7 +867,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, util.SetNodeOwnerFunc(klet.heartbeatClient, string(klet.nodeName))) // setup node shutdown manager - shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(kubeDeps.Recorder, nodeRef, klet.GetActivePods, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.syncNodeStatus, kubeCfg.ShutdownGracePeriod.Duration, kubeCfg.ShutdownGracePeriodCriticalPods.Duration) + shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(klet.probeManager, kubeDeps.Recorder, nodeRef, klet.GetActivePods, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.syncNodeStatus, kubeCfg.ShutdownGracePeriod.Duration, kubeCfg.ShutdownGracePeriodCriticalPods.Duration) klet.shutdownManager = shutdownManager klet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler) diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index fe36fb4892a..34eed35b5e8 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -314,7 +314,7 @@ func newTestKubeletWithImageList( kubelet.admitHandlers.AddPodAdmitHandler(evictionAdmitHandler) // setup shutdown manager - shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(fakeRecorder, nodeRef, kubelet.podManager.GetPods, killPodNow(kubelet.podWorkers, fakeRecorder), func() {}, 0 /* shutdownGracePeriodRequested*/, 0 /*shutdownGracePeriodCriticalPods */) + shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(kubelet.probeManager, fakeRecorder, nodeRef, kubelet.podManager.GetPods, killPodNow(kubelet.podWorkers, fakeRecorder), func() {}, 0 /* shutdownGracePeriodRequested*/, 0 /*shutdownGracePeriodCriticalPods */) kubelet.shutdownManager = shutdownManager kubelet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler) diff --git a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go index 6c5e8968c79..9b2d6521cde 100644 --- a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go +++ b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go @@ -34,6 +34,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/nodeshutdown/systemd" + "k8s.io/kubernetes/pkg/kubelet/prober" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/utils/clock" ) @@ -61,8 +62,9 @@ type dbusInhibiter interface { // Manager has functions that can be used to interact with the Node Shutdown Manager. type Manager struct { - recorder record.EventRecorder - nodeRef *v1.ObjectReference + recorder record.EventRecorder + nodeRef *v1.ObjectReference + probeManager prober.Manager shutdownGracePeriodRequested time.Duration shutdownGracePeriodCriticalPods time.Duration @@ -81,7 +83,7 @@ type Manager struct { } // NewManager returns a new node shutdown manager. -func NewManager(recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { +func NewManager(probeManager prober.Manager, recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { manager := &Manager{ recorder: recorder, nodeRef: nodeRef, @@ -91,6 +93,7 @@ func NewManager(recorder record.EventRecorder, nodeRef *v1.ObjectReference, getP shutdownGracePeriodRequested: shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods: shutdownGracePeriodCriticalPods, clock: clock.RealClock{}, + probeManager: probeManager, } return manager, manager } @@ -283,6 +286,9 @@ func (m *Manager) processShutdownEvent() error { gracePeriodOverride = int64(nonCriticalPodGracePeriod.Seconds()) } + // Stop probes for the pod + m.probeManager.RemovePod(pod) + // If the pod's spec specifies a termination gracePeriod which is less than the gracePeriodOverride calculated, use the pod spec termination gracePeriod. if pod.Spec.TerminationGracePeriodSeconds != nil && *pod.Spec.TerminationGracePeriodSeconds <= gracePeriodOverride { gracePeriodOverride = *pod.Spec.TerminationGracePeriodSeconds diff --git a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go index 15e671b4af0..4c64d4b4e13 100644 --- a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go +++ b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/pkg/apis/scheduling" pkgfeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/nodeshutdown/systemd" + probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing" testingclock "k8s.io/utils/clock/testing" ) @@ -236,10 +237,11 @@ func TestManager(t *testing.T) { } defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.GracefulNodeShutdown, true)() + proberManager := probetest.FakeManager{} fakeRecorder := &record.FakeRecorder{} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} - manager, _ := NewManager(fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, tc.shutdownGracePeriodCriticalPods) + manager, _ := NewManager(proberManager, fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, tc.shutdownGracePeriodCriticalPods) manager.clock = testingclock.NewFakeClock(time.Now()) err := manager.Start() @@ -315,10 +317,11 @@ func TestFeatureEnabled(t *testing.T) { } defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.GracefulNodeShutdown, tc.featureGateEnabled)() + proberManager := probetest.FakeManager{} fakeRecorder := &record.FakeRecorder{} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} - manager, _ := NewManager(fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, 0 /*shutdownGracePeriodCriticalPods*/) + manager, _ := NewManager(proberManager, fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, 0 /*shutdownGracePeriodCriticalPods*/) manager.clock = testingclock.NewFakeClock(time.Now()) assert.Equal(t, tc.expectEnabled, manager.isFeatureEnabled()) @@ -361,9 +364,10 @@ func TestRestart(t *testing.T) { return dbus, nil } + proberManager := probetest.FakeManager{} fakeRecorder := &record.FakeRecorder{} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} - manager, _ := NewManager(fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, syncNodeStatus, shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods) + manager, _ := NewManager(proberManager, fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, syncNodeStatus, shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods) err := manager.Start() lock.Unlock() diff --git a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go index 89a118a0421..49a5692a13f 100644 --- a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go +++ b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go @@ -26,13 +26,14 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/lifecycle" + "k8s.io/kubernetes/pkg/kubelet/prober" ) // Manager is a fake node shutdown manager for non linux platforms. type Manager struct{} // NewManager returns a fake node shutdown manager for non linux platforms. -func NewManager(recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { +func NewManager(proberManager prober.Manager, recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { m := &Manager{} return m, m }