diff --git a/pkg/kubelet/events/event.go b/pkg/kubelet/events/event.go index 74359e0514c..1b31dc7e605 100644 --- a/pkg/kubelet/events/event.go +++ b/pkg/kubelet/events/event.go @@ -65,6 +65,7 @@ const ( SuccessfulAttachVolume = "SuccessfulAttachVolume" SuccessfulMountVolume = "SuccessfulMountVolume" NodeRebooted = "Rebooted" + NodeShutdown = "Shutdown" ContainerGCFailed = "ContainerGCFailed" ImageGCFailed = "ImageGCFailed" FailedNodeAllocatableEnforcement = "FailedNodeAllocatableEnforcement" diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index bb3ec0c070f..fa72f734208 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -867,7 +867,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, util.SetNodeOwnerFunc(klet.heartbeatClient, string(klet.nodeName))) // setup node shutdown manager - shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(klet.GetActivePods, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.syncNodeStatus, kubeCfg.ShutdownGracePeriod.Duration, kubeCfg.ShutdownGracePeriodCriticalPods.Duration) + shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(kubeDeps.Recorder, nodeRef, klet.GetActivePods, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.syncNodeStatus, kubeCfg.ShutdownGracePeriod.Duration, kubeCfg.ShutdownGracePeriodCriticalPods.Duration) klet.shutdownManager = shutdownManager klet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler) diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 8f78d672659..b95d82f743d 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -313,7 +313,7 @@ func newTestKubeletWithImageList( kubelet.admitHandlers.AddPodAdmitHandler(evictionAdmitHandler) // setup shutdown manager - shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(kubelet.podManager.GetPods, killPodNow(kubelet.podWorkers, fakeRecorder), func() {}, 0 /* shutdownGracePeriodRequested*/, 0 /*shutdownGracePeriodCriticalPods */) + shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(fakeRecorder, nodeRef, kubelet.podManager.GetPods, killPodNow(kubelet.podWorkers, fakeRecorder), func() {}, 0 /* shutdownGracePeriodRequested*/, 0 /*shutdownGracePeriodCriticalPods */) kubelet.shutdownManager = shutdownManager kubelet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler) diff --git a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go index d3b0410adaa..0b85f0b4950 100644 --- a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go +++ b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go @@ -27,8 +27,10 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/clock" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/tools/record" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/features" + kubeletevents "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/nodeshutdown/systemd" @@ -58,6 +60,9 @@ type dbusInhibiter interface { // Manager has functions that can be used to interact with the Node Shutdown Manager. type Manager struct { + recorder record.EventRecorder + nodeRef *v1.ObjectReference + shutdownGracePeriodRequested time.Duration shutdownGracePeriodCriticalPods time.Duration @@ -75,8 +80,10 @@ type Manager struct { } // NewManager returns a new node shutdown manager. -func NewManager(getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { +func NewManager(recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { manager := &Manager{ + recorder: recorder, + nodeRef: nodeRef, getPods: getPodsFunc, killPodFunc: killPodFunc, syncNodeStatus: syncNodeStatus, @@ -192,6 +199,19 @@ func (m *Manager) start() (chan struct{}, error) { } klog.V(1).InfoS("Shutdown manager detected new shutdown event, isNodeShuttingDownNow", "event", isShuttingDown) + var shutdownType string + if isShuttingDown { + shutdownType = "shutdown" + } else { + shutdownType = "cancelled" + } + klog.V(1).InfoS("Shutdown manager detected new shutdown event", "event", shutdownType) + if isShuttingDown { + m.recorder.Event(m.nodeRef, v1.EventTypeNormal, kubeletevents.NodeShutdown, "Shutdown manager detected shutdown event") + } else { + m.recorder.Event(m.nodeRef, v1.EventTypeNormal, kubeletevents.NodeShutdown, "Shutdown manager detected shutdown cancellation") + } + m.nodeShuttingDownMutex.Lock() m.nodeShuttingDownNow = isShuttingDown m.nodeShuttingDownMutex.Unlock() diff --git a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go index 9d1338a7e97..94783313099 100644 --- a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go +++ b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/tools/record" featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/apis/scheduling" pkgfeatures "k8s.io/kubernetes/pkg/features" @@ -229,7 +230,10 @@ func TestManager(t *testing.T) { } defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.GracefulNodeShutdown, true)() - manager, _ := NewManager(activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, tc.shutdownGracePeriodCriticalPods) + fakeRecorder := &record.FakeRecorder{} + nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} + + manager, _ := NewManager(fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, tc.shutdownGracePeriodCriticalPods) manager.clock = clock.NewFakeClock(time.Now()) err := manager.Start() @@ -303,7 +307,10 @@ func TestFeatureEnabled(t *testing.T) { } defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.GracefulNodeShutdown, tc.featureGateEnabled)() - manager, _ := NewManager(activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, 0 /*shutdownGracePeriodCriticalPods*/) + fakeRecorder := &record.FakeRecorder{} + nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} + + manager, _ := NewManager(fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, 0 /*shutdownGracePeriodCriticalPods*/) manager.clock = clock.NewFakeClock(time.Now()) assert.Equal(t, tc.expectEnabled, manager.isFeatureEnabled()) @@ -345,7 +352,9 @@ func TestRestart(t *testing.T) { return dbus, nil } - manager, _ := NewManager(activePodsFunc, killPodsFunc, syncNodeStatus, shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods) + fakeRecorder := &record.FakeRecorder{} + nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} + manager, _ := NewManager(fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, syncNodeStatus, shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods) err := manager.Start() if err != nil { t.Errorf("unexpected error: %v", err) diff --git a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go index 44228f8be58..ddb050f0a31 100644 --- a/pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go +++ b/pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go @@ -21,6 +21,8 @@ package nodeshutdown import ( "time" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/lifecycle" ) @@ -29,7 +31,7 @@ import ( type Manager struct{} // NewManager returns a fake node shutdown manager for non linux platforms. -func NewManager(getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { +func NewManager(recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { m := &Manager{} return m, m }