github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-26 05:03:09 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #105215 from rphillips/add_probe_shutdown

Browse Source 
kubelet: add probe termination to graceful shutdowns
This commit is contained in:
Kubernetes Prow Robot 2021-10-11 21:19:46 -07:00 committed by GitHub
commit a923852ba0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 20 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/kubelet/kubelet.go
View File

@ -867,7 +867,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
util.SetNodeOwnerFunc(klet.heartbeatClient, string(klet.nodeName))) util.SetNodeOwnerFunc(klet.heartbeatClient, string(klet.nodeName)))
// setup node shutdown manager // setup node shutdown manager
shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(kubeDeps.Recorder, nodeRef, klet.GetActivePods, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.syncNodeStatus, kubeCfg.ShutdownGracePeriod.Duration, kubeCfg.ShutdownGracePeriodCriticalPods.Duration) shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(klet.probeManager, kubeDeps.Recorder, nodeRef, klet.GetActivePods, killPodNow(klet.podWorkers, kubeDeps.Recorder), klet.syncNodeStatus, kubeCfg.ShutdownGracePeriod.Duration, kubeCfg.ShutdownGracePeriodCriticalPods.Duration)
klet.shutdownManager = shutdownManager klet.shutdownManager = shutdownManager
klet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler) klet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler)

2
pkg/kubelet/kubelet_test.go
View File

@ -314,7 +314,7 @@ func newTestKubeletWithImageList(
kubelet.admitHandlers.AddPodAdmitHandler(evictionAdmitHandler) kubelet.admitHandlers.AddPodAdmitHandler(evictionAdmitHandler)
// setup shutdown manager // setup shutdown manager
shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(fakeRecorder, nodeRef, kubelet.podManager.GetPods, killPodNow(kubelet.podWorkers, fakeRecorder), func() {}, 0 /* shutdownGracePeriodRequested*/, 0 /*shutdownGracePeriodCriticalPods */) shutdownManager, shutdownAdmitHandler := nodeshutdown.NewManager(kubelet.probeManager, fakeRecorder, nodeRef, kubelet.podManager.GetPods, killPodNow(kubelet.podWorkers, fakeRecorder), func() {}, 0 /* shutdownGracePeriodRequested*/, 0 /*shutdownGracePeriodCriticalPods */)
kubelet.shutdownManager = shutdownManager kubelet.shutdownManager = shutdownManager
kubelet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler) kubelet.admitHandlers.AddPodAdmitHandler(shutdownAdmitHandler)

8
pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux.go
View File

@ -34,6 +34,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/eviction"
"k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/lifecycle"
"k8s.io/kubernetes/pkg/kubelet/nodeshutdown/systemd" "k8s.io/kubernetes/pkg/kubelet/nodeshutdown/systemd"
"k8s.io/kubernetes/pkg/kubelet/prober"
kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/utils/clock" "k8s.io/utils/clock"
) )
@ -63,6 +64,7 @@ type dbusInhibiter interface {
type Manager struct { type Manager struct {
recorder record.EventRecorder recorder record.EventRecorder
nodeRef *v1.ObjectReference nodeRef *v1.ObjectReference
probeManager prober.Manager
shutdownGracePeriodRequested time.Duration shutdownGracePeriodRequested time.Duration
shutdownGracePeriodCriticalPods time.Duration shutdownGracePeriodCriticalPods time.Duration
@ -81,7 +83,7 @@ type Manager struct {
} }
// NewManager returns a new node shutdown manager. // NewManager returns a new node shutdown manager.
func NewManager(recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { func NewManager(probeManager prober.Manager, recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) {
manager := &Manager{ manager := &Manager{
recorder: recorder, recorder: recorder,
nodeRef: nodeRef, nodeRef: nodeRef,
@ -91,6 +93,7 @@ func NewManager(recorder record.EventRecorder, nodeRef *v1.ObjectReference, getP
shutdownGracePeriodRequested: shutdownGracePeriodRequested, shutdownGracePeriodRequested: shutdownGracePeriodRequested,
shutdownGracePeriodCriticalPods: shutdownGracePeriodCriticalPods, shutdownGracePeriodCriticalPods: shutdownGracePeriodCriticalPods,
clock: clock.RealClock{}, clock: clock.RealClock{},
probeManager: probeManager,
} }
return manager, manager return manager, manager
} }
@ -283,6 +286,9 @@ func (m *Manager) processShutdownEvent() error {
gracePeriodOverride = int64(nonCriticalPodGracePeriod.Seconds()) gracePeriodOverride = int64(nonCriticalPodGracePeriod.Seconds())
} }
// Stop probes for the pod
m.probeManager.RemovePod(pod)
// If the pod's spec specifies a termination gracePeriod which is less than the gracePeriodOverride calculated, use the pod spec termination gracePeriod. // If the pod's spec specifies a termination gracePeriod which is less than the gracePeriodOverride calculated, use the pod spec termination gracePeriod.
if pod.Spec.TerminationGracePeriodSeconds != nil && *pod.Spec.TerminationGracePeriodSeconds <= gracePeriodOverride { if pod.Spec.TerminationGracePeriodSeconds != nil && *pod.Spec.TerminationGracePeriodSeconds <= gracePeriodOverride {
gracePeriodOverride = *pod.Spec.TerminationGracePeriodSeconds gracePeriodOverride = *pod.Spec.TerminationGracePeriodSeconds

10
pkg/kubelet/nodeshutdown/nodeshutdown_manager_linux_test.go
View File

@ -36,6 +36,7 @@ import (
"k8s.io/kubernetes/pkg/apis/scheduling" "k8s.io/kubernetes/pkg/apis/scheduling"
pkgfeatures "k8s.io/kubernetes/pkg/features" pkgfeatures "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/nodeshutdown/systemd" "k8s.io/kubernetes/pkg/kubelet/nodeshutdown/systemd"
probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing"
testingclock "k8s.io/utils/clock/testing" testingclock "k8s.io/utils/clock/testing"
) )
@ -236,10 +237,11 @@ func TestManager(t *testing.T) {
} }
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.GracefulNodeShutdown, true)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.GracefulNodeShutdown, true)()
proberManager := probetest.FakeManager{}
fakeRecorder := &record.FakeRecorder{} fakeRecorder := &record.FakeRecorder{}
nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
manager, _ := NewManager(fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, tc.shutdownGracePeriodCriticalPods) manager, _ := NewManager(proberManager, fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, tc.shutdownGracePeriodCriticalPods)
manager.clock = testingclock.NewFakeClock(time.Now()) manager.clock = testingclock.NewFakeClock(time.Now())
err := manager.Start() err := manager.Start()
@ -315,10 +317,11 @@ func TestFeatureEnabled(t *testing.T) {
} }
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.GracefulNodeShutdown, tc.featureGateEnabled)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.GracefulNodeShutdown, tc.featureGateEnabled)()
proberManager := probetest.FakeManager{}
fakeRecorder := &record.FakeRecorder{} fakeRecorder := &record.FakeRecorder{}
nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
manager, _ := NewManager(fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, 0 /*shutdownGracePeriodCriticalPods*/) manager, _ := NewManager(proberManager, fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, func() {}, tc.shutdownGracePeriodRequested, 0 /*shutdownGracePeriodCriticalPods*/)
manager.clock = testingclock.NewFakeClock(time.Now()) manager.clock = testingclock.NewFakeClock(time.Now())
assert.Equal(t, tc.expectEnabled, manager.isFeatureEnabled()) assert.Equal(t, tc.expectEnabled, manager.isFeatureEnabled())
@ -361,9 +364,10 @@ func TestRestart(t *testing.T) {
return dbus, nil return dbus, nil
} }
proberManager := probetest.FakeManager{}
fakeRecorder := &record.FakeRecorder{} fakeRecorder := &record.FakeRecorder{}
nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
manager, _ := NewManager(fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, syncNodeStatus, shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods) manager, _ := NewManager(proberManager, fakeRecorder, nodeRef, activePodsFunc, killPodsFunc, syncNodeStatus, shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods)
err := manager.Start() err := manager.Start()
lock.Unlock() lock.Unlock()

3
pkg/kubelet/nodeshutdown/nodeshutdown_manager_others.go
View File

@ -26,13 +26,14 @@ import (
"k8s.io/client-go/tools/record" "k8s.io/client-go/tools/record"
"k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/eviction"
"k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/lifecycle"
"k8s.io/kubernetes/pkg/kubelet/prober"
) )
// Manager is a fake node shutdown manager for non linux platforms. // Manager is a fake node shutdown manager for non linux platforms.
type Manager struct{} type Manager struct{}
// NewManager returns a fake node shutdown manager for non linux platforms. // NewManager returns a fake node shutdown manager for non linux platforms.
func NewManager(recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) { func NewManager(proberManager prober.Manager, recorder record.EventRecorder, nodeRef *v1.ObjectReference, getPodsFunc eviction.ActivePodsFunc, killPodFunc eviction.KillPodFunc, syncNodeStatus func(), shutdownGracePeriodRequested, shutdownGracePeriodCriticalPods time.Duration) (*Manager, lifecycle.PodAdmitHandler) {
m := &Manager{} m := &Manager{}
return m, m return m, m
} }