From 9bcb451d7d3962ae24d45a2395ce16e685327d0a Mon Sep 17 00:00:00 2001 From: Geonju Kim Date: Sat, 16 Jan 2021 00:57:48 +0900 Subject: [PATCH 1/2] kubelet: Delete static pods gracefully Add a new static pod after checking if its mirror pod is pending termination. --- pkg/kubelet/kubelet.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index a930b210d49..66bc2dc4b11 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1499,6 +1499,13 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error { return nil } + // If the pod is a static pod and its mirror pod is still gracefully terminating, + // we do not want to start the new static pod until the old static pod is gracefully terminated. + podFullName := kubecontainer.GetPodFullName(pod) + if kl.podKiller.IsMirrorPodPendingTerminationByPodName(podFullName) { + return fmt.Errorf("pod %q is pending termination", podFullName) + } + // Latency measurements for the main workflow are relative to the // first time the pod was seen by the API server. var firstSeenTime time.Time @@ -1634,7 +1641,6 @@ func (kl *Kubelet) syncPod(o syncPodOptions) error { // Create Mirror Pod for Static Pod if it doesn't already exist if kubetypes.IsStaticPod(pod) { - podFullName := kubecontainer.GetPodFullName(pod) deleted := false if mirrorPod != nil { if mirrorPod.DeletionTimestamp != nil || !kl.podManager.IsMirrorPodOf(mirrorPod, pod) { From 1563fb68e673bd854470e8c280ab0c5a92a062cd Mon Sep 17 00:00:00 2001 From: Geonju Kim Date: Sat, 16 Jan 2021 01:13:17 +0900 Subject: [PATCH 2/2] kubelet: Fix mirrorPodTerminationMap leak --- pkg/kubelet/kubelet.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 66bc2dc4b11..10e934824e8 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1771,6 +1771,9 @@ func (kl *Kubelet) deletePod(pod *v1.Pod) error { } podPair := kubecontainer.PodPair{APIPod: pod, RunningPod: &runningPod} + if _, ok := kl.podManager.GetMirrorPodByPod(pod); ok { + kl.podKiller.MarkMirrorPodPendingTermination(pod) + } kl.podKiller.KillPod(&podPair) // We leave the volume/directory cleanup to the periodic cleanup routine. @@ -2109,9 +2112,6 @@ func (kl *Kubelet) HandlePodRemoves(pods []*v1.Pod) { kl.handleMirrorPod(pod, start) continue } - if _, ok := kl.podManager.GetMirrorPodByPod(pod); ok { - kl.podKiller.MarkMirrorPodPendingTermination(pod) - } // Deletion is allowed to fail because the periodic cleanup routine // will trigger deletion again. if err := kl.deletePod(pod); err != nil {