github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-28 14:07:14 +00:00
Code Issues Projects Releases Wiki Activity

Handle eviction of pods in deleted namespace

Browse Source 
If a pod is already marked deleted, and the eviction
api returns an unauthorized response, ignore that
error since the pod is marked for deletion already.

If the pod is not already marked deleted, retry.
This commit is contained in:
Michael Gugino 2020-08-12 16:15:54 -04:00
parent 93f62dad6c
commit 8d2a2ffe01
1 changed files with 26 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
staging/src/k8s.io/kubectl/pkg/drain/drain.go
View File

@ -254,6 +254,7 @@ func (d *Helper) evictPods(pods []corev1.Pod, policyGroupVersion string, getPodF
defer cancel()
for _, pod := range pods {
go func(pod corev1.Pod, returnCh chan error) {
refreshPod := false
for {
switch d.DryRunStrategy {
case cmdutil.DryRunServer:
@ -268,20 +269,40 @@ func (d *Helper) evictPods(pods []corev1.Pod, policyGroupVersion string, getPodF
return
default:
}
err := d.EvictPod(pod, policyGroupVersion)
// Create a temporary pod so we don't mutate the pod in the loop.
activePod := pod
if refreshPod {
freshPod, err := getPodFn(pod.Namespace, pod.Name)
// We ignore errors and let eviction sort it out with
// the original pod.
if err == nil {
activePod = *freshPod
}
refreshPod = false
}
err := d.EvictPod(activePod, policyGroupVersion)
if err == nil {
break
} else if apierrors.IsNotFound(err) {
returnCh <- nil
return
} else if apierrors.IsTooManyRequests(err) {
fmt.Fprintf(d.ErrOut, "error when evicting pods/%q -n %q (will retry after 5s): %v\n", pod.Name, pod.Namespace, err)
fmt.Fprintf(d.ErrOut, "error when evicting pods/%q -n %q (will retry after 5s): %v\n", activePod.Name, activePod.Namespace, err)
time.Sleep(5 * time.Second)
} else if apierrors.IsForbidden(err) {
// an eviction request in a deleting namespace will throw a forbidden error
} else if !activePod.ObjectMeta.DeletionTimestamp.IsZero() && apierrors.IsForbidden(err) && apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) {
// an eviction request in a deleting namespace will throw a forbidden error,
// if the pod is already marked deleted, we can ignore this error, an eviction
// request will never succeed, but we will waitForDelete for this pod.
break
} else if apierrors.IsForbidden(err) && apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) {
// an eviction request in a deleting namespace will throw a forbidden error,
// if the pod is not marked deleted, we retry until it is.
fmt.Fprintf(d.ErrOut, "error when evicting pod %q (will retry after 5s): %v\n", activePod.Name, err)
time.Sleep(5 * time.Second)
} else {
returnCh <- fmt.Errorf("error when evicting pods/%q -n %q: %v", pod.Name, pod.Namespace, err)
returnCh <- fmt.Errorf("error when evicting pods/%q -n %q: %v", activePod.Name, activePod.Namespace, err)
return
}
}