diff --git a/pkg/controller/BUILD b/pkg/controller/BUILD index c077703f5fc..c1261c2e379 100644 --- a/pkg/controller/BUILD +++ b/pkg/controller/BUILD @@ -20,6 +20,7 @@ go_test( "//staging/src/k8s.io/api/apps/v1:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/pkg/controller/controller_utils.go b/pkg/controller/controller_utils.go index 77b6317ec9f..e2efe3bb2b5 100644 --- a/pkg/controller/controller_utils.go +++ b/pkg/controller/controller_utils.go @@ -602,7 +602,11 @@ func (r RealPodControl) DeletePod(namespace string, podID string, object runtime return fmt.Errorf("object does not have ObjectMeta, %v", err) } klog.V(2).Infof("Controller %v deleting pod %v/%v", accessor.GetName(), namespace, podID) - if err := r.KubeClient.CoreV1().Pods(namespace).Delete(context.TODO(), podID, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + if err := r.KubeClient.CoreV1().Pods(namespace).Delete(context.TODO(), podID, metav1.DeleteOptions{}); err != nil { + if apierrors.IsNotFound(err) { + klog.V(4).Infof("pod %v/%v has already been deleted.", namespace, podID) + return err + } r.Recorder.Eventf(object, v1.EventTypeWarning, FailedDeletePodReason, "Error deleting: %v", err) return fmt.Errorf("unable to delete pods: %v", err) } diff --git a/pkg/controller/controller_utils_test.go b/pkg/controller/controller_utils_test.go index 5ab7908a662..625695f3aaa 100644 --- a/pkg/controller/controller_utils_test.go +++ b/pkg/controller/controller_utils_test.go @@ -31,6 +31,7 @@ import ( apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -325,7 +326,7 @@ func TestDeletePodsAllowsMissing(t *testing.T) { controllerSpec := newReplicationController(1) err := podControl.DeletePod("namespace-name", "podName", controllerSpec) - assert.NoError(t, err, "unexpected error: %v", err) + assert.True(t, apierrors.IsNotFound(err)) } func TestActivePodFiltering(t *testing.T) { diff --git a/pkg/controller/daemon/daemon_controller.go b/pkg/controller/daemon/daemon_controller.go index a4ba5b87726..48a56c073c5 100644 --- a/pkg/controller/daemon/daemon_controller.go +++ b/pkg/controller/daemon/daemon_controller.go @@ -30,6 +30,7 @@ import ( v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -976,10 +977,12 @@ func (dsc *DaemonSetsController) syncNodes(ds *apps.DaemonSet, podsToDelete, nod go func(ix int) { defer deleteWait.Done() if err := dsc.podControl.DeletePod(ds.Namespace, podsToDelete[ix], ds); err != nil { - klog.V(2).Infof("Failed deletion, decrementing expectations for set %q/%q", ds.Namespace, ds.Name) dsc.expectations.DeletionObserved(dsKey) - errCh <- err - utilruntime.HandleError(err) + if !apierrors.IsNotFound(err) { + klog.V(2).Infof("Failed deletion, decremented expectations for set %q/%q", ds.Namespace, ds.Name) + errCh <- err + utilruntime.HandleError(err) + } } }(i) } diff --git a/pkg/controller/job/job_controller.go b/pkg/controller/job/job_controller.go index 4956b8fc2a9..7b5af385d5b 100644 --- a/pkg/controller/job/job_controller.go +++ b/pkg/controller/job/job_controller.go @@ -28,6 +28,7 @@ import ( batch "k8s.io/api/batch/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -43,10 +44,9 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/component-base/metrics/prometheus/ratelimiter" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/controller" "k8s.io/utils/integer" - - "k8s.io/klog/v2" ) const statusUpdateRetries = 3 @@ -618,7 +618,7 @@ func (jm *Controller) deleteJobPods(job *batch.Job, pods []*v1.Pod, errCh chan<- for i := int32(0); i < int32(nbPods); i++ { go func(ix int32) { defer wait.Done() - if err := jm.podControl.DeletePod(job.Namespace, pods[ix].Name, job); err != nil { + if err := jm.podControl.DeletePod(job.Namespace, pods[ix].Name, job); err != nil && !apierrors.IsNotFound(err) { defer utilruntime.HandleError(err) klog.V(2).Infof("Failed to delete %v, job %q/%q deadline exceeded", pods[ix].Name, job.Namespace, job.Name) errCh <- err @@ -715,14 +715,17 @@ func (jm *Controller) manageJob(activePods []*v1.Pod, succeeded int32, job *batc go func(ix int32) { defer wait.Done() if err := jm.podControl.DeletePod(job.Namespace, activePods[ix].Name, job); err != nil { - defer utilruntime.HandleError(err) // Decrement the expected number of deletes because the informer won't observe this deletion - klog.V(2).Infof("Failed to delete %v, decrementing expectations for job %q/%q", activePods[ix].Name, job.Namespace, job.Name) jm.expectations.DeletionObserved(jobKey) - activeLock.Lock() - active++ - activeLock.Unlock() - errCh <- err + if !apierrors.IsNotFound(err) { + klog.V(2).Infof("Failed to delete %v, decremented expectations for job %q/%q", activePods[ix].Name, job.Namespace, job.Name) + activeLock.Lock() + active++ + activeLock.Unlock() + errCh <- err + utilruntime.HandleError(err) + } + } }(i) } diff --git a/pkg/controller/replicaset/replica_set.go b/pkg/controller/replicaset/replica_set.go index 022ba715d84..5d4ba3b399f 100644 --- a/pkg/controller/replicaset/replica_set.go +++ b/pkg/controller/replicaset/replica_set.go @@ -39,6 +39,7 @@ import ( apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" @@ -620,9 +621,11 @@ func (rsc *ReplicaSetController) manageReplicas(filteredPods []*v1.Pod, rs *apps if err := rsc.podControl.DeletePod(rs.Namespace, targetPod.Name, rs); err != nil { // Decrement the expected number of deletes because the informer won't observe this deletion podKey := controller.PodKey(targetPod) - klog.V(2).Infof("Failed to delete %v, decrementing expectations for %v %s/%s", podKey, rsc.Kind, rs.Namespace, rs.Name) rsc.expectations.DeletionObserved(rsKey, podKey) - errCh <- err + if !apierrors.IsNotFound(err) { + klog.V(2).Infof("Failed to delete %v, decremented expectations for %v %s/%s", podKey, rsc.Kind, rs.Namespace, rs.Name) + errCh <- err + } } }(pod) }