Merge pull request #97980 from SergeyKanzhelev/revertSandboxCheckInStatus

Revert "Merge pull request #92817 from kmala/kubelet"

pkg/kubelet/BUILD

@@ -25,7 +25,6 @@ go_library(
         "kubelet_resources.go",
         "kubelet_volumes.go",
         "pod_container_deletor.go",
-        "pod_sandbox_deleter.go",
         "pod_workers.go",
         "reason_cache.go",
         "runonce.go",
@@ -186,7 +185,6 @@ go_test(
         "kubelet_volumes_linux_test.go",
         "kubelet_volumes_test.go",
         "pod_container_deletor_test.go",
-        "pod_sandbox_deleter_test.go",
         "pod_workers_test.go",
         "reason_cache_test.go",
         "runonce_test.go",
@@ -264,14 +262,21 @@ go_test(
         "//staging/src/k8s.io/client-go/util/testing:go_default_library",
         "//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
         "//staging/src/k8s.io/component-base/version:go_default_library",
-        "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library",
         "//staging/src/k8s.io/mount-utils:go_default_library",
         "//vendor/github.com/golang/groupcache/lru:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(

pkg/kubelet/container/runtime.go

@@ -114,8 +114,6 @@ type Runtime interface {
 	GetContainerLogs(ctx context.Context, pod *v1.Pod, containerID ContainerID, logOptions *v1.PodLogOptions, stdout, stderr io.Writer) (err error)
 	// Delete a container. If the container is still running, an error is returned.
 	DeleteContainer(containerID ContainerID) error
-	// DeleteSandbox deletes a sandbox.
-	DeleteSandbox(sandboxID string) error
 	// ImageService provides methods to image-related methods.
 	ImageService
 	// UpdatePodCIDR sends a new podCIDR to the runtime.
@@ -301,6 +299,7 @@ type PodStatus struct {
 	// Status of containers in the pod.
 	ContainerStatuses []*Status
 	// Status of the pod sandbox.
+	// Only for kuberuntime now, other runtime may keep it nil.
 	SandboxStatuses []*runtimeapi.PodSandboxStatus
 }
 
@@ -310,8 +309,6 @@ type Status struct {
 	ID ContainerID
 	// Name of the container.
 	Name string
-	// ID of the sandbox to which this container belongs.
-	PodSandboxID string
 	// Status of the container.
 	State State
 	// Creation time of the container.

pkg/kubelet/container/testing/fake_runtime.go

@@ -364,14 +364,6 @@ func (f *FakeRuntime) DeleteContainer(containerID kubecontainer.ContainerID) err
 	return f.Err
 }
 
-func (f *FakeRuntime) DeleteSandbox(sandboxID string) error {
-	f.Lock()
-	defer f.Unlock()
-
-	f.CalledFunctions = append(f.CalledFunctions, "DeleteSandbox")
-	return f.Err
-}
-
 func (f *FakeRuntime) ImageStats() (*kubecontainer.ImageStats, error) {
 	f.Lock()
 	defer f.Unlock()

pkg/kubelet/container/testing/runtime_mock.go

@@ -147,11 +147,6 @@ func (r *Mock) DeleteContainer(containerID kubecontainer.ContainerID) error {
 	return args.Error(0)
 }
 
-func (r *Mock) DeleteSandbox(sandboxID string) error {
-	args := r.Called(sandboxID)
-	return args.Error(0)
-}
-
 func (r *Mock) ImageStats() (*kubecontainer.ImageStats, error) {
 	args := r.Called()
 	return args.Get(0).(*kubecontainer.ImageStats), args.Error(1)

pkg/kubelet/cri/remote/fake/fake_runtime.go

@@ -112,7 +112,7 @@ func (f *RemoteRuntime) StopPodSandbox(ctx context.Context, req *kubeapi.StopPod
 // This call is idempotent, and must not return an error if the sandbox has
 // already been removed.
 func (f *RemoteRuntime) RemovePodSandbox(ctx context.Context, req *kubeapi.RemovePodSandboxRequest) (*kubeapi.RemovePodSandboxResponse, error) {
-	err := f.RuntimeService.RemovePodSandbox(req.PodSandboxId)
+	err := f.RuntimeService.StopPodSandbox(req.PodSandboxId)
 	if err != nil {
 		return nil, err
 	}

pkg/kubelet/kubelet.go

@@ -690,7 +690,6 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 	}
 	klet.containerGC = containerGC
 	klet.containerDeletor = newPodContainerDeletor(klet.containerRuntime, integer.IntMax(containerGCPolicy.MaxPerPodContainer, minDeadContainerInPod))
-	klet.sandboxDeleter = newPodSandboxDeleter(klet.containerRuntime)
 
 	// setup imageManager
 	imageManager, err := images.NewImageGCManager(klet.containerRuntime, klet.StatsProvider, kubeDeps.Recorder, nodeRef, imageGCPolicy, crOptions.PodSandboxImage)
@@ -1127,9 +1126,6 @@ type Kubelet struct {
 	// trigger deleting containers in a pod
 	containerDeletor *podContainerDeletor
 
-	// trigger deleting sandboxes in a pod
-	sandboxDeleter *podSandboxDeleter
-
 	// config iptables util rules
 	makeIPTablesUtilChains bool
 
@@ -1956,9 +1952,6 @@ func (kl *Kubelet) syncLoopIteration(configCh <-chan kubetypes.PodUpdate, handle
 				klog.V(4).Infof("SyncLoop (PLEG): ignore irrelevant event: %#v", e)
 			}
 		}
-		if e.Type == pleg.ContainerRemoved {
-			kl.deletePodSandbox(e.ID)
-		}
 
 		if e.Type == pleg.ContainerDied {
 			if containerID, ok := e.Data.(string); ok {
@@ -2282,16 +2275,6 @@ func (kl *Kubelet) fastStatusUpdateOnce() {
 	}
 }
 
-func (kl *Kubelet) deletePodSandbox(podID types.UID) {
-	if podStatus, err := kl.podCache.Get(podID); err == nil {
-		toKeep := 1
-		if kl.IsPodDeleted(podID) {
-			toKeep = 0
-		}
-		kl.sandboxDeleter.deleteSandboxesInPod(podStatus, toKeep)
-	}
-}
-
 // isSyncPodWorthy filters out events that are not worthy of pod syncing
 func isSyncPodWorthy(event *pleg.PodLifecycleEvent) bool {
 	// ContainerRemoved doesn't affect pod state

pkg/kubelet/kubelet_pods.go

@@ -966,16 +966,6 @@ func (kl *Kubelet) PodResourcesAreReclaimed(pod *v1.Pod, status v1.PodStatus) bo
 		klog.V(3).Infof("Pod %q is terminated, but some containers have not been cleaned up: %s", format.Pod(pod), statusStr)
 		return false
 	}
-	// pod's sandboxes should be deleted
-	if len(runtimeStatus.SandboxStatuses) > 0 {
-		var sandboxStr string
-		for _, sandbox := range runtimeStatus.SandboxStatuses {
-			sandboxStr += fmt.Sprintf("%+v ", *sandbox)
-		}
-		klog.V(3).Infof("Pod %q is terminated, but some pod sandboxes have not been cleaned up: %s", format.Pod(pod), sandboxStr)
-		return false
-	}
-
 	if kl.podVolumesExist(pod.UID) && !kl.keepTerminatedPodVolumes {
 		// We shouldn't delete pods whose volumes have not been cleaned up if we are not keeping terminated pod volumes
 		klog.V(3).Infof("Pod %q is terminated, but some volumes have not been cleaned up", format.Pod(pod))

pkg/kubelet/kubelet_pods_test.go

@@ -43,7 +43,6 @@ import (
 	// api.Registry.GroupOrDie(v1.GroupName).GroupVersions[0].String() is changed
 	// to "v1"?
 
-	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 	_ "k8s.io/kubernetes/pkg/apis/core/install"
 	"k8s.io/kubernetes/pkg/features"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -2424,73 +2423,6 @@ func TestTruncatePodHostname(t *testing.T) {
 	}
 }
 
-func TestPodResourcesAreReclaimed(t *testing.T) {
-
-	type args struct {
-		pod           *v1.Pod
-		status        v1.PodStatus
-		runtimeStatus kubecontainer.PodStatus
-	}
-	tests := []struct {
-		name string
-		args args
-		want bool
-	}{
-		{
-			"pod with running containers",
-			args{
-				pod: &v1.Pod{},
-				status: v1.PodStatus{
-					ContainerStatuses: []v1.ContainerStatus{
-						runningState("containerA"),
-						runningState("containerB"),
-					},
-				},
-			},
-			false,
-		},
-		{
-			"pod with containers in runtime cache",
-			args{
-				pod:    &v1.Pod{},
-				status: v1.PodStatus{},
-				runtimeStatus: kubecontainer.PodStatus{
-					ContainerStatuses: []*kubecontainer.Status{
-						{},
-					},
-				},
-			},
-			false,
-		},
-		{
-			"pod with sandbox present",
-			args{
-				pod:    &v1.Pod{},
-				status: v1.PodStatus{},
-				runtimeStatus: kubecontainer.PodStatus{
-					SandboxStatuses: []*runtimeapi.PodSandboxStatus{
-						{},
-					},
-				},
-			},
-			false,
-		},
-	}
-
-	testKubelet := newTestKubelet(t, false)
-	defer testKubelet.Cleanup()
-	kl := testKubelet.kubelet
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			testKubelet.fakeRuntime.PodStatus = tt.args.runtimeStatus
-			if got := kl.PodResourcesAreReclaimed(tt.args.pod, tt.args.status); got != tt.want {
-				t.Errorf("PodResourcesAreReclaimed() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func TestGenerateAPIPodStatusHostNetworkPodIPs(t *testing.T) {
 	testcases := []struct {
 		name          string

pkg/kubelet/kuberuntime/kuberuntime_container.go

@@ -474,7 +474,6 @@ func (m *kubeGenericRuntimeManager) getPodContainerStatuses(uid kubetypes.UID, n
 				cStatus.Message += tMessage
 			}
 		}
-		cStatus.PodSandboxID = c.PodSandboxId
 		statuses[i] = cStatus
 	}
 

pkg/kubelet/kuberuntime/kuberuntime_gc.go

@@ -161,13 +161,26 @@ func (cgc *containerGC) removeOldestNSandboxes(sandboxes []sandboxGCInfo, toRemo
 	// Remove from oldest to newest (last to first).
 	for i := len(sandboxes) - 1; i >= numToKeep; i-- {
 		if !sandboxes[i].active {
-			if err := cgc.manager.DeleteSandbox(sandboxes[i].id); err != nil {
-				klog.Errorf("Failed to remove sandbox %q: %v", sandboxes[i].id, err)
-			}
+			cgc.removeSandbox(sandboxes[i].id)
 		}
 	}
 }
 
+// removeSandbox removes the sandbox by sandboxID.
+func (cgc *containerGC) removeSandbox(sandboxID string) {
+	klog.V(4).Infof("Removing sandbox %q", sandboxID)
+	// In normal cases, kubelet should've already called StopPodSandbox before
+	// GC kicks in. To guard against the rare cases where this is not true, try
+	// stopping the sandbox before removing it.
+	if err := cgc.client.StopPodSandbox(sandboxID); err != nil {
+		klog.Errorf("Failed to stop sandbox %q before removing: %v", sandboxID, err)
+		return
+	}
+	if err := cgc.client.RemovePodSandbox(sandboxID); err != nil {
+		klog.Errorf("Failed to remove sandbox %q: %v", sandboxID, err)
+	}
+}
+
 // evictableContainers gets all containers that are evictable. Evictable containers are: not running
 // and created more than MinAge ago.
 func (cgc *containerGC) evictableContainers(minAge time.Duration) (containersByEvictUnit, error) {

pkg/kubelet/kuberuntime/kuberuntime_manager_test.go

@@ -318,7 +318,7 @@ func TestGetPodStatus(t *testing.T) {
 	}
 
 	// Set fake sandbox and faked containers to fakeRuntime.
-	sandbox, _ := makeAndSetFakePod(t, m, fakeRuntime, pod)
+	makeAndSetFakePod(t, m, fakeRuntime, pod)
 
 	podStatus, err := m.GetPodStatus(pod.UID, pod.Name, pod.Namespace)
 	assert.NoError(t, err)
@@ -326,9 +326,6 @@ func TestGetPodStatus(t *testing.T) {
 	assert.Equal(t, pod.Name, podStatus.Name)
 	assert.Equal(t, pod.Namespace, podStatus.Namespace)
 	assert.Equal(t, apitest.FakePodSandboxIPs, podStatus.IPs)
-	for _, containerStatus := range podStatus.ContainerStatuses {
-		assert.Equal(t, sandbox.Id, containerStatus.PodSandboxID)
-	}
 }
 
 func TestGetPods(t *testing.T) {

pkg/kubelet/kuberuntime/kuberuntime_sandbox.go

@@ -313,15 +313,3 @@ func (m *kubeGenericRuntimeManager) GetPortForward(podName, podNamespace string,
 	}
 	return url.Parse(resp.Url)
 }
-
-// DeleteSandbox removes the sandbox by sandboxID.
-func (m *kubeGenericRuntimeManager) DeleteSandbox(sandboxID string) error {
-	klog.V(4).Infof("Removing sandbox %q", sandboxID)
-	// stop sandbox is called as part of kill pod function but the error is ignored. So,
-	// we have to call stop sandbox again to make sure that all the resources like network
-	// are cleaned by runtime.
-	if err := m.runtimeService.StopPodSandbox(sandboxID); err != nil {
-		return err
-	}
-	return m.runtimeService.RemovePodSandbox(sandboxID)
-}

pkg/kubelet/kuberuntime/kuberuntime_sandbox_test.go

@@ -26,7 +26,6 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-	apitest "k8s.io/cri-api/pkg/apis/testing"
 	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
 	"k8s.io/kubernetes/pkg/kubelet/runtimeclass"
 	rctest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing"
@@ -173,40 +172,3 @@ func newSeccompPod(podFieldProfile, containerFieldProfile *v1.SeccompProfile, po
 	}
 	return pod
 }
-
-// TestDeleteSandbox tests removing the sandbox.
-func TestDeleteSandbox(t *testing.T) {
-	fakeRuntime, _, m, err := createTestRuntimeManager()
-	require.NoError(t, err)
-	pod := &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			UID:       "12345678",
-			Name:      "bar",
-			Namespace: "new",
-		},
-		Spec: v1.PodSpec{
-			Containers: []v1.Container{
-				{
-					Name:            "foo",
-					Image:           "busybox",
-					ImagePullPolicy: v1.PullIfNotPresent,
-				},
-			},
-		},
-	}
-
-	sandbox := makeFakePodSandbox(t, m, sandboxTemplate{
-		pod:       pod,
-		createdAt: fakeCreatedAt,
-		state:     runtimeapi.PodSandboxState_SANDBOX_NOTREADY,
-	})
-	fakeRuntime.SetFakeSandboxes([]*apitest.FakePodSandbox{sandbox})
-
-	err = m.DeleteSandbox(sandbox.Id)
-	assert.NoError(t, err)
-	assert.Contains(t, fakeRuntime.Called, "StopPodSandbox")
-	assert.Contains(t, fakeRuntime.Called, "RemovePodSandbox")
-	containers, err := fakeRuntime.ListPodSandbox(&runtimeapi.PodSandboxFilter{Id: sandbox.Id})
-	assert.NoError(t, err)
-	assert.Empty(t, containers)
-}

pkg/kubelet/pod_sandbox_deleter.go

@@ -1,82 +0,0 @@
-/*
-Copyright 2020 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubelet
-
-import (
-	"sort"
-
-	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apimachinery/pkg/util/wait"
-	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-	"k8s.io/klog/v2"
-	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
-)
-
-const (
-	// The number of sandboxes which can be deleted in parallel.
-	sandboxDeletionBufferLimit = 20
-)
-
-type sandboxStatusByCreatedList []*runtimeapi.PodSandboxStatus
-
-type podSandboxDeleter struct {
-	worker chan<- string
-}
-
-func (a sandboxStatusByCreatedList) Len() int      { return len(a) }
-func (a sandboxStatusByCreatedList) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a sandboxStatusByCreatedList) Less(i, j int) bool {
-	return a[i].CreatedAt > a[j].CreatedAt
-}
-
-func newPodSandboxDeleter(runtime kubecontainer.Runtime) *podSandboxDeleter {
-	buffer := make(chan string, sandboxDeletionBufferLimit)
-	go wait.Forever(func() {
-		for id := range buffer {
-			if err := runtime.DeleteSandbox(id); err != nil {
-				klog.Warningf("[pod_sandbox_deleter] DeleteSandbox returned error for (id=%v): %v", id, err)
-			}
-		}
-	}, 0)
-
-	return &podSandboxDeleter{
-		worker: buffer,
-	}
-}
-
-// deleteSandboxesInPod issues sandbox deletion requests for all inactive sandboxes after sorting by creation time
-// and skipping toKeep number of sandboxes
-func (p *podSandboxDeleter) deleteSandboxesInPod(podStatus *kubecontainer.PodStatus, toKeep int) {
-	sandboxIDs := sets.NewString()
-	for _, containerStatus := range podStatus.ContainerStatuses {
-		sandboxIDs.Insert(containerStatus.PodSandboxID)
-	}
-	sandboxStatuses := podStatus.SandboxStatuses
-	if toKeep > 0 {
-		sort.Sort(sandboxStatusByCreatedList(sandboxStatuses))
-	}
-
-	for i := len(sandboxStatuses) - 1; i >= toKeep; i-- {
-		if _, ok := sandboxIDs[sandboxStatuses[i].Id]; !ok && sandboxStatuses[i].State != runtimeapi.PodSandboxState_SANDBOX_READY {
-			select {
-			case p.worker <- sandboxStatuses[i].Id:
-			default:
-				klog.Warningf("Failed to issue the request to remove sandbox %v", sandboxStatuses[i].Id)
-			}
-		}
-	}
-}

pkg/kubelet/pod_sandbox_deleter_test.go

@@ -1,160 +0,0 @@
-/*
-Copyright 2020 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubelet
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"k8s.io/apimachinery/pkg/util/wait"
-	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
-	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
-)
-
-type testPodSandboxDeleter struct {
-	podSandboxDeleter
-	deletedSandoxes []string
-}
-
-func newTestPodSandboxDeleter() (*testPodSandboxDeleter, chan struct{}) {
-	buffer := make(chan string, 5)
-	stopCh := make(chan struct{})
-	testSandboxDeleter := &testPodSandboxDeleter{
-		podSandboxDeleter: podSandboxDeleter{
-			worker: buffer,
-		},
-		deletedSandoxes: []string{},
-	}
-	go wait.Until(func() {
-		for {
-			id, ok := <-buffer
-			if !ok {
-				close(stopCh)
-				break
-			}
-			testSandboxDeleter.deletedSandoxes = append(testSandboxDeleter.deletedSandoxes, id)
-		}
-	}, 0, stopCh)
-
-	return testSandboxDeleter, stopCh
-}
-
-func Test_podSandboxDeleter_deleteSandboxesInPod(t *testing.T) {
-	type args struct {
-		podStatus *kubecontainer.PodStatus
-		toKeep    int
-	}
-	tests := []struct {
-		name string
-		args args
-		want []string
-	}{
-		{
-			name: "ready sandboxes shouldn't be deleted ever",
-			args: args{
-				podStatus: &kubecontainer.PodStatus{
-					SandboxStatuses: []*runtimeapi.PodSandboxStatus{
-						{
-							Id:    "testsandbox",
-							State: runtimeapi.PodSandboxState_SANDBOX_READY,
-						},
-					},
-				},
-				toKeep: 0,
-			},
-			want: []string{},
-		},
-		{
-			name: "all unready sandboxes should be deleted if to keep is 0",
-			args: args{
-				podStatus: &kubecontainer.PodStatus{
-					SandboxStatuses: []*runtimeapi.PodSandboxStatus{
-						{
-							Id:    "testsandbox",
-							State: runtimeapi.PodSandboxState_SANDBOX_READY,
-						},
-						{
-							Id:    "testsandbox1",
-							State: runtimeapi.PodSandboxState_SANDBOX_NOTREADY,
-						},
-						{
-							Id:    "testsandbox2",
-							State: runtimeapi.PodSandboxState_SANDBOX_NOTREADY,
-						},
-					},
-				},
-				toKeep: 0,
-			},
-			want: []string{"testsandbox1", "testsandbox2"},
-		},
-		{
-			name: "sandboxes with containers shouldn't be deleted",
-			args: args{
-				podStatus: &kubecontainer.PodStatus{
-					ContainerStatuses: []*kubecontainer.Status{
-						{
-							PodSandboxID: "testsandbox1",
-						},
-					},
-					SandboxStatuses: []*runtimeapi.PodSandboxStatus{
-						{
-							Id:    "testsandbox1",
-							State: runtimeapi.PodSandboxState_SANDBOX_NOTREADY,
-						},
-						{
-							Id:    "testsandbox2",
-							State: runtimeapi.PodSandboxState_SANDBOX_NOTREADY,
-						},
-					},
-				},
-				toKeep: 0,
-			},
-			want: []string{"testsandbox2"},
-		},
-		{
-			name: "latest unready sandboxes shouldn't be deleted if to keep is 1",
-			args: args{
-				podStatus: &kubecontainer.PodStatus{
-					SandboxStatuses: []*runtimeapi.PodSandboxStatus{
-						{
-							Id:        "testsandbox1",
-							State:     runtimeapi.PodSandboxState_SANDBOX_NOTREADY,
-							CreatedAt: time.Now().Add(time.Second).UnixNano(),
-						},
-						{
-							Id:        "testsandbox2",
-							State:     runtimeapi.PodSandboxState_SANDBOX_NOTREADY,
-							CreatedAt: time.Now().Add(2 * time.Second).UnixNano(),
-						},
-					},
-				},
-				toKeep: 1,
-			},
-			want: []string{"testsandbox1"},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			p, stopCh := newTestPodSandboxDeleter()
-			p.deleteSandboxesInPod(tt.args.podStatus, tt.args.toKeep)
-			close(p.worker)
-			<-stopCh
-			assert.ElementsMatch(t, tt.want, p.deletedSandoxes, tt.name)
-		})
-	}
-}