diff --git a/pkg/kubelet/cm/containermap/container_map.go b/pkg/kubelet/cm/containermap/container_map.go
index c324e028b4f..7dc4689af62 100644
--- a/pkg/kubelet/cm/containermap/container_map.go
+++ b/pkg/kubelet/cm/containermap/container_map.go
@@ -69,3 +69,10 @@ func (cm ContainerMap) GetContainerRef(containerID string) (string, string, erro
 	}
 	return cm[containerID].podUID, cm[containerID].containerName, nil
 }
+
+// Visit invoke visitor function to walks all of the entries in the container map
+func (cm ContainerMap) Visit(visitor func(podUID, containerName, containerID string)) {
+	for k, v := range cm {
+		visitor(v.podUID, v.containerName, k)
+	}
+}
diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go
index a876d413870..dde49b6ec8c 100644
--- a/pkg/kubelet/cm/cpumanager/cpu_manager.go
+++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go
@@ -388,6 +388,16 @@ func (m *manager) removeStaleState() {
 			}
 		}
 	}
+
+	m.containerMap.Visit(func(podUID, containerName, containerID string) {
+		if _, ok := activeContainers[podUID][containerName]; !ok {
+			klog.ErrorS(nil, "RemoveStaleState: removing container", "podUID", podUID, "containerName", containerName)
+			err := m.policyRemoveContainerByRef(podUID, containerName)
+			if err != nil {
+				klog.ErrorS(err, "RemoveStaleState: failed to remove container", "podUID", podUID, "containerName", containerName)
+			}
+		}
+	})
 }
 
 func (m *manager) reconcileState() (success []reconciledContainer, failure []reconciledContainer) {
diff --git a/pkg/kubelet/cm/memorymanager/memory_manager.go b/pkg/kubelet/cm/memorymanager/memory_manager.go
index 906569d92dc..dad537e9a47 100644
--- a/pkg/kubelet/cm/memorymanager/memory_manager.go
+++ b/pkg/kubelet/cm/memorymanager/memory_manager.go
@@ -339,6 +339,13 @@ func (m *manager) removeStaleState() {
 			}
 		}
 	}
+
+	m.containerMap.Visit(func(podUID, containerName, containerID string) {
+		if _, ok := activeContainers[podUID][containerName]; !ok {
+			klog.InfoS("RemoveStaleState removing state", "podUID", podUID, "containerName", containerName)
+			m.policyRemoveContainerByRef(podUID, containerName)
+		}
+	})
 }
 
 func (m *manager) policyRemoveContainerByRef(podUID string, containerName string) {