diff --git a/pkg/kubelet/util/manager/cache_based_manager.go b/pkg/kubelet/util/manager/cache_based_manager.go
index 1929c5519d7..160f9ca0d29 100644
--- a/pkg/kubelet/util/manager/cache_based_manager.go
+++ b/pkg/kubelet/util/manager/cache_based_manager.go
@@ -92,7 +92,7 @@ func isObjectOlder(newObject, oldObject runtime.Object) bool {
 	return newVersion < oldVersion
 }
 
-func (s *objectStore) AddReference(namespace, name string, podUID types.UID) {
+func (s *objectStore) AddReference(namespace, name string, _ types.UID) {
 	key := objectKey{namespace: namespace, name: name}
 
 	// AddReference is called from RegisterPod, thus it needs to be efficient.
@@ -114,7 +114,7 @@ func (s *objectStore) AddReference(namespace, name string, podUID types.UID) {
 	item.data = nil
 }
 
-func (s *objectStore) DeleteReference(namespace, name string, podUID types.UID) {
+func (s *objectStore) DeleteReference(namespace, name string, _ types.UID) {
 	key := objectKey{namespace: namespace, name: name}
 
 	s.lock.Lock()
diff --git a/pkg/kubelet/util/manager/watch_based_manager.go b/pkg/kubelet/util/manager/watch_based_manager.go
index ab0ab1fac0b..c8d7d339026 100644
--- a/pkg/kubelet/util/manager/watch_based_manager.go
+++ b/pkg/kubelet/util/manager/watch_based_manager.go
@@ -45,7 +45,7 @@ type isImmutableFunc func(runtime.Object) bool
 
 // objectCacheItem is a single item stored in objectCache.
 type objectCacheItem struct {
-	refCount  int
+	refMap    map[types.UID]int
 	store     *cacheStore
 	reflector *cache.Reflector
 
@@ -232,7 +232,7 @@ func (c *objectCache) newReflectorLocked(namespace, name string) *objectCacheIte
 		0,
 	)
 	item := &objectCacheItem{
-		refCount:  0,
+		refMap:    make(map[types.UID]int),
 		store:     store,
 		reflector: reflector,
 		hasSynced: func() (bool, error) { return store.hasSynced(), nil },
@@ -261,7 +261,7 @@ func (c *objectCache) AddReference(namespace, name string, podUID types.UID) {
 		item = c.newReflectorLocked(namespace, name)
 		c.items[key] = item
 	}
-	item.refCount++
+	item.refMap[podUID]++
 }
 
 func (c *objectCache) DeleteReference(namespace, name string, podUID types.UID) {
@@ -270,8 +270,11 @@ func (c *objectCache) DeleteReference(namespace, name string, podUID types.UID)
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	if item, ok := c.items[key]; ok {
-		item.refCount--
-		if item.refCount == 0 {
+		item.refMap[podUID]--
+		if item.refMap[podUID] == 0 {
+			delete(item.refMap, podUID)
+		}
+		if len(item.refMap) == 0 {
 			// Stop the underlying reflector.
 			item.stop()
 			delete(c.items, key)