diff --git a/pkg/proxy/endpointslicecache.go b/pkg/proxy/endpointslicecache.go
index c967f9f5172..5671404fcd3 100644
--- a/pkg/proxy/endpointslicecache.go
+++ b/pkg/proxy/endpointslicecache.go
@@ -149,6 +149,9 @@ func (cache *EndpointSliceCache) checkoutChanges() map[types.NamespacedName]*end
 			}
 
 			delete(esTracker.pending, name)
+			if len(esTracker.applied) == 0 && len(esTracker.pending) == 0 {
+				delete(cache.trackerByServiceMap, serviceNN)
+			}
 		}
 
 		change.current = cache.getEndpointsMap(serviceNN, esTracker.applied)