diff --git a/tools/cache/listers.go b/tools/cache/listers.go
index ce377329..311ff8c4 100644
--- a/tools/cache/listers.go
+++ b/tools/cache/listers.go
@@ -31,7 +31,14 @@ import (
 type AppendFunc func(interface{})
 
 func ListAll(store Store, selector labels.Selector, appendFn AppendFunc) error {
+	selectAll := selector.Empty()
 	for _, m := range store.List() {
+		if selectAll {
+			// Avoid computing labels of the objects to speed up common flows
+			// of listing all objects.
+			appendFn(m)
+			continue
+		}
 		metadata, err := meta.Accessor(m)
 		if err != nil {
 			return err
@@ -44,8 +51,15 @@ func ListAll(store Store, selector labels.Selector, appendFn AppendFunc) error {
 }
 
 func ListAllByNamespace(indexer Indexer, namespace string, selector labels.Selector, appendFn AppendFunc) error {
+	selectAll := selector.Empty()
 	if namespace == metav1.NamespaceAll {
 		for _, m := range indexer.List() {
+			if selectAll {
+				// Avoid computing labels of the objects to speed up common flows
+				// of listing all objects.
+				appendFn(m)
+				continue
+			}
 			metadata, err := meta.Accessor(m)
 			if err != nil {
 				return err
@@ -74,6 +88,12 @@ func ListAllByNamespace(indexer Indexer, namespace string, selector labels.Selec
 		return nil
 	}
 	for _, m := range items {
+		if selectAll {
+			// Avoid computing labels of the objects to speed up common flows
+			// of listing all objects.
+			appendFn(m)
+			continue
+		}
 		metadata, err := meta.Accessor(m)
 		if err != nil {
 			return err