diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json
index 483ff51cf..0f8dbd728 100644
--- a/Godeps/Godeps.json
+++ b/Godeps/Godeps.json
@@ -348,7 +348,7 @@
 		},
 		{
 			"ImportPath": "k8s.io/api",
-			"Rev": "601adab5827b"
+			"Rev": "1daea2040a90"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery",
diff --git a/go.mod b/go.mod
index c45c2b0aa..5185aa3bb 100644
--- a/go.mod
+++ b/go.mod
@@ -28,7 +28,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
 	google.golang.org/appengine v1.5.0 // indirect
-	k8s.io/api v0.0.0-20191112131242-601adab5827b
+	k8s.io/api v0.0.0-20191113133434-1daea2040a90
 	k8s.io/apimachinery v0.0.0-20191112131242-71e349463c92
 	k8s.io/klog v1.0.0
 	k8s.io/utils v0.0.0-20191030222137-2b95a09bc58d
@@ -38,6 +38,6 @@ require (
 replace (
 	golang.org/x/sys => golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a
 	golang.org/x/tools => golang.org/x/tools v0.0.0-20190821162956-65e3620a7ae7
-	k8s.io/api => k8s.io/api v0.0.0-20191112131242-601adab5827b
+	k8s.io/api => k8s.io/api v0.0.0-20191113133434-1daea2040a90
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20191112131242-71e349463c92
 )
diff --git a/go.sum b/go.sum
index a58e7a19a..31ab2fd48 100644
--- a/go.sum
+++ b/go.sum
@@ -191,7 +191,7 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.0.0-20191112131242-601adab5827b/go.mod h1:DUQOBjXV8wQPg9ckcTr81Ya8TNJUXMCq2Gz6UGfV4+Y=
+k8s.io/api v0.0.0-20191113133434-1daea2040a90/go.mod h1:DUQOBjXV8wQPg9ckcTr81Ya8TNJUXMCq2Gz6UGfV4+Y=
 k8s.io/apimachinery v0.0.0-20191112131242-71e349463c92/go.mod h1:+6CX7hP4aLfX2sb91JYDMIp0VqDSog2kZu0BHe+lP+s=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
diff --git a/tools/cache/thread_safe_store.go b/tools/cache/thread_safe_store.go
index 33e6239a6..e72325147 100644
--- a/tools/cache/thread_safe_store.go
+++ b/tools/cache/thread_safe_store.go
@@ -292,6 +292,13 @@ func (c *threadSafeMap) deleteFromIndices(obj interface{}, key string) {
 			set := index[indexValue]
 			if set != nil {
 				set.Delete(key)
+
+				// If we don't delete the set when zero, indices with high cardinality
+				// short lived resources can cause memory to increase over time from
+				// unused empty sets. See `kubernetes/kubernetes/issues/84959`.
+				if len(set) == 0 {
+					delete(index, indexValue)
+				}
 			}
 		}
 	}
diff --git a/tools/cache/thread_safe_store_test.go b/tools/cache/thread_safe_store_test.go
new file mode 100644
index 000000000..267ebcbba
--- /dev/null
+++ b/tools/cache/thread_safe_store_test.go
@@ -0,0 +1,92 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"testing"
+)
+
+func TestThreadSafeStoreDeleteRemovesEmptySetsFromIndex(t *testing.T) {
+	testIndexer := "testIndexer"
+
+	indexers := Indexers{
+		testIndexer: func(obj interface{}) (strings []string, e error) {
+			indexes := []string{obj.(string)}
+			return indexes, nil
+		},
+	}
+
+	indices := Indices{}
+	store := NewThreadSafeStore(indexers, indices).(*threadSafeMap)
+
+	testKey := "testKey"
+
+	store.Add(testKey, testKey)
+
+	// Assumption check, there should be a set for the `testKey` with one element in the added index
+	set := store.indices[testIndexer][testKey]
+
+	if len(set) != 1 {
+		t.Errorf("Initial assumption of index backing string set having 1 element failed. Actual elements: %d", len(set))
+		return
+	}
+
+	store.Delete(testKey)
+	set, present := store.indices[testIndexer][testKey]
+
+	if present {
+		t.Errorf("Index backing string set not deleted from index. Set length: %d", len(set))
+	}
+}
+
+func TestThreadSafeStoreAddKeepsNonEmptySetPostDeleteFromIndex(t *testing.T) {
+	testIndexer := "testIndexer"
+	testIndex := "testIndex"
+
+	indexers := Indexers{
+		testIndexer: func(obj interface{}) (strings []string, e error) {
+			indexes := []string{testIndex}
+			return indexes, nil
+		},
+	}
+
+	indices := Indices{}
+	store := NewThreadSafeStore(indexers, indices).(*threadSafeMap)
+
+	store.Add("retain", "retain")
+	store.Add("delete", "delete")
+
+	// Assumption check, there should be a set for the `testIndex` with two elements
+	set := store.indices[testIndexer][testIndex]
+
+	if len(set) != 2 {
+		t.Errorf("Initial assumption of index backing string set having 2 elements failed. Actual elements: %d", len(set))
+		return
+	}
+
+	store.Delete("delete")
+	set, present := store.indices[testIndexer][testIndex]
+
+	if !present {
+		t.Errorf("Index backing string set erroneously deleted from index.")
+		return
+	}
+
+	if len(set) != 1 {
+		t.Errorf("Index backing string set has incorrect length, expect 1. Set length: %d", len(set))
+	}
+}