Merge pull request #133061 from serathius/apf_list

Account for caching and maximum object size when estimating LIST request

staging/src/k8s.io/apiserver/pkg/util/flowcontrol/request/list_work_estimator.go

@@ -33,6 +33,11 @@ import (
 const (
 	bytesPerSeat                     = 100_000
 	cacheWithStreamingMaxMemoryUsage = 1_000_000
+	// 1.5MB is the recommended client request size in byte
+	// the etcd server should accept. See
+	// https://github.com/etcd-io/etcd/blob/release-3.4/embed/config.go#L56.
+	maxObjectSize       = 1_500_000
+	infiniteObjectCount = 1_000_000_000
 )
 
 func newListWorkEstimator(countFn statsGetterFunc, config *WorkEstimatorConfig, maxSeatsFn maxSeatsFunc) *listWorkEstimator {
@@ -89,21 +94,24 @@ func (e *listWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLe
 		}
 	}
 	// TODO: Check whether watchcache is enabled.
+	var listFromStorage bool
 	result, err := delegator.ShouldDelegateListMeta(&listOptions, delegator.CacheWithoutSnapshots{})
 	if err != nil {
-		return WorkEstimate{InitialSeats: maxSeats}
+		// Assume worse case where we need to reach to etcd.
+		listFromStorage = true
+	} else {
+		listFromStorage = result.ShouldDelegate
 	}
-	listFromStorage := result.ShouldDelegate
 	isListFromCache := requestInfo.Verb == "watch" || !listFromStorage
 
 	stats, err := e.statsGetterFn(key(requestInfo))
 	switch {
 	case err == ObjectCountStaleErr:
 		// object count going stale is indicative of degradation, so we should
-		// be conservative here and allocate maximum seats to this list request.
+		// be conservative here and return maximum object count and size.
 		// NOTE: if a CRD is removed, its count will go stale first and then the
 		// pruner will eventually remove the CRD from the cache.
-		return WorkEstimate{InitialSeats: maxSeats}
+		stats = storage.Stats{ObjectCount: infiniteObjectCount, EstimatedAverageObjectSizeBytes: maxObjectSize}
 	case err == ObjectCountNotFoundErr:
 		// there are multiple scenarios in which we can see this error:
 		//  a. the type is truly unknown, a typo on the caller's part.
@@ -120,9 +128,9 @@ func (e *listWorkEstimator) estimate(r *http.Request, flowSchemaName, priorityLe
 		return WorkEstimate{InitialSeats: minSeats}
 	case err != nil:
 		// we should never be here since Get returns either ObjectCountStaleErr or
-		// ObjectCountNotFoundErr, return maximumSeats to be on the safe side.
+		// ObjectCountNotFoundErr, return maximum object count and size.
 		klog.ErrorS(err, "Unexpected error from object count tracker")
-		return WorkEstimate{InitialSeats: maxSeats}
+		stats = storage.Stats{ObjectCount: infiniteObjectCount, EstimatedAverageObjectSizeBytes: maxObjectSize}
 	}
 
 	var seats uint64
@@ -172,9 +180,8 @@ func (e *listWorkEstimator) seatsBasedOnObjectCount(stats storage.Stats, listOpt
 }
 
 func (e *listWorkEstimator) seatsBasedOnObjectSize(stats storage.Stats, listOptions metav1.ListOptions, isListFromCache bool, matchesSingle bool) uint64 {
-	// Size not available, fallback to count based estimate
 	if stats.EstimatedAverageObjectSizeBytes <= 0 && stats.ObjectCount != 0 {
-		return e.seatsBasedOnObjectCount(stats, listOptions, isListFromCache, matchesSingle)
+		stats.EstimatedAverageObjectSizeBytes = maxObjectSize
 	}
 	limited := stats.ObjectCount
 	if listOptions.Limit > 0 && listOptions.Limit < limited {

staging/src/k8s.io/apiserver/pkg/util/flowcontrol/request/width_test.go

@@ -47,14 +47,14 @@ func TestWorkEstimator(t *testing.T) {
 		additionalLatencyExpected time.Duration
 	}{
 		{
-			name:                 "request has no RequestInfo",
+			name:                 "request has no RequestInfo, expect maxSeats",
 			requestURI:           "http://server/apis/",
 			requestInfo:          nil,
-			maxSeats:             10,
-			initialSeatsExpected: 10,
+			maxSeats:             100,
+			initialSeatsExpected: 100,
 		},
 		{
-			name:       "request verb is not list",
+			name:       "request verb is not list, expect minSeats",
 			requestURI: "http://server/apis/",
 			requestInfo: &apirequest.RequestInfo{
 				Verb: "get",
@@ -63,67 +63,79 @@ func TestWorkEstimator(t *testing.T) {
 			initialSeatsExpected: 1,
 		},
 		{
-			name:       "request verb is list, conversion to ListOptions returns error",
+			name:       "request verb is list, conversion to ListOptions returns error, expect maxSeats",
 			requestURI: "http://server/apis/foo.bar/v1/events?limit=invalid",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             20,
-			initialSeatsExpected: 20,
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 100,
 		},
 		{
-			name:       "request verb is list, has limit and resource version is 1",
+			name:       "request verb is list, resource version 1, limit 399, expect read 4MB read from etcd",
 			requestURI: "http://server/apis/foo.bar/v1/events?limit=399&resourceVersion=1",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
-			initialSeatsExpected: 4,
+			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 40,
 		},
 		{
-			name:       "request verb is list, limit not set",
+			name:       "request verb is list, resource version 1, expect seats capped by cache",
 			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=1",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
+			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 10,
+		},
+		{
+			name:       "request verb is list, resource version 1, expect read 7MB read from cache",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=1",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 69, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
 			initialSeatsExpected: 7,
 		},
 		{
-			name:       "request verb is list, resource version not set",
+			name:       "request verb is list, limit 399, expect 4MB read from etcd",
 			requestURI: "http://server/apis/foo.bar/v1/events?limit=399",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
-			initialSeatsExpected: 4,
+			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 40,
 		},
 		{
-			name:       "request verb is list, no query parameters, count known",
+			name:       "request verb is list, expect read 4MB from etcd",
 			requestURI: "http://server/apis/foo.bar/v1/events",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 399, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
-			initialSeatsExpected: 4,
+			stats:                storage.Stats{ObjectCount: 399, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 40,
 		},
 		{
-			name:       "request verb is list, no query parameters, count not known",
+			name:       "request verb is list, count not known, expect minSeats",
 			requestURI: "http://server/apis/foo.bar/v1/events",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
@@ -131,131 +143,326 @@ func TestWorkEstimator(t *testing.T) {
 				Resource: "events",
 			},
 			statsErr:             ObjectCountNotFoundErr,
-			maxSeats:             10,
+			maxSeats:             100,
 			initialSeatsExpected: 1,
 		},
 		{
-			name:       "request verb is list, continuation is set",
+			name:       "request verb is list, continuation is set, limit 399, expect read 4MB from etcd",
 			requestURI: "http://server/apis/foo.bar/v1/events?continue=token&limit=399",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
-			initialSeatsExpected: 4,
+			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 40,
 		},
 		{
-			name:       "request verb is list, resource version is zero",
+			name:       "request verb is list, resource version is zero, limit 299, expect seats capped by cache",
 			requestURI: "http://server/apis/foo.bar/v1/events?limit=299&resourceVersion=0",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 399, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
-			initialSeatsExpected: 3,
+			stats:                storage.Stats{ObjectCount: 399, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 10,
 		},
 		{
-			name:       "request verb is list, resource version is zero, no limit",
+			name:       "request verb is list, resource version is zero, limit 10, expect read 400KB from cache",
+			requestURI: "http://server/apis/foo.bar/v1/events?limit=20&resourceVersion=0",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 399, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 2,
+		},
+		{
+			name:       "request verb is list, resource version is zero, expect seats capped by cache",
 			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=0",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_000},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 10,
+		},
+		{
+			name:       "request verb is list, resource version is zero, expect read 8MB from cache",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=0",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 79, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
 			initialSeatsExpected: 8,
 		},
 		{
-			name:       "request verb is list, resource version match is Exact",
+			name:       "request verb is list, resource version 1, match is Exact, expect read 4MB from etcd",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=1&resourceVersionMatch=Exact&limit=399",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 40,
+		},
+		{
+			name:       "request verb is list, resource version 1, match is NotOlderThan, expect seats capped by cache",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=1&resourceVersionMatch=NotOlderThan",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 10,
+		},
+		{
+			name:       "request verb is list, resource version 1, match is NotOlderThan, expect read 8 MB from cache",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=1&resourceVersionMatch=NotOlderThan",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 79, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 8,
+		},
+		{
+			name:       "request verb is list, resource version 1, match is Exact, expect seats capped by max",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=1&resourceVersionMatch=Exact",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 5000, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             20,
+			initialSeatsExpected: 20,
+		},
+		{
+			name:       "request verb is list, bad match, expect read 2MB from etcd",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersionMatch=foo",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 200, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 20,
+		},
+		{
+			name:       "request verb is list, bad match, limit 399, expect read 4MB from etcd",
 			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=foo&resourceVersionMatch=Exact&limit=399",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
-			initialSeatsExpected: 4,
+			stats:                storage.Stats{ObjectCount: 699, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 40,
 		},
 		{
-			name:       "request verb is list, resource version match is NotOlderThan, limit not specified",
-			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=foo&resourceVersionMatch=NotOlderThan",
+			name:       "request verb is list, resource version 1, match exact, expect seats capped by max seats",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=1&resourceVersionMatch=Exact",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
-			initialSeatsExpected: 8,
-		},
-		{
-			name:       "request verb is list, capped by watchcache max",
-			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=foo",
-			requestInfo: &apirequest.RequestInfo{
-				Verb:     "list",
-				APIGroup: "foo.bar",
-				Resource: "events",
-			},
-			stats:                storage.Stats{ObjectCount: 5000, EstimatedAverageObjectSizeBytes: 1_000},
+			stats:                storage.Stats{ObjectCount: 5000, EstimatedAverageObjectSizeBytes: 10_000},
 			maxSeats:             20,
+			initialSeatsExpected: 20,
+		},
+		{
+			name:       "request verb is list, resource version 0, count is not found, expect min seats",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=0",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			statsErr:             ObjectCountNotFoundErr,
+			maxSeats:             100,
+			initialSeatsExpected: 1,
+		},
+		{
+			name:       "request verb is list, object count is stale, expect max seats",
+			requestURI: "http://server/apis/foo.bar/v1/events",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 1, EstimatedAverageObjectSizeBytes: 1},
+			statsErr:             ObjectCountStaleErr,
+			maxSeats:             100,
+			initialSeatsExpected: 100,
+		},
+		{
+			name:       "request verb is list, no object size, expect seats capped by cache",
+			requestURI: "http://server/apis/foo.bar/v1/events",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 1},
+			maxSeats:             100,
+			initialSeatsExpected: 15,
+		},
+		{
+			name:       "request verb is list, object count is not found, expect min seats",
+			requestURI: "http://server/apis/foo.bar/v1/events",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			statsErr:             ObjectCountNotFoundErr,
+			maxSeats:             100,
+			initialSeatsExpected: 1,
+		},
+		{
+			name:       "request verb is list, count getter throws unknown error, expect max seats",
+			requestURI: "http://server/apis/foo.bar/v1/events",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			statsErr:             errors.New("unknown error"),
+			maxSeats:             100,
+			initialSeatsExpected: 100,
+		},
+		{
+			name:       "request verb is list, resource version 0, count not known, limit 1, expect min seats",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=0&limit=1",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			statsErr:             ObjectCountNotFoundErr,
+			maxSeats:             100,
+			initialSeatsExpected: 1,
+		},
+		{
+			name:       "request verb is list, object count is stale, limit 1, expect read max object size from etcd",
+			requestURI: "http://server/apis/foo.bar/v1/events?limit=1",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 1, EstimatedAverageObjectSizeBytes: 10_000},
+			statsErr:             ObjectCountStaleErr,
+			maxSeats:             100,
+			initialSeatsExpected: 15,
+		},
+		{
+			name:       "request verb is list, no object size, limit 1, expect read max object size from etcd",
+			requestURI: "http://server/apis/foo.bar/v1/events&limit=1",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 1},
+			maxSeats:             100,
+			initialSeatsExpected: 15,
+		},
+		{
+			name:       "request verb is list, object count is not found, limit 1, expect min seats",
+			requestURI: "http://server/apis/foo.bar/v1/events?limit=1",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			statsErr:             ObjectCountNotFoundErr,
+			maxSeats:             100,
+			initialSeatsExpected: 1,
+		},
+		{
+			name:       "request verb is list, count getter throws unknown error, limit 1, expect read max object size from etcd",
+			requestURI: "http://server/apis/foo.bar/v1/events?limit=1",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			statsErr:             errors.New("unknown error"),
+			maxSeats:             100,
+			initialSeatsExpected: 15,
+		},
+		{
+			name:       "request verb is list, resource version 0, count not known, limit 499, expect min seats",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=0&limit=499",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			statsErr:             ObjectCountNotFoundErr,
+			maxSeats:             100,
+			initialSeatsExpected: 1,
+		},
+		{
+			name:       "request verb is list, object count is stale, limit 499, expect max seats",
+			requestURI: "http://server/apis/foo.bar/v1/events?limit=499",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 1, EstimatedAverageObjectSizeBytes: 1},
+			statsErr:             ObjectCountStaleErr,
+			maxSeats:             100,
+			initialSeatsExpected: 100,
+		},
+		{
+			name:       "request verb is list, no object size, limit 499, expect read max object size from etcd",
+			requestURI: "http://server/apis/foo.bar/v1/events?limit=499",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 1},
+			maxSeats:             100,
+			initialSeatsExpected: 15,
+		},
+		{
+			name:       "request verb is list, no object size, resource version 0, limit 499, expect capped by cache",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=0&limit=499",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 1},
+			maxSeats:             100,
 			initialSeatsExpected: 10,
 		},
 		{
-			name:       "request verb is list, maximum is capped",
-			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=foo&resourceVersionMatch=Exact",
-			requestInfo: &apirequest.RequestInfo{
-				Verb:     "list",
-				APIGroup: "foo.bar",
-				Resource: "events",
-			},
-			stats:                storage.Stats{ObjectCount: 5000, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             20,
-			initialSeatsExpected: 20,
-		},
-		{
-			name:       "request verb is list, maximum is capped, lower max seats",
-			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=foo",
-			requestInfo: &apirequest.RequestInfo{
-				Verb:     "list",
-				APIGroup: "foo.bar",
-				Resource: "events",
-			},
-			stats:                storage.Stats{ObjectCount: 1999, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             5,
-			initialSeatsExpected: 5,
-		},
-		{
-			name:       "request verb is list, list from cache, count not known",
-			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=0&limit=799",
-			requestInfo: &apirequest.RequestInfo{
-				Verb:     "list",
-				APIGroup: "foo.bar",
-				Resource: "events",
-			},
-			statsErr:             ObjectCountNotFoundErr,
-			maxSeats:             10,
-			initialSeatsExpected: 1,
-		},
-		{
-			name:       "request verb is list, object count is stale",
-			requestURI: "http://server/apis/foo.bar/v1/events?limit=499",
-			requestInfo: &apirequest.RequestInfo{
-				Verb:     "list",
-				APIGroup: "foo.bar",
-				Resource: "events",
-			},
-			stats:                storage.Stats{ObjectCount: 1999, EstimatedAverageObjectSizeBytes: 1_000},
-			statsErr:             ObjectCountStaleErr,
-			maxSeats:             20,
-			initialSeatsExpected: 20,
-		},
-		{
-			name:       "request verb is list, object count is not found",
+			name:       "request verb is list, object count is not found, limit 499, expect min seats",
 			requestURI: "http://server/apis/foo.bar/v1/events?limit=499",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
@@ -263,11 +470,11 @@ func TestWorkEstimator(t *testing.T) {
 				Resource: "events",
 			},
 			statsErr:             ObjectCountNotFoundErr,
-			maxSeats:             10,
+			maxSeats:             100,
 			initialSeatsExpected: 1,
 		},
 		{
-			name:       "request verb is list, count getter throws unknown error",
+			name:       "request verb is list, count getter throws unknown error, limit 499, expect max seats",
 			requestURI: "http://server/apis/foo.bar/v1/events?limit=499",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
@@ -275,11 +482,11 @@ func TestWorkEstimator(t *testing.T) {
 				Resource: "events",
 			},
 			statsErr:             errors.New("unknown error"),
-			maxSeats:             20,
-			initialSeatsExpected: 20,
+			maxSeats:             100,
+			initialSeatsExpected: 100,
 		},
 		{
-			name:       "request verb is list, metadata.name specified",
+			name:       "request verb is list, metadata.name specified, expect read 200KB from etcd",
 			requestURI: "http://server/apis/foo.bar/v1/events?fieldSelector=metadata.name%3Dtest",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
@@ -287,12 +494,12 @@ func TestWorkEstimator(t *testing.T) {
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
-			initialSeatsExpected: 1,
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 200_000},
+			maxSeats:             100,
+			initialSeatsExpected: 2,
 		},
 		{
-			name:       "request verb is list, metadata.name specified, 1MB object",
+			name:       "request verb is list, metadata.name specified, expect read 1.5MB from etcd",
 			requestURI: "http://server/apis/foo.bar/v1/events?fieldSelector=metadata.name%3Dtest",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
@@ -300,12 +507,12 @@ func TestWorkEstimator(t *testing.T) {
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_000_000},
-			maxSeats:             20,
-			initialSeatsExpected: 10,
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_500_000},
+			maxSeats:             100,
+			initialSeatsExpected: 15,
 		},
 		{
-			name:       "request verb is list, metadata.name, resourceVersion and limit specified",
+			name:       "request verb is list, metadata.name, resource version 0, limit 500, expect read 200KB from cache",
 			requestURI: "http://server/apis/foo.bar/v1/events?fieldSelector=metadata.name%3Dtest&limit=500&resourceVersion=0",
 			requestInfo: &apirequest.RequestInfo{
 				Verb:     "list",
@@ -313,9 +520,82 @@ func TestWorkEstimator(t *testing.T) {
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_000},
-			maxSeats:             10,
-			initialSeatsExpected: 1,
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 200_000},
+			maxSeats:             100,
+			initialSeatsExpected: 2,
+		},
+		{
+			name:       "request verb is list, metadata.name, resource version 0, limit 500, expect seats capped by cache",
+			requestURI: "http://server/apis/foo.bar/v1/events?fieldSelector=metadata.name%3Dtest&limit=500&resourceVersion=0",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				Name:     "test",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: maxObjectSize},
+			maxSeats:             100,
+			initialSeatsExpected: 10,
+		},
+		{
+			name:       "request verb is list, labelSelector, expect read 8MB from etcd",
+			requestURI: "http://server/apis/foo.bar/v1/events?labelSelector=app%3Dtest",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 80,
+		},
+		{
+			name:       "request verb is list, labelSelector, limit 49, expect read 4MB from etcd by pagination",
+			requestURI: "http://server/apis/foo.bar/v1/events?labelSelector=app%3Dtest&limit=49",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 40,
+		},
+		{
+			name:       "request verb is list, labelSelector, limit 699, expect read 7MB from etcd",
+			requestURI: "http://server/apis/foo.bar/v1/events?labelSelector=app%3Dtest&limit=699",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 70,
+		},
+		{
+			name:       "request verb is list, labelSelector, resource version 0, expect seats capped cache",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=0&labelSelector=app%3Dtest",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 10,
+		},
+		{
+			name:       "request verb is list, labelSelector, resource version 0, limit 299, expect read 300KB from cache",
+			requestURI: "http://server/apis/foo.bar/v1/events?resourceVersion=0&labelSelector=app%3Dtest&limit=29",
+			requestInfo: &apirequest.RequestInfo{
+				Verb:     "list",
+				APIGroup: "foo.bar",
+				Resource: "events",
+			},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
+			initialSeatsExpected: 3,
 		},
 		{
 			name:       "request verb is watch, sendInitialEvents is nil",
@@ -325,7 +605,7 @@ func TestWorkEstimator(t *testing.T) {
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_000},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
 			initialSeatsExpected: 1,
 		},
 		{
@@ -336,7 +616,8 @@ func TestWorkEstimator(t *testing.T) {
 				APIGroup: "foo.bar",
 				Resource: "events",
 			},
-			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_000},
+			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 10_000},
+			maxSeats:             100,
 			initialSeatsExpected: 1,
 		},
 		{
@@ -348,6 +629,7 @@ func TestWorkEstimator(t *testing.T) {
 				Resource: "events",
 			},
 			stats:                storage.Stats{ObjectCount: 799, EstimatedAverageObjectSizeBytes: 1_000},
+			maxSeats:             100,
 			initialSeatsExpected: 8,
 		},
 		{