Use limit=-1 to disable default list limit

The default chunk size in the partition was set to 100000. It could be
overridden as a larger or smaller number, but not disabled altogether.
This change adds the ability for users to explicitly opt out of the
limit by specifying a negative number or zero. The default behavior is
the same.

README.md

@@ -50,6 +50,39 @@ generating a kubeconfig for a cluster, or installing an app from a catalog:
 POST /v1/catalog.cattle.io.clusterrepos/rancher-partner-charts?action=install
 ```
 
+#### `limit`
+
+Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`).
+
+Set the maximum number of results to retrieve from Kubernetes. The limit is
+passed on as a parameter to the Kubernetes request. The purpose of setting this
+limit is to prevent a huge response from overwhelming Steve and Rancher. For
+more information about setting limits, review the Kubernetes documentation on
+[retrieving results in
+chunks](https://kubernetes.io/docs/reference/using-api/api-concepts/#retrieving-large-results-sets-in-chunks).
+
+The limit controls the size of the set coming from Kubernetes, and then
+filtering, sorting, and pagination are applied on that set. Because of this, if
+the result set is partial, there is no guarantee that the result returned to
+the client is fully sorted across the entire list, only across the returned
+chunk.
+
+The returned response will include a `continue` token, which indicates that the
+result is partial and must be used in the subsequent request to retrieve the
+next chunk.
+
+The default limit is 100000. To override the default, set `limit=-1`.
+
+#### `continue`
+
+Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`).
+
+Continue retrieving the next chunk of a partial list. The continue token is
+included in the response of a limited list and indicates that the result is
+partial. This token can then be used as a query parameter to retrieve the next
+chunk. All chunks have been retrieved when the continue field in the response
+is empty.
+
 #### `filter`
 
 Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`).

pkg/stores/partition/listprocessor/processor.go

@@ -161,15 +161,12 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions {
 }
 
 // getLimit extracts the limit parameter from the request or sets a default of 100000.
-// Since a default is always set, this implies that clients must always be
+// The default limit can be explicitly disabled by setting it to zero or negative.
-// aware that the list may be incomplete.
+// If the default is accepted, clients must be aware that the list may be incomplete, and use the "continue" token to get the next chunk of results.
 func getLimit(apiOp *types.APIRequest) int {
 	limitString := apiOp.Request.URL.Query().Get(limitParam)
 	limit, err := strconv.Atoi(limitString)
 	if err != nil {
-		limit = 0
-	}
-	if limit <= 0 {
 		limit = defaultLimit
 	}
 	return limit

pkg/stores/partition/parallel.go

@@ -137,7 +137,7 @@ func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, l
 	}()
 
 	for i := indexOrZero(p.Partitions, state.PartitionName); i < len(p.Partitions); i++ {
-		if capacity <= 0 || isDone(ctx) {
+		if (limit > 0 && capacity <= 0) || isDone(ctx) {
 			break
 		}
 
@@ -197,7 +197,7 @@ func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, l
 
 				// Case 1: the capacity has been reached across all goroutines but the list is still only partial,
 				// so save the state so that the next page can be requested later.
-				if len(list.Items) > capacity {
+				if limit > 0 && len(list.Items) > capacity {
 					result <- list.Items[:capacity]
 					// save state to redo this list at this offset
 					p.state = &listState{

pkg/stores/partition/store_test.go

@@ -77,6 +77,7 @@ func TestList(t *testing.T) {
 				newRequest("limit=1", "user1"),
 				newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith")))))), "user1"),
 				newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin")))))), "user1"),
+				newRequest("limit=-1", "user1"),
 			},
 			access: []map[string]string{
 				{
@@ -88,6 +89,9 @@ func TestList(t *testing.T) {
 				{
 					"user1": "roleA",
 				},
+				{
+					"user1": "roleA",
+				},
 			},
 			partitions: map[string][]Partition{
 				"user1": {
@@ -126,6 +130,14 @@ func TestList(t *testing.T) {
 						newApple("crispin").toObj(),
 					},
 				},
+				{
+					Count: 3,
+					Objects: []types.APIObject{
+						newApple("fuji").toObj(),
+						newApple("granny-smith").toObj(),
+						newApple("crispin").toObj(),
+					},
+				},
 			},
 		},
 		{