diff --git a/README.md b/README.md index d06e331..8343c5a 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,39 @@ generating a kubeconfig for a cluster, or installing an app from a catalog: POST /v1/catalog.cattle.io.clusterrepos/rancher-partner-charts?action=install ``` +#### `limit` + +Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`). + +Set the maximum number of results to retrieve from Kubernetes. The limit is +passed on as a parameter to the Kubernetes request. The purpose of setting this +limit is to prevent a huge response from overwhelming Steve and Rancher. For +more information about setting limits, review the Kubernetes documentation on +[retrieving results in +chunks](https://kubernetes.io/docs/reference/using-api/api-concepts/#retrieving-large-results-sets-in-chunks). + +The limit controls the size of the set coming from Kubernetes, and then +filtering, sorting, and pagination are applied on that set. Because of this, if +the result set is partial, there is no guarantee that the result returned to +the client is fully sorted across the entire list, only across the returned +chunk. + +The returned response will include a `continue` token, which indicates that the +result is partial and must be used in the subsequent request to retrieve the +next chunk. + +The default limit is 100000. To override the default, set `limit=-1`. + +#### `continue` + +Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`). + +Continue retrieving the next chunk of a partial list. The continue token is +included in the response of a limited list and indicates that the result is +partial. This token can then be used as a query parameter to retrieve the next +chunk. All chunks have been retrieved when the continue field in the response +is empty. + #### `filter` Only applicable to list requests (`/v1/{type}` and `/v1/{type}/{namespace}`). diff --git a/pkg/stores/partition/listprocessor/processor.go b/pkg/stores/partition/listprocessor/processor.go index 2cff396..67cace3 100644 --- a/pkg/stores/partition/listprocessor/processor.go +++ b/pkg/stores/partition/listprocessor/processor.go @@ -161,15 +161,12 @@ func ParseQuery(apiOp *types.APIRequest) *ListOptions { } // getLimit extracts the limit parameter from the request or sets a default of 100000. -// Since a default is always set, this implies that clients must always be -// aware that the list may be incomplete. +// The default limit can be explicitly disabled by setting it to zero or negative. +// If the default is accepted, clients must be aware that the list may be incomplete, and use the "continue" token to get the next chunk of results. func getLimit(apiOp *types.APIRequest) int { limitString := apiOp.Request.URL.Query().Get(limitParam) limit, err := strconv.Atoi(limitString) if err != nil { - limit = 0 - } - if limit <= 0 { limit = defaultLimit } return limit diff --git a/pkg/stores/partition/parallel.go b/pkg/stores/partition/parallel.go index 49f380c..03c8089 100644 --- a/pkg/stores/partition/parallel.go +++ b/pkg/stores/partition/parallel.go @@ -137,7 +137,7 @@ func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, l }() for i := indexOrZero(p.Partitions, state.PartitionName); i < len(p.Partitions); i++ { - if capacity <= 0 || isDone(ctx) { + if (limit > 0 && capacity <= 0) || isDone(ctx) { break } @@ -197,7 +197,7 @@ func (p *ParallelPartitionLister) feeder(ctx context.Context, state listState, l // Case 1: the capacity has been reached across all goroutines but the list is still only partial, // so save the state so that the next page can be requested later. - if len(list.Items) > capacity { + if limit > 0 && len(list.Items) > capacity { result <- list.Items[:capacity] // save state to redo this list at this offset p.state = &listState{ diff --git a/pkg/stores/partition/store_test.go b/pkg/stores/partition/store_test.go index f1e295f..c0b2b46 100644 --- a/pkg/stores/partition/store_test.go +++ b/pkg/stores/partition/store_test.go @@ -77,6 +77,7 @@ func TestList(t *testing.T) { newRequest("limit=1", "user1"), newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("granny-smith")))))), "user1"), newRequest(fmt.Sprintf("limit=1&continue=%s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`{"p":"all","c":"%s","l":1}`, base64.StdEncoding.EncodeToString([]byte("crispin")))))), "user1"), + newRequest("limit=-1", "user1"), }, access: []map[string]string{ { @@ -88,6 +89,9 @@ func TestList(t *testing.T) { { "user1": "roleA", }, + { + "user1": "roleA", + }, }, partitions: map[string][]Partition{ "user1": { @@ -126,6 +130,14 @@ func TestList(t *testing.T) { newApple("crispin").toObj(), }, }, + { + Count: 3, + Objects: []types.APIObject{ + newApple("fuji").toObj(), + newApple("granny-smith").toObj(), + newApple("crispin").toObj(), + }, + }, }, }, {