Remove limit and continue param for SQL cache (#643)

pkg/sqlcache/informer/listoption_indexer.go

@@ -423,29 +423,18 @@ func (l *ListOptionIndexer) constructQuery(lo *sqltypes.ListOptions, partitions
 		}
 	}
 
-	// 4- Pagination: LIMIT clause (from lo.Pagination and/or lo.ChunkSize/lo.Resume)
+	// 4- Pagination: LIMIT clause (from lo.Pagination)
 
 	limitClause := ""
-	// take the smallest limit between lo.Pagination and lo.ChunkSize
 	limit := lo.Pagination.PageSize
-	if limit == 0 || (lo.ChunkSize > 0 && lo.ChunkSize < limit) {
-		limit = lo.ChunkSize
-	}
 	if limit > 0 {
 		limitClause = "\n  LIMIT ?"
 		params = append(params, limit)
 	}
 
-	// OFFSET clause (from lo.Pagination and/or lo.Resume)
+	// OFFSET clause (from lo.Pagination)
 	offsetClause := ""
 	offset := 0
-	if lo.Resume != "" {
-		offsetInt, err := strconv.Atoi(lo.Resume)
-		if err != nil {
-			return queryInfo, err
-		}
-		offset = offsetInt
-	}
 	if lo.Pagination.Page >= 1 {
 		offset += lo.Pagination.PageSize * (lo.Pagination.Page - 1)
 	}

pkg/sqlcache/informer/listoption_indexer_test.go

@@ -307,50 +307,6 @@ func TestListByOptions(t *testing.T) {
 		expectedContToken: "",
 		expectedErr:       nil,
 	})
-	tests = append(tests, testCase{
-		description: "ListByOptions with ChunkSize set should set limit in prepared sql.Stmt",
-		listOptions: sqltypes.ListOptions{ChunkSize: 2},
-		partitions:  []partition.Partition{},
-		ns:          "",
-		expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o
-  JOIN "something_fields" f ON o.key = f.key
-  WHERE
-    (FALSE)
-  ORDER BY f."metadata.name" ASC 
-  LIMIT ?`,
-		expectedStmtArgs: []interface{}{2},
-		expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o
-  JOIN "something_fields" f ON o.key = f.key
-  WHERE
-    (FALSE))`,
-		expectedCountStmtArgs: []any{},
-		returnList:            []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}},
-		expectedList:          &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}},
-		expectedContToken:     "",
-		expectedErr:           nil,
-	})
-	tests = append(tests, testCase{
-		description: "ListByOptions with Resume set should set offset in prepared sql.Stmt",
-		listOptions: sqltypes.ListOptions{Resume: "4"},
-		partitions:  []partition.Partition{},
-		ns:          "",
-		expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o
-  JOIN "something_fields" f ON o.key = f.key
-  WHERE
-    (FALSE)
-  ORDER BY f."metadata.name" ASC 
-  OFFSET ?`,
-		expectedStmtArgs: []interface{}{4},
-		expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o
-  JOIN "something_fields" f ON o.key = f.key
-  WHERE
-    (FALSE))`,
-		expectedCountStmtArgs: []any{},
-		returnList:            []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}},
-		expectedList:          &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}},
-		expectedContToken:     "",
-		expectedErr:           nil,
-	})
 	tests = append(tests, testCase{
 		description: "ListByOptions with 1 OrFilter set with 1 filter should select where that filter is true in prepared sql.Stmt",
 		listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{

pkg/sqlcache/sqltypes/types.go

@@ -25,8 +25,6 @@ const (
 
 // ListOptions represents the query parameters that may be included in a list request.
 type ListOptions struct {
-	ChunkSize  int
-	Resume     string
 	Filters    []OrFilter
 	SortList   SortList
 	Pagination Pagination

pkg/stores/sqlpartition/listprocessor/processor.go

@@ -21,8 +21,6 @@ import (
 
 const (
 	defaultLimit            = 100000
-	continueParam           = "continue"
-	limitParam              = "limit"
 	filterParam             = "filter"
 	sortParam               = "sort"
 	pageSizeParam           = "pagesize"
@@ -84,11 +82,7 @@ func k8sRequirementToOrFilter(requirement queryparser.Requirement) (sqltypes.Fil
 func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (sqltypes.ListOptions, error) {
 	opts := sqltypes.ListOptions{}
 
-	opts.ChunkSize = getLimit(apiOp)
-
 	q := apiOp.Request.URL.Query()
-	cont := q.Get(continueParam)
-	opts.Resume = cont
 
 	filterParams := q[filterParam]
 	filterOpts := []sqltypes.OrFilter{}
@@ -173,18 +167,6 @@ func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (sqltypes.ListOpt
 	return opts, nil
 }
 
-// getLimit extracts the limit parameter from the request or sets a default of 100000.
-// The default limit can be explicitly disabled by setting it to zero or negative.
-// If the default is accepted, clients must be aware that the list may be incomplete, and use the "continue" token to get the next chunk of results.
-func getLimit(apiOp *types.APIRequest) int {
-	limitString := apiOp.Request.URL.Query().Get(limitParam)
-	limit, err := strconv.Atoi(limitString)
-	if err != nil {
-		limit = defaultLimit
-	}
-	return limit
-}
-
 // splitQuery takes a single-string k8s object accessor and returns its separate fields in a slice.
 // "Simple" accessors of the form `metadata.labels.foo` => ["metadata", "labels", "foo"]
 // but accessors with square brackets need to be broken on the brackets, as in

pkg/stores/sqlpartition/listprocessor/processor_test.go

@@ -36,8 +36,7 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
+			Filters: make([]sqltypes.OrFilter, 0),
-			Filters:   make([]sqltypes.OrFilter, 0),
 			Pagination: sqltypes.Pagination{
 				Page: 1,
 			},
@@ -52,7 +51,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -112,7 +110,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -162,7 +159,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -214,7 +210,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -240,7 +235,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -266,7 +260,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -291,7 +284,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -316,7 +308,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -342,7 +333,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -368,7 +358,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -393,7 +382,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -419,7 +407,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -455,7 +442,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -491,7 +477,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -523,7 +508,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -555,7 +539,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -587,7 +570,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -613,7 +595,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -645,7 +626,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -671,7 +651,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -713,7 +692,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -751,7 +729,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			Filters: []sqltypes.OrFilter{
 				{
 					Filters: []sqltypes.Filter{
@@ -784,7 +761,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			SortList: sqltypes.SortList{
 				SortDirectives: []sqltypes.Sort{
 					{
@@ -808,7 +784,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			SortList: sqltypes.SortList{
 				SortDirectives: []sqltypes.Sort{
 					{
@@ -832,7 +807,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			SortList: sqltypes.SortList{
 				SortDirectives: []sqltypes.Sort{
 					{
@@ -860,7 +834,6 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
 			SortList: sqltypes.SortList{
 				SortDirectives: []sqltypes.Sort{
 					{
@@ -890,56 +863,6 @@ func TestParseQuery(t *testing.T) {
 			return nil
 		},
 	})
-	tests = append(tests, testCase{
-		description: "ParseQuery() with no errors returned should returned no errors. If continue params is given, resume" +
-			" should be set with assigned value.",
-		req: &types.APIRequest{
-			Request: &http.Request{
-				URL: &url.URL{RawQuery: "continue=5"},
-			},
-		},
-		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
-			Resume:    "5",
-			Filters:   make([]sqltypes.OrFilter, 0),
-			Pagination: sqltypes.Pagination{
-				Page: 1,
-			},
-		},
-	})
-	tests = append(tests, testCase{
-		description: "ParseQuery() with no errors returned should returned no errors. If continue param is given, resume" +
-			" should be set with assigned value.",
-		req: &types.APIRequest{
-			Request: &http.Request{
-				URL: &url.URL{RawQuery: "continue=5"},
-			},
-		},
-		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
-			Resume:    "5",
-			Filters:   make([]sqltypes.OrFilter, 0),
-			Pagination: sqltypes.Pagination{
-				Page: 1,
-			},
-		},
-	})
-	tests = append(tests, testCase{
-		description: "ParseQuery() with no errors returned should returned no errors. If limit param is given, chunksize" +
-			" should be set with assigned value.",
-		req: &types.APIRequest{
-			Request: &http.Request{
-				URL: &url.URL{RawQuery: "limit=3"},
-			},
-		},
-		expectedLO: sqltypes.ListOptions{
-			ChunkSize: 3,
-			Filters:   make([]sqltypes.OrFilter, 0),
-			Pagination: sqltypes.Pagination{
-				Page: 1,
-			},
-		},
-	})
 	tests = append(tests, testCase{
 		description: "ParseQuery() with no errors returned should returned no errors. If page param is given, page" +
 			" should be set with assigned value.",
@@ -949,8 +872,7 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
+			Filters: make([]sqltypes.OrFilter, 0),
-			Filters:   make([]sqltypes.OrFilter, 0),
 			Pagination: sqltypes.Pagination{
 				Page: 3,
 			},
@@ -965,8 +887,7 @@ func TestParseQuery(t *testing.T) {
 			},
 		},
 		expectedLO: sqltypes.ListOptions{
-			ChunkSize: defaultLimit,
+			Filters: make([]sqltypes.OrFilter, 0),
-			Filters:   make([]sqltypes.OrFilter, 0),
 			Pagination: sqltypes.Pagination{
 				PageSize: 20,
 				Page:     1,