rancher/steve
1
0
Fork 0
mirror of https://github.com/rancher/steve.git synced 2025-09-03 08:25:13 +00:00
Code Issues Projects Releases Wiki Activity

Remove limit and continue param for SQL cache (#643)

Browse Source
This commit is contained in:
Tom Lebreux
2025-05-27 09:53:10 -04:00
committed by GitHub
parent 1e5018e31a
commit cb0d9d6d54
5 changed files with 5 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
pkg/sqlcache/informer/listoption_indexer.go
View File

@@ -423,29 +423,18 @@ func (l *ListOptionIndexer) constructQuery(lo *sqltypes.ListOptions, partitions
}
}
// 4- Pagination: LIMIT clause (from lo.Pagination and/or lo.ChunkSize/lo.Resume)
// 4- Pagination: LIMIT clause (from lo.Pagination)
limitClause := ""
// take the smallest limit between lo.Pagination and lo.ChunkSize
limit := lo.Pagination.PageSize
if limit == 0 || (lo.ChunkSize > 0 && lo.ChunkSize < limit) {
limit = lo.ChunkSize
}
if limit > 0 {
limitClause = "\n LIMIT ?"
params = append(params, limit)
}
// OFFSET clause (from lo.Pagination and/or lo.Resume)
// OFFSET clause (from lo.Pagination)
offsetClause := ""
offset := 0
if lo.Resume != "" {
offsetInt, err := strconv.Atoi(lo.Resume)
if err != nil {
return queryInfo, err
}
offset = offsetInt
}
if lo.Pagination.Page >= 1 {
offset += lo.Pagination.PageSize * (lo.Pagination.Page - 1)
}

44
pkg/sqlcache/informer/listoption_indexer_test.go
View File

@@ -307,50 +307,6 @@ func TestListByOptions(t *testing.T) {
expectedContToken: "",
expectedErr: nil,
})
tests = append(tests, testCase{
description: "ListByOptions with ChunkSize set should set limit in prepared sql.Stmt",
listOptions: sqltypes.ListOptions{ChunkSize: 2},
partitions: []partition.Partition{},
ns: "",
expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o
JOIN "something_fields" f ON o.key = f.key
WHERE
(FALSE)
ORDER BY f."metadata.name" ASC
LIMIT ?`,
expectedStmtArgs: []interface{}{2},
expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o
JOIN "something_fields" f ON o.key = f.key
WHERE
(FALSE))`,
expectedCountStmtArgs: []any{},
returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}},
expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}},
expectedContToken: "",
expectedErr: nil,
})
tests = append(tests, testCase{
description: "ListByOptions with Resume set should set offset in prepared sql.Stmt",
listOptions: sqltypes.ListOptions{Resume: "4"},
partitions: []partition.Partition{},
ns: "",
expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o
JOIN "something_fields" f ON o.key = f.key
WHERE
(FALSE)
ORDER BY f."metadata.name" ASC
OFFSET ?`,
expectedStmtArgs: []interface{}{4},
expectedCountStmt: `SELECT COUNT(*) FROM (SELECT o.object, o.objectnonce, o.dekid FROM "something" o
JOIN "something_fields" f ON o.key = f.key
WHERE
(FALSE))`,
expectedCountStmtArgs: []any{},
returnList: []any{&unstructured.Unstructured{Object: unstrTestObjectMap}, &unstructured.Unstructured{Object: unstrTestObjectMap}},
expectedList: &unstructured.UnstructuredList{Object: map[string]interface{}{"items": []map[string]interface{}{unstrTestObjectMap, unstrTestObjectMap}}, Items: []unstructured.Unstructured{{Object: unstrTestObjectMap}, {Object: unstrTestObjectMap}}},
expectedContToken: "",
expectedErr: nil,
})
tests = append(tests, testCase{
description: "ListByOptions with 1 OrFilter set with 1 filter should select where that filter is true in prepared sql.Stmt",
listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{

2
pkg/sqlcache/sqltypes/types.go
View File

@@ -25,8 +25,6 @@ const (
// ListOptions represents the query parameters that may be included in a list request.
type ListOptions struct {
ChunkSize int
Resume string
Filters []OrFilter
SortList SortList
Pagination Pagination

18
pkg/stores/sqlpartition/listprocessor/processor.go
View File

@@ -21,8 +21,6 @@ import (
const (
defaultLimit = 100000
continueParam = "continue"
limitParam = "limit"
filterParam = "filter"
sortParam = "sort"
pageSizeParam = "pagesize"
@@ -84,11 +82,7 @@ func k8sRequirementToOrFilter(requirement queryparser.Requirement) (sqltypes.Fil
func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (sqltypes.ListOptions, error) {
opts := sqltypes.ListOptions{}
opts.ChunkSize = getLimit(apiOp)
q := apiOp.Request.URL.Query()
cont := q.Get(continueParam)
opts.Resume = cont
filterParams := q[filterParam]
filterOpts := []sqltypes.OrFilter{}
@@ -173,18 +167,6 @@ func ParseQuery(apiOp *types.APIRequest, namespaceCache Cache) (sqltypes.ListOpt
return opts, nil
}
// getLimit extracts the limit parameter from the request or sets a default of 100000.
// The default limit can be explicitly disabled by setting it to zero or negative.
// If the default is accepted, clients must be aware that the list may be incomplete, and use the "continue" token to get the next chunk of results.
func getLimit(apiOp *types.APIRequest) int {
limitString := apiOp.Request.URL.Query().Get(limitParam)
limit, err := strconv.Atoi(limitString)
if err != nil {
limit = defaultLimit
}
return limit
}
// splitQuery takes a single-string k8s object accessor and returns its separate fields in a slice.
// "Simple" accessors of the form `metadata.labels.foo` => ["metadata", "labels", "foo"]
// but accessors with square brackets need to be broken on the brackets, as in

85
pkg/stores/sqlpartition/listprocessor/processor_test.go
View File

@@ -36,8 +36,7 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: make([]sqltypes.OrFilter, 0),
Filters: make([]sqltypes.OrFilter, 0),
Pagination: sqltypes.Pagination{
Page: 1,
},
@@ -52,7 +51,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -112,7 +110,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -162,7 +159,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -214,7 +210,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -240,7 +235,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -266,7 +260,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -291,7 +284,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -316,7 +308,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -342,7 +333,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -368,7 +358,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -393,7 +382,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -419,7 +407,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -455,7 +442,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -491,7 +477,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -523,7 +508,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -555,7 +539,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -587,7 +570,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -613,7 +595,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -645,7 +626,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -671,7 +651,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -713,7 +692,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -751,7 +729,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: []sqltypes.OrFilter{
{
Filters: []sqltypes.Filter{
@@ -784,7 +761,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
SortList: sqltypes.SortList{
SortDirectives: []sqltypes.Sort{
{
@@ -808,7 +784,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
SortList: sqltypes.SortList{
SortDirectives: []sqltypes.Sort{
{
@@ -832,7 +807,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
SortList: sqltypes.SortList{
SortDirectives: []sqltypes.Sort{
{
@@ -860,7 +834,6 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
SortList: sqltypes.SortList{
SortDirectives: []sqltypes.Sort{
{
@@ -890,56 +863,6 @@ func TestParseQuery(t *testing.T) {
return nil
},
})
tests = append(tests, testCase{
description: "ParseQuery() with no errors returned should returned no errors. If continue params is given, resume" +
" should be set with assigned value.",
req: &types.APIRequest{
Request: &http.Request{
URL: &url.URL{RawQuery: "continue=5"},
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Resume: "5",
Filters: make([]sqltypes.OrFilter, 0),
Pagination: sqltypes.Pagination{
Page: 1,
},
},
})
tests = append(tests, testCase{
description: "ParseQuery() with no errors returned should returned no errors. If continue param is given, resume" +
" should be set with assigned value.",
req: &types.APIRequest{
Request: &http.Request{
URL: &url.URL{RawQuery: "continue=5"},
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Resume: "5",
Filters: make([]sqltypes.OrFilter, 0),
Pagination: sqltypes.Pagination{
Page: 1,
},
},
})
tests = append(tests, testCase{
description: "ParseQuery() with no errors returned should returned no errors. If limit param is given, chunksize" +
" should be set with assigned value.",
req: &types.APIRequest{
Request: &http.Request{
URL: &url.URL{RawQuery: "limit=3"},
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: 3,
Filters: make([]sqltypes.OrFilter, 0),
Pagination: sqltypes.Pagination{
Page: 1,
},
},
})
tests = append(tests, testCase{
description: "ParseQuery() with no errors returned should returned no errors. If page param is given, page" +
" should be set with assigned value.",
@@ -949,8 +872,7 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: make([]sqltypes.OrFilter, 0),
Filters: make([]sqltypes.OrFilter, 0),
Pagination: sqltypes.Pagination{
Page: 3,
},
@@ -965,8 +887,7 @@ func TestParseQuery(t *testing.T) {
},
},
expectedLO: sqltypes.ListOptions{
ChunkSize: defaultLimit,
Filters: make([]sqltypes.OrFilter, 0),
Filters: make([]sqltypes.OrFilter, 0),
Pagination: sqltypes.Pagination{
PageSize: 20,
Page: 1,