diff --git a/pkg/sqlcache/db/client.go b/pkg/sqlcache/db/client.go index 9178e2a5..549bd922 100644 --- a/pkg/sqlcache/db/client.go +++ b/pkg/sqlcache/db/client.go @@ -458,6 +458,9 @@ func (c *client) NewConnection(useTempDir bool) (string, error) { // if two transactions want to write at the same time, allow 2 minutes for the first to complete // before baling out "_pragma=busy_timeout=120000&"+ + // store temporary tables to memory, to speed up queries making use + // of temporary tables (eg: when using DISTINCT) + "_pragma=temp_store=2&"+ // default to IMMEDIATE mode for transactions. Setting this parameter is the only current way // to be able to switch between DEFERRED and IMMEDIATE modes in modernc.org/sqlite's implementation // of BeginTx diff --git a/pkg/sqlcache/informer/listoption_indexer.go b/pkg/sqlcache/informer/listoption_indexer.go index ddb9f36d..2eb64c01 100644 --- a/pkg/sqlcache/informer/listoption_indexer.go +++ b/pkg/sqlcache/informer/listoption_indexer.go @@ -653,7 +653,11 @@ func (l *ListOptionIndexer) constructQuery(lo *sqltypes.ListOptions, partitions params = withParams joinPartsToUse = joinParts } - query += fmt.Sprintf(`SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "%s" o`, dbName) + query += "SELECT " + if queryUsesLabels { + query += "DISTINCT " + } + query += fmt.Sprintf(`o.object, o.objectnonce, o.dekid FROM "%s" o`, dbName) query += "\n " query += fmt.Sprintf(`JOIN "%s_fields" f ON o.key = f.key`, dbName) if len(joinPartsToUse) > 0 { @@ -845,10 +849,13 @@ func (l *ListOptionIndexer) executeQuery(ctx context.Context, queryInfo *QueryIn var items []any err = l.WithTransaction(ctx, false, func(tx transaction.Client) error { txStmt := tx.Stmt(stmt) + now := time.Now() rows, err := txStmt.QueryContext(ctx, queryInfo.params...) if err != nil { return &db.QueryError{QueryString: queryInfo.query, Err: err} } + elapsed := time.Since(now) + logLongQuery(elapsed, queryInfo.query, queryInfo.params) items, err = l.ReadObjects(rows, l.GetType(), l.GetShouldEncrypt()) if err != nil { return fmt.Errorf("read objects: %w", err) @@ -864,10 +871,13 @@ func (l *ListOptionIndexer) executeQuery(ctx context.Context, queryInfo *QueryIn } }() txStmt := tx.Stmt(countStmt) + now = time.Now() rows, err := txStmt.QueryContext(ctx, queryInfo.countParams...) if err != nil { return &db.QueryError{QueryString: queryInfo.countQuery, Err: err} } + elapsed = time.Since(now) + logLongQuery(elapsed, queryInfo.countQuery, queryInfo.countParams) total, err = l.ReadInt(rows) if err != nil { return fmt.Errorf("error reading query results: %w", err) @@ -894,6 +904,14 @@ func (l *ListOptionIndexer) executeQuery(ctx context.Context, queryInfo *QueryIn return toUnstructuredList(items, latestRV), total, continueToken, nil } +func logLongQuery(elapsed time.Duration, query string, params []any) { + threshold := 500 * time.Millisecond + if elapsed < threshold { + return + } + logrus.Debugf("Query took more than %v (took %v): %s with params %v", threshold, elapsed, query, params) +} + func (l *ListOptionIndexer) validateColumn(column string) error { for _, v := range l.indexedFields { if v == column { diff --git a/pkg/sqlcache/informer/listoption_indexer_test.go b/pkg/sqlcache/informer/listoption_indexer_test.go index f7f7725e..5d7b3bdd 100644 --- a/pkg/sqlcache/informer/listoption_indexer_test.go +++ b/pkg/sqlcache/informer/listoption_indexer_test.go @@ -507,6 +507,62 @@ func TestNewListOptionIndexerEasy(t *testing.T) { expectedContToken: "", expectedErr: nil, }) + tests = append(tests, testCase{ + description: "ListByOptions with single object matching many labels with AND", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "cows"}, + Matches: []string{"milk"}, + Op: sqltypes.Eq, + }, + }, + }, + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "horses"}, + Matches: []string{"shoes"}, + Op: sqltypes.Eq, + }, + }, + }, + }, + }, + partitions: []partition.Partition{{All: true}}, + ns: "", + expectedList: makeList(t, obj02b_milk_shoes), + expectedTotal: 1, + expectedContToken: "", + expectedErr: nil, + }) + tests = append(tests, testCase{ + description: "ListByOptions with many objects matching many labels with OR", + listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ + { + []sqltypes.Filter{ + { + Field: []string{"metadata", "labels", "cows"}, + Matches: []string{"milk"}, + Op: sqltypes.Eq, + }, + { + Field: []string{"metadata", "labels", "horses"}, + Matches: []string{"shoes"}, + Op: sqltypes.Eq, + }, + }, + }, + }, + }, + partitions: []partition.Partition{{All: true}}, + ns: "", + expectedList: makeList(t, obj02_milk_saddles, obj02b_milk_shoes, obj03a_shoes, obj04_milk), + expectedTotal: 4, + expectedContToken: "", + expectedErr: nil, + }) tests = append(tests, testCase{ description: "ListByOptions with 1 OrFilter set with 1 filter should select where that filter is true", listOptions: sqltypes.ListOptions{Filters: []sqltypes.OrFilter{ @@ -835,6 +891,25 @@ func TestNewListOptionIndexerEasy(t *testing.T) { expectedContToken: "", expectedErr: nil, }) + tests = append(tests, testCase{ + description: "ListByOptions sorting on two existing labels, with no label filters, should sort correctly", + listOptions: sqltypes.ListOptions{ + SortList: sqltypes.SortList{ + SortDirectives: []sqltypes.Sort{ + { + Fields: []string{"metadata", "labels", "horses"}, + }, + { + Fields: []string{"metadata", "labels", "cows"}, + }, + }, + }, + }, + partitions: []partition.Partition{{All: true}}, + expectedList: makeList(t, obj02a_beef_saddles, obj02_milk_saddles, obj03_saddles, + obj02b_milk_shoes, obj03a_shoes, obj04_milk, obj01_no_labels, obj05__guard_lodgepole), + expectedTotal: len(allObjects), + }) tests = append(tests, testCase{ description: "ListByOptions with Pagination.PageSize set should set limit to PageSize", listOptions: sqltypes.ListOptions{ @@ -1270,7 +1345,7 @@ func TestConstructQuery(t *testing.T) { }, partitions: []partition.Partition{}, ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o JOIN "something_fields" f ON o.key = f.key WHERE (f."metadata.queryField1" IN (?)) AND @@ -1295,7 +1370,7 @@ func TestConstructQuery(t *testing.T) { }, partitions: []partition.Partition{}, ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o JOIN "something_fields" f ON o.key = f.key WHERE (f."metadata.queryField1" NOT IN (?)) AND @@ -1680,7 +1755,7 @@ func TestConstructQuery(t *testing.T) { }, partitions: []partition.Partition{}, ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o JOIN "something_fields" f ON o.key = f.key WHERE (extractBarredValue(f."spec.containers.image", "3") = ?) AND @@ -1703,7 +1778,7 @@ func TestConstructQuery(t *testing.T) { }, partitions: []partition.Partition{}, ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o JOIN "something_fields" f ON o.key = f.key WHERE (FALSE) @@ -1736,7 +1811,7 @@ func TestConstructQuery(t *testing.T) { }, partitions: []partition.Partition{}, ns: "", - expectedStmt: `SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o + expectedStmt: `SELECT o.object, o.objectnonce, o.dekid FROM "something" o JOIN "something_fields" f ON o.key = f.key WHERE (extractBarredValue(f."spec.containers.image", "3") = ?) AND @@ -1883,7 +1958,7 @@ func TestConstructQuery(t *testing.T) { SELECT key, value FROM "something_labels" WHERE label = ? ) -SELECT DISTINCT o.object, o.objectnonce, o.dekid FROM "something" o +SELECT o.object, o.objectnonce, o.dekid FROM "something" o JOIN "something_fields" f ON o.key = f.key LEFT OUTER JOIN lt1 ON o.key = lt1.key WHERE