rancher/steve
1
0
Fork 0
mirror of https://github.com/rancher/steve.git synced 2025-09-08 10:49:25 +00:00
Code Issues Projects Releases Wiki Activity

Hard-wire external associations: 5/7: update A=>B links when instances of A change (#646)

Browse Source 
* Continue rebasing.

* Wrote unit tests for external associations.

* Fix the generated SQL.

Some syntactic sugar (capitalizing the keywords), but use the 'ON' syntax on JOINs.

* We want "management.cattle.io.projects:spec.displayName" not "...spec.clusterName"

* Implement hard-wired external associations:

* The table is in sqlproxy.proxy_store
  - externalGVKDependencies - a map of GVKs to dependencies.
    When the key GVK is updated, it triggers the updates in the database for the dependent GVKs,
    replacing fields as specified in the table.

* This is done in an afterUpsert handler, but it's done after the transaction for the core
  GVK update is finished, because most likely the dependent GVK updates will depend on the
  final database values for the GVK being updated, and if we do it as part of the transaction
  the new values won't be committed to the database.

* When an object is modified/created, check for external deps that need updating.

* Stop emitting errors when joining tables if one of the tables doesn't exist.

* Update unit test syntax for SQL queries.

* And an override check

This ensures we don't overwrite good data when
pulling data from one table to another.

* Drop labels, and use mgmt.cattle.io/spec.displayName

There's no need to hardwire labels in proxy_store:typeSpecificIndexedFields
because all labels are indexed in the shadow labels table.

* Keep clusterName, add displayName for mgmt.cattle.io

* Fix rebase/merge breakage.

* Finish the merge: add the 'selfUpdateInfo' param where it didn't get inserted during merge.

* Patch up rebase failures.

* Now gomock generates named args. I give up.
This commit is contained in:
Eric Promislow
2025-07-03 14:35:09 -07:00
committed by GitHub
parent 61173104af
commit 3be82a28d1
11 changed files with 288 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
pkg/sqlcache/store/store_test.go
View File

@@ -14,7 +14,6 @@ import (
"context"
"database/sql"
"fmt"
"github.com/rancher/steve/pkg/sqlcache/sqltypes"
"reflect"
"regexp"
"strings"
@@ -22,6 +21,7 @@ import (
"github.com/rancher/steve/pkg/sqlcache/db"
"github.com/rancher/steve/pkg/sqlcache/db/transaction"
"github.com/rancher/steve/pkg/sqlcache/sqltypes"
"github.com/stretchr/testify/assert"
"go.uber.org/mock/gomock"
@@ -705,17 +705,102 @@ func WSIgnoringMatcher(expected string) gomock.Matcher {
}
}
func TestAddWithExternalUpdates(t *testing.T) {
func TestAddWithOneUpdate(t *testing.T) {
type testCase struct {
description string
updateExternal bool
updateSelf bool
}
testObject := testStoreObject{Id: "testStoreObject", Val: "a"}
var tests []testCase
tests = append(tests,
testCase{description: "Add external update",
updateExternal: true,
updateSelf: false,
})
tests = append(tests,
testCase{description: "Add self update",
updateExternal: false,
updateSelf: true,
})
t.Parallel()
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
c, txC := SetupMockDB(t)
stmts := NewMockStmt(gomock.NewController(t))
store := SetupStoreWithExternalDependencies(t, c, test.updateExternal, test.updateSelf)
c.EXPECT().Upsert(txC, store.upsertStmt, "testStoreObject", testObject, store.shouldEncrypt).Return(nil)
c.EXPECT().WithTransaction(gomock.Any(), true, gomock.Any()).Return(nil).Do(
func(ctx context.Context, shouldEncrypt bool, f db.WithTransactionFunction) {
err := f(txC)
if err != nil {
t.Fail()
}
}).Times(2)
rawStmt := `SELECT DISTINCT f.key, ex2."spec.displayName" FROM "_v1_Namespace_fields" f
LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON f.key = lt1.key
JOIN "management.cattle.io_v3_Project_fields" ex2 ON lt1.value = ex2."metadata.name"
WHERE lt1.label = ? AND f."spec.displayName" != ex2."spec.displayName"`
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt))
results1 := "field.cattle.io/projectId"
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results1)
c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields1", "moose1"}}, nil)
// Override check:
rawStmt2 := `SELECT f."spec.displayName" FROM "_v1_Namespace_fields" f WHERE f.key = ?`
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt2))
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), gomock.Any())
c.EXPECT().ReadStrings(gomock.Any())
rawStmt2a := `UPDATE "_v1_Namespace_fields" SET "spec.displayName" = ? WHERE key = ?`
c.EXPECT().Prepare(rawStmt2a)
txC.EXPECT().Stmt(gomock.Any()).Return(stmts)
stmts.EXPECT().Exec("moose1", "lego.cattle.io/fields1")
rawStmt3 := `SELECT f.key, ex2."spec.projectName" FROM "_v1_Pods_fields" f
JOIN "provisioner.cattle.io_v3_Cluster_fields" ex2 ON f."field.cattle.io/fixer" = ex2."metadata.name"
WHERE f."spec.projectName" != ex2."spec.projectName"`
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt3))
results2 := []any{"lego.cattle.io/fields2"}
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results2)
c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields2", "moose2"}}, nil)
// Override check:
rawStmt2 = `SELECT f."spec.projectName" FROM "_v1_Pods_fields" f WHERE f.key = ?`
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt2))
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), gomock.Any())
c.EXPECT().ReadStrings(gomock.Any())
rawStmt4 := `UPDATE "_v1_Pods_fields" SET "spec.projectName" = ? WHERE key = ?`
c.EXPECT().Prepare(rawStmt4)
txC.EXPECT().Stmt(gomock.Any()).Return(stmts)
stmts.EXPECT().Exec("moose2", "lego.cattle.io/fields2")
err := store.Add(testObject)
assert.Nil(t, err)
})
}
}
func TestAddWithBothUpdates(t *testing.T) {
type testCase struct {
description string
test func(t *testing.T)
}
testObject := testStoreObject{Id: "testStoreObject", Val: "a"}
var tests []testCase
tests = append(tests, testCase{description: "Add with no DB client errors", test: func(t *testing.T) {
tests = append(tests, testCase{description: "Update both external and self", test: func(t *testing.T) {
c, txC := SetupMockDB(t)
stmts := NewMockStmt(gomock.NewController(t))
store := SetupStoreWithExternalDependencies(t, c)
store := SetupStoreWithExternalDependencies(t, c, true, true)
rawStmt := `SELECT DISTINCT f.key, ex2."spec.displayName" FROM "_v1_Namespace_fields" f
LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON f.key = lt1.key
JOIN "management.cattle.io_v3_Project_fields" ex2 ON lt1.value = ex2."metadata.name"
WHERE lt1.label = ? AND f."spec.displayName" != ex2."spec.displayName"`
rawStmt3 := `SELECT f.key, ex2."spec.projectName" FROM "_v1_Pods_fields" f
JOIN "provisioner.cattle.io_v3_Cluster_fields" ex2 ON f."field.cattle.io/fixer" = ex2."metadata.name"
WHERE f."spec.projectName" != ex2."spec.projectName"`
c.EXPECT().Upsert(txC, store.upsertStmt, "testStoreObject", testObject, store.shouldEncrypt).Return(nil)
c.EXPECT().WithTransaction(gomock.Any(), true, gomock.Any()).Return(nil).Do(
@@ -724,32 +809,47 @@ func TestAddWithExternalUpdates(t *testing.T) {
if err != nil {
t.Fail()
}
}).Times(2)
rawStmt := `SELECT DISTINCT f.key, ex2."spec.displayName" FROM "_v1_Namespace_fields" f
LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON f.key = lt1.key
JOIN "management.cattle.io_v3_Project_fields" ex2 ON lt1.value = ex2."metadata.name"
WHERE lt1.label = ? AND f."spec.displayName" != ex2."spec.displayName"`
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt))
results1 := []any{"field.cattle.io/projectId"}
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results1)
c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields1", "moose1"}}, nil)
rawStmt2 := `UPDATE "_v1_Namespace_fields" SET "spec.displayName" = ? WHERE key = ?`
c.EXPECT().Prepare(rawStmt2)
txC.EXPECT().Stmt(gomock.Any()).Return(stmts)
stmts.EXPECT().Exec("moose1", "lego.cattle.io/fields1")
})
for _ = range 2 {
c.EXPECT().WithTransaction(gomock.Any(), true, gomock.Any()).Return(nil).Do(
func(ctx context.Context, shouldEncrypt bool, f db.WithTransactionFunction) {
err := f(txC)
if err != nil {
t.Fail()
}
})
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt))
results1 := "field.cattle.io/projectId"
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results1)
c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields1", "moose1"}}, nil)
// Override check:
rawStmt2 := `SELECT f."spec.displayName" FROM "_v1_Namespace_fields" f WHERE f.key = ?`
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt2))
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), gomock.Any())
c.EXPECT().ReadStrings(gomock.Any())
rawStmt3 := `SELECT f.key, ex2."spec.projectName" FROM "_v1_Pods_fields" f
JOIN "provisioner.cattle.io_v3_Cluster_fields" ex2 ON f."field.cattle.io/fixer" = ex2."metadata.name"
WHERE f."spec.projectName" != ex2."spec.projectName"`
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt3))
results2 := []any{"field.cattle.io/fixer"}
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results2)
rawStmt2a := `UPDATE "_v1_Namespace_fields" SET "spec.displayName" = ? WHERE key = ?`
c.EXPECT().Prepare(rawStmt2a)
txC.EXPECT().Stmt(gomock.Any()).Return(stmts)
stmts.EXPECT().Exec("moose1", "lego.cattle.io/fields1")
c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields2", "moose2"}}, nil)
rawStmt4 := `UPDATE "_v1_Pods_fields" SET "spec.projectName" = ? WHERE key = ?`
c.EXPECT().Prepare(rawStmt4)
txC.EXPECT().Stmt(gomock.Any()).Return(stmts)
stmts.EXPECT().Exec("moose2", "lego.cattle.io/fields2")
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt3))
results2 := []any{"field.cattle.io/fixer"}
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results2)
c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields2", "moose2"}}, nil)
// Override check:
rawStmt2 = `SELECT f."spec.projectName" FROM "_v1_Pods_fields" f WHERE f.key = ?`
c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt2))
c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), gomock.Any())
c.EXPECT().ReadStrings(gomock.Any())
rawStmt4 := `UPDATE "_v1_Pods_fields" SET "spec.projectName" = ? WHERE key = ?`
c.EXPECT().Prepare(rawStmt4)
txC.EXPECT().Stmt(gomock.Any()).Return(stmts)
stmts.EXPECT().Exec("moose2", "lego.cattle.io/fields2")
// And again for the other object
}
err := store.Add(testObject)
assert.Nil(t, err)
@@ -789,7 +889,7 @@ func SetupMockDB(t *testing.T) (*MockClient, *MockTXClient) {
func SetupStore(t *testing.T, client *MockClient, shouldEncrypt bool) *Store {
name := "testStoreObject"
gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: name}
store, err := NewStore(context.Background(), testStoreObject{}, testStoreKeyFunc, client, shouldEncrypt, gvk, name, nil)
store, err := NewStore(context.Background(), testStoreObject{}, testStoreKeyFunc, client, shouldEncrypt, gvk, name, nil, nil)
if err != nil {
t.Error(err)
}
@@ -800,7 +900,7 @@ func gvkKey(group, version, kind string) string {
return group + "_" + version + "_" + kind
}
func SetupStoreWithExternalDependencies(t *testing.T, client *MockClient) *Store {
func SetupStoreWithExternalDependencies(t *testing.T, client *MockClient, updateExternal bool, updateSelf bool) *Store {
name := "testStoreObject"
gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: name}
namespaceProjectLabelDep := sqltypes.ExternalLabelDependency{
@@ -822,7 +922,15 @@ func SetupStoreWithExternalDependencies(t *testing.T, client *MockClient) *Store
ExternalDependencies: []sqltypes.ExternalDependency{namespaceNonLabelDep},
ExternalLabelDependencies: []sqltypes.ExternalLabelDependency{namespaceProjectLabelDep},
}
store, err := NewStore(context.Background(), testStoreObject{}, testStoreKeyFunc, client, false, gvk, name, &updateInfo)
externalUpdateInfo := &updateInfo
selfUpdateInfo := &updateInfo
if !updateExternal {
externalUpdateInfo = nil
}
if !updateSelf {
selfUpdateInfo = nil
}
store, err := NewStore(context.Background(), testStoreObject{}, testStoreKeyFunc, client, false, gvk, name, externalUpdateInfo, selfUpdateInfo)
if err != nil {
t.Error(err)
}