diff --git a/pkg/sqlcache/informer/factory/informer_factory.go b/pkg/sqlcache/informer/factory/informer_factory.go index dd29f883..0b2e3592 100644 --- a/pkg/sqlcache/informer/factory/informer_factory.go +++ b/pkg/sqlcache/informer/factory/informer_factory.go @@ -48,7 +48,7 @@ type guardedInformer struct { mutex *sync.Mutex } -type newInformer func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespace bool, watchable bool, maxEventsCount int) (*informer.Informer, error) +type newInformer func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespace bool, watchable bool, maxEventsCount int) (*informer.Informer, error) type Cache struct { informer.ByOptionsLister @@ -109,7 +109,7 @@ func NewCacheFactory(opts CacheFactoryOptions) (*CacheFactory, error) { // CacheFor returns an informer for given GVK, using sql store indexed with fields, using the specified client. For virtual fields, they must be added by the transform function // and specified by fields to be used for later fields. -func (f *CacheFactory) CacheFor(ctx context.Context, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool, watchable bool) (Cache, error) { +func (f *CacheFactory) CacheFor(ctx context.Context, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool, watchable bool) (Cache, error) { // First of all block Reset() until we are done f.mutex.RLock() defer f.mutex.RUnlock() @@ -146,7 +146,7 @@ func (f *CacheFactory) CacheFor(ctx context.Context, fields [][]string, external _, encryptResourceAlways := defaultEncryptedResourceTypes[gvk] shouldEncrypt := f.encryptAll || encryptResourceAlways maxEventsCount := f.getMaximumEventsCount(gvk) - i, err := f.newInformer(ctx, client, fields, externalUpdateInfo, transform, gvk, f.dbClient, shouldEncrypt, namespaced, watchable, maxEventsCount) + i, err := f.newInformer(ctx, client, fields, externalUpdateInfo, selfUpdateInfo, transform, gvk, f.dbClient, shouldEncrypt, namespaced, watchable, maxEventsCount) if err != nil { return Cache{}, err } diff --git a/pkg/sqlcache/informer/factory/informer_factory_test.go b/pkg/sqlcache/informer/factory/informer_factory_test.go index 92d30510..875f412b 100644 --- a/pkg/sqlcache/informer/factory/informer_factory_test.go +++ b/pkg/sqlcache/informer/factory/informer_factory_test.go @@ -75,7 +75,7 @@ func TestCacheFor(t *testing.T) { expectedC := Cache{ ByOptionsLister: i, } - testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { + testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { assert.Equal(t, client, dynamicClient) assert.Equal(t, fields, fields) assert.Equal(t, expectedGVK, gvk) @@ -99,12 +99,12 @@ func TestCacheFor(t *testing.T) { }() var c Cache var err error - c, err = f.CacheFor(context.Background(), fields, nil, nil, dynamicClient, expectedGVK, false, true) + c, err = f.CacheFor(context.Background(), fields, nil, nil, nil, dynamicClient, expectedGVK, false, true) assert.Nil(t, err) assert.Equal(t, expectedC, c) // this sleep is critical to the test. It ensure there has been enough time for expected function like Run to be invoked in their go routines. time.Sleep(1 * time.Second) - c2, err := f.CacheFor(context.Background(), fields, nil, nil, dynamicClient, expectedGVK, false, true) + c2, err := f.CacheFor(context.Background(), fields, nil, nil, nil, dynamicClient, expectedGVK, false, true) assert.Nil(t, err) assert.Equal(t, c, c2) }}) @@ -122,7 +122,7 @@ func TestCacheFor(t *testing.T) { // need to set this so Run function is not nil SharedIndexInformer: sii, } - testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { + testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { assert.Equal(t, client, dynamicClient) assert.Equal(t, fields, fields) assert.Equal(t, expectedGVK, gvk) @@ -144,7 +144,7 @@ func TestCacheFor(t *testing.T) { close(f.stopCh) }() var err error - _, err = f.CacheFor(context.Background(), fields, nil, nil, dynamicClient, expectedGVK, false, true) + _, err = f.CacheFor(context.Background(), fields, nil, nil, nil, dynamicClient, expectedGVK, false, true) assert.NotNil(t, err) time.Sleep(2 * time.Second) }}) @@ -166,7 +166,7 @@ func TestCacheFor(t *testing.T) { expectedC := Cache{ ByOptionsLister: i, } - testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { + testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { assert.Equal(t, client, dynamicClient) assert.Equal(t, fields, fields) assert.Equal(t, expectedGVK, gvk) @@ -186,7 +186,7 @@ func TestCacheFor(t *testing.T) { close(f.stopCh) var c Cache var err error - c, err = f.CacheFor(context.Background(), fields, nil, nil, dynamicClient, expectedGVK, false, true) + c, err = f.CacheFor(context.Background(), fields, nil, nil, nil, dynamicClient, expectedGVK, false, true) assert.Nil(t, err) assert.Equal(t, expectedC, c) time.Sleep(1 * time.Second) @@ -207,7 +207,7 @@ func TestCacheFor(t *testing.T) { expectedC := Cache{ ByOptionsLister: i, } - testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { + testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { assert.Equal(t, client, dynamicClient) assert.Equal(t, fields, fields) assert.Equal(t, expectedGVK, gvk) @@ -231,7 +231,7 @@ func TestCacheFor(t *testing.T) { }() var c Cache var err error - c, err = f.CacheFor(context.Background(), fields, nil, nil, dynamicClient, expectedGVK, false, true) + c, err = f.CacheFor(context.Background(), fields, nil, nil, nil, dynamicClient, expectedGVK, false, true) assert.Nil(t, err) assert.Equal(t, expectedC, c) time.Sleep(1 * time.Second) @@ -257,7 +257,7 @@ func TestCacheFor(t *testing.T) { expectedC := Cache{ ByOptionsLister: i, } - testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced, watchable bool, maxEventsCount int) (*informer.Informer, error) { + testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced, watchable bool, maxEventsCount int) (*informer.Informer, error) { assert.Equal(t, client, dynamicClient) assert.Equal(t, fields, fields) assert.Equal(t, expectedGVK, gvk) @@ -281,7 +281,7 @@ func TestCacheFor(t *testing.T) { }() var c Cache var err error - c, err = f.CacheFor(context.Background(), fields, nil, nil, dynamicClient, expectedGVK, false, true) + c, err = f.CacheFor(context.Background(), fields, nil, nil, nil, dynamicClient, expectedGVK, false, true) assert.Nil(t, err) assert.Equal(t, expectedC, c) time.Sleep(1 * time.Second) @@ -306,7 +306,7 @@ func TestCacheFor(t *testing.T) { expectedC := Cache{ ByOptionsLister: i, } - testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced, watchable bool, maxEventsCount int) (*informer.Informer, error) { + testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced, watchable bool, maxEventsCount int) (*informer.Informer, error) { assert.Equal(t, client, dynamicClient) assert.Equal(t, fields, fields) assert.Equal(t, expectedGVK, gvk) @@ -330,7 +330,7 @@ func TestCacheFor(t *testing.T) { }() var c Cache var err error - c, err = f.CacheFor(context.Background(), fields, nil, nil, dynamicClient, expectedGVK, false, true) + c, err = f.CacheFor(context.Background(), fields, nil, nil, nil, dynamicClient, expectedGVK, false, true) assert.Nil(t, err) assert.Equal(t, expectedC, c) time.Sleep(1 * time.Second) @@ -355,7 +355,7 @@ func TestCacheFor(t *testing.T) { expectedC := Cache{ ByOptionsLister: i, } - testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { + testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { // we can't test func == func, so instead we check if the output was as expected input := "someinput" ouput, err := transform(input) @@ -387,7 +387,7 @@ func TestCacheFor(t *testing.T) { }() var c Cache var err error - c, err = f.CacheFor(context.Background(), fields, nil, transformFunc, dynamicClient, expectedGVK, false, true) + c, err = f.CacheFor(context.Background(), fields, nil, nil, transformFunc, dynamicClient, expectedGVK, false, true) assert.Nil(t, err) assert.Equal(t, expectedC, c) time.Sleep(1 * time.Second) @@ -408,7 +408,7 @@ func TestCacheFor(t *testing.T) { expectedC := Cache{ ByOptionsLister: i, } - testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { + testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { assert.Equal(t, client, dynamicClient) assert.Equal(t, fields, fields) assert.Equal(t, expectedGVK, gvk) @@ -434,7 +434,7 @@ func TestCacheFor(t *testing.T) { var c Cache var err error // CacheFor(ctx context.Context, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool, watchable bool) - c, err = f.CacheFor(context.Background(), fields, nil, nil, dynamicClient, expectedGVK, false, true) + c, err = f.CacheFor(context.Background(), fields, nil, nil, nil, dynamicClient, expectedGVK, false, true) assert.Nil(t, err) assert.Equal(t, expectedC, c) time.Sleep(1 * time.Second) @@ -459,7 +459,7 @@ func TestCacheFor(t *testing.T) { expectedC := Cache{ ByOptionsLister: i, } - testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { + testNewInformer := func(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt, namespaced bool, watchable bool, maxEventsCount int) (*informer.Informer, error) { assert.Equal(t, client, dynamicClient) assert.Equal(t, fields, fields) assert.Equal(t, expectedGVK, gvk) @@ -487,7 +487,7 @@ func TestCacheFor(t *testing.T) { }() var c Cache var err error - c, err = f.CacheFor(context.Background(), fields, nil, nil, dynamicClient, expectedGVK, false, true) + c, err = f.CacheFor(context.Background(), fields, nil, nil, nil, dynamicClient, expectedGVK, false, true) assert.Nil(t, err) assert.Equal(t, expectedC, c) time.Sleep(1 * time.Second) diff --git a/pkg/sqlcache/informer/informer.go b/pkg/sqlcache/informer/informer.go index cb46427f..87d953a7 100644 --- a/pkg/sqlcache/informer/informer.go +++ b/pkg/sqlcache/informer/informer.go @@ -55,7 +55,7 @@ var newInformer = cache.NewSharedIndexInformer // NewInformer returns a new SQLite-backed Informer for the type specified by schema in unstructured.Unstructured form // using the specified client -func NewInformer(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespaced bool, watchable bool, maxEventsCount int) (*Informer, error) { +func NewInformer(ctx context.Context, client dynamic.ResourceInterface, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, gvk schema.GroupVersionKind, db db.Client, shouldEncrypt bool, namespaced bool, watchable bool, maxEventsCount int) (*Informer, error) { watchFunc := func(options metav1.ListOptions) (watch.Interface, error) { return client.Watch(ctx, options) } @@ -112,7 +112,7 @@ func NewInformer(ctx context.Context, client dynamic.ResourceInterface, fields [ name := informerNameFromGVK(gvk) - s, err := sqlStore.NewStore(ctx, example, cache.DeletionHandlingMetaNamespaceKeyFunc, db, shouldEncrypt, gvk, name, externalUpdateInfo) + s, err := sqlStore.NewStore(ctx, example, cache.DeletionHandlingMetaNamespaceKeyFunc, db, shouldEncrypt, gvk, name, externalUpdateInfo, selfUpdateInfo) if err != nil { return nil, err } diff --git a/pkg/sqlcache/informer/informer_test.go b/pkg/sqlcache/informer/informer_test.go index d7f5ed30..63be28af 100644 --- a/pkg/sqlcache/informer/informer_test.go +++ b/pkg/sqlcache/informer/informer_test.go @@ -81,7 +81,7 @@ func TestNewInformer(t *testing.T) { } }) - informer, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, gvk, dbClient, false, true, true, 0) + informer, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, nil, gvk, dbClient, false, true, true, 0) assert.Nil(t, err) assert.NotNil(t, informer.ByOptionsLister) assert.NotNil(t, informer.SharedIndexInformer) @@ -105,7 +105,7 @@ func TestNewInformer(t *testing.T) { } }) - _, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, gvk, dbClient, false, true, true, 0) + _, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, nil, gvk, dbClient, false, true, true, 0) assert.NotNil(t, err) }}) tests = append(tests, testCase{description: "NewInformer() with errors returned from NewIndexer(), should return an error", test: func(t *testing.T) { @@ -140,7 +140,7 @@ func TestNewInformer(t *testing.T) { } }) - _, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, gvk, dbClient, false, true, true, 0) + _, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, nil, gvk, dbClient, false, true, true, 0) assert.NotNil(t, err) }}) tests = append(tests, testCase{description: "NewInformer() with errors returned from NewListOptionIndexer(), should return an error", test: func(t *testing.T) { @@ -193,7 +193,7 @@ func TestNewInformer(t *testing.T) { } }) - _, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, gvk, dbClient, false, true, true, 0) + _, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, nil, gvk, dbClient, false, true, true, 0) assert.NotNil(t, err) }}) tests = append(tests, testCase{description: "NewInformer() with transform func", test: func(t *testing.T) { @@ -257,7 +257,7 @@ func TestNewInformer(t *testing.T) { transformFunc := func(input interface{}) (interface{}, error) { return "someoutput", nil } - informer, err := NewInformer(context.Background(), dynamicClient, fields, nil, transformFunc, gvk, dbClient, false, true, true, 0) + informer, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, transformFunc, gvk, dbClient, false, true, true, 0) assert.Nil(t, err) assert.NotNil(t, informer.ByOptionsLister) assert.NotNil(t, informer.SharedIndexInformer) @@ -265,10 +265,10 @@ func TestNewInformer(t *testing.T) { // we can't test func == func, so instead we check if the output was as expected input := "someinput" - ouput, err := mockInformer.transformFunc(input) + output, err := mockInformer.transformFunc(input) assert.Nil(t, err) - outputStr, ok := ouput.(string) - assert.True(t, ok, "ouput from transform was expected to be a string") + outputStr, ok := output.(string) + assert.True(t, ok, "output from transform was expected to be a string") assert.Equal(t, "someoutput", outputStr) newInformer = cache.NewSharedIndexInformer @@ -293,7 +293,7 @@ func TestNewInformer(t *testing.T) { transformFunc := func(input interface{}) (interface{}, error) { return "someoutput", nil } - _, err := NewInformer(context.Background(), dynamicClient, fields, nil, transformFunc, gvk, dbClient, false, true, true, 0) + _, err := NewInformer(context.Background(), dynamicClient, fields, nil, nil, transformFunc, gvk, dbClient, false, true, true, 0) assert.Error(t, err) newInformer = cache.NewSharedIndexInformer }}) diff --git a/pkg/sqlcache/informer/listoption_indexer_test.go b/pkg/sqlcache/informer/listoption_indexer_test.go index a6000b66..95c35027 100644 --- a/pkg/sqlcache/informer/listoption_indexer_test.go +++ b/pkg/sqlcache/informer/listoption_indexer_test.go @@ -50,7 +50,7 @@ func makeListOptionIndexer(ctx context.Context, opts ListOptionIndexerOptions) ( return nil, "", err } - s, err := store.NewStore(ctx, example, cache.DeletionHandlingMetaNamespaceKeyFunc, db, false, gvk, name, nil) + s, err := store.NewStore(ctx, example, cache.DeletionHandlingMetaNamespaceKeyFunc, db, false, gvk, name, nil, nil) if err != nil { return nil, "", err } diff --git a/pkg/sqlcache/integration_test.go b/pkg/sqlcache/integration_test.go index eb9e599c..5bc4fd0b 100644 --- a/pkg/sqlcache/integration_test.go +++ b/pkg/sqlcache/integration_test.go @@ -323,7 +323,7 @@ func (i *IntegrationSuite) createCacheAndFactory(fields [][]string, transformFun Resource: "configmaps", } dynamicResource := dynamicClient.Resource(configMapGVR).Namespace(testNamespace) - cache, err := cacheFactory.CacheFor(context.Background(), fields, nil, transformFunc, dynamicResource, configMapGVK, true, true) + cache, err := cacheFactory.CacheFor(context.Background(), fields, nil, nil, transformFunc, dynamicResource, configMapGVK, true, true) if err != nil { return nil, nil, fmt.Errorf("unable to make cache: %w", err) } diff --git a/pkg/sqlcache/store/store.go b/pkg/sqlcache/store/store.go index 6472c29e..c8e66295 100644 --- a/pkg/sqlcache/store/store.go +++ b/pkg/sqlcache/store/store.go @@ -8,6 +8,7 @@ import ( "database/sql" "fmt" "reflect" + "strings" "github.com/rancher/lasso/pkg/log" "github.com/rancher/steve/pkg/sqlcache/db" @@ -44,6 +45,7 @@ type Store struct { gvk schema.GroupVersionKind name string externalUpdateInfo *sqltypes.ExternalGVKUpdates + selfUpdateInfo *sqltypes.ExternalGVKUpdates typ reflect.Type keyFunc cache.KeyFunc shouldEncrypt bool @@ -72,12 +74,13 @@ type Store struct { var _ cache.Store = (*Store)(nil) // NewStore creates a SQLite-backed cache.Store for objects of the given example type -func NewStore(ctx context.Context, example any, keyFunc cache.KeyFunc, c db.Client, shouldEncrypt bool, gvk schema.GroupVersionKind, name string, externalUpdateInfo *sqltypes.ExternalGVKUpdates) (*Store, error) { +func NewStore(ctx context.Context, example any, keyFunc cache.KeyFunc, c db.Client, shouldEncrypt bool, gvk schema.GroupVersionKind, name string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates) (*Store, error) { s := &Store{ ctx: ctx, name: name, gvk: gvk, externalUpdateInfo: externalUpdateInfo, + selfUpdateInfo: selfUpdateInfo, typ: reflect.TypeOf(example), Client: c, keyFunc: keyFunc, @@ -121,19 +124,37 @@ func NewStore(ctx context.Context, example any, keyFunc cache.KeyFunc, c db.Clie return s, nil } -func (s *Store) checkUpdateExternalInfo(key string) error { - if s.externalUpdateInfo == nil { - return nil - } - return s.WithTransaction(s.ctx, true, func(tx transaction.Client) error { - if err := s.updateExternalInfo(tx, key, s.externalUpdateInfo); err != nil { - // Just report and ignore errors - logrus.Errorf("Error updating external info %v: %s", s.externalUpdateInfo, err) - } - return nil - }) +func isDBError(e error) bool { + return strings.Contains(e.Error(), "SQL logic error: no such table:") } +func (s *Store) checkUpdateExternalInfo(key string) { + for _, updateBlock := range []*sqltypes.ExternalGVKUpdates{s.externalUpdateInfo, s.selfUpdateInfo} { + if updateBlock != nil { + s.WithTransaction(s.ctx, true, func(tx transaction.Client) error { + err := s.updateExternalInfo(tx, key, updateBlock) + if err != nil && !isDBError(err) { + // Just report and ignore errors + logrus.Errorf("Error updating external info %v: %s", s.externalUpdateInfo, err) + } + return nil + }) + } + } +} + +// This function is called in two different conditions: +// Let's say resource B has a field X that we want to copy into resource A +// When a B is upserted, we update any A's that depend on it +// When an A is upserted, we check to see if any B's have that info +// The `key` argument here can belong to either an A or a B, depending on which resource is being updated. +// So it's only used in debug messages. +// The SELECT queries are more generic -- find *all* the instances of A that have a connection to B, +// ignoring any cases where A.X == B.X, as there's no need to update those. +// +// Some code later on in the function verifies that we aren't overwriting a non-empty value +// with the empty string. I assume this is never desired. + func (s *Store) updateExternalInfo(tx transaction.Client, key string, externalUpdateInfo *sqltypes.ExternalGVKUpdates) error { for _, labelDep := range externalUpdateInfo.ExternalLabelDependencies { rawGetStmt := fmt.Sprintf(`SELECT DISTINCT f.key, ex2."%s" FROM "%s_fields" f @@ -151,7 +172,9 @@ func (s *Store) updateExternalInfo(tx transaction.Client, key string, externalUp getStmt := s.Prepare(rawGetStmt) rows, err := s.QueryForRows(s.ctx, getStmt, labelDep.SourceLabelName) if err != nil { - logrus.Infof("Error getting external info for table %s, key %s: %v", labelDep.TargetGVK, key, &db.QueryError{QueryString: rawGetStmt, Err: err}) + if !isDBError(err) { + logrus.Infof("Error getting external info for table %s, key %s: %v", labelDep.TargetGVK, key, &db.QueryError{QueryString: rawGetStmt, Err: err}) + } continue } result, err := s.ReadStrings2(rows) @@ -165,6 +188,10 @@ func (s *Store) updateExternalInfo(tx transaction.Client, key string, externalUp for _, innerResult := range result { sourceKey := innerResult[0] finalTargetValue := innerResult[1] + ignoreUpdate, err := s.overrideCheck(labelDep.TargetFinalFieldName, labelDep.SourceGVK, sourceKey, finalTargetValue) + if ignoreUpdate || err != nil { + continue + } rawStmt := fmt.Sprintf(`UPDATE "%s_fields" SET "%s" = ? WHERE key = ?`, labelDep.SourceGVK, labelDep.TargetFinalFieldName) preparedStmt := s.Prepare(rawStmt) @@ -191,7 +218,9 @@ func (s *Store) updateExternalInfo(tx transaction.Client, key string, externalUp getStmt := s.Prepare(rawGetStmt) rows, err := s.QueryForRows(s.ctx, getStmt, nonLabelDep.SourceFieldName) if err != nil { - logrus.Infof("Error getting external info for table %s, key %s: %v", nonLabelDep.TargetGVK, key, &db.QueryError{QueryString: rawGetStmt, Err: err}) + if !isDBError(err) { + logrus.Infof("Error getting external info for table %s, key %s: %v", nonLabelDep.TargetGVK, key, &db.QueryError{QueryString: rawGetStmt, Err: err}) + } continue } result, err := s.ReadStrings2(rows) @@ -205,6 +234,10 @@ func (s *Store) updateExternalInfo(tx transaction.Client, key string, externalUp for _, innerResult := range result { sourceKey := innerResult[0] finalTargetValue := innerResult[1] + ignoreUpdate, err := s.overrideCheck(nonLabelDep.TargetFinalFieldName, nonLabelDep.SourceGVK, sourceKey, finalTargetValue) + if ignoreUpdate || err != nil { + continue + } rawStmt := fmt.Sprintf(`UPDATE "%s_fields" SET "%s" = ? WHERE key = ?`, nonLabelDep.SourceGVK, nonLabelDep.TargetFinalFieldName) preparedStmt := s.Prepare(rawStmt) @@ -213,12 +246,46 @@ func (s *Store) updateExternalInfo(tx transaction.Client, key string, externalUp logrus.Infof("Error running %s(%s, %s): %s", rawStmt, finalTargetValue, sourceKey, err) continue } + logrus.Debugf("QQQ: non-label-Updated %s[%s].%s to %s", + nonLabelDep.SourceGVK, + sourceKey, + nonLabelDep.TargetFinalFieldName, + finalTargetValue) } } return nil } +// If the new value will change a non-empty current value, return [true, error:nil] +func (s *Store) overrideCheck(finalFieldName, sourceGVK, sourceKey, finalTargetValue string) (bool, error) { + rawGetValueStmt := fmt.Sprintf(`SELECT f."%s" FROM "%s_fields" f WHERE f.key = ?`, + finalFieldName, sourceGVK) + getValueStmt := s.Prepare(rawGetValueStmt) + rows, err := s.QueryForRows(s.ctx, getValueStmt, sourceKey) + if err != nil { + logrus.Debugf("Checking the field, got error %s", err) + return false, err + } + results, err := s.ReadStrings(rows) + if err != nil { + logrus.Infof("Checking the field for table %s, key %s, got error %s", sourceGVK, sourceKey, err) + return false, err + } + if len(results) == 1 { + currentValue := results[0] + if len(currentValue) > 0 && len(finalTargetValue) == 0 { + logrus.Debugf("Don't override %s key %s, field %s=%s with an empty string", + sourceGVK, + sourceKey, + finalFieldName, + currentValue) + return true, nil + } + } + return false, nil +} + // deleteByKey deletes the object associated with key, if it exists in this Store func (s *Store) deleteByKey(key string, obj any) error { return s.WithTransaction(s.ctx, true, func(tx transaction.Client) error { @@ -282,7 +349,8 @@ func (s *Store) Add(obj any) error { log.Errorf("Error in Store.Add for type %v: %v", s.name, err) return err } - return s.checkUpdateExternalInfo(key) + s.checkUpdateExternalInfo(key) + return nil } // Update saves an obj, or updates it if it exists in this Store @@ -309,7 +377,8 @@ func (s *Store) Update(obj any) error { log.Errorf("Error in Store.Update for type %v: %v", s.name, err) return err } - return s.checkUpdateExternalInfo(key) + s.checkUpdateExternalInfo(key) + return nil } // Delete deletes the given object, if it exists in this Store diff --git a/pkg/sqlcache/store/store_test.go b/pkg/sqlcache/store/store_test.go index 97c96474..18cb3557 100644 --- a/pkg/sqlcache/store/store_test.go +++ b/pkg/sqlcache/store/store_test.go @@ -14,7 +14,6 @@ import ( "context" "database/sql" "fmt" - "github.com/rancher/steve/pkg/sqlcache/sqltypes" "reflect" "regexp" "strings" @@ -22,6 +21,7 @@ import ( "github.com/rancher/steve/pkg/sqlcache/db" "github.com/rancher/steve/pkg/sqlcache/db/transaction" + "github.com/rancher/steve/pkg/sqlcache/sqltypes" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" @@ -705,17 +705,102 @@ func WSIgnoringMatcher(expected string) gomock.Matcher { } } -func TestAddWithExternalUpdates(t *testing.T) { +func TestAddWithOneUpdate(t *testing.T) { + type testCase struct { + description string + updateExternal bool + updateSelf bool + } + testObject := testStoreObject{Id: "testStoreObject", Val: "a"} + var tests []testCase + tests = append(tests, + testCase{description: "Add external update", + updateExternal: true, + updateSelf: false, + }) + tests = append(tests, + testCase{description: "Add self update", + updateExternal: false, + updateSelf: true, + }) + t.Parallel() + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + c, txC := SetupMockDB(t) + stmts := NewMockStmt(gomock.NewController(t)) + store := SetupStoreWithExternalDependencies(t, c, test.updateExternal, test.updateSelf) + + c.EXPECT().Upsert(txC, store.upsertStmt, "testStoreObject", testObject, store.shouldEncrypt).Return(nil) + c.EXPECT().WithTransaction(gomock.Any(), true, gomock.Any()).Return(nil).Do( + func(ctx context.Context, shouldEncrypt bool, f db.WithTransactionFunction) { + err := f(txC) + if err != nil { + t.Fail() + } + }).Times(2) + rawStmt := `SELECT DISTINCT f.key, ex2."spec.displayName" FROM "_v1_Namespace_fields" f + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON f.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ex2 ON lt1.value = ex2."metadata.name" + WHERE lt1.label = ? AND f."spec.displayName" != ex2."spec.displayName"` + c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt)) + results1 := "field.cattle.io/projectId" + c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results1) + c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields1", "moose1"}}, nil) + // Override check: + rawStmt2 := `SELECT f."spec.displayName" FROM "_v1_Namespace_fields" f WHERE f.key = ?` + c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt2)) + c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), gomock.Any()) + c.EXPECT().ReadStrings(gomock.Any()) + + rawStmt2a := `UPDATE "_v1_Namespace_fields" SET "spec.displayName" = ? WHERE key = ?` + c.EXPECT().Prepare(rawStmt2a) + txC.EXPECT().Stmt(gomock.Any()).Return(stmts) + stmts.EXPECT().Exec("moose1", "lego.cattle.io/fields1") + + rawStmt3 := `SELECT f.key, ex2."spec.projectName" FROM "_v1_Pods_fields" f + JOIN "provisioner.cattle.io_v3_Cluster_fields" ex2 ON f."field.cattle.io/fixer" = ex2."metadata.name" + WHERE f."spec.projectName" != ex2."spec.projectName"` + c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt3)) + results2 := []any{"lego.cattle.io/fields2"} + c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results2) + + c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields2", "moose2"}}, nil) + // Override check: + rawStmt2 = `SELECT f."spec.projectName" FROM "_v1_Pods_fields" f WHERE f.key = ?` + c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt2)) + c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), gomock.Any()) + c.EXPECT().ReadStrings(gomock.Any()) + + rawStmt4 := `UPDATE "_v1_Pods_fields" SET "spec.projectName" = ? WHERE key = ?` + c.EXPECT().Prepare(rawStmt4) + txC.EXPECT().Stmt(gomock.Any()).Return(stmts) + stmts.EXPECT().Exec("moose2", "lego.cattle.io/fields2") + + err := store.Add(testObject) + assert.Nil(t, err) + }) + } +} + +func TestAddWithBothUpdates(t *testing.T) { type testCase struct { description string test func(t *testing.T) } testObject := testStoreObject{Id: "testStoreObject", Val: "a"} var tests []testCase - tests = append(tests, testCase{description: "Add with no DB client errors", test: func(t *testing.T) { + tests = append(tests, testCase{description: "Update both external and self", test: func(t *testing.T) { c, txC := SetupMockDB(t) stmts := NewMockStmt(gomock.NewController(t)) - store := SetupStoreWithExternalDependencies(t, c) + store := SetupStoreWithExternalDependencies(t, c, true, true) + + rawStmt := `SELECT DISTINCT f.key, ex2."spec.displayName" FROM "_v1_Namespace_fields" f + LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON f.key = lt1.key + JOIN "management.cattle.io_v3_Project_fields" ex2 ON lt1.value = ex2."metadata.name" + WHERE lt1.label = ? AND f."spec.displayName" != ex2."spec.displayName"` + rawStmt3 := `SELECT f.key, ex2."spec.projectName" FROM "_v1_Pods_fields" f + JOIN "provisioner.cattle.io_v3_Cluster_fields" ex2 ON f."field.cattle.io/fixer" = ex2."metadata.name" + WHERE f."spec.projectName" != ex2."spec.projectName"` c.EXPECT().Upsert(txC, store.upsertStmt, "testStoreObject", testObject, store.shouldEncrypt).Return(nil) c.EXPECT().WithTransaction(gomock.Any(), true, gomock.Any()).Return(nil).Do( @@ -724,32 +809,47 @@ func TestAddWithExternalUpdates(t *testing.T) { if err != nil { t.Fail() } - }).Times(2) - rawStmt := `SELECT DISTINCT f.key, ex2."spec.displayName" FROM "_v1_Namespace_fields" f - LEFT OUTER JOIN "_v1_Namespace_labels" lt1 ON f.key = lt1.key - JOIN "management.cattle.io_v3_Project_fields" ex2 ON lt1.value = ex2."metadata.name" - WHERE lt1.label = ? AND f."spec.displayName" != ex2."spec.displayName"` - c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt)) - results1 := []any{"field.cattle.io/projectId"} - c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results1) - c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields1", "moose1"}}, nil) - rawStmt2 := `UPDATE "_v1_Namespace_fields" SET "spec.displayName" = ? WHERE key = ?` - c.EXPECT().Prepare(rawStmt2) - txC.EXPECT().Stmt(gomock.Any()).Return(stmts) - stmts.EXPECT().Exec("moose1", "lego.cattle.io/fields1") + }) + for _ = range 2 { + c.EXPECT().WithTransaction(gomock.Any(), true, gomock.Any()).Return(nil).Do( + func(ctx context.Context, shouldEncrypt bool, f db.WithTransactionFunction) { + err := f(txC) + if err != nil { + t.Fail() + } + }) + c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt)) + results1 := "field.cattle.io/projectId" + c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results1) + c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields1", "moose1"}}, nil) + // Override check: + rawStmt2 := `SELECT f."spec.displayName" FROM "_v1_Namespace_fields" f WHERE f.key = ?` + c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt2)) + c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), gomock.Any()) + c.EXPECT().ReadStrings(gomock.Any()) - rawStmt3 := `SELECT f.key, ex2."spec.projectName" FROM "_v1_Pods_fields" f - JOIN "provisioner.cattle.io_v3_Cluster_fields" ex2 ON f."field.cattle.io/fixer" = ex2."metadata.name" - WHERE f."spec.projectName" != ex2."spec.projectName"` - c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt3)) - results2 := []any{"field.cattle.io/fixer"} - c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results2) + rawStmt2a := `UPDATE "_v1_Namespace_fields" SET "spec.displayName" = ? WHERE key = ?` + c.EXPECT().Prepare(rawStmt2a) + txC.EXPECT().Stmt(gomock.Any()).Return(stmts) + stmts.EXPECT().Exec("moose1", "lego.cattle.io/fields1") - c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields2", "moose2"}}, nil) - rawStmt4 := `UPDATE "_v1_Pods_fields" SET "spec.projectName" = ? WHERE key = ?` - c.EXPECT().Prepare(rawStmt4) - txC.EXPECT().Stmt(gomock.Any()).Return(stmts) - stmts.EXPECT().Exec("moose2", "lego.cattle.io/fields2") + c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt3)) + results2 := []any{"field.cattle.io/fixer"} + c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), results2) + + c.EXPECT().ReadStrings2(gomock.Any()).Return([][]string{{"lego.cattle.io/fields2", "moose2"}}, nil) + // Override check: + rawStmt2 = `SELECT f."spec.projectName" FROM "_v1_Pods_fields" f WHERE f.key = ?` + c.EXPECT().Prepare(WSIgnoringMatcher(rawStmt2)) + c.EXPECT().QueryForRows(gomock.Any(), gomock.Any(), gomock.Any()) + c.EXPECT().ReadStrings(gomock.Any()) + + rawStmt4 := `UPDATE "_v1_Pods_fields" SET "spec.projectName" = ? WHERE key = ?` + c.EXPECT().Prepare(rawStmt4) + txC.EXPECT().Stmt(gomock.Any()).Return(stmts) + stmts.EXPECT().Exec("moose2", "lego.cattle.io/fields2") + // And again for the other object + } err := store.Add(testObject) assert.Nil(t, err) @@ -789,7 +889,7 @@ func SetupMockDB(t *testing.T) (*MockClient, *MockTXClient) { func SetupStore(t *testing.T, client *MockClient, shouldEncrypt bool) *Store { name := "testStoreObject" gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: name} - store, err := NewStore(context.Background(), testStoreObject{}, testStoreKeyFunc, client, shouldEncrypt, gvk, name, nil) + store, err := NewStore(context.Background(), testStoreObject{}, testStoreKeyFunc, client, shouldEncrypt, gvk, name, nil, nil) if err != nil { t.Error(err) } @@ -800,7 +900,7 @@ func gvkKey(group, version, kind string) string { return group + "_" + version + "_" + kind } -func SetupStoreWithExternalDependencies(t *testing.T, client *MockClient) *Store { +func SetupStoreWithExternalDependencies(t *testing.T, client *MockClient, updateExternal bool, updateSelf bool) *Store { name := "testStoreObject" gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: name} namespaceProjectLabelDep := sqltypes.ExternalLabelDependency{ @@ -822,7 +922,15 @@ func SetupStoreWithExternalDependencies(t *testing.T, client *MockClient) *Store ExternalDependencies: []sqltypes.ExternalDependency{namespaceNonLabelDep}, ExternalLabelDependencies: []sqltypes.ExternalLabelDependency{namespaceProjectLabelDep}, } - store, err := NewStore(context.Background(), testStoreObject{}, testStoreKeyFunc, client, false, gvk, name, &updateInfo) + externalUpdateInfo := &updateInfo + selfUpdateInfo := &updateInfo + if !updateExternal { + externalUpdateInfo = nil + } + if !updateSelf { + selfUpdateInfo = nil + } + store, err := NewStore(context.Background(), testStoreObject{}, testStoreKeyFunc, client, false, gvk, name, externalUpdateInfo, selfUpdateInfo) if err != nil { t.Error(err) } diff --git a/pkg/stores/sqlproxy/proxy_mocks_test.go b/pkg/stores/sqlproxy/proxy_mocks_test.go index ac8a403e..40828f5d 100644 --- a/pkg/stores/sqlproxy/proxy_mocks_test.go +++ b/pkg/stores/sqlproxy/proxy_mocks_test.go @@ -267,18 +267,18 @@ func (m *MockCacheFactory) EXPECT() *MockCacheFactoryMockRecorder { } // CacheFor mocks base method. -func (m *MockCacheFactory) CacheFor(ctx context.Context, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced, watchable bool) (factory.Cache, error) { +func (m *MockCacheFactory) CacheFor(ctx context.Context, fields [][]string, externalUpdateInfo, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced, watchable bool) (factory.Cache, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CacheFor", ctx, fields, externalUpdateInfo, transform, client, gvk, namespaced, watchable) + ret := m.ctrl.Call(m, "CacheFor", ctx, fields, externalUpdateInfo, selfUpdateInfo, transform, client, gvk, namespaced, watchable) ret0, _ := ret[0].(factory.Cache) ret1, _ := ret[1].(error) return ret0, ret1 } // CacheFor indicates an expected call of CacheFor. -func (mr *MockCacheFactoryMockRecorder) CacheFor(ctx, fields, externalUpdateInfo, transform, client, gvk, namespaced, watchable any) *gomock.Call { +func (mr *MockCacheFactoryMockRecorder) CacheFor(ctx, fields, externalUpdateInfo, selfUpdateInfo, transform, client, gvk, namespaced, watchable any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFor", reflect.TypeOf((*MockCacheFactory)(nil).CacheFor), ctx, fields, externalUpdateInfo, transform, client, gvk, namespaced, watchable) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFor", reflect.TypeOf((*MockCacheFactory)(nil).CacheFor), ctx, fields, externalUpdateInfo, selfUpdateInfo, transform, client, gvk, namespaced, watchable) } // Reset mocks base method. diff --git a/pkg/stores/sqlproxy/proxy_store.go b/pkg/stores/sqlproxy/proxy_store.go index c782ff78..3bf08496 100644 --- a/pkg/stores/sqlproxy/proxy_store.go +++ b/pkg/stores/sqlproxy/proxy_store.go @@ -65,8 +65,6 @@ var ( paramCodec = runtime.NewParameterCodec(paramScheme) // Please keep the gvkKey entries in alphabetical order, on a field-by-field basis typeSpecificIndexedFields = map[string][][]string{ - gvkKey("", "v1", "ConfigMap"): { - {"metadata", "labels", "harvesterhci.io/cloud-init-template"}}, gvkKey("", "v1", "Event"): { {"_type"}, {"involvedObject", "kind"}, @@ -75,7 +73,6 @@ var ( {"reason"}, }, gvkKey("", "v1", "Namespace"): { - {"metadata", "labels", "field.cattle.io/projectId"}, {"spec", "displayName"}, }, gvkKey("", "v1", "Node"): { @@ -143,7 +140,6 @@ var ( gvkKey("cluster.x-k8s.io", "v1beta1", "MachineDeployment"): { {"spec", "clusterName"}}, gvkKey("management.cattle.io", "v3", "Cluster"): { - {"metadata", "labels", "provider.cattle.io"}, {"spec", "internal"}, {"spec", "displayName"}, {"status", "connected"}, @@ -156,8 +152,8 @@ var ( gvkKey("management.cattle.io", "v3", "NodeTemplate"): { {"spec", "clusterName"}}, gvkKey("management.cattle.io", "v3", "Project"): { - {"spec", "displayName"}, {"spec", "clusterName"}, + {"spec", "displayName"}, }, gvkKey("networking.k8s.io", "v1", "Ingress"): { {"spec", "rules", "host"}, @@ -165,7 +161,6 @@ var ( }, gvkKey("provisioning.cattle.io", "v1", "Cluster"): { {"metadata", "annotations", "provisioning.cattle.io/management-cluster-display-name"}, - {"metadata", "labels", "provider.cattle.io"}, {"status", "clusterName"}, {"status", "provider"}, }, @@ -192,6 +187,8 @@ var ( }, }, } + namespaceGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} + mcioProjectGvk = schema.GroupVersionKind{Group: "management.cattle.io", Version: "v3", Kind: "Project"} namespaceProjectLabelDep = sqltypes.ExternalLabelDependency{ SourceGVK: gvkKey("", "v1", "Namespace"), SourceLabelName: "field.cattle.io/projectId", @@ -199,12 +196,16 @@ var ( TargetKeyFieldName: "metadata.name", TargetFinalFieldName: "spec.displayName", } + namespaceUpdates = sqltypes.ExternalGVKUpdates{ + AffectedGVK: namespaceGVK, + ExternalDependencies: nil, + ExternalLabelDependencies: []sqltypes.ExternalLabelDependency{namespaceProjectLabelDep}, + } externalGVKDependencies = sqltypes.ExternalGVKDependency{ - schema.GroupVersionKind{Group: "management.cattle.io", Version: "v3", Kind: "Project"}: &sqltypes.ExternalGVKUpdates{ - AffectedGVK: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"}, - ExternalDependencies: nil, - ExternalLabelDependencies: []sqltypes.ExternalLabelDependency{namespaceProjectLabelDep}, - }, + mcioProjectGvk: &namespaceUpdates, + } + selfGVKDependencies = sqltypes.ExternalGVKDependency{ + namespaceGVK: &namespaceUpdates, } ) @@ -278,7 +279,7 @@ type Store struct { type CacheFactoryInitializer func() (CacheFactory, error) type CacheFactory interface { - CacheFor(ctx context.Context, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool, watchable bool) (factory.Cache, error) + CacheFor(ctx context.Context, fields [][]string, externalUpdateInfo *sqltypes.ExternalGVKUpdates, selfUpdateInfo *sqltypes.ExternalGVKUpdates, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool, watchable bool) (factory.Cache, error) Reset() error } @@ -360,7 +361,7 @@ func (s *Store) initializeNamespaceCache() error { // get the ns informer tableClient := &tablelistconvert.Client{ResourceInterface: client} - nsInformer, err := s.cacheFactory.CacheFor(s.ctx, fields, externalGVKDependencies[gvk], transformFunc, tableClient, gvk, false, true) + nsInformer, err := s.cacheFactory.CacheFor(s.ctx, fields, externalGVKDependencies[gvk], selfGVKDependencies[gvk], transformFunc, tableClient, gvk, false, true) if err != nil { return err } @@ -565,9 +566,8 @@ func (s *Store) watch(apiOp *types.APIRequest, schema *types.APISchema, w types. fields = append(fields, getFieldForGVK(gvk)...) transformFunc := s.transformBuilder.GetTransformFunc(gvk, nil) tableClient := &tablelistconvert.Client{ResourceInterface: client} - attrs := attributes.GVK(schema) ns := attributes.Namespaced(schema) - inf, err := s.cacheFactory.CacheFor(s.ctx, fields, externalGVKDependencies[gvk], transformFunc, tableClient, attrs, ns, controllerschema.IsListWatchable(schema)) + inf, err := s.cacheFactory.CacheFor(s.ctx, fields, externalGVKDependencies[gvk], selfGVKDependencies[gvk], transformFunc, tableClient, gvk, ns, controllerschema.IsListWatchable(schema)) if err != nil { return nil, err } @@ -778,9 +778,8 @@ func (s *Store) ListByPartitions(apiOp *types.APIRequest, apiSchema *types.APISc transformFunc := s.transformBuilder.GetTransformFunc(gvk, cols) tableClient := &tablelistconvert.Client{ResourceInterface: client} - attrs := attributes.GVK(apiSchema) ns := attributes.Namespaced(apiSchema) - inf, err := s.cacheFactory.CacheFor(s.ctx, fields, externalGVKDependencies[gvk], transformFunc, tableClient, attrs, ns, controllerschema.IsListWatchable(apiSchema)) + inf, err := s.cacheFactory.CacheFor(s.ctx, fields, externalGVKDependencies[gvk], selfGVKDependencies[gvk], transformFunc, tableClient, gvk, ns, controllerschema.IsListWatchable(apiSchema)) if err != nil { return nil, 0, "", err } diff --git a/pkg/stores/sqlproxy/proxy_store_test.go b/pkg/stores/sqlproxy/proxy_store_test.go index 8074552c..5135828a 100644 --- a/pkg/stores/sqlproxy/proxy_store_test.go +++ b/pkg/stores/sqlproxy/proxy_store_test.go @@ -84,8 +84,7 @@ func TestNewProxyStore(t *testing.T) { nsSchema := baseNSSchema scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) - cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}, {"spec", "displayName"}}, nil, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(c, nil) - + cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"spec", "displayName"}}, gomock.Any(), gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(c, nil) s, err := NewProxyStore(context.Background(), scc, cg, rn, nil, cf) assert.Nil(t, err) assert.Equal(t, scc, s.columnSetter) @@ -151,7 +150,7 @@ func TestNewProxyStore(t *testing.T) { nsSchema := baseNSSchema scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) - cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}, {"spec", "displayName"}}, nil, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(factory.Cache{}, fmt.Errorf("error")) + cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"spec", "displayName"}}, gomock.Any(), gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(factory.Cache{}, fmt.Errorf("error")) s, err := NewProxyStore(context.Background(), scc, cg, rn, nil, cf) assert.Nil(t, err) @@ -244,7 +243,7 @@ func TestListByPartitions(t *testing.T) { assert.Nil(t, err) cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil) // This tests that fields are being extracted from schema columns and the type specific fields map - cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, nil, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil) + cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil) tb.EXPECT().GetTransformFunc(attributes.GVK(schema), []common.ColumnDefinition{{Field: "some.field"}}).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil) list, total, contToken, err := s.ListByPartitions(req, schema, partitions) @@ -460,7 +459,7 @@ func TestListByPartitions(t *testing.T) { // This tests that fields are being extracted from schema columns and the type specific fields map // note also the watchable bool is expected to be false - cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, nil, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), false).Return(c, nil) + cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), false).Return(c, nil) tb.EXPECT().GetTransformFunc(attributes.GVK(schema), []common.ColumnDefinition{{Field: "some.field"}}).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil) @@ -535,7 +534,7 @@ func TestListByPartitions(t *testing.T) { cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil) // This tests that fields are being extracted from schema columns and the type specific fields map tb.EXPECT().GetTransformFunc(attributes.GVK(schema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) - cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, nil, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(factory.Cache{}, fmt.Errorf("error")) + cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(factory.Cache{}, fmt.Errorf("error")) _, _, _, err = s.ListByPartitions(req, schema, partitions) assert.NotNil(t, err) @@ -611,7 +610,7 @@ func TestListByPartitions(t *testing.T) { assert.Nil(t, err) cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil) // This tests that fields are being extracted from schema columns and the type specific fields map - cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, nil, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil) + cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil) bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(nil, 0, "", fmt.Errorf("error")) tb.EXPECT().GetTransformFunc(attributes.GVK(schema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) @@ -724,7 +723,7 @@ func TestListByPartitionWithUserAccess(t *testing.T) { } attributes.SetGVK(theSchema, gvk) cg.EXPECT().TableAdminClient(apiOp, theSchema, "", &WarningBuffer{}).Return(ri, nil) - cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {"id"}, {"metadata", "state", "name"}}, gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(theSchema), attributes.Namespaced(theSchema), true).Return(c, nil) + cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {"id"}, {"metadata", "state", "name"}}, gomock.Any(), gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(theSchema), attributes.Namespaced(theSchema), true).Return(c, nil) tb.EXPECT().GetTransformFunc(attributes.GVK(theSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) listToReturn := &unstructured.UnstructuredList{ @@ -766,7 +765,7 @@ func TestReset(t *testing.T) { cf.EXPECT().Reset().Return(nil) cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) - cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}, {"spec", "displayName"}}, nil, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(nsc2, nil) + cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"spec", "displayName"}}, gomock.Any(), gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(nsc2, nil) tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) err := s.Reset() assert.Nil(t, err) @@ -873,7 +872,7 @@ func TestReset(t *testing.T) { cf.EXPECT().Reset().Return(nil) cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) - cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}, {"spec", "displayName"}}, nil, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(factory.Cache{}, fmt.Errorf("error")) + cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"spec", "displayName"}}, gomock.Any(), gomock.Any(), gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(factory.Cache{}, fmt.Errorf("error")) tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) err := s.Reset() assert.NotNil(t, err)