From dd27bd0c8d30a036f448613d488d04829357081a Mon Sep 17 00:00:00 2001 From: Silvio Moioli Date: Wed, 2 Oct 2024 14:59:54 +0200 Subject: [PATCH] [v2.9] Virtual Resource filters (#288) Adds logic which adds virtual fields resources. This allows these fields to be sorted/filtered on when the SQL cache is enabled. Id and metadata.state.name were added as the first two fields. Co-authored-by: Michael Bolot --- README.md | 11 ++ go.mod | 6 +- go.sum | 14 +- pkg/resources/common/formatter.go | 32 ++-- pkg/resources/schema.go | 4 +- pkg/resources/virtual/common/common.go | 116 ++++++++++++ pkg/resources/virtual/common/common_test.go | 188 ++++++++++++++++++++ pkg/resources/virtual/common/util.go | 40 +++++ pkg/resources/virtual/virtual.go | 28 +++ pkg/server/server.go | 2 +- pkg/stores/partition/store.go | 20 +-- pkg/stores/sqlpartition/store.go | 24 ++- pkg/stores/sqlproxy/proxy_mocks_test.go | 48 ++++- pkg/stores/sqlproxy/proxy_store.go | 71 +++++--- pkg/stores/sqlproxy/proxy_store_test.go | 129 ++++++++------ 15 files changed, 611 insertions(+), 122 deletions(-) create mode 100644 pkg/resources/virtual/common/common.go create mode 100644 pkg/resources/virtual/common/common_test.go create mode 100644 pkg/resources/virtual/common/util.go create mode 100644 pkg/resources/virtual/virtual.go diff --git a/README.md b/README.md index 22f275be..712efbd8 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,17 @@ item is included in the list. /v1/{type}?filter=spec.containers.image=alpine ``` +**If SQLite caching is enabled** (`server.Options.SQLCache=true`), +filtering is only supported for a subset of attributes: +- `id`, `metadata.name`, `metadata.namespace`, `metadata.state.name`, and `metadata.timestamp` for any resource kind +- a short list of hardcoded attributes for a selection of specific types listed +in [typeSpecificIndexFields](https://github.com/rancher/steve/blob/main/pkg/stores/sqlproxy/proxy_store.go#L52-L58) +- the special string `metadata.fields[N]`, with N starting at 0, for all columns +displayed by `kubectl get $TYPE`. For example `secrets` have `"metadata.fields[0]"`, +`"metadata.fields[1]"` , `"metadata.fields[2]"`, and `"metadata.fields[3]"` respectively +corresponding to `"name"`, `"type"`, `"data"`, and `"age"`. For CRDs, these come from +[Additional printer columns](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#additional-printer-columns) + #### `projectsornamespaces` Resources can also be filtered by the Rancher projects their namespaces belong diff --git a/go.mod b/go.mod index dffc06fe..a158dfaf 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/rancher/apiserver v0.0.0-20240708202538-39a6f2535146 github.com/rancher/dynamiclistener v0.6.0-rc2 github.com/rancher/kubernetes-provider-detector v0.1.5 - github.com/rancher/lasso v0.0.0-20240705194423-b2a060d103c1 + github.com/rancher/lasso v0.0.0-20240923125127-ae858d002589 github.com/rancher/norman v0.0.0-20240708202514-a0127673d1b9 github.com/rancher/remotedialer v0.3.2 github.com/rancher/wrangler/v3 v3.0.0 @@ -80,7 +80,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -109,7 +109,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect modernc.org/libc v1.49.3 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/go.sum b/go.sum index 606eba72..2d63f024 100644 --- a/go.sum +++ b/go.sum @@ -184,16 +184,16 @@ github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUo github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rancher/apiserver v0.0.0-20240708202538-39a6f2535146 h1:6I4Z7PAGmned9+EYxbMS7kvajId3r8+ZwAR5wB7X3kg= github.com/rancher/apiserver v0.0.0-20240708202538-39a6f2535146/go.mod h1:ZNk+LcRGwQYHqgbsJijRrI49KFbX31/QzoUBq4rAeV0= github.com/rancher/dynamiclistener v0.6.0-rc2 h1:ASh61tOKTa2OJyKMc9stcmv7W6Xn/rwA8Me0yEIUe7s= github.com/rancher/dynamiclistener v0.6.0-rc2/go.mod h1:7VNEQhAwzbYJ08S1MYb6B4vili6K7CcrG4cNZXq1j+s= github.com/rancher/kubernetes-provider-detector v0.1.5 h1:hWRAsWuJOemzGjz/XrbTlM7QmfO4OedvFE3QwXiH60I= github.com/rancher/kubernetes-provider-detector v0.1.5/go.mod h1:ypuJS7kP7rUiAn330xG46mj+Nhvym05GM8NqMVekpH0= -github.com/rancher/lasso v0.0.0-20240705194423-b2a060d103c1 h1:vv1jDlYbd4KhGbPNxmjs8CYgEHUrQm2bMtmULfXJ6iw= -github.com/rancher/lasso v0.0.0-20240705194423-b2a060d103c1/go.mod h1:A/y3BLQkxZXYD60MNDRwAG9WGxXfvd6Z6gWR/a8wPw8= +github.com/rancher/lasso v0.0.0-20240923125127-ae858d002589 h1:c3IIzpVo5Jhd1T7Dih/kbEueWC21j4xTZ5EQO9rhTUg= +github.com/rancher/lasso v0.0.0-20240923125127-ae858d002589/go.mod h1:+GJaJqWpk1MZYCDrMC+4Jee0M/aScoru5T8V2A//rgg= github.com/rancher/norman v0.0.0-20240708202514-a0127673d1b9 h1:AlRMRs5mHJcdiK83KKJyFVeybPMZ7dOUzC0l3k9aUa8= github.com/rancher/norman v0.0.0-20240708202514-a0127673d1b9/go.mod h1:dyjfXBsNiroPWOdUZe7diUOUSLf6HQ/r2kEpwH/8zas= github.com/rancher/remotedialer v0.3.2 h1:kstZbRwPS5gPWpGg8VjEHT2poHtArs+Fc317YM8JCzU= @@ -252,6 +252,8 @@ go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lI go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -412,8 +414,8 @@ k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3 h1:SbdLaI6mM6ffDSJCadEaD4IkuPzepLDGlkd2xV0t1uA= k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= modernc.org/cc/v4 v4.20.0 h1:45Or8mQfbUqJOG9WaxvlFYOAQO0lQ5RvqBcFCXngjxk= modernc.org/cc/v4 v4.20.0/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= modernc.org/ccgo/v4 v4.16.0 h1:ofwORa6vx2FMm0916/CkZjpFPSR70VwTjUCe2Eg5BnA= diff --git a/pkg/resources/common/formatter.go b/pkg/resources/common/formatter.go index 1bda5986..a876f89c 100644 --- a/pkg/resources/common/formatter.go +++ b/pkg/resources/common/formatter.go @@ -23,18 +23,19 @@ import ( func DefaultTemplate(clientGetter proxy.ClientGetter, summaryCache *summarycache.SummaryCache, asl accesscontrol.AccessSetLookup, - namespaceCache corecontrollers.NamespaceCache) schema.Template { + namespaceCache corecontrollers.NamespaceCache, + sqlCache bool) schema.Template { return schema.Template{ Store: metricsStore.NewMetricsStore(proxy.NewProxyStore(clientGetter, summaryCache, asl, namespaceCache)), - Formatter: formatter(summaryCache), + Formatter: formatter(summaryCache, sqlCache), } } // DefaultTemplateForStore provides a default schema template which uses a provided, pre-initialized store. Primarily used when creating a Template that uses a Lasso SQL store internally. -func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache) schema.Template { +func DefaultTemplateForStore(store types.Store, summaryCache *summarycache.SummaryCache, sqlCache bool) schema.Template { return schema.Template{ Store: store, - Formatter: formatter(summaryCache), + Formatter: formatter(summaryCache, sqlCache), } } @@ -71,7 +72,7 @@ func selfLink(gvr schema2.GroupVersionResource, meta metav1.Object) (prefix stri return buf.String() } -func formatter(summarycache *summarycache.SummaryCache) types.Formatter { +func formatter(summarycache *summarycache.SummaryCache, sqlCache bool) types.Formatter { return func(request *types.APIRequest, resource *types.RawResource) { if resource.Schema == nil { return @@ -104,17 +105,20 @@ func formatter(summarycache *summarycache.SummaryCache) types.Formatter { } if unstr, ok := resource.APIObject.Object.(*unstructured.Unstructured); ok { - s, rel := summarycache.SummaryAndRelationship(unstr) - data.PutValue(unstr.Object, map[string]interface{}{ - "name": s.State, - "error": s.Error, - "transitioning": s.Transitioning, - "message": strings.Join(s.Message, ":"), - }, "metadata", "state") - data.PutValue(unstr.Object, rel, "metadata", "relationships") + if !sqlCache { + // with the sql cache, these were already added by the indexer + s, rel := summarycache.SummaryAndRelationship(unstr) + data.PutValue(unstr.Object, map[string]interface{}{ + "name": s.State, + "error": s.Error, + "transitioning": s.Transitioning, + "message": strings.Join(s.Message, ":"), + }, "metadata", "state") + data.PutValue(unstr.Object, rel, "metadata", "relationships") - summary.NormalizeConditions(unstr) + summary.NormalizeConditions(unstr) + } includeFields(request, unstr) excludeFields(request, unstr) excludeValues(request, unstr) diff --git a/pkg/resources/schema.go b/pkg/resources/schema.go index a96e45f7..298807c2 100644 --- a/pkg/resources/schema.go +++ b/pkg/resources/schema.go @@ -49,7 +49,7 @@ func DefaultSchemaTemplates(cf *client.Factory, discovery discovery.DiscoveryInterface, namespaceCache corecontrollers.NamespaceCache) []schema.Template { return []schema.Template{ - common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache), + common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache, false), apigroups.Template(discovery), { ID: "configmap", @@ -79,7 +79,7 @@ func DefaultSchemaTemplatesForStore(store types.Store, discovery discovery.DiscoveryInterface) []schema.Template { return []schema.Template{ - common.DefaultTemplateForStore(store, summaryCache), + common.DefaultTemplateForStore(store, summaryCache, true), apigroups.Template(discovery), { ID: "configmap", diff --git a/pkg/resources/virtual/common/common.go b/pkg/resources/virtual/common/common.go new file mode 100644 index 00000000..97fa7778 --- /dev/null +++ b/pkg/resources/virtual/common/common.go @@ -0,0 +1,116 @@ +// Package common provides cache.TransformFunc's which are common to all types +package common + +import ( + "fmt" + "strings" + + "github.com/rancher/steve/pkg/summarycache" + "github.com/rancher/wrangler/v3/pkg/data" + wranglerSummary "github.com/rancher/wrangler/v3/pkg/summary" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/cache" +) + +// SummaryCache provides an interface to get a summary/relationships for an object. Implemented by the summaryCache +// struct from pkg/summarycache +type SummaryCache interface { + SummaryAndRelationship(runtime.Object) (*wranglerSummary.SummarizedObject, []summarycache.Relationship) +} + +// DefaultFields produces a VirtualTransformFunc through GetTransform() that applies to all k8s types +type DefaultFields struct { + Cache SummaryCache +} + +// GetTransform produces the default transformation func +func (d *DefaultFields) GetTransform() cache.TransformFunc { + return d.transform +} + +// transform implements virtual.VirtualTransformFunc, and adds reserved fields/summary +func (d *DefaultFields) transform(obj any) (any, error) { + raw, isSignal, err := getUnstructured(obj) + if isSignal { + return obj, nil + } + if err != nil { + return nil, err + } + raw = addIDField(raw) + raw, err = addSummaryFields(raw, d.Cache) + if err != nil { + return nil, fmt.Errorf("unable to add summary fields: %w", err) + } + return raw, nil +} + +// addSummaryFields adds the virtual fields for object state. +func addSummaryFields(raw *unstructured.Unstructured, cache SummaryCache) (*unstructured.Unstructured, error) { + s, relationships := cache.SummaryAndRelationship(raw) + if s != nil { + data.PutValue(raw.Object, map[string]interface{}{ + "name": s.State, + "error": s.Error, + "transitioning": s.Transitioning, + "message": strings.Join(s.Message, ":"), + }, "metadata", "state") + + } + var rels []any + for _, relationship := range relationships { + rel, err := toMap(relationship) + if err != nil { + return nil, fmt.Errorf("unable to convert relationship to map: %w", err) + } + rels = append(rels, rel) + } + data.PutValue(raw.Object, rels, "metadata", "relationships") + + normalizeConditions(raw) + return raw, nil +} + +// addIDField adds the ID field based on namespace/name, and moves the current id field to _id if present +func addIDField(raw *unstructured.Unstructured) *unstructured.Unstructured { + objectID := raw.GetName() + namespace := raw.GetNamespace() + if namespace != "" { + objectID = fmt.Sprintf("%s/%s", namespace, objectID) + } + currentIDValue, ok := raw.Object["id"] + if ok { + raw.Object["_id"] = currentIDValue + } + raw.Object["id"] = objectID + return raw +} + +func normalizeConditions(raw *unstructured.Unstructured) { + var ( + obj data.Object + newConditions []any + ) + + obj = raw.Object + for _, condition := range obj.Slice("status", "conditions") { + var summary wranglerSummary.Summary + for _, summarizer := range wranglerSummary.ConditionSummarizers { + summary = summarizer(obj, []wranglerSummary.Condition{{Object: condition}}, summary) + } + condition.Set("error", summary.Error) + condition.Set("transitioning", summary.Transitioning) + + if condition.String("lastUpdateTime") == "" { + condition.Set("lastUpdateTime", condition.String("lastTransitionTime")) + } + // needs to be reconverted back to a map[string]any or we can have encoding problems with unregistered types + var mapCondition map[string]any = condition + newConditions = append(newConditions, mapCondition) + } + + if len(newConditions) > 0 { + obj.SetNested(newConditions, "status", "conditions") + } +} diff --git a/pkg/resources/virtual/common/common_test.go b/pkg/resources/virtual/common/common_test.go new file mode 100644 index 00000000..78141d57 --- /dev/null +++ b/pkg/resources/virtual/common/common_test.go @@ -0,0 +1,188 @@ +package common_test + +import ( + "testing" + + "github.com/rancher/steve/pkg/resources/virtual/common" + "github.com/rancher/steve/pkg/summarycache" + "github.com/rancher/wrangler/v3/pkg/summary" + "github.com/stretchr/testify/require" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/cache" +) + +func TestTransform(t *testing.T) { + tests := []struct { + name string + input any + hasSummary *summary.SummarizedObject + hasRelationships []summarycache.Relationship + wantOutput any + wantError bool + }{ + { + name: "signal error", + input: cache.DeletedFinalStateUnknown{ + Key: "some-ns/some-name", + }, + wantOutput: cache.DeletedFinalStateUnknown{ + Key: "some-ns/some-name", + }, + wantError: false, + }, + { + name: "not unstructured", + input: map[string]any{ + "somekey": "someval", + }, + wantError: true, + }, + { + name: "add summary + relationships + reserved fields", + hasSummary: &summary.SummarizedObject{ + PartialObjectMetadata: v1.PartialObjectMetadata{ + ObjectMeta: v1.ObjectMeta{ + Name: "testobj", + Namespace: "test-ns", + }, + TypeMeta: v1.TypeMeta{ + APIVersion: "test.cattle.io/v1", + Kind: "TestResource", + }, + }, + Summary: summary.Summary{ + State: "success", + Transitioning: false, + Error: false, + Message: []string{"resource 1 rolled out", "resource 2 rolled out"}, + }, + }, + hasRelationships: []summarycache.Relationship{ + { + ToID: "1345", + ToType: "SomeType", + ToNamespace: "some-ns", + FromID: "78901", + FromType: "TestResource", + Rel: "uses", + }, + }, + input: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cattle.io/v1", + "kind": "TestResource", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + }, + "id": "old-id", + }, + }, + wantOutput: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cattle.io/v1", + "kind": "TestResource", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + "state": map[string]interface{}{ + "name": "success", + "error": false, + "transitioning": false, + "message": "resource 1 rolled out:resource 2 rolled out", + }, + "relationships": []any{ + map[string]any{ + "toId": "1345", + "toType": "SomeType", + "toNamespace": "some-ns", + "fromId": "78901", + "fromType": "TestResource", + "rel": "uses", + }, + }, + }, + "id": "test-ns/testobj", + "_id": "old-id", + }, + }, + }, + { + name: "add conditions + reserved fields", + input: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cattle.io/v1", + "kind": "TestResource", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + }, + "status": map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{ + "status": "False", + "reason": "Error", + "message": "some error", + "lastTransitionTime": "2024-01-01", + }, + }, + }, + }, + }, + wantOutput: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cattle.io/v1", + "kind": "TestResource", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + "relationships": []any(nil), + }, + "status": map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{ + "status": "False", + "reason": "Error", + "transitioning": false, + "error": true, + "message": "some error", + "lastTransitionTime": "2024-01-01", + "lastUpdateTime": "2024-01-01", + }, + }, + }, + "id": "test-ns/testobj", + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeCache := fakeSummaryCache{ + summarizedObject: test.hasSummary, + relationships: test.hasRelationships, + } + df := common.DefaultFields{ + Cache: &fakeCache, + } + output, err := df.GetTransform()(test.input) + require.Equal(t, test.wantOutput, output) + if test.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +type fakeSummaryCache struct { + summarizedObject *summary.SummarizedObject + relationships []summarycache.Relationship +} + +func (f *fakeSummaryCache) SummaryAndRelationship(runtime.Object) (*summary.SummarizedObject, []summarycache.Relationship) { + return f.summarizedObject, f.relationships +} diff --git a/pkg/resources/virtual/common/util.go b/pkg/resources/virtual/common/util.go new file mode 100644 index 00000000..fb496ea8 --- /dev/null +++ b/pkg/resources/virtual/common/util.go @@ -0,0 +1,40 @@ +package common + +import ( + "encoding/json" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/tools/cache" +) + +// GetUnstructured retrieves an unstructured object from the provided input. If this is a signal +// object (like cache.DeletedFinalStateUnknown), returns true, indicating that this wasn't an +// unstructured object, but doesn't need to be processed by our transform function +func getUnstructured(obj any) (*unstructured.Unstructured, bool, error) { + raw, ok := obj.(*unstructured.Unstructured) + if !ok { + _, isFinalUnknown := obj.(cache.DeletedFinalStateUnknown) + if isFinalUnknown { + // As documented in the TransformFunc interface + return nil, true, nil + } + return nil, false, fmt.Errorf("object was of type %T, not unstructured", raw) + } + return raw, false, nil +} + +// toMap converts an object to a map[string]any which can be stored/retrieved from the cache. Currently +// uses json encoding to take advantage of tag names +func toMap(obj any) (map[string]any, error) { + bytes, err := json.Marshal(obj) + if err != nil { + return nil, fmt.Errorf("unable to marshal object: %w", err) + } + var retObj map[string]any + err = json.Unmarshal(bytes, &retObj) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal object: %w", err) + } + return retObj, nil +} diff --git a/pkg/resources/virtual/virtual.go b/pkg/resources/virtual/virtual.go new file mode 100644 index 00000000..ec297e31 --- /dev/null +++ b/pkg/resources/virtual/virtual.go @@ -0,0 +1,28 @@ +// Package virtual provides functions/resources to define virtual fields (fields which don't exist in k8s +// but should be visible in the API) on resources +package virtual + +import ( + "github.com/rancher/steve/pkg/resources/virtual/common" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/cache" +) + +// TransformBuilder builds transform functions for specified GVKs through GetTransformFunc +type TransformBuilder struct { + defaultFields *common.DefaultFields +} + +// NewTransformBuilder returns a TransformBuilder using the given summary cache +func NewTransformBuilder(cache common.SummaryCache) *TransformBuilder { + return &TransformBuilder{ + &common.DefaultFields{ + Cache: cache, + }, + } +} + +// GetTransformFunc retrieves a TransformFunc for a given GVK. Currently only returns a transformFunc for defaultFields +func (t *TransformBuilder) GetTransformFunc(_ schema.GroupVersionKind) cache.TransformFunc { + return t.defaultFields.GetTransform() +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 5cd96a8e..472b7d5b 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -163,7 +163,7 @@ func setup(ctx context.Context, server *Server) error { var onSchemasHandler schemacontroller.SchemasHandlerFunc if server.SQLCache { - s, err := sqlproxy.NewProxyStore(cols, cf, summaryCache, nil) + s, err := sqlproxy.NewProxyStore(cols, cf, summaryCache, summaryCache, nil) if err != nil { panic(err) } diff --git a/pkg/stores/partition/store.go b/pkg/stores/partition/store.go index 6796611b..c018fc4b 100644 --- a/pkg/stores/partition/store.go +++ b/pkg/stores/partition/store.go @@ -111,7 +111,7 @@ func (s *Store) Delete(apiOp *types.APIRequest, schema *types.APISchema, id stri if err != nil { return types.APIObject{}, err } - return ToAPI(schema, obj, warnings), nil + return ToAPI(schema, obj, warnings, types.ReservedFields), nil } // ByID looks up a single object by its ID. @@ -125,7 +125,7 @@ func (s *Store) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string if err != nil { return types.APIObject{}, err } - return ToAPI(schema, obj, warnings), nil + return ToAPI(schema, obj, warnings, types.ReservedFields), nil } func (s *Store) listPartition(ctx context.Context, apiOp *types.APIRequest, schema *types.APISchema, partition Partition, @@ -227,7 +227,7 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP for _, item := range list { item := item.DeepCopy() - result.Objects = append(result.Objects, ToAPI(schema, item, nil)) + result.Objects = append(result.Objects, ToAPI(schema, item, nil, types.ReservedFields)) } result.Pages = pages @@ -267,7 +267,7 @@ func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, data ty if err != nil { return types.APIObject{}, err } - return ToAPI(schema, obj, warnings), nil + return ToAPI(schema, obj, warnings, types.ReservedFields), nil } // Update updates a single object in the store. @@ -281,7 +281,7 @@ func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, data ty if err != nil { return types.APIObject{}, err } - return ToAPI(schema, obj, warnings), nil + return ToAPI(schema, obj, warnings, types.ReservedFields), nil } // Watch returns a channel of events for a list or resource. @@ -327,13 +327,13 @@ func (s *Store) Watch(apiOp *types.APIRequest, schema *types.APISchema, wr types return response, nil } -func ToAPI(schema *types.APISchema, obj runtime.Object, warnings []types.Warning) types.APIObject { +func ToAPI(schema *types.APISchema, obj runtime.Object, warnings []types.Warning, reservedFields map[string]bool) types.APIObject { if obj == nil || reflect.ValueOf(obj).IsNil() { return types.APIObject{} } if unstr, ok := obj.(*unstructured.Unstructured); ok { - obj = moveToUnderscore(unstr) + obj = moveToUnderscore(unstr, reservedFields) } apiObject := types.APIObject{ @@ -357,12 +357,12 @@ func ToAPI(schema *types.APISchema, obj runtime.Object, warnings []types.Warning return apiObject } -func moveToUnderscore(obj *unstructured.Unstructured) *unstructured.Unstructured { +func moveToUnderscore(obj *unstructured.Unstructured, reservedFields map[string]bool) *unstructured.Unstructured { if obj == nil { return nil } - for k := range types.ReservedFields { + for k := range reservedFields { v, ok := obj.Object[k] if ok { delete(obj.Object, k) @@ -394,7 +394,7 @@ func ToAPIEvent(apiOp *types.APIRequest, schema *types.APISchema, event watch.Ev return apiEvent } - apiEvent.Object = ToAPI(schema, event.Object, nil) + apiEvent.Object = ToAPI(schema, event.Object, nil, types.ReservedFields) m, err := meta.Accessor(event.Object) if err != nil { diff --git a/pkg/stores/sqlpartition/store.go b/pkg/stores/sqlpartition/store.go index 460a5f0d..145d5e72 100644 --- a/pkg/stores/sqlpartition/store.go +++ b/pkg/stores/sqlpartition/store.go @@ -24,8 +24,9 @@ type SchemaColumnSetter interface { // Store implements types.proxyStore for partitions. type Store struct { - Partitioner Partitioner - asl accesscontrol.AccessSetLookup + Partitioner Partitioner + asl accesscontrol.AccessSetLookup + sqlReservedFields map[string]bool } // NewStore creates a types.proxyStore implementation with a partitioner @@ -36,6 +37,14 @@ func NewStore(store UnstructuredStore, asl accesscontrol.AccessSetLookup) *Store }, asl: asl, } + sqlReservedFields := map[string]bool{} + for key, value := range types.ReservedFields { + if key == "id" { + continue + } + sqlReservedFields[key] = value + } + s.sqlReservedFields = sqlReservedFields return s } @@ -48,7 +57,7 @@ func (s *Store) Delete(apiOp *types.APIRequest, schema *types.APISchema, id stri if err != nil { return types.APIObject{}, err } - return partition.ToAPI(schema, obj, warnings), nil + return partition.ToAPI(schema, obj, warnings, types.ReservedFields), nil } // ByID looks up a single object by its ID. @@ -59,7 +68,7 @@ func (s *Store) ByID(apiOp *types.APIRequest, schema *types.APISchema, id string if err != nil { return types.APIObject{}, err } - return partition.ToAPI(schema, obj, warnings), nil + return partition.ToAPI(schema, obj, warnings, types.ReservedFields), nil } // List returns a list of objects across all applicable partitions. @@ -85,7 +94,8 @@ func (s *Store) List(apiOp *types.APIRequest, schema *types.APISchema) (types.AP for _, item := range list { item := item.DeepCopy() - result.Objects = append(result.Objects, partition.ToAPI(schema, item, nil)) + // the sql cache automatically adds the ID through a transformFunc. Because of this, we have a different set of reserved fields for the SQL cache + result.Objects = append(result.Objects, partition.ToAPI(schema, item, nil, s.sqlReservedFields)) } result.Revision = "" @@ -101,7 +111,7 @@ func (s *Store) Create(apiOp *types.APIRequest, schema *types.APISchema, data ty if err != nil { return types.APIObject{}, err } - return partition.ToAPI(schema, obj, warnings), nil + return partition.ToAPI(schema, obj, warnings, types.ReservedFields), nil } // Update updates a single object in the store. @@ -112,7 +122,7 @@ func (s *Store) Update(apiOp *types.APIRequest, schema *types.APISchema, data ty if err != nil { return types.APIObject{}, err } - return partition.ToAPI(schema, obj, warnings), nil + return partition.ToAPI(schema, obj, warnings, types.ReservedFields), nil } // Watch returns a channel of events for a list or resource. diff --git a/pkg/stores/sqlproxy/proxy_mocks_test.go b/pkg/stores/sqlproxy/proxy_mocks_test.go index e92836f4..ca1499ae 100644 --- a/pkg/stores/sqlproxy/proxy_mocks_test.go +++ b/pkg/stores/sqlproxy/proxy_mocks_test.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/rancher/steve/pkg/stores/sqlproxy (interfaces: Cache,ClientGetter,CacheFactory,SchemaColumnSetter,RelationshipNotifier) +// Source: github.com/rancher/steve/pkg/stores/sqlproxy (interfaces: Cache,ClientGetter,CacheFactory,SchemaColumnSetter,RelationshipNotifier,TransformBuilder) // Package sqlproxy is a generated GoMock package. package sqlproxy @@ -19,6 +19,7 @@ import ( dynamic "k8s.io/client-go/dynamic" kubernetes "k8s.io/client-go/kubernetes" rest "k8s.io/client-go/rest" + cache "k8s.io/client-go/tools/cache" ) // MockCache is a mock of Cache interface. @@ -257,18 +258,18 @@ func (m *MockCacheFactory) EXPECT() *MockCacheFactoryMockRecorder { } // CacheFor mocks base method. -func (m *MockCacheFactory) CacheFor(arg0 [][]string, arg1 dynamic.ResourceInterface, arg2 schema.GroupVersionKind, arg3 bool) (factory.Cache, error) { +func (m *MockCacheFactory) CacheFor(arg0 [][]string, arg1 cache.TransformFunc, arg2 dynamic.ResourceInterface, arg3 schema.GroupVersionKind, arg4 bool) (factory.Cache, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CacheFor", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "CacheFor", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(factory.Cache) ret1, _ := ret[1].(error) return ret0, ret1 } // CacheFor indicates an expected call of CacheFor. -func (mr *MockCacheFactoryMockRecorder) CacheFor(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockCacheFactoryMockRecorder) CacheFor(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFor", reflect.TypeOf((*MockCacheFactory)(nil).CacheFor), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFor", reflect.TypeOf((*MockCacheFactory)(nil).CacheFor), arg0, arg1, arg2, arg3, arg4) } // Reset mocks base method. @@ -358,3 +359,40 @@ func (mr *MockRelationshipNotifierMockRecorder) OnInboundRelationshipChange(arg0 mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnInboundRelationshipChange", reflect.TypeOf((*MockRelationshipNotifier)(nil).OnInboundRelationshipChange), arg0, arg1, arg2) } + +// MockTransformBuilder is a mock of TransformBuilder interface. +type MockTransformBuilder struct { + ctrl *gomock.Controller + recorder *MockTransformBuilderMockRecorder +} + +// MockTransformBuilderMockRecorder is the mock recorder for MockTransformBuilder. +type MockTransformBuilderMockRecorder struct { + mock *MockTransformBuilder +} + +// NewMockTransformBuilder creates a new mock instance. +func NewMockTransformBuilder(ctrl *gomock.Controller) *MockTransformBuilder { + mock := &MockTransformBuilder{ctrl: ctrl} + mock.recorder = &MockTransformBuilderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockTransformBuilder) EXPECT() *MockTransformBuilderMockRecorder { + return m.recorder +} + +// GetTransformFunc mocks base method. +func (m *MockTransformBuilder) GetTransformFunc(arg0 schema.GroupVersionKind) cache.TransformFunc { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTransformFunc", arg0) + ret0, _ := ret[0].(cache.TransformFunc) + return ret0 +} + +// GetTransformFunc indicates an expected call of GetTransformFunc. +func (mr *MockTransformBuilderMockRecorder) GetTransformFunc(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTransformFunc", reflect.TypeOf((*MockTransformBuilder)(nil).GetTransformFunc), arg0) +} diff --git a/pkg/stores/sqlproxy/proxy_store.go b/pkg/stores/sqlproxy/proxy_store.go index c6e5953a..e12d87af 100644 --- a/pkg/stores/sqlproxy/proxy_store.go +++ b/pkg/stores/sqlproxy/proxy_store.go @@ -15,6 +15,15 @@ import ( "sync" "github.com/pkg/errors" + + "github.com/rancher/steve/pkg/attributes" + "github.com/rancher/steve/pkg/resources/common" + "github.com/rancher/steve/pkg/resources/virtual" + virtualCommon "github.com/rancher/steve/pkg/resources/virtual/common" + metricsStore "github.com/rancher/steve/pkg/stores/metrics" + "github.com/rancher/steve/pkg/stores/sqlpartition/listprocessor" + "github.com/rancher/steve/pkg/stores/sqlproxy/tablelistconvert" + "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/api/meta" @@ -39,11 +48,7 @@ import ( "github.com/rancher/wrangler/v3/pkg/schemas/validation" "github.com/rancher/wrangler/v3/pkg/summary" - "github.com/rancher/steve/pkg/attributes" - "github.com/rancher/steve/pkg/resources/common" - metricsStore "github.com/rancher/steve/pkg/stores/metrics" - "github.com/rancher/steve/pkg/stores/sqlpartition/listprocessor" - "github.com/rancher/steve/pkg/stores/sqlproxy/tablelistconvert" + "k8s.io/client-go/tools/cache" ) const ( @@ -63,6 +68,10 @@ var ( "management.cattle.io_v3_Node": {{`status`, `nodeName`}}, } + commonIndexFields = [][]string{ + {`id`}, + {`metadata`, `state`, `name`}, + } baseNSSchema = types.APISchema{ Schema: &schemas.Schema{ Attributes: map[string]interface{}{ @@ -124,29 +133,35 @@ type RelationshipNotifier interface { OnInboundRelationshipChange(ctx context.Context, schema *types.APISchema, namespace string) <-chan *summary.Relationship } +type TransformBuilder interface { + GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc +} + type Store struct { - clientGetter ClientGetter - notifier RelationshipNotifier - cacheFactory CacheFactory - cfInitializer CacheFactoryInitializer - namespaceCache Cache - lock sync.Mutex - columnSetter SchemaColumnSetter + clientGetter ClientGetter + notifier RelationshipNotifier + cacheFactory CacheFactory + cfInitializer CacheFactoryInitializer + namespaceCache Cache + lock sync.Mutex + columnSetter SchemaColumnSetter + transformBuilder TransformBuilder } type CacheFactoryInitializer func() (CacheFactory, error) type CacheFactory interface { - CacheFor(fields [][]string, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool) (factory.Cache, error) + CacheFor(fields [][]string, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool) (factory.Cache, error) Reset() error } // NewProxyStore returns a Store implemented directly on top of kubernetes. -func NewProxyStore(c SchemaColumnSetter, clientGetter ClientGetter, notifier RelationshipNotifier, factory CacheFactory) (*Store, error) { +func NewProxyStore(c SchemaColumnSetter, clientGetter ClientGetter, notifier RelationshipNotifier, scache virtualCommon.SummaryCache, factory CacheFactory) (*Store, error) { store := &Store{ - clientGetter: clientGetter, - notifier: notifier, - columnSetter: c, + clientGetter: clientGetter, + notifier: notifier, + columnSetter: c, + transformBuilder: virtual.NewTransformBuilder(scache), } if factory == nil { @@ -203,14 +218,18 @@ func (s *Store) initializeNamespaceCache() error { return err } + gvk := attributes.GVK(&nsSchema) // get fields from schema's columns fields := getFieldsFromSchema(&nsSchema) // get any type-specific fields that steve is interested in - fields = append(fields, getFieldForGVK(attributes.GVK(&nsSchema))...) + fields = append(fields, getFieldForGVK(gvk)...) + + // get the type-specifc transform func + transformFunc := s.transformBuilder.GetTransformFunc(gvk) // get the ns informer - nsInformer, err := s.cacheFactory.CacheFor(fields, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(&nsSchema), false) + nsInformer, err := s.cacheFactory.CacheFor(fields, transformFunc, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(&nsSchema), false) if err != nil { return err } @@ -220,7 +239,13 @@ func (s *Store) initializeNamespaceCache() error { } func getFieldForGVK(gvk schema.GroupVersionKind) [][]string { - return typeSpecificIndexedFields[keyFromGVK(gvk)] + fields := [][]string{} + fields = append(fields, commonIndexFields...) + typeFields := typeSpecificIndexedFields[keyFromGVK(gvk)] + if typeFields != nil { + fields = append(fields, typeFields...) + } + return fields } func keyFromGVK(gvk schema.GroupVersionKind) string { @@ -639,10 +664,12 @@ func (s *Store) ListByPartitions(apiOp *types.APIRequest, schema *types.APISchem if err != nil { return nil, 0, "", err } + gvk := attributes.GVK(schema) fields := getFieldsFromSchema(schema) - fields = append(fields, getFieldForGVK(attributes.GVK(schema))...) + fields = append(fields, getFieldForGVK(gvk)...) + transformFunc := s.transformBuilder.GetTransformFunc(gvk) - inf, err := s.cacheFactory.CacheFor(fields, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(schema), attributes.Namespaced(schema)) + inf, err := s.cacheFactory.CacheFor(fields, transformFunc, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(schema), attributes.Namespaced(schema)) if err != nil { return nil, 0, "", err } diff --git a/pkg/stores/sqlproxy/proxy_store_test.go b/pkg/stores/sqlproxy/proxy_store_test.go index 9a876dac..54f31f0d 100644 --- a/pkg/stores/sqlproxy/proxy_store_test.go +++ b/pkg/stores/sqlproxy/proxy_store_test.go @@ -41,7 +41,7 @@ import ( clientgotesting "k8s.io/client-go/testing" ) -//go:generate mockgen --build_flags=--mod=mod -package sqlproxy -destination ./proxy_mocks_test.go github.com/rancher/steve/pkg/stores/sqlproxy Cache,ClientGetter,CacheFactory,SchemaColumnSetter,RelationshipNotifier +//go:generate mockgen --build_flags=--mod=mod -package sqlproxy -destination ./proxy_mocks_test.go github.com/rancher/steve/pkg/stores/sqlproxy Cache,ClientGetter,CacheFactory,SchemaColumnSetter,RelationshipNotifier,TransformBuilder //go:generate mockgen --build_flags=--mod=mod -package sqlproxy -destination ./sql_informer_mocks_test.go github.com/rancher/lasso/pkg/cache/sql/informer ByOptionsLister //go:generate mockgen --build_flags=--mod=mod -package sqlproxy -destination ./dynamic_mocks_test.go k8s.io/client-go/dynamic ResourceInterface @@ -82,9 +82,9 @@ func TestNewProxyStore(t *testing.T) { nsSchema := baseNSSchema scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) - cf.EXPECT().CacheFor([][]string{{"metadata", "labels[field.cattle.io/projectId]"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(c, nil) + cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(c, nil) - s, err := NewProxyStore(scc, cg, rn, cf) + s, err := NewProxyStore(scc, cg, rn, nil, cf) assert.Nil(t, err) assert.Equal(t, scc, s.columnSetter) assert.Equal(t, cg, s.clientGetter) @@ -105,7 +105,7 @@ func TestNewProxyStore(t *testing.T) { nsSchema := baseNSSchema scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(fmt.Errorf("error")) - s, err := NewProxyStore(scc, cg, rn, cf) + s, err := NewProxyStore(scc, cg, rn, nil, cf) assert.Nil(t, err) assert.Equal(t, scc, s.columnSetter) assert.Equal(t, cg, s.clientGetter) @@ -127,7 +127,7 @@ func TestNewProxyStore(t *testing.T) { scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(nil, fmt.Errorf("error")) - s, err := NewProxyStore(scc, cg, rn, cf) + s, err := NewProxyStore(scc, cg, rn, nil, cf) assert.Nil(t, err) assert.Equal(t, scc, s.columnSetter) assert.Equal(t, cg, s.clientGetter) @@ -149,9 +149,9 @@ func TestNewProxyStore(t *testing.T) { nsSchema := baseNSSchema scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) - cf.EXPECT().CacheFor([][]string{{"metadata", "labels[field.cattle.io/projectId]"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error")) + cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error")) - s, err := NewProxyStore(scc, cg, rn, cf) + s, err := NewProxyStore(scc, cg, rn, nil, cf) assert.Nil(t, err) assert.Equal(t, scc, s.columnSetter) assert.Equal(t, cg, s.clientGetter) @@ -181,6 +181,7 @@ func TestListByPartitions(t *testing.T) { cf := NewMockCacheFactory(gomock.NewController(t)) ri := NewMockResourceInterface(gomock.NewController(t)) bloi := NewMockByOptionsLister(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) inf := &informer.Informer{ ByOptionsLister: bloi, } @@ -188,9 +189,10 @@ func TestListByPartitions(t *testing.T) { ByOptionsLister: inf, } s := &Store{ - namespaceCache: nsi, - clientGetter: cg, - cacheFactory: cf, + namespaceCache: nsi, + clientGetter: cg, + cacheFactory: cf, + transformBuilder: tb, } var partitions []partition.Partition req := &types.APIRequest{ @@ -238,7 +240,8 @@ func TestListByPartitions(t *testing.T) { assert.Nil(t, err) cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil) // This tests that fields are being extracted from schema columns and the type specific fields map - cf.EXPECT().CacheFor([][]string{{"some", "field"}, {"gvk", "specific", "fields"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil) + cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil) + tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil) list, total, contToken, err := s.ListByPartitions(req, schema, partitions) assert.Nil(t, err) @@ -253,11 +256,13 @@ func TestListByPartitions(t *testing.T) { nsi := NewMockCache(gomock.NewController(t)) cg := NewMockClientGetter(gomock.NewController(t)) cf := NewMockCacheFactory(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) s := &Store{ - namespaceCache: nsi, - clientGetter: cg, - cacheFactory: cf, + namespaceCache: nsi, + clientGetter: cg, + cacheFactory: cf, + transformBuilder: tb, } var partitions []partition.Partition req := &types.APIRequest{ @@ -317,11 +322,13 @@ func TestListByPartitions(t *testing.T) { nsi := NewMockCache(gomock.NewController(t)) cg := NewMockClientGetter(gomock.NewController(t)) cf := NewMockCacheFactory(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) s := &Store{ - namespaceCache: nsi, - clientGetter: cg, - cacheFactory: cf, + namespaceCache: nsi, + clientGetter: cg, + cacheFactory: cf, + transformBuilder: tb, } var partitions []partition.Partition req := &types.APIRequest{ @@ -380,11 +387,13 @@ func TestListByPartitions(t *testing.T) { cg := NewMockClientGetter(gomock.NewController(t)) cf := NewMockCacheFactory(gomock.NewController(t)) ri := NewMockResourceInterface(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) s := &Store{ - namespaceCache: nsi, - clientGetter: cg, - cacheFactory: cf, + namespaceCache: nsi, + clientGetter: cg, + cacheFactory: cf, + transformBuilder: tb, } var partitions []partition.Partition req := &types.APIRequest{ @@ -432,7 +441,8 @@ func TestListByPartitions(t *testing.T) { assert.Nil(t, err) cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil) // This tests that fields are being extracted from schema columns and the type specific fields map - cf.EXPECT().CacheFor([][]string{{"some", "field"}, {"gvk", "specific", "fields"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(factory.Cache{}, fmt.Errorf("error")) + tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) + cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(factory.Cache{}, fmt.Errorf("error")) _, _, _, err = s.ListByPartitions(req, schema, partitions) assert.NotNil(t, err) @@ -447,6 +457,7 @@ func TestListByPartitions(t *testing.T) { cf := NewMockCacheFactory(gomock.NewController(t)) ri := NewMockResourceInterface(gomock.NewController(t)) bloi := NewMockByOptionsLister(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) inf := &informer.Informer{ ByOptionsLister: bloi, } @@ -454,9 +465,10 @@ func TestListByPartitions(t *testing.T) { ByOptionsLister: inf, } s := &Store{ - namespaceCache: nsi, - clientGetter: cg, - cacheFactory: cf, + namespaceCache: nsi, + clientGetter: cg, + cacheFactory: cf, + transformBuilder: tb, } var partitions []partition.Partition req := &types.APIRequest{ @@ -504,8 +516,9 @@ func TestListByPartitions(t *testing.T) { assert.Nil(t, err) cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil) // This tests that fields are being extracted from schema columns and the type specific fields map - cf.EXPECT().CacheFor([][]string{{"some", "field"}, {"gvk", "specific", "fields"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil) + cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil) bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(nil, 0, "", fmt.Errorf("error")) + tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) _, _, _, err = s.ListByPartitions(req, schema, partitions) assert.NotNil(t, err) @@ -531,19 +544,22 @@ func TestReset(t *testing.T) { cf := NewMockCacheFactory(gomock.NewController(t)) cs := NewMockSchemaColumnSetter(gomock.NewController(t)) ri := NewMockResourceInterface(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) nsc2 := factory.Cache{} s := &Store{ - namespaceCache: nsc, - clientGetter: cg, - cacheFactory: cf, - columnSetter: cs, - cfInitializer: func() (CacheFactory, error) { return cf, nil }, + namespaceCache: nsc, + clientGetter: cg, + cacheFactory: cf, + columnSetter: cs, + cfInitializer: func() (CacheFactory, error) { return cf, nil }, + transformBuilder: tb, } nsSchema := baseNSSchema cf.EXPECT().Reset().Return(nil) cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) - cf.EXPECT().CacheFor([][]string{{"metadata", "labels[field.cattle.io/projectId]"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(nsc2, nil) + cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(nsc2, nil) + tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) err := s.Reset() assert.Nil(t, err) assert.Equal(t, nsc2, s.namespaceCache) @@ -556,13 +572,15 @@ func TestReset(t *testing.T) { cg := NewMockClientGetter(gomock.NewController(t)) cf := NewMockCacheFactory(gomock.NewController(t)) cs := NewMockSchemaColumnSetter(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) s := &Store{ - namespaceCache: nsi, - clientGetter: cg, - cacheFactory: cf, - columnSetter: cs, - cfInitializer: func() (CacheFactory, error) { return cf, nil }, + namespaceCache: nsi, + clientGetter: cg, + cacheFactory: cf, + columnSetter: cs, + cfInitializer: func() (CacheFactory, error) { return cf, nil }, + transformBuilder: tb, } cf.EXPECT().Reset().Return(fmt.Errorf("error")) @@ -577,13 +595,15 @@ func TestReset(t *testing.T) { cg := NewMockClientGetter(gomock.NewController(t)) cf := NewMockCacheFactory(gomock.NewController(t)) cs := NewMockSchemaColumnSetter(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) s := &Store{ - namespaceCache: nsi, - clientGetter: cg, - cacheFactory: cf, - columnSetter: cs, - cfInitializer: func() (CacheFactory, error) { return cf, nil }, + namespaceCache: nsi, + clientGetter: cg, + cacheFactory: cf, + columnSetter: cs, + cfInitializer: func() (CacheFactory, error) { return cf, nil }, + transformBuilder: tb, } cf.EXPECT().Reset().Return(nil) @@ -599,13 +619,15 @@ func TestReset(t *testing.T) { cg := NewMockClientGetter(gomock.NewController(t)) cf := NewMockCacheFactory(gomock.NewController(t)) cs := NewMockSchemaColumnSetter(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) s := &Store{ - namespaceCache: nsi, - clientGetter: cg, - cacheFactory: cf, - columnSetter: cs, - cfInitializer: func() (CacheFactory, error) { return cf, nil }, + namespaceCache: nsi, + clientGetter: cg, + cacheFactory: cf, + columnSetter: cs, + cfInitializer: func() (CacheFactory, error) { return cf, nil }, + transformBuilder: tb, } nsSchema := baseNSSchema @@ -624,20 +646,23 @@ func TestReset(t *testing.T) { cf := NewMockCacheFactory(gomock.NewController(t)) cs := NewMockSchemaColumnSetter(gomock.NewController(t)) ri := NewMockResourceInterface(gomock.NewController(t)) + tb := NewMockTransformBuilder(gomock.NewController(t)) s := &Store{ - namespaceCache: nsc, - clientGetter: cg, - cacheFactory: cf, - columnSetter: cs, - cfInitializer: func() (CacheFactory, error) { return cf, nil }, + namespaceCache: nsc, + clientGetter: cg, + cacheFactory: cf, + columnSetter: cs, + cfInitializer: func() (CacheFactory, error) { return cf, nil }, + transformBuilder: tb, } nsSchema := baseNSSchema cf.EXPECT().Reset().Return(nil) cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil) cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil) - cf.EXPECT().CacheFor([][]string{{"metadata", "labels[field.cattle.io/projectId]"}}, &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error")) + cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error")) + tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil }) err := s.Reset() assert.NotNil(t, err) },