rancher/steve
1
0
Fork 0
mirror of https://github.com/rancher/steve.git synced 2025-09-01 15:37:31 +00:00
Code Issues Projects Releases Wiki Activity

[v2.10] SQL cache backports (#435)

Browse Source 
Co-authored-by: Tom Lebreux <tom.lebreux@suse.com>
This commit is contained in:
Silvio Moioli
2025-01-11 12:26:12 +01:00
committed by GitHub
parent a672f2f12a
commit c48ac64c4d
6 changed files with 158 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pkg/stores/sqlproxy/proxy_mocks_test.go
View File

@@ -263,18 +263,18 @@ func (m *MockCacheFactory) EXPECT() *MockCacheFactoryMockRecorder {
}
// CacheFor mocks base method.
func (m *MockCacheFactory) CacheFor(arg0 [][]string, arg1 cache.TransformFunc, arg2 dynamic.ResourceInterface, arg3 schema.GroupVersionKind, arg4 bool) (factory.Cache, error) {
func (m *MockCacheFactory) CacheFor(arg0 [][]string, arg1 cache.TransformFunc, arg2 dynamic.ResourceInterface, arg3 schema.GroupVersionKind, arg4, arg5 bool) (factory.Cache, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "CacheFor", arg0, arg1, arg2, arg3, arg4)
ret := m.ctrl.Call(m, "CacheFor", arg0, arg1, arg2, arg3, arg4, arg5)
ret0, _ := ret[0].(factory.Cache)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// CacheFor indicates an expected call of CacheFor.
func (mr *MockCacheFactoryMockRecorder) CacheFor(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call {
func (mr *MockCacheFactoryMockRecorder) CacheFor(arg0, arg1, arg2, arg3, arg4, arg5 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFor", reflect.TypeOf((*MockCacheFactory)(nil).CacheFor), arg0, arg1, arg2, arg3, arg4)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CacheFor", reflect.TypeOf((*MockCacheFactory)(nil).CacheFor), arg0, arg1, arg2, arg3, arg4, arg5)
}
// Reset mocks base method.

63
pkg/stores/sqlproxy/proxy_store.go
View File

@@ -41,6 +41,7 @@ import (
"github.com/rancher/wrangler/v3/pkg/summary"
"github.com/rancher/steve/pkg/attributes"
controllerschema "github.com/rancher/steve/pkg/controllers/schema"
"github.com/rancher/steve/pkg/resources/common"
"github.com/rancher/steve/pkg/resources/virtual"
virtualCommon "github.com/rancher/steve/pkg/resources/virtual/common"
@@ -59,6 +60,8 @@ var (
paramScheme = runtime.NewScheme()
paramCodec = runtime.NewParameterCodec(paramScheme)
typeSpecificIndexedFields = map[string][][]string{
gvkKey("", "v1", "ConfigMap"): {
{"metadata", "labels[harvesterhci.io/cloud-init-template]"}},
gvkKey("", "v1", "Event"): {
{"_type"},
{"involvedObject", "kind"},
@@ -70,11 +73,43 @@ var (
gvkKey("", "v1", "Node"): {
{"status", "nodeInfo", "kubeletVersion"},
{"status", "nodeInfo", "operatingSystem"}},
gvkKey("", "v1", "PersistentVolume"): {
{"status", "reason"},
{"spec", "persistentVolumeReclaimPolicy"},
},
gvkKey("", "v1", "PersistentVolumeClaim"): {
{"spec", "volumeName"}},
gvkKey("", "v1", "Pod"): {
{"spec", "containers", "image"},
{"spec", "nodeName"}},
gvkKey("", "v1", "ConfigMap"): {
{"metadata", "labels[harvesterhci.io/cloud-init-template]"}},
gvkKey("", "v1", "Service"): {
{"spec", "clusterIP"},
{"spec", "type"},
},
gvkKey("apps", "v1", "DaemonSet"): {
{"metadata", "annotations[field.cattle.io/publicEndpoints]"},
},
gvkKey("apps", "v1", "Deployment"): {
{"metadata", "annotations[field.cattle.io/publicEndpoints]"},
},
gvkKey("apps", "v1", "StatefulSet"): {
{"metadata", "annotations[field.cattle.io/publicEndpoints]"},
},
gvkKey("autoscaling", "v2", "HorizontalPodAutoscaler"): {
{"spec", "scaleTargetRef", "name"},
{"spec", "minReplicas"},
{"spec", "maxReplicas"},
{"status", "currentReplicas"},
},
gvkKey("batch", "v1", "CronJob"): {
{"metadata", "annotations[field.cattle.io/publicEndpoints]"},
},
gvkKey("batch", "v1", "Job"): {
{"metadata", "annotations[field.cattle.io/publicEndpoints]"},
},
gvkKey("catalog.cattle.io", "v1", "App"): {
{"spec", "chart", "metadata", "name"},
},
gvkKey("catalog.cattle.io", "v1", "ClusterRepo"): {
{"metadata", "annotations[clusterrepo.cattle.io/hidden]"},
{"spec", "gitBranch"},
@@ -87,18 +122,30 @@ var (
},
gvkKey("cluster.x-k8s.io", "v1beta1", "Machine"): {
{"spec", "clusterName"}},
gvkKey("management.cattle.io", "v3", "Cluster"): {
{"metadata", "labels[provider.cattle.io]"},
{"spec", "internal"},
{"spec", "displayName"},
{"status", "provider"},
},
gvkKey("management.cattle.io", "v3", "Node"): {
{"status", "nodeName"}},
gvkKey("management.cattle.io", "v3", "NodePool"): {
{"spec", "clusterName"}},
gvkKey("management.cattle.io", "v3", "NodeTemplate"): {
{"spec", "clusterName"}},
gvkKey("networking.k8s.io", "v1", "Ingress"): {
{"spec", "rules", "host"},
{"spec", "ingressClassName"},
},
gvkKey("provisioning.cattle.io", "v1", "Cluster"): {
{"metadata", "labels[provider.cattle.io]"},
{"status", "clusterName"},
{"status", "provider"},
{"status", "allocatable", "cpu"},
{"status", "allocatable", "memory"},
{"status", "allocatable", "pods"},
},
gvkKey("storage.k8s.io", "v1", "StorageClass"): {
{"provisioner"},
{"metadata", "annotations[storageclass.kubernetes.io/is-default-class]"},
},
}
commonIndexFields = [][]string{
@@ -184,7 +231,7 @@ type Store struct {
type CacheFactoryInitializer func() (CacheFactory, error)
type CacheFactory interface {
CacheFor(fields [][]string, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool) (factory.Cache, error)
CacheFor(fields [][]string, transform cache.TransformFunc, client dynamic.ResourceInterface, gvk schema.GroupVersionKind, namespaced bool, watchable bool) (factory.Cache, error)
Reset() error
}
@@ -262,7 +309,7 @@ func (s *Store) initializeNamespaceCache() error {
transformFunc := s.transformBuilder.GetTransformFunc(gvk)
// get the ns informer
nsInformer, err := s.cacheFactory.CacheFor(fields, transformFunc, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(&nsSchema), false)
nsInformer, err := s.cacheFactory.CacheFor(fields, transformFunc, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(&nsSchema), false, true)
if err != nil {
return err
}
@@ -702,7 +749,7 @@ func (s *Store) ListByPartitions(apiOp *types.APIRequest, schema *types.APISchem
fields = append(fields, getFieldForGVK(gvk)...)
transformFunc := s.transformBuilder.GetTransformFunc(gvk)
inf, err := s.cacheFactory.CacheFor(fields, transformFunc, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(schema), attributes.Namespaced(schema))
inf, err := s.cacheFactory.CacheFor(fields, transformFunc, &tablelistconvert.Client{ResourceInterface: client}, attributes.GVK(schema), attributes.Namespaced(schema), controllerschema.IsListWatchable(schema))
if err != nil {
return nil, 0, "", err
}

101
pkg/stores/sqlproxy/proxy_store_test.go
View File

@@ -82,7 +82,7 @@ func TestNewProxyStore(t *testing.T) {
nsSchema := baseNSSchema
scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil)
cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(c, nil)
cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(c, nil)
s, err := NewProxyStore(scc, cg, rn, nil, cf)
assert.Nil(t, err)
@@ -149,7 +149,7 @@ func TestNewProxyStore(t *testing.T) {
nsSchema := baseNSSchema
scc.EXPECT().SetColumns(context.Background(), &nsSchema).Return(nil)
cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error"))
cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(factory.Cache{}, fmt.Errorf("error"))
s, err := NewProxyStore(scc, cg, rn, nil, cf)
assert.Nil(t, err)
@@ -207,6 +207,7 @@ func TestListByPartitions(t *testing.T) {
Field: "some.field",
},
},
"verbs": []string{"list", "watch"},
}},
}
expectedItems := []unstructured.Unstructured{
@@ -240,7 +241,7 @@ func TestListByPartitions(t *testing.T) {
assert.Nil(t, err)
cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
// This tests that fields are being extracted from schema columns and the type specific fields map
cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil)
cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil)
tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil)
list, total, contToken, err := s.ListByPartitions(req, schema, partitions)
@@ -277,6 +278,7 @@ func TestListByPartitions(t *testing.T) {
Field: "some.field",
},
},
"verbs": []string{"list", "watch"},
}},
}
expectedItems := []unstructured.Unstructured{
@@ -343,6 +345,7 @@ func TestListByPartitions(t *testing.T) {
Field: "some.field",
},
},
"verbs": []string{"list", "watch"},
}},
}
expectedItems := []unstructured.Unstructured{
@@ -380,6 +383,88 @@ func TestListByPartitions(t *testing.T) {
assert.NotNil(t, err)
},
})
tests = append(tests, testCase{
description: "client ListByPartitions() should detect listable-but-unwatchable schema, still work normally",
test: func(t *testing.T) {
nsi := NewMockCache(gomock.NewController(t))
cg := NewMockClientGetter(gomock.NewController(t))
cf := NewMockCacheFactory(gomock.NewController(t))
ri := NewMockResourceInterface(gomock.NewController(t))
bloi := NewMockByOptionsLister(gomock.NewController(t))
tb := NewMockTransformBuilder(gomock.NewController(t))
inf := &informer.Informer{
ByOptionsLister: bloi,
}
c := factory.Cache{
ByOptionsLister: inf,
}
s := &Store{
namespaceCache: nsi,
clientGetter: cg,
cacheFactory: cf,
transformBuilder: tb,
}
var partitions []partition.Partition
req := &types.APIRequest{
Request: &http.Request{
URL: &url.URL{},
},
}
schema := &types.APISchema{
Schema: &schemas.Schema{Attributes: map[string]interface{}{
"columns": []common.ColumnDefinition{
{
Field: "some.field",
},
},
// note: no watch here
"verbs": []string{"list"},
}},
}
expectedItems := []unstructured.Unstructured{
{
Object: map[string]interface{}{
"kind": "apple",
"metadata": map[string]interface{}{
"name": "fuji",
},
"data": map[string]interface{}{
"color": "pink",
},
},
},
}
listToReturn := &unstructured.UnstructuredList{
Items: make([]unstructured.Unstructured, len(expectedItems), len(expectedItems)),
}
gvk := schema2.GroupVersionKind{
Group: "some",
Version: "test",
Kind: "gvk",
}
typeSpecificIndexedFields["some_test_gvk"] = [][]string{{"gvk", "specific", "fields"}}
attributes.SetGVK(schema, gvk)
// ListByPartitions copies point so we need some original record of items to ensure as asserting listToReturn's
// items is equal to the list returned by ListByParititons doesn't ensure no mutation happened
copy(listToReturn.Items, expectedItems)
opts, err := listprocessor.ParseQuery(req, nil)
assert.Nil(t, err)
cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
// This tests that fields are being extracted from schema columns and the type specific fields map
// note also the watchable bool is expected to be false
cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), false).Return(c, nil)
tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil)
list, total, contToken, err := s.ListByPartitions(req, schema, partitions)
assert.Nil(t, err)
assert.Equal(t, expectedItems, list)
assert.Equal(t, len(expectedItems), total)
assert.Equal(t, "", contToken)
},
})
tests = append(tests, testCase{
description: "client ListByPartitions() with CacheFor() error returned should returned an errors. Should pass fields",
test: func(t *testing.T) {
@@ -408,6 +493,7 @@ func TestListByPartitions(t *testing.T) {
Field: "some.field",
},
},
"verbs": []string{"list", "watch"},
}},
}
expectedItems := []unstructured.Unstructured{
@@ -442,7 +528,7 @@ func TestListByPartitions(t *testing.T) {
cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
// This tests that fields are being extracted from schema columns and the type specific fields map
tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(factory.Cache{}, fmt.Errorf("error"))
cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(factory.Cache{}, fmt.Errorf("error"))
_, _, _, err = s.ListByPartitions(req, schema, partitions)
assert.NotNil(t, err)
@@ -483,6 +569,7 @@ func TestListByPartitions(t *testing.T) {
Field: "some.field",
},
},
"verbs": []string{"list", "watch"},
}},
}
expectedItems := []unstructured.Unstructured{
@@ -516,7 +603,7 @@ func TestListByPartitions(t *testing.T) {
assert.Nil(t, err)
cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
// This tests that fields are being extracted from schema columns and the type specific fields map
cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema)).Return(c, nil)
cf.EXPECT().CacheFor([][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil)
bloi.EXPECT().ListByOptions(req.Context(), opts, partitions, req.Namespace).Return(nil, 0, "", fmt.Errorf("error"))
tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
@@ -558,7 +645,7 @@ func TestReset(t *testing.T) {
cf.EXPECT().Reset().Return(nil)
cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil)
cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(nsc2, nil)
cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(nsc2, nil)
tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
err := s.Reset()
assert.Nil(t, err)
@@ -661,7 +748,7 @@ func TestReset(t *testing.T) {
cf.EXPECT().Reset().Return(nil)
cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil)
cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false).Return(factory.Cache{}, fmt.Errorf("error"))
cf.EXPECT().CacheFor([][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels[field.cattle.io/projectId]"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(factory.Cache{}, fmt.Errorf("error"))
tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
err := s.Reset()
assert.NotNil(t, err)