1
0
mirror of https://github.com/rancher/steve.git synced 2025-08-30 21:50:05 +00:00

[v2.9] Backport PR 271: Index more sqlite cache fields. (#315)

* [v2.9] Backport PR 271: Index more sqlite cache fields.

* go mod tidy

Signed-off-by: Silvio Moioli <silvio@moioli.net>

---------

Signed-off-by: Silvio Moioli <silvio@moioli.net>
Co-authored-by: Silvio Moioli <silvio@moioli.net>
This commit is contained in:
Eric Promislow 2024-10-31 10:48:05 -07:00 committed by GitHub
parent ddd2e373b7
commit c4ebbe629f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 433 additions and 49 deletions

View File

@ -34,7 +34,7 @@ func main() {
func run(_ *cli.Context) error {
ctx := signals.SetupSignalContext()
debugconfig.MustSetupDebug()
s, err := config.ToServer(ctx, false)
s, err := config.ToServer(ctx, debugconfig.SQLCache)
if err != nil {
return err
}

View File

@ -13,6 +13,7 @@ import (
type Config struct {
Debug bool
DebugLevel int
SQLCache bool
}
func (c *Config) MustSetupDebug() {
@ -54,6 +55,10 @@ func Flags(config *Config) []cli.Flag {
Value: 7,
Destination: &config.DebugLevel,
},
cli.BoolFlag{
Name: "sql-cache",
Destination: &config.SQLCache,
},
}
}
@ -68,5 +73,9 @@ func FlagsV2(config *Config) []cliv2.Flag {
Value: 7,
Destination: &config.DebugLevel,
},
&cliv2.BoolFlag{
Name: "sql-cache",
Destination: &config.SQLCache,
},
}
}

View File

@ -10,7 +10,6 @@ import (
wranglerSummary "github.com/rancher/wrangler/v3/pkg/summary"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
)
// SummaryCache provides an interface to get a summary/relationships for an object. Implemented by the summaryCache
@ -24,26 +23,14 @@ type DefaultFields struct {
Cache SummaryCache
}
// GetTransform produces the default transformation func
func (d *DefaultFields) GetTransform() cache.TransformFunc {
return d.transform
}
// transform implements virtual.VirtualTransformFunc, and adds reserved fields/summary
func (d *DefaultFields) transform(obj any) (any, error) {
raw, isSignal, err := getUnstructured(obj)
if isSignal {
return obj, nil
}
if err != nil {
return nil, err
}
raw = addIDField(raw)
raw, err = addSummaryFields(raw, d.Cache)
// TransformCommon implements virtual.VirtualTransformFunc, and adds reserved fields/summary
func (d *DefaultFields) TransformCommon(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
obj = addIDField(obj)
obj, err := addSummaryFields(obj, d.Cache)
if err != nil {
return nil, fmt.Errorf("unable to add summary fields: %w", err)
}
return raw, nil
return obj, nil
}
// addSummaryFields adds the virtual fields for object state.

View File

@ -9,11 +9,10 @@ import (
"github.com/stretchr/testify/require"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
)
func TestTransform(t *testing.T) {
func TestTransformCommonObjects(t *testing.T) {
tests := []struct {
name string
input any
@ -160,29 +159,29 @@ func TestTransform(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeCache := fakeSummaryCache{
summarizedObject: test.hasSummary,
relationships: test.hasRelationships,
fakeCache := common.FakeSummaryCache{
SummarizedObject: test.hasSummary,
Relationships: test.hasRelationships,
}
df := common.DefaultFields{
Cache: &fakeCache,
}
output, err := df.GetTransform()(test.input)
require.Equal(t, test.wantOutput, output)
raw, isSignal, err := common.GetUnstructured(test.input)
if err != nil {
require.True(t, test.wantError)
return
}
if isSignal {
require.Equal(t, test.input, test.wantOutput)
return
}
output, err := df.TransformCommon(raw)
if test.wantError {
require.Error(t, err)
} else {
require.Equal(t, test.wantOutput, output)
require.NoError(t, err)
}
})
}
}
type fakeSummaryCache struct {
summarizedObject *summary.SummarizedObject
relationships []summarycache.Relationship
}
func (f *fakeSummaryCache) SummaryAndRelationship(runtime.Object) (*summary.SummarizedObject, []summarycache.Relationship) {
return f.summarizedObject, f.relationships
}

View File

@ -0,0 +1,16 @@
package common
import (
"github.com/rancher/steve/pkg/summarycache"
"github.com/rancher/wrangler/v3/pkg/summary"
"k8s.io/apimachinery/pkg/runtime"
)
type FakeSummaryCache struct {
SummarizedObject *summary.SummarizedObject
Relationships []summarycache.Relationship
}
func (f *FakeSummaryCache) SummaryAndRelationship(runtime.Object) (*summary.SummarizedObject, []summarycache.Relationship) {
return f.SummarizedObject, f.Relationships
}

View File

@ -11,7 +11,7 @@ import (
// GetUnstructured retrieves an unstructured object from the provided input. If this is a signal
// object (like cache.DeletedFinalStateUnknown), returns true, indicating that this wasn't an
// unstructured object, but doesn't need to be processed by our transform function
func getUnstructured(obj any) (*unstructured.Unstructured, bool, error) {
func GetUnstructured(obj any) (*unstructured.Unstructured, bool, error) {
raw, ok := obj.(*unstructured.Unstructured)
if !ok {
_, isFinalUnknown := obj.(cache.DeletedFinalStateUnknown)

View File

@ -0,0 +1,16 @@
// Package common provides cache.TransformFunc's for /v1 Event objects
package events
import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
// TransformEventObject does special-case handling on event objects
// 1. (only one so far): replaces the _type field with the contents of the field named "type", if it exists
func TransformEventObject(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
currentTypeValue, ok := obj.Object["type"]
if ok {
obj.Object["_type"] = currentTypeValue
}
return obj, nil
}

View File

@ -0,0 +1,93 @@
package events_test
import (
"testing"
"github.com/rancher/steve/pkg/resources/virtual/events"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func TestTransformEvents(t *testing.T) {
tests := []struct {
name string
input any
wantOutput any
wantError bool
}{
{
name: "fix event fields",
input: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "/v1",
"kind": "Event",
"metadata": map[string]interface{}{
"name": "gregsFarm",
"namespace": "gregsNamespace",
},
"id": "eventTest1id",
"type": "Gorniplatz",
},
},
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "/v1",
"kind": "Event",
"metadata": map[string]interface{}{
"name": "gregsFarm",
"namespace": "gregsNamespace",
},
"id": "eventTest1id",
"type": "Gorniplatz",
"_type": "Gorniplatz",
},
},
},
{
name: "don't fix non-default-group event fields",
input: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "palau.io/v1",
"kind": "Event",
"metadata": map[string]interface{}{
"name": "gregsFarm",
"namespace": "gregsNamespace",
},
"id": "eventTest1id",
"type": "Gorniplatz",
},
},
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "palau.io/v1",
"kind": "Event",
"metadata": map[string]interface{}{
"name": "gregsFarm",
"namespace": "gregsNamespace",
},
"id": "eventTest1id",
"type": "Gorniplatz",
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var output interface{}
var err error
raw, ok := test.input.(*unstructured.Unstructured)
if ok && raw.GetKind() == "Event" && raw.GetAPIVersion() == "/v1" {
output, err = events.TransformEventObject(raw)
} else {
output = raw
err = nil
}
require.Equal(t, test.wantOutput, output)
if test.wantError {
require.Error(t, err)
} else {
require.NoError(t, err)
}
})
}
}

View File

@ -3,7 +3,11 @@
package virtual
import (
"fmt"
"github.com/rancher/steve/pkg/resources/virtual/common"
"github.com/rancher/steve/pkg/resources/virtual/events"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/cache"
)
@ -16,13 +20,36 @@ type TransformBuilder struct {
// NewTransformBuilder returns a TransformBuilder using the given summary cache
func NewTransformBuilder(cache common.SummaryCache) *TransformBuilder {
return &TransformBuilder{
&common.DefaultFields{
defaultFields: &common.DefaultFields{
Cache: cache,
},
}
}
// GetTransformFunc retrieves a TransformFunc for a given GVK. Currently only returns a transformFunc for defaultFields
func (t *TransformBuilder) GetTransformFunc(_ schema.GroupVersionKind) cache.TransformFunc {
return t.defaultFields.GetTransform()
// GetTransformFunc returns the func to transform a raw object into a fixed object, if needed
func (t *TransformBuilder) GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc {
converters := make([]func(*unstructured.Unstructured) (*unstructured.Unstructured, error), 0)
if gvk.Kind == "Event" && gvk.Group == "" && gvk.Version == "v1" {
converters = append(converters, events.TransformEventObject)
}
converters = append(converters, t.defaultFields.TransformCommon)
return func(raw interface{}) (interface{}, error) {
obj, isSignal, err := common.GetUnstructured(raw)
if isSignal {
// isSignal= true overrides any error
return raw, err
}
if err != nil {
return nil, fmt.Errorf("GetUnstructured: failed to get underlying object: %w", err)
}
// Conversions are run in this loop:
for _, f := range converters {
obj, err = f(obj)
if err != nil {
return nil, err
}
}
return obj, nil
}
}

View File

@ -0,0 +1,202 @@
package virtual_test
import (
"github.com/rancher/steve/pkg/resources/virtual"
"k8s.io/apimachinery/pkg/runtime/schema"
"strings"
"testing"
"github.com/rancher/steve/pkg/resources/virtual/common"
"github.com/rancher/steve/pkg/summarycache"
"github.com/rancher/wrangler/v3/pkg/summary"
"github.com/stretchr/testify/require"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func TestTransformChain(t *testing.T) {
tests := []struct {
name string
input any
hasSummary *summary.SummarizedObject
hasRelationships []summarycache.Relationship
wantOutput any
wantError bool
}{
{
name: "add summary + relationships + reserved fields",
hasSummary: &summary.SummarizedObject{
PartialObjectMetadata: v1.PartialObjectMetadata{
ObjectMeta: v1.ObjectMeta{
Name: "testobj",
Namespace: "test-ns",
},
TypeMeta: v1.TypeMeta{
APIVersion: "test.cattle.io/v1",
Kind: "TestResource",
},
},
Summary: summary.Summary{
State: "success",
Transitioning: false,
Error: false,
Message: []string{"resource 1 rolled out", "resource 2 rolled out"},
},
},
hasRelationships: []summarycache.Relationship{
{
ToID: "1345",
ToType: "SomeType",
ToNamespace: "some-ns",
FromID: "78901",
FromType: "TestResource",
Rel: "uses",
},
},
input: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "test.cattle.io/v1",
"kind": "TestResource",
"metadata": map[string]interface{}{
"name": "testobj",
"namespace": "test-ns",
},
"id": "old-id",
},
},
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "test.cattle.io/v1",
"kind": "TestResource",
"metadata": map[string]interface{}{
"name": "testobj",
"namespace": "test-ns",
"state": map[string]interface{}{
"name": "success",
"error": false,
"transitioning": false,
"message": "resource 1 rolled out:resource 2 rolled out",
},
"relationships": []any{
map[string]any{
"toId": "1345",
"toType": "SomeType",
"toNamespace": "some-ns",
"fromId": "78901",
"fromType": "TestResource",
"rel": "uses",
},
},
},
"id": "test-ns/testobj",
"_id": "old-id",
},
},
},
{
name: "processable event",
input: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "/v1",
"kind": "Event",
"metadata": map[string]interface{}{
"name": "oswaldsFarm",
"namespace": "oswaldsNamespace",
},
"status": map[string]interface{}{
"conditions": []interface{}{
map[string]interface{}{
"status": "False",
"reason": "Error",
"message": "some error",
"lastTransitionTime": "2024-01-01",
},
},
},
"id": "eventTest2id",
"type": "Gorniplatz",
},
},
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "/v1",
"kind": "Event",
"metadata": map[string]interface{}{
"name": "oswaldsFarm",
"namespace": "oswaldsNamespace",
"relationships": []any(nil),
},
"status": map[string]interface{}{
"conditions": []interface{}{
map[string]interface{}{
"status": "False",
"reason": "Error",
"transitioning": false,
"error": true,
"message": "some error",
"lastTransitionTime": "2024-01-01",
"lastUpdateTime": "2024-01-01",
},
},
},
"id": "oswaldsNamespace/oswaldsFarm",
"_id": "eventTest2id",
"type": "Gorniplatz",
"_type": "Gorniplatz",
},
},
},
{
name: "don't fix non-default-group event fields",
input: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "palau.io/v1",
"kind": "Event",
"metadata": map[string]interface{}{
"name": "gregsFarm",
"namespace": "gregsNamespace",
"relationships": []any(nil),
},
"id": "eventTest1id",
"type": "Gorniplatz",
},
},
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "palau.io/v1",
"kind": "Event",
"metadata": map[string]interface{}{
"name": "gregsFarm",
"namespace": "gregsNamespace",
"relationships": []any(nil),
},
"id": "gregsNamespace/gregsFarm",
"_id": "eventTest1id",
"type": "Gorniplatz",
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeCache := common.FakeSummaryCache{
SummarizedObject: test.hasSummary,
Relationships: test.hasRelationships,
}
tb := virtual.NewTransformBuilder(&fakeCache)
raw, isSignal, err := common.GetUnstructured(test.input)
require.False(t, isSignal)
require.Nil(t, err)
apiVersion := raw.GetAPIVersion()
parts := strings.Split(apiVersion, "/")
gvk := schema.GroupVersionKind{Group: parts[0], Version: parts[1], Kind: raw.GetKind()}
output, err := tb.GetTransformFunc(gvk)(test.input)
require.Equal(t, test.wantOutput, output)
if test.wantError {
require.Error(t, err)
} else {
require.NoError(t, err)
}
})
}
}

View File

@ -61,12 +61,47 @@ var (
paramScheme = runtime.NewScheme()
paramCodec = runtime.NewParameterCodec(paramScheme)
typeSpecificIndexedFields = map[string][][]string{
"_v1_Namespace": {{`metadata`, `labels[field.cattle.io/projectId]`}},
"_v1_Node": {{`status`, `nodeInfo`, `kubeletVersion`}, {`status`, `nodeInfo`, `operatingSystem`}},
"_v1_Pod": {{`spec`, `containers`, `image`}, {`spec`, `nodeName`}},
"_v1_ConfigMap": {{`metadata`, `labels[harvesterhci.io/cloud-init-template]`}},
"management.cattle.io_v3_Node": {{`status`, `nodeName`}},
gvkKey("", "v1", "Event"): {
{"_type"},
{"involvedObject", "kind"},
{"message"},
{"reason"},
},
gvkKey("", "v1", "Namespace"): {
{"metadata", "labels[field.cattle.io/projectId]"}},
gvkKey("", "v1", "Node"): {
{"status", "nodeInfo", "kubeletVersion"},
{"status", "nodeInfo", "operatingSystem"}},
gvkKey("", "v1", "Pod"): {
{"spec", "containers", "image"},
{"spec", "nodeName"}},
gvkKey("", "v1", "ConfigMap"): {
{"metadata", "labels[harvesterhci.io/cloud-init-template]"}},
gvkKey("catalog.cattle.io", "v1", "ClusterRepo"): {
{"metadata", "annotations[clusterrepo.cattle.io/hidden]"},
{"spec", "gitBranch"},
{"spec", "gitRepo"},
},
gvkKey("catalog.cattle.io", "v1", "Operation"): {
{"status", "action"},
{"status", "namespace"},
{"status", "releaseName"},
},
gvkKey("cluster.x-k8s.io", "v1beta1", "Machine"): {
{"spec", "clusterName"}},
gvkKey("management.cattle.io", "v3", "Node"): {
{"status", "nodeName"}},
gvkKey("management.cattle.io", "v3", "NodePool"): {
{"spec", "clusterName"}},
gvkKey("management.cattle.io", "v3", "NodeTemplate"): {
{"spec", "clusterName"}},
gvkKey("provisioning.cattle.io", "v1", "Cluster"): {
{"metadata", "labels[provider.cattle.io]"},
{"status", "provider"},
{"status", "allocatable", "cpu"},
{"status", "allocatable", "memory"},
{"status", "allocatable", "pods"},
},
}
commonIndexFields = [][]string{
{`id`},
@ -241,15 +276,15 @@ func (s *Store) initializeNamespaceCache() error {
func getFieldForGVK(gvk schema.GroupVersionKind) [][]string {
fields := [][]string{}
fields = append(fields, commonIndexFields...)
typeFields := typeSpecificIndexedFields[keyFromGVK(gvk)]
typeFields := typeSpecificIndexedFields[gvkKey(gvk.Group, gvk.Version, gvk.Kind)]
if typeFields != nil {
fields = append(fields, typeFields...)
}
return fields
}
func keyFromGVK(gvk schema.GroupVersionKind) string {
return gvk.Group + "_" + gvk.Version + "_" + gvk.Kind
func gvkKey(group, version, kind string) string {
return group + "_" + version + "_" + kind
}
// getFieldsFromSchema converts object field names from types.APISchema's format into lasso's