rancher/steve
1
0
Fork 0
mirror of https://github.com/rancher/steve.git synced 2025-07-15 15:42:13 +00:00
Code Issues Projects Releases Wiki Activity

#48673 - Added Timestamp Cache Handling to metadata.fields (#648)

Browse Source 
* added timestamp convertion to metadata.fields

* fixed duration parsing

* fixed tests

* removed tags file

* added comments

* added better error handling

* changed ParseHumanDuration to use Fscanf

* added builtins handling

* adding mock updates

* fixing tests

* another try

* added timestamp convertion to metadata.fields

* addressing comments from @ericpromislow

* converting error to warning

* added template options
This commit is contained in:
Felipe Gehrke 2025-06-16 19:33:28 -03:00 committed by GitHub
parent 2e8a0f2851
commit b3539616e0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 513 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
pkg/resources/common/duration.go Normal file
View File

@ -0,0 +1,35 @@
package common
import (
"fmt"
"strings"
"time"
)
func ParseHumanReadableDuration(s string) (time.Duration, error) {
var total time.Duration
var val int
var unit byte
r := strings.NewReader(s)
for r.Len() > 0 {
if _, err := fmt.Fscanf(r, "%d%c", &val, &unit); err != nil {
return 0, fmt.Errorf("invalid duration in %s: %w", s, err)
}
switch unit {
case 'd':
total += time.Duration(val) * 24 * time.Hour
case 'h':
total += time.Duration(val) * time.Hour
case 'm':
total += time.Duration(val) * time.Minute
case 's':
total += time.Duration(val) * time.Second
default:
return 0, fmt.Errorf("invalid duration unit %s in %s", string(unit), s)
}
}
return total, nil
}

74
pkg/resources/common/duration_test.go Normal file
View File

@ -0,0 +1,74 @@
package common
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestParseHumanDuration(t *testing.T) {
testCases := []struct {
name string
input string
expected time.Duration
expectedErr bool
}{
{
name: "days + hours + mins + secs",
input: "1d23h45m56s",
expected: 24*time.Hour + 23*time.Hour + 45*time.Minute + 56*time.Second,
},
{
name: "hours + mins + secs",
input: "12h34m56s",
expected: 12*time.Hour + 34*time.Minute + 56*time.Second,
},
{
name: "days + hours",
input: "1d2h",
expected: 24*time.Hour + 2*time.Hour,
},
{
name: "hours + secs",
input: "1d2s",
expected: 24*time.Hour + 2*time.Second,
},
{
name: "mins + secs",
input: "1d2m",
expected: 24*time.Hour + 2*time.Minute,
},
{
name: "hours",
input: "1h",
expected: 1 * time.Hour,
},
{
name: "mins",
input: "1m",
expected: 1 * time.Minute,
},
{
name: "secs",
input: "0s",
expected: 0 * time.Second,
},
{
name: "invalid input",
input: "<invalid>",
expectedErr: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
output, err := ParseHumanReadableDuration(tc.input)
if tc.expectedErr {
assert.Error(t, err)
} else {
assert.Equal(t, tc.expected, output)
}
})
}
}

77
pkg/resources/common/formatter.go
View File

@ -2,12 +2,17 @@ package common
import (
"net/http"
"slices"
"strconv"
"strings"
"time"
"github.com/rancher/apiserver/pkg/types"
"github.com/rancher/steve/pkg/accesscontrol"
"github.com/rancher/steve/pkg/attributes"
"github.com/rancher/steve/pkg/resources/virtual/common"
"github.com/sirupsen/logrus"
"github.com/rancher/steve/pkg/schema"
metricsStore "github.com/rancher/steve/pkg/stores/metrics"
"github.com/rancher/steve/pkg/stores/proxy"
@ -19,25 +24,32 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
schema2 "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/duration"
)
type TemplateOptions struct {
InSQLMode bool
}
func DefaultTemplate(clientGetter proxy.ClientGetter,
summaryCache *summarycache.SummaryCache,
asl accesscontrol.AccessSetLookup,
namespaceCache corecontrollers.NamespaceCache) schema.Template {
namespaceCache corecontrollers.NamespaceCache,
options TemplateOptions) schema.Template {
return schema.Template{
Store: metricsStore.NewMetricsStore(proxy.NewProxyStore(clientGetter, summaryCache, asl, namespaceCache)),
Formatter: formatter(summaryCache, asl),
Formatter: formatter(summaryCache, asl, options),
}
}
// DefaultTemplateForStore provides a default schema template which uses a provided, pre-initialized store. Primarily used when creating a Template that uses a Lasso SQL store internally.
func DefaultTemplateForStore(store types.Store,
summaryCache *summarycache.SummaryCache,
asl accesscontrol.AccessSetLookup) schema.Template {
asl accesscontrol.AccessSetLookup,
options TemplateOptions) schema.Template {
return schema.Template{
Store: store,
Formatter: formatter(summaryCache, asl),
Formatter: formatter(summaryCache, asl, options),
}
}
@ -83,7 +95,7 @@ func buildBasePath(gvr schema2.GroupVersionResource, namespace string, includeNa
return buf.String()
}
func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLookup) types.Formatter {
func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLookup, options TemplateOptions) types.Formatter {
return func(request *types.APIRequest, resource *types.RawResource) {
if resource.Schema == nil {
return
@ -140,6 +152,7 @@ func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLook
delete(resource.Links, "patch")
}
gvk := attributes.GVK(resource.Schema)
if unstr, ok := resource.APIObject.Object.(*unstructured.Unstructured); ok {
// with the sql cache, these were already added by the indexer. However, the sql cache
// is only used for lists, so we need to re-add here for get/watch
@ -157,6 +170,10 @@ func formatter(summarycache common.SummaryCache, asl accesscontrol.AccessSetLook
includeFields(request, unstr)
excludeFields(request, unstr)
excludeValues(request, unstr)
if options.InSQLMode {
convertMetadataTimestampFields(request, gvk, unstr)
}
}
if permsQuery := request.Query.Get("checkPermissions"); permsQuery != "" {
@ -212,6 +229,56 @@ func excludeFields(request *types.APIRequest, unstr *unstructured.Unstructured)
}
}
// convertMetadataTimestampFields updates metadata timestamp fields to ensure they remain fresh and human-readable when sent back
// to the client. Internally, fields are stored as Unix timestamps; on each request, we calculate the elapsed time since
// those timestamps by subtracting them from time.Now(), then format the resulting duration into a human-friendly string.
// This prevents cached durations (e.g. “2d” - 2 days) from becoming stale over time.
func convertMetadataTimestampFields(request *types.APIRequest, gvk schema2.GroupVersionKind, unstr *unstructured.Unstructured) {
if request.Schema != nil {
cols := GetColumnDefinitions(request.Schema)
for _, col := range cols {
gvkDateFields, gvkFound := DateFieldsByGVKBuiltins[gvk]
if col.Type == "date" || (gvkFound && slices.Contains(gvkDateFields, col.Name)) {
index := GetIndexValueFromString(col.Field)
if index == -1 {
logrus.Errorf("field index not found at column.Field struct variable: %s", col.Field)
return
}
curValue, got, err := unstructured.NestedSlice(unstr.Object, "metadata", "fields")
if err != nil {
logrus.Errorf("failed to get metadata.fields slice from unstr.Object: %s", err.Error())
}
if !got {
logrus.Debugf("couldn't find metadata.fields at unstr.Object")
return
}
timeValue, ok := curValue[index].(string)
if !ok {
logrus.Debugf("time field isn't a string")
return
}
millis, err := strconv.ParseInt(timeValue, 10, 64)
if err != nil {
logrus.Warnf("failed to convert timestamp value: %s", err.Error())
return
}
timestamp := time.Unix(0, millis*int64(time.Millisecond))
dur := time.Since(timestamp)
curValue[index] = duration.HumanDuration(dur)
if err := unstructured.SetNestedSlice(unstr.Object, curValue, "metadata", "fields"); err != nil {
logrus.Errorf("failed to set value back to metadata.fields slice: %s", err.Error())
return
}
}
}
}
}
func excludeValues(request *types.APIRequest, unstr *unstructured.Unstructured) {
if values, ok := request.Query["excludeValues"]; ok {
for _, f := range values {

4
pkg/resources/common/formatter_test.go
View File

@ -1068,7 +1068,7 @@ func Test_formatterLinks(t *testing.T) {
APIObject: test.apiObject,
Links: test.currentLinks,
}
fmtter := formatter(nil, asl)
fmtter := formatter(nil, asl, TemplateOptions{InSQLMode: false})
fmtter(request, resource)
require.Equal(t, test.wantLinks, resource.Links)
@ -1269,7 +1269,7 @@ func TestFormatterAddsResourcePermissions(t *testing.T) {
asl.EXPECT().AccessFor(&defaultUserInfo).Return(&accessSet).AnyTimes()
formatter := formatter(fakeCache, asl)
formatter := formatter(fakeCache, asl, TemplateOptions{InSQLMode: false})
formatter(req, resource)
// Extract the resultant resourcePermissions

78
pkg/resources/common/gvkdatefields.go Normal file
View File

@ -0,0 +1,78 @@
package common
import (
"k8s.io/apimachinery/pkg/runtime/schema"
)
var DateFieldsByGVKBuiltins = map[schema.GroupVersionKind][]string{
{Group: "", Version: "v1", Kind: "ConfigMap"}: {"Age"},
{Group: "", Version: "v1", Kind: "Endpoints"}: {"Age"},
{Group: "", Version: "v1", Kind: "Event"}: {"Last Seen", "First Seen"},
{Group: "", Version: "v1", Kind: "Namespace"}: {"Age"},
{Group: "", Version: "v1", Kind: "Node"}: {"Age"},
{Group: "", Version: "v1", Kind: "PersistentVolume"}: {"Age"},
{Group: "", Version: "v1", Kind: "PersistentVolumeClaim"}: {"Age"},
{Group: "", Version: "v1", Kind: "Pod"}: {"Age"},
{Group: "", Version: "v1", Kind: "ReplicationController"}: {"Age"},
{Group: "", Version: "v1", Kind: "ResourceQuota"}: {"Age"},
{Group: "", Version: "v1", Kind: "Secret"}: {"Age"},
{Group: "", Version: "v1", Kind: "Service"}: {"Age"},
{Group: "", Version: "v1", Kind: "ServiceAccount"}: {"Age"},
{Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "MutatingAdmissionPolicy"}: {"Age"},
{Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "MutatingAdmissionPolicyBinding"}: {"Age"},
{Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "ValidatingAdmissionPolicy"}: {"Age"},
{Group: "admissionregistration.k8s.io", Version: "v1alpha2", Kind: "ValidatingAdmissionPolicyBinding"}: {"Age"},
{Group: "admissionregistration.k8s.io", Version: "v1beta1", Kind: "MutatingWebhookConfiguration"}: {"Age"},
{Group: "admissionregistration.k8s.io", Version: "v1beta1", Kind: "ValidatingWebhookConfiguration"}: {"Age"},
{Group: "apiserverinternal.k8s.io", Version: "v1alpha1", Kind: "StorageVersion"}: {"Age"},
{Group: "apps", Version: "v1", Kind: "DaemonSet"}: {"Age"},
{Group: "apps", Version: "v1", Kind: "Deployment"}: {"Age"},
{Group: "apps", Version: "v1", Kind: "ReplicaSet"}: {"Age"},
{Group: "apps", Version: "v1", Kind: "StatefulSet"}: {"Age"},
{Group: "apps", Version: "v1beta1", Kind: "ControllerRevision"}: {"Age"},
{Group: "autoscaling", Version: "v1", Kind: "Scale"}: {"Age"},
{Group: "autoscaling", Version: "v2beta1", Kind: "HorizontalPodAutoscaler"}: {"Age"},
{Group: "batch", Version: "v1", Kind: "Job"}: {"Duration", "Age"},
{Group: "batch", Version: "v1beta1", Kind: "CronJob"}: {"Last Schedule", "Age"},
{Group: "certificates.k8s.io", Version: "v1beta1", Kind: "CertificateSigningRequest"}: {"Age"},
{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}: {"Age"},
{Group: "coordination.k8s.io", Version: "v1alpha2", Kind: "LeaseCandidate"}: {"Age"},
{Group: "discovery.k8s.io", Version: "v1beta1", Kind: "EndpointSlice"}: {"Age"},
{Group: "flowcontrol.k8s.io", Version: "v1", Kind: "FlowSchema"}: {"Age"},
{Group: "flowcontrol.k8s.io", Version: "v1", Kind: "PriorityLevelConfiguration"}: {"Age"},
{Group: "networking.k8s.io", Version: "v1beta1", Kind: "Ingress"}: {"Age"},
{Group: "networking.k8s.io", Version: "v1beta1", Kind: "IngressClass"}: {"Age"},
{Group: "networking.k8s.io", Version: "v1beta1", Kind: "IPAddress"}: {"Age"},
{Group: "networking.k8s.io", Version: "v1beta1", Kind: "NetworkPolicy"}: {"Age"},
{Group: "networking.k8s.io", Version: "v1beta1", Kind: "ServiceCIDR"}: {"Age"},
{Group: "node.k8s.io", Version: "v1", Kind: "RuntimeClass"}: {"Age"},
{Group: "policy", Version: "v1", Kind: "PodDisruptionBudget"}: {"Age"},
{Group: "rbac.authorization.k8s.io", Version: "v1beta1", Kind: "ClusterRoleBinding"}: {"Age"},
{Group: "rbac.authorization.k8s.io", Version: "v1beta1", Kind: "RoleBinding"}: {"Age"},
{Group: "resource.k8s.io", Version: "v1beta1", Kind: "DeviceClass"}: {"Age"},
{Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceClaim"}: {"Age"},
{Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceClaimTemplate"}: {"Age"},
{Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceSlice"}: {"Age"},
{Group: "scheduling.k8s.io", Version: "v1", Kind: "PriorityClass"}: {"Age"},
{Group: "storage.k8s.io", Version: "v1", Kind: "CSIDriver"}: {"Age"},
{Group: "storage.k8s.io", Version: "v1", Kind: "CSINode"}: {"Age"},
{Group: "storage.k8s.io", Version: "v1", Kind: "StorageClass"}: {"Age"},
{Group: "storage.k8s.io", Version: "v1", Kind: "VolumeAttachment"}: {"Age"},
{Group: "storage.k8s.io", Version: "v1beta1", Kind: "VolumeAttributesClass"}: {"Age"},
}

41
pkg/resources/common/util.go Normal file
View File

@ -0,0 +1,41 @@
package common
import (
"strconv"
"strings"
"github.com/rancher/apiserver/pkg/types"
"github.com/rancher/steve/pkg/attributes"
)
// GetIndexValueFromString looks for values between [ ].
// e.g: $.metadata.fields[2], in this case it would return 2
// In case it doesn't find any value between brackets it returns -1
func GetIndexValueFromString(pathString string) int {
idxStart := strings.Index(pathString, "[")
if idxStart == -1 {
return -1
}
idxEnd := strings.Index(pathString[idxStart+1:], "]")
if idxEnd == -1 {
return -1
}
idx, err := strconv.Atoi(pathString[idxStart+1 : idxStart+1+idxEnd])
if err != nil {
return -1
}
return idx
}
// GetColumnDefinitions returns ColumnDefinitions from an APISchema
func GetColumnDefinitions(schema *types.APISchema) []ColumnDefinition {
columns := attributes.Columns(schema)
if columns == nil {
return nil
}
colDefs, ok := columns.([]ColumnDefinition)
if !ok {
return nil
}
return colDefs
}

10
pkg/resources/schema.go
View File

@ -47,9 +47,10 @@ func DefaultSchemaTemplates(cf *client.Factory,
summaryCache *summarycache.SummaryCache,
lookup accesscontrol.AccessSetLookup,
discovery discovery.DiscoveryInterface,
namespaceCache corecontrollers.NamespaceCache) []schema.Template {
namespaceCache corecontrollers.NamespaceCache,
options common.TemplateOptions) []schema.Template {
return []schema.Template{
common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache),
common.DefaultTemplate(cf, summaryCache, lookup, namespaceCache, options),
apigroups.Template(discovery),
{
ID: "configmap",
@ -77,10 +78,11 @@ func DefaultSchemaTemplatesForStore(store types.Store,
baseSchemas *types.APISchemas,
summaryCache *summarycache.SummaryCache,
lookup accesscontrol.AccessSetLookup,
discovery discovery.DiscoveryInterface) []schema.Template {
discovery discovery.DiscoveryInterface,
options common.TemplateOptions) []schema.Template {
return []schema.Template{
common.DefaultTemplateForStore(store, summaryCache, lookup),
common.DefaultTemplateForStore(store, summaryCache, lookup, options),
apigroups.Template(discovery),
{
ID: "configmap",

46
pkg/resources/virtual/virtual.go
View File

@ -4,7 +4,10 @@ package virtual
import (
"fmt"
"slices"
"time"
rescommon "github.com/rancher/steve/pkg/resources/common"
"github.com/rancher/steve/pkg/resources/virtual/clusters"
"github.com/rancher/steve/pkg/resources/virtual/common"
"github.com/rancher/steve/pkg/resources/virtual/events"
@ -13,6 +16,8 @@ import (
"k8s.io/client-go/tools/cache"
)
var now = time.Now()
// TransformBuilder builds transform functions for specified GVKs through GetTransformFunc
type TransformBuilder struct {
defaultFields *common.DefaultFields
@ -28,13 +33,52 @@ func NewTransformBuilder(cache common.SummaryCache) *TransformBuilder {
}
// GetTransformFunc returns the func to transform a raw object into a fixed object, if needed
func (t *TransformBuilder) GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc {
func (t *TransformBuilder) GetTransformFunc(gvk schema.GroupVersionKind, columns []rescommon.ColumnDefinition) cache.TransformFunc {
converters := make([]func(*unstructured.Unstructured) (*unstructured.Unstructured, error), 0)
if gvk.Kind == "Event" && gvk.Group == "" && gvk.Version == "v1" {
converters = append(converters, events.TransformEventObject)
} else if gvk.Kind == "Cluster" && gvk.Group == "management.cattle.io" && gvk.Version == "v3" {
converters = append(converters, clusters.TransformManagedCluster)
}
// Detecting if we need to convert date fields
for _, col := range columns {
gvkDateFields, gvkFound := rescommon.DateFieldsByGVKBuiltins[gvk]
if col.Type == "date" || (gvkFound && slices.Contains(gvkDateFields, col.Name)) {
converters = append(converters, func(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
index := rescommon.GetIndexValueFromString(col.Field)
if index == -1 {
return nil, fmt.Errorf("field index not found at column.Field struct variable: %s", col.Field)
}
curValue, got, err := unstructured.NestedSlice(obj.Object, "metadata", "fields")
if !got {
return obj, nil
}
if err != nil {
return nil, err
}
value, cast := curValue[index].(string)
if !cast {
return nil, fmt.Errorf("could not cast metadata.fields %d to string", index)
}
duration, err := rescommon.ParseHumanReadableDuration(value)
if err != nil {
return nil, err
}
curValue[index] = fmt.Sprintf("%d", now.Add(-duration).UnixMilli())
if err := unstructured.SetNestedSlice(obj.Object, curValue, "metadata", "fields"); err != nil {
return nil, err
}
return obj, nil
})
}
}
converters = append(converters, t.defaultFields.TransformCommon)
return func(raw interface{}) (interface{}, error) {

143
pkg/resources/virtual/virtual_test.go
View File

@ -1,11 +1,12 @@
package virtual_test
package virtual
import (
"fmt"
"strings"
"testing"
"time"
"github.com/rancher/steve/pkg/resources/virtual"
rescommon "github.com/rancher/steve/pkg/resources/common"
"github.com/rancher/steve/pkg/resources/virtual/common"
"github.com/rancher/steve/pkg/summarycache"
"github.com/rancher/wrangler/v3/pkg/summary"
@ -16,11 +17,14 @@ import (
)
func TestTransformChain(t *testing.T) {
now = func() time.Time { return time.Date(1992, 9, 2, 0, 0, 0, 0, time.UTC) }()
noColumns := []rescommon.ColumnDefinition{}
tests := []struct {
name string
input any
hasSummary *summary.SummarizedObject
hasRelationships []summarycache.Relationship
columns []rescommon.ColumnDefinition
wantOutput any
wantError bool
}{
@ -65,6 +69,7 @@ func TestTransformChain(t *testing.T) {
"id": "old-id",
},
},
columns: noColumns,
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "test.cattle.io/v1",
@ -94,6 +99,131 @@ func TestTransformChain(t *testing.T) {
},
},
},
{
name: "CRD metadata.fields has a date field - should convert to timestamp",
hasSummary: &summary.SummarizedObject{
PartialObjectMetadata: v1.PartialObjectMetadata{
ObjectMeta: v1.ObjectMeta{
Name: "testobj",
Namespace: "test-ns",
},
TypeMeta: v1.TypeMeta{
APIVersion: "test.cattle.io/v1",
Kind: "TestResource",
},
},
Summary: summary.Summary{
State: "success",
Transitioning: false,
Error: false,
},
},
input: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "test.cattle.io/v1",
"kind": "TestResource",
"metadata": map[string]interface{}{
"name": "testobj",
"namespace": "test-ns",
"fields": []interface{}{"1d"},
},
"id": "old-id",
},
},
columns: []rescommon.ColumnDefinition{
{
Field: "metadata.fields[0]",
TableColumnDefinition: v1.TableColumnDefinition{
Name: "Age",
Type: "date",
},
},
},
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "test.cattle.io/v1",
"kind": "TestResource",
"metadata": map[string]interface{}{
"name": "testobj",
"namespace": "test-ns",
"relationships": []any(nil),
"state": map[string]interface{}{
"name": "success",
"error": false,
"transitioning": false,
"message": "",
},
"fields": []interface{}{
fmt.Sprintf("%d", now.Add(-24*time.Hour).UnixMilli()),
},
},
"id": "test-ns/testobj",
"_id": "old-id",
},
},
},
{
name: "built-in type metadata.fields has a date field - should convert to timestamp",
hasSummary: &summary.SummarizedObject{
PartialObjectMetadata: v1.PartialObjectMetadata{
ObjectMeta: v1.ObjectMeta{
Name: "testobj",
Namespace: "test-ns",
},
TypeMeta: v1.TypeMeta{
APIVersion: "apps/v1",
Kind: "Deployment",
},
},
Summary: summary.Summary{
State: "success",
Transitioning: false,
Error: false,
},
},
input: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "testobj",
"namespace": "test-ns",
"fields": []interface{}{"1d"},
},
"id": "old-id",
},
},
columns: []rescommon.ColumnDefinition{
{
TableColumnDefinition: v1.TableColumnDefinition{
Name: "Age",
},
Field: "metadata.fields[0]",
},
},
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "testobj",
"namespace": "test-ns",
"relationships": []any(nil),
"state": map[string]interface{}{
"name": "success",
"error": false,
"transitioning": false,
"message": "",
},
"fields": []interface{}{
fmt.Sprintf("%d", now.Add(-24*time.Hour).UnixMilli()),
},
},
"id": "test-ns/testobj",
"_id": "old-id",
},
},
},
{
name: "processable event",
input: &unstructured.Unstructured{
@ -118,6 +248,7 @@ func TestTransformChain(t *testing.T) {
"type": "Gorniplatz",
},
},
columns: noColumns,
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "/v1",
@ -162,6 +293,7 @@ func TestTransformChain(t *testing.T) {
"type": "Gorniplatz",
},
},
columns: noColumns,
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "palau.io/v1",
@ -218,6 +350,7 @@ func TestTransformChain(t *testing.T) {
},
},
},
columns: noColumns,
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "management.cattle.io/v3",
@ -302,6 +435,7 @@ func TestTransformChain(t *testing.T) {
},
},
},
columns: noColumns,
wantOutput: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "management.cattle.io/v3",
@ -346,13 +480,14 @@ func TestTransformChain(t *testing.T) {
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeCache := common.FakeSummaryCache{
SummarizedObject: test.hasSummary,
Relationships: test.hasRelationships,
}
tb := virtual.NewTransformBuilder(&fakeCache)
tb := NewTransformBuilder(&fakeCache)
raw, isSignal, err := common.GetUnstructured(test.input)
require.False(t, isSignal)
require.Nil(t, err)
@ -362,7 +497,7 @@ func TestTransformChain(t *testing.T) {
if test.name == "a non-ready cluster" {
fmt.Printf("Stop here")
}
output, err := tb.GetTransformFunc(gvk)(test.input)
output, err := tb.GetTransformFunc(gvk, test.columns)(test.input)
require.Equal(t, test.wantOutput, output)
if test.wantError {
require.Error(t, err)

4
pkg/server/server.go
View File

@ -224,7 +224,7 @@ func setup(ctx context.Context, server *Server) error {
store := metricsStore.NewMetricsStore(errStore)
// end store setup code
for _, template := range resources.DefaultSchemaTemplatesForStore(store, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery()) {
for _, template := range resources.DefaultSchemaTemplatesForStore(store, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), common.TemplateOptions{InSQLMode: true}) {
sf.AddTemplate(template)
}
@ -238,7 +238,7 @@ func setup(ctx context.Context, server *Server) error {
return nil
}
} else {
for _, template := range resources.DefaultSchemaTemplates(cf, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), server.controllers.Core.Namespace().Cache()) {
for _, template := range resources.DefaultSchemaTemplates(cf, server.BaseSchemas, summaryCache, asl, server.controllers.K8s.Discovery(), server.controllers.Core.Namespace().Cache(), common.TemplateOptions{InSQLMode: false}) {
sf.AddTemplate(template)
}
onSchemasHandler = ccache.OnSchemas

9
pkg/stores/sqlproxy/proxy_mocks_test.go
View File

@ -14,6 +14,7 @@ import (
reflect "reflect"
types "github.com/rancher/apiserver/pkg/types"
common "github.com/rancher/steve/pkg/resources/common"
factory "github.com/rancher/steve/pkg/sqlcache/informer/factory"
partition "github.com/rancher/steve/pkg/sqlcache/partition"
sqltypes "github.com/rancher/steve/pkg/sqlcache/sqltypes"
@ -389,15 +390,15 @@ func (m *MockTransformBuilder) EXPECT() *MockTransformBuilderMockRecorder {
}
// GetTransformFunc mocks base method.
func (m *MockTransformBuilder) GetTransformFunc(arg0 schema.GroupVersionKind) cache.TransformFunc {
func (m *MockTransformBuilder) GetTransformFunc(arg0 schema.GroupVersionKind, arg1 []common.ColumnDefinition) cache.TransformFunc {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetTransformFunc", arg0)
ret := m.ctrl.Call(m, "GetTransformFunc", arg0, arg1)
ret0, _ := ret[0].(cache.TransformFunc)
return ret0
}
// GetTransformFunc indicates an expected call of GetTransformFunc.
func (mr *MockTransformBuilderMockRecorder) GetTransformFunc(arg0 any) *gomock.Call {
func (mr *MockTransformBuilderMockRecorder) GetTransformFunc(arg0, arg1 any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTransformFunc", reflect.TypeOf((*MockTransformBuilder)(nil).GetTransformFunc), arg0)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTransformFunc", reflect.TypeOf((*MockTransformBuilder)(nil).GetTransformFunc), arg0, arg1)
}

11
pkg/stores/sqlproxy/proxy_store.go
View File

@ -240,7 +240,7 @@ type RelationshipNotifier interface {
}
type TransformBuilder interface {
GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc
GetTransformFunc(gvk schema.GroupVersionKind, colDefs []common.ColumnDefinition) cache.TransformFunc
}
type Store struct {
@ -335,9 +335,10 @@ func (s *Store) initializeNamespaceCache() error {
// get any type-specific fields that steve is interested in
fields = append(fields, getFieldForGVK(gvk)...)
cols := common.GetColumnDefinitions(&nsSchema)
// get the type-specific transform func
transformFunc := s.transformBuilder.GetTransformFunc(gvk)
transformFunc := s.transformBuilder.GetTransformFunc(gvk, cols)
// get the ns informer
tableClient := &tablelistconvert.Client{ResourceInterface: client}
@ -545,7 +546,7 @@ func (s *Store) watch(apiOp *types.APIRequest, schema *types.APISchema, w types.
gvk := attributes.GVK(schema)
fields := getFieldsFromSchema(schema)
fields = append(fields, getFieldForGVK(gvk)...)
transformFunc := s.transformBuilder.GetTransformFunc(gvk)
transformFunc := s.transformBuilder.GetTransformFunc(gvk, nil)
tableClient := &tablelistconvert.Client{ResourceInterface: client}
attrs := attributes.GVK(schema)
ns := attributes.Namespaced(schema)
@ -756,7 +757,9 @@ func (s *Store) ListByPartitions(apiOp *types.APIRequest, apiSchema *types.APISc
gvk := attributes.GVK(apiSchema)
fields := getFieldsFromSchema(apiSchema)
fields = append(fields, getFieldForGVK(gvk)...)
transformFunc := s.transformBuilder.GetTransformFunc(gvk)
cols := common.GetColumnDefinitions(apiSchema)
transformFunc := s.transformBuilder.GetTransformFunc(gvk, cols)
tableClient := &tablelistconvert.Client{ResourceInterface: client}
attrs := attributes.GVK(apiSchema)
ns := attributes.Namespaced(apiSchema)

14
pkg/stores/sqlproxy/proxy_store_test.go
View File

@ -245,7 +245,7 @@ func TestListByPartitions(t *testing.T) {
cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
// This tests that fields are being extracted from schema columns and the type specific fields map
cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil)
tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
tb.EXPECT().GetTransformFunc(attributes.GVK(schema), []common.ColumnDefinition{{Field: "some.field"}}).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil)
list, total, contToken, err := s.ListByPartitions(req, schema, partitions)
assert.Nil(t, err)
@ -462,7 +462,7 @@ func TestListByPartitions(t *testing.T) {
// note also the watchable bool is expected to be false
cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), false).Return(c, nil)
tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
tb.EXPECT().GetTransformFunc(attributes.GVK(schema), []common.ColumnDefinition{{Field: "some.field"}}).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(listToReturn, len(listToReturn.Items), "", nil)
list, total, contToken, err := s.ListByPartitions(req, schema, partitions)
assert.Nil(t, err)
@ -534,7 +534,7 @@ func TestListByPartitions(t *testing.T) {
assert.Nil(t, err)
cg.EXPECT().TableAdminClient(req, schema, "", &WarningBuffer{}).Return(ri, nil)
// This tests that fields are being extracted from schema columns and the type specific fields map
tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
tb.EXPECT().GetTransformFunc(attributes.GVK(schema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(factory.Cache{}, fmt.Errorf("error"))
_, _, _, err = s.ListByPartitions(req, schema, partitions)
@ -613,7 +613,7 @@ func TestListByPartitions(t *testing.T) {
// This tests that fields are being extracted from schema columns and the type specific fields map
cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {`id`}, {`metadata`, `state`, `name`}, {"gvk", "specific", "fields"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(schema), attributes.Namespaced(schema), true).Return(c, nil)
bloi.EXPECT().ListByOptions(req.Context(), &opts, partitions, req.Namespace).Return(nil, 0, "", fmt.Errorf("error"))
tb.EXPECT().GetTransformFunc(attributes.GVK(schema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
tb.EXPECT().GetTransformFunc(attributes.GVK(schema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
_, _, _, err = s.ListByPartitions(req, schema, partitions)
assert.NotNil(t, err)
@ -725,7 +725,7 @@ func TestListByPartitionWithUserAccess(t *testing.T) {
attributes.SetGVK(theSchema, gvk)
cg.EXPECT().TableAdminClient(apiOp, theSchema, "", &WarningBuffer{}).Return(ri, nil)
cf.EXPECT().CacheFor(context.Background(), [][]string{{"some", "field"}, {"id"}, {"metadata", "state", "name"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(theSchema), attributes.Namespaced(theSchema), true).Return(c, nil)
tb.EXPECT().GetTransformFunc(attributes.GVK(theSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
tb.EXPECT().GetTransformFunc(attributes.GVK(theSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
listToReturn := &unstructured.UnstructuredList{
Items: make([]unstructured.Unstructured, 0, 0),
@ -767,7 +767,7 @@ func TestReset(t *testing.T) {
cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil)
cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(nsc2, nil)
tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
err := s.Reset()
assert.Nil(t, err)
assert.Equal(t, nsc2, s.namespaceCache)
@ -874,7 +874,7 @@ func TestReset(t *testing.T) {
cs.EXPECT().SetColumns(gomock.Any(), gomock.Any()).Return(nil)
cg.EXPECT().TableAdminClient(nil, &nsSchema, "", &WarningBuffer{}).Return(ri, nil)
cf.EXPECT().CacheFor(context.Background(), [][]string{{`id`}, {`metadata`, `state`, `name`}, {"metadata", "labels", "field.cattle.io/projectId"}}, gomock.Any(), &tablelistconvert.Client{ResourceInterface: ri}, attributes.GVK(&nsSchema), false, true).Return(factory.Cache{}, fmt.Errorf("error"))
tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema)).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
tb.EXPECT().GetTransformFunc(attributes.GVK(&nsSchema), gomock.Any()).Return(func(obj interface{}) (interface{}, error) { return obj, nil })
err := s.Reset()
assert.NotNil(t, err)
},