From d4cfe78364788943a31bc7b55427688b71174fa9 Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Tue, 12 Apr 2022 16:17:28 -0700 Subject: [PATCH] Add field filtering for resources This change enables steve to work with three new query parameters: "include": only include the named fields from the kubernetes object. Subfields are denoted with ".". Subfields within arrays are ignored. Multiple fields can be included by repeating the parameter. Example: GET /v1/configmaps?include=kind&include=apiVersion => { "type": "collection", "links": { "self": "http://server/v1/configmaps" }, "createTypes": { "configmap": "http://server/v1/configmaps" }, "actions": {}, "resourceType": "configmap", "revision": "327238", "data": [ { "id": "c-m-w466b2vg/kube-root-ca.crt", "type": "configmap", "links": { "remove": "http://server/v1/configmaps/c-m-w466b2vg/kube-root-ca.crt", "self": "http://server/v1/configmaps/c-m-w466b2vg/kube-root-ca.crt", "update": "http://server/v1/configmaps/c-m-w466b2vg/kube-root-ca.crt", "view": "http://server/api/v1/namespaces/c-m-w466b2vg/configmaps/kube-root-ca.crt" }, "apiVersion": "v1", "kind": "ConfigMap" }, } ... } "exclude": exclude the named fields from the kubernetes object. Subfields are denoted with ".". Subfields within arrays are ignored. Multiple fields can be excluded by repeating the parameter. Example: GET /v1/configmaps?exclude=data&exclude=metadata.managedFields => { "type": "collection", "links": { "self": "http://server/v1/configmaps" }, "createTypes": { "configmap": "http://server/v1/configmaps" }, "actions": {}, "resourceType": "configmap", "revision": "328086", "data": [ { "id": "c-m-w466b2vg/kube-root-ca.crt", "type": "configmap", "links": { "remove": "http://server/v1/configmaps/c-m-w466b2vg/kube-root-ca.crt", "self": "http://server/v1/configmaps/c-m-w466b2vg/kube-root-ca.crt", "update": "http://server/v1/configmaps/c-m-w466b2vg/kube-root-ca.crt", "view": "http://server/api/v1/namespaces/c-m-w466b2vg/configmaps/kube-root-ca.crt" }, "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "creationTimestamp": "2022-04-11T22:05:27Z", "fields": [ "kube-root-ca.crt", 1, "25h" ], "name": "kube-root-ca.crt", "namespace": "c-m-w466b2vg", "relationships": null, "resourceVersion": "36948", "state": { "error": false, "message": "Resource is always ready", "name": "active", "transitioning": false }, "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b" } }, ... } "excludeValues": replace the values of an object with empty strings, leaving the keys in place. Useful for showing a summary of an object with large values, such as the data in a ConfigMap. Only works on fields that are object. Multiple fields can have values excluded by repeating the parameter. Example: GET /v1/configmaps?excludeValues=data => { "type": "collection", ... "data": [ { ... "data": { "ca.crt": "" }, ... }, ... ] } --- pkg/resources/common/formatter.go | 41 ++ pkg/resources/common/formatter_test.go | 543 +++++++++++++++++++++++++ 2 files changed, 584 insertions(+) create mode 100644 pkg/resources/common/formatter_test.go diff --git a/pkg/resources/common/formatter.go b/pkg/resources/common/formatter.go index cc28616..06dd345 100644 --- a/pkg/resources/common/formatter.go +++ b/pkg/resources/common/formatter.go @@ -93,6 +93,47 @@ func formatter(summarycache *summarycache.SummaryCache) types.Formatter { data.PutValue(unstr.Object, rel, "metadata", "relationships") summary.NormalizeConditions(unstr) + + includeFields(request, unstr) + excludeFields(request, unstr) + excludeValues(request, unstr) + } + + } +} + +func includeFields(request *types.APIRequest, unstr *unstructured.Unstructured) { + if fields, ok := request.Query["include"]; ok { + newObj := map[string]interface{}{} + for _, f := range fields { + fieldParts := strings.Split(f, ".") + if val, ok := data.GetValue(unstr.Object, fieldParts...); ok { + data.PutValue(newObj, val, fieldParts...) + } + } + unstr.Object = newObj + } +} + +func excludeFields(request *types.APIRequest, unstr *unstructured.Unstructured) { + if fields, ok := request.Query["exclude"]; ok { + for _, f := range fields { + fieldParts := strings.Split(f, ".") + data.RemoveValue(unstr.Object, fieldParts...) + } + } +} + +func excludeValues(request *types.APIRequest, unstr *unstructured.Unstructured) { + if values, ok := request.Query["excludeValues"]; ok { + for _, f := range values { + fieldParts := strings.Split(f, ".") + fieldValues := data.GetValueN(unstr.Object, fieldParts...) + if obj, ok := fieldValues.(map[string]interface{}); ok { + for k := range obj { + data.PutValue(unstr.Object, "", append(fieldParts, k)...) + } + } } } } diff --git a/pkg/resources/common/formatter_test.go b/pkg/resources/common/formatter_test.go new file mode 100644 index 0000000..635fec5 --- /dev/null +++ b/pkg/resources/common/formatter_test.go @@ -0,0 +1,543 @@ +package common + +import ( + "net/url" + "testing" + + "github.com/rancher/apiserver/pkg/types" + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func Test_includeFields(t *testing.T) { + tests := []struct { + name string + request *types.APIRequest + unstr *unstructured.Unstructured + want *unstructured.Unstructured + }{ + { + name: "include top level field", + request: &types.APIRequest{ + Query: url.Values{ + "include": []string{"metadata"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + }, + }, + }, + { + name: "include sub field", + request: &types.APIRequest{ + Query: url.Values{ + "include": []string{"metadata.managedFields"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + "managedFields": []map[string]interface{}{ + { + "apiVersion": "v1", + "fieldsType": "FieldsV1", + "fieldsV1": map[string]interface{}{ + "f:data": map[string]interface{}{ + ".": map[string]interface{}{}, + "f:ca.crt": map[string]interface{}{}, + }, + }, + "manager": "kube-controller-manager", + "operation": "Update", + "time": "2022-04-11T22:05:27Z", + }, + }, + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "managedFields": []map[string]interface{}{ + { + "apiVersion": "v1", + "fieldsType": "FieldsV1", + "fieldsV1": map[string]interface{}{ + "f:data": map[string]interface{}{ + ".": map[string]interface{}{}, + "f:ca.crt": map[string]interface{}{}, + }, + }, + "manager": "kube-controller-manager", + "operation": "Update", + "time": "2022-04-11T22:05:27Z", + }, + }, + }, + }, + }, + }, + { + name: "include invalid field", + request: &types.APIRequest{ + Query: url.Values{ + "include": []string{"foo.bar"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + }, + { + name: "include multiple fields", + request: &types.APIRequest{ + Query: url.Values{ + "include": []string{"kind", "apiVersion", "metadata.name"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "kube-root-ca.crt", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + includeFields(tt.request, tt.unstr) + assert.Equal(t, tt.want, tt.unstr) + }) + } +} + +func Test_excludeFields(t *testing.T) { + tests := []struct { + name string + request *types.APIRequest + unstr *unstructured.Unstructured + want *unstructured.Unstructured + }{ + { + name: "exclude top level field", + request: &types.APIRequest{ + Query: url.Values{ + "exclude": []string{"metadata"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + }, + { + name: "exclude sub field", + request: &types.APIRequest{ + Query: url.Values{ + "exclude": []string{"metadata.managedFields"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + "managedFields": []map[string]interface{}{ + { + "apiVersion": "v1", + "fieldsType": "FieldsV1", + "fieldsV1": map[string]interface{}{ + "f:data": map[string]interface{}{ + ".": map[string]interface{}{}, + "f:ca.crt": map[string]interface{}{}, + }, + }, + "manager": "kube-controller-manager", + "operation": "Update", + "time": "2022-04-11T22:05:27Z", + }, + }, + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + }, + { + name: "exclude invalid field", + request: &types.APIRequest{ + Query: url.Values{ + "exclude": []string{"foo.bar"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + }, + { + name: "exclude multiple fields", + request: &types.APIRequest{ + Query: url.Values{ + "exclude": []string{"kind", "apiVersion", "metadata.name"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + excludeFields(tt.request, tt.unstr) + assert.Equal(t, tt.want, tt.unstr) + }) + } +} + +func Test_excludeValues(t *testing.T) { + tests := []struct { + name string + request *types.APIRequest + unstr *unstructured.Unstructured + want *unstructured.Unstructured + }{ + { + name: "exclude top level value", + request: &types.APIRequest{ + Query: url.Values{ + "excludeValues": []string{"data"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "", + }, + }, + }, + }, + { + name: "exclude sub field value", + request: &types.APIRequest{ + Query: url.Values{ + "excludeValues": []string{"metadata.annotations"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{ + "deployment.kubernetes.io/revision": "2", + "meta.helm.sh/release-name": "fleet-agent-local", + "meta.helm.sh/release-namespace": "cattle-fleet-local-system", + }, + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "fleet-agent", + "namespace": "cattle-fleet-local-system", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "spec": map[string]interface{}{ + "replicas": 1, + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{ + "deployment.kubernetes.io/revision": "", + "meta.helm.sh/release-name": "", + "meta.helm.sh/release-namespace": "", + }, + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "fleet-agent", + "namespace": "cattle-fleet-local-system", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "spec": map[string]interface{}{ + "replicas": 1, + }, + }, + }, + }, + { + name: "exclude invalid value", + request: &types.APIRequest{ + Query: url.Values{ + "excludeValues": []string{"foo.bar"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "kube-root-ca.crt", + "namespace": "c-m-w466b2vg", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "data": map[string]interface{}{ + "ca.crt": "-----BEGIN CERTIFICATE-----\nMIIC5zCCAc+gAwIBAg\n-----END CERTIFICATE-----\n", + }, + }, + }, + }, + { + name: "exclude multiple values", + request: &types.APIRequest{ + Query: url.Values{ + "excludeValues": []string{"metadata.annotations", "metadata.labels"}, + }, + }, + unstr: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{ + "deployment.kubernetes.io/revision": "2", + "meta.helm.sh/release-name": "fleet-agent-local", + "meta.helm.sh/release-namespace": "cattle-fleet-local-system", + }, + "labels": map[string]interface{}{ + "app.kubernetes.io/managed-by": "Helm", + "objectset.rio.cattle.io/hash": "362023f752e7f1989d8b652e029bd2c658ae7c44", + }, + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "fleet-agent", + "namespace": "cattle-fleet-local-system", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "spec": map[string]interface{}{ + "replicas": 1, + }, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{ + "deployment.kubernetes.io/revision": "", + "meta.helm.sh/release-name": "", + "meta.helm.sh/release-namespace": "", + }, + "labels": map[string]interface{}{ + "app.kubernetes.io/managed-by": "", + "objectset.rio.cattle.io/hash": "", + }, + "creationTimestamp": "2022-04-11T22:05:27Z", + "name": "fleet-agent", + "namespace": "cattle-fleet-local-system", + "resourceVersion": "36948", + "uid": "1c497934-52cb-42ab-a613-dedfd5fb207b", + }, + "spec": map[string]interface{}{ + "replicas": 1, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + excludeValues(tt.request, tt.unstr) + assert.Equal(t, tt.want, tt.unstr) + }) + } +}