From f01f2b122923798d60893c6c189a204b94fe5723 Mon Sep 17 00:00:00 2001 From: Maciej Borsz Date: Thu, 18 Feb 2021 10:56:51 +0100 Subject: [PATCH] Use more real world examples in BenchmarkSerializeObject --- .../endpoints/handlers/responsewriters/BUILD | 3 + .../responsewriters/testdata/pod.json | 162 ++++++++++++++++++ .../handlers/responsewriters/writers_test.go | 117 +++++++++++-- 3 files changed, 271 insertions(+), 11 deletions(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/testdata/pod.json diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/BUILD b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/BUILD index 86356415235..21c17545f06 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/BUILD @@ -13,14 +13,17 @@ go_test( "status_test.go", "writers_test.go", ], + data = glob(["testdata/**"]), embed = [":go_default_library"], deps = [ + "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authentication/user:go_default_library", "//staging/src/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/testdata/pod.json b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/testdata/pod.json new file mode 100644 index 00000000000..39811fdfe7e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/testdata/pod.json @@ -0,0 +1,162 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2021-02-18T09:46:18Z", + "generateName": "nginx-deployment-7b88ccfd76-", + "labels": { + "app": "nginx", + "pod-template-hash": "7b88ccfd76" + }, + "managedFields": [ + { + "apiVersion": "v1", + "fieldsType": "FieldsV1", + "fieldsV1": {"f:metadata":{"f:generateName":{},"f:labels":{".":{},"f:app":{},"f:pod-template-hash":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"c9f927f9-8f7a-43c4-a100-b4bdf306dd93\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:spec":{"f:containers":{"k:{\"name\":\"nginx\"}":{".":{},"f:image":{},"f:imagePullPolicy":{},"f:name":{},"f:ports":{".":{},"k:{\"containerPort\":80,\"protocol\":\"TCP\"}":{".":{},"f:containerPort":{},"f:protocol":{}}},"f:resources":{".":{},"f:requests":{".":{},"f:cpu":{}}},"f:terminationMessagePath":{},"f:terminationMessagePolicy":{}}},"f:dnsPolicy":{},"f:enableServiceLinks":{},"f:restartPolicy":{},"f:schedulerName":{},"f:securityContext":{},"f:terminationGracePeriodSeconds":{}}}, + "manager": "kube-controller-manager", + "operation": "Update", + "time": "2021-02-18T09:46:18Z" + }, + { + "apiVersion": "v1", + "fieldsType": "FieldsV1", + "fieldsV1": {"f:status":{"f:conditions":{"k:{\"type\":\"ContainersReady\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Initialized\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}},"k:{\"type\":\"Ready\"}":{".":{},"f:lastProbeTime":{},"f:lastTransitionTime":{},"f:status":{},"f:type":{}}},"f:containerStatuses":{},"f:hostIP":{},"f:phase":{},"f:podIP":{},"f:podIPs":{".":{},"k:{\"ip\":\"10.224.98.9\"}":{".":{},"f:ip":{}}},"f:startTime":{}}}, + "manager": "kubelet", + "operation": "Update", + "time": "2021-02-18T09:46:19Z" + } + ], + "name": "nginx-deployment-7b88ccfd76-24jf7", + "namespace": "default", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "nginx-deployment-7b88ccfd76", + "uid": "c9f927f9-8f7a-43c4-a100-b4bdf306dd93" + } + ], + "resourceVersion": "10345330", + "uid": "3be4f64b-3687-47ff-9910-b72366ea798e" + }, + "spec": { + "containers": [ + { + "image": "nginx:1.14.2", + "imagePullPolicy": "IfNotPresent", + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "protocol": "TCP" + } + ], + "resources": { + "requests": { + "cpu": "1m" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "default-token-hz58m", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "gke-pf-default-pool-ad46e28b-kdh9", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "default", + "serviceAccountName": "default", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "default-token-hz58m", + "secret": { + "defaultMode": 420, + "secretName": "default-token-hz58m" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2021-02-18T09:46:18Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2021-02-18T09:46:19Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2021-02-18T09:46:19Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2021-02-18T09:46:18Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://cea8a81981cd4780fd0b705049a533ca9d83c4596b9f7f4e67915863a2ca76a2", + "image": "nginx:1.14.2", + "imageID": "docker-pullable://nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d", + "lastState": {}, + "name": "nginx", + "ready": true, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2021-02-18T09:46:19Z" + } + } + } + ], + "hostIP": "10.223.96.100", + "phase": "Running", + "podIP": "10.224.98.9", + "podIPs": [ + { + "ip": "10.224.98.9" + } + ], + "qosClass": "Burstable", + "startTime": "2021-02-18T09:46:18Z" + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers_test.go index 266e4edc724..1a8d98be33c 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers_test.go @@ -20,25 +20,34 @@ import ( "bytes" "compress/gzip" "encoding/hex" + "encoding/json" "errors" "fmt" "io" "io/ioutil" + "math/rand" "net/http" "net/http/httptest" "net/url" + "os" "reflect" + "strconv" "testing" + "time" + v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" ) +const benchmarkSeed = 100 + func TestSerializeObjectParallel(t *testing.T) { largePayload := bytes.Repeat([]byte("0123456789abcdef"), defaultGzipThresholdBytes/16+1) type test struct { @@ -366,8 +375,88 @@ func TestSerializeObject(t *testing.T) { } } -func benchmarkSerializeObject(b *testing.B, size int) { - largePayload := bytes.Repeat([]byte("0123456789abcdef"), size/16+1) +func randTime(t *time.Time, r *rand.Rand) { + *t = time.Unix(r.Int63n(1000*365*24*60*60), r.Int63()) +} + +func randIP(s *string, r *rand.Rand) { + *s = fmt.Sprintf("10.20.%d.%d", r.Int31n(256), r.Int31n(256)) +} + +// randPod changes fields in pod to mimic another pod from the same replicaset. +// The list fields here has been generated by picking two pods in the same replicaset +// and checking diff of their jsons. +func randPod(b *testing.B, pod *v1.Pod, r *rand.Rand) { + pod.Name = fmt.Sprintf("%s-%x", pod.GenerateName, r.Int63n(1000)) + pod.UID = uuid.NewUUID() + pod.ResourceVersion = strconv.Itoa(r.Int()) + pod.Spec.NodeName = fmt.Sprintf("some-node-prefix-%x", r.Int63n(1000)) + + randTime(&pod.CreationTimestamp.Time, r) + randTime(&pod.Status.StartTime.Time, r) + for i := range pod.Status.Conditions { + randTime(&pod.Status.Conditions[i].LastTransitionTime.Time, r) + } + for i := range pod.Status.ContainerStatuses { + containerStatus := &pod.Status.ContainerStatuses[i] + state := &containerStatus.State + if state.Running != nil { + randTime(&state.Running.StartedAt.Time, r) + } + containerStatus.ContainerID = fmt.Sprintf("docker://%x%x%x%x", r.Int63(), r.Int63(), r.Int63(), r.Int63()) + } + for i := range pod.ManagedFields { + randTime(&pod.ManagedFields[i].Time.Time, r) + } + + randIP(&pod.Status.HostIP, r) + randIP(&pod.Status.PodIP, r) +} + +func benchmarkItems(b *testing.B, file string, n int) *v1.PodList { + pod := v1.Pod{} + f, err := os.Open(file) + if err != nil { + b.Fatalf("Failed to open %q: %v", file, err) + } + defer f.Close() + err = json.NewDecoder(f).Decode(&pod) + if err != nil { + b.Fatalf("Failed to decode %q: %v", file, err) + } + + list := &v1.PodList{ + Items: make([]v1.Pod, n), + } + + r := rand.New(rand.NewSource(benchmarkSeed)) + for i := 0; i < n; i++ { + list.Items[i] = *pod.DeepCopy() + randPod(b, &list.Items[i], r) + } + return list +} + +func toProtoBuf(b *testing.B, list *v1.PodList) []byte { + out, err := list.Marshal() + if err != nil { + b.Fatalf("Failed to marshal list to protobuf: %v", err) + } + return out +} + +func toJSON(b *testing.B, list *v1.PodList) []byte { + out, err := json.Marshal(list) + if err != nil { + b.Fatalf("Failed to marshal list to json: %v", err) + } + return out +} + +func benchmarkSerializeObject(b *testing.B, payload []byte) { + input, output := len(payload), len(gzipContent(payload, defaultGzipContentEncodingLevel)) + b.Logf("Payload size: %d, expected output size: %d, ratio: %.2f", input, output, float64(output)/float64(input)) + req := &http.Request{ Header: http.Header{ "Accept-Encoding": []string{"gzip"}, @@ -377,7 +466,7 @@ func benchmarkSerializeObject(b *testing.B, size int) { defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.APIResponseCompression, true)() encoder := &fakeEncoder{ - buf: largePayload, + buf: payload, } b.ResetTimer() @@ -391,18 +480,24 @@ func benchmarkSerializeObject(b *testing.B, size int) { } } -func BenchmarkSerializeObject10KB(b *testing.B) { - benchmarkSerializeObject(b, 10*1024) +func BenchmarkSerializeObject1000PodsPB(b *testing.B) { + benchmarkSerializeObject(b, toProtoBuf(b, benchmarkItems(b, "testdata/pod.json", 1000))) +} +func BenchmarkSerializeObject10000PodsPB(b *testing.B) { + benchmarkSerializeObject(b, toProtoBuf(b, benchmarkItems(b, "testdata/pod.json", 10000))) +} +func BenchmarkSerializeObject100000PodsPB(b *testing.B) { + benchmarkSerializeObject(b, toProtoBuf(b, benchmarkItems(b, "testdata/pod.json", 100000))) } -func BenchmarkSerializeObject10MB(b *testing.B) { - benchmarkSerializeObject(b, 10*1024*1024) +func BenchmarkSerializeObject1000PodsJSON(b *testing.B) { + benchmarkSerializeObject(b, toJSON(b, benchmarkItems(b, "testdata/pod.json", 1000))) } -func BenchmarkSerializeObject100MB(b *testing.B) { - benchmarkSerializeObject(b, 100*1024*1024) +func BenchmarkSerializeObject10000PodsJSON(b *testing.B) { + benchmarkSerializeObject(b, toJSON(b, benchmarkItems(b, "testdata/pod.json", 10000))) } -func BenchmarkSerializeObject1GB(b *testing.B) { - benchmarkSerializeObject(b, 1024*1024*1024) +func BenchmarkSerializeObject100000PodsJSON(b *testing.B) { + benchmarkSerializeObject(b, toJSON(b, benchmarkItems(b, "testdata/pod.json", 100000))) } type fakeResponseRecorder struct {