Merge pull request #59800 from porridge/export-benchmark-data

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Save benchmark data in perfdash-friendly format.

**What this PR does / why we need it**:
Saves data in a format which perfdash can ingest directly.

**Which issue(s) this PR fixes**
Step towards resolving #58154

**Release note**:
```release-note
NONE
```

/cc @shyamjvs

Godeps/Godeps.json

@@ -2992,6 +2992,10 @@
 			"ImportPath": "golang.org/x/time/rate",
 			"Rev": "f51c12702a4d776e4c1fa9b0fabab841babae631"
 		},
+		{
+			"ImportPath": "golang.org/x/tools/benchmark/parse",
+			"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32"
+		},
 		{
 			"ImportPath": "golang.org/x/tools/container/intsets",
 			"Rev": "2382e3994d48b1d22acc2c86bcad0a2aff028e32"

Godeps/LICENSES

@@ -90221,6 +90221,41 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ================================================================================
 
 
+================================================================================
+= vendor/golang.org/x/tools/benchmark/parse licensed under: =
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+= vendor/golang.org/x/tools/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707
+================================================================================
+
+
 ================================================================================
 = vendor/golang.org/x/tools/container/intsets licensed under: =
 

hack/jenkins/benchmark-dockerized.sh

@@ -48,4 +48,6 @@ cd /go/src/k8s.io/kubernetes
 
 # Run the benchmark tests and pretty-print the results into a separate file.
 make test-integration WHAT="$*" KUBE_TEST_ARGS="-run='XXX' -bench=. -benchmem" \
-  | tee >(prettybench -no-passthrough > ${ARTIFACTS_DIR}/BenchmarkResults.txt)
+  | tee \
+   >(prettybench -no-passthrough > ${ARTIFACTS_DIR}/BenchmarkResults.txt) \
+   >(go run test/integration/benchmark/jsonify/main.go ${ARTIFACTS_DIR}/BenchmarkResults_benchmark_$(date -u +%Y-%m-%dT%H:%M:%SZ).json || cat > /dev/null)

test/integration/BUILD

@@ -37,6 +37,7 @@ filegroup(
         ":package-srcs",
         "//test/integration/apiserver:all-srcs",
         "//test/integration/auth:all-srcs",
+        "//test/integration/benchmark/jsonify:all-srcs",
         "//test/integration/client:all-srcs",
         "//test/integration/configmap:all-srcs",
         "//test/integration/daemonset:all-srcs",

test/integration/benchmark/jsonify/BUILD

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "k8s.io/kubernetes/test/integration/benchmark/jsonify",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//test/e2e/perftype:go_default_library",
+        "//vendor/golang.org/x/tools/benchmark/parse:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "jsonify",
+    embed = [":go_default_library"],
+    importpath = "k8s.io/kubernetes/test/integration/benchmark/jsonify",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/integration/benchmark/jsonify/main.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	benchparse "golang.org/x/tools/benchmark/parse"
+	"k8s.io/kubernetes/test/e2e/perftype"
+)
+
+func main() {
+	err := run()
+	if err != nil {
+		panic(err)
+	}
+}
+
+func run() error {
+	if len(os.Args) < 2 {
+		return fmt.Errorf("output filename is a required argument")
+	}
+	benchmarkSet, err := benchparse.ParseSet(os.Stdin)
+	if err != nil {
+		return err
+	}
+	data := perftype.PerfData{Version: "v1"}
+	for _, benchMarks := range benchmarkSet {
+		for _, benchMark := range benchMarks {
+			data.DataItems = appendIfMeasured(data.DataItems, benchMark, benchparse.NsPerOp, "time", "μs", benchMark.NsPerOp/1000.0)
+			data.DataItems = appendIfMeasured(data.DataItems, benchMark, benchparse.MBPerS, "throughput", "MBps", benchMark.MBPerS)
+			data.DataItems = appendIfMeasured(data.DataItems, benchMark, benchparse.AllocedBytesPerOp, "allocated", "bytes", float64(benchMark.AllocedBytesPerOp))
+			data.DataItems = appendIfMeasured(data.DataItems, benchMark, benchparse.AllocsPerOp, "allocations", "1", float64(benchMark.AllocsPerOp))
+			data.DataItems = appendIfMeasured(data.DataItems, benchMark, 0, "iterations", "1", float64(benchMark.N))
+		}
+	}
+	output := &bytes.Buffer{}
+	if err := json.NewEncoder(output).Encode(data); err != nil {
+		return err
+	}
+	formatted := &bytes.Buffer{}
+	if err := json.Indent(formatted, output.Bytes(), "", "  "); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(os.Args[1], formatted.Bytes(), 0664)
+}
+
+func appendIfMeasured(items []perftype.DataItem, benchmark *benchparse.Benchmark, metricType int, metricName string, unit string, value float64) []perftype.DataItem {
+	if metricType != 0 && (benchmark.Measured&metricType) == 0 {
+		return items
+	}
+	return append(items, perftype.DataItem{
+		Unit: unit,
+		Labels: map[string]string{
+			"benchmark":  benchmark.Name,
+			"metricName": metricName},
+		Data: map[string]float64{
+			"value": value}})
+}

vendor/BUILD

@@ -387,6 +387,7 @@ filegroup(
         "//vendor/golang.org/x/text/unicode/norm:all-srcs",
         "//vendor/golang.org/x/text/width:all-srcs",
         "//vendor/golang.org/x/time/rate:all-srcs",
+        "//vendor/golang.org/x/tools/benchmark/parse:all-srcs",
         "//vendor/golang.org/x/tools/container/intsets:all-srcs",
         "//vendor/golang.org/x/tools/go/ast/astutil:all-srcs",
         "//vendor/golang.org/x/tools/go/vcs:all-srcs",

vendor/golang.org/x/tools/benchmark/parse/BUILD

@@ -0,0 +1,22 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["parse.go"],
+    importpath = "golang.org/x/tools/benchmark/parse",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/golang.org/x/tools/benchmark/parse/parse.go

@@ -0,0 +1,131 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package parse provides support for parsing benchmark results as
+// generated by 'go test -bench'.
+package parse
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+)
+
+// Flags used by Benchmark.Measured to indicate
+// which measurements a Benchmark contains.
+const (
+	NsPerOp = 1 << iota
+	MBPerS
+	AllocedBytesPerOp
+	AllocsPerOp
+)
+
+// Benchmark is one run of a single benchmark.
+type Benchmark struct {
+	Name              string  // benchmark name
+	N                 int     // number of iterations
+	NsPerOp           float64 // nanoseconds per iteration
+	AllocedBytesPerOp uint64  // bytes allocated per iteration
+	AllocsPerOp       uint64  // allocs per iteration
+	MBPerS            float64 // MB processed per second
+	Measured          int     // which measurements were recorded
+	Ord               int     // ordinal position within a benchmark run
+}
+
+// ParseLine extracts a Benchmark from a single line of testing.B
+// output.
+func ParseLine(line string) (*Benchmark, error) {
+	fields := strings.Fields(line)
+
+	// Two required, positional fields: Name and iterations.
+	if len(fields) < 2 {
+		return nil, fmt.Errorf("two fields required, have %d", len(fields))
+	}
+	if !strings.HasPrefix(fields[0], "Benchmark") {
+		return nil, fmt.Errorf(`first field does not start with "Benchmark"`)
+	}
+	n, err := strconv.Atoi(fields[1])
+	if err != nil {
+		return nil, err
+	}
+	b := &Benchmark{Name: fields[0], N: n}
+
+	// Parse any remaining pairs of fields; we've parsed one pair already.
+	for i := 1; i < len(fields)/2; i++ {
+		b.parseMeasurement(fields[i*2], fields[i*2+1])
+	}
+	return b, nil
+}
+
+func (b *Benchmark) parseMeasurement(quant string, unit string) {
+	switch unit {
+	case "ns/op":
+		if f, err := strconv.ParseFloat(quant, 64); err == nil {
+			b.NsPerOp = f
+			b.Measured |= NsPerOp
+		}
+	case "MB/s":
+		if f, err := strconv.ParseFloat(quant, 64); err == nil {
+			b.MBPerS = f
+			b.Measured |= MBPerS
+		}
+	case "B/op":
+		if i, err := strconv.ParseUint(quant, 10, 64); err == nil {
+			b.AllocedBytesPerOp = i
+			b.Measured |= AllocedBytesPerOp
+		}
+	case "allocs/op":
+		if i, err := strconv.ParseUint(quant, 10, 64); err == nil {
+			b.AllocsPerOp = i
+			b.Measured |= AllocsPerOp
+		}
+	}
+}
+
+func (b *Benchmark) String() string {
+	buf := new(bytes.Buffer)
+	fmt.Fprintf(buf, "%s %d", b.Name, b.N)
+	if (b.Measured & NsPerOp) != 0 {
+		fmt.Fprintf(buf, " %.2f ns/op", b.NsPerOp)
+	}
+	if (b.Measured & MBPerS) != 0 {
+		fmt.Fprintf(buf, " %.2f MB/s", b.MBPerS)
+	}
+	if (b.Measured & AllocedBytesPerOp) != 0 {
+		fmt.Fprintf(buf, " %d B/op", b.AllocedBytesPerOp)
+	}
+	if (b.Measured & AllocsPerOp) != 0 {
+		fmt.Fprintf(buf, " %d allocs/op", b.AllocsPerOp)
+	}
+	return buf.String()
+}
+
+// Set is a collection of benchmarks from one
+// testing.B run, keyed by name to facilitate comparison.
+type Set map[string][]*Benchmark
+
+// ParseSet extracts a Set from testing.B output.
+// ParseSet preserves the order of benchmarks that have identical
+// names.
+func ParseSet(r io.Reader) (Set, error) {
+	bb := make(Set)
+	scan := bufio.NewScanner(r)
+	ord := 0
+	for scan.Scan() {
+		if b, err := ParseLine(scan.Text()); err == nil {
+			b.Ord = ord
+			ord++
+			bb[b.Name] = append(bb[b.Name], b)
+		}
+	}
+
+	if err := scan.Err(); err != nil {
+		return nil, err
+	}
+
+	return bb, nil
+}