Merge pull request #67971 from Katharine/coverage-instrumentation

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

Add ability to build with runtime coverage instrumentation

**What this PR does / why we need it**:

This PR adds the ability to instrument a subset of kubernetes binaries to report code coverage information. The specific use-case is to help determine coverage of our end-to-end Conformance tests, as well as provide data that can be used to help determine where to focus. This PR focuses on making it possible to build with instrumentation; collecting and using the generated coverage data will be done in later PRs. For more details as to the intent, see the [design doc](https://docs.google.com/document/d/1FKMBFxz7vtA-6ZgUkA47F8m6yR00fwqLcXMVJqsHt0g/edit?usp=sharing) (google doc; requires kubernetes-dev membership).

Specifically, this PR adds a new `KUBE_BUILD_WITH_COVERAGE` make variable, which when set will cause `kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, `kube-proxy` and `kubelet` to be built with coverage instrumentation. These coverage-instrumented binaries will flush coverage information to disk every five seconds, defaulting to a temporary directory unless the `KUBE_COVERAGE_FILE` environment variable is set at launch, in which case it will write to that file instead.

The mechanism used to achieve coverage instrumentation is to build the targeted binaries as "unit tests" with coverage enabled, and then rigging the unit tests to just execute the binary's usual entry point. This is implemented only for the bash build system.

/sig testing

```release-note
NONE
```
This commit is contained in:
Kubernetes Submit Queue 2018-09-01 01:32:52 -07:00 committed by GitHub
commit 68d22a878d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 346 additions and 12 deletions

1
.gitignore vendored
View File

@ -111,6 +111,7 @@ kubernetes.tar.gz
# TODO(thockin): uncomment this when we stop committing the generated files.
#zz_generated.*
zz_generated.openapi.go
zz_generated_*_test.go
# make-related metadata
/.make/

View File

@ -596,6 +596,7 @@ function kube::build::run_build_command_ex() {
--env "KUBE_FASTBUILD=${KUBE_FASTBUILD:-false}"
--env "KUBE_BUILDER_OS=${OSTYPE:-notdetected}"
--env "KUBE_VERBOSE=${KUBE_VERBOSE}"
--env "KUBE_BUILD_WITH_COVERAGE=${KUBE_BUILD_WITH_COVERAGE:-}"
--env "GOFLAGS=${GOFLAGS:-}"
--env "GOLDFLAGS=${GOLDFLAGS:-}"
--env "GOGCFLAGS=${GOGCFLAGS:-}"

View File

@ -228,6 +228,15 @@ readonly KUBE_STATIC_LIBRARIES=(
kubectl
)
# Fully-qualified package names that we want to instrument for coverage information.
readonly KUBE_COVERAGE_INSTRUMENTED_PACKAGES=(
k8s.io/kubernetes/cmd/kube-apiserver
k8s.io/kubernetes/cmd/kube-controller-manager
k8s.io/kubernetes/cmd/kube-scheduler
k8s.io/kubernetes/cmd/kube-proxy
k8s.io/kubernetes/cmd/kubelet
)
# KUBE_CGO_OVERRIDES is a space-separated list of binaries which should be built
# with CGO enabled, assuming CGO is supported on the target platform.
# This overrides any entry in KUBE_STATIC_LIBRARIES.
@ -458,6 +467,100 @@ kube::golang::outfile_for_binary() {
echo "${output_path}/${bin}"
}
# Argument: the name of a Kubernetes package.
# Returns 0 if the binary can be built with coverage, 1 otherwise.
# NB: this ignores whether coverage is globally enabled or not.
kube::golang::is_instrumented_package() {
return $(kube::util::array_contains "$1" "${KUBE_COVERAGE_INSTRUMENTED_PACKAGES[@]}")
}
# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler)
# Echos the path to a dummy test used for coverage information.
kube::golang::path_for_coverage_dummy_test() {
local package="$1"
local path="${KUBE_GOPATH}/src/${package}"
local name=$(basename "${package}")
echo "$path/zz_generated_${name}_test.go"
}
# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler).
# Creates a dummy unit test on disk in the source directory for the given package.
# This unit test will invoke the package's standard entry point when run.
kube::golang::create_coverage_dummy_test() {
local package="$1"
local name="$(basename "${package}")"
cat <<EOF > $(kube::golang::path_for_coverage_dummy_test "${package}")
package main
import (
"testing"
"k8s.io/kubernetes/pkg/util/coverage"
)
func TestMain(m *testing.M) {
// Get coverage running
coverage.InitCoverage("${name}")
// Go!
main()
// Make sure we actually write the profiling information to disk, if we make it here.
// On long-running services, or anything that calls os.Exit(), this is insufficient,
// so we also flush periodically with a default period of five seconds (configurable by
// the KUBE_COVERAGE_FLUSH_INTERVAL environment variable).
coverage.FlushCoverage()
}
EOF
}
# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler).
# Deletes a test generated by kube::golang::create_coverage_dummy_test.
# It is not an error to call this for a nonexistent test.
kube::golang::delete_coverage_dummy_test() {
local package="$1"
rm -f $(kube::golang::path_for_coverage_dummy_test "${package}")
}
# Arguments: a list of kubernetes packages to build.
# Expected variables: ${build_args} should be set to an array of Go build arguments.
# In addition, ${package} and ${platform} should have been set earlier, and if
# ${build_with_coverage} is set, coverage instrumentation will be enabled.
#
# Invokes Go to actually build some packages. If coverage is disabled, simply invokes
# go install. If coverage is enabled, builds covered binaries using go test, temporarily
# producing the required unit test files and then cleaning up after itself.
# Non-covered binaries are then built using go install as usual.
kube::golang::build_some_binaries() {
if [[ -n "${build_with_coverage:-}" ]]; then
local -a uncovered=()
for package in "$@"; do
if kube::golang::is_instrumented_package "${package}"; then
V=2 kube::log::info "Building ${package} with coverage..."
kube::golang::create_coverage_dummy_test "${package}"
kube::util::trap_add "kube::golang::delete_coverage_dummy_test \"${package}\"" EXIT
go test -c -o "$(kube::golang::outfile_for_binary "${package}" "${platform}")" \
-covermode count \
-coverpkg k8s.io/... \
"${build_args[@]}" \
-tags coverage \
"${package}"
else
uncovered+=("${package}")
fi
done
if [[ "${#uncovered[@]}" != 0 ]]; then
V=2 kube::log::info "Building ${uncovered[@]} without coverage..."
go install "${build_args[@]}" "${uncovered[@]}"
else
V=2 kube::log::info "Nothing to build without coverage."
fi
else
V=2 kube::log::info "Coverage is disabled."
go install "${build_args[@]}" "$@"
fi
}
kube::golang::build_binaries_for_platform() {
local platform=$1
@ -477,18 +580,24 @@ kube::golang::build_binaries_for_platform() {
fi
done
local -a build_args
if [[ "${#statics[@]}" != 0 ]]; then
CGO_ENABLED=0 go install -installsuffix static "${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${statics[@]:+${statics[@]}}"
build_args=(
-installsuffix static
${goflags:+"${goflags[@]}"}
-gcflags "${gogcflags:-}"
-ldflags "${goldflags:-}"
)
CGO_ENABLED=0 kube::golang::build_some_binaries "${statics[@]}"
fi
if [[ "${#nonstatics[@]}" != 0 ]]; then
go install "${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
"${nonstatics[@]:+${nonstatics[@]}}"
build_args=(
${goflags:+"${goflags[@]}"}
-gcflags "${gogcflags:-}"
-ldflags "${goldflags:-}"
)
kube::golang::build_some_binaries "${nonstatics[@]}"
fi
for test in "${tests[@]:+${tests[@]}}"; do
@ -497,9 +606,9 @@ kube::golang::build_binaries_for_platform() {
mkdir -p "$(dirname ${outfile})"
go test -c \
"${goflags[@]:+${goflags[@]}}" \
-gcflags "${gogcflags}" \
-ldflags "${goldflags}" \
${goflags:+"${goflags[@]}"} \
-gcflags "${gogcflags:-}" \
-ldflags "${goldflags:-}" \
-o "${outfile}" \
"${testpkg}"
done
@ -552,10 +661,11 @@ kube::golang::build_binaries() {
host_platform=$(kube::golang::host_platform)
# Use eval to preserve embedded quoted strings.
local goflags goldflags gogcflags
local goflags goldflags gogcflags build_with_coverage
eval "goflags=(${GOFLAGS:-})"
goldflags="${GOLDFLAGS:-} $(kube::version::ldflags)"
gogcflags="${GOGCFLAGS:-}"
build_with_coverage="${KUBE_BUILD_WITH_COVERAGE:-}"
local -a targets=()
local arg

View File

@ -18,6 +18,20 @@ kube::util::sortable_date() {
date "+%Y%m%d-%H%M%S"
}
# arguments: target, item1, item2, item3, ...
# returns 0 if target is in the given items, 1 otherwise.
kube::util::array_contains() {
local search="$1"
local element
shift
for element; do
if [[ "${element}" == "${search}" ]]; then
return 0
fi
done
return 1
}
kube::util::wait_for_url() {
local url=$1
local prefix=${2:-}

View File

@ -16,6 +16,7 @@ filegroup(
"//pkg/util/config:all-srcs",
"//pkg/util/configz:all-srcs",
"//pkg/util/conntrack:all-srcs",
"//pkg/util/coverage:all-srcs",
"//pkg/util/dbus:all-srcs",
"//pkg/util/ebtables:all-srcs",
"//pkg/util/env:all-srcs",

25
pkg/util/coverage/BUILD Normal file
View File

@ -0,0 +1,25 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"coverage_disabled.go",
"fake_test_deps.go",
],
importpath = "k8s.io/kubernetes/pkg/util/coverage",
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

8
pkg/util/coverage/OWNERS Normal file
View File

@ -0,0 +1,8 @@
approvers:
- bentheelder
- spiffxp
reviewers:
- bentheelder
- spiffxp
labels:
- sig/testing

View File

@ -0,0 +1,91 @@
// +build coverage
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package coverage provides tools for coverage-instrumented binaries to collect and
// flush coverage information.
package coverage
import (
"flag"
"fmt"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/util/wait"
"os"
"testing"
"time"
)
var coverageFile string
// tempCoveragePath returns a temporary file to write coverage information to.
// The file is in the same directory as the destination, ensuring os.Rename will work.
func tempCoveragePath() string {
return coverageFile + ".tmp"
}
// InitCoverage is called from the dummy unit test to prepare Go's coverage framework.
// Clients should never need to call it.
func InitCoverage(name string) {
// We read the coverage destination in from the KUBE_COVERAGE_FILE env var,
// or if it's empty we just use a default in /tmp
coverageFile = os.Getenv("KUBE_COVERAGE_FILE")
if coverageFile == "" {
coverageFile = "/tmp/k8s-" + name + ".cov"
}
fmt.Println("Dumping coverage information to " + coverageFile)
flushInterval := 5 * time.Second
requestedInterval := os.Getenv("KUBE_COVERAGE_FLUSH_INTERVAL")
if requestedInterval != "" {
if duration, err := time.ParseDuration(requestedInterval); err == nil {
flushInterval = duration
} else {
panic("Invalid KUBE_COVERAGE_FLUSH_INTERVAL value; try something like '30s'.")
}
}
// Set up the unit test framework with the required arguments to activate test coverage.
flag.CommandLine.Parse([]string{"-test.coverprofile", tempCoveragePath()})
// Begin periodic logging
go wait.Forever(FlushCoverage, flushInterval)
}
// FlushCoverage flushes collected coverage information to disk.
// The destination file is configured at startup and cannot be changed.
// Calling this function also sends a line like "coverage: 5% of statements" to stdout.
func FlushCoverage() {
// We're not actually going to run any tests, but we need Go to think we did so it writes
// coverage information to disk. To achieve this, we create a bunch of empty test suites and
// have it "run" them.
tests := []testing.InternalTest{}
benchmarks := []testing.InternalBenchmark{}
examples := []testing.InternalExample{}
var deps fakeTestDeps
dummyRun := testing.MainStart(deps, tests, benchmarks, examples)
dummyRun.Run()
// Once it writes to the temporary path, we move it to the intended path.
// This gets us atomic updates from the perspective of another process trying to access
// the file.
if err := os.Rename(tempCoveragePath(), coverageFile); err != nil {
glog.Errorf("Couldn't move coverage file from %s to %s", coverageFile, tempCoveragePath())
}
}

View File

@ -0,0 +1,29 @@
// +build !coverage
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package coverage
// InitCoverage is illegal when not running with coverage.
func InitCoverage(name string) {
panic("Called InitCoverage when not built with coverage instrumentation.")
}
// FlushCoverage is a no-op when not running with coverage.
func FlushCoverage() {
}

View File

@ -0,0 +1,54 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package coverage
import (
"io"
)
// This is an implementation of testing.testDeps. It doesn't need to do anything, because
// no tests are actually run. It does need a concrete implementation of at least ImportPath,
// which is called unconditionally when running tests.
type fakeTestDeps struct{}
func (fakeTestDeps) ImportPath() string {
return ""
}
func (fakeTestDeps) MatchString(pat, str string) (bool, error) {
return false, nil
}
func (fakeTestDeps) StartCPUProfile(io.Writer) error {
return nil
}
func (fakeTestDeps) StopCPUProfile() {}
func (fakeTestDeps) StartTestLog(io.Writer) {}
func (fakeTestDeps) StopTestLog() error {
return nil
}
func (fakeTestDeps) WriteHeapProfile(io.Writer) error {
return nil
}
func (fakeTestDeps) WriteProfileTo(string, io.Writer, int) error {
return nil
}