diff --git a/.gitignore b/.gitignore index 9aa7a78aa3c..02a2a4b1a78 100644 --- a/.gitignore +++ b/.gitignore @@ -111,6 +111,7 @@ kubernetes.tar.gz # TODO(thockin): uncomment this when we stop committing the generated files. #zz_generated.* zz_generated.openapi.go +zz_generated_*_test.go # make-related metadata /.make/ diff --git a/build/common.sh b/build/common.sh index 1066a7d8c1e..ea865f72297 100755 --- a/build/common.sh +++ b/build/common.sh @@ -596,6 +596,7 @@ function kube::build::run_build_command_ex() { --env "KUBE_FASTBUILD=${KUBE_FASTBUILD:-false}" --env "KUBE_BUILDER_OS=${OSTYPE:-notdetected}" --env "KUBE_VERBOSE=${KUBE_VERBOSE}" + --env "KUBE_BUILD_WITH_COVERAGE=${KUBE_BUILD_WITH_COVERAGE:-}" --env "GOFLAGS=${GOFLAGS:-}" --env "GOLDFLAGS=${GOLDFLAGS:-}" --env "GOGCFLAGS=${GOGCFLAGS:-}" diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index 437c8fa2176..9eb52a7d20b 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -228,6 +228,15 @@ readonly KUBE_STATIC_LIBRARIES=( kubectl ) +# Fully-qualified package names that we want to instrument for coverage information. +readonly KUBE_COVERAGE_INSTRUMENTED_PACKAGES=( + k8s.io/kubernetes/cmd/kube-apiserver + k8s.io/kubernetes/cmd/kube-controller-manager + k8s.io/kubernetes/cmd/kube-scheduler + k8s.io/kubernetes/cmd/kube-proxy + k8s.io/kubernetes/cmd/kubelet +) + # KUBE_CGO_OVERRIDES is a space-separated list of binaries which should be built # with CGO enabled, assuming CGO is supported on the target platform. # This overrides any entry in KUBE_STATIC_LIBRARIES. @@ -458,6 +467,100 @@ kube::golang::outfile_for_binary() { echo "${output_path}/${bin}" } +# Argument: the name of a Kubernetes package. +# Returns 0 if the binary can be built with coverage, 1 otherwise. +# NB: this ignores whether coverage is globally enabled or not. +kube::golang::is_instrumented_package() { + return $(kube::util::array_contains "$1" "${KUBE_COVERAGE_INSTRUMENTED_PACKAGES[@]}") +} + +# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler) +# Echos the path to a dummy test used for coverage information. +kube::golang::path_for_coverage_dummy_test() { + local package="$1" + local path="${KUBE_GOPATH}/src/${package}" + local name=$(basename "${package}") + echo "$path/zz_generated_${name}_test.go" +} + +# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler). +# Creates a dummy unit test on disk in the source directory for the given package. +# This unit test will invoke the package's standard entry point when run. +kube::golang::create_coverage_dummy_test() { + local package="$1" + local name="$(basename "${package}")" + cat < $(kube::golang::path_for_coverage_dummy_test "${package}") +package main +import ( + "testing" + "k8s.io/kubernetes/pkg/util/coverage" +) + +func TestMain(m *testing.M) { + // Get coverage running + coverage.InitCoverage("${name}") + + // Go! + main() + + // Make sure we actually write the profiling information to disk, if we make it here. + // On long-running services, or anything that calls os.Exit(), this is insufficient, + // so we also flush periodically with a default period of five seconds (configurable by + // the KUBE_COVERAGE_FLUSH_INTERVAL environment variable). + coverage.FlushCoverage() +} +EOF +} + +# Argument: the name of a Kubernetes package (e.g. k8s.io/kubernetes/cmd/kube-scheduler). +# Deletes a test generated by kube::golang::create_coverage_dummy_test. +# It is not an error to call this for a nonexistent test. +kube::golang::delete_coverage_dummy_test() { + local package="$1" + rm -f $(kube::golang::path_for_coverage_dummy_test "${package}") +} + +# Arguments: a list of kubernetes packages to build. +# Expected variables: ${build_args} should be set to an array of Go build arguments. +# In addition, ${package} and ${platform} should have been set earlier, and if +# ${build_with_coverage} is set, coverage instrumentation will be enabled. +# +# Invokes Go to actually build some packages. If coverage is disabled, simply invokes +# go install. If coverage is enabled, builds covered binaries using go test, temporarily +# producing the required unit test files and then cleaning up after itself. +# Non-covered binaries are then built using go install as usual. +kube::golang::build_some_binaries() { + if [[ -n "${build_with_coverage:-}" ]]; then + local -a uncovered=() + for package in "$@"; do + if kube::golang::is_instrumented_package "${package}"; then + V=2 kube::log::info "Building ${package} with coverage..." + + kube::golang::create_coverage_dummy_test "${package}" + kube::util::trap_add "kube::golang::delete_coverage_dummy_test \"${package}\"" EXIT + + go test -c -o "$(kube::golang::outfile_for_binary "${package}" "${platform}")" \ + -covermode count \ + -coverpkg k8s.io/... \ + "${build_args[@]}" \ + -tags coverage \ + "${package}" + else + uncovered+=("${package}") + fi + done + if [[ "${#uncovered[@]}" != 0 ]]; then + V=2 kube::log::info "Building ${uncovered[@]} without coverage..." + go install "${build_args[@]}" "${uncovered[@]}" + else + V=2 kube::log::info "Nothing to build without coverage." + fi + else + V=2 kube::log::info "Coverage is disabled." + go install "${build_args[@]}" "$@" + fi +} + kube::golang::build_binaries_for_platform() { local platform=$1 @@ -477,18 +580,24 @@ kube::golang::build_binaries_for_platform() { fi done + local -a build_args if [[ "${#statics[@]}" != 0 ]]; then - CGO_ENABLED=0 go install -installsuffix static "${goflags[@]:+${goflags[@]}}" \ - -gcflags "${gogcflags}" \ - -ldflags "${goldflags}" \ - "${statics[@]:+${statics[@]}}" + build_args=( + -installsuffix static + ${goflags:+"${goflags[@]}"} + -gcflags "${gogcflags:-}" + -ldflags "${goldflags:-}" + ) + CGO_ENABLED=0 kube::golang::build_some_binaries "${statics[@]}" fi if [[ "${#nonstatics[@]}" != 0 ]]; then - go install "${goflags[@]:+${goflags[@]}}" \ - -gcflags "${gogcflags}" \ - -ldflags "${goldflags}" \ - "${nonstatics[@]:+${nonstatics[@]}}" + build_args=( + ${goflags:+"${goflags[@]}"} + -gcflags "${gogcflags:-}" + -ldflags "${goldflags:-}" + ) + kube::golang::build_some_binaries "${nonstatics[@]}" fi for test in "${tests[@]:+${tests[@]}}"; do @@ -497,9 +606,9 @@ kube::golang::build_binaries_for_platform() { mkdir -p "$(dirname ${outfile})" go test -c \ - "${goflags[@]:+${goflags[@]}}" \ - -gcflags "${gogcflags}" \ - -ldflags "${goldflags}" \ + ${goflags:+"${goflags[@]}"} \ + -gcflags "${gogcflags:-}" \ + -ldflags "${goldflags:-}" \ -o "${outfile}" \ "${testpkg}" done @@ -552,10 +661,11 @@ kube::golang::build_binaries() { host_platform=$(kube::golang::host_platform) # Use eval to preserve embedded quoted strings. - local goflags goldflags gogcflags + local goflags goldflags gogcflags build_with_coverage eval "goflags=(${GOFLAGS:-})" goldflags="${GOLDFLAGS:-} $(kube::version::ldflags)" gogcflags="${GOGCFLAGS:-}" + build_with_coverage="${KUBE_BUILD_WITH_COVERAGE:-}" local -a targets=() local arg diff --git a/hack/lib/util.sh b/hack/lib/util.sh index a24b1093591..19220c93c3b 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -18,6 +18,20 @@ kube::util::sortable_date() { date "+%Y%m%d-%H%M%S" } +# arguments: target, item1, item2, item3, ... +# returns 0 if target is in the given items, 1 otherwise. +kube::util::array_contains() { + local search="$1" + local element + shift + for element; do + if [[ "${element}" == "${search}" ]]; then + return 0 + fi + done + return 1 +} + kube::util::wait_for_url() { local url=$1 local prefix=${2:-} diff --git a/pkg/util/BUILD b/pkg/util/BUILD index 2cd593108e7..d80a3832de1 100644 --- a/pkg/util/BUILD +++ b/pkg/util/BUILD @@ -16,6 +16,7 @@ filegroup( "//pkg/util/config:all-srcs", "//pkg/util/configz:all-srcs", "//pkg/util/conntrack:all-srcs", + "//pkg/util/coverage:all-srcs", "//pkg/util/dbus:all-srcs", "//pkg/util/ebtables:all-srcs", "//pkg/util/env:all-srcs", diff --git a/pkg/util/coverage/BUILD b/pkg/util/coverage/BUILD new file mode 100644 index 00000000000..30b93dc274c --- /dev/null +++ b/pkg/util/coverage/BUILD @@ -0,0 +1,25 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "coverage_disabled.go", + "fake_test_deps.go", + ], + importpath = "k8s.io/kubernetes/pkg/util/coverage", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/util/coverage/OWNERS b/pkg/util/coverage/OWNERS new file mode 100644 index 00000000000..f517735525c --- /dev/null +++ b/pkg/util/coverage/OWNERS @@ -0,0 +1,8 @@ +approvers: + - bentheelder + - spiffxp +reviewers: + - bentheelder + - spiffxp +labels: + - sig/testing diff --git a/pkg/util/coverage/coverage.go b/pkg/util/coverage/coverage.go new file mode 100644 index 00000000000..a6cdb2e73d4 --- /dev/null +++ b/pkg/util/coverage/coverage.go @@ -0,0 +1,91 @@ +// +build coverage + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package coverage provides tools for coverage-instrumented binaries to collect and +// flush coverage information. +package coverage + +import ( + "flag" + "fmt" + "github.com/golang/glog" + "k8s.io/apimachinery/pkg/util/wait" + "os" + "testing" + "time" +) + +var coverageFile string + +// tempCoveragePath returns a temporary file to write coverage information to. +// The file is in the same directory as the destination, ensuring os.Rename will work. +func tempCoveragePath() string { + return coverageFile + ".tmp" +} + +// InitCoverage is called from the dummy unit test to prepare Go's coverage framework. +// Clients should never need to call it. +func InitCoverage(name string) { + // We read the coverage destination in from the KUBE_COVERAGE_FILE env var, + // or if it's empty we just use a default in /tmp + coverageFile = os.Getenv("KUBE_COVERAGE_FILE") + if coverageFile == "" { + coverageFile = "/tmp/k8s-" + name + ".cov" + } + fmt.Println("Dumping coverage information to " + coverageFile) + + flushInterval := 5 * time.Second + requestedInterval := os.Getenv("KUBE_COVERAGE_FLUSH_INTERVAL") + if requestedInterval != "" { + if duration, err := time.ParseDuration(requestedInterval); err == nil { + flushInterval = duration + } else { + panic("Invalid KUBE_COVERAGE_FLUSH_INTERVAL value; try something like '30s'.") + } + } + + // Set up the unit test framework with the required arguments to activate test coverage. + flag.CommandLine.Parse([]string{"-test.coverprofile", tempCoveragePath()}) + + // Begin periodic logging + go wait.Forever(FlushCoverage, flushInterval) +} + +// FlushCoverage flushes collected coverage information to disk. +// The destination file is configured at startup and cannot be changed. +// Calling this function also sends a line like "coverage: 5% of statements" to stdout. +func FlushCoverage() { + // We're not actually going to run any tests, but we need Go to think we did so it writes + // coverage information to disk. To achieve this, we create a bunch of empty test suites and + // have it "run" them. + tests := []testing.InternalTest{} + benchmarks := []testing.InternalBenchmark{} + examples := []testing.InternalExample{} + + var deps fakeTestDeps + + dummyRun := testing.MainStart(deps, tests, benchmarks, examples) + dummyRun.Run() + + // Once it writes to the temporary path, we move it to the intended path. + // This gets us atomic updates from the perspective of another process trying to access + // the file. + if err := os.Rename(tempCoveragePath(), coverageFile); err != nil { + glog.Errorf("Couldn't move coverage file from %s to %s", coverageFile, tempCoveragePath()) + } +} diff --git a/pkg/util/coverage/coverage_disabled.go b/pkg/util/coverage/coverage_disabled.go new file mode 100644 index 00000000000..2e2f12f6347 --- /dev/null +++ b/pkg/util/coverage/coverage_disabled.go @@ -0,0 +1,29 @@ +// +build !coverage + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package coverage + +// InitCoverage is illegal when not running with coverage. +func InitCoverage(name string) { + panic("Called InitCoverage when not built with coverage instrumentation.") +} + +// FlushCoverage is a no-op when not running with coverage. +func FlushCoverage() { + +} diff --git a/pkg/util/coverage/fake_test_deps.go b/pkg/util/coverage/fake_test_deps.go new file mode 100644 index 00000000000..8ca0b9b0934 --- /dev/null +++ b/pkg/util/coverage/fake_test_deps.go @@ -0,0 +1,54 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package coverage + +import ( + "io" +) + +// This is an implementation of testing.testDeps. It doesn't need to do anything, because +// no tests are actually run. It does need a concrete implementation of at least ImportPath, +// which is called unconditionally when running tests. +type fakeTestDeps struct{} + +func (fakeTestDeps) ImportPath() string { + return "" +} + +func (fakeTestDeps) MatchString(pat, str string) (bool, error) { + return false, nil +} + +func (fakeTestDeps) StartCPUProfile(io.Writer) error { + return nil +} + +func (fakeTestDeps) StopCPUProfile() {} + +func (fakeTestDeps) StartTestLog(io.Writer) {} + +func (fakeTestDeps) StopTestLog() error { + return nil +} + +func (fakeTestDeps) WriteHeapProfile(io.Writer) error { + return nil +} + +func (fakeTestDeps) WriteProfileTo(string, io.Writer, int) error { + return nil +}