diff --git a/hack/test-integration.sh b/hack/test-integration.sh index 9f47d1e257a..0094dee472f 100755 --- a/hack/test-integration.sh +++ b/hack/test-integration.sh @@ -52,7 +52,7 @@ runTests() { # KUBE_RACE="-race" KUBE_GOFLAGS="-tags 'integration no-docker' " \ KUBE_RACE="" \ - KUBE_TIMEOUT="${KUBE_TIMEOUT}" \ + KUBE_TEST_ARGS="--integration-timeout=${KUBE_TIMEOUT}" KUBE_TEST_API_VERSIONS="$1" \ "${KUBE_ROOT}/hack/test-go.sh" test/integration diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 6b601fba7da..f7aa5be06ca 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -206,6 +206,7 @@ insecure-bind-address insecure-port insecure-skip-tls-verify instance-name-prefix +integration-timeout iptables-masquerade-bit iptables-sync-period ir-data-source @@ -448,6 +449,7 @@ target-port tcp-services terminated-pod-gc-threshold test-timeout +timeout-full-stack tls-cert-file tls-private-key-file to-version diff --git a/test/integration/main_test.go b/test/integration/main_test.go new file mode 100644 index 00000000000..400321fae59 --- /dev/null +++ b/test/integration/main_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "flag" + "fmt" + "os" + "runtime" + "strings" + "testing" + "time" +) + +var integrationTimeout = flag.Duration("integration-timeout", 10*time.Minute, "If non-zero timeout all tests after this amount of time.") +var fullStack = flag.Bool("timeout-full-stack", false, "If true, don't filter stacks on timeout") +var timer *time.Timer + +func printInterestingStacks() { + buf := make([]byte, 2<<20) + buf = buf[:runtime.Stack(buf, true)] + stacks := [][]string{} + for _, stack := range strings.Split(string(buf), "\n\n") { + lines := strings.Split(stack, "\n") + for _, line := range lines { + if strings.Contains(line, "src/k8s.io") || *fullStack { + stacks = append(stacks, lines) + break + } + } + } + for _, stack := range stacks { + for _, line := range stack { + fmt.Printf("%s\n", line) + } + fmt.Printf("\n") + } + return +} + +func startAlarm() { + if *integrationTimeout > 0 { + timer = time.AfterFunc(*integrationTimeout, func() { + printInterestingStacks() + fmt.Printf("Integration tests timed out after %v\n", integrationTimeout.String()) + os.Exit(1) + }) + } +} + +func stopAlarm() { + if timer != nil { + timer.Stop() + } +} + +/** + * Adds a custom test main handler (see: TestMain in https://golang.org/pkg/testing/ for details) + * This custom does the standard timeout, but strips non-kubernetes stacks out of the stack trace. + * It filters things like: + +goroutine 466 [IO wait, 7 minutes]: +net.runtime_pollWait(0x7fd74c4672c0, 0x72, 0xc821614000) + /usr/local/go/src/runtime/netpoll.go:160 +0x60 +net.(*pollDesc).Wait(0xc8215c21b0, 0x72, 0x0, 0x0) + /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a +net.(*pollDesc).WaitRead(0xc8215c21b0, 0x0, 0x0) + /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36 +net.(*netFD).Read(0xc8215c2150, 0xc821614000, 0x1000, 0x1000, 0x0, 0x7fd74c491050, 0xc820014058) + /usr/local/go/src/net/fd_unix.go:250 +0x23a +net.(*conn).Read(0xc820a5a090, 0xc821614000, 0x1000, 0x1000, 0x0, 0x0, 0x0) + /usr/local/go/src/net/net.go:172 +0xe4 +net/http.noteEOFReader.Read(0x7fd74c465258, 0xc820a5a090, 0xc8215f0068, 0xc821614000, 0x1000, 0x1000, 0x405773, 0x0, 0x0) + /usr/local/go/src/net/http/transport.go:1687 +0x67 +net/http.(*noteEOFReader).Read(0xc8215ae1a0, 0xc821614000, 0x1000, 0x1000, 0xc82159ad1d, 0x0, 0x0) + :284 +0xd0 +bufio.(*Reader).fill(0xc8202a2b40) + /usr/local/go/src/bufio/bufio.go:97 +0x1e9 +bufio.(*Reader).Peek(0xc8202a2b40, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0) + /usr/local/go/src/bufio/bufio.go:132 +0xcc +net/http.(*persistConn).readLoop(0xc8215f0000) + /usr/local/go/src/net/http/transport.go:1073 +0x177 +created by net/http.(*Transport).dialConn + /usr/local/go/src/net/http/transport.go:857 +0x10a6 + + * See printInterestingStacks above for details. + * We may want to get even more aggressive in the future. +*/ +// TestMain replaces the built-in golang Main for running tests. See https://golang.org/pkg/testing/ +func TestMain(m *testing.M) { + flag.Parse() + startAlarm() + v := m.Run() + stopAlarm() + os.Exit(v) +}