github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-14 05:36:12 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #38576 from fejta/log

Browse Source 
Automatic merge from submit-queue

Interrupt/Terminate all child processes on timeout. Auto-gen stepName

Signal all children of the finishRunning command. See http://stackoverflow.com/questions/22470193/why-wont-go-kill-a-child-process-correctly

Most commands started by e2e.go are shell scripts like `hack/ginkgo-e2e.sh`, `hack/e2e-internal/e2e-down.sh`, etc which quickly starts a subprocess such as the ginkgo binary. We spend ~1s in ginkgo-e2e.sh and hours inside the ginkgo binary. Therefore when we want to timeout it is important that we signal the child processes as well, which we can accomplish by starting the command in a new process group, and signaling `-pid` instead of `pid`

Ref https://github.com/kubernetes/test-infra/issues/1316 https://github.com/kubernetes/test-infra/issues/1250
Test results: https://github.com/kubernetes/kubernetes/pull/37868
This commit is contained in:
Kubernetes Submit Queue
2016-12-12 14:16:34 -08:00
committed by GitHub
parent 7671ccd402 bf7aa86932
commit d6e27fac03
1 changed files with 60 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
hack/e2e.go
View File

@@ -32,6 +32,7 @@ import (
"regexp"
"strconv"
"strings"
"syscall"
"text/template"
"time"
)
@@ -250,7 +251,7 @@ func run(deploy deployer) error {
os.Setenv("KUBE_RUNTIME_CONFIG", "batch/v2alpha1=true")
if *up {
if err := xmlWrap("TearDown", deploy.Down); err != nil {
if err := xmlWrap("TearDown Previous", deploy.Down); err != nil {
return fmt.Errorf("error tearing down previous cluster: %s", err)
}
}
@@ -292,18 +293,7 @@ func run(deploy deployer) error {
return fmt.Errorf("starting e2e cluster: %s", err)
}
if *dump != "" {
cmd := exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "get", "nodes", "-oyaml")
b, err := cmd.CombinedOutput()
if *verbose {
log.Printf("kubectl get nodes:\n%s", string(b))
}
if err == nil {
if err := ioutil.WriteFile(filepath.Join(*dump, "nodes.yaml"), b, 0644); err != nil {
errs = appendError(errs, fmt.Errorf("error writing nodes.yaml: %v", err))
}
} else {
errs = appendError(errs, fmt.Errorf("error running get nodes: %v", err))
}
errs = appendError(errs, xmlWrap("list nodes", listNodes))
}
}
@@ -323,22 +313,21 @@ func run(deploy deployer) error {
if *test {
errs = appendError(errs, xmlWrap("get kubeconfig", deploy.SetupKubecfg))
errs = appendError(errs, xmlWrap("kubectl version", func() error {
return finishRunning("kubectl version", exec.Command("./cluster/kubectl.sh", "version", "--match-server-version=false"))
return finishRunning(exec.Command("./cluster/kubectl.sh", "version", "--match-server-version=false"))
}))
// Individual tests will create their own JUnit, so don't xmlWrap.
if *skewTests {
errs = appendError(errs, SkewTest())
errs = appendError(errs, xmlWrap("SkewTest", SkewTest))
} else {
if err := xmlWrap("IsUp", deploy.IsUp); err != nil {
errs = appendError(errs, err)
} else {
errs = appendError(errs, Test())
errs = appendError(errs, xmlWrap("Test", Test))
}
}
}
if *kubemark {
errs = appendError(errs, KubemarkTest())
errs = appendError(errs, xmlWrap("Kubemark", KubemarkTest))
}
if len(errs) > 0 && *dump != "" {
@@ -379,6 +368,18 @@ func run(deploy deployer) error {
return nil
}
func listNodes() error {
cmd := exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "get", "nodes", "-oyaml")
b, err := cmd.CombinedOutput()
if *verbose {
log.Printf("kubectl get nodes:\n%s", string(b))
}
if err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(*dump, "nodes.yaml"), b, 0644)
}
func DiffResources(before, clusterUp, clusterDown, after []byte, location string) error {
if location == "" {
var err error
@@ -460,7 +461,7 @@ func Build() error {
// it's OK to download the docker image.
cmd := exec.Command("make", "quick-release")
cmd.Stdin = os.Stdin
if err := finishRunning("build-release", cmd); err != nil {
if err := finishRunning(cmd); err != nil {
return fmt.Errorf("error building kubernetes: %v", err)
}
return nil
@@ -489,11 +490,11 @@ func getDeployer() (deployer, error) {
type bash struct{}
func (b bash) Up() error {
return finishRunning("up", exec.Command("./hack/e2e-internal/e2e-up.sh"))
return finishRunning(exec.Command("./hack/e2e-internal/e2e-up.sh"))
}
func (b bash) IsUp() error {
return finishRunning("get status", exec.Command("./hack/e2e-internal/e2e-status.sh"))
return finishRunning(exec.Command("./hack/e2e-internal/e2e-status.sh"))
}
func (b bash) SetupKubecfg() error {
@@ -501,7 +502,7 @@ func (b bash) SetupKubecfg() error {
}
func (b bash) Down() error {
return finishRunning("teardown", exec.Command("./hack/e2e-internal/e2e-down.sh"))
return finishRunning(exec.Command("./hack/e2e-internal/e2e-down.sh"))
}
type kops struct {
@@ -588,10 +589,10 @@ func (k kops) Up() error {
if k.kubeVersion != "" {
createArgs = append(createArgs, "--kubernetes-version", k.kubeVersion)
}
if err := finishRunning("kops config", exec.Command(k.path, createArgs...)); err != nil {
if err := finishRunning(exec.Command(k.path, createArgs...)); err != nil {
return fmt.Errorf("kops configuration failed: %v", err)
}
if err := finishRunning("kops update", exec.Command(k.path, "update", "cluster", k.cluster, "--yes")); err != nil {
if err := finishRunning(exec.Command(k.path, "update", "cluster", k.cluster, "--yes")); err != nil {
return fmt.Errorf("kops bringup failed: %v", err)
}
// TODO(zmerlynn): More cluster validation. This should perhaps be
@@ -613,7 +614,7 @@ func (k kops) SetupKubecfg() error {
// Assume that if we already have it, it's good.
return nil
}
if err := finishRunning("kops export", exec.Command(k.path, "export", "kubecfg", k.cluster)); err != nil {
if err := finishRunning(exec.Command(k.path, "export", "kubecfg", k.cluster)); err != nil {
return fmt.Errorf("Failure exporting kops kubecfg: %v", err)
}
return nil
@@ -623,12 +624,12 @@ func (k kops) Down() error {
// We do a "kops get" first so the exit status of "kops delete" is
// more sensical in the case of a non-existant cluster. ("kops
// delete" will exit with status 1 on a non-existant cluster)
err := finishRunning("kops get", exec.Command(k.path, "get", "clusters", k.cluster))
err := finishRunning(exec.Command(k.path, "get", "clusters", k.cluster))
if err != nil {
// This is expected if the cluster doesn't exist.
return nil
}
return finishRunning("kops delete", exec.Command(k.path, "delete", "cluster", k.cluster, "--yes"))
return finishRunning(exec.Command(k.path, "delete", "cluster", k.cluster, "--yes"))
}
type kubernetesAnywhere struct {
@@ -706,7 +707,7 @@ func (k kubernetesAnywhere) writeConfig() error {
func (k kubernetesAnywhere) Up() error {
cmd := exec.Command("make", "-C", k.path, "WAIT_FOR_KUBECONFIG=y", "deploy-cluster")
if err := finishRunning("deploy-cluster", cmd); err != nil {
if err := finishRunning(cmd); err != nil {
return err
}
@@ -732,12 +733,12 @@ func (k kubernetesAnywhere) SetupKubecfg() error {
}
func (k kubernetesAnywhere) Down() error {
err := finishRunning("get kubeconfig-path", exec.Command("make", "-C", k.path, "kubeconfig-path"))
err := finishRunning(exec.Command("make", "-C", k.path, "kubeconfig-path"))
if err != nil {
// This is expected if the cluster doesn't exist.
return nil
}
return finishRunning("destroy-cluster", exec.Command("make", "-C", k.path, "FORCE_DESTROY=y", "destroy-cluster"))
return finishRunning(exec.Command("make", "-C", k.path, "FORCE_DESTROY=y", "destroy-cluster"))
}
func clusterSize(deploy deployer) (int, error) {
@@ -812,12 +813,12 @@ func waitForNodes(d deployer, nodes int, timeout time.Duration) error {
func DumpClusterLogs(location string) error {
log.Printf("Dumping cluster logs to: %v", location)
return finishRunning("dump cluster logs", exec.Command("./cluster/log-dump.sh", location))
return finishRunning(exec.Command("./cluster/log-dump.sh", location))
}
func KubemarkTest() error {
// Stop previous run
err := finishRunning("Stop kubemark", exec.Command("./test/kubemark/stop-kubemark.sh"))
err := finishRunning(exec.Command("./test/kubemark/stop-kubemark.sh"))
if err != nil {
return err
}
@@ -828,7 +829,7 @@ func KubemarkTest() error {
// stop the leaking resources for now, we want to be on the safe side
// and call it explictly in defer if the other one is not called.
defer xmlWrap("Deferred Stop kubemark", func() error {
return finishRunning("Stop kubemark", exec.Command("./test/kubemark/stop-kubemark.sh"))
return finishRunning(exec.Command("./test/kubemark/stop-kubemark.sh"))
})
// Start new run
@@ -844,7 +845,7 @@ func KubemarkTest() error {
os.Setenv("NUM_NODES", os.Getenv("KUBEMARK_NUM_NODES"))
os.Setenv("MASTER_SIZE", os.Getenv("KUBEMARK_MASTER_SIZE"))
err = xmlWrap("Start kubemark", func() error {
return finishRunning("Start kubemark", exec.Command("./test/kubemark/start-kubemark.sh"))
return finishRunning(exec.Command("./test/kubemark/start-kubemark.sh"))
})
if err != nil {
return err
@@ -857,13 +858,13 @@ func KubemarkTest() error {
}
test_args := os.Getenv("KUBEMARK_TEST_ARGS")
err = finishRunning("Run kubemark tests", exec.Command("./test/kubemark/run-e2e-tests.sh", "--ginkgo.focus="+focus, test_args))
err = finishRunning(exec.Command("./test/kubemark/run-e2e-tests.sh", "--ginkgo.focus="+focus, test_args))
if err != nil {
return err
}
err = xmlWrap("Stop kubemark", func() error {
return finishRunning("Stop kubemark", exec.Command("./test/kubemark/stop-kubemark.sh"))
return finishRunning(exec.Command("./test/kubemark/stop-kubemark.sh"))
})
if err != nil {
return err
@@ -896,13 +897,12 @@ func UpgradeTest(args string) error {
defer os.Unsetenv("E2E_REPORT_PREFIX")
}
os.Setenv("E2E_REPORT_PREFIX", "upgrade")
return finishRunning("Upgrade Ginkgo tests",
exec.Command(
"go", "run", "./hack/e2e.go",
"--test",
"--test_args="+args,
fmt.Sprintf("--v=%t", *verbose),
fmt.Sprintf("--check_version_skew=%t", *checkVersionSkew)))
return finishRunning(exec.Command(
"go", "run", "./hack/e2e.go",
"--test",
"--test_args="+args,
fmt.Sprintf("--v=%t", *verbose),
fmt.Sprintf("--check_version_skew=%t", *checkVersionSkew)))
}
func SkewTest() error {
@@ -911,28 +911,28 @@ func SkewTest() error {
return err
}
defer os.Chdir(old)
return finishRunning("Skewed Ginkgo tests",
exec.Command(
"go", "run", "./hack/e2e.go",
"--test",
"--test_args="+*testArgs,
fmt.Sprintf("--v=%t", *verbose),
fmt.Sprintf("--check_version_skew=%t", *checkVersionSkew)))
return finishRunning(exec.Command(
"go", "run", "./hack/e2e.go",
"--test",
"--test_args="+*testArgs,
fmt.Sprintf("--v=%t", *verbose),
fmt.Sprintf("--check_version_skew=%t", *checkVersionSkew)))
}
func Test() error {
// TODO(fejta): add a --federated or something similar
if os.Getenv("FEDERATION") != "true" {
return finishRunning("Ginkgo tests", exec.Command("./hack/ginkgo-e2e.sh", strings.Fields(*testArgs)...))
return finishRunning(exec.Command("./hack/ginkgo-e2e.sh", strings.Fields(*testArgs)...))
}
if *testArgs == "" {
*testArgs = "--ginkgo.focus=\\[Feature:Federation\\]"
}
return finishRunning("Federated Ginkgo tests", exec.Command("./hack/federated-ginkgo-e2e.sh", strings.Fields(*testArgs)...))
return finishRunning(exec.Command("./hack/federated-ginkgo-e2e.sh", strings.Fields(*testArgs)...))
}
func finishRunning(stepName string, cmd *exec.Cmd) error {
func finishRunning(cmd *exec.Cmd) error {
stepName := strings.Join(cmd.Args, " ")
if *verbose {
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
@@ -942,6 +942,7 @@ func finishRunning(stepName string, cmd *exec.Cmd) error {
log.Printf("Step '%s' finished in %s", stepName, time.Since(start))
}(time.Now())
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
if err := cmd.Start(); err != nil {
return fmt.Errorf("error starting %v: %v", stepName, err)
}
@@ -956,12 +957,17 @@ func finishRunning(stepName string, cmd *exec.Cmd) error {
select {
case <-terminate.C:
terminate.Reset(time.Duration(0)) // Kill subsequent processes immediately.
syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
cmd.Process.Kill()
return fmt.Errorf("Terminate testing after 15m after %s timeout during %s", *timeout, stepName)
case <-interrupt.C:
log.Printf("Interrupt testing after %s timeout. Will terminate in another 15m", *timeout)
terminate.Reset(15 * time.Minute)
cmd.Process.Signal(os.Interrupt)
if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGINT); err != nil {
log.Printf("Failed to interrupt %v. Will terminate immediately: %v", stepName, err)
syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM)
cmd.Process.Kill()
}
case err := <-finished:
return err
}