diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 1a838958b4a..fc32dfcdfca 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -525,6 +525,7 @@ rkt-stage1-image root-ca-file root-dir route-reconciliation-period +run-kubelet-mode run-proxy run-services-mode runtime-cgroups diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index 8f8715e6b60..a1d3b7978f6 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -51,6 +51,7 @@ var e2es *services.E2EServices // TODO(random-liu): Change the following modes to sub-command. var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.") +var runKubeletMode = flag.Bool("run-kubelet-mode", false, "If true, only start kubelet, and not run test.") var systemValidateMode = flag.Bool("system-validate-mode", false, "If true, only run system validation in current process, and not run test.") func init() { @@ -81,6 +82,11 @@ func TestE2eNode(t *testing.T) { services.RunE2EServices() return } + if *runKubeletMode { + // If run-kubelet-mode is specified, only start kubelet. + services.RunKubelet() + return + } if *systemValidateMode { // If system-validate-mode is specified, only run system validation in current process. if framework.TestContext.NodeConformance { diff --git a/test/e2e_node/jenkins/conformance/conformance-jenkins.sh b/test/e2e_node/jenkins/conformance/conformance-jenkins.sh new file mode 100755 index 00000000000..751768eb06c --- /dev/null +++ b/test/e2e_node/jenkins/conformance/conformance-jenkins.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2016 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script executed by jenkins to run node conformance test against gce +# Usage: test/e2e_node/jenkins/conformance-node-jenkins.sh + +set -e +set -x + +: "${1:?Usage test/e2e_node/jenkins/conformance-node-jenkins.sh }" + +. $1 + +make generated_files + +WORKSPACE=${WORKSPACE:-"/tmp/"} +ARTIFACTS=${WORKSPACE}/_artifacts +TIMEOUT=${TIMEOUT:-"45m"} + +mkdir -p ${ARTIFACTS} + +go run test/e2e_node/runner/remote/run_remote.go conformance \ + --logtostderr --vmodule=*=4 --ssh-env="gce" \ + --zone="$GCE_ZONE" --project="$GCE_PROJECT" --hosts="$GCE_HOSTS" \ + --images="$GCE_IMAGES" --image-project="$GCE_IMAGE_PROJECT" \ + --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \ + --results-dir="$ARTIFACTS" --test-timeout="$TIMEOUT" \ + --test_args="--kubelet-flags=\"$KUBELET_ARGS\"" \ + --instance-metadata="$GCE_INSTANCE_METADATA" diff --git a/test/e2e_node/jenkins/conformance/jenkins-conformance.properties b/test/e2e_node/jenkins/conformance/jenkins-conformance.properties new file mode 100644 index 00000000000..f465d8081a9 --- /dev/null +++ b/test/e2e_node/jenkins/conformance/jenkins-conformance.properties @@ -0,0 +1,6 @@ +GCE_HOSTS= +GCE_IMAGE_CONFIG_PATH=test/e2e_node/jenkins/image-config.yaml +GCE_ZONE=us-central1-f +GCE_PROJECT=k8s-jkns-ci-node-e2e +CLEANUP=true +KUBELET_ARGS='--experimental-cgroups-per-qos=true --cgroup-root=/' diff --git a/test/e2e_node/remote/BUILD b/test/e2e_node/remote/BUILD index ef751ff955f..d73c46558d1 100644 --- a/test/e2e_node/remote/BUILD +++ b/test/e2e_node/remote/BUILD @@ -10,8 +10,12 @@ load( go_library( name = "go_default_library", srcs = [ + "node_conformance.go", + "node_e2e.go", "remote.go", "ssh.go", + "types.go", + "utils.go", ], tags = ["automanaged"], deps = [ diff --git a/test/e2e_node/remote/node_conformance.go b/test/e2e_node/remote/node_conformance.go new file mode 100644 index 00000000000..8423a671d6e --- /dev/null +++ b/test/e2e_node/remote/node_conformance.go @@ -0,0 +1,298 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/golang/glog" + + "k8s.io/kubernetes/test/e2e_node/builder" +) + +// ConformanceRemote contains the specific functions in the node conformance test suite. +type ConformanceRemote struct{} + +func InitConformanceRemote() TestSuite { + return &ConformanceRemote{} +} + +// getConformanceDirectory gets node conformance test build directory. +func getConformanceDirectory() (string, error) { + k8sRoot, err := builder.GetK8sRootDir() + if err != nil { + return "", err + } + return filepath.Join(k8sRoot, "test", "e2e_node", "conformance", "build"), nil +} + +// commandToString is a helper function which formats command to string. +func commandToString(c *exec.Cmd) string { + return strings.Join(append([]string{c.Path}, c.Args[1:]...), " ") +} + +// Image path constants. +const ( + conformanceRegistry = "gcr.io/google_containers" + conformanceArch = runtime.GOARCH + conformanceTarfile = "node_conformance.tar" + conformanceTestBinary = "e2e_node.test" + conformanceImageLoadTimeout = time.Duration(30) * time.Second +) + +// timestamp is used as an unique id of current test. +var timestamp = getTimestamp() + +// getConformanceImageRepo returns conformance image full repo name. +func getConformanceImageRepo() string { + return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp) +} + +// buildConformanceTest builds node conformance test image tarball into binDir. +func buildConformanceTest(binDir string) error { + // Get node conformance directory. + conformancePath, err := getConformanceDirectory() + if err != nil { + return fmt.Errorf("failed to get node conformance directory: %v", err) + } + // Build docker image. + cmd := exec.Command("make", "-C", conformancePath, "BIN_DIR="+binDir, + "REGISTRY="+conformanceRegistry, + "ARCH="+conformanceArch, + "VERSION="+timestamp) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to build node conformance docker image: command - %q, error - %v, output - %q", + commandToString(cmd), err, output) + } + // Save docker image into tar file. + cmd = exec.Command("docker", "save", getConformanceImageRepo(), "-o", filepath.Join(binDir, conformanceTarfile)) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to save node conformance docker image into tar file: command - %q, error - %v, output - %q", + commandToString(cmd), err, output) + } + return nil +} + +// SetupTestPackage sets up the test package with binaries k8s required for node conformance test +func (c *ConformanceRemote) SetupTestPackage(tardir string) error { + // Build the executables + if err := builder.BuildGo(); err != nil { + return fmt.Errorf("failed to build the depedencies: %v", err) + } + + // Make sure we can find the newly built binaries + buildOutputDir, err := builder.GetK8sBuildOutputDir() + if err != nil { + return fmt.Errorf("failed to locate kubernetes build output directory %v", err) + } + + // Build node conformance tarball. + if err := buildConformanceTest(buildOutputDir); err != nil { + return fmt.Errorf("failed to build node conformance test %v", err) + } + + // Copy files + requiredFiles := []string{"kubelet", conformanceTestBinary, conformanceTarfile} + for _, file := range requiredFiles { + source := filepath.Join(buildOutputDir, file) + if _, err := os.Stat(source); err != nil { + return fmt.Errorf("failed to locate test file %s: %v", file, err) + } + output, err := exec.Command("cp", source, filepath.Join(tardir, file)).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to copy %q: error - %v output - %q", file, err, output) + } + } + + return nil +} + +// loadConformanceImage loads node conformance image from tar file. +func loadConformanceImage(host, workspace string) error { + tarfile := filepath.Join(workspace, conformanceTarfile) + if output, err := SSH(host, "timeout", conformanceImageLoadTimeout.String(), + "docker", "load", "-i", tarfile); err != nil { + return fmt.Errorf("failed to load node conformance image from tar file %q: error - %v output - %q", + tarfile, err, output) + } + return nil +} + +// kubeletLauncherLog is the log of kubelet launcher. +const kubeletLauncherLog = "kubelet-launcher.log" + +// kubeletPodManifestPath is a fixed known pod manifest path. We can not use the random pod +// manifest directory generated in e2e_node.test because we need to mount the directory into +// the conformance test container, it's easier if it's a known directory. +// TODO(random-liu): Get rid of this once we switch to cluster e2e node bootstrap script. +var kubeletPodManifestPath = "conformance-pod-manifest-" + timestamp + +// getPodManifestPath returns pod manifest full path. +func getPodManifestPath(workspace string) string { + return filepath.Join(workspace, kubeletPodManifestPath) +} + +// isSystemd returns whether the node is a systemd node. +func isSystemd(host string) (bool, error) { + // Returns "systemd" if /run/systemd/system is found, empty string otherwise. + output, err := SSH(host, "test", "-e", "/run/systemd/system", "&&", "echo", "systemd", "||", "true") + if err != nil { + return false, fmt.Errorf("failed to check systemd: error - %v output - %q", err, output) + } + return strings.TrimSpace(output) != "", nil +} + +// launchKubelet launches kubelet by running e2e_node.test binary in run-kubelet-mode. +// This is a temporary solution, we should change node e2e to use the same node bootstrap +// with cluster e2e and launch kubelet outside of the test for both regular node e2e and +// node conformance test. +// TODO(random-liu): Switch to use standard node bootstrap script. +func launchKubelet(host, workspace, results, testArgs string) error { + podManifestPath := getPodManifestPath(workspace) + if output, err := SSH(host, "mkdir", podManifestPath); err != nil { + return fmt.Errorf("failed to create kubelet pod manifest path %q: error - %v output - %q", + podManifestPath, err, output) + } + startKubeletCmd := fmt.Sprintf("./%s --run-kubelet-mode --logtostderr --node-name=%s"+ + " --report-dir=%s %s --kubelet-flags=--pod-manifest-path=%s > %s 2>&1", + conformanceTestBinary, host, results, testArgs, podManifestPath, filepath.Join(results, kubeletLauncherLog)) + var cmd []string + systemd, err := isSystemd(host) + if err != nil { + return fmt.Errorf("failed to check systemd: %v", err) + } + if systemd { + cmd = []string{ + "systemd-run", "sh", "-c", getSSHCommand(" && ", + // Switch to workspace. + fmt.Sprintf("cd %s", workspace), + // Launch kubelet by running e2e_node.test in run-kubelet-mode. + startKubeletCmd, + ), + } + } else { + cmd = []string{ + "sh", "-c", getSSHCommand(" && ", + // Switch to workspace. + fmt.Sprintf("cd %s", workspace), + // Launch kubelet by running e2e_node.test in run-kubelet-mode with nohup. + fmt.Sprintf("(nohup %s &)", startKubeletCmd), + ), + } + } + glog.V(2).Infof("Launch kubelet with command: %v", cmd) + output, err := SSH(host, cmd...) + if err != nil { + return fmt.Errorf("failed to launch kubelet with command %v: error - %v output - %q", + cmd, err, output) + } + glog.Info("Successfully launch kubelet") + return nil +} + +// kubeletStopGracePeriod is the grace period to wait before forcibly killing kubelet. +const kubeletStopGracePeriod = 10 * time.Second + +// stopKubelet stops kubelet launcher and kubelet gracefully. +func stopKubelet(host, workspace string) error { + glog.Info("Gracefully stop kubelet launcher") + if output, err := SSH(host, "pkill", conformanceTestBinary); err != nil { + return fmt.Errorf("failed to gracefully stop kubelet launcher: error - %v output - %q", + err, output) + } + glog.Info("Wait for kubelet launcher to stop") + stopped := false + for start := time.Now(); time.Since(start) < kubeletStopGracePeriod; time.Sleep(time.Second) { + // Check whehther the process is still running. + output, err := SSH(host, "pidof", conformanceTestBinary, "||", "true") + if err != nil { + return fmt.Errorf("failed to check kubelet stopping: error - %v output -%q", + err, output) + } + // Kubelet is stopped + if strings.TrimSpace(output) == "" { + stopped = true + break + } + } + if !stopped { + glog.Info("Forcibly stop kubelet") + if output, err := SSH(host, "pkill", "-SIGKILL", conformanceTestBinary); err != nil { + return fmt.Errorf("failed to forcibly stop kubelet: error - %v output - %q", + err, output) + } + } + glog.Info("Successfully stop kubelet") + // Clean up the pod manifest path + podManifestPath := getPodManifestPath(workspace) + if output, err := SSH(host, "rm", "-f", filepath.Join(workspace, podManifestPath)); err != nil { + return fmt.Errorf("failed to cleanup pod manifest directory %q: error - %v, output - %q", + podManifestPath, err, output) + } + return nil +} + +// RunTest runs test on the node. +func (c *ConformanceRemote) RunTest(host, workspace, results, junitFilePrefix, testArgs, _ string, timeout time.Duration) (string, error) { + // Install the cni plugin. + if err := installCNI(host, workspace); err != nil { + return "", err + } + + // Configure iptables firewall rules. + if err := configureFirewall(host); err != nil { + return "", err + } + + // Kill any running node processes. + cleanupNodeProcesses(host) + + // Load node conformance image. + if err := loadConformanceImage(host, workspace); err != nil { + return "", err + } + + // Launch kubelet. + if err := launchKubelet(host, workspace, results, testArgs); err != nil { + return "", err + } + // Stop kubelet. + defer func() { + if err := stopKubelet(host, workspace); err != nil { + // Only log an error if failed to stop kubelet because it is not critical. + glog.Errorf("failed to stop kubelet: %v", err) + } + }() + + // Run the tests + glog.V(2).Infof("Starting tests on %q", host) + podManifestPath := getPodManifestPath(workspace) + cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result %s'", + timeout.Seconds(), podManifestPath, podManifestPath, results, getConformanceImageRepo()) + testOutput, err := SSH(host, "sh", "-c", cmd) + if err != nil { + return testOutput, err + } + + return testOutput, nil +} diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go new file mode 100644 index 00000000000..1f5c74818a9 --- /dev/null +++ b/test/e2e_node/remote/node_e2e.go @@ -0,0 +1,164 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/golang/glog" + + "k8s.io/kubernetes/test/e2e_node/builder" +) + +// NodeE2ERemote contains the specific functions in the node e2e test suite. +type NodeE2ERemote struct{} + +func InitNodeE2ERemote() TestSuite { + // TODO: Register flags. + return &NodeE2ERemote{} +} + +const localGCIMounterPath = "cluster/gce/gci/mounter/mounter" + +// SetupTestPackage sets up the test package with binaries k8s required for node e2e tests +func (n *NodeE2ERemote) SetupTestPackage(tardir string) error { + // Build the executables + if err := builder.BuildGo(); err != nil { + return fmt.Errorf("failed to build the depedencies: %v", err) + } + + // Make sure we can find the newly built binaries + buildOutputDir, err := builder.GetK8sBuildOutputDir() + if err != nil { + return fmt.Errorf("failed to locate kubernetes build output directory %v", err) + } + + // Copy binaries + requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo"} + for _, bin := range requiredBins { + source := filepath.Join(buildOutputDir, bin) + if _, err := os.Stat(source); err != nil { + return fmt.Errorf("failed to locate test binary %s: %v", bin, err) + } + out, err := exec.Command("cp", source, filepath.Join(tardir, bin)).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to copy %q: %v Output: %q", bin, err, out) + } + } + + // Include the GCI mounter artifacts in the deployed tarball + k8sDir, err := builder.GetK8sRootDir() + if err != nil { + return fmt.Errorf("Could not find K8s root dir! Err: %v", err) + } + source := filepath.Join(k8sDir, localGCIMounterPath) + + // Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves + if _, err := os.Stat(source); err != nil { + return fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err) + } + + bindir := "cluster/gce/gci/mounter" + bin := "mounter" + destdir := filepath.Join(tardir, bindir) + dest := filepath.Join(destdir, bin) + out, err := exec.Command("mkdir", "-p", filepath.Join(tardir, bindir)).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to create directory %q for GCI mounter script. Err: %v. Output:\n%s", destdir, err, out) + } + out, err = exec.Command("cp", source, dest).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to copy GCI mounter script to the archive bin. Err: %v. Output:\n%s", err, out) + } + return nil +} + +// updateGCIMounterPath updates kubelet flags to set gci mounter path. This will only take effect for +// GCI image. +func updateGCIMounterPath(args, host, workspace string) (string, error) { + // Determine if tests will run on a GCI node. + output, err := SSH(host, "cat", "/etc/os-release") + if err != nil { + return args, fmt.Errorf("issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output) + } + if !strings.Contains(output, "ID=gci") { + // This is not a GCI image + return args, nil + } + + // If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args. + // We do this here because the local var `workspace` tells us which /tmp/node-e2e-%d is relevant to the current test run. + + // Determine if the GCI mounter script exists locally. + k8sDir, err := builder.GetK8sRootDir() + if err != nil { + return args, fmt.Errorf("could not find K8s root dir! Err: %v", err) + } + source := filepath.Join(k8sDir, localGCIMounterPath) + + // Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves + if _, err = os.Stat(source); err != nil { + return args, fmt.Errorf("could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err) + } + + glog.V(2).Infof("GCI node and GCI mounter both detected, modifying --experimental-mounter-path accordingly") + // Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error + // here will tell us to update the remote test runner. + mounterPath := filepath.Join(workspace, localGCIMounterPath) + output, err = SSH(host, "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath)) + if err != nil { + return args, fmt.Errorf("unabled to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output) + } + // Insert args at beginning of test args, so any values from command line take precedence + args = fmt.Sprintf("--kubelet-flags=--experimental-mounter-path=%s ", mounterPath) + args + return args, nil +} + +// RunTest runs test on the node. +func (n *NodeE2ERemote) RunTest(host, workspace, results, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) { + // Install the cni plugin. + if err := installCNI(host, workspace); err != nil { + return "", err + } + + // Configure iptables firewall rules + if err := configureFirewall(host); err != nil { + return "", err + } + + // Kill any running node processes + cleanupNodeProcesses(host) + + testArgs, err := updateGCIMounterPath(testArgs, host, workspace) + if err != nil { + return "", err + } + + // Run the tests + glog.V(2).Infof("Starting tests on %q", host) + cmd := getSSHCommand(" && ", + fmt.Sprintf("cd %s", workspace), + fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s %s", + timeout.Seconds(), ginkgoArgs, host, results, junitFilePrefix, testArgs), + ) + return SSH(host, "sh", "-c", cmd) +} diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index be02298dfdd..1e162cb5b86 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -23,87 +23,33 @@ import ( "os" "os/exec" "path/filepath" - "strings" "time" "github.com/golang/glog" utilerrors "k8s.io/kubernetes/pkg/util/errors" - "k8s.io/kubernetes/test/e2e_node/builder" ) var testTimeoutSeconds = flag.Duration("test-timeout", 45*time.Minute, "How long (in golang duration format) to wait for ginkgo tests to complete.") var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test results to.") -const ( - archiveName = "e2e_node_test.tar.gz" - CNIRelease = "07a8a28637e97b22eb8dfe710eeae1344f69d16e" - CNIDirectory = "cni" -) +const archiveName = "e2e_node_test.tar.gz" -var CNIURL = fmt.Sprintf("https://storage.googleapis.com/kubernetes-release/network-plugins/cni-%s.tar.gz", CNIRelease) - -// CreateTestArchive builds the local source and creates a tar archive e2e_node_test.tar.gz containing -// the binaries k8s required for node e2e tests -func CreateTestArchive() (string, error) { - // Build the executables - if err := builder.BuildGo(); err != nil { - return "", fmt.Errorf("failed to build the depedencies: %v", err) - } - - // Make sure we can find the newly built binaries - buildOutputDir, err := builder.GetK8sBuildOutputDir() - if err != nil { - return "", fmt.Errorf("failed to locate kubernetes build output directory %v", err) - } - - glog.Infof("Building archive...") +func CreateTestArchive(suite TestSuite) (string, error) { + glog.V(2).Infof("Building archive...") tardir, err := ioutil.TempDir("", "node-e2e-archive") if err != nil { return "", fmt.Errorf("failed to create temporary directory %v.", err) } defer os.RemoveAll(tardir) - // Copy binaries - requiredBins := []string{"kubelet", "e2e_node.test", "ginkgo"} - for _, bin := range requiredBins { - source := filepath.Join(buildOutputDir, bin) - if _, err := os.Stat(source); err != nil { - return "", fmt.Errorf("failed to locate test binary %s: %v", bin, err) - } - out, err := exec.Command("cp", source, filepath.Join(tardir, bin)).CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to copy %q: %v Output: %q", bin, err, out) - } - } - - // Include the GCI mounter artifacts in the deployed tarball - k8sDir, err := builder.GetK8sRootDir() + // Call the suite function to setup the test package. + err = suite.SetupTestPackage(tardir) if err != nil { - return "", fmt.Errorf("Could not find K8s root dir! Err: %v", err) - } - localSource := "cluster/gce/gci/mounter/mounter" - source := filepath.Join(k8sDir, localSource) - - // Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves - if _, err := os.Stat(source); err != nil { - return "", fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err) - } - - bindir := "cluster/gce/gci/mounter" - bin := "mounter" - destdir := filepath.Join(tardir, bindir) - dest := filepath.Join(destdir, bin) - out, err := exec.Command("mkdir", "-p", filepath.Join(tardir, bindir)).CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to create directory %q for GCI mounter script. Err: %v. Output:\n%s", destdir, err, out) - } - out, err = exec.Command("cp", source, dest).CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to copy GCI mounter script to the archive bin. Err: %v. Output:\n%s", err, out) + return "", fmt.Errorf("failed to setup test package %q: %v", tardir, err) } // Build the tar - out, err = exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput() + out, err := exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput() if err != nil { return "", fmt.Errorf("failed to build tar %v. Output:\n%s", err, out) } @@ -116,170 +62,60 @@ func CreateTestArchive() (string, error) { } // Returns the command output, whether the exit was ok, and any errors -func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string, testArgs string, ginkgoFlags string) (string, bool, error) { +// TODO(random-liu): junitFilePrefix is not prefix actually, the file name is junit-junitFilePrefix.xml. Change the variable name. +func RunRemote(suite TestSuite, archive string, host string, cleanup bool, junitFilePrefix string, testArgs string, ginkgoArgs string) (string, bool, error) { // Create the temp staging directory - glog.Infof("Staging test binaries on %s", host) + glog.V(2).Infof("Staging test binaries on %q", host) workspace := fmt.Sprintf("/tmp/node-e2e-%s", getTimestamp()) // Do not sudo here, so that we can use scp to copy test archive to the directdory. if output, err := SSHNoSudo(host, "mkdir", workspace); err != nil { // Exit failure with the error - return "", false, fmt.Errorf("failed to create workspace directory: %v output: %q", err, output) + return "", false, fmt.Errorf("failed to create workspace directory %q on host %q: %v output: %q", workspace, host, err, output) } if cleanup { defer func() { output, err := SSH(host, "rm", "-rf", workspace) if err != nil { - glog.Errorf("failed to cleanup workspace %s on host %v. Output:\n%s", workspace, err, output) + glog.Errorf("failed to cleanup workspace %q on host %q: %v. Output:\n%s", workspace, host, err, output) } }() } - // Install the cni plugin. - cniPath := filepath.Join(workspace, CNIDirectory) - cmd := getSSHCommand(" ; ", - fmt.Sprintf("mkdir -p %s", cniPath), - fmt.Sprintf("wget -O - %s | tar -xz -C %s", CNIURL, cniPath), - ) - if output, err := SSH(host, "sh", "-c", cmd); err != nil { - // Exit failure with the error - return "", false, fmt.Errorf("failed to install cni plugin: %v output: %q", err, output) - } - - // Configure iptables firewall rules - // TODO: consider calling bootstrap script to configure host based on OS - output, err := SSH(host, "iptables", "-L", "INPUT") - if err != nil { - return "", false, fmt.Errorf("failed to get iptables INPUT: %v output: %q", err, output) - } - if strings.Contains(output, "Chain INPUT (policy DROP)") { - cmd = getSSHCommand("&&", - "(iptables -C INPUT -w -p TCP -j ACCEPT || iptables -A INPUT -w -p TCP -j ACCEPT)", - "(iptables -C INPUT -w -p UDP -j ACCEPT || iptables -A INPUT -w -p UDP -j ACCEPT)", - "(iptables -C INPUT -w -p ICMP -j ACCEPT || iptables -A INPUT -w -p ICMP -j ACCEPT)") - output, err := SSH(host, "sh", "-c", cmd) - if err != nil { - return "", false, fmt.Errorf("failed to configured firewall: %v output: %v", err, output) - } - } - output, err = SSH(host, "iptables", "-L", "FORWARD") - if err != nil { - return "", false, fmt.Errorf("failed to get iptables FORWARD: %v output: %q", err, output) - } - if strings.Contains(output, "Chain FORWARD (policy DROP)") { - cmd = getSSHCommand("&&", - "(iptables -C FORWARD -w -p TCP -j ACCEPT || iptables -A FORWARD -w -p TCP -j ACCEPT)", - "(iptables -C FORWARD -w -p UDP -j ACCEPT || iptables -A FORWARD -w -p UDP -j ACCEPT)", - "(iptables -C FORWARD -w -p ICMP -j ACCEPT || iptables -A FORWARD -w -p ICMP -j ACCEPT)") - output, err = SSH(host, "sh", "-c", cmd) - if err != nil { - return "", false, fmt.Errorf("failed to configured firewall: %v output: %v", err, output) - } - } - // Copy the archive to the staging directory - if output, err = runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), workspace)); err != nil { + if output, err := runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), workspace)); err != nil { // Exit failure with the error return "", false, fmt.Errorf("failed to copy test archive: %v, output: %q", err, output) } - // Kill any running node processes - cmd = getSSHCommand(" ; ", - "pkill kubelet", - "pkill kube-apiserver", - "pkill etcd", - ) - // No need to log an error if pkill fails since pkill will fail if the commands are not running. - // If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd - // logs about failing to bind the required ports. - glog.Infof("Killing any existing node processes on %s", host) - SSH(host, "sh", "-c", cmd) - // Extract the archive - cmd = getSSHCommand(" && ", + cmd := getSSHCommand(" && ", fmt.Sprintf("cd %s", workspace), fmt.Sprintf("tar -xzvf ./%s", archiveName), ) - glog.Infof("Extracting tar on %s", host) - if output, err = SSH(host, "sh", "-c", cmd); err != nil { + glog.V(2).Infof("Extracting tar on %q", host) + if output, err := SSH(host, "sh", "-c", cmd); err != nil { // Exit failure with the error return "", false, fmt.Errorf("failed to extract test archive: %v, output: %q", err, output) } - // If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args. - // We do this here because the local var `workspace` tells us which /tmp/node-e2e-%d is relevant to the current test run. - - // Determine if the GCI mounter script exists locally. - k8sDir, err := builder.GetK8sRootDir() - if err != nil { - return "", false, fmt.Errorf("Could not find K8s root dir! Err: %v", err) - } - localSource := "cluster/gce/gci/mounter/mounter" - source := filepath.Join(k8sDir, localSource) - - // Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves - if _, err = os.Stat(source); err != nil { - return "", false, fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err) + // Create the test result directory. + resultDir := filepath.Join(workspace, "results") + if output, err := SSHNoSudo(host, "mkdir", resultDir); err != nil { + // Exit failure with the error + return "", false, fmt.Errorf("failed to create test result directory %q on host %q: %v output: %q", resultDir, host, err, output) } - // Determine if tests will run on a GCI node. - output, err = SSH(host, "sh", "-c", "'cat /etc/os-release'") - if err != nil { - glog.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output) - return "", false, fmt.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output) - } - if strings.Contains(output, "ID=gci") { - glog.Infof("GCI node and GCI mounter both detected, modifying --experimental-mounter-path accordingly") - // Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error - // here will tell us to update the remote test runner. - mounterPath := filepath.Join(workspace, "cluster/gce/gci/mounter/mounter") - output, err = SSH(host, "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath)) - if err != nil { - glog.Errorf("Unable to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output) - return "", false, err - } - // Insert args at beginning of testArgs, so any values from command line take precedence - testArgs = fmt.Sprintf("--kubelet-flags=--experimental-mounter-path=%s ", mounterPath) + testArgs - } + glog.V(2).Infof("Running test on %q", host) + output, err := suite.RunTest(host, workspace, resultDir, junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds) - // Run the tests - cmd = getSSHCommand(" && ", - fmt.Sprintf("cd %s", workspace), - fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s/results --report-prefix=%s %s", - testTimeoutSeconds.Seconds(), ginkgoFlags, host, workspace, junitFilePrefix, testArgs), - ) aggErrs := []error{} - - glog.Infof("Starting tests on %s", host) - output, err = SSH(host, "sh", "-c", cmd) // Do not log the output here, let the caller deal with the test output. if err != nil { aggErrs = append(aggErrs, err) - - // Encountered an unexpected error. The remote test harness may not - // have finished retrieved and stored all the logs in this case. Try - // to get some logs for debugging purposes. - // TODO: This is a best-effort, temporary hack that only works for - // journald nodes. We should have a more robust way to collect logs. - var ( - logName = "system.log" - logPath = fmt.Sprintf("/tmp/%s-%s", getTimestamp(), logName) - destPath = fmt.Sprintf("%s/%s-%s", *resultsDir, host, logName) - ) - glog.Infof("Test failed unexpectedly. Attempting to retreiving system logs (only works for nodes with journald)") - // Try getting the system logs from journald and store it to a file. - // Don't reuse the original test directory on the remote host because - // it could've be been removed if the node was rebooted. - if output, err := SSH(host, "sh", "-c", fmt.Sprintf("'journalctl --system --all > %s'", logPath)); err == nil { - glog.Infof("Got the system logs from journald; copying it back...") - if output, err := runSSHCommand("scp", fmt.Sprintf("%s:%s", GetHostnameOrIp(host), logPath), destPath); err != nil { - glog.Infof("Failed to copy the log: err: %v, output: %q", err, output) - } - } else { - glog.Infof("Failed to run journactl (normal if it doesn't exist on the node): %v, output: %q", err, output) - } + collectSystemLog(host, workspace) } - glog.Infof("Copying test artifacts from %s", host) + glog.V(2).Infof("Copying test artifacts from %q", host) scpErr := getTestArtifacts(host, workspace) if scpErr != nil { aggErrs = append(aggErrs, scpErr) @@ -313,6 +149,33 @@ func getTestArtifacts(host, testDir string) error { return nil } +// collectSystemLog is a temporary hack to collect system log when encountered on +// unexpected error. +func collectSystemLog(host, workspace string) { + // Encountered an unexpected error. The remote test harness may not + // have finished retrieved and stored all the logs in this case. Try + // to get some logs for debugging purposes. + // TODO: This is a best-effort, temporary hack that only works for + // journald nodes. We should have a more robust way to collect logs. + var ( + logName = "system.log" + logPath = fmt.Sprintf("/tmp/%s-%s", getTimestamp(), logName) + destPath = fmt.Sprintf("%s/%s-%s", *resultsDir, host, logName) + ) + glog.V(2).Infof("Test failed unexpectedly. Attempting to retreiving system logs (only works for nodes with journald)") + // Try getting the system logs from journald and store it to a file. + // Don't reuse the original test directory on the remote host because + // it could've be been removed if the node was rebooted. + if output, err := SSH(host, "sh", "-c", fmt.Sprintf("'journalctl --system --all > %s'", logPath)); err == nil { + glog.V(2).Infof("Got the system logs from journald; copying it back...") + if output, err := runSSHCommand("scp", fmt.Sprintf("%s:%s", GetHostnameOrIp(host), logPath), destPath); err != nil { + glog.V(2).Infof("Failed to copy the log: err: %v, output: %q", err, output) + } + } else { + glog.V(2).Infof("Failed to run journactl (normal if it doesn't exist on the node): %v, output: %q", err, output) + } +} + // WriteLog is a temporary function to make it possible to write log // in the runner. This is used to collect serial console log. // TODO(random-liu): Use the log-dump script in cluster e2e. diff --git a/test/e2e_node/remote/types.go b/test/e2e_node/remote/types.go new file mode 100644 index 00000000000..52b80a54222 --- /dev/null +++ b/test/e2e_node/remote/types.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "time" +) + +// TestSuite is the interface of a test suite, such as node e2e, node conformance, +// node soaking, cri validation etc. +type TestSuite interface { + // SetupTestPackage setup the test package in the given directory. TestSuite + // should put all necessary binaries and dependencies into the path. The caller + // will: + // * create a tarball with the directory. + // * deploy the tarball to the testing host. + // * untar the tarball to the testing workspace on the testing host. + SetupTestPackage(path string) error + // RunTest runs test on the node in the given workspace and returns test output + // and test error if there is any. + // * host is the target node to run the test. + // * workspace is the directory on the testing host the test is running in. Note + // that the test package is unpacked in the workspace before running the test. + // * results is the directory the test should write result into. All logs should be + // saved as *.log, all junit file should start with junit*. + // * junitFilePrefix is the prefix of output junit file. + // * testArgs is the arguments passed to test. + // * ginkgoArgs is the arguments passed to ginkgo. + // * timeout is the test timeout. + RunTest(host, workspace, results, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) +} diff --git a/test/e2e_node/remote/utils.go b/test/e2e_node/remote/utils.go new file mode 100644 index 00000000000..0f66bea2ee9 --- /dev/null +++ b/test/e2e_node/remote/utils.go @@ -0,0 +1,97 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package remote + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/golang/glog" +) + +// utils.go contains functions used accross test suites. + +const ( + cniRelease = "07a8a28637e97b22eb8dfe710eeae1344f69d16e" + cniDirectory = "cni" + cniURL = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-" + cniRelease + ".tar.gz" +) + +// Install the cni plugin. +func installCNI(host, workspace string) error { + glog.V(2).Infof("Install CNI on %q", host) + cniPath := filepath.Join(workspace, cniDirectory) + cmd := getSSHCommand(" ; ", + fmt.Sprintf("mkdir -p %s", cniPath), + fmt.Sprintf("wget -O - %s | tar -xz -C %s", cniURL, cniPath), + ) + if output, err := SSH(host, "sh", "-c", cmd); err != nil { + return fmt.Errorf("failed to install cni plugin on %q: %v output: %q", host, err, output) + } + return nil +} + +// configureFirewall configures iptable firewall rules. +func configureFirewall(host string) error { + glog.V(2).Infof("Configure iptables firewall rules on %q", host) + // TODO: consider calling bootstrap script to configure host based on OS + output, err := SSH(host, "iptables", "-L", "INPUT") + if err != nil { + return fmt.Errorf("failed to get iptables INPUT on %q: %v output: %q", host, err, output) + } + if strings.Contains(output, "Chain INPUT (policy DROP)") { + cmd := getSSHCommand("&&", + "(iptables -C INPUT -w -p TCP -j ACCEPT || iptables -A INPUT -w -p TCP -j ACCEPT)", + "(iptables -C INPUT -w -p UDP -j ACCEPT || iptables -A INPUT -w -p UDP -j ACCEPT)", + "(iptables -C INPUT -w -p ICMP -j ACCEPT || iptables -A INPUT -w -p ICMP -j ACCEPT)") + output, err := SSH(host, "sh", "-c", cmd) + if err != nil { + return fmt.Errorf("failed to configured firewall on %q: %v output: %v", host, err, output) + } + } + output, err = SSH(host, "iptables", "-L", "FORWARD") + if err != nil { + return fmt.Errorf("failed to get iptables FORWARD on %q: %v output: %q", host, err, output) + } + if strings.Contains(output, "Chain FORWARD (policy DROP)") { + cmd := getSSHCommand("&&", + "(iptables -C FORWARD -w -p TCP -j ACCEPT || iptables -A FORWARD -w -p TCP -j ACCEPT)", + "(iptables -C FORWARD -w -p UDP -j ACCEPT || iptables -A FORWARD -w -p UDP -j ACCEPT)", + "(iptables -C FORWARD -w -p ICMP -j ACCEPT || iptables -A FORWARD -w -p ICMP -j ACCEPT)") + output, err = SSH(host, "sh", "-c", cmd) + if err != nil { + return fmt.Errorf("failed to configured firewall on %q: %v output: %v", host, err, output) + } + } + return nil +} + +// cleanupNodeProcesses kills all running node processes may conflict with the test. +func cleanupNodeProcesses(host string) { + glog.V(2).Infof("Killing any existing node processes on %q", host) + cmd := getSSHCommand(" ; ", + "pkill kubelet", + "pkill kube-apiserver", + "pkill etcd", + "pkill e2e_node.test", + ) + // No need to log an error if pkill fails since pkill will fail if the commands are not running. + // If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd + // logs about failing to bind the required ports. + SSH(host, "sh", "-c", cmd) +} diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index 7f1b95e6f98..c2bd9f229e8 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -66,6 +66,7 @@ const ( var ( computeService *compute.Service arc Archive + suite remote.TestSuite ) type Archive struct { @@ -125,12 +126,32 @@ type internalGCEImage struct { tests []string } +// parseFlags parse subcommands and flags +func parseFlags() { + if len(os.Args) <= 1 { + glog.Fatalf("Too few flags specified: %v", os.Args) + } + // Parse subcommand. + subcommand := os.Args[1] + switch subcommand { + case "conformance": + suite = remote.InitConformanceRemote() + // TODO: Add subcommand for node soaking, node conformance, cri validation. + default: + // Use node e2e suite by default if no subcommand is specified. + suite = remote.InitNodeE2ERemote() + } + // Parse test flags. + flag.CommandLine.Parse(os.Args[2:]) +} + func main() { - flag.Parse() + parseFlags() + rand.Seed(time.Now().UTC().UnixNano()) if *buildOnly { // Build the archive and exit - remote.CreateTestArchive() + remote.CreateTestArchive(suite) return } @@ -301,7 +322,7 @@ func callGubernator(gubernator bool) { } func (a *Archive) getArchive() (string, error) { - a.Do(func() { a.path, a.err = remote.CreateTestArchive() }) + a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite) }) return a.path, a.err } @@ -363,7 +384,7 @@ func testHost(host string, deleteFiles bool, junitFilePrefix string, ginkgoFlags } } - output, exitOk, err := remote.RunRemote(path, host, deleteFiles, junitFilePrefix, *testArgs, ginkgoFlagsStr) + output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, junitFilePrefix, *testArgs, ginkgoFlagsStr) return &TestResult{ output: output, err: err, diff --git a/test/e2e_node/services/BUILD b/test/e2e_node/services/BUILD index ad3a97f8112..b56f927528d 100644 --- a/test/e2e_node/services/BUILD +++ b/test/e2e_node/services/BUILD @@ -17,6 +17,7 @@ go_library( "namespace_controller.go", "server.go", "services.go", + "util.go", ], tags = ["automanaged"], deps = [ diff --git a/test/e2e_node/services/internal_services.go b/test/e2e_node/services/internal_services.go index a38163ddd93..344883b2088 100644 --- a/test/e2e_node/services/internal_services.go +++ b/test/e2e_node/services/internal_services.go @@ -19,8 +19,6 @@ package services import ( "io/ioutil" "os" - "os/signal" - "syscall" "github.com/golang/glog" ) @@ -38,10 +36,6 @@ func newE2EServices() *e2eServices { return &e2eServices{} } -// terminationSignals are signals that cause the program to exit in the -// supported platforms (linux, darwin, windows). -var terminationSignals = []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT} - // run starts all e2e services and wait for the termination signal. Once receives the // termination signal, it will stop the e2e services gracefully. func (es *e2eServices) run() error { @@ -50,9 +44,7 @@ func (es *e2eServices) run() error { return err } // Wait until receiving a termination signal. - sig := make(chan os.Signal, 1) - signal.Notify(sig, terminationSignals...) - <-sig + waitForTerminationSignal() return nil } diff --git a/test/e2e_node/services/kubelet.go b/test/e2e_node/services/kubelet.go index 5ea5b633243..0be6a0e8dbe 100644 --- a/test/e2e_node/services/kubelet.go +++ b/test/e2e_node/services/kubelet.go @@ -63,6 +63,22 @@ func init() { flag.Var(&kubeletArgs, "kubelet-flags", "Kubelet flags passed to kubelet, this will override default kubelet flags in the test. Flags specified in multiple kubelet-flags will be concatenate.") } +// RunKubelet starts kubelet and waits for termination signal. Once receives the +// termination signal, it will stop the kubelet gracefully. +func RunKubelet() { + var err error + // Enable monitorParent to make sure kubelet will receive termination signal + // when test process exits. + e := NewE2EServices(true /* monitorParent */) + defer e.Stop() + e.kubelet, err = e.startKubelet() + if err != nil { + glog.Fatalf("Failed to start kubelet: %v", err) + } + // Wait until receiving a termination signal. + waitForTerminationSignal() +} + const ( // Ports of different e2e services. kubeletPort = "10250" diff --git a/test/e2e_node/services/util.go b/test/e2e_node/services/util.go new file mode 100644 index 00000000000..558be7d52d7 --- /dev/null +++ b/test/e2e_node/services/util.go @@ -0,0 +1,34 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "os" + "os/signal" + "syscall" +) + +// terminationSignals are signals that cause the program to exit in the +// supported platforms (linux, darwin, windows). +var terminationSignals = []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT} + +// waitForTerminationSignal waits for termination signal. +func waitForTerminationSignal() { + sig := make(chan os.Signal, 1) + signal.Notify(sig, terminationSignals...) + <-sig +}