From d9ddd64c9cac6ef4528d3cd4d240645fc9698d23 Mon Sep 17 00:00:00 2001 From: Random-Liu Date: Fri, 28 Oct 2016 22:56:23 -0700 Subject: [PATCH 1/2] Reorganize node e2e output directories. --- hack/make-rules/test-e2e-node.sh | 25 ++++------- test/e2e_node/remote/remote.go | 48 ++++++++++++--------- test/e2e_node/services/internal_services.go | 3 +- test/e2e_node/services/services.go | 6 ++- 4 files changed, 44 insertions(+), 38 deletions(-) diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh index 7198af9df36..a5fe0d42fb5 100755 --- a/hack/make-rules/test-e2e-node.sh +++ b/hack/make-rules/test-e2e-node.sh @@ -26,8 +26,7 @@ skip=${SKIP:-""} # Currently, parallelism only affects when REMOTE=true. For local test, # ginkgo default parallelism (cores - 1) is used. parallelism=${PARALLELISM:-8} -report=${REPORT:-"/tmp/"} -artifacts=${ARTIFACTS:-"/tmp/_artifacts"} +artifacts=${ARTIFACTS:-"/tmp/_artifacts/`date +%y%m%dT%H%M%S`"} remote=${REMOTE:-"false"} run_until_failure=${RUN_UNTIL_FAILURE:-"false"} test_args=${TEST_ARGS:-""} @@ -50,6 +49,12 @@ if [[ $run_until_failure != "" ]]; then ginkgoflags="$ginkgoflags -untilItFails=$run_until_failure " fi +# Setup the directory to copy test artifacts (logs, junit.xml, etc) from remote host to local host +if [ ! -d "${artifacts}" ]; then + echo "Creating artifacts directory at ${artifacts}" + mkdir -p ${artifacts} +fi +echo "Test artifacts will be written to ${artifacts}" if [ $remote = true ] ; then # The following options are only valid in remote run. @@ -74,18 +79,6 @@ if [ $remote = true ] ; then cleanup=${CLEANUP:-"true"} delete_instances=${DELETE_INSTANCES:-"false"} - # Setup the directory to copy test artifacts (logs, junit.xml, etc) from remote host to local host - if [[ $gubernator = true && -d "${artifacts}" ]]; then - echo "Removing artifacts directory at ${artifacts}" - rm -r ${artifacts} - fi - - if [ ! -d "${artifacts}" ]; then - echo "Creating artifacts directory at ${artifacts}" - mkdir -p ${artifacts} - fi - echo "Test artifacts will be written to ${artifacts}" - # Get the compute zone zone=$(gcloud info --format='value(config.properties.compute.zone)') if [[ $zone == "" ]]; then @@ -155,7 +148,7 @@ else # Test using the host the script was run on # Provided for backwards compatibility go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \ - --test-flags="--alsologtostderr --v 4 --report-dir=${report} --node-name $(hostname) \ - $test_args" --build-dependencies=true + --test-flags="--alsologtostderr --v 4 --report-dir=${artifacts} --node-name $(hostname) \ + $test_args" --build-dependencies=true 2>&1 | tee "${artifacts}/build-log.txt" exit $? fi diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index bcc2cf353f8..fee32d2b0c6 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -20,7 +20,6 @@ import ( "flag" "fmt" "io/ioutil" - "math/rand" "os" "os/exec" "os/user" @@ -169,26 +168,24 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // Create the temp staging directory glog.Infof("Staging test binaries on %s", host) - dirName := fmt.Sprintf("gcloud-e2e-%d", rand.Int31()) - tmp := fmt.Sprintf("/tmp/%s", dirName) - + workspace := fmt.Sprintf("/tmp/node-e2e-%s", getTimestamp()) // Do not sudo here, so that we can use scp to copy test archive to the directdory. - _, err := SSHNoSudo(host, "mkdir", tmp) + _, err := SSHNoSudo(host, "mkdir", workspace) if err != nil { // Exit failure with the error return "", false, err } if cleanup { defer func() { - output, err := SSH(host, "rm", "-rf", tmp) + output, err := SSH(host, "rm", "-rf", workspace) if err != nil { - glog.Errorf("failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output) + glog.Errorf("failed to cleanup workspace %s on host %v. Output:\n%s", workspace, err, output) } }() } // Install the cni plugin. - cniPath := filepath.Join(tmp, CNIDirectory) + cniPath := filepath.Join(workspace, CNIDirectory) cmd := getSSHCommand(" ; ", fmt.Sprintf("mkdir -p %s", cniPath), fmt.Sprintf("wget -O - %s | tar -xz -C %s", CNIURL, cniPath), @@ -220,7 +217,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string } // Copy the archive to the staging directory - _, err = runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), tmp)) + _, err = runSSHCommand("scp", archive, fmt.Sprintf("%s:%s/", GetHostnameOrIp(host), workspace)) if err != nil { // Exit failure with the error return "", false, err @@ -240,7 +237,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // Extract the archive cmd = getSSHCommand(" && ", - fmt.Sprintf("cd %s", tmp), + fmt.Sprintf("cd %s", workspace), fmt.Sprintf("tar -xzvf ./%s", archiveName), ) glog.Infof("Extracting tar on %s", host) @@ -251,7 +248,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string } // If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args. - // We do this here because the local var `tmp` tells us which /tmp/gcloud-e2e-%d is relevant to the current test run. + // We do this here because the local var `workspace` tells us which /tmp/node-e2e-%d is relevant to the current test run. // Determine if the GCI mounter script exists locally. k8sDir, err := builder.GetK8sRootDir() @@ -275,7 +272,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string if strings.Contains(output, "ID=gci") { // Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error // here will tell us to update the remote test runner. - mounterPath := filepath.Join(tmp, "cluster/gce/gci/mounter/mounter") + mounterPath := filepath.Join(workspace, "cluster/gce/gci/mounter/mounter") output, err = SSH(host, "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath)) if err != nil { glog.Errorf("Unable to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output) @@ -291,9 +288,9 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // Run the tests cmd = getSSHCommand(" && ", - fmt.Sprintf("cd %s", tmp), + fmt.Sprintf("cd %s", workspace), fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s/results --report-prefix=%s %s", - testTimeoutSeconds.Seconds(), ginkgoFlags, host, tmp, junitFilePrefix, testArgs), + testTimeoutSeconds.Seconds(), ginkgoFlags, host, workspace, junitFilePrefix, testArgs), ) aggErrs := []error{} @@ -311,8 +308,8 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string // TODO: This is a best-effort, temporary hack that only works for // journald nodes. We should have a more robust way to collect logs. var ( - logName = fmt.Sprintf("%s-system.log", dirName) - logPath = fmt.Sprintf("/tmp/%s-system.log", dirName) + logName = "system.log" + logPath = filepath.Join(workspace, logName) destPath = fmt.Sprintf("%s/%s-%s", *resultsDir, host, logName) ) glog.Infof("Test failed unexpectedly. Attempting to retreiving system logs (only works for nodes with journald)") @@ -331,7 +328,7 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string } glog.Infof("Copying test artifacts from %s", host) - scpErr := getTestArtifacts(host, tmp) + scpErr := getTestArtifacts(host, workspace) if scpErr != nil { aggErrs = append(aggErrs, scpErr) } @@ -339,14 +336,25 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string return output, len(aggErrs) == 0, utilerrors.NewAggregate(aggErrs) } +// timestampFormat is the timestamp format used in the node e2e directory name. +const timestampFormat = "20060102T150405" + +func getTimestamp() string { + return fmt.Sprintf(time.Now().Format(timestampFormat)) +} + func getTestArtifacts(host, testDir string) error { - _, err := runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/", GetHostnameOrIp(host), testDir), fmt.Sprintf("%s/%s", *resultsDir, host)) + logPath := filepath.Join(*resultsDir, host) + if err := os.MkdirAll(logPath, 0755); err != nil { + return fmt.Errorf("failed to create log directory %q: %v", logPath, err) + } + // Copy logs to artifacts/hostname + _, err := runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/*.log", GetHostnameOrIp(host), testDir), logPath) if err != nil { return err } - // Copy junit to the top of artifacts - _, err = runSSHCommand("scp", fmt.Sprintf("%s:%s/results/junit*", GetHostnameOrIp(host), testDir), fmt.Sprintf("%s/", *resultsDir)) + _, err = runSSHCommand("scp", fmt.Sprintf("%s:%s/results/junit*", GetHostnameOrIp(host), testDir), *resultsDir) if err != nil { return err } diff --git a/test/e2e_node/services/internal_services.go b/test/e2e_node/services/internal_services.go index b1edf9e4784..a38163ddd93 100644 --- a/test/e2e_node/services/internal_services.go +++ b/test/e2e_node/services/internal_services.go @@ -115,7 +115,8 @@ func (es *e2eServices) stop() { // startEtcd starts the embedded etcd instance or returns an error. func (es *e2eServices) startEtcd() error { glog.Info("Starting etcd") - dataDir, err := ioutil.TempDir("", "node-e2e") + // Create data directory in current working space. + dataDir, err := ioutil.TempDir(".", "etcd") if err != nil { return err } diff --git a/test/e2e_node/services/services.go b/test/e2e_node/services/services.go index 8b998735e93..581f22e27fb 100644 --- a/test/e2e_node/services/services.go +++ b/test/e2e_node/services/services.go @@ -82,7 +82,11 @@ func (e *E2EServices) Start() error { // Start kubelet // Create the manifest path for kubelet. // TODO(random-liu): Remove related logic when we move kubelet starting logic out of the test. - framework.TestContext.ManifestPath, err = ioutil.TempDir("", "node-e2e-pod") + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current working directory: %v", err) + } + framework.TestContext.ManifestPath, err = ioutil.TempDir(cwd, "pod-manifest") if err != nil { return fmt.Errorf("failed to create static pod manifest directory: %v", err) } From f6029fb99576c278e7a368b0b2a0a3c645e71b01 Mon Sep 17 00:00:00 2001 From: Random-Liu Date: Mon, 31 Oct 2016 21:59:57 -0700 Subject: [PATCH 2/2] Handle interrupt properly, wrap focus and skip with quote. --- hack/make-rules/test-e2e-node.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh index a5fe0d42fb5..c01442dcce7 100755 --- a/hack/make-rules/test-e2e-node.sh +++ b/hack/make-rules/test-e2e-node.sh @@ -38,11 +38,11 @@ if [[ $parallelism > 1 ]]; then fi if [[ $focus != "" ]]; then - ginkgoflags="$ginkgoflags -focus=$focus " + ginkgoflags="$ginkgoflags -focus='$focus' " fi if [[ $skip != "" ]]; then - ginkgoflags="$ginkgoflags -skip=$skip " + ginkgoflags="$ginkgoflags -skip='$skip' " fi if [[ $run_until_failure != "" ]]; then @@ -130,7 +130,7 @@ if [ $remote = true ] ; then --results-dir="$artifacts" --ginkgo-flags="$ginkgoflags" \ --image-project="$image_project" --instance-name-prefix="$instance_prefix" --setup-node="true" \ --delete-instances="$delete_instances" --test_args="$test_args" --instance-metadata="$metadata" \ - 2>&1 | tee "${artifacts}/build-log.txt" + 2>&1 | tee -i "${artifacts}/build-log.txt" exit $? else @@ -149,6 +149,6 @@ else # Provided for backwards compatibility go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \ --test-flags="--alsologtostderr --v 4 --report-dir=${artifacts} --node-name $(hostname) \ - $test_args" --build-dependencies=true 2>&1 | tee "${artifacts}/build-log.txt" + $test_args" --build-dependencies=true 2>&1 | tee -i "${artifacts}/build-log.txt" exit $? fi