diff --git a/hack/ginkgo-e2e.sh b/hack/ginkgo-e2e.sh index 97f1c13817d..daf1e4ad2c6 100755 --- a/hack/ginkgo-e2e.sh +++ b/hack/ginkgo-e2e.sh @@ -30,6 +30,9 @@ e2e_test=$(kube::util::find-binary "e2e.test") GINKGO_PARALLEL=${GINKGO_PARALLEL:-n} # set to 'y' to run tests in parallel +# If 'y', will rerun failed tests once to give them a second chance. +GINKGO_TOLERATE_FLAKES=${GINKGO_TOLERATE_FLAKES:-n} + # The number of tests that can run in parallel depends on what tests # are running and on the size of the cluster. Too many, and tests will # fail due to resource contention. 25 is a reasonable default for a @@ -101,6 +104,11 @@ elif [[ ${GINKGO_PARALLEL} =~ ^[yY]$ ]]; then ginkgo_args+=("--nodes=25") fi +FLAKE_ATTEMPTS=1 +if [[ "${GINKGO_TOLERATE_FLAKES}" == "y" ]]; then + FLAKE_ATTEMPTS=2 +fi + # The --host setting is used only when providing --auth_config # If --kubeconfig is used, the host to use is retrieved from the .kubeconfig # file and the one provided with --host is ignored. @@ -108,6 +116,7 @@ fi export PATH=$(dirname "${e2e_test}"):"${PATH}" "${ginkgo}" "${ginkgo_args[@]:+${ginkgo_args[@]}}" "${e2e_test}" -- \ "${auth_config[@]:+${auth_config[@]}}" \ + --ginkgo.flakeAttempts="${FLAKE_ATTEMPTS}" \ --host="${KUBE_MASTER_URL}" \ --provider="${KUBERNETES_PROVIDER}" \ --gce-project="${PROJECT:-}" \ diff --git a/hack/jenkins/upload-to-gcs.sh b/hack/jenkins/upload-to-gcs.sh index 47229aeaa4c..fcae3a6ef73 100755 --- a/hack/jenkins/upload-to-gcs.sh +++ b/hack/jenkins/upload-to-gcs.sh @@ -46,13 +46,19 @@ fi if [[ ${JOB_NAME} =~ -pull- ]]; then : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/pull/${ghprbPullId:-unknown}"} + : ${JENKINS_GCS_LATEST_PATH:="gs://kubernetes-jenkins/pr-logs/directory"} + : ${JENKINS_GCS_LOGS_INDIRECT:="gs://kubernetes-jenkins/pr-logs/directory/${JOB_NAME}"} else : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"} + : ${JENKINS_GCS_LATEST_PATH:="gs://kubernetes-jenkins/logs"} + : ${JENKINS_GCS_LOGS_INDIRECT:=""} fi readonly artifacts_path="${WORKSPACE}/_artifacts" readonly gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}" readonly gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}" +readonly gcs_latest_path="${JENKINS_GCS_LATEST_PATH}/${JOB_NAME}" +readonly gcs_indirect_path="${JENKINS_GCS_LOGS_INDIRECT}" readonly gcs_acl="public-read" readonly results_url=${gcs_build_path//"gs:/"/"https://console.cloud.google.com/storage/browser"} readonly timestamp=$(date +%s) @@ -115,11 +121,25 @@ function upload_artifacts_and_build_result() { echo "Uploading build log" gsutil -q cp -Z -a "${gcs_acl}" "${WORKSPACE}/build-log.txt" "${gcs_build_path}" fi + + # For pull jobs, keep a canonical ordering for tools that want to examine + # the output. + if [[ "${gcs_indirect_path}" != "" ]]; then + echo "Writing ${gcs_build_path} to ${gcs_indirect_path}/${BUILD_NUMBER}.txt" + echo "${gcs_build_path}" | \ + gsutil -q -h "Content-Type:text/plain" \ + cp -a "${gcs_acl}" - "${gcs_indirect_path}/${BUILD_NUMBER}.txt" || continue + echo "Marking build ${BUILD_NUMBER} as the latest completed build for this PR" + echo "${BUILD_NUMBER}" | \ + gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \ + cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue + fi + # Mark this build as the latest completed. echo "Marking build ${BUILD_NUMBER} as the latest completed build" echo "${BUILD_NUMBER}" | \ gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \ - cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue + cp -a "${gcs_acl}" - "${gcs_latest_path}/latest-build.txt" || continue break # all uploads succeeded if we hit this point done diff --git a/vendor/github.com/onsi/ginkgo/config/config.go b/vendor/github.com/onsi/ginkgo/config/config.go index 410f7b7ac25..4b5485b4157 100644 --- a/vendor/github.com/onsi/ginkgo/config/config.go +++ b/vendor/github.com/onsi/ginkgo/config/config.go @@ -31,6 +31,7 @@ type GinkgoConfigType struct { SkipMeasurements bool FailOnPending bool FailFast bool + FlakeAttempts int EmitSpecProgress bool DryRun bool @@ -75,6 +76,8 @@ func Flags(flagSet *flag.FlagSet, prefix string, includeParallelFlags bool) { flagSet.BoolVar(&(GinkgoConfig.RegexScansFilePath), prefix+"regexScansFilePath", false, "If set, ginkgo regex matching also will look at the file path (code location).") + flagSet.IntVar(&(GinkgoConfig.FlakeAttempts), prefix+"flakeAttempts", 1, "Make up to this many attempts to run each spec. Please note that if any of the attempts succeed, the suite will not be failed. But any failures will still be recorded.") + flagSet.BoolVar(&(GinkgoConfig.EmitSpecProgress), prefix+"progress", false, "If set, ginkgo will emit progress information as each spec runs to the GinkgoWriter.") if includeParallelFlags { @@ -128,6 +131,10 @@ func BuildFlagArgs(prefix string, ginkgo GinkgoConfigType, reporter DefaultRepor result = append(result, fmt.Sprintf("--%sskip=%s", prefix, ginkgo.SkipString)) } + if ginkgo.FlakeAttempts > 1 { + result = append(result, fmt.Sprintf("--%sflakeAttempts=%d", prefix, ginkgo.FlakeAttempts)) + } + if ginkgo.EmitSpecProgress { result = append(result, fmt.Sprintf("--%sprogress", prefix)) } diff --git a/vendor/github.com/onsi/ginkgo/ginkgo/help_command.go b/vendor/github.com/onsi/ginkgo/ginkgo/help_command.go index a42d4f8aae7..23b1d2f1178 100644 --- a/vendor/github.com/onsi/ginkgo/ginkgo/help_command.go +++ b/vendor/github.com/onsi/ginkgo/ginkgo/help_command.go @@ -9,7 +9,7 @@ func BuildHelpCommand() *Command { return &Command{ Name: "help", FlagSet: flag.NewFlagSet("help", flag.ExitOnError), - UsageCommand: "ginkgo help ", + UsageCommand: "ginkgo help ", Usage: []string{ "Print usage information. If a command is passed in, print usage information just for that command.", }, diff --git a/vendor/github.com/onsi/ginkgo/ginkgo/interrupthandler/sigquit_swallower_unix.go b/vendor/github.com/onsi/ginkgo/ginkgo/interrupthandler/sigquit_swallower_unix.go index 14c94210ee0..43c18544a8d 100644 --- a/vendor/github.com/onsi/ginkgo/ginkgo/interrupthandler/sigquit_swallower_unix.go +++ b/vendor/github.com/onsi/ginkgo/ginkgo/interrupthandler/sigquit_swallower_unix.go @@ -1,4 +1,4 @@ -// +build freebsd openbsd netbsd dragonfly darwin linux +// +build freebsd openbsd netbsd dragonfly darwin linux solaris package interrupthandler diff --git a/vendor/github.com/onsi/ginkgo/ginkgo/testrunner/test_runner.go b/vendor/github.com/onsi/ginkgo/ginkgo/testrunner/test_runner.go index cc9805c9900..973229d94f6 100644 --- a/vendor/github.com/onsi/ginkgo/ginkgo/testrunner/test_runner.go +++ b/vendor/github.com/onsi/ginkgo/ginkgo/testrunner/test_runner.go @@ -51,7 +51,7 @@ func New(suite testsuite.TestSuite, numCPU int, parallelStream bool, race bool, if !suite.Precompiled { dir, err := ioutil.TempDir("", "ginkgo") if err != nil { - panic(fmt.Sprintf("coulnd't create temporary directory... might be time to rm -rf:\n%s", err.Error())) + panic(fmt.Sprintf("couldn't create temporary directory... might be time to rm -rf:\n%s", err.Error())) } runner.compilationTargetPath = filepath.Join(dir, suite.PackageName+".test") } diff --git a/vendor/github.com/onsi/ginkgo/ginkgo_dsl.go b/vendor/github.com/onsi/ginkgo/ginkgo_dsl.go index 1d8e593052a..0220549f7cb 100644 --- a/vendor/github.com/onsi/ginkgo/ginkgo_dsl.go +++ b/vendor/github.com/onsi/ginkgo/ginkgo_dsl.go @@ -168,6 +168,8 @@ func CurrentGinkgoTestDescription() GinkgoTestDescription { // //You use the Time() function to time how long the passed in body function takes to run //You use the RecordValue() function to track arbitrary numerical measurements. +//The RecordValueWithPrecision() function can be used alternatively to provide the unit +//and resolution of the numeric measurement. //The optional info argument is passed to the test reporter and can be used to // provide the measurement data to a custom reporter with context. // @@ -175,6 +177,7 @@ func CurrentGinkgoTestDescription() GinkgoTestDescription { type Benchmarker interface { Time(name string, body func(), info ...interface{}) (elapsedTime time.Duration) RecordValue(name string, value float64, info ...interface{}) + RecordValueWithPrecision(name string, value float64, units string, precision int, info ...interface{}) } //RunSpecs is the entry point for the Ginkgo test runner. diff --git a/vendor/github.com/onsi/ginkgo/internal/leafnodes/benchmarker.go b/vendor/github.com/onsi/ginkgo/internal/leafnodes/benchmarker.go index bc0dd1a627a..9c3eed2b6fe 100644 --- a/vendor/github.com/onsi/ginkgo/internal/leafnodes/benchmarker.go +++ b/vendor/github.com/onsi/ginkgo/internal/leafnodes/benchmarker.go @@ -28,20 +28,27 @@ func (b *benchmarker) Time(name string, body func(), info ...interface{}) (elaps b.mu.Lock() defer b.mu.Unlock() - measurement := b.getMeasurement(name, "Fastest Time", "Slowest Time", "Average Time", "s", info...) + measurement := b.getMeasurement(name, "Fastest Time", "Slowest Time", "Average Time", "s", 3, info...) measurement.Results = append(measurement.Results, elapsedTime.Seconds()) return } func (b *benchmarker) RecordValue(name string, value float64, info ...interface{}) { - measurement := b.getMeasurement(name, "Smallest", " Largest", " Average", "", info...) + measurement := b.getMeasurement(name, "Smallest", " Largest", " Average", "", 3, info...) b.mu.Lock() defer b.mu.Unlock() measurement.Results = append(measurement.Results, value) } -func (b *benchmarker) getMeasurement(name string, smallestLabel string, largestLabel string, averageLabel string, units string, info ...interface{}) *types.SpecMeasurement { +func (b *benchmarker) RecordValueWithPrecision(name string, value float64, units string, precision int, info ...interface{}) { + measurement := b.getMeasurement(name, "Smallest", " Largest", " Average", units, precision, info...) + b.mu.Lock() + defer b.mu.Unlock() + measurement.Results = append(measurement.Results, value) +} + +func (b *benchmarker) getMeasurement(name string, smallestLabel string, largestLabel string, averageLabel string, units string, precision int, info ...interface{}) *types.SpecMeasurement { measurement, ok := b.measurements[name] if !ok { var computedInfo interface{} @@ -57,6 +64,7 @@ func (b *benchmarker) getMeasurement(name string, smallestLabel string, largestL LargestLabel: largestLabel, AverageLabel: averageLabel, Units: units, + Precision: precision, Results: make([]float64, 0), } b.measurements[name] = measurement diff --git a/vendor/github.com/onsi/ginkgo/internal/remote/output_interceptor_unix.go b/vendor/github.com/onsi/ginkgo/internal/remote/output_interceptor_unix.go index 1235ad00cbb..980065da575 100644 --- a/vendor/github.com/onsi/ginkgo/internal/remote/output_interceptor_unix.go +++ b/vendor/github.com/onsi/ginkgo/internal/remote/output_interceptor_unix.go @@ -1,4 +1,4 @@ -// +build freebsd openbsd netbsd dragonfly darwin linux +// +build freebsd openbsd netbsd dragonfly darwin linux solaris package remote diff --git a/vendor/github.com/onsi/ginkgo/internal/remote/syscall_dup_solaris.go b/vendor/github.com/onsi/ginkgo/internal/remote/syscall_dup_solaris.go new file mode 100644 index 00000000000..75ef7fb78e3 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/internal/remote/syscall_dup_solaris.go @@ -0,0 +1,9 @@ +// +build solaris + +package remote + +import "golang.org/x/sys/unix" + +func syscallDup(oldfd int, newfd int) (err error) { + return unix.Dup2(oldfd, newfd) +} diff --git a/vendor/github.com/onsi/ginkgo/internal/remote/syscall_dup_unix.go b/vendor/github.com/onsi/ginkgo/internal/remote/syscall_dup_unix.go index e7fad5bbb03..ef625596007 100644 --- a/vendor/github.com/onsi/ginkgo/internal/remote/syscall_dup_unix.go +++ b/vendor/github.com/onsi/ginkgo/internal/remote/syscall_dup_unix.go @@ -1,5 +1,6 @@ // +build !linux !arm64 // +build !windows +// +build !solaris package remote diff --git a/vendor/github.com/onsi/ginkgo/internal/spec/spec.go b/vendor/github.com/onsi/ginkgo/internal/spec/spec.go index ef788b76a48..d32dec6997c 100644 --- a/vendor/github.com/onsi/ginkgo/internal/spec/spec.go +++ b/vendor/github.com/onsi/ginkgo/internal/spec/spec.go @@ -17,9 +17,10 @@ type Spec struct { containers []*containernode.ContainerNode - state types.SpecState - runTime time.Duration - failure types.SpecFailure + state types.SpecState + runTime time.Duration + failure types.SpecFailure + previousFailures bool } func New(subject leafnodes.SubjectNode, containers []*containernode.ContainerNode, announceProgress bool) *Spec { @@ -58,6 +59,10 @@ func (spec *Spec) Passed() bool { return spec.state == types.SpecStatePassed } +func (spec *Spec) Flaked() bool { + return spec.state == types.SpecStatePassed && spec.previousFailures +} + func (spec *Spec) Pending() bool { return spec.state == types.SpecStatePending } @@ -109,6 +114,10 @@ func (spec *Spec) ConcatenatedString() string { } func (spec *Spec) Run(writer io.Writer) { + if spec.state == types.SpecStateFailed { + spec.previousFailures = true + } + startTime := time.Now() defer func() { spec.runTime = time.Since(startTime) diff --git a/vendor/github.com/onsi/ginkgo/internal/specrunner/spec_runner.go b/vendor/github.com/onsi/ginkgo/internal/specrunner/spec_runner.go index 7ca7740ba9f..869aaecaebb 100644 --- a/vendor/github.com/onsi/ginkgo/internal/specrunner/spec_runner.go +++ b/vendor/github.com/onsi/ginkgo/internal/specrunner/spec_runner.go @@ -137,21 +137,20 @@ func (runner *SpecRunner) runSpecs() bool { if skipRemainingSpecs { spec.Skip() } - runner.reportSpecWillRun(spec.Summary(runner.suiteID)) if !spec.Skipped() && !spec.Pending() { - runner.runningSpec = spec - spec.Run(runner.writer) - runner.runningSpec = nil - if spec.Failed() { + if passed := runner.runSpec(spec); !passed { suiteFailed = true } } else if spec.Pending() && runner.config.FailOnPending { + runner.reportSpecWillRun(spec.Summary(runner.suiteID)) suiteFailed = true + runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed()) + } else { + runner.reportSpecWillRun(spec.Summary(runner.suiteID)) + runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed()) } - runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed()) - if spec.Failed() && runner.config.FailFast { skipRemainingSpecs = true } @@ -160,6 +159,26 @@ func (runner *SpecRunner) runSpecs() bool { return !suiteFailed } +func (runner *SpecRunner) runSpec(spec *spec.Spec) (passed bool) { + maxAttempts := 1 + if runner.config.FlakeAttempts > 0 { + // uninitialized configs count as 1 + maxAttempts = runner.config.FlakeAttempts + } + + for i := 0; i < maxAttempts; i++ { + runner.reportSpecWillRun(spec.Summary(runner.suiteID)) + runner.runningSpec = spec + spec.Run(runner.writer) + runner.runningSpec = nil + runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed()) + if !spec.Failed() { + return true + } + } + return false +} + func (runner *SpecRunner) CurrentSpecSummary() (*types.SpecSummary, bool) { if runner.runningSpec == nil { return nil, false @@ -300,6 +319,10 @@ func (runner *SpecRunner) summary(success bool) *types.SuiteSummary { return ex.Passed() }) + numberOfFlakedSpecs := runner.countSpecsSatisfying(func(ex *spec.Spec) bool { + return ex.Flaked() + }) + numberOfFailedSpecs := runner.countSpecsSatisfying(func(ex *spec.Spec) bool { return ex.Failed() }) @@ -320,5 +343,6 @@ func (runner *SpecRunner) summary(success bool) *types.SuiteSummary { NumberOfSkippedSpecs: numberOfSkippedSpecs, NumberOfPassedSpecs: numberOfPassedSpecs, NumberOfFailedSpecs: numberOfFailedSpecs, + NumberOfFlakedSpecs: numberOfFlakedSpecs, } } diff --git a/vendor/github.com/onsi/ginkgo/reporters/stenographer/stenographer.go b/vendor/github.com/onsi/ginkgo/reporters/stenographer/stenographer.go index 5b5d905da16..b588924756e 100644 --- a/vendor/github.com/onsi/ginkgo/reporters/stenographer/stenographer.go +++ b/vendor/github.com/onsi/ginkgo/reporters/stenographer/stenographer.go @@ -506,15 +506,15 @@ func (s *consoleStenographer) measurementReport(spec *types.SpecSummary, succinc message = append(message, fmt.Sprintf(" %s - %s: %s%s, %s: %s%s ± %s%s, %s: %s%s", s.colorize(boldStyle, "%s", measurement.Name), measurement.SmallestLabel, - s.colorize(greenColor, "%.3f", measurement.Smallest), + s.colorize(greenColor, measurement.PrecisionFmt(), measurement.Smallest), measurement.Units, measurement.AverageLabel, - s.colorize(cyanColor, "%.3f", measurement.Average), + s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.Average), measurement.Units, - s.colorize(cyanColor, "%.3f", measurement.StdDeviation), + s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.StdDeviation), measurement.Units, measurement.LargestLabel, - s.colorize(redColor, "%.3f", measurement.Largest), + s.colorize(redColor, measurement.PrecisionFmt(), measurement.Largest), measurement.Units, )) } @@ -531,15 +531,15 @@ func (s *consoleStenographer) measurementReport(spec *types.SpecSummary, succinc s.colorize(boldStyle, "%s", measurement.Name), info, measurement.SmallestLabel, - s.colorize(greenColor, "%.3f", measurement.Smallest), + s.colorize(greenColor, measurement.PrecisionFmt(), measurement.Smallest), measurement.Units, measurement.LargestLabel, - s.colorize(redColor, "%.3f", measurement.Largest), + s.colorize(redColor, measurement.PrecisionFmt(), measurement.Largest), measurement.Units, measurement.AverageLabel, - s.colorize(cyanColor, "%.3f", measurement.Average), + s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.Average), measurement.Units, - s.colorize(cyanColor, "%.3f", measurement.StdDeviation), + s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.StdDeviation), measurement.Units, )) } diff --git a/vendor/github.com/onsi/ginkgo/types/types.go b/vendor/github.com/onsi/ginkgo/types/types.go index 889612e0a72..dcf95ba51fa 100644 --- a/vendor/github.com/onsi/ginkgo/types/types.go +++ b/vendor/github.com/onsi/ginkgo/types/types.go @@ -1,6 +1,9 @@ package types -import "time" +import ( + "strconv" + "time" +) const GINKGO_FOCUS_EXIT_CODE = 197 @@ -16,7 +19,10 @@ type SuiteSummary struct { NumberOfSkippedSpecs int NumberOfPassedSpecs int NumberOfFailedSpecs int - RunTime time.Duration + // Flaked specs are those that failed initially, but then passed on a + // subsequent try. + NumberOfFlakedSpecs int + RunTime time.Duration } type SpecSummary struct { @@ -100,6 +106,17 @@ type SpecMeasurement struct { LargestLabel string AverageLabel string Units string + Precision int +} + +func (s SpecMeasurement) PrecisionFmt() string { + if s.Precision == 0 { + return "%f" + } + + str := strconv.Itoa(s.Precision) + + return "%." + str + "f" } type SpecState uint