diff --git a/test/e2e/BUILD b/test/e2e/BUILD index e77d250a9e6..e5ffc1ba5a4 100644 --- a/test/e2e/BUILD +++ b/test/e2e/BUILD @@ -76,6 +76,7 @@ go_library( "//test/e2e/framework/providers/vsphere:go_default_library", "//test/e2e/framework/testfiles:go_default_library", "//test/e2e/manifest:go_default_library", + "//test/e2e/reporters:go_default_library", "//test/utils:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/ginkgo/config:go_default_library", @@ -113,6 +114,7 @@ filegroup( "//test/e2e/network:all-srcs", "//test/e2e/node:all-srcs", "//test/e2e/perftype:all-srcs", + "//test/e2e/reporters:all-srcs", "//test/e2e/scheduling:all-srcs", "//test/e2e/servicecatalog:all-srcs", "//test/e2e/storage:all-srcs", diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index b26ebd56f6e..bbecc6ce55e 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -41,6 +41,7 @@ import ( e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/manifest" + e2ereporters "k8s.io/kubernetes/test/e2e/reporters" testutils "k8s.io/kubernetes/test/utils" utilnet "k8s.io/utils/net" @@ -100,8 +101,11 @@ func RunE2ETests(t *testing.T) { r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode)))) } } - klog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode) + // Stream the progress to stdout and optionally a URL accepting progress updates. + r = append(r, e2ereporters.NewProgressReporter(framework.TestContext.ProgressReportURL)) + + klog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode) ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r) } diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index e30b5cf543a..77f5e6ffa24 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -166,6 +166,9 @@ type TestContextType struct { // NonblockingTaints is the comma-delimeted string given by the user to specify taints which should not stop the test framework from running tests. NonblockingTaints string + + // ProgressReportURL is the URL which progress updates will be posted to as tests complete. If empty, no updates are sent. + ProgressReportURL string } // NodeKillerConfig describes configuration of NodeKiller -- a utility to @@ -292,6 +295,8 @@ func RegisterCommonFlags(flags *flag.FlagSet) { flags.BoolVar(&TestContext.ListImages, "list-images", false, "If true, will show list of images used for runnning tests.") flags.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.") + + flags.StringVar(&TestContext.ProgressReportURL, "progress-report-url", "", "The URL to POST progress updates to as the suite runs to assist in aiding integrations. If empty, no messages sent.") } // RegisterClusterFlags registers flags specific to the cluster e2e test suite. diff --git a/test/e2e/reporters/BUILD b/test/e2e/reporters/BUILD new file mode 100644 index 00000000000..a3c54e1219c --- /dev/null +++ b/test/e2e/reporters/BUILD @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["progress.go"], + importpath = "k8s.io/kubernetes/test/e2e/reporters", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/onsi/ginkgo/config:go_default_library", + "//vendor/github.com/onsi/ginkgo/types:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/test/e2e/reporters/progress.go b/test/e2e/reporters/progress.go new file mode 100644 index 00000000000..0c30698cad8 --- /dev/null +++ b/test/e2e/reporters/progress.go @@ -0,0 +1,153 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reporters + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" + + "k8s.io/klog" + + "github.com/onsi/ginkgo/config" + "github.com/onsi/ginkgo/types" +) + +// ProgressReporter is a ginkgo reporter which tracks the total number of tests to be run/passed/failed/skipped. +// As new tests are completed it updates the values and prints them to stdout and optionally, sends the updates +// to the configured URL. +type ProgressReporter struct { + LastMsg string `json:"msg"` + + TestsTotal int `json:"total"` + TestsCompleted int `json:"completed"` + TestsSkipped int `json:"skipped"` + TestsFailed int `json:"failed"` + + Failures []string `json:"failures,omitempty"` + + progressURL string + client *http.Client +} + +// NewProgressReporter returns a progress reporter which posts updates to the given URL. +func NewProgressReporter(progressReportURL string) *ProgressReporter { + rep := &ProgressReporter{ + Failures: []string{}, + progressURL: progressReportURL, + } + if len(progressReportURL) > 0 { + rep.client = &http.Client{ + Timeout: time.Second * 10, + } + } + return rep +} + +// SpecSuiteWillBegin is invoked by ginkgo when the suite is about to start and is the first point in which we can +// antipate the number of tests which will be run. +func (reporter *ProgressReporter) SpecSuiteWillBegin(cfg config.GinkgoConfigType, summary *types.SuiteSummary) { + reporter.TestsTotal = summary.NumberOfSpecsThatWillBeRun + reporter.LastMsg = "Test Suite starting" + reporter.sendUpdates() +} + +// SpecSuiteDidEnd is the last method invoked by Ginkgo after all the specs are run. +func (reporter *ProgressReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { + reporter.LastMsg = "Test Suite completed" + reporter.sendUpdates() +} + +// SpecDidComplete is invoked by Ginkgo each time a spec is completed (including skipped specs). +func (reporter *ProgressReporter) SpecDidComplete(specSummary *types.SpecSummary) { + testname := strings.Join(specSummary.ComponentTexts[1:], " ") + switch specSummary.State { + case types.SpecStateFailed: + if len(specSummary.ComponentTexts) > 0 { + reporter.Failures = append(reporter.Failures, testname) + } else { + reporter.Failures = append(reporter.Failures, "Unknown test name") + } + reporter.TestsFailed++ + reporter.LastMsg = fmt.Sprintf("FAILED %v", testname) + case types.SpecStatePassed: + reporter.TestsCompleted++ + reporter.LastMsg = fmt.Sprintf("PASSED %v", testname) + case types.SpecStateSkipped: + reporter.TestsSkipped++ + return + default: + return + } + + reporter.sendUpdates() +} + +// sendUpdates serializes the current progress and prints it to stdout and also posts it to the configured endpoint if set. +func (reporter *ProgressReporter) sendUpdates() { + b := reporter.serialize() + fmt.Println(string(b)) + go reporter.postProgressToURL(b) +} + +func (reporter *ProgressReporter) postProgressToURL(b []byte) { + // If a progressURL and client is set/available then POST to it. Noop otherwise. + if reporter.client == nil || len(reporter.progressURL) == 0 { + return + } + + resp, err := reporter.client.Post(reporter.progressURL, "application/json", bytes.NewReader(b)) + if err != nil { + klog.Errorf("Failed to post progress update to %v: %v", reporter.progressURL, err) + return + } + if resp.StatusCode >= 400 { + klog.Errorf("Unexpected response when posting progress update to %v: %v", reporter.progressURL, resp.StatusCode) + if resp.Body != nil { + defer resp.Body.Close() + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + klog.Errorf("Failed to read response body from posting progress: %v", err) + return + } + klog.Errorf("Response body from posting progress update: %v", respBody) + } + + return + } +} + +func (reporter *ProgressReporter) serialize() []byte { + b, err := json.Marshal(reporter) + if err != nil { + return []byte(fmt.Sprintf(`{"msg":"%v", "error":"%v"}`, reporter.LastMsg, err)) + } + return b +} + +// SpecWillRun is implemented as a noop to satisfy the reporter interface for ginkgo. +func (reporter *ProgressReporter) SpecWillRun(specSummary *types.SpecSummary) {} + +// BeforeSuiteDidRun is implemented as a noop to satisfy the reporter interface for ginkgo. +func (reporter *ProgressReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {} + +// AfterSuiteDidRun is implemented as a noop to satisfy the reporter interface for ginkgo. +func (reporter *ProgressReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {}