Print progress updates to stdout and publish to URL

One common frustration of end users running the e2e suite is that
they take a significant amount of time and it is difficult to
gauge progress.

Even if tailing the logs it can be difficult to see where one
test starts and another ends or understand the if there have been
failures in the past 1h of logs.

This change adds a new custom reporter which prints summary information
as tests complete. This includes the number of tests to run and how
many have been passed/failed/skipped along with which tests have failed.

A new flag can be set which pushes these values to an endpoint. This is
intended for integration with Sonobuoy but any endpoint could consume and
surface this data to the user so they can better understand the state
of the current test run.

test/e2e/BUILD

@@ -76,6 +76,7 @@ go_library(
         "//test/e2e/framework/providers/vsphere:go_default_library",
         "//test/e2e/framework/testfiles:go_default_library",
         "//test/e2e/manifest:go_default_library",
+        "//test/e2e/reporters:go_default_library",
         "//test/utils:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/ginkgo/config:go_default_library",
@@ -113,6 +114,7 @@ filegroup(
         "//test/e2e/network:all-srcs",
         "//test/e2e/node:all-srcs",
         "//test/e2e/perftype:all-srcs",
+        "//test/e2e/reporters:all-srcs",
         "//test/e2e/scheduling:all-srcs",
         "//test/e2e/servicecatalog:all-srcs",
         "//test/e2e/storage:all-srcs",

test/e2e/e2e.go

@@ -41,6 +41,7 @@ import (
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	"k8s.io/kubernetes/test/e2e/manifest"
+	e2ereporters "k8s.io/kubernetes/test/e2e/reporters"
 	testutils "k8s.io/kubernetes/test/utils"
 	utilnet "k8s.io/utils/net"
 
@@ -100,8 +101,11 @@ func RunE2ETests(t *testing.T) {
 			r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode))))
 		}
 	}
-	klog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
 
+	// Stream the progress to stdout and optionally a URL accepting progress updates.
+	r = append(r, e2ereporters.NewProgressReporter(framework.TestContext.ProgressReportURL))
+
+	klog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
 	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r)
 }
 

test/e2e/framework/test_context.go

@@ -166,6 +166,9 @@ type TestContextType struct {
 
 	// NonblockingTaints is the comma-delimeted string given by the user to specify taints which should not stop the test framework from running tests.
 	NonblockingTaints string
+
+	// ProgressReportURL is the URL which progress updates will be posted to as tests complete. If empty, no updates are sent.
+	ProgressReportURL string
 }
 
 // NodeKillerConfig describes configuration of NodeKiller -- a utility to
@@ -292,6 +295,8 @@ func RegisterCommonFlags(flags *flag.FlagSet) {
 
 	flags.BoolVar(&TestContext.ListImages, "list-images", false, "If true, will show list of images used for runnning tests.")
 	flags.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.")
+
+	flags.StringVar(&TestContext.ProgressReportURL, "progress-report-url", "", "The URL to POST progress updates to as the suite runs to assist in aiding integrations. If empty, no messages sent.")
 }
 
 // RegisterClusterFlags registers flags specific to the cluster e2e test suite.

test/e2e/reporters/BUILD

@@ -0,0 +1,27 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["progress.go"],
+    importpath = "k8s.io/kubernetes/test/e2e/reporters",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/onsi/ginkgo/config:go_default_library",
+        "//vendor/github.com/onsi/ginkgo/types:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

test/e2e/reporters/progress.go

@@ -0,0 +1,153 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reporters
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"k8s.io/klog"
+
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/ginkgo/types"
+)
+
+// ProgressReporter is a ginkgo reporter which tracks the total number of tests to be run/passed/failed/skipped.
+// As new tests are completed it updates the values and prints them to stdout and optionally, sends the updates
+// to the configured URL.
+type ProgressReporter struct {
+	LastMsg string `json:"msg"`
+
+	TestsTotal     int `json:"total"`
+	TestsCompleted int `json:"completed"`
+	TestsSkipped   int `json:"skipped"`
+	TestsFailed    int `json:"failed"`
+
+	Failures []string `json:"failures,omitempty"`
+
+	progressURL string
+	client      *http.Client
+}
+
+// NewProgressReporter returns a progress reporter which posts updates to the given URL.
+func NewProgressReporter(progressReportURL string) *ProgressReporter {
+	rep := &ProgressReporter{
+		Failures:    []string{},
+		progressURL: progressReportURL,
+	}
+	if len(progressReportURL) > 0 {
+		rep.client = &http.Client{
+			Timeout: time.Second * 10,
+		}
+	}
+	return rep
+}
+
+// SpecSuiteWillBegin is invoked by ginkgo when the suite is about to start and is the first point in which we can
+// antipate the number of tests which will be run.
+func (reporter *ProgressReporter) SpecSuiteWillBegin(cfg config.GinkgoConfigType, summary *types.SuiteSummary) {
+	reporter.TestsTotal = summary.NumberOfSpecsThatWillBeRun
+	reporter.LastMsg = "Test Suite starting"
+	reporter.sendUpdates()
+}
+
+// SpecSuiteDidEnd is the last method invoked by Ginkgo after all the specs are run.
+func (reporter *ProgressReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) {
+	reporter.LastMsg = "Test Suite completed"
+	reporter.sendUpdates()
+}
+
+// SpecDidComplete is invoked by Ginkgo each time a spec is completed (including skipped specs).
+func (reporter *ProgressReporter) SpecDidComplete(specSummary *types.SpecSummary) {
+	testname := strings.Join(specSummary.ComponentTexts[1:], " ")
+	switch specSummary.State {
+	case types.SpecStateFailed:
+		if len(specSummary.ComponentTexts) > 0 {
+			reporter.Failures = append(reporter.Failures, testname)
+		} else {
+			reporter.Failures = append(reporter.Failures, "Unknown test name")
+		}
+		reporter.TestsFailed++
+		reporter.LastMsg = fmt.Sprintf("FAILED %v", testname)
+	case types.SpecStatePassed:
+		reporter.TestsCompleted++
+		reporter.LastMsg = fmt.Sprintf("PASSED %v", testname)
+	case types.SpecStateSkipped:
+		reporter.TestsSkipped++
+		return
+	default:
+		return
+	}
+
+	reporter.sendUpdates()
+}
+
+// sendUpdates serializes the current progress and prints it to stdout and also posts it to the configured endpoint if set.
+func (reporter *ProgressReporter) sendUpdates() {
+	b := reporter.serialize()
+	fmt.Println(string(b))
+	go reporter.postProgressToURL(b)
+}
+
+func (reporter *ProgressReporter) postProgressToURL(b []byte) {
+	// If a progressURL and client is set/available then POST to it. Noop otherwise.
+	if reporter.client == nil || len(reporter.progressURL) == 0 {
+		return
+	}
+
+	resp, err := reporter.client.Post(reporter.progressURL, "application/json", bytes.NewReader(b))
+	if err != nil {
+		klog.Errorf("Failed to post progress update to %v: %v", reporter.progressURL, err)
+		return
+	}
+	if resp.StatusCode >= 400 {
+		klog.Errorf("Unexpected response when posting progress update to %v: %v", reporter.progressURL, resp.StatusCode)
+		if resp.Body != nil {
+			defer resp.Body.Close()
+			respBody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				klog.Errorf("Failed to read response body from posting progress: %v", err)
+				return
+			}
+			klog.Errorf("Response body from posting progress update: %v", respBody)
+		}
+
+		return
+	}
+}
+
+func (reporter *ProgressReporter) serialize() []byte {
+	b, err := json.Marshal(reporter)
+	if err != nil {
+		return []byte(fmt.Sprintf(`{"msg":"%v", "error":"%v"}`, reporter.LastMsg, err))
+	}
+	return b
+}
+
+// SpecWillRun is implemented as a noop to satisfy the reporter interface for ginkgo.
+func (reporter *ProgressReporter) SpecWillRun(specSummary *types.SpecSummary) {}
+
+// BeforeSuiteDidRun is implemented as a noop to satisfy the reporter interface for ginkgo.
+func (reporter *ProgressReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {}
+
+// AfterSuiteDidRun is implemented as a noop to satisfy the reporter interface for ginkgo.
+func (reporter *ProgressReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {}