Merge pull request #66296 from shyamjvs/flake-reporting-util

Automatic merge from submit-queue (batch tested with PRs 66296, 66382). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add flake-reporting utility to testing framework

One step towards https://github.com/kubernetes/kubernetes/issues/66239

/cc @wojtek-t @mborsz 
(whoever can review first)

```release-note
NONE
```

test/e2e/framework/BUILD

@@ -15,6 +15,7 @@ go_library(
         "deployment_util.go",
         "exec_util.go",
         "firewall_util.go",
+        "flake_reporting_util.go",
         "framework.go",
         "get-kubemark-resource-usage.go",
         "google_compute.go",

test/e2e/framework/flake_reporting_util.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+)
+
+type FlakeReport struct {
+	lock       sync.RWMutex
+	Flakes     []string `json:"flakes"`
+	FlakeCount int      `json:"flakeCount"`
+}
+
+func NewFlakeReport() *FlakeReport {
+	return &FlakeReport{
+		Flakes: []string{},
+	}
+}
+
+func buildDescription(optionalDescription ...interface{}) string {
+	switch len(optionalDescription) {
+	case 0:
+		return ""
+	default:
+		return fmt.Sprintf(optionalDescription[0].(string), optionalDescription[1:]...)
+	}
+}
+
+// RecordFlakeIfError records the error (if non-nil) as a flake along with an optional description.
+// This can be used as a replacement of framework.ExpectNoError() for non-critical errors that can
+// be considered as 'flakes' to avoid causing failures in tests.
+func (f *FlakeReport) RecordFlakeIfError(err error, optionalDescription ...interface{}) {
+	if err == nil {
+		return
+	}
+	msg := fmt.Sprintf("Unexpected error occurred: %v", err)
+	desc := buildDescription(optionalDescription)
+	if desc != "" {
+		msg = fmt.Sprintf("%v (Description: %v)", msg, desc)
+	}
+	Logf(msg)
+	f.lock.Lock()
+	defer f.lock.Unlock()
+	f.Flakes = append(f.Flakes, msg)
+	f.FlakeCount++
+}
+
+func (f *FlakeReport) GetFlakeCount() int {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.FlakeCount
+}
+
+func (f *FlakeReport) PrintHumanReadable() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	buf := bytes.Buffer{}
+	buf.WriteString(fmt.Sprintf("FlakeCount: %v\n", f.FlakeCount))
+	buf.WriteString("Flakes:\n")
+	for _, flake := range f.Flakes {
+		buf.WriteString(fmt.Sprintf("%v\n", flake))
+	}
+	return buf.String()
+}
+
+func (f *FlakeReport) PrintJSON() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return PrettyPrintJSON(f)
+}
+
+func (f *FlakeReport) SummaryKind() string {
+	return "FlakeReport"
+}

test/e2e/framework/framework.go

@@ -90,6 +90,9 @@ type Framework struct {
 	logsSizeCloseChannel chan bool
 	logsSizeVerifier     *LogsSizeVerifier
 
+	// Flaky operation failures in an e2e test can be captured through this.
+	flakeReport *FlakeReport
+
 	// To make sure that this framework cleans up after itself, no matter what,
 	// we install a Cleanup action before each test and clear it after.  If we
 	// should abort, the AfterSuite hook should run all Cleanup actions.
@@ -269,6 +272,8 @@ func (f *Framework) BeforeEach() {
 		}
 
 	}
+
+	f.flakeReport = NewFlakeReport()
 }
 
 // AfterEach deletes the namespace, after reading its events.
@@ -382,6 +387,12 @@ func (f *Framework) AfterEach() {
 		close(f.kubemarkControllerCloseChannel)
 	}
 
+	// Report any flakes that were observed in the e2e test and reset.
+	if f.flakeReport != nil && f.flakeReport.GetFlakeCount() > 0 {
+		f.TestSummaries = append(f.TestSummaries, f.flakeReport)
+		f.flakeReport = nil
+	}
+
 	PrintSummaries(f.TestSummaries, f.BaseName)
 
 	// Check whether all nodes are ready after the test.
@@ -409,6 +420,10 @@ func (f *Framework) CreateNamespace(baseName string, labels map[string]string) (
 	return ns, err
 }
 
+func (f *Framework) RecordFlakeIfError(err error, optionalDescription ...interface{}) {
+	f.flakeReport.RecordFlakeIfError(err, optionalDescription)
+}
+
 // AddNamespacesToDelete adds one or more namespaces to be deleted when the test
 // completes.
 func (f *Framework) AddNamespacesToDelete(namespaces ...*v1.Namespace) {