Merge pull request #66296 from shyamjvs/flake-reporting-util

Automatic merge from submit-queue (batch tested with PRs 66296, 66382). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add flake-reporting utility to testing framework

One step towards https://github.com/kubernetes/kubernetes/issues/66239

/cc @wojtek-t @mborsz 
(whoever can review first)

```release-note
NONE
```
This commit is contained in:
Kubernetes Submit Queue 2018-07-23 17:26:01 -07:00 committed by GitHub
commit adc14cc30c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 107 additions and 0 deletions

View File

@ -15,6 +15,7 @@ go_library(
"deployment_util.go",
"exec_util.go",
"firewall_util.go",
"flake_reporting_util.go",
"framework.go",
"get-kubemark-resource-usage.go",
"google_compute.go",

View File

@ -0,0 +1,91 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package framework
import (
"bytes"
"fmt"
"sync"
)
type FlakeReport struct {
lock sync.RWMutex
Flakes []string `json:"flakes"`
FlakeCount int `json:"flakeCount"`
}
func NewFlakeReport() *FlakeReport {
return &FlakeReport{
Flakes: []string{},
}
}
func buildDescription(optionalDescription ...interface{}) string {
switch len(optionalDescription) {
case 0:
return ""
default:
return fmt.Sprintf(optionalDescription[0].(string), optionalDescription[1:]...)
}
}
// RecordFlakeIfError records the error (if non-nil) as a flake along with an optional description.
// This can be used as a replacement of framework.ExpectNoError() for non-critical errors that can
// be considered as 'flakes' to avoid causing failures in tests.
func (f *FlakeReport) RecordFlakeIfError(err error, optionalDescription ...interface{}) {
if err == nil {
return
}
msg := fmt.Sprintf("Unexpected error occurred: %v", err)
desc := buildDescription(optionalDescription)
if desc != "" {
msg = fmt.Sprintf("%v (Description: %v)", msg, desc)
}
Logf(msg)
f.lock.Lock()
defer f.lock.Unlock()
f.Flakes = append(f.Flakes, msg)
f.FlakeCount++
}
func (f *FlakeReport) GetFlakeCount() int {
f.lock.RLock()
defer f.lock.RUnlock()
return f.FlakeCount
}
func (f *FlakeReport) PrintHumanReadable() string {
f.lock.RLock()
defer f.lock.RUnlock()
buf := bytes.Buffer{}
buf.WriteString(fmt.Sprintf("FlakeCount: %v\n", f.FlakeCount))
buf.WriteString("Flakes:\n")
for _, flake := range f.Flakes {
buf.WriteString(fmt.Sprintf("%v\n", flake))
}
return buf.String()
}
func (f *FlakeReport) PrintJSON() string {
f.lock.RLock()
defer f.lock.RUnlock()
return PrettyPrintJSON(f)
}
func (f *FlakeReport) SummaryKind() string {
return "FlakeReport"
}

View File

@ -90,6 +90,9 @@ type Framework struct {
logsSizeCloseChannel chan bool
logsSizeVerifier *LogsSizeVerifier
// Flaky operation failures in an e2e test can be captured through this.
flakeReport *FlakeReport
// To make sure that this framework cleans up after itself, no matter what,
// we install a Cleanup action before each test and clear it after. If we
// should abort, the AfterSuite hook should run all Cleanup actions.
@ -269,6 +272,8 @@ func (f *Framework) BeforeEach() {
}
}
f.flakeReport = NewFlakeReport()
}
// AfterEach deletes the namespace, after reading its events.
@ -382,6 +387,12 @@ func (f *Framework) AfterEach() {
close(f.kubemarkControllerCloseChannel)
}
// Report any flakes that were observed in the e2e test and reset.
if f.flakeReport != nil && f.flakeReport.GetFlakeCount() > 0 {
f.TestSummaries = append(f.TestSummaries, f.flakeReport)
f.flakeReport = nil
}
PrintSummaries(f.TestSummaries, f.BaseName)
// Check whether all nodes are ready after the test.
@ -409,6 +420,10 @@ func (f *Framework) CreateNamespace(baseName string, labels map[string]string) (
return ns, err
}
func (f *Framework) RecordFlakeIfError(err error, optionalDescription ...interface{}) {
f.flakeReport.RecordFlakeIfError(err, optionalDescription)
}
// AddNamespacesToDelete adds one or more namespaces to be deleted when the test
// completes.
func (f *Framework) AddNamespacesToDelete(namespaces ...*v1.Namespace) {