Don't fail performance tests on single fail event

test/e2e/density.go

@@ -18,6 +18,7 @@ package e2e
 
 import (
 	"fmt"
+	"math"
 	"strconv"
 	"sync"
 	"time"
@@ -108,7 +109,7 @@ var _ = Describe("Density", func() {
 			nameStr := strconv.Itoa(totalPods) + "-" + string(util.NewUUID())
 			RCName = "my-hostname-density" + nameStr
 
-			// Create a listener for events
+			// Create a listener for events.
 			events := make([](*api.Event), 0)
 			_, controller := framework.NewInformer(
 				&cache.ListWatch{
@@ -130,8 +131,11 @@ var _ = Describe("Density", func() {
 			stop := make(chan struct{})
 			go controller.Run(stop)
 
-			// Start the replication controller
+			// Start the replication controller.
+			startTime := time.Now()
 			expectNoError(RunRC(c, RCName, ns, "gcr.io/google_containers/pause:go", totalPods))
+			e2eStartupTime := time.Now().Sub(startTime)
+			Logf("E2E startup time for %d pods: %v", totalPods, e2eStartupTime)
 
 			By("Waiting for all events to be recorded")
 			last := -1
@@ -148,13 +152,9 @@ var _ = Describe("Density", func() {
 			}
 			Logf("Found %d events", current)
 
-			// Verify there were no pod killings or failures
-			By("Verifying there were no pod killings or failures")
-			for _, e := range events {
-				for _, s := range []string{"kill", "fail"} {
-					Expect(e.Reason).NotTo(ContainSubstring(s), "event:' %s', reason: '%s', message: '%s', field path: '%s'", e, e.ObjectMeta.Name, e.Message, e.InvolvedObject.FieldPath)
-				}
-			}
+			// Tune the threshold for allowed failures.
+			badEvents := BadEvents(events)
+			Expect(badEvents).NotTo(BeNumerically(">", int(math.Floor(0.01*float64(totalPods)))))
 		})
 	}