diff --git a/test/e2e/density.go b/test/e2e/density.go index c2fc23761b4..00c7ad44ad2 100644 --- a/test/e2e/density.go +++ b/test/e2e/density.go @@ -148,7 +148,6 @@ var _ = Describe("Density", func() { expectNoError(writePerfData(c, fmt.Sprintf(testContext.OutputDir+"/%s", uuid), "after")) // Verify latency metrics - // TODO: We should reset metrics before the test. Currently previous tests influence latency metrics. highLatencyRequests, err := HighLatencyRequests(c, 3*time.Second, util.NewStringSet("events")) expectNoError(err) Expect(highLatencyRequests).NotTo(BeNumerically(">", 0), "There should be no high-latency requests") @@ -386,6 +385,16 @@ var _ = Describe("Density", func() { printLatencies(schedToWatchLag, "worst scheduled-to-end total latencies") printLatencies(e2eLag, "worst e2e total latencies") + // Test whether e2e pod startup time is acceptable. + // TODO: Switch it to 5 seconds once we are sure our tests are passing. + podStartupThreshold := 8 * time.Second + e2ePodStartupTime50perc := e2eLag[len(e2eLag)/2].Latency + e2ePodStartupTime90perc := e2eLag[len(e2eLag)*9/10].Latency + e2ePodStartupTime99perc := e2eLag[len(e2eLag)*99/100].Latency + Expect(e2ePodStartupTime50perc < podStartupThreshold).To(Equal(true), "Too high pod startup time 50th percentile") + Expect(e2ePodStartupTime90perc < podStartupThreshold).To(Equal(true), "Too high pod startup time 90th percentile") + Expect(e2ePodStartupTime99perc < podStartupThreshold).To(Equal(true), "Too high pod startup time 99th percentile") + // Log suspicious latency metrics/docker errors from all nodes that had slow startup times for _, l := range startupLag { if l.Latency > NodeStartupThreshold {