github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-20 01:01:22 +00:00
Code Issues Projects Releases Wiki Activity

Scalability tests: Increase sample size for pod startup latency measurement.

Browse Source
This commit is contained in:
Maciej Borsz 2018-08-02 13:14:24 +02:00
parent 4bec356e01
commit 2632a6a619
1 changed files with 68 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
test/e2e/scalability/density.go
View File

@ -56,6 +56,7 @@ const (
MinSaturationThreshold = 2 * time.Minute
MinPodsPerSecondThroughput = 8
DensityPollInterval = 10 * time.Second
MinPodStartupMeasurements = 500
)
// Maximum container failures this test tolerates before failing.
@ -696,7 +697,10 @@ var _ = SIGDescribe("Density", func() {
}
e2eStartupTime = runDensityTest(dConfig, testPhaseDurations, &scheduleThroughputs)
if itArg.runLatencyTest {
By("Scheduling additional Pods to measure startup latencies")
// Pick latencyPodsIterations so that:
// latencyPodsIterations * nodeCount >= MinPodStartupMeasurements.
latencyPodsIterations := (MinPodStartupMeasurements + nodeCount - 1) / nodeCount
By(fmt.Sprintf("Scheduling additional %d Pods to measure startup latencies", latencyPodsIterations*nodeCount))
createTimes := make(map[string]metav1.Time, 0)
nodeNames := make(map[string]string, 0)
@ -775,9 +779,14 @@ var _ = SIGDescribe("Density", func() {
go controller.Run(stopCh)
}
for latencyPodsIteration := 0; latencyPodsIteration < latencyPodsIterations; latencyPodsIteration++ {
podIndexOffset := latencyPodsIteration * nodeCount
framework.Logf("Creating %d latency pods in range [%d, %d]", nodeCount, podIndexOffset+1, podIndexOffset+nodeCount)
watchTimesLen := len(watchTimes)
// Create some additional pods with throughput ~5 pods/sec.
latencyPodStartupPhase := testPhaseDurations.StartPhase(800, "latency pods creation")
latencyPodStartupPhase := testPhaseDurations.StartPhase(800+latencyPodsIteration*10, "latency pods creation")
defer latencyPodStartupPhase.End()
var wg sync.WaitGroup
wg.Add(nodeCount)
@ -795,7 +804,7 @@ var _ = SIGDescribe("Density", func() {
}
rcNameToNsMap := map[string]string{}
for i := 1; i <= nodeCount; i++ {
name := additionalPodsPrefix + "-" + strconv.Itoa(i)
name := additionalPodsPrefix + "-" + strconv.Itoa(podIndexOffset+i)
nsName := namespaces[i%len(namespaces)].Name
rcNameToNsMap[name] = nsName
go createRunningPodFromRC(&wg, c, name, nsName, imageutils.GetPauseImageName(), additionalPodsPrefix, cpuRequest, memRequest)
@ -804,16 +813,15 @@ var _ = SIGDescribe("Density", func() {
wg.Wait()
latencyPodStartupPhase.End()
latencyMeasurementPhase := testPhaseDurations.StartPhase(810, "pod startup latencies measurement")
latencyMeasurementPhase := testPhaseDurations.StartPhase(801+latencyPodsIteration*10, "pod startup latencies measurement")
defer latencyMeasurementPhase.End()
By("Waiting for all Pods begin observed by the watch...")
waitTimeout := 10 * time.Minute
for start := time.Now(); len(watchTimes) < nodeCount; time.Sleep(10 * time.Second) {
for start := time.Now(); len(watchTimes) < watchTimesLen+nodeCount; time.Sleep(10 * time.Second) {
if time.Since(start) < waitTimeout {
framework.Failf("Timeout reached waiting for all Pods being observed by the watch.")
}
}
close(stopCh)
nodeToLatencyPods := make(map[string]int)
for i := range latencyPodStores {
@ -827,6 +835,20 @@ var _ = SIGDescribe("Density", func() {
}
}
}
latencyMeasurementPhase.End()
By("Removing additional replication controllers")
podDeletionPhase := testPhaseDurations.StartPhase(802+latencyPodsIteration*10, "latency pods deletion")
defer podDeletionPhase.End()
deleteRC := func(i int) {
defer GinkgoRecover()
name := additionalPodsPrefix + "-" + strconv.Itoa(podIndexOffset+i+1)
framework.ExpectNoError(framework.DeleteRCAndWaitForGC(c, rcNameToNsMap[name], name))
}
workqueue.Parallelize(25, nodeCount, deleteRC)
podDeletionPhase.End()
}
close(stopCh)
for i := 0; i < len(namespaces); i++ {
nsName := namespaces[i].Name
@ -914,18 +936,6 @@ var _ = SIGDescribe("Density", func() {
framework.ExpectNoError(framework.VerifyLatencyWithinThreshold(podStartupLatencyThreshold, podStartupLatency.E2ELatency, "pod startup"))
framework.LogSuspiciousLatency(startupLag, e2eLag, nodeCount, c)
latencyMeasurementPhase.End()
By("Removing additional replication controllers")
podDeletionPhase := testPhaseDurations.StartPhase(820, "latency pods deletion")
defer podDeletionPhase.End()
deleteRC := func(i int) {
defer GinkgoRecover()
name := additionalPodsPrefix + "-" + strconv.Itoa(i+1)
framework.ExpectNoError(framework.DeleteRCAndWaitForGC(c, rcNameToNsMap[name], name))
}
workqueue.Parallelize(25, nodeCount, deleteRC)
podDeletionPhase.End()
}
cleanupDensityTest(dConfig, testPhaseDurations)
})