From 90bc19959df6a96223f4848366e9f1c2cc7ea103 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Thu, 6 Oct 2016 10:57:46 +0200 Subject: [PATCH] Extend logging in cacher to understand its bottleneck --- pkg/storage/cacher.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/storage/cacher.go b/pkg/storage/cacher.go index 8b230ffcb03..b5f7aa28bd6 100644 --- a/pkg/storage/cacher.go +++ b/pkg/storage/cacher.go @@ -713,9 +713,23 @@ func (c *cacheWatcher) sendWatchCacheEvent(event watchCacheEvent) { func (c *cacheWatcher) process(initEvents []watchCacheEvent, resourceVersion uint64) { defer utilruntime.HandleCrash() + // Check how long we are processing initEvents. + // As long as these are not processed, we are not processing + // any incoming events, so if it takes long, we may actually + // block all watchers for some time. + // TODO: If it appears to be long in some cases, we may consider + // - longer result buffers if there are a lot of initEvents + // - try some parallelization + const initProcessThreshold = 5 * time.Millisecond + startTime := time.Now() for _, event := range initEvents { c.sendWatchCacheEvent(event) } + processingTime := time.Since(startTime) + if processingTime > initProcessThreshold { + glog.V(2).Infof("processing %d initEvents took %v", len(initEvents), processingTime) + } + defer close(c.result) defer c.Stop() for {