From 88e281cb14a1ba1b22e4d77a79159d38ee22b804 Mon Sep 17 00:00:00 2001 From: Ace-Tang Date: Tue, 13 Aug 2019 12:58:13 +0800 Subject: [PATCH] monitor: enlarge watch buffer enlarge watch buffer, or no one consume monitor watcher if shim.Wait and shim.watchSandbox simultaneously receive signal goroutine 60 [semacquire, 641 minutes]: sync.runtime_SemacquireMutex(0xc00037a144, 0x42cd00) /usr/local/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc00037a140) /usr/local/go/src/sync/mutex.go:134 +0x109 github.com/kata-containers/runtime/virtcontainers.(*monitor).stop(0xc00037a140) /go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:95 +0x5f github.com/kata-containers/runtime/virtcontainers.(*Sandbox).Delete(0xc0003c8160, 0x78effdc01, 0x0) /go/src/github.com/kata-containers/runtime/virtcontainers/sandbox.go:773 +0x4fb github.com/kata-containers/runtime/containerd-shim-v2.wait(0xc000478b80, 0xc000338240, 0x0, 0x0, 0x107d540, 0xc0000100f0, 0x107d520) /go/src/github.com/kata-containers/runtime/containerd-shim-v2/wait.go:60 +0x3e5 created by github.com/kata-containers/runtime/containerd-shim-v2.startContainer /go/src/github.com/kata-containers/runtime/containerd-shim-v2/start.go:74 +0x3e5 goroutine 53 [chan send, 641 minutes]: github.com/kata-containers/runtime/virtcontainers.(*monitor).notify(0xc00037a140, 0x107cfe0, 0xc0001ec160) /go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:87 +0xed github.com/kata-containers/runtime/virtcontainers.(*monitor).watchAgent(0xc00037a140) /go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:125 +0xab github.com/kata-containers/runtime/virtcontainers.(*monitor).newWatcher.func1(0xc00037a140) /go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:59 +0x72 created by github.com/kata-containers/runtime/virtcontainers.(*monitor).newWatcher /go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:49 +0x125 Fixes: #1981 Signed-off-by: Ace-Tang --- virtcontainers/monitor.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/virtcontainers/monitor.go b/virtcontainers/monitor.go index ff5b4d0348..5b8f5f517b 100644 --- a/virtcontainers/monitor.go +++ b/virtcontainers/monitor.go @@ -12,7 +12,10 @@ import ( "github.com/pkg/errors" ) -const defaultCheckInterval = 1 * time.Second +const ( + defaultCheckInterval = 1 * time.Second + watcherChannelSize = 128 +) type monitor struct { sync.Mutex @@ -37,7 +40,7 @@ func (m *monitor) newWatcher() (chan error, error) { m.Lock() defer m.Unlock() - watcher := make(chan error, 1) + watcher := make(chan error, watcherChannelSize) m.watchers = append(m.watchers, watcher) if !m.running { @@ -83,7 +86,14 @@ func (m *monitor) notify(err error) { }() for _, c := range m.watchers { - c <- err + // throw away message can not write to channel + // make it not stuck, the first error is useful. + select { + case c <- err: + + default: + virtLog.WithField("channel-size", watcherChannelSize).Warnf("watcher channel is full, throw notify message") + } } } @@ -98,8 +108,8 @@ func (m *monitor) stop() { return } + m.stopCh <- true defer func() { - m.stopCh <- true m.watchers = nil m.running = false }()