github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-06-29 16:57:18 +00:00
Code Issues Projects Releases Wiki Activity

monitor: enlarge watch buffer

Browse Source 
enlarge watch buffer, or no one consume monitor watcher if shim.Wait and shim.watchSandbox
simultaneously receive signal

goroutine 60 [semacquire, 641 minutes]:
sync.runtime_SemacquireMutex(0xc00037a144, 0x42cd00)
/usr/local/go/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xc00037a140)
/usr/local/go/src/sync/mutex.go:134 +0x109
github.com/kata-containers/runtime/virtcontainers.(*monitor).stop(0xc00037a140)
/go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:95 +0x5f
github.com/kata-containers/runtime/virtcontainers.(*Sandbox).Delete(0xc0003c8160, 0x78effdc01, 0x0)
/go/src/github.com/kata-containers/runtime/virtcontainers/sandbox.go:773 +0x4fb
github.com/kata-containers/runtime/containerd-shim-v2.wait(0xc000478b80, 0xc000338240, 0x0, 0x0, 0x107d540, 0xc0000100f0, 0x107d520)
/go/src/github.com/kata-containers/runtime/containerd-shim-v2/wait.go:60 +0x3e5
created by github.com/kata-containers/runtime/containerd-shim-v2.startContainer
/go/src/github.com/kata-containers/runtime/containerd-shim-v2/start.go:74 +0x3e5

goroutine 53 [chan send, 641 minutes]:
github.com/kata-containers/runtime/virtcontainers.(*monitor).notify(0xc00037a140, 0x107cfe0, 0xc0001ec160)
/go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:87 +0xed
github.com/kata-containers/runtime/virtcontainers.(*monitor).watchAgent(0xc00037a140)
/go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:125 +0xab
github.com/kata-containers/runtime/virtcontainers.(*monitor).newWatcher.func1(0xc00037a140)
/go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:59 +0x72
created by github.com/kata-containers/runtime/virtcontainers.(*monitor).newWatcher
/go/src/github.com/kata-containers/runtime/virtcontainers/monitor.go:49 +0x125

Fixes: #1981

Signed-off-by: Ace-Tang <aceapril@126.com>
This commit is contained in:
Ace-Tang 2019-08-13 12:58:13 +08:00
parent 7019ce5c9b
commit 88e281cb14
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
virtcontainers/monitor.go
View File

@ -12,7 +12,10 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
const defaultCheckInterval = 1 * time.Second const (
defaultCheckInterval = 1 * time.Second
watcherChannelSize = 128
)
type monitor struct { type monitor struct {
sync.Mutex sync.Mutex
@ -37,7 +40,7 @@ func (m *monitor) newWatcher() (chan error, error) {
m.Lock() m.Lock()
defer m.Unlock() defer m.Unlock()
watcher := make(chan error, 1) watcher := make(chan error, watcherChannelSize)
m.watchers = append(m.watchers, watcher) m.watchers = append(m.watchers, watcher)
if !m.running { if !m.running {
@ -83,7 +86,14 @@ func (m *monitor) notify(err error) {
}() }()
for _, c := range m.watchers { for _, c := range m.watchers {
c <- err // throw away message can not write to channel
// make it not stuck, the first error is useful.
select {
case c <- err:
default:
virtLog.WithField("channel-size", watcherChannelSize).Warnf("watcher channel is full, throw notify message")
}
} }
} }
@ -98,8 +108,8 @@ func (m *monitor) stop() {
return return
} }
m.stopCh <- true
defer func() { defer func() {
m.stopCh <- true
m.watchers = nil m.watchers = nil
m.running = false m.running = false
}() }()