diff --git a/listener.go b/listener.go
index 28db3ff..1c0bb21 100644
--- a/listener.go
+++ b/listener.go
@@ -237,8 +237,8 @@ func (c *closeWrapper) close() error {
 }
 
 func (c *closeWrapper) Close() error {
-	c.l.Lock()
-	defer c.l.Unlock()
+	c.l.connLock.Lock()
+	defer c.l.connLock.Unlock()
 	return c.close()
 }