mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-01-04 23:17:50 +00:00
Merge pull request #16053 from saad-ali/attachDetachMutextFix
Fix GCE Cloud/Attach/Detach stability issues
This commit is contained in:
Daniel Smith
82
pkg/util/keymutex/keymutex.go
Normal file
82
pkg/util/keymutex/keymutex.go
Normal file
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package keymutex
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/golang/glog"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// KeyMutex is a thread-safe interface for aquiring locks on arbitrary strings.
|
||||
type KeyMutex interface {
|
||||
// Aquires a lock associated with the specified ID, creates the lock if one doesn't already exist.
|
||||
LockKey(id string)
|
||||
|
||||
// Releases the lock associated with the specified ID.
|
||||
// Returns an error if the specified ID doesn't exist.
|
||||
UnlockKey(id string) error
|
||||
}
|
||||
|
||||
// Returns a new instance of a key mutex.
|
||||
func NewKeyMutex() KeyMutex {
|
||||
return &keyMutex{
|
||||
mutexMap: make(map[string]*sync.Mutex),
|
||||
}
|
||||
}
|
||||
|
||||
type keyMutex struct {
|
||||
sync.RWMutex
|
||||
mutexMap map[string]*sync.Mutex
|
||||
}
|
||||
|
||||
// Aquires a lock associated with the specified ID (creates the lock if one doesn't already exist).
|
||||
func (km *keyMutex) LockKey(id string) {
|
||||
glog.V(5).Infof("LockKey(...) called for id %q\r\n", id)
|
||||
mutex := km.getOrCreateLock(id)
|
||||
mutex.Lock()
|
||||
glog.V(5).Infof("LockKey(...) for id %q completed.\r\n", id)
|
||||
}
|
||||
|
||||
// Releases the lock associated with the specified ID.
|
||||
// Returns an error if the specified ID doesn't exist.
|
||||
func (km *keyMutex) UnlockKey(id string) error {
|
||||
glog.V(5).Infof("UnlockKey(...) called for id %q\r\n", id)
|
||||
km.RLock()
|
||||
defer km.RUnlock()
|
||||
mutex, exists := km.mutexMap[id]
|
||||
if !exists {
|
||||
return fmt.Errorf("id %q not found", id)
|
||||
}
|
||||
glog.V(5).Infof("UnlockKey(...) for id. Mutex found, trying to unlock it. %q\r\n", id)
|
||||
|
||||
mutex.Unlock()
|
||||
glog.V(5).Infof("UnlockKey(...) for id %q completed.\r\n", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns lock associated with the specified ID, or creates the lock if one doesn't already exist.
|
||||
func (km *keyMutex) getOrCreateLock(id string) *sync.Mutex {
|
||||
km.Lock()
|
||||
defer km.Unlock()
|
||||
|
||||
if _, exists := km.mutexMap[id]; !exists {
|
||||
km.mutexMap[id] = &sync.Mutex{}
|
||||
}
|
||||
|
||||
return km.mutexMap[id]
|
||||
}
|
||||
111
pkg/util/keymutex/keymutex_test.go
Normal file
111
pkg/util/keymutex/keymutex_test.go
Normal file
@@ -0,0 +1,111 @@
|
||||
/*
|
||||
Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package keymutex
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
callbackTimeout = 1 * time.Second
|
||||
)
|
||||
|
||||
func Test_SingleLock_NoUnlock(t *testing.T) {
|
||||
// Arrange
|
||||
km := NewKeyMutex()
|
||||
key := "fakeid"
|
||||
callbackCh := make(chan interface{})
|
||||
|
||||
// Act
|
||||
go lockAndCallback(km, key, callbackCh)
|
||||
|
||||
// Assert
|
||||
verifyCallbackHappens(t, callbackCh)
|
||||
}
|
||||
|
||||
func Test_SingleLock_SingleUnlock(t *testing.T) {
|
||||
// Arrange
|
||||
km := NewKeyMutex()
|
||||
key := "fakeid"
|
||||
callbackCh := make(chan interface{})
|
||||
|
||||
// Act & Assert
|
||||
go lockAndCallback(km, key, callbackCh)
|
||||
verifyCallbackHappens(t, callbackCh)
|
||||
km.UnlockKey(key)
|
||||
}
|
||||
|
||||
func Test_DoubleLock_DoubleUnlock(t *testing.T) {
|
||||
// Arrange
|
||||
km := NewKeyMutex()
|
||||
key := "fakeid"
|
||||
callbackCh1stLock := make(chan interface{})
|
||||
callbackCh2ndLock := make(chan interface{})
|
||||
|
||||
// Act & Assert
|
||||
go lockAndCallback(km, key, callbackCh1stLock)
|
||||
verifyCallbackHappens(t, callbackCh1stLock)
|
||||
go lockAndCallback(km, key, callbackCh2ndLock)
|
||||
verifyCallbackDoesntHappens(t, callbackCh2ndLock)
|
||||
km.UnlockKey(key)
|
||||
verifyCallbackHappens(t, callbackCh2ndLock)
|
||||
km.UnlockKey(key)
|
||||
}
|
||||
|
||||
func lockAndCallback(km KeyMutex, id string, callbackCh chan<- interface{}) {
|
||||
km.LockKey(id)
|
||||
callbackCh <- true
|
||||
}
|
||||
|
||||
func verifyCallbackHappens(t *testing.T, callbackCh <-chan interface{}) bool {
|
||||
select {
|
||||
case <-callbackCh:
|
||||
return true
|
||||
case <-time.After(callbackTimeout):
|
||||
t.Fatalf("Timed out waiting for callback.")
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func verifyCallbackDoesntHappens(t *testing.T, callbackCh <-chan interface{}) bool {
|
||||
select {
|
||||
case <-callbackCh:
|
||||
t.Fatalf("Unexpected callback.")
|
||||
return false
|
||||
case <-time.After(callbackTimeout):
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func verifyNoError(t *testing.T, err error, name string) {
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected response on %q. Expected: <no error> Actual: <%v>", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyError(t *testing.T, err error, name string) {
|
||||
if err == nil {
|
||||
t.Fatalf("Unexpected response on %q. Expected: <error> Actual: <no error>", name)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyMsg(t *testing.T, expected, actual string) {
|
||||
if actual != expected {
|
||||
t.Fatalf("Unexpected testMsg value. Expected: <%v> Actual: <%v>", expected, actual)
|
||||
}
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package operationmanager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Operation Manager is a thread-safe interface for keeping track of multiple pending async operations.
|
||||
type OperationManager interface {
|
||||
// Called when the operation with the given ID has started.
|
||||
// Creates a new channel with specified buffer size tracked with the specified ID.
|
||||
// Returns a read-only version of the newly created channel.
|
||||
// Returns an error if an entry with the specified ID already exists (previous entry must be removed by calling Close).
|
||||
Start(id string, bufferSize uint) (<-chan interface{}, error)
|
||||
|
||||
// Called when the operation with the given ID has terminated.
|
||||
// Closes and removes the channel associated with ID.
|
||||
// Returns an error if no associated channel exists.
|
||||
Close(id string) error
|
||||
|
||||
// Attempts to send msg to the channel associated with ID.
|
||||
// Returns an error if no associated channel exists.
|
||||
Send(id string, msg interface{}) error
|
||||
|
||||
// Returns true if an entry with the specified ID already exists.
|
||||
Exists(id string) bool
|
||||
}
|
||||
|
||||
// Returns a new instance of a channel manager.
|
||||
func NewOperationManager() OperationManager {
|
||||
return &operationManager{
|
||||
chanMap: make(map[string]chan interface{}),
|
||||
}
|
||||
}
|
||||
|
||||
type operationManager struct {
|
||||
sync.RWMutex
|
||||
chanMap map[string]chan interface{}
|
||||
}
|
||||
|
||||
// Called when the operation with the given ID has started.
|
||||
// Creates a new channel with specified buffer size tracked with the specified ID.
|
||||
// Returns a read-only version of the newly created channel.
|
||||
// Returns an error if an entry with the specified ID already exists (previous entry must be removed by calling Close).
|
||||
func (cm *operationManager) Start(id string, bufferSize uint) (<-chan interface{}, error) {
|
||||
cm.Lock()
|
||||
defer cm.Unlock()
|
||||
if _, exists := cm.chanMap[id]; exists {
|
||||
return nil, fmt.Errorf("id %q already exists", id)
|
||||
}
|
||||
cm.chanMap[id] = make(chan interface{}, bufferSize)
|
||||
return cm.chanMap[id], nil
|
||||
}
|
||||
|
||||
// Called when the operation with the given ID has terminated.
|
||||
// Closes and removes the channel associated with ID.
|
||||
// Returns an error if no associated channel exists.
|
||||
func (cm *operationManager) Close(id string) error {
|
||||
cm.Lock()
|
||||
defer cm.Unlock()
|
||||
if _, exists := cm.chanMap[id]; !exists {
|
||||
return fmt.Errorf("id %q not found", id)
|
||||
}
|
||||
close(cm.chanMap[id])
|
||||
delete(cm.chanMap, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Attempts to send msg to the channel associated with ID.
|
||||
// Returns an error if no associated channel exists.
|
||||
func (cm *operationManager) Send(id string, msg interface{}) error {
|
||||
cm.RLock()
|
||||
defer cm.RUnlock()
|
||||
if _, exists := cm.chanMap[id]; !exists {
|
||||
return fmt.Errorf("id %q not found", id)
|
||||
}
|
||||
cm.chanMap[id] <- msg
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns true if an entry with the specified ID already exists.
|
||||
func (cm *operationManager) Exists(id string) (exists bool) {
|
||||
cm.RLock()
|
||||
defer cm.RUnlock()
|
||||
_, exists = cm.chanMap[id]
|
||||
return
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
/*
|
||||
Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Channel Manager keeps track of multiple channels
|
||||
package operationmanager
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStart(t *testing.T) {
|
||||
// Arrange
|
||||
cm := NewOperationManager()
|
||||
chanId := "testChanId"
|
||||
testMsg := "test message"
|
||||
|
||||
// Act
|
||||
ch, startErr := cm.Start(chanId, 1 /* bufferSize */)
|
||||
sigErr := cm.Send(chanId, testMsg)
|
||||
|
||||
// Assert
|
||||
verifyNoError(t, startErr, "Start")
|
||||
verifyNoError(t, sigErr, "Send")
|
||||
actualMsg := <-ch
|
||||
verifyMsg(t, testMsg /* expected */, actualMsg.(string) /* actual */)
|
||||
}
|
||||
|
||||
func TestStartIdExists(t *testing.T) {
|
||||
// Arrange
|
||||
cm := NewOperationManager()
|
||||
chanId := "testChanId"
|
||||
|
||||
// Act
|
||||
_, startErr1 := cm.Start(chanId, 1 /* bufferSize */)
|
||||
_, startErr2 := cm.Start(chanId, 1 /* bufferSize */)
|
||||
|
||||
// Assert
|
||||
verifyNoError(t, startErr1, "Start1")
|
||||
verifyError(t, startErr2, "Start2")
|
||||
}
|
||||
|
||||
func TestStartAndAdd2Chans(t *testing.T) {
|
||||
// Arrange
|
||||
cm := NewOperationManager()
|
||||
chanId1 := "testChanId1"
|
||||
chanId2 := "testChanId2"
|
||||
testMsg1 := "test message 1"
|
||||
testMsg2 := "test message 2"
|
||||
|
||||
// Act
|
||||
ch1, startErr1 := cm.Start(chanId1, 1 /* bufferSize */)
|
||||
ch2, startErr2 := cm.Start(chanId2, 1 /* bufferSize */)
|
||||
sigErr1 := cm.Send(chanId1, testMsg1)
|
||||
sigErr2 := cm.Send(chanId2, testMsg2)
|
||||
|
||||
// Assert
|
||||
verifyNoError(t, startErr1, "Start1")
|
||||
verifyNoError(t, startErr2, "Start2")
|
||||
verifyNoError(t, sigErr1, "Send1")
|
||||
verifyNoError(t, sigErr2, "Send2")
|
||||
actualMsg1 := <-ch1
|
||||
actualMsg2 := <-ch2
|
||||
verifyMsg(t, testMsg1 /* expected */, actualMsg1.(string) /* actual */)
|
||||
verifyMsg(t, testMsg2 /* expected */, actualMsg2.(string) /* actual */)
|
||||
}
|
||||
|
||||
func TestStartAndAdd2ChansAndClose(t *testing.T) {
|
||||
// Arrange
|
||||
cm := NewOperationManager()
|
||||
chanId1 := "testChanId1"
|
||||
chanId2 := "testChanId2"
|
||||
testMsg1 := "test message 1"
|
||||
testMsg2 := "test message 2"
|
||||
|
||||
// Act
|
||||
ch1, startErr1 := cm.Start(chanId1, 1 /* bufferSize */)
|
||||
ch2, startErr2 := cm.Start(chanId2, 1 /* bufferSize */)
|
||||
sigErr1 := cm.Send(chanId1, testMsg1)
|
||||
sigErr2 := cm.Send(chanId2, testMsg2)
|
||||
cm.Close(chanId1)
|
||||
sigErr3 := cm.Send(chanId1, testMsg1)
|
||||
|
||||
// Assert
|
||||
verifyNoError(t, startErr1, "Start1")
|
||||
verifyNoError(t, startErr2, "Start2")
|
||||
verifyNoError(t, sigErr1, "Send1")
|
||||
verifyNoError(t, sigErr2, "Send2")
|
||||
verifyError(t, sigErr3, "Send3")
|
||||
actualMsg1 := <-ch1
|
||||
actualMsg2 := <-ch2
|
||||
verifyMsg(t, testMsg1 /* expected */, actualMsg1.(string) /* actual */)
|
||||
verifyMsg(t, testMsg2 /* expected */, actualMsg2.(string) /* actual */)
|
||||
}
|
||||
|
||||
func TestExists(t *testing.T) {
|
||||
// Arrange
|
||||
cm := NewOperationManager()
|
||||
chanId1 := "testChanId1"
|
||||
chanId2 := "testChanId2"
|
||||
|
||||
// Act & Assert
|
||||
verifyExists(t, cm, chanId1, false /* expected */)
|
||||
verifyExists(t, cm, chanId2, false /* expected */)
|
||||
|
||||
_, startErr1 := cm.Start(chanId1, 1 /* bufferSize */)
|
||||
verifyNoError(t, startErr1, "Start1")
|
||||
verifyExists(t, cm, chanId1, true /* expected */)
|
||||
verifyExists(t, cm, chanId2, false /* expected */)
|
||||
|
||||
_, startErr2 := cm.Start(chanId2, 1 /* bufferSize */)
|
||||
verifyNoError(t, startErr2, "Start2")
|
||||
verifyExists(t, cm, chanId1, true /* expected */)
|
||||
verifyExists(t, cm, chanId2, true /* expected */)
|
||||
|
||||
cm.Close(chanId1)
|
||||
verifyExists(t, cm, chanId1, false /* expected */)
|
||||
verifyExists(t, cm, chanId2, true /* expected */)
|
||||
|
||||
cm.Close(chanId2)
|
||||
verifyExists(t, cm, chanId1, false /* expected */)
|
||||
verifyExists(t, cm, chanId2, false /* expected */)
|
||||
}
|
||||
|
||||
func verifyExists(t *testing.T, cm OperationManager, id string, expected bool) {
|
||||
if actual := cm.Exists(id); expected != actual {
|
||||
t.Fatalf("Unexpected Exists(%q) response. Expected: <%v> Actual: <%v>", id, expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyNoError(t *testing.T, err error, name string) {
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected response on %q. Expected: <no error> Actual: <%v>", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyError(t *testing.T, err error, name string) {
|
||||
if err == nil {
|
||||
t.Fatalf("Unexpected response on %q. Expected: <error> Actual: <no error>", name)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyMsg(t *testing.T, expected, actual string) {
|
||||
if actual != expected {
|
||||
t.Fatalf("Unexpected testMsg value. Expected: <%v> Actual: <%v>", expected, actual)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user