Add docs about process of discovering disks from new nodes

This commit is contained in:
Hemant Kumar 2020-11-18 22:13:03 -05:00
parent 46a57b0f5c
commit b6ae233c40
3 changed files with 67 additions and 44 deletions

View File

@ -611,15 +611,15 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN
// BuildMissingVolumeNodeMap builds a map of volumes and nodes which are not known to attach detach controller. // BuildMissingVolumeNodeMap builds a map of volumes and nodes which are not known to attach detach controller.
// There could be nodes in cluster which do not have any pods with vsphere volumes running on them // There could be nodes in cluster which do not have any pods with vsphere volumes running on them
// such nodes won't be part of disk verificatio check because attach-detach controller does not keep track // such nodes won't be part of disk verification check because attach-detach controller does not keep track
// such nodes. But such nodes may still have dangling volumes on them and hence we need to scan all the // such nodes. But such nodes may still have dangling volumes on them and hence we need to scan all the
// remaining nodes which weren't scanned by code previously. // remaining nodes which weren't scanned by code previously.
func (vs *VSphere) BuildMissingVolumeNodeMap(ctx context.Context) { func (vs *VSphere) BuildMissingVolumeNodeMap(ctx context.Context) {
nodeDetails := vs.nodeManager.GetNodeNames() nodeNames := vs.nodeManager.GetNodeNames()
// Segregate nodes according to VC-DC // Segregate nodes according to VC-DC
dcNodes := make(map[string][]k8stypes.NodeName) dcNodes := make(map[string][]k8stypes.NodeName)
for _, nodeName := range nodeDetails { for _, nodeName := range nodeNames {
// if given node is not in node volume map // if given node is not in node volume map
if !vs.vsphereVolumeMap.CheckForNode(nodeName) { if !vs.vsphereVolumeMap.CheckForNode(nodeName) {
nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName) nodeInfo, err := vs.nodeManager.GetNodeInfo(nodeName)

View File

@ -36,14 +36,14 @@ type nodeVolumeStatus struct {
// VsphereVolumeMap stores last known state of node and volume mapping // VsphereVolumeMap stores last known state of node and volume mapping
type VsphereVolumeMap struct { type VsphereVolumeMap struct {
volumeNodeMap map[volumePath]nodeVolumeStatus volumeNodeMap map[volumePath]nodeVolumeStatus
knownNodes map[k8stypes.NodeName]bool nodeMap map[k8stypes.NodeName]bool
lock sync.RWMutex lock sync.RWMutex
} }
func NewVsphereVolumeMap() *VsphereVolumeMap { func NewVsphereVolumeMap() *VsphereVolumeMap {
return &VsphereVolumeMap{ return &VsphereVolumeMap{
volumeNodeMap: map[volumePath]nodeVolumeStatus{}, volumeNodeMap: map[volumePath]nodeVolumeStatus{},
knownNodes: map[k8stypes.NodeName]bool{}, nodeMap: map[k8stypes.NodeName]bool{},
} }
} }
@ -56,6 +56,9 @@ func (vsphereVolume *VsphereVolumeMap) StartDiskVerification() {
v.verified = false v.verified = false
vsphereVolume.volumeNodeMap[k] = v vsphereVolume.volumeNodeMap[k] = v
} }
// reset nodeMap to empty so that any node we could not verify via usual verification process
// can still be verified.
vsphereVolume.nodeMap = map[k8stypes.NodeName]bool{}
} }
// CheckForVolume verifies if disk is attached to some node in the cluster. // CheckForVolume verifies if disk is attached to some node in the cluster.
@ -74,11 +77,8 @@ func (vsphereVolume *VsphereVolumeMap) CheckForVolume(path string) (k8stypes.Nod
func (vsphereVolume *VsphereVolumeMap) CheckForNode(nodeName k8stypes.NodeName) bool { func (vsphereVolume *VsphereVolumeMap) CheckForNode(nodeName k8stypes.NodeName) bool {
vsphereVolume.lock.RLock() vsphereVolume.lock.RLock()
defer vsphereVolume.lock.RUnlock() defer vsphereVolume.lock.RUnlock()
_, ok := vsphereVolume.knownNodes[nodeName] _, ok := vsphereVolume.nodeMap[nodeName]
if ok { return ok
return true
}
return false
} }
// Add all devices found on a node to the device map // Add all devices found on a node to the device map
@ -91,7 +91,7 @@ func (vsphereVolume *VsphereVolumeMap) Add(node k8stypes.NodeName, vmDevices obj
if backing, ok := virtualDevice.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok { if backing, ok := virtualDevice.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok {
filename := volumePath(backing.FileName) filename := volumePath(backing.FileName)
vsphereVolume.volumeNodeMap[filename] = nodeVolumeStatus{node, true} vsphereVolume.volumeNodeMap[filename] = nodeVolumeStatus{node, true}
vsphereVolume.knownNodes[node] = true vsphereVolume.nodeMap[node] = true
} }
} }
} }
@ -104,7 +104,7 @@ func (vsphereVolume *VsphereVolumeMap) RemoveUnverified() {
for k, v := range vsphereVolume.volumeNodeMap { for k, v := range vsphereVolume.volumeNodeMap {
if !v.verified { if !v.verified {
delete(vsphereVolume.volumeNodeMap, k) delete(vsphereVolume.volumeNodeMap, k)
delete(vsphereVolume.knownNodes, v.nodeName) delete(vsphereVolume.nodeMap, v.nodeName)
} }
} }
} }

View File

@ -28,34 +28,66 @@ import (
func TestVsphereVolumeMap(t *testing.T) { func TestVsphereVolumeMap(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
deviceToAdd object.VirtualDeviceList deviceToAdd object.VirtualDeviceList
nodeToAdd k8stypes.NodeName nodeToAdd k8stypes.NodeName
volumeToCheck string checkRunner func(volumeMap *VsphereVolumeMap)
runVerification bool
expectInMap bool
}{ }{
{ {
name: "adding new volume", name: "adding new volume",
deviceToAdd: getVirtualDeviceList("[foobar] kubevols/foo.vmdk"), deviceToAdd: getVirtualDeviceList("[foobar] kubevols/foo.vmdk"),
nodeToAdd: convertToK8sType("node1.lan"), nodeToAdd: convertToK8sType("node1.lan"),
volumeToCheck: "[foobar] kubevols/foo.vmdk", checkRunner: func(volumeMap *VsphereVolumeMap) {
expectInMap: true, volumeToCheck := "[foobar] kubevols/foo.vmdk"
_, ok := volumeMap.CheckForVolume(volumeToCheck)
if !ok {
t.Errorf("error checking volume %s, expected true got %v", volumeToCheck, ok)
}
},
}, },
{ {
name: "mismatching volume", name: "mismatching volume",
deviceToAdd: getVirtualDeviceList("[foobar] kubevols/foo.vmdk"), deviceToAdd: getVirtualDeviceList("[foobar] kubevols/foo.vmdk"),
nodeToAdd: convertToK8sType("node1.lan"), nodeToAdd: convertToK8sType("node1.lan"),
volumeToCheck: "[foobar] kubevols/bar.vmdk", checkRunner: func(volumeMap *VsphereVolumeMap) {
expectInMap: false, volumeToCheck := "[foobar] kubevols/bar.vmdk"
_, ok := volumeMap.CheckForVolume(volumeToCheck)
if ok {
t.Errorf("error checking volume %s, expected false got %v", volumeToCheck, ok)
}
},
}, },
{ {
name: "should remove unverified devices", name: "should remove unverified devices",
deviceToAdd: getVirtualDeviceList("[foobar] kubevols/foo.vmdk"), deviceToAdd: getVirtualDeviceList("[foobar] kubevols/foo.vmdk"),
nodeToAdd: convertToK8sType("node1.lan"), nodeToAdd: convertToK8sType("node1.lan"),
volumeToCheck: "[foobar] kubevols/foo.vmdk", checkRunner: func(volumeMap *VsphereVolumeMap) {
runVerification: true, volumeMap.StartDiskVerification()
expectInMap: false, volumeMap.RemoveUnverified()
volumeToCheck := "[foobar] kubevols/foo.vmdk"
_, ok := volumeMap.CheckForVolume(volumeToCheck)
if ok {
t.Errorf("error checking volume %s, expected false got %v", volumeToCheck, ok)
}
node := k8stypes.NodeName("node1.lan")
ok = volumeMap.CheckForNode(node)
if ok {
t.Errorf("unexpected node %s in node map", node)
}
},
},
{
name: "node check should return false for previously added node",
deviceToAdd: getVirtualDeviceList("[foobar] kubevols/foo.vmdk"),
nodeToAdd: convertToK8sType("node1.lan"),
checkRunner: func(volumeMap *VsphereVolumeMap) {
volumeMap.StartDiskVerification()
node := k8stypes.NodeName("node1.lan")
ok := volumeMap.CheckForNode(node)
if ok {
t.Errorf("unexpected node %s in node map", node)
}
},
}, },
} }
@ -63,16 +95,7 @@ func TestVsphereVolumeMap(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
vMap := NewVsphereVolumeMap() vMap := NewVsphereVolumeMap()
vMap.Add(tc.nodeToAdd, tc.deviceToAdd) vMap.Add(tc.nodeToAdd, tc.deviceToAdd)
tc.checkRunner(vMap)
if tc.runVerification {
vMap.StartDiskVerification()
vMap.RemoveUnverified()
}
_, ok := vMap.CheckForVolume(tc.volumeToCheck)
nodeOk := vMap.CheckForNode(tc.nodeToAdd)
if ok != tc.expectInMap && nodeOk != tc.expectInMap {
t.Errorf("error checking volume %s, expected %v got %v", tc.volumeToCheck, tc.expectInMap, ok)
}
}) })
} }
} }