github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-07-17 11:28:58 +00:00
Code Issues Projects Releases Wiki Activity

Ensure CSINode belongs to current node on init

Browse Source 
When a CSI plugin attempts to register on a node, the node checks if the CSINode object exists. If it does but the ownerReference of the CSINode does not match the UID of the current node, possibly because the node object was recreated with the same name, we end up in a race condition where the CSINode object will be updated but subsequently deleted by the GC controller.
In this situation, the CSINode object will be gone and won't be recreated unless the CSI plugin or the kubelet are restarted.
This commit fixes this race by checking that the CSINode object belong to the current node during initialization.
If it doesn't, it means that the CSINode object is left over from a previous node and it must be removed first. Once removed, registration can progress as usual.
This commit is contained in:
Baptiste Girard-Carrabin
2025-03-04 15:29:42 +01:00
parent bdda0a530e
commit a375b0aa36
1 changed files with 38 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
pkg/volume/csi/nodeinfomanager/nodeinfomanager.go
View File

@@ -63,6 +63,7 @@ var (
// the Node and CSINode objects.
type nodeInfoManager struct {
nodeName types.NodeName
nodeID types.UID
volumeHost volume.VolumeHost
migratedPlugins map[string](func() bool)
// lock protects changes to node.
@@ -430,6 +431,12 @@ func (nim *nodeInfoManager) tryInitializeCSINodeWithAnnotation(csiKubeClient cli
nim.lock.Lock()
defer nim.lock.Unlock()
node, err := csiKubeClient.CoreV1().Nodes().Get(context.TODO(), string(nim.nodeName), metav1.GetOptions{})
if err != nil {
return err
}
nim.nodeID = node.UID
nodeInfo, err := csiKubeClient.StorageV1().CSINodes().Get(context.TODO(), string(nim.nodeName), metav1.GetOptions{})
if nodeInfo == nil || errors.IsNotFound(err) {
// CreateCSINode will set the annotation
@@ -439,6 +446,16 @@ func (nim *nodeInfoManager) tryInitializeCSINodeWithAnnotation(csiKubeClient cli
return err
}
if !nim.nodeOwnsCSINode(nodeInfo) {
klog.V(2).Infof("existing CSINode %q is owned by different node, cleaning up...", nodeInfo.Name)
err = nim.DeleteCSINode()
if err != nil {
return fmt.Errorf("error deleting existing CSINode %q: %w", nodeInfo.Name, err)
}
// Returning now so that the next attempt can create a new CSINode object
return fmt.Errorf("CSINode %q was owned by different node, deleted it", nodeInfo.Name)
}
annotationModified := setMigrationAnnotation(nim.migratedPlugins, nodeInfo)
if annotationModified {
@@ -449,6 +466,15 @@ func (nim *nodeInfoManager) tryInitializeCSINodeWithAnnotation(csiKubeClient cli
}
func (nim *nodeInfoManager) nodeOwnsCSINode(nodeInfo *storagev1.CSINode) bool {
for _, ownerRef := range nodeInfo.OwnerReferences {
if ownerRef.Kind == nodeKind.Kind && ownerRef.Name == string(nim.nodeName) && ownerRef.UID == nim.nodeID {
return true
}
}
return false
}
func (nim *nodeInfoManager) CreateCSINode() (*storagev1.CSINode, error) {
csiKubeClient := nim.volumeHost.GetKubeClient()
@@ -456,11 +482,6 @@ func (nim *nodeInfoManager) CreateCSINode() (*storagev1.CSINode, error) {
return nil, fmt.Errorf("error getting CSI client")
}
node, err := csiKubeClient.CoreV1().Nodes().Get(context.TODO(), string(nim.nodeName), metav1.GetOptions{})
if err != nil {
return nil, err
}
nodeInfo := &storagev1.CSINode{
ObjectMeta: metav1.ObjectMeta{
Name: string(nim.nodeName),
@@ -468,8 +489,8 @@ func (nim *nodeInfoManager) CreateCSINode() (*storagev1.CSINode, error) {
{
APIVersion: nodeKind.Version,
Kind: nodeKind.Kind,
Name: node.Name,
UID: node.UID,
Name: string(nim.nodeName),
UID: nim.nodeID,
},
},
},
@@ -483,6 +504,16 @@ func (nim *nodeInfoManager) CreateCSINode() (*storagev1.CSINode, error) {
return csiKubeClient.StorageV1().CSINodes().Create(context.TODO(), nodeInfo, metav1.CreateOptions{})
}
func (nim *nodeInfoManager) DeleteCSINode() error {
csiKubeClient := nim.volumeHost.GetKubeClient()
if csiKubeClient == nil {
return fmt.Errorf("error getting CSI client")
}
return csiKubeClient.StorageV1().CSINodes().Delete(context.TODO(), string(nim.nodeName), metav1.DeleteOptions{})
}
func setMigrationAnnotation(migratedPlugins map[string](func() bool), nodeInfo *storagev1.CSINode) (modified bool) {
if migratedPlugins == nil {
return false