diff --git a/pkg/cloudprovider/providers/vsphere/nodemanager.go b/pkg/cloudprovider/providers/vsphere/nodemanager.go index 25f8d58c9e2..81aa928e99f 100644 --- a/pkg/cloudprovider/providers/vsphere/nodemanager.go +++ b/pkg/cloudprovider/providers/vsphere/nodemanager.go @@ -45,10 +45,13 @@ type NodeManager struct { nodeInfoMap map[string]*NodeInfo // Maps node name to node structure registeredNodes map[string]*v1.Node + //CredentialsManager + credentialManager *SecretCredentialManager // Mutexes - registeredNodesLock sync.RWMutex - nodeInfoLock sync.RWMutex + registeredNodesLock sync.RWMutex + nodeInfoLock sync.RWMutex + credentialManagerLock sync.Mutex } type NodeDetails struct { @@ -119,7 +122,7 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := vsi.conn.Connect(ctx) + err := nm.vcConnect(ctx, vsi) if err != nil { glog.V(4).Info("Discovering node error vc:", err) setGlobalErr(err) @@ -297,30 +300,17 @@ func (nm *NodeManager) GetNodeInfo(nodeName k8stypes.NodeName) (NodeInfo, error) // // This method is a getter but it can cause side-effect of updating NodeInfo objects. func (nm *NodeManager) GetNodeDetails() ([]NodeDetails, error) { - nm.nodeInfoLock.RLock() - defer nm.nodeInfoLock.RUnlock() + nm.registeredNodesLock.Lock() + defer nm.registeredNodesLock.Unlock() var nodeDetails []NodeDetails - vsphereSessionRefreshMap := make(map[string]bool) - for nodeName, nodeInfo := range nm.nodeInfoMap { - var n *NodeInfo - var err error - if vsphereSessionRefreshMap[nodeInfo.vcServer] { - // vSphere connection already refreshed. Just refresh VM and Datacenter. - glog.V(4).Infof("Renewing NodeInfo %+v for node %q. No new connection needed.", nodeInfo, nodeName) - n, err = nm.renewNodeInfo(nodeInfo, false) - } else { - // Refresh vSphere connection, VM and Datacenter. - glog.V(4).Infof("Renewing NodeInfo %+v for node %q with new vSphere connection.", nodeInfo, nodeName) - n, err = nm.renewNodeInfo(nodeInfo, true) - vsphereSessionRefreshMap[nodeInfo.vcServer] = true - } + for nodeName, nodeObj := range nm.registeredNodes { + nodeInfo, err := nm.GetNodeInfoWithNodeObject(nodeObj) if err != nil { return nil, err } - nm.nodeInfoMap[nodeName] = n glog.V(4).Infof("Updated NodeInfo %q for node %q.", nodeInfo, nodeName) - nodeDetails = append(nodeDetails, NodeDetails{nodeName, n.vm, n.vmUUID}) + nodeDetails = append(nodeDetails, NodeDetails{nodeName, nodeInfo.vm, nodeInfo.vmUUID}) } return nodeDetails, nil } @@ -355,7 +345,7 @@ func (nm *NodeManager) renewNodeInfo(nodeInfo *NodeInfo, reconnect bool) (*NodeI return nil, err } if reconnect { - err := vsphereInstance.conn.Connect(ctx) + err := nm.vcConnect(ctx, vsphereInstance) if err != nil { return nil, err } @@ -370,3 +360,82 @@ func (nodeInfo *NodeInfo) VM() *vclib.VirtualMachine { } return nodeInfo.vm } + +// vcConnect connects to vCenter with existing credentials +// If credentials are invalid: +// 1. It will fetch credentials from credentialManager +// 2. Update the credentials +// 3. Connects again to vCenter with fetched credentials +func (nm *NodeManager) vcConnect(ctx context.Context, vsphereInstance *VSphereInstance) error { + err := vsphereInstance.conn.Connect(ctx) + if err == nil { + return nil + } + + credentialManager := nm.CredentialManager() + if !vclib.IsInvalidCredentialsError(err) || credentialManager == nil { + glog.Errorf("Cannot connect to vCenter with err: %v", err) + return err + } + + glog.V(4).Infof("Invalid credentials. Cannot connect to server %q. "+ + "Fetching credentials from secrets.", vsphereInstance.conn.Hostname) + + // Get latest credentials from SecretCredentialManager + credentials, err := credentialManager.GetCredential(vsphereInstance.conn.Hostname) + if err != nil { + glog.Errorf("Failed to get credentials from Secret Credential Manager with err: %v", err) + return err + } + vsphereInstance.conn.UpdateCredentials(credentials.User, credentials.Password) + return vsphereInstance.conn.Connect(ctx) +} + +// GetNodeInfoWithNodeObject returns a NodeInfo which datacenter, vm and vc server ip address. +// This method returns an error if it is unable find node VCs and DCs listed in vSphere.conf +// NodeInfo returned may not be updated to reflect current VM location. +// +// This method is a getter but it can cause side-effect of updating NodeInfo object. +func (nm *NodeManager) GetNodeInfoWithNodeObject(node *v1.Node) (NodeInfo, error) { + nodeName := node.Name + getNodeInfo := func(nodeName string) *NodeInfo { + nm.nodeInfoLock.RLock() + nodeInfo := nm.nodeInfoMap[nodeName] + nm.nodeInfoLock.RUnlock() + return nodeInfo + } + nodeInfo := getNodeInfo(nodeName) + var err error + if nodeInfo == nil { + // Rediscover node if no NodeInfo found. + glog.V(4).Infof("No VM found for node %q. Initiating rediscovery.", nodeName) + err = nm.DiscoverNode(node) + if err != nil { + glog.Errorf("Error %q node info for node %q not found", err, nodeName) + return NodeInfo{}, err + } + nodeInfo = getNodeInfo(nodeName) + } else { + // Renew the found NodeInfo to avoid stale vSphere connection. + glog.V(4).Infof("Renewing NodeInfo %+v for node %q", nodeInfo, nodeName) + nodeInfo, err = nm.renewNodeInfo(nodeInfo, true) + if err != nil { + glog.Errorf("Error %q occurred while renewing NodeInfo for %q", err, nodeName) + return NodeInfo{}, err + } + nm.addNodeInfo(nodeName, nodeInfo) + } + return *nodeInfo, nil +} + +func (nm *NodeManager) CredentialManager() *SecretCredentialManager { + nm.credentialManagerLock.Lock() + defer nm.credentialManagerLock.Unlock() + return nm.credentialManager +} + +func (nm *NodeManager) UpdateCredentialManager(credentialManager *SecretCredentialManager) { + nm.credentialManagerLock.Lock() + defer nm.credentialManagerLock.Unlock() + nm.credentialManager = credentialManager +} diff --git a/pkg/cloudprovider/providers/vsphere/vclib/utils.go b/pkg/cloudprovider/providers/vsphere/vclib/utils.go index 0704948b802..ac67f5150d0 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/utils.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/utils.go @@ -172,6 +172,14 @@ func IsManagedObjectNotFoundError(err error) bool { return isManagedObjectNotFoundError } +func IsInvalidCredentialsError(err error) bool { + isInvalidCredentialsError := false + if soap.IsSoapFault(err) { + _, isInvalidCredentialsError = soap.ToSoapFault(err).VimFault().(types.InvalidLogin) + } + return isInvalidCredentialsError +} + // VerifyVolumePathsForVM verifies if the volume paths (volPaths) are attached to VM. func VerifyVolumePathsForVM(vmMo mo.VirtualMachine, volPaths []string, nodeName string, nodeVolumeMap map[string]map[string]bool) { // Verify if the volume paths are present on the VM backing virtual disk devices