Fix nodes power state on Azure

pkg/cloudprovider/providers/azure/azure_fakes.go

@@ -894,6 +894,6 @@ func (f *fakeVMSet) GetDataDisks(nodeName types.NodeName) ([]compute.DataDisk, e
 	return nil, fmt.Errorf("unimplemented")
 }
 
-func (f *fakeVMSet) GetProvisioningStateByNodeName(name string) (string, error) {
+func (f *fakeVMSet) GetPowerStatusByNodeName(name string) (string, error) {
 	return "", fmt.Errorf("unimplemented")
 }

pkg/cloudprovider/providers/azure/azure_instances.go

@@ -28,6 +28,12 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 )
 
+const (
+	vmPowerStatePrefix      = "PowerState/"
+	vmPowerStateStopped     = "stopped"
+	vmPowerStateDeallocated = "deallocated"
+)
+
 // NodeAddresses returns the addresses of the specified instance.
 func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) {
 	// Returns nil for unmanaged nodes because azure cloud provider couldn't fetch information for them.
@@ -148,12 +154,13 @@ func (az *Cloud) InstanceShutdownByProviderID(ctx context.Context, providerID st
 		return false, err
 	}
 
-	provisioningState, err := az.vmSet.GetProvisioningStateByNodeName(string(nodeName))
+	powerStatus, err := az.vmSet.GetPowerStatusByNodeName(string(nodeName))
 	if err != nil {
 		return false, err
 	}
+	glog.V(5).Infof("InstanceShutdownByProviderID gets power status %q for node %q", powerStatus, nodeName)
 
-	return strings.ToLower(provisioningState) == "stopped" || strings.ToLower(provisioningState) == "deallocated", nil
+	return strings.ToLower(powerStatus) == vmPowerStateStopped || strings.ToLower(powerStatus) == vmPowerStateDeallocated, nil
 }
 
 // getComputeMetadata gets compute information from instance metadata.

pkg/cloudprovider/providers/azure/azure_standard.go

@@ -346,13 +346,24 @@ func (as *availabilitySet) GetInstanceIDByNodeName(name string) (string, error)
 	return *machine.ID, nil
 }
 
-func (as *availabilitySet) GetProvisioningStateByNodeName(name string) (provisioningState string, err error) {
+// GetPowerStatusByNodeName returns the power state of the specified node.
+func (as *availabilitySet) GetPowerStatusByNodeName(name string) (powerState string, err error) {
 	vm, err := as.getVirtualMachine(types.NodeName(name))
 	if err != nil {
-		return provisioningState, err
+		return powerState, err
 	}
 
-	return *vm.ProvisioningState, nil
+	if vm.InstanceView != nil && vm.InstanceView.Statuses != nil {
+		statuses := *vm.InstanceView.Statuses
+		for _, status := range statuses {
+			state := to.String(status.Code)
+			if strings.HasPrefix(state, vmPowerStatePrefix) {
+				return strings.TrimPrefix(state, vmPowerStatePrefix), nil
+			}
+		}
+	}
+
+	return "", fmt.Errorf("failed to get power status for node %q", name)
 }
 
 // GetNodeNameByProviderID gets the node name by provider ID.

pkg/cloudprovider/providers/azure/azure_vmsets.go

@@ -65,6 +65,6 @@ type VMSet interface {
 	// GetDataDisks gets a list of data disks attached to the node.
 	GetDataDisks(nodeName types.NodeName) ([]compute.DataDisk, error)
 
-	// GetProvisioningStateByNodeName gets the provisioning state by node name.
+	// GetPowerStatusByNodeName returns the power state of the specified node.
-	GetProvisioningStateByNodeName(name string) (string, error)
+	GetPowerStatusByNodeName(name string) (string, error)
 }

pkg/cloudprovider/providers/azure/azure_vmss.go

@@ -128,13 +128,24 @@ func (ss *scaleSet) getVmssVM(nodeName string) (ssName, instanceID string, vm co
 	return ssName, instanceID, *(cachedVM.(*compute.VirtualMachineScaleSetVM)), nil
 }
 
-func (ss *scaleSet) GetProvisioningStateByNodeName(name string) (provisioningState string, err error) {
+// GetPowerStatusByNodeName returns the power state of the specified node.
+func (ss *scaleSet) GetPowerStatusByNodeName(name string) (powerState string, err error) {
 	_, _, vm, err := ss.getVmssVM(name)
 	if err != nil {
-		return provisioningState, err
+		return powerState, err
 	}
 
-	return *vm.ProvisioningState, nil
+	if vm.InstanceView != nil && vm.InstanceView.Statuses != nil {
+		statuses := *vm.InstanceView.Statuses
+		for _, status := range statuses {
+			state := to.String(status.Code)
+			if strings.HasPrefix(state, vmPowerStatePrefix) {
+				return strings.TrimPrefix(state, vmPowerStatePrefix), nil
+			}
+		}
+	}
+
+	return "", fmt.Errorf("failed to get power status for node %q", name)
 }
 
 // getCachedVirtualMachineByInstanceID gets scaleSetVMInfo from cache.

pkg/cloudprovider/providers/azure/azure_vmss_cache.go

@@ -199,6 +199,24 @@ func (ss *scaleSet) newVmssVMCache() (*timedCache, error) {
 			return nil, nil
 		}
 
+		// Get instanceView for vmssVM.
+		if result.InstanceView == nil {
+			viewCtx, viewCancel := getContextWithCancel()
+			defer viewCancel()
+			view, err := ss.VirtualMachineScaleSetVMsClient.GetInstanceView(viewCtx, resourceGroup, ssName, instanceID)
+			// It is possible that the vmssVM gets removed just before this call. So check whether the VM exist again.
+			exists, message, realErr = checkResourceExistsFromError(err)
+			if realErr != nil {
+				return nil, realErr
+			}
+			if !exists {
+				glog.V(2).Infof("Virtual machine scale set VM %q not found with message: %q", key, message)
+				return nil, nil
+			}
+
+			result.InstanceView = &view
+		}
+
 		return &result, nil
 	}