diff --git a/pkg/cloudprovider/providers/azure/azure_instances.go b/pkg/cloudprovider/providers/azure/azure_instances.go index e2d5b3cb754..7eb403a7b29 100644 --- a/pkg/cloudprovider/providers/azure/azure_instances.go +++ b/pkg/cloudprovider/providers/azure/azure_instances.go @@ -30,6 +30,16 @@ import ( // NodeAddresses returns the addresses of the specified instance. func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) { + // Returns nil for unmanaged nodes because azure cloud provider couldn't fetch information for them. + unmanaged, err := az.IsNodeUnmanaged(string(name)) + if err != nil { + return nil, err + } + if unmanaged { + glog.V(4).Infof("NodeAddresses: omitting unmanaged node %q", name) + return nil, nil + } + addressGetter := func(nodeName types.NodeName) ([]v1.NodeAddress, error) { ip, publicIP, err := az.GetIPForMachineWithRetry(nodeName) if err != nil { @@ -92,6 +102,12 @@ func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.N // This method will not be called from the node that is requesting this ID. i.e. metadata service // and other local methods cannot be used here func (az *Cloud) NodeAddressesByProviderID(ctx context.Context, providerID string) ([]v1.NodeAddress, error) { + // Returns nil for unmanaged nodes because azure cloud provider couldn't fetch information for them. + if az.IsNodeUnmanagedByProviderID(providerID) { + glog.V(4).Infof("NodeAddressesByProviderID: omitting unmanaged node %q", providerID) + return nil, nil + } + name, err := az.vmSet.GetNodeNameByProviderID(providerID) if err != nil { return nil, err @@ -103,6 +119,12 @@ func (az *Cloud) NodeAddressesByProviderID(ctx context.Context, providerID strin // InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running. // If false is returned with no error, the instance will be immediately deleted by the cloud controller manager. func (az *Cloud) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) { + // Returns true for unmanaged nodes because azure cloud provider always assumes them exists. + if az.IsNodeUnmanagedByProviderID(providerID) { + glog.V(4).Infof("InstanceExistsByProviderID: assuming unmanaged node %q exists", providerID) + return true, nil + } + name, err := az.vmSet.GetNodeNameByProviderID(providerID) if err != nil { return false, err @@ -154,6 +176,15 @@ func (az *Cloud) isCurrentInstance(name types.NodeName, metadataVMName string) ( // Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound) func (az *Cloud) InstanceID(ctx context.Context, name types.NodeName) (string, error) { nodeName := mapNodeNameToVMName(name) + unmanaged, err := az.IsNodeUnmanaged(nodeName) + if err != nil { + return "", err + } + if unmanaged { + // InstanceID is same with nodeName for unmanaged nodes. + glog.V(4).Infof("InstanceID: getting ID %q for unmanaged node %q", name, name) + return nodeName, nil + } if az.UseInstanceMetadata { computeMetadata, err := az.getComputeMetadata() @@ -202,6 +233,12 @@ func (az *Cloud) InstanceID(ctx context.Context, name types.NodeName) (string, e // This method will not be called from the node that is requesting this ID. i.e. metadata service // and other local methods cannot be used here func (az *Cloud) InstanceTypeByProviderID(ctx context.Context, providerID string) (string, error) { + // Returns "" for unmanaged nodes because azure cloud provider couldn't fetch information for them. + if az.IsNodeUnmanagedByProviderID(providerID) { + glog.V(4).Infof("InstanceTypeByProviderID: omitting unmanaged node %q", providerID) + return "", nil + } + name, err := az.vmSet.GetNodeNameByProviderID(providerID) if err != nil { return "", err @@ -215,6 +252,16 @@ func (az *Cloud) InstanceTypeByProviderID(ctx context.Context, providerID string // (Implementer Note): This is used by kubelet. Kubelet will label the node. Real log from kubelet: // Adding node label from cloud provider: beta.kubernetes.io/instance-type=[value] func (az *Cloud) InstanceType(ctx context.Context, name types.NodeName) (string, error) { + // Returns "" for unmanaged nodes because azure cloud provider couldn't fetch information for them. + unmanaged, err := az.IsNodeUnmanaged(string(name)) + if err != nil { + return "", err + } + if unmanaged { + glog.V(4).Infof("InstanceType: omitting unmanaged node %q", name) + return "", nil + } + if az.UseInstanceMetadata { computeMetadata, err := az.getComputeMetadata() if err != nil { diff --git a/pkg/cloudprovider/providers/azure/azure_routes.go b/pkg/cloudprovider/providers/azure/azure_routes.go index 7f627a98a1d..20a49064cb5 100644 --- a/pkg/cloudprovider/providers/azure/azure_routes.go +++ b/pkg/cloudprovider/providers/azure/azure_routes.go @@ -107,6 +107,16 @@ func (az *Cloud) createRouteTable() error { // route.Name will be ignored, although the cloud-provider may use nameHint // to create a more user-meaningful name. func (az *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint string, kubeRoute *cloudprovider.Route) error { + // Returns for unmanaged nodes because azure cloud provider couldn't fetch information for them. + unmanaged, err := az.IsNodeUnmanaged(string(kubeRoute.TargetNode)) + if err != nil { + return err + } + if unmanaged { + glog.V(2).Infof("CreateRoute: omitting unmanaged node %q", kubeRoute.TargetNode) + return nil + } + glog.V(2).Infof("CreateRoute: creating route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) if err := az.createRouteTableIfNotExists(clusterName, kubeRoute); err != nil { return err @@ -150,6 +160,16 @@ func (az *Cloud) CreateRoute(ctx context.Context, clusterName string, nameHint s // DeleteRoute deletes the specified managed route // Route should be as returned by ListRoutes func (az *Cloud) DeleteRoute(ctx context.Context, clusterName string, kubeRoute *cloudprovider.Route) error { + // Returns for unmanaged nodes because azure cloud provider couldn't fetch information for them. + unmanaged, err := az.IsNodeUnmanaged(string(kubeRoute.TargetNode)) + if err != nil { + return err + } + if unmanaged { + glog.V(2).Infof("DeleteRoute: omitting unmanaged node %q", kubeRoute.TargetNode) + return nil + } + glog.V(2).Infof("DeleteRoute: deleting route. clusterName=%q instance=%q cidr=%q", clusterName, kubeRoute.TargetNode, kubeRoute.DestinationCIDR) ctx, cancel := getContextWithCancel() diff --git a/pkg/cloudprovider/providers/azure/azure_standard.go b/pkg/cloudprovider/providers/azure/azure_standard.go index 8fec048c256..f5b39274c22 100644 --- a/pkg/cloudprovider/providers/azure/azure_standard.go +++ b/pkg/cloudprovider/providers/azure/azure_standard.go @@ -625,7 +625,7 @@ func (as *availabilitySet) ensureHostInPool(serviceName string, nodeName types.N } if nic.ProvisioningState != nil && *nic.ProvisioningState == nicFailedState { - glog.V(3).Infof("ensureHostInPool skips node %s because its primdary nic %s is in Failed state", nodeName, nic.Name) + glog.V(3).Infof("ensureHostInPool skips node %s because its primary nic %s is in Failed state", nodeName, *nic.Name) return nil } diff --git a/pkg/cloudprovider/providers/azure/azure_wrap.go b/pkg/cloudprovider/providers/azure/azure_wrap.go index f3d2e642805..86bb87325cc 100644 --- a/pkg/cloudprovider/providers/azure/azure_wrap.go +++ b/pkg/cloudprovider/providers/azure/azure_wrap.go @@ -19,6 +19,7 @@ package azure import ( "fmt" "net/http" + "regexp" "strings" "time" @@ -36,6 +37,8 @@ var ( lbCacheTTL = 2 * time.Minute nsgCacheTTL = 2 * time.Minute rtCacheTTL = 2 * time.Minute + + azureNodeProviderIDRE = regexp.MustCompile(`^azure:///subscriptions/(?:.*)/resourceGroups/(?:.*)/providers/Microsoft.Compute/(?:.*)`) ) // checkExistsFromError inspects an error and returns a true if err is nil, @@ -283,3 +286,21 @@ func (az *Cloud) useStandardLoadBalancer() bool { func (az *Cloud) excludeMasterNodesFromStandardLB() bool { return az.ExcludeMasterFromStandardLB != nil && *az.ExcludeMasterFromStandardLB } + +// IsNodeUnmanaged returns true if the node is not managed by Azure cloud provider. +// Those nodes includes on-prem or VMs from other clouds. They will not be added to load balancer +// backends. Azure routes and managed disks are also not supported for them. +func (az *Cloud) IsNodeUnmanaged(nodeName string) (bool, error) { + unmanagedNodes, err := az.GetUnmanagedNodes() + if err != nil { + return false, err + } + + return unmanagedNodes.Has(nodeName), nil +} + +// IsNodeUnmanagedByProviderID returns true if the node is not managed by Azure cloud provider. +// All managed node's providerIDs are in format 'azure:///subscriptions//resourceGroups//providers/Microsoft.Compute/.*' +func (az *Cloud) IsNodeUnmanagedByProviderID(providerID string) bool { + return azureNodeProviderIDRE.Match([]byte(providerID)) +} diff --git a/pkg/cloudprovider/providers/azure/azure_zones.go b/pkg/cloudprovider/providers/azure/azure_zones.go index 0d4195acc68..08aab6d07e4 100644 --- a/pkg/cloudprovider/providers/azure/azure_zones.go +++ b/pkg/cloudprovider/providers/azure/azure_zones.go @@ -103,6 +103,12 @@ func (az *Cloud) getZoneFromFaultDomain() (cloudprovider.Zone, error) { // This is particularly useful in external cloud providers where the kubelet // does not initialize node data. func (az *Cloud) GetZoneByProviderID(ctx context.Context, providerID string) (cloudprovider.Zone, error) { + // Returns nil for unmanaged nodes because azure cloud provider couldn't fetch information for them. + if az.IsNodeUnmanagedByProviderID(providerID) { + glog.V(2).Infof("GetZoneByProviderID: omitting unmanaged node %q", providerID) + return cloudprovider.Zone{}, nil + } + nodeName, err := az.vmSet.GetNodeNameByProviderID(providerID) if err != nil { return cloudprovider.Zone{}, err @@ -115,6 +121,16 @@ func (az *Cloud) GetZoneByProviderID(ctx context.Context, providerID string) (cl // This is particularly useful in external cloud providers where the kubelet // does not initialize node data. func (az *Cloud) GetZoneByNodeName(ctx context.Context, nodeName types.NodeName) (cloudprovider.Zone, error) { + // Returns "" for unmanaged nodes because azure cloud provider couldn't fetch information for them. + unmanaged, err := az.IsNodeUnmanaged(string(nodeName)) + if err != nil { + return cloudprovider.Zone{}, err + } + if unmanaged { + glog.V(2).Infof("GetZoneByNodeName: omitting unmanaged node %q", nodeName) + return cloudprovider.Zone{}, nil + } + return az.vmSet.GetZoneByNodeName(string(nodeName)) }