From 6a7bb312510f2e883c93b645d8ff835c15399d81 Mon Sep 17 00:00:00 2001 From: louisgong Date: Wed, 13 May 2020 11:24:31 +0800 Subject: [PATCH] implement azure InstanceMetadataByProviderID function --- .../azure/azure_instances.go | 250 ++++++++++++------ .../azure/azure_instances_test.go | 149 +++++++++++ 2 files changed, 312 insertions(+), 87 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances.go index 8f1b123e3b5..1fe7237c1b0 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances.go @@ -41,6 +41,26 @@ var ( errNodeNotInitialized = fmt.Errorf("providerID is empty, the node is not initialized yet") ) +func (az *Cloud) addressGetter(nodeName types.NodeName) ([]v1.NodeAddress, error) { + ip, publicIP, err := az.getIPForMachine(nodeName) + if err != nil { + klog.V(2).Infof("NodeAddresses(%s) abort backoff: %v", nodeName, err) + return nil, err + } + + addresses := []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: ip}, + {Type: v1.NodeHostName, Address: string(nodeName)}, + } + if len(publicIP) > 0 { + addresses = append(addresses, v1.NodeAddress{ + Type: v1.NodeExternalIP, + Address: publicIP, + }) + } + return addresses, nil +} + // NodeAddresses returns the addresses of the specified instance. func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) { // Returns nil for unmanaged nodes because azure cloud provider couldn't fetch information for them. @@ -53,26 +73,6 @@ func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.N return nil, nil } - addressGetter := func(nodeName types.NodeName) ([]v1.NodeAddress, error) { - ip, publicIP, err := az.getIPForMachine(nodeName) - if err != nil { - klog.V(2).Infof("NodeAddresses(%s) abort backoff: %v", nodeName, err) - return nil, err - } - - addresses := []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: ip}, - {Type: v1.NodeHostName, Address: string(name)}, - } - if len(publicIP) > 0 { - addresses = append(addresses, v1.NodeAddress{ - Type: v1.NodeExternalIP, - Address: publicIP, - }) - } - return addresses, nil - } - if az.UseInstanceMetadata { metadata, err := az.metadata.GetMetadata(azcache.CacheReadTypeDefault) if err != nil { @@ -91,59 +91,62 @@ func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.N // Not local instance, get addresses from Azure ARM API. if !isLocalInstance { if az.vmSet != nil { - return addressGetter(name) + return az.addressGetter(name) } // vmSet == nil indicates credentials are not provided. return nil, fmt.Errorf("no credentials provided for Azure cloud provider") } - if len(metadata.Network.Interface) == 0 { - return nil, fmt.Errorf("no interface is found for the instance") - } - - // Use ip address got from instance metadata. - ipAddress := metadata.Network.Interface[0] - addresses := []v1.NodeAddress{ - {Type: v1.NodeHostName, Address: string(name)}, - } - if len(ipAddress.IPV4.IPAddress) > 0 && len(ipAddress.IPV4.IPAddress[0].PrivateIP) > 0 { - address := ipAddress.IPV4.IPAddress[0] - addresses = append(addresses, v1.NodeAddress{ - Type: v1.NodeInternalIP, - Address: address.PrivateIP, - }) - if len(address.PublicIP) > 0 { - addresses = append(addresses, v1.NodeAddress{ - Type: v1.NodeExternalIP, - Address: address.PublicIP, - }) - } - } - if len(ipAddress.IPV6.IPAddress) > 0 && len(ipAddress.IPV6.IPAddress[0].PrivateIP) > 0 { - address := ipAddress.IPV6.IPAddress[0] - addresses = append(addresses, v1.NodeAddress{ - Type: v1.NodeInternalIP, - Address: address.PrivateIP, - }) - if len(address.PublicIP) > 0 { - addresses = append(addresses, v1.NodeAddress{ - Type: v1.NodeExternalIP, - Address: address.PublicIP, - }) - } - } - - if len(addresses) == 1 { - // No IP addresses is got from instance metadata service, clean up cache and report errors. - az.metadata.imsCache.Delete(metadataCacheKey) - return nil, fmt.Errorf("get empty IP addresses from instance metadata service") - } - - return addresses, nil + return az.getLocalInstanceNodeAddresses(metadata.Network.Interface, string(name)) } - return addressGetter(name) + return az.addressGetter(name) +} + +func (az *Cloud) getLocalInstanceNodeAddresses(netInterfaces []NetworkInterface, nodeName string) ([]v1.NodeAddress, error) { + if len(netInterfaces) == 0 { + return nil, fmt.Errorf("no interface is found for the instance") + } + + // Use ip address got from instance metadata. + netInterface := netInterfaces[0] + addresses := []v1.NodeAddress{ + {Type: v1.NodeHostName, Address: nodeName}, + } + if len(netInterface.IPV4.IPAddress) > 0 && len(netInterface.IPV4.IPAddress[0].PrivateIP) > 0 { + address := netInterface.IPV4.IPAddress[0] + addresses = append(addresses, v1.NodeAddress{ + Type: v1.NodeInternalIP, + Address: address.PrivateIP, + }) + if len(address.PublicIP) > 0 { + addresses = append(addresses, v1.NodeAddress{ + Type: v1.NodeExternalIP, + Address: address.PublicIP, + }) + } + } + if len(netInterface.IPV6.IPAddress) > 0 && len(netInterface.IPV6.IPAddress[0].PrivateIP) > 0 { + address := netInterface.IPV6.IPAddress[0] + addresses = append(addresses, v1.NodeAddress{ + Type: v1.NodeInternalIP, + Address: address.PrivateIP, + }) + if len(address.PublicIP) > 0 { + addresses = append(addresses, v1.NodeAddress{ + Type: v1.NodeExternalIP, + Address: address.PublicIP, + }) + } + } + + if len(addresses) == 1 { + // No IP addresses is got from instance metadata service, clean up cache and report errors. + az.metadata.imsCache.Delete(metadataCacheKey) + return nil, fmt.Errorf("get empty IP addresses from instance metadata service") + } + return addresses, nil } // NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID @@ -232,7 +235,77 @@ func (az *Cloud) InstanceShutdownByProviderID(ctx context.Context, providerID st // InstanceMetadataByProviderID returns metadata of the specified instance. func (az *Cloud) InstanceMetadataByProviderID(ctx context.Context, providerID string) (*cloudprovider.InstanceMetadata, error) { - return nil, fmt.Errorf("unimplemented") + if providerID == "" { + return nil, errNodeNotInitialized + } + + nodeName, err := az.vmSet.GetNodeNameByProviderID(providerID) + if err != nil { + return nil, err + } + + // Returns "" for unmanaged nodes because azure cloud provider couldn't fetch information for them. + unmanaged, err := az.IsNodeUnmanaged(string(nodeName)) + if err != nil { + return nil, err + } + if unmanaged { + klog.V(4).Infof("InstanceType: omitting unmanaged node %q", string(nodeName)) + return nil, nil + } + + md := &cloudprovider.InstanceMetadata{} + md.ProviderID = providerID + if az.UseInstanceMetadata { + metadata, err := az.metadata.GetMetadata(azcache.CacheReadTypeUnsafe) + if err != nil { + return nil, err + } + + if metadata.Compute == nil || metadata.Network == nil { + return nil, fmt.Errorf("failure of getting instance metadata") + } + + isLocalInstance, err := az.isCurrentInstance(nodeName, metadata.Compute.Name) + if err != nil { + return nil, err + } + + // Not local instance, get metadata from Azure ARM API. + if !isLocalInstance { + if az.vmSet != nil { + if md.Type, err = az.vmSet.GetInstanceTypeByNodeName(string(nodeName)); err != nil { + return nil, err + } + if md.NodeAddresses, err = az.addressGetter(nodeName); err != nil { + return nil, err + } + return md, nil + } + // vmSet == nil indicates credentials are not provided. + return nil, fmt.Errorf("no credentials provided for Azure cloud provider") + } + + if metadata.Compute.VMSize != "" { + md.Type = metadata.Compute.VMSize + } else { + if md.Type, err = az.vmSet.GetInstanceTypeByNodeName(string(nodeName)); err != nil { + return nil, err + } + } + if md.NodeAddresses, err = az.getLocalInstanceNodeAddresses(metadata.Network.Interface, string(nodeName)); err != nil { + return nil, err + } + return md, nil + } + + if md.Type, err = az.vmSet.GetInstanceTypeByNodeName(string(nodeName)); err != nil { + return nil, err + } + if md.NodeAddresses, err = az.addressGetter(nodeName); err != nil { + return nil, err + } + return md, err } func (az *Cloud) isCurrentInstance(name types.NodeName, metadataVMName string) (bool, error) { @@ -292,32 +365,35 @@ func (az *Cloud) InstanceID(ctx context.Context, name types.NodeName) (string, e // vmSet == nil indicates credentials are not provided. return "", fmt.Errorf("no credentials provided for Azure cloud provider") } - - // Get resource group name and subscription ID. - resourceGroup := strings.ToLower(metadata.Compute.ResourceGroup) - subscriptionID := strings.ToLower(metadata.Compute.SubscriptionID) - - // Compose instanceID based on nodeName for standard instance. - if metadata.Compute.VMScaleSetName == "" { - return az.getStandardMachineID(subscriptionID, resourceGroup, nodeName), nil - } - - // Get scale set name and instanceID from vmName for vmss. - ssName, instanceID, err := extractVmssVMName(metadata.Compute.Name) - if err != nil { - if err == ErrorNotVmssInstance { - // Compose machineID for standard Node. - return az.getStandardMachineID(subscriptionID, resourceGroup, nodeName), nil - } - return "", err - } - // Compose instanceID based on ssName and instanceID for vmss instance. - return az.getVmssMachineID(subscriptionID, resourceGroup, ssName, instanceID), nil + return az.getLocalInstanceProviderID(metadata, nodeName) } return az.vmSet.GetInstanceIDByNodeName(nodeName) } +func (az *Cloud) getLocalInstanceProviderID(metadata *InstanceMetadata, nodeName string) (string, error) { + // Get resource group name and subscription ID. + resourceGroup := strings.ToLower(metadata.Compute.ResourceGroup) + subscriptionID := strings.ToLower(metadata.Compute.SubscriptionID) + + // Compose instanceID based on nodeName for standard instance. + if metadata.Compute.VMScaleSetName == "" { + return az.getStandardMachineID(subscriptionID, resourceGroup, nodeName), nil + } + + // Get scale set name and instanceID from vmName for vmss. + ssName, instanceID, err := extractVmssVMName(metadata.Compute.Name) + if err != nil { + if err == ErrorNotVmssInstance { + // Compose machineID for standard Node. + return az.getStandardMachineID(subscriptionID, resourceGroup, nodeName), nil + } + return "", err + } + // Compose instanceID based on ssName and instanceID for vmss instance. + return az.getVmssMachineID(subscriptionID, resourceGroup, ssName, instanceID), nil +} + // InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID // This method will not be called from the node that is requesting this ID. i.e. metadata service // and other local methods cannot be used here diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances_test.go index 1f989c2ab94..ed364c5f835 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances_test.go @@ -57,6 +57,9 @@ func setTestVirtualMachines(c *Cloud, vmList map[string]string, isDataDisksFull }, } vm.VirtualMachineProperties = &compute.VirtualMachineProperties{ + HardwareProfile: &compute.HardwareProfile{ + VMSize: compute.VirtualMachineSizeTypesStandardA0, + }, InstanceView: &compute.VirtualMachineInstanceView{ Statuses: &status, }, @@ -395,6 +398,152 @@ func TestNodeAddresses(t *testing.T) { } } +func TestInstanceMetadataByProviderID(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + cloud := GetTestCloud(ctrl) + cloud.Config.UseInstanceMetadata = true + metadataTemplate := `{"compute":{"name":"%s","subscriptionId":"subscription","resourceGroupName":"rg"},"network":{"interface":[{"ipv4":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]},"ipv6":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]}}]}}` + + testcases := []struct { + name string + vmList []string + nodeName string + ipV4 string + ipV6 string + ipV4Public string + ipV6Public string + providerID string + expectedAddr []v1.NodeAddress + expectError bool + }{ + { + name: "NodeAddresses should get both ipV4 and ipV6 private addresses, InstanceID should get instanceID if node's name are equal to metadataName", + vmList: []string{"vm1"}, + nodeName: "vm1", + ipV4: "10.240.0.1", + ipV6: "1111:11111:00:00:1111:1111:000:111", + providerID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1", + expectedAddr: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: "vm1", + }, + { + Type: v1.NodeInternalIP, + Address: "10.240.0.1", + }, + { + Type: v1.NodeInternalIP, + Address: "1111:11111:00:00:1111:1111:000:111", + }, + }, + }, + { + name: "NodeAddresses should report error when IPs are empty", + nodeName: "vm1", + expectError: true, + }, + { + name: "NodeAddresses should get ipV4 private and public addresses, InstanceID should get instanceID from Azure API if node is not local instance", + vmList: []string{"vm2"}, + nodeName: "vm2", + providerID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm2", + ipV4: "10.240.0.1", + ipV4Public: "9.9.9.9", + expectedAddr: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: "vm2", + }, + { + Type: v1.NodeInternalIP, + Address: "10.240.0.1", + }, + { + Type: v1.NodeExternalIP, + Address: "9.9.9.9", + }, + }, + }, + { + name: "InstanceID should report error if VM doesn't exist", + vmList: []string{"vm1"}, + nodeName: "vm3", + expectError: true, + }, + { + name: "NodeAddresses should get ipV6 private and public addresses", + nodeName: "vm1", + providerID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1", + ipV6: "1111:11111:00:00:1111:1111:000:111", + ipV6Public: "2222:22221:00:00:2222:2222:000:111", + expectedAddr: []v1.NodeAddress{ + { + Type: v1.NodeHostName, + Address: "vm1", + }, + { + Type: v1.NodeInternalIP, + Address: "1111:11111:00:00:1111:1111:000:111", + }, + { + Type: v1.NodeExternalIP, + Address: "2222:22221:00:00:2222:2222:000:111", + }, + }, + }, + } + + for _, test := range testcases { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Errorf("Test [%s] unexpected error: %v", test.name, err) + } + + mux := http.NewServeMux() + mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, fmt.Sprintf(metadataTemplate, test.nodeName, test.ipV4, test.ipV4Public, test.ipV6, test.ipV6Public)) + })) + go func() { + http.Serve(listener, mux) + }() + defer listener.Close() + + cloud.metadata, err = NewInstanceMetadataService("http://" + listener.Addr().String() + "/") + if err != nil { + t.Errorf("Test [%s] unexpected error: %v", test.name, err) + } + + vmListWithPowerState := make(map[string]string) + for _, vm := range test.vmList { + vmListWithPowerState[vm] = "" + } + expectedVMs := setTestVirtualMachines(cloud, vmListWithPowerState, false) + mockVMsClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) + for _, vm := range expectedVMs { + mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, *vm.Name, gomock.Any()).Return(vm, nil).AnyTimes() + } + mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "vm3", gomock.Any()).Return(compute.VirtualMachine{}, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes() + mockVMsClient.EXPECT().Update(gomock.Any(), cloud.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + md, err := cloud.InstanceMetadataByProviderID(context.Background(), test.providerID) + if test.expectError { + if err == nil { + t.Errorf("Test [%s] unexpected nil err", test.name) + } + } else { + if err != nil { + t.Errorf("Test [%s] unexpected error: %v", test.name, err) + } + } + + if len(test.expectedAddr) > 0 && !reflect.DeepEqual(md.NodeAddresses, test.expectedAddr) { + t.Errorf("Test [%s] unexpected ipAddresses: %s, expected %q", test.name, md.NodeAddresses, test.expectedAddr) + } + } +} + func TestIsCurrentInstance(t *testing.T) { cloud := &Cloud{ Config: Config{