github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-24 04:06:03 +00:00
Code Issues Projects Releases Wiki Activity

implement azure InstanceMetadataByProviderID function

Browse Source
This commit is contained in:
louisgong 2020-05-13 11:24:31 +08:00
parent da4b5e8dd9
commit 6a7bb31251
2 changed files with 312 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

250
staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances.go
View File

@ -41,6 +41,26 @@ var (
errNodeNotInitialized = fmt.Errorf("providerID is empty, the node is not initialized yet")
)
func (az *Cloud) addressGetter(nodeName types.NodeName) ([]v1.NodeAddress, error) {
ip, publicIP, err := az.getIPForMachine(nodeName)
if err != nil {
klog.V(2).Infof("NodeAddresses(%s) abort backoff: %v", nodeName, err)
return nil, err
}
addresses := []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: ip},
{Type: v1.NodeHostName, Address: string(nodeName)},
}
if len(publicIP) > 0 {
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: publicIP,
})
}
return addresses, nil
}
// NodeAddresses returns the addresses of the specified instance.
func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) {
// Returns nil for unmanaged nodes because azure cloud provider couldn't fetch information for them.
@ -53,26 +73,6 @@ func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.N
return nil, nil
}
addressGetter := func(nodeName types.NodeName) ([]v1.NodeAddress, error) {
ip, publicIP, err := az.getIPForMachine(nodeName)
if err != nil {
klog.V(2).Infof("NodeAddresses(%s) abort backoff: %v", nodeName, err)
return nil, err
}
addresses := []v1.NodeAddress{
{Type: v1.NodeInternalIP, Address: ip},
{Type: v1.NodeHostName, Address: string(name)},
}
if len(publicIP) > 0 {
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: publicIP,
})
}
return addresses, nil
}
if az.UseInstanceMetadata {
metadata, err := az.metadata.GetMetadata(azcache.CacheReadTypeDefault)
if err != nil {
@ -91,59 +91,62 @@ func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.N
// Not local instance, get addresses from Azure ARM API.
if !isLocalInstance {
if az.vmSet != nil {
return addressGetter(name)
return az.addressGetter(name)
}
// vmSet == nil indicates credentials are not provided.
return nil, fmt.Errorf("no credentials provided for Azure cloud provider")
}
if len(metadata.Network.Interface) == 0 {
return nil, fmt.Errorf("no interface is found for the instance")
}
// Use ip address got from instance metadata.
ipAddress := metadata.Network.Interface[0]
addresses := []v1.NodeAddress{
{Type: v1.NodeHostName, Address: string(name)},
}
if len(ipAddress.IPV4.IPAddress) > 0 && len(ipAddress.IPV4.IPAddress[0].PrivateIP) > 0 {
address := ipAddress.IPV4.IPAddress[0]
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeInternalIP,
Address: address.PrivateIP,
})
if len(address.PublicIP) > 0 {
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: address.PublicIP,
})
}
}
if len(ipAddress.IPV6.IPAddress) > 0 && len(ipAddress.IPV6.IPAddress[0].PrivateIP) > 0 {
address := ipAddress.IPV6.IPAddress[0]
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeInternalIP,
Address: address.PrivateIP,
})
if len(address.PublicIP) > 0 {
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: address.PublicIP,
})
}
}
if len(addresses) == 1 {
// No IP addresses is got from instance metadata service, clean up cache and report errors.
az.metadata.imsCache.Delete(metadataCacheKey)
return nil, fmt.Errorf("get empty IP addresses from instance metadata service")
}
return addresses, nil
return az.getLocalInstanceNodeAddresses(metadata.Network.Interface, string(name))
}
return addressGetter(name)
return az.addressGetter(name)
}
func (az *Cloud) getLocalInstanceNodeAddresses(netInterfaces []NetworkInterface, nodeName string) ([]v1.NodeAddress, error) {
if len(netInterfaces) == 0 {
return nil, fmt.Errorf("no interface is found for the instance")
}
// Use ip address got from instance metadata.
netInterface := netInterfaces[0]
addresses := []v1.NodeAddress{
{Type: v1.NodeHostName, Address: nodeName},
}
if len(netInterface.IPV4.IPAddress) > 0 && len(netInterface.IPV4.IPAddress[0].PrivateIP) > 0 {
address := netInterface.IPV4.IPAddress[0]
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeInternalIP,
Address: address.PrivateIP,
})
if len(address.PublicIP) > 0 {
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: address.PublicIP,
})
}
}
if len(netInterface.IPV6.IPAddress) > 0 && len(netInterface.IPV6.IPAddress[0].PrivateIP) > 0 {
address := netInterface.IPV6.IPAddress[0]
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeInternalIP,
Address: address.PrivateIP,
})
if len(address.PublicIP) > 0 {
addresses = append(addresses, v1.NodeAddress{
Type: v1.NodeExternalIP,
Address: address.PublicIP,
})
}
}
if len(addresses) == 1 {
// No IP addresses is got from instance metadata service, clean up cache and report errors.
az.metadata.imsCache.Delete(metadataCacheKey)
return nil, fmt.Errorf("get empty IP addresses from instance metadata service")
}
return addresses, nil
}
// NodeAddressesByProviderID returns the node addresses of an instances with the specified unique providerID
@ -232,7 +235,77 @@ func (az *Cloud) InstanceShutdownByProviderID(ctx context.Context, providerID st
// InstanceMetadataByProviderID returns metadata of the specified instance.
func (az *Cloud) InstanceMetadataByProviderID(ctx context.Context, providerID string) (*cloudprovider.InstanceMetadata, error) {
return nil, fmt.Errorf("unimplemented")
if providerID == "" {
return nil, errNodeNotInitialized
}
nodeName, err := az.vmSet.GetNodeNameByProviderID(providerID)
if err != nil {
return nil, err
}
// Returns "" for unmanaged nodes because azure cloud provider couldn't fetch information for them.
unmanaged, err := az.IsNodeUnmanaged(string(nodeName))
if err != nil {
return nil, err
}
if unmanaged {
klog.V(4).Infof("InstanceType: omitting unmanaged node %q", string(nodeName))
return nil, nil
}
md := &cloudprovider.InstanceMetadata{}
md.ProviderID = providerID
if az.UseInstanceMetadata {
metadata, err := az.metadata.GetMetadata(azcache.CacheReadTypeUnsafe)
if err != nil {
return nil, err
}
if metadata.Compute == nil || metadata.Network == nil {
return nil, fmt.Errorf("failure of getting instance metadata")
}
isLocalInstance, err := az.isCurrentInstance(nodeName, metadata.Compute.Name)
if err != nil {
return nil, err
}
// Not local instance, get metadata from Azure ARM API.
if !isLocalInstance {
if az.vmSet != nil {
if md.Type, err = az.vmSet.GetInstanceTypeByNodeName(string(nodeName)); err != nil {
return nil, err
}
if md.NodeAddresses, err = az.addressGetter(nodeName); err != nil {
return nil, err
}
return md, nil
}
// vmSet == nil indicates credentials are not provided.
return nil, fmt.Errorf("no credentials provided for Azure cloud provider")
}
if metadata.Compute.VMSize != "" {
md.Type = metadata.Compute.VMSize
} else {
if md.Type, err = az.vmSet.GetInstanceTypeByNodeName(string(nodeName)); err != nil {
return nil, err
}
}
if md.NodeAddresses, err = az.getLocalInstanceNodeAddresses(metadata.Network.Interface, string(nodeName)); err != nil {
return nil, err
}
return md, nil
}
if md.Type, err = az.vmSet.GetInstanceTypeByNodeName(string(nodeName)); err != nil {
return nil, err
}
if md.NodeAddresses, err = az.addressGetter(nodeName); err != nil {
return nil, err
}
return md, err
}
func (az *Cloud) isCurrentInstance(name types.NodeName, metadataVMName string) (bool, error) {
@ -292,32 +365,35 @@ func (az *Cloud) InstanceID(ctx context.Context, name types.NodeName) (string, e
// vmSet == nil indicates credentials are not provided.
return "", fmt.Errorf("no credentials provided for Azure cloud provider")
}
// Get resource group name and subscription ID.
resourceGroup := strings.ToLower(metadata.Compute.ResourceGroup)
subscriptionID := strings.ToLower(metadata.Compute.SubscriptionID)
// Compose instanceID based on nodeName for standard instance.
if metadata.Compute.VMScaleSetName == "" {
return az.getStandardMachineID(subscriptionID, resourceGroup, nodeName), nil
}
// Get scale set name and instanceID from vmName for vmss.
ssName, instanceID, err := extractVmssVMName(metadata.Compute.Name)
if err != nil {
if err == ErrorNotVmssInstance {
// Compose machineID for standard Node.
return az.getStandardMachineID(subscriptionID, resourceGroup, nodeName), nil
}
return "", err
}
// Compose instanceID based on ssName and instanceID for vmss instance.
return az.getVmssMachineID(subscriptionID, resourceGroup, ssName, instanceID), nil
return az.getLocalInstanceProviderID(metadata, nodeName)
}
return az.vmSet.GetInstanceIDByNodeName(nodeName)
}
func (az *Cloud) getLocalInstanceProviderID(metadata *InstanceMetadata, nodeName string) (string, error) {
// Get resource group name and subscription ID.
resourceGroup := strings.ToLower(metadata.Compute.ResourceGroup)
subscriptionID := strings.ToLower(metadata.Compute.SubscriptionID)
// Compose instanceID based on nodeName for standard instance.
if metadata.Compute.VMScaleSetName == "" {
return az.getStandardMachineID(subscriptionID, resourceGroup, nodeName), nil
}
// Get scale set name and instanceID from vmName for vmss.
ssName, instanceID, err := extractVmssVMName(metadata.Compute.Name)
if err != nil {
if err == ErrorNotVmssInstance {
// Compose machineID for standard Node.
return az.getStandardMachineID(subscriptionID, resourceGroup, nodeName), nil
}
return "", err
}
// Compose instanceID based on ssName and instanceID for vmss instance.
return az.getVmssMachineID(subscriptionID, resourceGroup, ssName, instanceID), nil
}
// InstanceTypeByProviderID returns the cloudprovider instance type of the node with the specified unique providerID
// This method will not be called from the node that is requesting this ID. i.e. metadata service
// and other local methods cannot be used here

149
staging/src/k8s.io/legacy-cloud-providers/azure/azure_instances_test.go
View File

@ -57,6 +57,9 @@ func setTestVirtualMachines(c *Cloud, vmList map[string]string, isDataDisksFull
},
}
vm.VirtualMachineProperties = &compute.VirtualMachineProperties{
HardwareProfile: &compute.HardwareProfile{
VMSize: compute.VirtualMachineSizeTypesStandardA0,
},
InstanceView: &compute.VirtualMachineInstanceView{
Statuses: &status,
},
@ -395,6 +398,152 @@ func TestNodeAddresses(t *testing.T) {
}
}
func TestInstanceMetadataByProviderID(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
cloud := GetTestCloud(ctrl)
cloud.Config.UseInstanceMetadata = true
metadataTemplate := `{"compute":{"name":"%s","subscriptionId":"subscription","resourceGroupName":"rg"},"network":{"interface":[{"ipv4":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]},"ipv6":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]}}]}}`
testcases := []struct {
name string
vmList []string
nodeName string
ipV4 string
ipV6 string
ipV4Public string
ipV6Public string
providerID string
expectedAddr []v1.NodeAddress
expectError bool
}{
{
name: "NodeAddresses should get both ipV4 and ipV6 private addresses, InstanceID should get instanceID if node's name are equal to metadataName",
vmList: []string{"vm1"},
nodeName: "vm1",
ipV4: "10.240.0.1",
ipV6: "1111:11111:00:00:1111:1111:000:111",
providerID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1",
expectedAddr: []v1.NodeAddress{
{
Type: v1.NodeHostName,
Address: "vm1",
},
{
Type: v1.NodeInternalIP,
Address: "10.240.0.1",
},
{
Type: v1.NodeInternalIP,
Address: "1111:11111:00:00:1111:1111:000:111",
},
},
},
{
name: "NodeAddresses should report error when IPs are empty",
nodeName: "vm1",
expectError: true,
},
{
name: "NodeAddresses should get ipV4 private and public addresses, InstanceID should get instanceID from Azure API if node is not local instance",
vmList: []string{"vm2"},
nodeName: "vm2",
providerID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm2",
ipV4: "10.240.0.1",
ipV4Public: "9.9.9.9",
expectedAddr: []v1.NodeAddress{
{
Type: v1.NodeHostName,
Address: "vm2",
},
{
Type: v1.NodeInternalIP,
Address: "10.240.0.1",
},
{
Type: v1.NodeExternalIP,
Address: "9.9.9.9",
},
},
},
{
name: "InstanceID should report error if VM doesn't exist",
vmList: []string{"vm1"},
nodeName: "vm3",
expectError: true,
},
{
name: "NodeAddresses should get ipV6 private and public addresses",
nodeName: "vm1",
providerID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1",
ipV6: "1111:11111:00:00:1111:1111:000:111",
ipV6Public: "2222:22221:00:00:2222:2222:000:111",
expectedAddr: []v1.NodeAddress{
{
Type: v1.NodeHostName,
Address: "vm1",
},
{
Type: v1.NodeInternalIP,
Address: "1111:11111:00:00:1111:1111:000:111",
},
{
Type: v1.NodeExternalIP,
Address: "2222:22221:00:00:2222:2222:000:111",
},
},
},
}
for _, test := range testcases {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Errorf("Test [%s] unexpected error: %v", test.name, err)
}
mux := http.NewServeMux()
mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, fmt.Sprintf(metadataTemplate, test.nodeName, test.ipV4, test.ipV4Public, test.ipV6, test.ipV6Public))
}))
go func() {
http.Serve(listener, mux)
}()
defer listener.Close()
cloud.metadata, err = NewInstanceMetadataService("http://" + listener.Addr().String() + "/")
if err != nil {
t.Errorf("Test [%s] unexpected error: %v", test.name, err)
}
vmListWithPowerState := make(map[string]string)
for _, vm := range test.vmList {
vmListWithPowerState[vm] = ""
}
expectedVMs := setTestVirtualMachines(cloud, vmListWithPowerState, false)
mockVMsClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface)
for _, vm := range expectedVMs {
mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, *vm.Name, gomock.Any()).Return(vm, nil).AnyTimes()
}
mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "vm3", gomock.Any()).Return(compute.VirtualMachine{}, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes()
mockVMsClient.EXPECT().Update(gomock.Any(), cloud.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
md, err := cloud.InstanceMetadataByProviderID(context.Background(), test.providerID)
if test.expectError {
if err == nil {
t.Errorf("Test [%s] unexpected nil err", test.name)
}
} else {
if err != nil {
t.Errorf("Test [%s] unexpected error: %v", test.name, err)
}
}
if len(test.expectedAddr) > 0 && !reflect.DeepEqual(md.NodeAddresses, test.expectedAddr) {
t.Errorf("Test [%s] unexpected ipAddresses: %s, expected %q", test.name, md.NodeAddresses, test.expectedAddr)
}
}
}
func TestIsCurrentInstance(t *testing.T) {
cloud := &Cloud{
Config: Config{