Merge pull request #59531 from feiskyer/metadata

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add useInstanceMetadata param back in Azure cloud provider

This reverts commit bb1e797b28.



**What this PR does / why we need it**:

Add useInstanceMetadata param back in Azure cloud provider.

@jdumars and @brendandburns are working to make cloud controller manager using node's local instance metadata. It could help reducing cloud API calls for large clusters.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
This reverts #57647 and #57646.

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

pkg/cloudprovider/providers/azure/BUILD

@@ -17,6 +17,7 @@ go_library(
         "azure_controllerCommon.go",
         "azure_fakes.go",
         "azure_file.go",
+        "azure_instance_metadata.go",
         "azure_instances.go",
         "azure_loadbalancer.go",
         "azure_managedDiskController.go",

pkg/cloudprovider/providers/azure/azure.go

@@ -102,6 +102,12 @@ type Config struct {
 	// Rate limit Bucket Size
 	CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket" yaml:"cloudProviderRateLimitBucket"`
 
+	// Use instance metadata service where possible
+	UseInstanceMetadata bool `json:"useInstanceMetadata" yaml:"useInstanceMetadata"`
+
+	// Use managed service identity for the virtual machine to access Azure ARM APIs
+	UseManagedIdentityExtension bool `json:"useManagedIdentityExtension"`
+
 	// Maximum allowed LoadBalancer Rule Count is the limit enforced by Azure Load balancer
 	MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount"`
 }
@@ -122,6 +128,7 @@ type Cloud struct {
 	DisksClient             DisksClient
 	FileClient              FileClient
 	resourceRequestBackoff  wait.Backoff
+	metadata                *InstanceMetadata
 	vmSet                   VMSet
 
 	// Clients for vmss.
@@ -225,6 +232,8 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
 			az.CloudProviderBackoffJitter)
 	}
 
+	az.metadata = NewInstanceMetadata()
+
 	if az.MaximumLoadBalancerRuleCount == 0 {
 		az.MaximumLoadBalancerRuleCount = maximumLoadBalancerRuleCount
 	}

pkg/cloudprovider/providers/azure/azure_instance_metadata.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+)
+
+const metadataURL = "http://169.254.169.254/metadata/"
+
+// NetworkMetadata contains metadata about an instance's network
+type NetworkMetadata struct {
+	Interface []NetworkInterface `json:"interface"`
+}
+
+// NetworkInterface represents an instances network interface.
+type NetworkInterface struct {
+	IPV4 NetworkData `json:"ipv4"`
+	IPV6 NetworkData `json:"ipv6"`
+	MAC  string      `json:"macAddress"`
+}
+
+// NetworkData contains IP information for a network.
+type NetworkData struct {
+	IPAddress []IPAddress `json:"ipAddress"`
+	Subnet    []Subnet    `json:"subnet"`
+}
+
+// IPAddress represents IP address information.
+type IPAddress struct {
+	PrivateIP string `json:"privateIPAddress"`
+	PublicIP  string `json:"publicIPAddress"`
+}
+
+// Subnet represents subnet information.
+type Subnet struct {
+	Address string `json:"address"`
+	Prefix  string `json:"prefix"`
+}
+
+// InstanceMetadata knows how to query the Azure instance metadata server.
+type InstanceMetadata struct {
+	baseURL string
+}
+
+// NewInstanceMetadata creates an instance of the InstanceMetadata accessor object.
+func NewInstanceMetadata() *InstanceMetadata {
+	return &InstanceMetadata{
+		baseURL: metadataURL,
+	}
+}
+
+// makeMetadataURL makes a complete metadata URL from the given path.
+func (i *InstanceMetadata) makeMetadataURL(path string) string {
+	return i.baseURL + path
+}
+
+// Object queries the metadata server and populates the passed in object
+func (i *InstanceMetadata) Object(path string, obj interface{}) error {
+	data, err := i.queryMetadataBytes(path, "json")
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(data, obj)
+}
+
+// Text queries the metadata server and returns the corresponding text
+func (i *InstanceMetadata) Text(path string) (string, error) {
+	data, err := i.queryMetadataBytes(path, "text")
+	if err != nil {
+		return "", err
+	}
+	return string(data), err
+}
+
+func (i *InstanceMetadata) queryMetadataBytes(path, format string) ([]byte, error) {
+	client := &http.Client{}
+
+	req, err := http.NewRequest("GET", i.makeMetadataURL(path), nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Metadata", "True")
+
+	q := req.URL.Query()
+	q.Add("format", format)
+	q.Add("api-version", "2017-04-02")
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	return ioutil.ReadAll(resp.Body)
+}

pkg/cloudprovider/providers/azure/azure_instances.go

@@ -29,6 +29,26 @@ import (
 
 // NodeAddresses returns the addresses of the specified instance.
 func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) {
+	if az.UseInstanceMetadata {
+		ipAddress := IPAddress{}
+		err := az.metadata.Object("instance/network/interface/0/ipv4/ipAddress/0", &ipAddress)
+		if err != nil {
+			return nil, err
+		}
+		addresses := []v1.NodeAddress{
+			{Type: v1.NodeInternalIP, Address: ipAddress.PrivateIP},
+			{Type: v1.NodeHostName, Address: string(name)},
+		}
+		if len(ipAddress.PublicIP) > 0 {
+			addr := v1.NodeAddress{
+				Type:    v1.NodeExternalIP,
+				Address: ipAddress.PublicIP,
+			}
+			addresses = append(addresses, addr)
+		}
+		return addresses, nil
+	}
+
 	ip, err := az.GetIPForMachineWithRetry(name)
 	if err != nil {
 		glog.V(2).Infof("NodeAddresses(%s) abort backoff", name)
@@ -77,9 +97,28 @@ func (az *Cloud) InstanceExistsByProviderID(ctx context.Context, providerID stri
 	return true, nil
 }
 
+func (az *Cloud) isCurrentInstance(name types.NodeName) (bool, error) {
+	nodeName := mapNodeNameToVMName(name)
+	metadataName, err := az.metadata.Text("instance/compute/name")
+	return (metadataName == nodeName), err
+}
+
 // InstanceID returns the cloud provider ID of the specified instance.
 // Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound)
 func (az *Cloud) InstanceID(ctx context.Context, name types.NodeName) (string, error) {
+	if az.UseInstanceMetadata {
+		isLocalInstance, err := az.isCurrentInstance(name)
+		if err != nil {
+			return "", err
+		}
+		if isLocalInstance {
+			externalInstanceID, err := az.metadata.Text("instance/compute/vmId")
+			if err == nil {
+				return externalInstanceID, nil
+			}
+		}
+	}
+
 	return az.vmSet.GetInstanceIDByNodeName(string(name))
 }
 
@@ -100,6 +139,19 @@ func (az *Cloud) InstanceTypeByProviderID(ctx context.Context, providerID string
 // (Implementer Note): This is used by kubelet. Kubelet will label the node. Real log from kubelet:
 //       Adding node label from cloud provider: beta.kubernetes.io/instance-type=[value]
 func (az *Cloud) InstanceType(ctx context.Context, name types.NodeName) (string, error) {
+	if az.UseInstanceMetadata {
+		isLocalInstance, err := az.isCurrentInstance(name)
+		if err != nil {
+			return "", err
+		}
+		if isLocalInstance {
+			machineType, err := az.metadata.Text("instance/compute/vmSize")
+			if err == nil {
+				return machineType, nil
+			}
+		}
+	}
+
 	return az.vmSet.GetInstanceTypeByNodeName(string(name))
 }
 

pkg/cloudprovider/providers/azure/azure_test.go

@@ -18,10 +18,12 @@ package azure
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -1677,6 +1679,73 @@ func TestGetNodeNameByProviderID(t *testing.T) {
 	}
 }
 
+func TestMetadataURLGeneration(t *testing.T) {
+	metadata := NewInstanceMetadata()
+	fullPath := metadata.makeMetadataURL("some/path")
+	if fullPath != "http://169.254.169.254/metadata/some/path" {
+		t.Errorf("Expected http://169.254.169.254/metadata/some/path saw %s", fullPath)
+	}
+}
+
+func TestMetadataParsing(t *testing.T) {
+	data := `
+{
+    "interface": [
+      {
+        "ipv4": {
+          "ipAddress": [
+            {
+              "privateIpAddress": "10.0.1.4",
+              "publicIpAddress": "X.X.X.X"
+            }
+          ],
+          "subnet": [
+            {
+              "address": "10.0.1.0",
+              "prefix": "24"
+            }
+          ]
+        },
+        "ipv6": {
+          "ipAddress": [
+
+          ]
+        },
+        "macAddress": "002248020E1E"
+      }
+    ]
+}
+`
+
+	network := NetworkMetadata{}
+	if err := json.Unmarshal([]byte(data), &network); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	ip := network.Interface[0].IPV4.IPAddress[0].PrivateIP
+	if ip != "10.0.1.4" {
+		t.Errorf("Unexpected value: %s, expected 10.0.1.4", ip)
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, data)
+	}))
+	defer server.Close()
+
+	metadata := &InstanceMetadata{
+		baseURL: server.URL,
+	}
+
+	networkJSON := NetworkMetadata{}
+	if err := metadata.Object("/some/path", &networkJSON); err != nil {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(network, networkJSON) {
+		t.Errorf("Unexpected inequality:\n%#v\nvs\n%#v", network, networkJSON)
+	}
+}
+
 func addTestSubnet(t *testing.T, az *Cloud, svc *v1.Service) {
 	if svc.Annotations[ServiceAnnotationLoadBalancerInternal] != "true" {
 		t.Error("Subnet added to non-internal service")