From 4e4fde93a7b154ea57d12b5e247dec671086801d Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Thu, 8 Feb 2018 12:40:35 +0800 Subject: [PATCH] Add useInstanceMetadata param back in Azure cloud provider This reverts commit bb1e797b2816701ed316a48846eb27e4182220e5. --- pkg/cloudprovider/providers/azure/BUILD | 1 + pkg/cloudprovider/providers/azure/azure.go | 9 ++ .../azure/azure_instance_metadata.go | 113 ++++++++++++++++++ .../providers/azure/azure_instances.go | 52 ++++++++ .../providers/azure/azure_test.go | 69 +++++++++++ 5 files changed, 244 insertions(+) create mode 100644 pkg/cloudprovider/providers/azure/azure_instance_metadata.go diff --git a/pkg/cloudprovider/providers/azure/BUILD b/pkg/cloudprovider/providers/azure/BUILD index 8cb737eed7a..66e9eb2dca2 100644 --- a/pkg/cloudprovider/providers/azure/BUILD +++ b/pkg/cloudprovider/providers/azure/BUILD @@ -17,6 +17,7 @@ go_library( "azure_controllerCommon.go", "azure_fakes.go", "azure_file.go", + "azure_instance_metadata.go", "azure_instances.go", "azure_loadbalancer.go", "azure_managedDiskController.go", diff --git a/pkg/cloudprovider/providers/azure/azure.go b/pkg/cloudprovider/providers/azure/azure.go index 9958f744565..241920e5f04 100644 --- a/pkg/cloudprovider/providers/azure/azure.go +++ b/pkg/cloudprovider/providers/azure/azure.go @@ -102,6 +102,12 @@ type Config struct { // Rate limit Bucket Size CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket" yaml:"cloudProviderRateLimitBucket"` + // Use instance metadata service where possible + UseInstanceMetadata bool `json:"useInstanceMetadata" yaml:"useInstanceMetadata"` + + // Use managed service identity for the virtual machine to access Azure ARM APIs + UseManagedIdentityExtension bool `json:"useManagedIdentityExtension"` + // Maximum allowed LoadBalancer Rule Count is the limit enforced by Azure Load balancer MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount"` } @@ -122,6 +128,7 @@ type Cloud struct { DisksClient DisksClient FileClient FileClient resourceRequestBackoff wait.Backoff + metadata *InstanceMetadata vmSet VMSet // Clients for vmss. @@ -225,6 +232,8 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) { az.CloudProviderBackoffJitter) } + az.metadata = NewInstanceMetadata() + if az.MaximumLoadBalancerRuleCount == 0 { az.MaximumLoadBalancerRuleCount = maximumLoadBalancerRuleCount } diff --git a/pkg/cloudprovider/providers/azure/azure_instance_metadata.go b/pkg/cloudprovider/providers/azure/azure_instance_metadata.go new file mode 100644 index 00000000000..a9d29ec5a51 --- /dev/null +++ b/pkg/cloudprovider/providers/azure/azure_instance_metadata.go @@ -0,0 +1,113 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "encoding/json" + "io/ioutil" + "net/http" +) + +const metadataURL = "http://169.254.169.254/metadata/" + +// NetworkMetadata contains metadata about an instance's network +type NetworkMetadata struct { + Interface []NetworkInterface `json:"interface"` +} + +// NetworkInterface represents an instances network interface. +type NetworkInterface struct { + IPV4 NetworkData `json:"ipv4"` + IPV6 NetworkData `json:"ipv6"` + MAC string `json:"macAddress"` +} + +// NetworkData contains IP information for a network. +type NetworkData struct { + IPAddress []IPAddress `json:"ipAddress"` + Subnet []Subnet `json:"subnet"` +} + +// IPAddress represents IP address information. +type IPAddress struct { + PrivateIP string `json:"privateIPAddress"` + PublicIP string `json:"publicIPAddress"` +} + +// Subnet represents subnet information. +type Subnet struct { + Address string `json:"address"` + Prefix string `json:"prefix"` +} + +// InstanceMetadata knows how to query the Azure instance metadata server. +type InstanceMetadata struct { + baseURL string +} + +// NewInstanceMetadata creates an instance of the InstanceMetadata accessor object. +func NewInstanceMetadata() *InstanceMetadata { + return &InstanceMetadata{ + baseURL: metadataURL, + } +} + +// makeMetadataURL makes a complete metadata URL from the given path. +func (i *InstanceMetadata) makeMetadataURL(path string) string { + return i.baseURL + path +} + +// Object queries the metadata server and populates the passed in object +func (i *InstanceMetadata) Object(path string, obj interface{}) error { + data, err := i.queryMetadataBytes(path, "json") + if err != nil { + return err + } + return json.Unmarshal(data, obj) +} + +// Text queries the metadata server and returns the corresponding text +func (i *InstanceMetadata) Text(path string) (string, error) { + data, err := i.queryMetadataBytes(path, "text") + if err != nil { + return "", err + } + return string(data), err +} + +func (i *InstanceMetadata) queryMetadataBytes(path, format string) ([]byte, error) { + client := &http.Client{} + + req, err := http.NewRequest("GET", i.makeMetadataURL(path), nil) + if err != nil { + return nil, err + } + req.Header.Add("Metadata", "True") + + q := req.URL.Query() + q.Add("format", format) + q.Add("api-version", "2017-04-02") + req.URL.RawQuery = q.Encode() + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + return ioutil.ReadAll(resp.Body) +} diff --git a/pkg/cloudprovider/providers/azure/azure_instances.go b/pkg/cloudprovider/providers/azure/azure_instances.go index bc34d901d96..7e25f2f79b6 100644 --- a/pkg/cloudprovider/providers/azure/azure_instances.go +++ b/pkg/cloudprovider/providers/azure/azure_instances.go @@ -29,6 +29,26 @@ import ( // NodeAddresses returns the addresses of the specified instance. func (az *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) { + if az.UseInstanceMetadata { + ipAddress := IPAddress{} + err := az.metadata.Object("instance/network/interface/0/ipv4/ipAddress/0", &ipAddress) + if err != nil { + return nil, err + } + addresses := []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: ipAddress.PrivateIP}, + {Type: v1.NodeHostName, Address: string(name)}, + } + if len(ipAddress.PublicIP) > 0 { + addr := v1.NodeAddress{ + Type: v1.NodeExternalIP, + Address: ipAddress.PublicIP, + } + addresses = append(addresses, addr) + } + return addresses, nil + } + ip, err := az.GetIPForMachineWithRetry(name) if err != nil { glog.V(2).Infof("NodeAddresses(%s) abort backoff", name) @@ -77,9 +97,28 @@ func (az *Cloud) InstanceExistsByProviderID(ctx context.Context, providerID stri return true, nil } +func (az *Cloud) isCurrentInstance(name types.NodeName) (bool, error) { + nodeName := mapNodeNameToVMName(name) + metadataName, err := az.metadata.Text("instance/compute/name") + return (metadataName == nodeName), err +} + // InstanceID returns the cloud provider ID of the specified instance. // Note that if the instance does not exist or is no longer running, we must return ("", cloudprovider.InstanceNotFound) func (az *Cloud) InstanceID(ctx context.Context, name types.NodeName) (string, error) { + if az.UseInstanceMetadata { + isLocalInstance, err := az.isCurrentInstance(name) + if err != nil { + return "", err + } + if isLocalInstance { + externalInstanceID, err := az.metadata.Text("instance/compute/vmId") + if err == nil { + return externalInstanceID, nil + } + } + } + return az.vmSet.GetInstanceIDByNodeName(string(name)) } @@ -100,6 +139,19 @@ func (az *Cloud) InstanceTypeByProviderID(ctx context.Context, providerID string // (Implementer Note): This is used by kubelet. Kubelet will label the node. Real log from kubelet: // Adding node label from cloud provider: beta.kubernetes.io/instance-type=[value] func (az *Cloud) InstanceType(ctx context.Context, name types.NodeName) (string, error) { + if az.UseInstanceMetadata { + isLocalInstance, err := az.isCurrentInstance(name) + if err != nil { + return "", err + } + if isLocalInstance { + machineType, err := az.metadata.Text("instance/compute/vmSize") + if err == nil { + return machineType, nil + } + } + } + return az.vmSet.GetInstanceTypeByNodeName(string(name)) } diff --git a/pkg/cloudprovider/providers/azure/azure_test.go b/pkg/cloudprovider/providers/azure/azure_test.go index 1895081fe3b..21fcf0bbe94 100644 --- a/pkg/cloudprovider/providers/azure/azure_test.go +++ b/pkg/cloudprovider/providers/azure/azure_test.go @@ -18,10 +18,12 @@ package azure import ( "context" + "encoding/json" "fmt" "math" "net/http" "net/http/httptest" + "reflect" "strings" "testing" @@ -1677,6 +1679,73 @@ func TestGetNodeNameByProviderID(t *testing.T) { } } +func TestMetadataURLGeneration(t *testing.T) { + metadata := NewInstanceMetadata() + fullPath := metadata.makeMetadataURL("some/path") + if fullPath != "http://169.254.169.254/metadata/some/path" { + t.Errorf("Expected http://169.254.169.254/metadata/some/path saw %s", fullPath) + } +} + +func TestMetadataParsing(t *testing.T) { + data := ` +{ + "interface": [ + { + "ipv4": { + "ipAddress": [ + { + "privateIpAddress": "10.0.1.4", + "publicIpAddress": "X.X.X.X" + } + ], + "subnet": [ + { + "address": "10.0.1.0", + "prefix": "24" + } + ] + }, + "ipv6": { + "ipAddress": [ + + ] + }, + "macAddress": "002248020E1E" + } + ] +} +` + + network := NetworkMetadata{} + if err := json.Unmarshal([]byte(data), &network); err != nil { + t.Errorf("Unexpected error: %v", err) + } + + ip := network.Interface[0].IPV4.IPAddress[0].PrivateIP + if ip != "10.0.1.4" { + t.Errorf("Unexpected value: %s, expected 10.0.1.4", ip) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, data) + })) + defer server.Close() + + metadata := &InstanceMetadata{ + baseURL: server.URL, + } + + networkJSON := NetworkMetadata{} + if err := metadata.Object("/some/path", &networkJSON); err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if !reflect.DeepEqual(network, networkJSON) { + t.Errorf("Unexpected inequality:\n%#v\nvs\n%#v", network, networkJSON) + } +} + func addTestSubnet(t *testing.T, az *Cloud, svc *v1.Service) { if svc.Annotations[ServiceAnnotationLoadBalancerInternal] != "true" { t.Error("Subnet added to non-internal service")