mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-25 20:53:33 +00:00
Merge pull request #100690 from feiskyer/fix-node-pip
azure: fix node public IP not able to fetch issues from IMDS
This commit is contained in:
commit
7a0092f96a
@ -482,7 +482,7 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret bool) erro
|
|||||||
az.Config = *config
|
az.Config = *config
|
||||||
az.Environment = *env
|
az.Environment = *env
|
||||||
az.ResourceRequestBackoff = resourceRequestBackoff
|
az.ResourceRequestBackoff = resourceRequestBackoff
|
||||||
az.metadata, err = NewInstanceMetadataService(metadataURL)
|
az.metadata, err = NewInstanceMetadataService(imdsServer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -25,13 +25,18 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"k8s.io/klog/v2"
|
||||||
azcache "k8s.io/legacy-cloud-providers/azure/cache"
|
azcache "k8s.io/legacy-cloud-providers/azure/cache"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
metadataCacheTTL = time.Minute
|
metadataCacheTTL = time.Minute
|
||||||
metadataCacheKey = "InstanceMetadata"
|
metadataCacheKey = "InstanceMetadata"
|
||||||
metadataURL = "http://169.254.169.254/metadata/instance"
|
imdsInstanceAPIVersion = "2019-03-11"
|
||||||
|
imdsLoadBalancerAPIVersion = "2020-10-01"
|
||||||
|
imdsServer = "http://169.254.169.254"
|
||||||
|
imdsInstanceURI = "/metadata/instance"
|
||||||
|
imdsLoadBalancerURI = "/metadata/loadbalancer"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NetworkMetadata contains metadata about an instance's network
|
// NetworkMetadata contains metadata about an instance's network
|
||||||
@ -86,19 +91,35 @@ type InstanceMetadata struct {
|
|||||||
Network *NetworkMetadata `json:"network,omitempty"`
|
Network *NetworkMetadata `json:"network,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PublicIPMetadata represents the public IP metadata.
|
||||||
|
type PublicIPMetadata struct {
|
||||||
|
FrontendIPAddress string `json:"frontendIpAddress,omitempty"`
|
||||||
|
PrivateIPAddress string `json:"privateIpAddress,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadbalancerProfile represents load balancer profile in IMDS.
|
||||||
|
type LoadbalancerProfile struct {
|
||||||
|
PublicIPAddresses []PublicIPMetadata `json:"publicIpAddresses,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadBalancerMetadata represents load balancer metadata.
|
||||||
|
type LoadBalancerMetadata struct {
|
||||||
|
LoadBalancer *LoadbalancerProfile `json:"loadbalancer,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// InstanceMetadataService knows how to query the Azure instance metadata server.
|
// InstanceMetadataService knows how to query the Azure instance metadata server.
|
||||||
type InstanceMetadataService struct {
|
type InstanceMetadataService struct {
|
||||||
metadataURL string
|
imdsServer string
|
||||||
imsCache *azcache.TimedCache
|
imsCache *azcache.TimedCache
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewInstanceMetadataService creates an instance of the InstanceMetadataService accessor object.
|
// NewInstanceMetadataService creates an instance of the InstanceMetadataService accessor object.
|
||||||
func NewInstanceMetadataService(metadataURL string) (*InstanceMetadataService, error) {
|
func NewInstanceMetadataService(imdsServer string) (*InstanceMetadataService, error) {
|
||||||
ims := &InstanceMetadataService{
|
ims := &InstanceMetadataService{
|
||||||
metadataURL: metadataURL,
|
imdsServer: imdsServer,
|
||||||
}
|
}
|
||||||
|
|
||||||
imsCache, err := azcache.NewTimedcache(metadataCacheTTL, ims.getInstanceMetadata)
|
imsCache, err := azcache.NewTimedcache(metadataCacheTTL, ims.getMetadata)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -107,8 +128,52 @@ func NewInstanceMetadataService(metadataURL string) (*InstanceMetadataService, e
|
|||||||
return ims, nil
|
return ims, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ims *InstanceMetadataService) getInstanceMetadata(key string) (interface{}, error) {
|
func (ims *InstanceMetadataService) getMetadata(key string) (interface{}, error) {
|
||||||
req, err := http.NewRequest("GET", ims.metadataURL, nil)
|
instanceMetadata, err := ims.getInstanceMetadata(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if instanceMetadata.Network != nil && len(instanceMetadata.Network.Interface) > 0 {
|
||||||
|
netInterface := instanceMetadata.Network.Interface[0]
|
||||||
|
if (len(netInterface.IPV4.IPAddress) > 0 && len(netInterface.IPV4.IPAddress[0].PublicIP) > 0) ||
|
||||||
|
(len(netInterface.IPV6.IPAddress) > 0 && len(netInterface.IPV6.IPAddress[0].PublicIP) > 0) {
|
||||||
|
// Return if public IP address has already part of instance metadata.
|
||||||
|
return instanceMetadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
loadBalancerMetadata, err := ims.getLoadBalancerMetadata()
|
||||||
|
if err != nil || loadBalancerMetadata == nil || loadBalancerMetadata.LoadBalancer == nil {
|
||||||
|
// Log a warning since loadbalancer metadata may not be available when the VM
|
||||||
|
// is not in standard LoadBalancer backend address pool.
|
||||||
|
klog.V(4).Infof("Warning: failed to get loadbalancer metadata: %v", err)
|
||||||
|
return instanceMetadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
publicIPs := loadBalancerMetadata.LoadBalancer.PublicIPAddresses
|
||||||
|
if len(netInterface.IPV4.IPAddress) > 0 && len(netInterface.IPV4.IPAddress[0].PrivateIP) > 0 {
|
||||||
|
for _, pip := range publicIPs {
|
||||||
|
if pip.PrivateIPAddress == netInterface.IPV4.IPAddress[0].PrivateIP {
|
||||||
|
netInterface.IPV4.IPAddress[0].PublicIP = pip.FrontendIPAddress
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(netInterface.IPV6.IPAddress) > 0 && len(netInterface.IPV6.IPAddress[0].PrivateIP) > 0 {
|
||||||
|
for _, pip := range publicIPs {
|
||||||
|
if pip.PrivateIPAddress == netInterface.IPV6.IPAddress[0].PrivateIP {
|
||||||
|
netInterface.IPV6.IPAddress[0].PublicIP = pip.FrontendIPAddress
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return instanceMetadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ims *InstanceMetadataService) getInstanceMetadata(key string) (*InstanceMetadata, error) {
|
||||||
|
req, err := http.NewRequest("GET", ims.imdsServer+imdsInstanceURI, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -117,7 +182,7 @@ func (ims *InstanceMetadataService) getInstanceMetadata(key string) (interface{}
|
|||||||
|
|
||||||
q := req.URL.Query()
|
q := req.URL.Query()
|
||||||
q.Add("format", "json")
|
q.Add("format", "json")
|
||||||
q.Add("api-version", "2019-03-11")
|
q.Add("api-version", imdsInstanceAPIVersion)
|
||||||
req.URL.RawQuery = q.Encode()
|
req.URL.RawQuery = q.Encode()
|
||||||
|
|
||||||
client := &http.Client{}
|
client := &http.Client{}
|
||||||
@ -145,6 +210,44 @@ func (ims *InstanceMetadataService) getInstanceMetadata(key string) (interface{}
|
|||||||
return &obj, nil
|
return &obj, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (ims *InstanceMetadataService) getLoadBalancerMetadata() (*LoadBalancerMetadata, error) {
|
||||||
|
req, err := http.NewRequest("GET", ims.imdsServer+imdsLoadBalancerURI, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Header.Add("Metadata", "True")
|
||||||
|
req.Header.Add("User-Agent", "golang/kubernetes-cloud-provider")
|
||||||
|
|
||||||
|
q := req.URL.Query()
|
||||||
|
q.Add("format", "json")
|
||||||
|
q.Add("api-version", imdsLoadBalancerAPIVersion)
|
||||||
|
req.URL.RawQuery = q.Encode()
|
||||||
|
|
||||||
|
client := &http.Client{}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("failure of getting loadbalancer metadata with response %q", resp.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
obj := LoadBalancerMetadata{}
|
||||||
|
err = json.Unmarshal(data, &obj)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &obj, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetMetadata gets instance metadata from cache.
|
// GetMetadata gets instance metadata from cache.
|
||||||
// crt determines if we can get data from stalled cache/need fresh if cache expired.
|
// crt determines if we can get data from stalled cache/need fresh if cache expired.
|
||||||
func (ims *InstanceMetadataService) GetMetadata(crt azcache.AzureCacheReadType) (*InstanceMetadata, error) {
|
func (ims *InstanceMetadataService) GetMetadata(crt azcache.AzureCacheReadType) (*InstanceMetadata, error) {
|
||||||
|
@ -23,6 +23,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute"
|
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute"
|
||||||
@ -399,7 +400,7 @@ func TestNodeAddresses(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
metadataTemplate := `{"compute":{"name":"%s"},"network":{"interface":[{"ipv4":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]},"ipv6":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]}}]}}`
|
metadataTemplate := `{"compute":{"name":"%s"},"network":{"interface":[{"ipv4":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]},"ipv6":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]}}]}}`
|
||||||
|
loadbalancerTemplate := `{"loadbalancer": {"publicIpAddresses": [{"frontendIpAddress": "%s","privateIpAddress": "%s"},{"frontendIpAddress": "%s","privateIpAddress": "%s"}]}}`
|
||||||
testcases := []struct {
|
testcases := []struct {
|
||||||
name string
|
name string
|
||||||
nodeName string
|
nodeName string
|
||||||
@ -410,6 +411,7 @@ func TestNodeAddresses(t *testing.T) {
|
|||||||
ipV6 string
|
ipV6 string
|
||||||
ipV4Public string
|
ipV4Public string
|
||||||
ipV6Public string
|
ipV6Public string
|
||||||
|
loadBalancerSku string
|
||||||
expectedAddress []v1.NodeAddress
|
expectedAddress []v1.NodeAddress
|
||||||
useInstanceMetadata bool
|
useInstanceMetadata bool
|
||||||
useCustomImsCache bool
|
useCustomImsCache bool
|
||||||
@ -484,7 +486,7 @@ func TestNodeAddresses(t *testing.T) {
|
|||||||
expectedAddress: expectedNodeAddress,
|
expectedAddress: expectedNodeAddress,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "NodeAddresses should get IP addresses from local if node's name is equal to metadataName",
|
name: "NodeAddresses should get IP addresses from local IMDS if node's name is equal to metadataName",
|
||||||
nodeName: "vm1",
|
nodeName: "vm1",
|
||||||
metadataName: "vm1",
|
metadataName: "vm1",
|
||||||
vmType: vmTypeStandard,
|
vmType: vmTypeStandard,
|
||||||
@ -492,6 +494,41 @@ func TestNodeAddresses(t *testing.T) {
|
|||||||
ipV4Public: "192.168.1.12",
|
ipV4Public: "192.168.1.12",
|
||||||
ipV6: "1111:11111:00:00:1111:1111:000:111",
|
ipV6: "1111:11111:00:00:1111:1111:000:111",
|
||||||
ipV6Public: "2222:22221:00:00:2222:2222:000:111",
|
ipV6Public: "2222:22221:00:00:2222:2222:000:111",
|
||||||
|
loadBalancerSku: "basic",
|
||||||
|
useInstanceMetadata: true,
|
||||||
|
expectedAddress: []v1.NodeAddress{
|
||||||
|
{
|
||||||
|
Type: v1.NodeHostName,
|
||||||
|
Address: "vm1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: v1.NodeInternalIP,
|
||||||
|
Address: "10.240.0.1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: v1.NodeExternalIP,
|
||||||
|
Address: "192.168.1.12",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: v1.NodeInternalIP,
|
||||||
|
Address: "1111:11111:00:00:1111:1111:000:111",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Type: v1.NodeExternalIP,
|
||||||
|
Address: "2222:22221:00:00:2222:2222:000:111",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "NodeAddresses should get IP addresses from local IMDS for standard LoadBalancer if node's name is equal to metadataName",
|
||||||
|
nodeName: "vm1",
|
||||||
|
metadataName: "vm1",
|
||||||
|
vmType: vmTypeStandard,
|
||||||
|
ipV4: "10.240.0.1",
|
||||||
|
ipV4Public: "192.168.1.12",
|
||||||
|
ipV6: "1111:11111:00:00:1111:1111:000:111",
|
||||||
|
ipV6Public: "2222:22221:00:00:2222:2222:000:111",
|
||||||
|
loadBalancerSku: "standard",
|
||||||
useInstanceMetadata: true,
|
useInstanceMetadata: true,
|
||||||
expectedAddress: []v1.NodeAddress{
|
expectedAddress: []v1.NodeAddress{
|
||||||
{
|
{
|
||||||
@ -533,10 +570,19 @@ func TestNodeAddresses(t *testing.T) {
|
|||||||
|
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if strings.Contains(r.RequestURI, imdsLoadBalancerURI) {
|
||||||
|
fmt.Fprintf(w, loadbalancerTemplate, test.ipV4Public, test.ipV4, test.ipV6Public, test.ipV6)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if test.metadataTemplate != "" {
|
if test.metadataTemplate != "" {
|
||||||
fmt.Fprintf(w, test.metadataTemplate)
|
fmt.Fprintf(w, test.metadataTemplate)
|
||||||
} else {
|
} else {
|
||||||
fmt.Fprintf(w, metadataTemplate, test.metadataName, test.ipV4, test.ipV4Public, test.ipV6, test.ipV6Public)
|
if test.loadBalancerSku == "standard" {
|
||||||
|
fmt.Fprintf(w, metadataTemplate, test.metadataName, test.ipV4, "", test.ipV6, "")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(w, metadataTemplate, test.metadataName, test.ipV4, test.ipV4Public, test.ipV6, test.ipV6Public)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
go func() {
|
go func() {
|
||||||
|
Loading…
Reference in New Issue
Block a user