mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Merge pull request #53557 from yolo3301/az_lb_pipid
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Allow use resource ID to specify public IP address in azure_loadbalancer **What this PR does / why we need it**: Currently the Azure load balancer assumes that a Public IP address is in the same resource group as the cluster. This is not necessarily true in all environments, in addition to accepting a Public IP, we should allow an annotation to the `Service` object that indicates what resource group the IP is present in. **Which issue this PR fixes**: fixes #53274 #52129 **Special notes for your reviewer**: *first time golang user, please forgive the amateurness* Release note ```release-note Allow use resource ID to specify public IP address in azure_loadbalancer ```
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
02be3eb07c
@ -243,23 +243,23 @@ func (az *Cloud) ListLBWithRetry() ([]network.LoadBalancer, error) {
|
||||
return allLBs, nil
|
||||
}
|
||||
|
||||
// ListPIPWithRetry list the PIP resources in az.ResourceGroup
|
||||
func (az *Cloud) ListPIPWithRetry() ([]network.PublicIPAddress, error) {
|
||||
// ListPIPWithRetry list the PIP resources in the given resource group
|
||||
func (az *Cloud) ListPIPWithRetry(pipResourceGroup string) ([]network.PublicIPAddress, error) {
|
||||
allPIPs := []network.PublicIPAddress{}
|
||||
var result network.PublicIPAddressListResult
|
||||
err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
|
||||
var retryErr error
|
||||
az.operationPollRateLimiter.Accept()
|
||||
glog.V(10).Infof("PublicIPAddressesClient.List(%v): start", az.ResourceGroup)
|
||||
result, retryErr = az.PublicIPAddressesClient.List(az.ResourceGroup)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.List(%v): end", az.ResourceGroup)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.List(%v): start", pipResourceGroup)
|
||||
result, retryErr = az.PublicIPAddressesClient.List(pipResourceGroup)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.List(%v): end", pipResourceGroup)
|
||||
if retryErr != nil {
|
||||
glog.Errorf("PublicIPAddressesClient.List(%v) - backoff: failure, will retry,err=%v",
|
||||
az.ResourceGroup,
|
||||
pipResourceGroup,
|
||||
retryErr)
|
||||
return false, retryErr
|
||||
}
|
||||
glog.V(2).Infof("PublicIPAddressesClient.List(%v) - backoff: success", az.ResourceGroup)
|
||||
glog.V(2).Infof("PublicIPAddressesClient.List(%v) - backoff: success", pipResourceGroup)
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
@ -271,21 +271,21 @@ func (az *Cloud) ListPIPWithRetry() ([]network.PublicIPAddress, error) {
|
||||
allPIPs = append(allPIPs, *result.Value...)
|
||||
appendResults = false
|
||||
|
||||
// follow the next link to get all the vms for resource group
|
||||
// follow the next link to get all the pip resources for resource group
|
||||
if result.NextLink != nil {
|
||||
err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
|
||||
var retryErr error
|
||||
az.operationPollRateLimiter.Accept()
|
||||
glog.V(10).Infof("PublicIPAddressesClient.ListNextResults(%v): start", az.ResourceGroup)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.ListNextResults(%v): start", pipResourceGroup)
|
||||
result, retryErr = az.PublicIPAddressesClient.ListNextResults(result)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.ListNextResults(%v): end", az.ResourceGroup)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.ListNextResults(%v): end", pipResourceGroup)
|
||||
if retryErr != nil {
|
||||
glog.Errorf("PublicIPAddressesClient.ListNextResults(%v) - backoff: failure, will retry,err=%v",
|
||||
az.ResourceGroup,
|
||||
pipResourceGroup,
|
||||
retryErr)
|
||||
return false, retryErr
|
||||
}
|
||||
glog.V(2).Infof("PublicIPAddressesClient.ListNextResults(%v) - backoff: success", az.ResourceGroup)
|
||||
glog.V(2).Infof("PublicIPAddressesClient.ListNextResults(%v) - backoff: success", pipResourceGroup)
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
@ -299,14 +299,14 @@ func (az *Cloud) ListPIPWithRetry() ([]network.PublicIPAddress, error) {
|
||||
}
|
||||
|
||||
// CreateOrUpdatePIPWithRetry invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry
|
||||
func (az *Cloud) CreateOrUpdatePIPWithRetry(pip network.PublicIPAddress) error {
|
||||
func (az *Cloud) CreateOrUpdatePIPWithRetry(pipResourceGroup string, pip network.PublicIPAddress) error {
|
||||
return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
|
||||
az.operationPollRateLimiter.Accept()
|
||||
glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s): start", *pip.Name)
|
||||
respChan, errChan := az.PublicIPAddressesClient.CreateOrUpdate(az.ResourceGroup, *pip.Name, pip, nil)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s, %s): start", pipResourceGroup, *pip.Name)
|
||||
respChan, errChan := az.PublicIPAddressesClient.CreateOrUpdate(pipResourceGroup, *pip.Name, pip, nil)
|
||||
resp := <-respChan
|
||||
err := <-errChan
|
||||
glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s): end", *pip.Name)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s, %s): end", pipResourceGroup, *pip.Name)
|
||||
return processRetryResponse(resp.Response, err)
|
||||
})
|
||||
}
|
||||
@ -325,14 +325,14 @@ func (az *Cloud) CreateOrUpdateInterfaceWithRetry(nic network.Interface) error {
|
||||
}
|
||||
|
||||
// DeletePublicIPWithRetry invokes az.PublicIPAddressesClient.Delete with exponential backoff retry
|
||||
func (az *Cloud) DeletePublicIPWithRetry(pipName string) error {
|
||||
func (az *Cloud) DeletePublicIPWithRetry(pipResourceGroup string, pipName string) error {
|
||||
return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
|
||||
az.operationPollRateLimiter.Accept()
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Delete(%s): start", pipName)
|
||||
respChan, errChan := az.PublicIPAddressesClient.Delete(az.ResourceGroup, pipName, nil)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Delete(%s, %s): start", pipResourceGroup, pipName)
|
||||
respChan, errChan := az.PublicIPAddressesClient.Delete(pipResourceGroup, pipName, nil)
|
||||
resp := <-respChan
|
||||
err := <-errChan
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Delete(%s): end", pipName)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Delete(%s, %s): end", pipResourceGroup, pipName)
|
||||
return processRetryResponse(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
@ -67,6 +67,10 @@ const (
|
||||
ServiceAnnotationSharedSecurityRule = "service.beta.kubernetes.io/azure-shared-securityrule"
|
||||
)
|
||||
|
||||
// ServiceAnnotationLoadBalancerResourceGroup is the annotation used on the service
|
||||
// to specify the resource group of load balancer objects that are not in the same resource group as the cluster.
|
||||
const ServiceAnnotationLoadBalancerResourceGroup = "service.beta.kubernetes.io/azure-load-balancer-resource-group"
|
||||
|
||||
// GetLoadBalancer returns whether the specified load balancer exists, and
|
||||
// if so, what its status is.
|
||||
func (az *Cloud) GetLoadBalancer(clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) {
|
||||
@ -82,7 +86,7 @@ func (az *Cloud) GetLoadBalancer(clusterName string, service *v1.Service) (statu
|
||||
return status, true, nil
|
||||
}
|
||||
|
||||
func getPublicIPLabel(service *v1.Service) string {
|
||||
func getPublicIPDomainNameLabel(service *v1.Service) string {
|
||||
if labelName, found := service.Annotations[ServiceAnnotationDNSLabelName]; found {
|
||||
return labelName
|
||||
}
|
||||
@ -316,7 +320,7 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress Name from ID(%s)", serviceName, *lb.Name, *pipID)
|
||||
}
|
||||
pip, existsPip, err := az.getPublicIPAddress(pipName)
|
||||
pip, existsPip, err := az.getPublicIPAddress(az.getPublicIPAddressResourceGroup(service), pipName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -338,7 +342,9 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service)
|
||||
return getPublicIPName(clusterName, service), nil
|
||||
}
|
||||
|
||||
pips, err := az.ListPIPWithRetry()
|
||||
pipResourceGroup := az.getPublicIPAddressResourceGroup(service)
|
||||
|
||||
pips, err := az.ListPIPWithRetry(pipResourceGroup)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@ -349,7 +355,7 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service)
|
||||
return *pip.Name, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("user supplied IP Address %s was not found", loadBalancerIP)
|
||||
return "", fmt.Errorf("user supplied IP Address %s was not found in resource group %s", loadBalancerIP, pipResourceGroup)
|
||||
}
|
||||
|
||||
func flipServiceInternalAnnotation(service *v1.Service) *v1.Service {
|
||||
@ -386,8 +392,9 @@ func (az *Cloud) findServiceIPAddress(clusterName string, service *v1.Service, i
|
||||
return lbStatus.Ingress[0].IP, nil
|
||||
}
|
||||
|
||||
func (az *Cloud) ensurePublicIPExists(serviceName, pipName, domainNameLabel string) (*network.PublicIPAddress, error) {
|
||||
pip, existsPip, err := az.getPublicIPAddress(pipName)
|
||||
func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domainNameLabel string) (*network.PublicIPAddress, error) {
|
||||
pipResourceGroup := az.getPublicIPAddressResourceGroup(service)
|
||||
pip, existsPip, err := az.getPublicIPAddress(pipResourceGroup, pipName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -395,6 +402,7 @@ func (az *Cloud) ensurePublicIPExists(serviceName, pipName, domainNameLabel stri
|
||||
return &pip, nil
|
||||
}
|
||||
|
||||
serviceName := getServiceName(service)
|
||||
pip.Name = to.StringPtr(pipName)
|
||||
pip.Location = to.StringPtr(az.Location)
|
||||
pip.PublicIPAddressPropertiesFormat = &network.PublicIPAddressPropertiesFormat{
|
||||
@ -408,18 +416,18 @@ func (az *Cloud) ensurePublicIPExists(serviceName, pipName, domainNameLabel stri
|
||||
pip.Tags = &map[string]*string{"service": &serviceName}
|
||||
glog.V(3).Infof("ensure(%s): pip(%s) - creating", serviceName, *pip.Name)
|
||||
az.operationPollRateLimiter.Accept()
|
||||
glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%q): start", *pip.Name)
|
||||
err = az.CreateOrUpdatePIPWithRetry(pip)
|
||||
glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%s, %q): start", pipResourceGroup, *pip.Name)
|
||||
err = az.CreateOrUpdatePIPWithRetry(pipResourceGroup, pip)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("ensure(%s) abort backoff: pip(%s) - creating", serviceName, *pip.Name)
|
||||
return nil, err
|
||||
}
|
||||
glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%q): end", *pip.Name)
|
||||
glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%s, %q): end", pipResourceGroup, *pip.Name)
|
||||
|
||||
az.operationPollRateLimiter.Accept()
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Get(%q): start", *pip.Name)
|
||||
pip, err = az.PublicIPAddressesClient.Get(az.ResourceGroup, *pip.Name, "")
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Get(%q): end", *pip.Name)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Get(%s, %q): start", pipResourceGroup, *pip.Name)
|
||||
pip, err = az.PublicIPAddressesClient.Get(pipResourceGroup, *pip.Name, "")
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Get(%s, %q): end", pipResourceGroup, *pip.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -546,8 +554,8 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
domainNameLabel := getPublicIPLabel(service)
|
||||
pip, err := az.ensurePublicIPExists(serviceName, pipName, domainNameLabel)
|
||||
domainNameLabel := getPublicIPDomainNameLabel(service)
|
||||
pip, err := az.ensurePublicIPExists(service, pipName, domainNameLabel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -1137,7 +1145,9 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, want
|
||||
}
|
||||
}
|
||||
|
||||
pips, err := az.ListPIPWithRetry()
|
||||
pipResourceGroup := az.getPublicIPAddressResourceGroup(service)
|
||||
|
||||
pips, err := az.ListPIPWithRetry(pipResourceGroup)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -1154,14 +1164,14 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, want
|
||||
} else {
|
||||
glog.V(2).Infof("ensure(%s): pip(%s) - deleting", serviceName, pipName)
|
||||
az.operationPollRateLimiter.Accept()
|
||||
glog.V(10).Infof("DeletePublicIPWithRetry(%q): start", pipName)
|
||||
err = az.DeletePublicIPWithRetry(pipName)
|
||||
glog.V(10).Infof("DeletePublicIPWithRetry(%s, %q): start", pipResourceGroup, pipName)
|
||||
err = az.DeletePublicIPWithRetry(pipResourceGroup, pipName)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("ensure(%s) abort backoff: pip(%s) - deleting", serviceName, pipName)
|
||||
// We let err to pass through
|
||||
// It may be ignorable
|
||||
}
|
||||
glog.V(10).Infof("DeletePublicIPWithRetry(%q): end", pipName) // response not read yet...
|
||||
glog.V(10).Infof("DeletePublicIPWithRetry(%s, %q): end", pipResourceGroup, pipName) // response not read yet...
|
||||
|
||||
err = ignoreStatusNotFoundFromError(err)
|
||||
if err != nil {
|
||||
@ -1176,8 +1186,8 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, want
|
||||
if !isInternal && wantLb {
|
||||
// Confirm desired public ip resource exists
|
||||
var pip *network.PublicIPAddress
|
||||
domainNameLabel := getPublicIPLabel(service)
|
||||
if pip, err = az.ensurePublicIPExists(serviceName, desiredPipName, domainNameLabel); err != nil {
|
||||
domainNameLabel := getPublicIPDomainNameLabel(service)
|
||||
if pip, err = az.ensurePublicIPExists(service, desiredPipName, domainNameLabel); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pip, nil
|
||||
@ -1240,9 +1250,17 @@ func findSecurityRule(rules []network.SecurityRule, rule network.SecurityRule) b
|
||||
return false
|
||||
}
|
||||
|
||||
func (az *Cloud) getPublicIPAddressResourceGroup(service *v1.Service) string {
|
||||
if resourceGroup, found := service.Annotations[ServiceAnnotationLoadBalancerResourceGroup]; found {
|
||||
return resourceGroup
|
||||
}
|
||||
|
||||
return az.ResourceGroup
|
||||
}
|
||||
|
||||
// Check if service requires an internal load balancer.
|
||||
func requiresInternalLoadBalancer(service *v1.Service) bool {
|
||||
if l, ok := service.Annotations[ServiceAnnotationLoadBalancerInternal]; ok {
|
||||
if l, found := service.Annotations[ServiceAnnotationLoadBalancerInternal]; found {
|
||||
return l == "true"
|
||||
}
|
||||
|
||||
@ -1251,7 +1269,7 @@ func requiresInternalLoadBalancer(service *v1.Service) bool {
|
||||
|
||||
func subnet(service *v1.Service) *string {
|
||||
if requiresInternalLoadBalancer(service) {
|
||||
if l, ok := service.Annotations[ServiceAnnotationLoadBalancerInternalSubnet]; ok {
|
||||
if l, found := service.Annotations[ServiceAnnotationLoadBalancerInternalSubnet]; found {
|
||||
return &l
|
||||
}
|
||||
}
|
||||
|
||||
@ -156,13 +156,17 @@ func (az *Cloud) listLoadBalancers() (lbListResult network.LoadBalancerListResul
|
||||
return lbListResult, exists, err
|
||||
}
|
||||
|
||||
func (az *Cloud) getPublicIPAddress(name string) (pip network.PublicIPAddress, exists bool, err error) {
|
||||
var realErr error
|
||||
func (az *Cloud) getPublicIPAddress(pipResourceGroup string, pipName string) (pip network.PublicIPAddress, exists bool, err error) {
|
||||
resourceGroup := az.ResourceGroup
|
||||
if pipResourceGroup != "" {
|
||||
resourceGroup = pipResourceGroup
|
||||
}
|
||||
|
||||
var realErr error
|
||||
az.operationPollRateLimiter.Accept()
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Get(%s): start", name)
|
||||
pip, err = az.PublicIPAddressesClient.Get(az.ResourceGroup, name, "")
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Get(%s): end", name)
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Get(%s, %s): start", resourceGroup, pipName)
|
||||
pip, err = az.PublicIPAddressesClient.Get(resourceGroup, pipName, "")
|
||||
glog.V(10).Infof("PublicIPAddressesClient.Get(%s, %s): end", resourceGroup, pipName)
|
||||
|
||||
exists, realErr = checkResourceExistsFromError(err)
|
||||
if realErr != nil {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user