diff --git a/pkg/cloudprovider/providers/azure/azure_backoff.go b/pkg/cloudprovider/providers/azure/azure_backoff.go index 3cf5d8930fa..33fc9064095 100644 --- a/pkg/cloudprovider/providers/azure/azure_backoff.go +++ b/pkg/cloudprovider/providers/azure/azure_backoff.go @@ -243,23 +243,23 @@ func (az *Cloud) ListLBWithRetry() ([]network.LoadBalancer, error) { return allLBs, nil } -// ListPIPWithRetry list the PIP resources in az.ResourceGroup -func (az *Cloud) ListPIPWithRetry() ([]network.PublicIPAddress, error) { +// ListPIPWithRetry list the PIP resources in the given resource group +func (az *Cloud) ListPIPWithRetry(pipResourceGroup string) ([]network.PublicIPAddress, error) { allPIPs := []network.PublicIPAddress{} var result network.PublicIPAddressListResult err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { var retryErr error az.operationPollRateLimiter.Accept() - glog.V(10).Infof("PublicIPAddressesClient.List(%v): start", az.ResourceGroup) - result, retryErr = az.PublicIPAddressesClient.List(az.ResourceGroup) - glog.V(10).Infof("PublicIPAddressesClient.List(%v): end", az.ResourceGroup) + glog.V(10).Infof("PublicIPAddressesClient.List(%v): start", pipResourceGroup) + result, retryErr = az.PublicIPAddressesClient.List(pipResourceGroup) + glog.V(10).Infof("PublicIPAddressesClient.List(%v): end", pipResourceGroup) if retryErr != nil { glog.Errorf("PublicIPAddressesClient.List(%v) - backoff: failure, will retry,err=%v", - az.ResourceGroup, + pipResourceGroup, retryErr) return false, retryErr } - glog.V(2).Infof("PublicIPAddressesClient.List(%v) - backoff: success", az.ResourceGroup) + glog.V(2).Infof("PublicIPAddressesClient.List(%v) - backoff: success", pipResourceGroup) return true, nil }) if err != nil { @@ -271,21 +271,21 @@ func (az *Cloud) ListPIPWithRetry() ([]network.PublicIPAddress, error) { allPIPs = append(allPIPs, *result.Value...) appendResults = false - // follow the next link to get all the vms for resource group + // follow the next link to get all the pip resources for resource group if result.NextLink != nil { err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { var retryErr error az.operationPollRateLimiter.Accept() - glog.V(10).Infof("PublicIPAddressesClient.ListNextResults(%v): start", az.ResourceGroup) + glog.V(10).Infof("PublicIPAddressesClient.ListNextResults(%v): start", pipResourceGroup) result, retryErr = az.PublicIPAddressesClient.ListNextResults(result) - glog.V(10).Infof("PublicIPAddressesClient.ListNextResults(%v): end", az.ResourceGroup) + glog.V(10).Infof("PublicIPAddressesClient.ListNextResults(%v): end", pipResourceGroup) if retryErr != nil { glog.Errorf("PublicIPAddressesClient.ListNextResults(%v) - backoff: failure, will retry,err=%v", - az.ResourceGroup, + pipResourceGroup, retryErr) return false, retryErr } - glog.V(2).Infof("PublicIPAddressesClient.ListNextResults(%v) - backoff: success", az.ResourceGroup) + glog.V(2).Infof("PublicIPAddressesClient.ListNextResults(%v) - backoff: success", pipResourceGroup) return true, nil }) if err != nil { @@ -299,14 +299,14 @@ func (az *Cloud) ListPIPWithRetry() ([]network.PublicIPAddress, error) { } // CreateOrUpdatePIPWithRetry invokes az.PublicIPAddressesClient.CreateOrUpdate with exponential backoff retry -func (az *Cloud) CreateOrUpdatePIPWithRetry(pip network.PublicIPAddress) error { +func (az *Cloud) CreateOrUpdatePIPWithRetry(pipResourceGroup string, pip network.PublicIPAddress) error { return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { az.operationPollRateLimiter.Accept() - glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s): start", *pip.Name) - respChan, errChan := az.PublicIPAddressesClient.CreateOrUpdate(az.ResourceGroup, *pip.Name, pip, nil) + glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s, %s): start", pipResourceGroup, *pip.Name) + respChan, errChan := az.PublicIPAddressesClient.CreateOrUpdate(pipResourceGroup, *pip.Name, pip, nil) resp := <-respChan err := <-errChan - glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s): end", *pip.Name) + glog.V(10).Infof("PublicIPAddressesClient.CreateOrUpdate(%s, %s): end", pipResourceGroup, *pip.Name) return processRetryResponse(resp.Response, err) }) } @@ -325,14 +325,14 @@ func (az *Cloud) CreateOrUpdateInterfaceWithRetry(nic network.Interface) error { } // DeletePublicIPWithRetry invokes az.PublicIPAddressesClient.Delete with exponential backoff retry -func (az *Cloud) DeletePublicIPWithRetry(pipName string) error { +func (az *Cloud) DeletePublicIPWithRetry(pipResourceGroup string, pipName string) error { return wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) { az.operationPollRateLimiter.Accept() - glog.V(10).Infof("PublicIPAddressesClient.Delete(%s): start", pipName) - respChan, errChan := az.PublicIPAddressesClient.Delete(az.ResourceGroup, pipName, nil) + glog.V(10).Infof("PublicIPAddressesClient.Delete(%s, %s): start", pipResourceGroup, pipName) + respChan, errChan := az.PublicIPAddressesClient.Delete(pipResourceGroup, pipName, nil) resp := <-respChan err := <-errChan - glog.V(10).Infof("PublicIPAddressesClient.Delete(%s): end", pipName) + glog.V(10).Infof("PublicIPAddressesClient.Delete(%s, %s): end", pipResourceGroup, pipName) return processRetryResponse(resp, err) }) } diff --git a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go index 0ff061a495b..54e108fcc84 100644 --- a/pkg/cloudprovider/providers/azure/azure_loadbalancer.go +++ b/pkg/cloudprovider/providers/azure/azure_loadbalancer.go @@ -67,6 +67,10 @@ const ( ServiceAnnotationSharedSecurityRule = "service.beta.kubernetes.io/azure-shared-securityrule" ) +// ServiceAnnotationLoadBalancerResourceGroup is the annotation used on the service +// to specify the resource group of load balancer objects that are not in the same resource group as the cluster. +const ServiceAnnotationLoadBalancerResourceGroup = "service.beta.kubernetes.io/azure-load-balancer-resource-group" + // GetLoadBalancer returns whether the specified load balancer exists, and // if so, what its status is. func (az *Cloud) GetLoadBalancer(clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) { @@ -82,7 +86,7 @@ func (az *Cloud) GetLoadBalancer(clusterName string, service *v1.Service) (statu return status, true, nil } -func getPublicIPLabel(service *v1.Service) string { +func getPublicIPDomainNameLabel(service *v1.Service) string { if labelName, found := service.Annotations[ServiceAnnotationDNSLabelName]; found { return labelName } @@ -316,7 +320,7 @@ func (az *Cloud) getServiceLoadBalancerStatus(service *v1.Service, lb *network.L if err != nil { return nil, fmt.Errorf("get(%s): lb(%s) - failed to get LB PublicIPAddress Name from ID(%s)", serviceName, *lb.Name, *pipID) } - pip, existsPip, err := az.getPublicIPAddress(pipName) + pip, existsPip, err := az.getPublicIPAddress(az.getPublicIPAddressResourceGroup(service), pipName) if err != nil { return nil, err } @@ -338,7 +342,9 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service) return getPublicIPName(clusterName, service), nil } - pips, err := az.ListPIPWithRetry() + pipResourceGroup := az.getPublicIPAddressResourceGroup(service) + + pips, err := az.ListPIPWithRetry(pipResourceGroup) if err != nil { return "", err } @@ -349,7 +355,7 @@ func (az *Cloud) determinePublicIPName(clusterName string, service *v1.Service) return *pip.Name, nil } } - return "", fmt.Errorf("user supplied IP Address %s was not found", loadBalancerIP) + return "", fmt.Errorf("user supplied IP Address %s was not found in resource group %s", loadBalancerIP, pipResourceGroup) } func flipServiceInternalAnnotation(service *v1.Service) *v1.Service { @@ -386,8 +392,9 @@ func (az *Cloud) findServiceIPAddress(clusterName string, service *v1.Service, i return lbStatus.Ingress[0].IP, nil } -func (az *Cloud) ensurePublicIPExists(serviceName, pipName, domainNameLabel string) (*network.PublicIPAddress, error) { - pip, existsPip, err := az.getPublicIPAddress(pipName) +func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domainNameLabel string) (*network.PublicIPAddress, error) { + pipResourceGroup := az.getPublicIPAddressResourceGroup(service) + pip, existsPip, err := az.getPublicIPAddress(pipResourceGroup, pipName) if err != nil { return nil, err } @@ -395,6 +402,7 @@ func (az *Cloud) ensurePublicIPExists(serviceName, pipName, domainNameLabel stri return &pip, nil } + serviceName := getServiceName(service) pip.Name = to.StringPtr(pipName) pip.Location = to.StringPtr(az.Location) pip.PublicIPAddressPropertiesFormat = &network.PublicIPAddressPropertiesFormat{ @@ -408,18 +416,18 @@ func (az *Cloud) ensurePublicIPExists(serviceName, pipName, domainNameLabel stri pip.Tags = &map[string]*string{"service": &serviceName} glog.V(3).Infof("ensure(%s): pip(%s) - creating", serviceName, *pip.Name) az.operationPollRateLimiter.Accept() - glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%q): start", *pip.Name) - err = az.CreateOrUpdatePIPWithRetry(pip) + glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%s, %q): start", pipResourceGroup, *pip.Name) + err = az.CreateOrUpdatePIPWithRetry(pipResourceGroup, pip) if err != nil { glog.V(2).Infof("ensure(%s) abort backoff: pip(%s) - creating", serviceName, *pip.Name) return nil, err } - glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%q): end", *pip.Name) + glog.V(10).Infof("CreateOrUpdatePIPWithRetry(%s, %q): end", pipResourceGroup, *pip.Name) az.operationPollRateLimiter.Accept() - glog.V(10).Infof("PublicIPAddressesClient.Get(%q): start", *pip.Name) - pip, err = az.PublicIPAddressesClient.Get(az.ResourceGroup, *pip.Name, "") - glog.V(10).Infof("PublicIPAddressesClient.Get(%q): end", *pip.Name) + glog.V(10).Infof("PublicIPAddressesClient.Get(%s, %q): start", pipResourceGroup, *pip.Name) + pip, err = az.PublicIPAddressesClient.Get(pipResourceGroup, *pip.Name, "") + glog.V(10).Infof("PublicIPAddressesClient.Get(%s, %q): end", pipResourceGroup, *pip.Name) if err != nil { return nil, err } @@ -546,8 +554,8 @@ func (az *Cloud) reconcileLoadBalancer(clusterName string, service *v1.Service, if err != nil { return nil, err } - domainNameLabel := getPublicIPLabel(service) - pip, err := az.ensurePublicIPExists(serviceName, pipName, domainNameLabel) + domainNameLabel := getPublicIPDomainNameLabel(service) + pip, err := az.ensurePublicIPExists(service, pipName, domainNameLabel) if err != nil { return nil, err } @@ -1137,7 +1145,9 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, want } } - pips, err := az.ListPIPWithRetry() + pipResourceGroup := az.getPublicIPAddressResourceGroup(service) + + pips, err := az.ListPIPWithRetry(pipResourceGroup) if err != nil { return nil, err } @@ -1154,14 +1164,14 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, want } else { glog.V(2).Infof("ensure(%s): pip(%s) - deleting", serviceName, pipName) az.operationPollRateLimiter.Accept() - glog.V(10).Infof("DeletePublicIPWithRetry(%q): start", pipName) - err = az.DeletePublicIPWithRetry(pipName) + glog.V(10).Infof("DeletePublicIPWithRetry(%s, %q): start", pipResourceGroup, pipName) + err = az.DeletePublicIPWithRetry(pipResourceGroup, pipName) if err != nil { glog.V(2).Infof("ensure(%s) abort backoff: pip(%s) - deleting", serviceName, pipName) // We let err to pass through // It may be ignorable } - glog.V(10).Infof("DeletePublicIPWithRetry(%q): end", pipName) // response not read yet... + glog.V(10).Infof("DeletePublicIPWithRetry(%s, %q): end", pipResourceGroup, pipName) // response not read yet... err = ignoreStatusNotFoundFromError(err) if err != nil { @@ -1176,8 +1186,8 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, want if !isInternal && wantLb { // Confirm desired public ip resource exists var pip *network.PublicIPAddress - domainNameLabel := getPublicIPLabel(service) - if pip, err = az.ensurePublicIPExists(serviceName, desiredPipName, domainNameLabel); err != nil { + domainNameLabel := getPublicIPDomainNameLabel(service) + if pip, err = az.ensurePublicIPExists(service, desiredPipName, domainNameLabel); err != nil { return nil, err } return pip, nil @@ -1240,9 +1250,17 @@ func findSecurityRule(rules []network.SecurityRule, rule network.SecurityRule) b return false } +func (az *Cloud) getPublicIPAddressResourceGroup(service *v1.Service) string { + if resourceGroup, found := service.Annotations[ServiceAnnotationLoadBalancerResourceGroup]; found { + return resourceGroup + } + + return az.ResourceGroup +} + // Check if service requires an internal load balancer. func requiresInternalLoadBalancer(service *v1.Service) bool { - if l, ok := service.Annotations[ServiceAnnotationLoadBalancerInternal]; ok { + if l, found := service.Annotations[ServiceAnnotationLoadBalancerInternal]; found { return l == "true" } @@ -1251,7 +1269,7 @@ func requiresInternalLoadBalancer(service *v1.Service) bool { func subnet(service *v1.Service) *string { if requiresInternalLoadBalancer(service) { - if l, ok := service.Annotations[ServiceAnnotationLoadBalancerInternalSubnet]; ok { + if l, found := service.Annotations[ServiceAnnotationLoadBalancerInternalSubnet]; found { return &l } } diff --git a/pkg/cloudprovider/providers/azure/azure_wrap.go b/pkg/cloudprovider/providers/azure/azure_wrap.go index ec045fee007..a8a2a960d05 100644 --- a/pkg/cloudprovider/providers/azure/azure_wrap.go +++ b/pkg/cloudprovider/providers/azure/azure_wrap.go @@ -156,13 +156,17 @@ func (az *Cloud) listLoadBalancers() (lbListResult network.LoadBalancerListResul return lbListResult, exists, err } -func (az *Cloud) getPublicIPAddress(name string) (pip network.PublicIPAddress, exists bool, err error) { - var realErr error +func (az *Cloud) getPublicIPAddress(pipResourceGroup string, pipName string) (pip network.PublicIPAddress, exists bool, err error) { + resourceGroup := az.ResourceGroup + if pipResourceGroup != "" { + resourceGroup = pipResourceGroup + } + var realErr error az.operationPollRateLimiter.Accept() - glog.V(10).Infof("PublicIPAddressesClient.Get(%s): start", name) - pip, err = az.PublicIPAddressesClient.Get(az.ResourceGroup, name, "") - glog.V(10).Infof("PublicIPAddressesClient.Get(%s): end", name) + glog.V(10).Infof("PublicIPAddressesClient.Get(%s, %s): start", resourceGroup, pipName) + pip, err = az.PublicIPAddressesClient.Get(resourceGroup, pipName, "") + glog.V(10).Infof("PublicIPAddressesClient.Get(%s, %s): end", resourceGroup, pipName) exists, realErr = checkResourceExistsFromError(err) if realErr != nil {