diff --git a/federation/pkg/federation-controller/service/endpoint_helper.go b/federation/pkg/federation-controller/service/endpoint_helper.go index 044f2d27623..96ecf08375d 100644 --- a/federation/pkg/federation-controller/service/endpoint_helper.go +++ b/federation/pkg/federation-controller/service/endpoint_helper.go @@ -120,27 +120,54 @@ func (cc *clusterClientCache) processEndpointDeletion(cachedService *cachedServi func (cc *clusterClientCache) processEndpointUpdate(cachedService *cachedService, endpoint *v1.Endpoints, clusterName string, serviceController *ServiceController) error { glog.V(4).Infof("Processing endpoint update for %s/%s, cluster %s", endpoint.Namespace, endpoint.Name, clusterName) cachedService.rwlock.Lock() + var reachable bool defer cachedService.rwlock.Unlock() - for _, subset := range endpoint.Subsets { - if len(subset.Addresses) > 0 { - cachedService.endpointMap[clusterName] = 1 - } - } _, ok := cachedService.endpointMap[clusterName] if !ok { - // first time get endpoints, update dns record - glog.V(4).Infof("Cached endpoint was not found for %s/%s, cluster %s, building one", endpoint.Namespace, endpoint.Name, clusterName) - cachedService.endpointMap[clusterName] = 1 - if err := serviceController.ensureDnsRecords(clusterName, cachedService); err != nil { - glog.V(4).Infof("Error ensuring DNS Records: %v", err) - for i := 0; i < clientRetryCount; i++ { - time.Sleep(cachedService.nextDNSUpdateDelay()) - err := serviceController.ensureDnsRecords(clusterName, cachedService) - if err == nil { - return nil - } + for _, subset := range endpoint.Subsets { + if len(subset.Addresses) > 0 { + reachable = true + break + } + } + if reachable { + // first time get endpoints, update dns record + glog.V(4).Infof("Reachable endpoint was found for %s/%s, cluster %s, building endpointMap", endpoint.Namespace, endpoint.Name, clusterName) + cachedService.endpointMap[clusterName] = 1 + if err := serviceController.ensureDnsRecords(clusterName, cachedService); err != nil { + glog.V(4).Infof("Error ensuring DNS Records: %v", err) + for i := 0; i < clientRetryCount; i++ { + time.Sleep(cachedService.nextDNSUpdateDelay()) + err := serviceController.ensureDnsRecords(clusterName, cachedService) + if err == nil { + return nil + } + } + return err + } + } + } else { + for _, subset := range endpoint.Subsets { + if len(subset.Addresses) > 0 { + reachable = true + break + } + } + if !reachable { + // first time get endpoints, update dns record + glog.V(4).Infof("Reachable endpoint was lost for %s/%s, cluster %s, deleting endpointMap", endpoint.Namespace, endpoint.Name, clusterName) + delete(cachedService.endpointMap, clusterName) + if err := serviceController.ensureDnsRecords(clusterName, cachedService); err != nil { + glog.V(4).Infof("Error ensuring DNS Records: %v", err) + for i := 0; i < clientRetryCount; i++ { + time.Sleep(cachedService.nextDNSUpdateDelay()) + err := serviceController.ensureDnsRecords(clusterName, cachedService) + if err == nil { + return nil + } + } + return err } - return err } } return nil diff --git a/federation/pkg/federation-controller/service/servicecontroller.go b/federation/pkg/federation-controller/service/servicecontroller.go index 32ea24a00bc..48882094f1e 100644 --- a/federation/pkg/federation-controller/service/servicecontroller.go +++ b/federation/pkg/federation-controller/service/servicecontroller.go @@ -76,7 +76,7 @@ type cachedService struct { appliedState *v1.Service // cluster endpoint map hold subset info from kubernetes clusters // key clusterName - // value is a flag that if there is ready address, 1 means there is ready address, 0 means no ready address + // value is a flag that if there is ready address, 1 means there is ready address endpointMap map[string]int // serviceStatusMap map holds service status info from kubernetes clusters, keyed on clusterName serviceStatusMap map[string]v1.LoadBalancerStatus @@ -101,7 +101,7 @@ type ServiceController struct { dns dnsprovider.Interface federationClient federation_release_1_3.Interface federationName string - zoneName string + zoneName string // each federation should be configured with a single zone (e.g. "mycompany.com") dnsZones dnsprovider.Zones serviceCache *serviceCache @@ -134,7 +134,7 @@ func New(federationClient federation_release_1_3.Interface, dns dnsprovider.Inte dns: dns, federationClient: federationClient, federationName: federationName, - zoneName: zoneName, + zoneName: zoneName, serviceCache: &serviceCache{fedServiceMap: make(map[string]*cachedService)}, clusterCache: &clusterClientCache{ rwlock: sync.Mutex{},