Ensure sync failures are correctly retried

Only track the last synced resources when all preceding steps have
completed to ensure that failures will be correctly retried.
This commit is contained in:
Dan Mace 2017-11-27 16:29:18 -05:00
parent d89f58fcc8
commit 9b2886df29

View File

@ -179,9 +179,8 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.DiscoveryInterface, p
return
}
// Something has changed, so track the new state and perform a sync.
// Something has changed, time to sync.
glog.V(2).Infof("syncing garbage collector with updated resources from discovery: %v", newResources)
oldResources = newResources
// Ensure workers are paused to avoid processing events before informers
// have resynced.
@ -208,7 +207,13 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.DiscoveryInterface, p
}
if !controller.WaitForCacheSync("garbage collector", stopCh, gc.dependencyGraphBuilder.IsSynced) {
utilruntime.HandleError(fmt.Errorf("timed out waiting for dependency graph builder sync during GC sync"))
return
}
// Finally, keep track of our new state. Do this after all preceding steps
// have succeeded to ensure we'll retry on subsequent syncs if an error
// occured.
oldResources = newResources
}, period, stopCh)
}