diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index a2f132434ac..f642a328cce 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -224,8 +224,10 @@ fi if [[ "${E2E_DOWN,,}" == "true" ]]; then # Sleep before deleting the cluster to give the controller manager time to # delete any cloudprovider resources still around from the last test. - # 60 seconds was chosen as an arbitrary bound for how long it should take - # to delete load balancer resources. - sleep 60 + # This is calibrated to allow enough time for 3 attempts to delete the + # resources. Each attempt is allocated 5 seconds for requests to the + # cloudprovider plus the processingRetryInterval from servicecontroller.go + # for the wait between attempts. + sleep 30 go run ./hack/e2e.go ${E2E_OPT} -v --down fi diff --git a/pkg/cloudprovider/servicecontroller/servicecontroller.go b/pkg/cloudprovider/servicecontroller/servicecontroller.go index 9a6657d3ca7..1e2863f60a4 100644 --- a/pkg/cloudprovider/servicecontroller/servicecontroller.go +++ b/pkg/cloudprovider/servicecontroller/servicecontroller.go @@ -38,6 +38,11 @@ import ( const ( workerGoroutines = 10 + // How long to wait before retrying the processing of a service change. + // If this changes, the sleep in hack/jenkins/e2e.sh before downing a cluster + // should be changed appropriately. + processingRetryInterval = 5 * time.Second + clientRetryCount = 5 clientRetryInterval = 5 * time.Second @@ -151,7 +156,7 @@ func (s *ServiceController) watchServices(serviceQueue *cache.DeltaFIFO) { if shouldRetry { // Add the failed service back to the queue so we'll retry it. glog.Errorf("Failed to process service delta. Retrying: %v", err) - time.Sleep(5 * time.Second) + time.Sleep(processingRetryInterval) serviceQueue.AddIfNotPresent(deltas) } else if err != nil { util.HandleError(fmt.Errorf("Failed to process service delta. Not retrying: %v", err))