Merge pull request #8831 from a-robinson/tp

Adjust the sleep time before tearing down e2e clusters and add a rough formula
This commit is contained in:
Tim Hockin 2015-05-27 12:42:49 -07:00
commit f69eb0af67
2 changed files with 11 additions and 4 deletions

View File

@ -224,8 +224,10 @@ fi
if [[ "${E2E_DOWN,,}" == "true" ]]; then if [[ "${E2E_DOWN,,}" == "true" ]]; then
# Sleep before deleting the cluster to give the controller manager time to # Sleep before deleting the cluster to give the controller manager time to
# delete any cloudprovider resources still around from the last test. # delete any cloudprovider resources still around from the last test.
# 60 seconds was chosen as an arbitrary bound for how long it should take # This is calibrated to allow enough time for 3 attempts to delete the
# to delete load balancer resources. # resources. Each attempt is allocated 5 seconds for requests to the
sleep 60 # cloudprovider plus the processingRetryInterval from servicecontroller.go
# for the wait between attempts.
sleep 30
go run ./hack/e2e.go ${E2E_OPT} -v --down go run ./hack/e2e.go ${E2E_OPT} -v --down
fi fi

View File

@ -38,6 +38,11 @@ import (
const ( const (
workerGoroutines = 10 workerGoroutines = 10
// How long to wait before retrying the processing of a service change.
// If this changes, the sleep in hack/jenkins/e2e.sh before downing a cluster
// should be changed appropriately.
processingRetryInterval = 5 * time.Second
clientRetryCount = 5 clientRetryCount = 5
clientRetryInterval = 5 * time.Second clientRetryInterval = 5 * time.Second
@ -151,7 +156,7 @@ func (s *ServiceController) watchServices(serviceQueue *cache.DeltaFIFO) {
if shouldRetry { if shouldRetry {
// Add the failed service back to the queue so we'll retry it. // Add the failed service back to the queue so we'll retry it.
glog.Errorf("Failed to process service delta. Retrying: %v", err) glog.Errorf("Failed to process service delta. Retrying: %v", err)
time.Sleep(5 * time.Second) time.Sleep(processingRetryInterval)
serviceQueue.AddIfNotPresent(deltas) serviceQueue.AddIfNotPresent(deltas)
} else if err != nil { } else if err != nil {
util.HandleError(fmt.Errorf("Failed to process service delta. Not retrying: %v", err)) util.HandleError(fmt.Errorf("Failed to process service delta. Not retrying: %v", err))