From e691f7d493b9df24fab8a4936f33200b2c71e18f Mon Sep 17 00:00:00 2001 From: Ruslan Gustomiasov Date: Fri, 31 May 2019 23:00:33 +0200 Subject: [PATCH 1/4] change aws encryptedCheck to exponential backoff --- .../k8s.io/legacy-cloud-providers/aws/aws.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go index d74f89c7e15..87d7c4ccc29 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go @@ -225,12 +225,13 @@ const ( createTagFactor = 2.0 createTagSteps = 9 - // encryptedCheck* is configuration of poll for created volume to check + // encryptedCheck* is configuration of exponential backoff for created volume to check // it has not been silently removed by AWS. // On a random AWS account (shared among several developers) it took 4s on - // average. - encryptedCheckInterval = 1 * time.Second - encryptedCheckTimeout = 30 * time.Second + // average, 8s max. + encryptedCheckInitialDelay = 1 * time.Second + encryptedCheckFactor = 2.0 + encryptedCheckSteps = 8 // Number of node names that can be added to a filter. The AWS limit is 200 // but we are using a lower limit on purpose @@ -2419,8 +2420,12 @@ func (c *Cloud) waitUntilVolumeAvailable(volumeName KubernetesVolumeID) error { // Unreachable code return err } - - err = wait.Poll(encryptedCheckInterval, encryptedCheckTimeout, func() (done bool, err error) { + backoff := wait.Backoff{ + Duration: encryptedCheckInitialDelay, + Factor: encryptedCheckFactor, + Steps: encryptedCheckSteps, + } + err = wait.ExponentialBackoff(backoff, func() (done bool, err error) { vol, err := disk.describeVolume() if err != nil { return true, err From 98dd1336b0abae25f56c48be64f3c1f9053741e5 Mon Sep 17 00:00:00 2001 From: Ruslan Gustomiasov Date: Sat, 8 Jun 2019 20:10:05 +0200 Subject: [PATCH 2/4] rename and change aws backoff vars --- staging/src/k8s.io/legacy-cloud-providers/aws/aws.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go index 87d7c4ccc29..b2f9a0ff318 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go @@ -229,9 +229,9 @@ const ( // it has not been silently removed by AWS. // On a random AWS account (shared among several developers) it took 4s on // average, 8s max. - encryptedCheckInitialDelay = 1 * time.Second - encryptedCheckFactor = 2.0 - encryptedCheckSteps = 8 + volumeCreateInitialDelay = 5 * time.Second + volumeCreateBackoffFactor = 1.2 + volumeCreateBackoffSteps = 10 // Number of node names that can be added to a filter. The AWS limit is 200 // but we are using a lower limit on purpose @@ -2421,9 +2421,9 @@ func (c *Cloud) waitUntilVolumeAvailable(volumeName KubernetesVolumeID) error { return err } backoff := wait.Backoff{ - Duration: encryptedCheckInitialDelay, - Factor: encryptedCheckFactor, - Steps: encryptedCheckSteps, + Duration: volumeCreateInitialDelay, + Factor: volumeCreateBackoffFactor, + Steps: volumeCreateBackoffSteps, } err = wait.ExponentialBackoff(backoff, func() (done bool, err error) { vol, err := disk.describeVolume() From ee5d623fe6ce7d1e29c4ab9a271ce22eb31dc806 Mon Sep 17 00:00:00 2001 From: Ruslan Gustomiasov Date: Sat, 8 Jun 2019 20:11:38 +0200 Subject: [PATCH 3/4] fix volumecreate comments --- staging/src/k8s.io/legacy-cloud-providers/aws/aws.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go index b2f9a0ff318..01674464298 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go @@ -225,7 +225,7 @@ const ( createTagFactor = 2.0 createTagSteps = 9 - // encryptedCheck* is configuration of exponential backoff for created volume to check + // volumeCreate* is configuration of exponential backoff for created volume to check // it has not been silently removed by AWS. // On a random AWS account (shared among several developers) it took 4s on // average, 8s max. From 2eeb863da80c6e4b64a80c2a36fb877bff5066fe Mon Sep 17 00:00:00 2001 From: Ruslan Gustomiasov Date: Mon, 24 Jun 2019 19:58:46 +0200 Subject: [PATCH 4/4] add sleep 5 before exponential backoff in waitUntilVolumeAvailable --- staging/src/k8s.io/legacy-cloud-providers/aws/aws.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go index 01674464298..d06a5f2cf58 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go @@ -225,8 +225,7 @@ const ( createTagFactor = 2.0 createTagSteps = 9 - // volumeCreate* is configuration of exponential backoff for created volume to check - // it has not been silently removed by AWS. + // volumeCreate* is configuration of exponential backoff for created volume. // On a random AWS account (shared among several developers) it took 4s on // average, 8s max. volumeCreateInitialDelay = 5 * time.Second @@ -2420,6 +2419,7 @@ func (c *Cloud) waitUntilVolumeAvailable(volumeName KubernetesVolumeID) error { // Unreachable code return err } + time.Sleep(5 * time.Second) backoff := wait.Backoff{ Duration: volumeCreateInitialDelay, Factor: volumeCreateBackoffFactor,