github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-04 18:00:08 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #7633 from derekwaynecarr/retry_quota

Browse Source 
Retry incrementing quota if there is a conflict
This commit is contained in:
Eric Tune 2015-05-01 15:41:24 -07:00
commit fae435c127
1 changed files with 52 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
plugin/pkg/admission/resourcequota/admission.go
View File

@ -19,7 +19,9 @@ package resourcequota
import ( import (
"fmt" "fmt"
"io" "io"
"math/rand"
"strings" "strings"
"time"
"github.com/GoogleCloudPlatform/kubernetes/pkg/admission" "github.com/GoogleCloudPlatform/kubernetes/pkg/admission"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@ -78,6 +80,13 @@ func (q *quota) Admit(a admission.Attributes) (err error) {
Name: "", Name: "",
}, },
} }
// concurrent operations that modify quota tracked resources can cause a conflict when incrementing usage
// as a result, we will attempt to increment quota usage per request up to numRetries limit
// we fuzz each retry with an interval period to attempt to improve end-user experience during concurrent operations
numRetries := 10
interval := time.Duration(rand.Int63n(90)+int64(10)) * time.Millisecond
items, err := q.indexer.Index("namespace", key) items, err := q.indexer.Index("namespace", key)
if err != nil { if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Unable to %s %s at this time because there was an error enforcing quota", a.GetOperation(), a.GetResource())) return admission.NewForbidden(a, fmt.Errorf("Unable to %s %s at this time because there was an error enforcing quota", a.GetOperation(), a.GetResource()))
@ -87,8 +96,11 @@ func (q *quota) Admit(a admission.Attributes) (err error) {
} }
for i := range items { for i := range items {
quota := items[i].(*api.ResourceQuota) quota := items[i].(*api.ResourceQuota)
for retry := 1; retry <= numRetries; retry++ {
// we cannot modify the value directly in the cache, so we copy // we cannot modify the value directly in the cache, so we copy
status := &api.ResourceQuotaStatus{ status := &api.ResourceQuotaStatus{
Hard: api.ResourceList{}, Hard: api.ResourceList{},
@ -118,8 +130,20 @@ func (q *quota) Admit(a admission.Attributes) (err error) {
} }
usage.Status = *status usage.Status = *status
_, err = q.client.ResourceQuotas(usage.Namespace).UpdateStatus(&usage) _, err = q.client.ResourceQuotas(usage.Namespace).UpdateStatus(&usage)
if err == nil {
break
}
// we have concurrent requests to update quota, so look to retry if needed
if retry == numRetries {
return admission.NewForbidden(a, fmt.Errorf("Unable to %s %s at this time because there are too many concurrent requests to increment quota", a.GetOperation(), a.GetResource()))
}
time.Sleep(interval)
// manually get the latest quota
quota, err = q.client.ResourceQuotas(usage.Namespace).Get(quota.Name)
if err != nil { if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Unable to %s %s at this time because there was an error enforcing quota", a.GetOperation(), a.GetResource())) return admission.NewForbidden(a, err)
}
} }
} }
} }