github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-28 22:17:14 +00:00
Code Issues Projects Releases Wiki Activity

Retry incrementing quota if there is a conflict

Browse Source
This commit is contained in:
derekwaynecarr 2015-05-01 13:31:51 -04:00
parent 72708d74b9
commit 6d08cc30ed
1 changed files with 52 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
plugin/pkg/admission/resourcequota/admission.go
View File

@ -19,7 +19,9 @@ package resourcequota
import ( import (
"fmt" "fmt"
"io" "io"
"math/rand"
"strings" "strings"
"time"
"github.com/GoogleCloudPlatform/kubernetes/pkg/admission" "github.com/GoogleCloudPlatform/kubernetes/pkg/admission"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@ -78,6 +80,13 @@ func (q *quota) Admit(a admission.Attributes) (err error) {
Name: "", Name: "",
}, },
} }
// concurrent operations that modify quota tracked resources can cause a conflict when incrementing usage
// as a result, we will attempt to increment quota usage per request up to numRetries limit
// we fuzz each retry with an interval period to attempt to improve end-user experience during concurrent operations
numRetries := 10
interval := time.Duration(rand.Int63n(90)+int64(10)) * time.Millisecond
items, err := q.indexer.Index("namespace", key) items, err := q.indexer.Index("namespace", key)
if err != nil { if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Unable to %s %s at this time because there was an error enforcing quota", a.GetOperation(), a.GetResource())) return admission.NewForbidden(a, fmt.Errorf("Unable to %s %s at this time because there was an error enforcing quota", a.GetOperation(), a.GetResource()))
@ -87,39 +96,54 @@ func (q *quota) Admit(a admission.Attributes) (err error) {
} }
for i := range items { for i := range items {
quota := items[i].(*api.ResourceQuota) quota := items[i].(*api.ResourceQuota)
// we cannot modify the value directly in the cache, so we copy for retry := 1; retry <= numRetries; retry++ {
status := &api.ResourceQuotaStatus{
Hard: api.ResourceList{},
Used: api.ResourceList{},
}
for k, v := range quota.Status.Hard {
status.Hard[k] = *v.Copy()
}
for k, v := range quota.Status.Used {
status.Used[k] = *v.Copy()
}
dirty, err := IncrementUsage(a, status, q.client) // we cannot modify the value directly in the cache, so we copy
if err != nil { status := &api.ResourceQuotaStatus{
return admission.NewForbidden(a, err) Hard: api.ResourceList{},
} Used: api.ResourceList{},
if dirty {
// construct a usage record
usage := api.ResourceQuota{
ObjectMeta: api.ObjectMeta{
Name: quota.Name,
Namespace: quota.Namespace,
ResourceVersion: quota.ResourceVersion,
Labels: quota.Labels,
Annotations: quota.Annotations},
} }
usage.Status = *status for k, v := range quota.Status.Hard {
_, err = q.client.ResourceQuotas(usage.Namespace).UpdateStatus(&usage) status.Hard[k] = *v.Copy()
}
for k, v := range quota.Status.Used {
status.Used[k] = *v.Copy()
}
dirty, err := IncrementUsage(a, status, q.client)
if err != nil { if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Unable to %s %s at this time because there was an error enforcing quota", a.GetOperation(), a.GetResource())) return admission.NewForbidden(a, err)
}
if dirty {
// construct a usage record
usage := api.ResourceQuota{
ObjectMeta: api.ObjectMeta{
Name: quota.Name,
Namespace: quota.Namespace,
ResourceVersion: quota.ResourceVersion,
Labels: quota.Labels,
Annotations: quota.Annotations},
}
usage.Status = *status
_, err = q.client.ResourceQuotas(usage.Namespace).UpdateStatus(&usage)
if err == nil {
break
}
// we have concurrent requests to update quota, so look to retry if needed
if retry == numRetries {
return admission.NewForbidden(a, fmt.Errorf("Unable to %s %s at this time because there are too many concurrent requests to increment quota", a.GetOperation(), a.GetResource()))
}
time.Sleep(interval)
// manually get the latest quota
quota, err = q.client.ResourceQuotas(usage.Namespace).Get(quota.Name)
if err != nil {
return admission.NewForbidden(a, err)
}
} }
} }
} }