Merge pull request #58394 from deads2k/controller-08-redeliver

Automatic merge from submit-queue (batch tested with PRs 58412, 56132, 58506, 58542, 58394). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

don't stop informer delivery on error

If an informer delivery fails today, we stop delivering to it entirely.  The pull updates the code to skip that particular notification, delay, and continue delivery with the next time.

/assign derekwaynecarr
/assign ncdc
/assign ash2k

@derekwaynecarr This would change the "the controller isn't doing anything?!" to "the controller missed my (individual) resource!"

```release-note
NONE
```

Kubernetes-commit: 71426ba59fd4a37e5da7deac6298ab33101bb5b6
This commit is contained in:
Kubernetes Publisher 2018-01-22 22:57:47 -08:00
commit b044414c7d
3 changed files with 82 additions and 65 deletions

106
Godeps/Godeps.json generated
View File

@ -400,215 +400,215 @@
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/equality",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/errors",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/meta",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/resource",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/testing",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/testing/fuzzer",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/api/testing/roundtrip",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apimachinery",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apimachinery/announced",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/internalversion",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1alpha1",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/conversion",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/conversion/queryparams",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/fields",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/labels",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/schema",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/json",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/protobuf",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/recognizer",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/streaming",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/versioning",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/selection",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/types",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/cache",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/clock",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/diff",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/errors",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/framer",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream/spdy",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/intstr",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/json",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/mergepatch",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/net",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/remotecommand",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/runtime",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/sets",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/strategicpatch",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/validation",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/validation/field",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/wait",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/util/yaml",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/version",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/pkg/watch",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/json",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/netutil",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect",
"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
},
{
"ImportPath": "k8s.io/kube-openapi/pkg/util/proto",

1
tools/cache/BUILD vendored
View File

@ -83,6 +83,7 @@ go_library(
"//vendor/k8s.io/client-go/rest:go_default_library",
"//vendor/k8s.io/client-go/tools/pager:go_default_library",
"//vendor/k8s.io/client-go/util/buffer:go_default_library",
"//vendor/k8s.io/client-go/util/retry:go_default_library",
],
)

View File

@ -26,6 +26,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/buffer"
"k8s.io/client-go/util/retry"
"github.com/golang/glog"
)
@ -540,20 +541,35 @@ func (p *processorListener) pop() {
}
func (p *processorListener) run() {
defer utilruntime.HandleCrash()
// this call blocks until the channel is closed. When a panic happens during the notification
// we will catch it, **the offending item will be skipped!**, and after a short delay (one second)
// the next notification will be attempted. This is usually better than the alternative of never
// delivering again.
stopCh := make(chan struct{})
wait.Until(func() {
// this gives us a few quick retries before a long pause and then a few more quick retries
err := wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) {
for next := range p.nextCh {
switch notification := next.(type) {
case updateNotification:
p.handler.OnUpdate(notification.oldObj, notification.newObj)
case addNotification:
p.handler.OnAdd(notification.newObj)
case deleteNotification:
p.handler.OnDelete(notification.oldObj)
default:
utilruntime.HandleError(fmt.Errorf("unrecognized notification: %#v", next))
}
}
// the only way to get here is if the p.nextCh is empty and closed
return true, nil
})
for next := range p.nextCh {
switch notification := next.(type) {
case updateNotification:
p.handler.OnUpdate(notification.oldObj, notification.newObj)
case addNotification:
p.handler.OnAdd(notification.newObj)
case deleteNotification:
p.handler.OnDelete(notification.oldObj)
default:
utilruntime.HandleError(fmt.Errorf("unrecognized notification: %#v", next))
// the only way to get here is if the p.nextCh is empty and closed
if err == nil {
close(stopCh)
}
}
}, 1*time.Minute, stopCh)
}
// shouldResync deterimines if the listener needs a resync. If the listener's resyncPeriod is 0,