Merge pull request #58394 from deads2k/controller-08-redeliver

Automatic merge from submit-queue (batch tested with PRs 58412, 56132, 58506, 58542, 58394). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

don't stop informer delivery on error

If an informer delivery fails today, we stop delivering to it entirely.  The pull updates the code to skip that particular notification, delay, and continue delivery with the next time.

/assign derekwaynecarr
/assign ncdc
/assign ash2k

@derekwaynecarr This would change the "the controller isn't doing anything?!" to "the controller missed my (individual) resource!"

```release-note
NONE
```

Kubernetes-commit: 71426ba59fd4a37e5da7deac6298ab33101bb5b6

Godeps/Godeps.json

@@ -400,215 +400,215 @@
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/equality",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/errors",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/meta",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/resource",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/testing",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/testing/fuzzer",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/testing/roundtrip",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apimachinery",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apimachinery/announced",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apimachinery/registered",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/internalversion",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1alpha1",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/conversion",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/conversion/queryparams",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/fields",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/labels",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/schema",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/json",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/protobuf",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/recognizer",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/streaming",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/versioning",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/selection",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/types",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/cache",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/clock",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/diff",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/errors",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/framer",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream/spdy",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/intstr",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/json",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/mergepatch",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/net",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/remotecommand",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/runtime",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/sets",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/strategicpatch",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/validation",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/validation/field",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/wait",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/yaml",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/version",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/watch",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/json",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/netutil",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect",
-			"Rev": "baafc3365acf1aa0c6c9c9aa169340d9fc2d3c6c"
+			"Rev": "2f1e02d3e57b8fb5206c5326bcb65217edc63a8e"
 		},
 		{
 			"ImportPath": "k8s.io/kube-openapi/pkg/util/proto",

tools/cache/BUILD

@@ -83,6 +83,7 @@ go_library(
         "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/tools/pager:go_default_library",
         "//vendor/k8s.io/client-go/util/buffer:go_default_library",
+        "//vendor/k8s.io/client-go/util/retry:go_default_library",
     ],
 )
 

tools/cache/shared_informer.go

@@ -26,6 +26,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/buffer"
+	"k8s.io/client-go/util/retry"
 
 	"github.com/golang/glog"
 )
@@ -540,8 +541,14 @@ func (p *processorListener) pop() {
 }
 
 func (p *processorListener) run() {
-	defer utilruntime.HandleCrash()
+	// this call blocks until the channel is closed.  When a panic happens during the notification
-
+	// we will catch it, **the offending item will be skipped!**, and after a short delay (one second)
+	// the next notification will be attempted.  This is usually better than the alternative of never
+	// delivering again.
+	stopCh := make(chan struct{})
+	wait.Until(func() {
+		// this gives us a few quick retries before a long pause and then a few more quick retries
+		err := wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) {
 			for next := range p.nextCh {
 				switch notification := next.(type) {
 				case updateNotification:
@@ -554,6 +561,15 @@ func (p *processorListener) run() {
 					utilruntime.HandleError(fmt.Errorf("unrecognized notification: %#v", next))
 				}
 			}
+			// the only way to get here is if the p.nextCh is empty and closed
+			return true, nil
+		})
+
+		// the only way to get here is if the p.nextCh is empty and closed
+		if err == nil {
+			close(stopCh)
+		}
+	}, 1*time.Minute, stopCh)
 }
 
 // shouldResync deterimines if the listener needs a resync. If the listener's resyncPeriod is 0,