Merge pull request #73937 from smarterclayton/report_errors

Report a watch error instead of eating it when we can't decode

Kubernetes-commit: c28b3b1fdd0d00972461653ac33599ffe5eccb2e

Godeps/Godeps.json

@@ -420,207 +420,207 @@
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/apitesting",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/apitesting/fuzzer",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/apitesting/roundtrip",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/equality",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/errors",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/meta",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/resource",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/fuzzer",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/internalversion",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/apis/meta/v1beta1",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/conversion",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/conversion/queryparams",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/fields",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/labels",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/schema",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/json",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/protobuf",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/recognizer",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/streaming",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/runtime/serializer/versioning",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/selection",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/types",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/cache",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/clock",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/diff",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/errors",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/framer",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/httpstream/spdy",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/intstr",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/json",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/mergepatch",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/naming",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/net",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/remotecommand",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/runtime",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/sets",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/strategicpatch",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/validation",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/validation/field",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/wait",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/util/yaml",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/version",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/pkg/watch",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/json",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/netutil",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/apimachinery/third_party/forked/golang/reflect",
-			"Rev": "4ceb6b6c5db56a2f8f454dd837c07160a3d6e131"
+			"Rev": "e508a7b04a89f09abafe336772f800bddedb9584"
 		},
 		{
 			"ImportPath": "k8s.io/klog",

rest/request.go

@@ -595,7 +595,12 @@ func (r *Request) WatchWithSpecificDecoders(wrapperDecoderFn func(io.ReadCloser)
 		return nil, fmt.Errorf("for request %s, got status: %v", url, resp.StatusCode)
 	}
 	wrapperDecoder := wrapperDecoderFn(resp.Body)
-	return watch.NewStreamWatcher(restclientwatch.NewDecoder(wrapperDecoder, embeddedDecoder)), nil
+	return watch.NewStreamWatcher(
+		restclientwatch.NewDecoder(wrapperDecoder, embeddedDecoder),
+		// use 500 to indicate that the cause of the error is unknown - other error codes
+		// are more specific to HTTP interactions, and set a reason
+		errors.NewClientErrorReporter(http.StatusInternalServerError, r.verb, "ClientWatchDecoding"),
+	), nil
 }
 
 // updateURLMetrics is a convenience function for pushing metrics.

rest/request_test.go

@@ -37,7 +37,7 @@ import (
 
 	"k8s.io/klog"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -879,9 +879,17 @@ func TestTransformUnstructuredError(t *testing.T) {
 	}
 }
 
+type errorReader struct {
+	err error
+}
+
+func (r errorReader) Read(data []byte) (int, error) { return 0, r.err }
+func (r errorReader) Close() error                  { return nil }
+
 func TestRequestWatch(t *testing.T) {
 	testCases := []struct {
 		Request *Request
+		Expect  []watch.Event
 		Err     bool
 		ErrFn   func(error) bool
 		Empty   bool
@@ -903,6 +911,40 @@ func TestRequestWatch(t *testing.T) {
 			},
 			Err: true,
 		},
+		{
+			Request: &Request{
+				content:     defaultContentConfig(),
+				serializers: defaultSerializers(t),
+				client: clientFunc(func(req *http.Request) (*http.Response, error) {
+					resp := &http.Response{StatusCode: http.StatusOK, Body: errorReader{err: errors.New("test error")}}
+					return resp, nil
+				}),
+				baseURL: &url.URL{},
+			},
+			Expect: []watch.Event{
+				{
+					Type: watch.Error,
+					Object: &metav1.Status{
+						Status:  "Failure",
+						Code:    500,
+						Reason:  "InternalError",
+						Message: `an error on the server ("unable to decode an event from the watch stream: test error") has prevented the request from succeeding`,
+						Details: &metav1.StatusDetails{
+							Causes: []metav1.StatusCause{
+								{
+									Type:    "UnexpectedServerResponse",
+									Message: "unable to decode an event from the watch stream: test error",
+								},
+								{
+									Type:    "ClientWatchDecoding",
+									Message: "unable to decode an event from the watch stream: test error",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 		{
 			Request: &Request{
 				content:     defaultContentConfig(),
@@ -999,20 +1041,18 @@ func TestRequestWatch(t *testing.T) {
 		},
 	}
 	for i, testCase := range testCases {
-		t.Logf("testcase %v", testCase.Request)
+		t.Run("", func(t *testing.T) {
 			testCase.Request.backoffMgr = &NoBackoff{}
 			watch, err := testCase.Request.Watch()
 			hasErr := err != nil
 			if hasErr != testCase.Err {
-			t.Errorf("%d: expected %t, got %t: %v", i, testCase.Err, hasErr, err)
+				t.Fatalf("%d: expected %t, got %t: %v", i, testCase.Err, hasErr, err)
-			continue
 			}
 			if testCase.ErrFn != nil && !testCase.ErrFn(err) {
 				t.Errorf("%d: error not valid: %v", i, err)
 			}
 			if hasErr && watch != nil {
-			t.Errorf("%d: watch should be nil when error is returned", i)
+				t.Fatalf("%d: watch should be nil when error is returned", i)
-			continue
 			}
 			if testCase.Empty {
 				_, ok := <-watch.ResultChan()
@@ -1020,6 +1060,18 @@ func TestRequestWatch(t *testing.T) {
 					t.Errorf("%d: expected the watch to be empty: %#v", i, watch)
 				}
 			}
+			if testCase.Expect != nil {
+				for i, evt := range testCase.Expect {
+					out, ok := <-watch.ResultChan()
+					if !ok {
+						t.Fatalf("Watch closed early, %d/%d read", i, len(testCase.Expect))
+					}
+					if !reflect.DeepEqual(evt, out) {
+						t.Fatalf("Event %d does not match: %s", i, diff.ObjectReflectDiff(evt, out))
+					}
+				}
+			}
+		})
 	}
 }