Update dependencies to v0.26.3 tag

Merge pull request #116437 from seans3/automated-cherry-pick-of-#116145-#115865-origin-release-1.26
Automated cherry pick of #116145: Plumb stale GroupVersions through aggregated discovery #115865: Removes old discovery hack ignoring 403 and 404 Kubernetes-commit: 052842af4ddb7f01ed8c7d248b59e80ad28175ad
2026-05-15 11:43:33 +00:00 · 2023-03-17 21:01:42 +00:00 · 2023-03-10 14:34:48 -08:00 · 2023-02-17 12:47:05 -08:00 · 2023-02-28 19:44:34 +00:00 · 2023-03-08 23:24:00 -08:00
15 changed files with 1083 additions and 364 deletions
--- a/applyconfigurations/internal/internal.go
+++ b/applyconfigurations/internal/internal.go
@@ -6553,6 +6553,8 @@ var schemaYAML = typed.YAMLObject(`types:
          elementType:
            namedType: io.k8s.api.core.v1.ResourceClaim
          elementRelationship: associative
+          keys:
+          - name
    - name: limits
      type:
        map:
@@ -11659,6 +11661,8 @@ var schemaYAML = typed.YAMLObject(`types:
          elementType:
            namedType: io.k8s.api.resource.v1alpha1.ResourceClaimConsumerReference
          elementRelationship: associative
+          keys:
+          - uid
 - name: io.k8s.api.resource.v1alpha1.ResourceClaimTemplate
  map:
    fields:
--- a/discovery/aggregated_discovery.go
+++ b/discovery/aggregated_discovery.go
@@ -24,19 +24,36 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )

+// StaleGroupVersionError encasulates failed GroupVersion marked "stale"
+// in the returned AggregatedDiscovery format.
+type StaleGroupVersionError struct {
+	gv schema.GroupVersion
+}
+
+func (s StaleGroupVersionError) Error() string {
+	return fmt.Sprintf("stale GroupVersion discovery: %v", s.gv)
+}
+
 // SplitGroupsAndResources transforms "aggregated" discovery top-level structure into
 // the previous "unaggregated" discovery groups and resources.
-func SplitGroupsAndResources(aggregatedGroups apidiscovery.APIGroupDiscoveryList) (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList) {
+func SplitGroupsAndResources(aggregatedGroups apidiscovery.APIGroupDiscoveryList) (
+	*metav1.APIGroupList,
+	map[schema.GroupVersion]*metav1.APIResourceList,
+	map[schema.GroupVersion]error) {
 	// Aggregated group list will contain the entirety of discovery, including
-	// groups, versions, and resources.
+	// groups, versions, and resources. GroupVersions marked "stale" are failed.
 	groups := []*metav1.APIGroup{}
+	failedGVs := map[schema.GroupVersion]error{}
 	resourcesByGV := map[schema.GroupVersion]*metav1.APIResourceList{}
 	for _, aggGroup := range aggregatedGroups.Items {
-		group, resources := convertAPIGroup(aggGroup)
+		group, resources, failed := convertAPIGroup(aggGroup)
 		groups = append(groups, group)
 		for gv, resourceList := range resources {
 			resourcesByGV[gv] = resourceList
 		}
+		for gv, err := range failed {
+			failedGVs[gv] = err
+		}
 	}
 	// Transform slice of groups to group list before returning.
 	groupList := &metav1.APIGroupList{}
@@ -44,23 +61,32 @@ func SplitGroupsAndResources(aggregatedGroups apidiscovery.APIGroupDiscoveryList
 	for _, group := range groups {
 		groupList.Groups = append(groupList.Groups, *group)
 	}
-	return groupList, resourcesByGV
+	return groupList, resourcesByGV, failedGVs
 }

 // convertAPIGroup tranforms an "aggregated" APIGroupDiscovery to an "legacy" APIGroup,
 // also returning the map of APIResourceList for resources within GroupVersions.
-func convertAPIGroup(g apidiscovery.APIGroupDiscovery) (*metav1.APIGroup, map[schema.GroupVersion]*metav1.APIResourceList) {
+func convertAPIGroup(g apidiscovery.APIGroupDiscovery) (
+	*metav1.APIGroup,
+	map[schema.GroupVersion]*metav1.APIResourceList,
+	map[schema.GroupVersion]error) {
 	// Iterate through versions to convert to group and resources.
 	group := &metav1.APIGroup{}
 	gvResources := map[schema.GroupVersion]*metav1.APIResourceList{}
+	failedGVs := map[schema.GroupVersion]error{}
 	group.Name = g.ObjectMeta.Name
-	for i, v := range g.Versions {
-		version := metav1.GroupVersionForDiscovery{}
+	for _, v := range g.Versions {
 		gv := schema.GroupVersion{Group: g.Name, Version: v.Version}
+		if v.Freshness == apidiscovery.DiscoveryFreshnessStale {
+			failedGVs[gv] = StaleGroupVersionError{gv: gv}
+			continue
+		}
+		version := metav1.GroupVersionForDiscovery{}
 		version.GroupVersion = gv.String()
 		version.Version = v.Version
 		group.Versions = append(group.Versions, version)
-		if i == 0 {
+		// PreferredVersion is first non-stale Version
+		if group.PreferredVersion == (metav1.GroupVersionForDiscovery{}) {
 			group.PreferredVersion = version
 		}
 		resourceList := &metav1.APIResourceList{}
@@ -76,7 +102,7 @@ func convertAPIGroup(g apidiscovery.APIGroupDiscovery) (*metav1.APIGroup, map[sc
 		}
 		gvResources[gv] = resourceList
 	}
-	return group, gvResources
+	return group, gvResources, failedGVs
 }

 // convertAPIResource tranforms a APIResourceDiscovery to an APIResource.
--- a/discovery/aggregated_discovery_test.go
+++ b/discovery/aggregated_discovery_test.go
@@ -31,6 +31,7 @@ func TestSplitGroupsAndResources(t *testing.T) {
 		agg                 apidiscovery.APIGroupDiscoveryList
 		expectedGroups      metav1.APIGroupList
 		expectedGVResources map[schema.GroupVersion]*metav1.APIResourceList
+		expectedFailedGVs   map[schema.GroupVersion]error
 	}{
 		{
 			name: "Aggregated discovery: core/v1 group and pod resource",
@@ -90,6 +91,7 @@ func TestSplitGroupsAndResources(t *testing.T) {
 					},
 				},
 			},
+			expectedFailedGVs: map[schema.GroupVersion]error{},
 		},
 		{
 			name: "Aggregated discovery: 1 group/1 resources at /api, 1 group/2 versions/1 resources at /apis",
@@ -179,6 +181,7 @@ func TestSplitGroupsAndResources(t *testing.T) {
 					},
 				},
 			},
+			expectedFailedGVs: map[schema.GroupVersion]error{},
 		},
 		{
 			name: "Aggregated discovery: 1 group/2 resources at /api, 1 group/2 resources at /apis",
@@ -313,6 +316,7 @@ func TestSplitGroupsAndResources(t *testing.T) {
 					},
 				},
 			},
+			expectedFailedGVs: map[schema.GroupVersion]error{},
 		},
 		{
 			name: "Aggregated discovery: multiple groups with cluster-scoped resources",
@@ -447,6 +451,7 @@ func TestSplitGroupsAndResources(t *testing.T) {
 					},
 				},
 			},
+			expectedFailedGVs: map[schema.GroupVersion]error{},
 		},
 		{
 			name: "Aggregated discovery with single subresource",
@@ -534,6 +539,7 @@ func TestSplitGroupsAndResources(t *testing.T) {
 					},
 				},
 			},
+			expectedFailedGVs: map[schema.GroupVersion]error{},
 		},
 		{
 			name: "Aggregated discovery with multiple subresources",
@@ -633,11 +639,185 @@ func TestSplitGroupsAndResources(t *testing.T) {
 					},
 				},
 			},
+			expectedFailedGVs: map[schema.GroupVersion]error{},
+		},
+		{
+			name: "Aggregated discovery: single failed GV at /api",
+			agg: apidiscovery.APIGroupDiscoveryList{
+				Items: []apidiscovery.APIGroupDiscovery{
+					{
+						Versions: []apidiscovery.APIVersionDiscovery{
+							{
+								Version: "v1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "pods",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "",
+											Version: "v1",
+											Kind:    "Pod",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "services",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "",
+											Version: "v1",
+											Kind:    "Service",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
+						},
+					},
+				},
+			},
+			// Single core Group/Version is stale, so no Version within Group.
+			expectedGroups: metav1.APIGroupList{
+				Groups: []metav1.APIGroup{{Name: ""}},
+			},
+			// Single core Group/Version is stale, so there are no expected resources.
+			expectedGVResources: map[schema.GroupVersion]*metav1.APIResourceList{},
+			expectedFailedGVs: map[schema.GroupVersion]error{
+				{Group: "", Version: "v1"}: StaleGroupVersionError{gv: schema.GroupVersion{Group: "", Version: "v1"}},
+			},
+		},
+		{
+			name: "Aggregated discovery: single failed GV at /apis",
+			agg: apidiscovery.APIGroupDiscoveryList{
+				Items: []apidiscovery.APIGroupDiscovery{
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "apps",
+						},
+						Versions: []apidiscovery.APIVersionDiscovery{
+							{
+								Version: "v1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "statefulsets",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1",
+											Kind:    "StatefulSets",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
+						},
+					},
+				},
+			},
+			// Single apps/v1 Group/Version is stale, so no Version within Group.
+			expectedGroups: metav1.APIGroupList{
+				Groups: []metav1.APIGroup{{Name: "apps"}},
+			},
+			// Single apps/v1 Group/Version is stale, so there are no expected resources.
+			expectedGVResources: map[schema.GroupVersion]*metav1.APIResourceList{},
+			expectedFailedGVs: map[schema.GroupVersion]error{
+				{Group: "apps", Version: "v1"}: StaleGroupVersionError{gv: schema.GroupVersion{Group: "apps", Version: "v1"}},
+			},
+		},
+		{
+			name: "Aggregated discovery: 1 group/2 versions/1 failed GV at /apis",
+			agg: apidiscovery.APIGroupDiscoveryList{
+				Items: []apidiscovery.APIGroupDiscovery{
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "apps",
+						},
+						Versions: []apidiscovery.APIVersionDiscovery{
+							// Stale v2 should report failed GV.
+							{
+								Version: "v2",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "daemonsets",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v2",
+											Kind:    "DaemonSets",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
+							{
+								Version: "v1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			// Only apps/v1 is non-stale expected Group/Version
+			expectedGroups: metav1.APIGroupList{
+				Groups: []metav1.APIGroup{
+					{
+						Name: "apps",
+						Versions: []metav1.GroupVersionForDiscovery{
+							{
+								GroupVersion: "apps/v1",
+								Version:      "v1",
+							},
+						},
+						// PreferredVersion must be apps/v1
+						PreferredVersion: metav1.GroupVersionForDiscovery{
+							GroupVersion: "apps/v1",
+							Version:      "v1",
+						},
+					},
+				},
+			},
+			// Only apps/v1 resources expected.
+			expectedGVResources: map[schema.GroupVersion]*metav1.APIResourceList{
+				{Group: "apps", Version: "v1"}: {
+					GroupVersion: "apps/v1",
+					APIResources: []metav1.APIResource{
+						{
+							Name:       "deployments",
+							Namespaced: true,
+							Group:      "apps",
+							Version:    "v1",
+							Kind:       "Deployment",
+						},
+					},
+				},
+			},
+			expectedFailedGVs: map[schema.GroupVersion]error{
+				{Group: "apps", Version: "v2"}: StaleGroupVersionError{gv: schema.GroupVersion{Group: "apps", Version: "v2"}},
+			},
 		},
 	}

 	for _, test := range tests {
-		apiGroups, resourcesByGV := SplitGroupsAndResources(test.agg)
+		apiGroups, resourcesByGV, failedGVs := SplitGroupsAndResources(test.agg)
+		assert.Equal(t, test.expectedFailedGVs, failedGVs)
 		assert.Equal(t, test.expectedGroups, *apiGroups)
 		assert.Equal(t, test.expectedGVResources, resourcesByGV)
 	}
--- a/discovery/cached/memory/memcache.go
+++ b/discovery/cached/memory/memcache.go
@@ -33,6 +33,7 @@ import (
 	"k8s.io/client-go/openapi"
 	cachedopenapi "k8s.io/client-go/openapi/cached"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
 )

 type cacheEntry struct {
@@ -61,6 +62,15 @@ var (
 	ErrCacheNotFound = errors.New("not found")
 )

+// Server returning empty ResourceList for Group/Version.
+type emptyResponseError struct {
+	gv string
+}
+
+func (e *emptyResponseError) Error() string {
+	return fmt.Sprintf("received empty response for: %s", e.gv)
+}
+
 var _ discovery.CachedDiscoveryInterface = &memCacheClient{}

 // isTransientConnectionError checks whether given error is "Connection refused" or
@@ -103,7 +113,13 @@ func (d *memCacheClient) ServerResourcesForGroupVersion(groupVersion string) (*m
 	if cachedVal.err != nil && isTransientError(cachedVal.err) {
 		r, err := d.serverResourcesForGroupVersion(groupVersion)
 		if err != nil {
-			utilruntime.HandleError(fmt.Errorf("couldn't get resource list for %v: %v", groupVersion, err))
+			// Don't log "empty response" as an error; it is a common response for metrics.
+			if _, emptyErr := err.(*emptyResponseError); emptyErr {
+				// Log at same verbosity as disk cache.
+				klog.V(3).Infof("%v", err)
+			} else {
+				utilruntime.HandleError(fmt.Errorf("couldn't get resource list for %v: %v", groupVersion, err))
+			}
 		}
 		cachedVal = &cacheEntry{r, err}
 		d.groupToServerResources[groupVersion] = cachedVal
@@ -120,32 +136,38 @@ func (d *memCacheClient) ServerGroupsAndResources() ([]*metav1.APIGroup, []*meta
 // GroupsAndMaybeResources returns the list of APIGroups, and possibly the map of group/version
 // to resources. The returned groups will never be nil, but the resources map can be nil
 // if there are no cached resources.
-func (d *memCacheClient) GroupsAndMaybeResources() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, error) {
+func (d *memCacheClient) GroupsAndMaybeResources() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error, error) {
 	d.lock.Lock()
 	defer d.lock.Unlock()

 	if !d.cacheValid {
 		if err := d.refreshLocked(); err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 	}
 	// Build the resourceList from the cache?
 	var resourcesMap map[schema.GroupVersion]*metav1.APIResourceList
+	var failedGVs map[schema.GroupVersion]error
 	if d.receivedAggregatedDiscovery && len(d.groupToServerResources) > 0 {
 		resourcesMap = map[schema.GroupVersion]*metav1.APIResourceList{}
+		failedGVs = map[schema.GroupVersion]error{}
 		for gv, cacheEntry := range d.groupToServerResources {
 			groupVersion, err := schema.ParseGroupVersion(gv)
 			if err != nil {
-				return nil, nil, fmt.Errorf("failed to parse group version (%v): %v", gv, err)
+				return nil, nil, nil, fmt.Errorf("failed to parse group version (%v): %v", gv, err)
+			}
+			if cacheEntry.err != nil {
+				failedGVs[groupVersion] = cacheEntry.err
+			} else {
+				resourcesMap[groupVersion] = cacheEntry.resourceList
 			}
-			resourcesMap[groupVersion] = cacheEntry.resourceList
 		}
 	}
-	return d.groupList, resourcesMap, nil
+	return d.groupList, resourcesMap, failedGVs, nil
 }

 func (d *memCacheClient) ServerGroups() (*metav1.APIGroupList, error) {
-	groups, _, err := d.GroupsAndMaybeResources()
+	groups, _, _, err := d.GroupsAndMaybeResources()
 	if err != nil {
 		return nil, err
 	}
@@ -219,7 +241,8 @@ func (d *memCacheClient) refreshLocked() error {

 	if ad, ok := d.delegate.(discovery.AggregatedDiscoveryInterface); ok {
 		var resources map[schema.GroupVersion]*metav1.APIResourceList
-		gl, resources, err = ad.GroupsAndMaybeResources()
+		var failedGVs map[schema.GroupVersion]error
+		gl, resources, failedGVs, err = ad.GroupsAndMaybeResources()
 		if resources != nil && err == nil {
 			// Cache the resources.
 			d.groupToServerResources = map[string]*cacheEntry{}
@@ -227,6 +250,10 @@ func (d *memCacheClient) refreshLocked() error {
 			for gv, resources := range resources {
 				d.groupToServerResources[gv.String()] = &cacheEntry{resources, nil}
 			}
+			// Cache GroupVersion discovery errors
+			for gv, err := range failedGVs {
+				d.groupToServerResources[gv.String()] = &cacheEntry{nil, err}
+			}
 			d.receivedAggregatedDiscovery = true
 			d.cacheValid = true
 			return nil
@@ -252,7 +279,13 @@ func (d *memCacheClient) refreshLocked() error {

 				r, err := d.serverResourcesForGroupVersion(gv)
 				if err != nil {
-					utilruntime.HandleError(fmt.Errorf("couldn't get resource list for %v: %v", gv, err))
+					// Don't log "empty response" as an error; it is a common response for metrics.
+					if _, emptyErr := err.(*emptyResponseError); emptyErr {
+						// Log at same verbosity as disk cache.
+						klog.V(3).Infof("%v", err)
+					} else {
+						utilruntime.HandleError(fmt.Errorf("couldn't get resource list for %v: %v", gv, err))
+					}
 				}

 				resultLock.Lock()
@@ -274,7 +307,7 @@ func (d *memCacheClient) serverResourcesForGroupVersion(groupVersion string) (*m
 		return r, err
 	}
 	if len(r.APIResources) == 0 {
-		return r, fmt.Errorf("Got empty response for: %v", groupVersion)
+		return r, &emptyResponseError{gv: groupVersion}
 	}
 	return r, nil
 }
--- a/discovery/cached/memory/memcache_test.go
+++ b/discovery/cached/memory/memcache_test.go
@@ -32,6 +32,7 @@ import (
 	errorsutil "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/discovery/fake"
@@ -599,10 +600,11 @@ func TestMemCacheGroupsAndMaybeResources(t *testing.T) {
 			groupToServerResources: map[string]*cacheEntry{},
 		}
 		assert.False(t, memClient.Fresh())
-		apiGroupList, resourcesMap, err := memClient.GroupsAndMaybeResources()
+		apiGroupList, resourcesMap, failedGVs, err := memClient.GroupsAndMaybeResources()
 		require.NoError(t, err)
 		// "Unaggregated" discovery always returns nil for resources.
 		assert.Nil(t, resourcesMap)
+		assert.True(t, len(failedGVs) == 0, "expected empty failed GroupVersions, got (%d)", len(failedGVs))
 		assert.False(t, memClient.receivedAggregatedDiscovery)
 		assert.True(t, memClient.Fresh())
 		// Test the expected groups are returned for the aggregated format.
@@ -618,7 +620,7 @@ func TestMemCacheGroupsAndMaybeResources(t *testing.T) {
 		// Invalidate the cache and retrieve the server groups and resources again.
 		memClient.Invalidate()
 		assert.False(t, memClient.Fresh())
-		apiGroupList, resourcesMap, err = memClient.GroupsAndMaybeResources()
+		apiGroupList, resourcesMap, _, err = memClient.GroupsAndMaybeResources()
 		require.NoError(t, err)
 		assert.Nil(t, resourcesMap)
 		assert.False(t, memClient.receivedAggregatedDiscovery)
@@ -638,6 +640,7 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 		expectedGroupNames    []string
 		expectedGroupVersions []string
 		expectedGVKs          []string
+		expectedFailedGVs     []string
 	}{
 		{
 			name: "Aggregated discovery: 1 group/1 resources at /api, 1 group/1 resources at /apis",
@@ -694,9 +697,10 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 				"/v1/Pod",
 				"apps/v1/Deployment",
 			},
+			expectedFailedGVs: []string{},
 		},
 		{
-			name: "Aggregated discovery: 1 group/1 resources at /api, 1 group/2 versions/1 resources at /apis",
+			name: "Aggregated discovery: 1 group/1 resources at /api, 1 group/2 versions/1 resources/1 stale GV at /apis",
 			corev1: &apidiscovery.APIGroupDiscoveryList{
 				Items: []apidiscovery.APIGroupDiscovery{
 					{
@@ -741,6 +745,7 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 								},
 							},
 							{
+								// Stale Version is not included in discovery.
 								Version: "v2",
 								Resources: []apidiscovery.APIResourceDiscovery{
 									{
@@ -753,18 +758,19 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 										Scope: apidiscovery.ScopeNamespace,
 									},
 								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
 							},
 						},
 					},
 				},
 			},
 			expectedGroupNames:    []string{"", "apps"},
-			expectedGroupVersions: []string{"v1", "apps/v1", "apps/v2"},
+			expectedGroupVersions: []string{"v1", "apps/v1"},
 			expectedGVKs: []string{
 				"/v1/Pod",
 				"apps/v1/Deployment",
-				"apps/v2/Deployment",
 			},
+			expectedFailedGVs: []string{"apps/v2"},
 		},
 		{
 			name: "Aggregated discovery: 1 group/2 resources at /api, 1 group/2 resources at /apis",
@@ -841,9 +847,10 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 				"apps/v1/Deployment",
 				"apps/v1/StatefulSet",
 			},
+			expectedFailedGVs: []string{},
 		},
 		{
-			name: "Aggregated discovery: 1 group/2 resources at /api, 2 group/2 resources at /apis",
+			name: "Aggregated discovery: 1 group/2 resources at /api, 2 group/2 resources/1 stale GV at /apis",
 			corev1: &apidiscovery.APIGroupDiscoveryList{
 				Items: []apidiscovery.APIGroupDiscovery{
 					{
@@ -905,6 +912,31 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 									},
 								},
 							},
+							{
+								// Stale version is not included in discovery.
+								Version: "v1beta1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1beta1",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "statefulsets",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1beta1",
+											Kind:    "StatefulSet",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
 						},
 					},
 					{
@@ -949,9 +981,10 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 				"batch/v1/Job",
 				"batch/v1/CronJob",
 			},
+			expectedFailedGVs: []string{"apps/v1beta1"},
 		},
 		{
-			name:   "Aggregated discovery: /api returns nothing, 2 groups/2 resources at /apis",
+			name:   "Aggregated discovery: /api returns nothing, 2 groups/2 resources/2 stale GV at /apis",
 			corev1: &apidiscovery.APIGroupDiscoveryList{},
 			apis: &apidiscovery.APIGroupDiscoveryList{
 				Items: []apidiscovery.APIGroupDiscovery{
@@ -960,6 +993,7 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 							Name: "apps",
 						},
 						Versions: []apidiscovery.APIVersionDiscovery{
+							// Statel "v1" Version is not included in discovery.
 							{
 								Version: "v1",
 								Resources: []apidiscovery.APIResourceDiscovery{
@@ -982,6 +1016,30 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 										Scope: apidiscovery.ScopeNamespace,
 									},
 								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
+							{
+								Version: "v1beta1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1beta1",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "statefulsets",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1beta1",
+											Kind:    "StatefulSet",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
 							},
 						},
 					},
@@ -1013,18 +1071,35 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 									},
 								},
 							},
+							{
+								// Stale Version is not included in discovery.
+								Version: "v1beta1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "jobs",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "batch",
+											Version: "v1beta1",
+											Kind:    "Job",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
 						},
 					},
 				},
 			},
 			expectedGroupNames:    []string{"apps", "batch"},
-			expectedGroupVersions: []string{"apps/v1", "batch/v1"},
+			expectedGroupVersions: []string{"apps/v1beta1", "batch/v1"},
 			expectedGVKs: []string{
-				"apps/v1/Deployment",
-				"apps/v1/StatefulSet",
+				"apps/v1beta1/Deployment",
+				"apps/v1beta1/StatefulSet",
 				"batch/v1/Job",
 				"batch/v1/CronJob",
 			},
+			expectedFailedGVs: []string{"apps/v1", "batch/v1beta1"},
 		},
 	}

@@ -1054,7 +1129,7 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 			groupToServerResources: map[string]*cacheEntry{},
 		}
 		assert.False(t, memClient.Fresh())
-		apiGroupList, resourcesMap, err := memClient.GroupsAndMaybeResources()
+		apiGroupList, resourcesMap, failedGVs, err := memClient.GroupsAndMaybeResources()
 		require.NoError(t, err)
 		assert.True(t, memClient.receivedAggregatedDiscovery)
 		assert.True(t, memClient.Fresh())
@@ -1077,10 +1152,15 @@ func TestAggregatedMemCacheGroupsAndMaybeResources(t *testing.T) {
 		actualGVKs := sets.NewString(groupVersionKinds(resources)...)
 		assert.True(t, expectedGVKs.Equal(actualGVKs),
 			"%s: Expected GVKs (%s), got (%s)", test.name, expectedGVKs.List(), actualGVKs.List())
+		// Test the returned failed GroupVersions are correct.
+		expectedFailedGVs := sets.NewString(test.expectedFailedGVs...)
+		actualFailedGVs := sets.NewString(failedGroupVersions(failedGVs)...)
+		assert.True(t, expectedFailedGVs.Equal(actualFailedGVs),
+			"%s: Expected Failed GroupVersions (%s), got (%s)", test.name, expectedFailedGVs.List(), actualFailedGVs.List())
 		// Invalidate the cache and retrieve the server groups again.
 		memClient.Invalidate()
 		assert.False(t, memClient.Fresh())
-		apiGroupList, _, err = memClient.GroupsAndMaybeResources()
+		apiGroupList, _, _, err = memClient.GroupsAndMaybeResources()
 		require.NoError(t, err)
 		// Test the expected groups are returned for the aggregated format.
 		actualGroupNames = sets.NewString(groupNamesFromList(apiGroupList)...)
@@ -1410,3 +1490,11 @@ func groupVersionKinds(resources []*metav1.APIResourceList) []string {
 	}
 	return result
 }
+
+func failedGroupVersions(gvs map[schema.GroupVersion]error) []string {
+	result := []string{}
+	for gv := range gvs {
+		result = append(result, gv.String())
+	}
+	return result
+}
--- a/discovery/discovery_client.go
+++ b/discovery/discovery_client.go
@@ -86,7 +86,7 @@ type DiscoveryInterface interface {
 type AggregatedDiscoveryInterface interface {
 	DiscoveryInterface

-	GroupsAndMaybeResources() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, error)
+	GroupsAndMaybeResources() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error, error)
 }

 // CachedDiscoveryInterface is a DiscoveryInterface with cache invalidation and freshness.
@@ -186,18 +186,23 @@ func apiVersionsToAPIGroup(apiVersions *metav1.APIVersions) (apiGroup metav1.API
 // and resources from /api and /apis (either aggregated or not). Legacy groups
 // must be ordered first. The server will either return both endpoints (/api, /apis)
 // as aggregated discovery format or legacy format. For safety, resources will only
-// be returned if both endpoints returned resources.
-func (d *DiscoveryClient) GroupsAndMaybeResources() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, error) {
+// be returned if both endpoints returned resources. Returned "failedGVs" can be
+// empty, but will only be nil in the case an error is returned.
+func (d *DiscoveryClient) GroupsAndMaybeResources() (
+	*metav1.APIGroupList,
+	map[schema.GroupVersion]*metav1.APIResourceList,
+	map[schema.GroupVersion]error,
+	error) {
 	// Legacy group ordered first (there is only one -- core/v1 group). Returned groups must
 	// be non-nil, but it could be empty. Returned resources, apiResources map could be nil.
-	groups, resources, err := d.downloadLegacy()
+	groups, resources, failedGVs, err := d.downloadLegacy()
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	// Discovery groups and (possibly) resources downloaded from /apis.
-	apiGroups, apiResources, aerr := d.downloadAPIs()
-	if err != nil {
-		return nil, nil, aerr
+	apiGroups, apiResources, failedApisGVs, aerr := d.downloadAPIs()
+	if aerr != nil {
+		return nil, nil, nil, aerr
 	}
 	// Merge apis groups into the legacy groups.
 	for _, group := range apiGroups.Groups {
@@ -211,14 +216,23 @@ func (d *DiscoveryClient) GroupsAndMaybeResources() (*metav1.APIGroupList, map[s
 	} else if resources != nil {
 		resources = nil
 	}
-	return groups, resources, err
+	// Merge failed GroupVersions from /api and /apis
+	for gv, err := range failedApisGVs {
+		failedGVs[gv] = err
+	}
+	return groups, resources, failedGVs, err
 }

 // downloadLegacy returns the discovery groups and possibly resources
 // for the legacy v1 GVR at /api, or an error if one occurred. It is
 // possible for the resource map to be nil if the server returned
-// the unaggregated discovery.
-func (d *DiscoveryClient) downloadLegacy() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, error) {
+// the unaggregated discovery. Returned "failedGVs" can be empty, but
+// will only be nil in the case of a returned error.
+func (d *DiscoveryClient) downloadLegacy() (
+	*metav1.APIGroupList,
+	map[schema.GroupVersion]*metav1.APIResourceList,
+	map[schema.GroupVersion]error,
+	error) {
 	accept := acceptDiscoveryFormats
 	if d.UseLegacyDiscovery {
 		accept = AcceptV1
@@ -230,16 +244,19 @@ func (d *DiscoveryClient) downloadLegacy() (*metav1.APIGroupList, map[schema.Gro
 		Do(context.TODO()).
 		ContentType(&responseContentType).
 		Raw()
-	// Special error handling for 403 or 404 to be compatible with older v1.0 servers.
-	// Return empty group list to be merged with /apis.
-	if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) {
-		return nil, nil, err
-	}
-	if err != nil && (errors.IsNotFound(err) || errors.IsForbidden(err)) {
-		return &metav1.APIGroupList{}, nil, nil
+	apiGroupList := &metav1.APIGroupList{}
+	failedGVs := map[schema.GroupVersion]error{}
+	if err != nil {
+		// Tolerate 404, since aggregated api servers can return it.
+		if errors.IsNotFound(err) {
+			// Return empty structures and no error.
+			emptyGVMap := map[schema.GroupVersion]*metav1.APIResourceList{}
+			return apiGroupList, emptyGVMap, failedGVs, nil
+		} else {
+			return nil, nil, nil, err
+		}
 	}

-	apiGroupList := &metav1.APIGroupList{}
 	var resourcesByGV map[schema.GroupVersion]*metav1.APIResourceList
 	// Switch on content-type server responded with: aggregated or unaggregated.
 	switch responseContentType {
@@ -247,7 +264,7 @@ func (d *DiscoveryClient) downloadLegacy() (*metav1.APIGroupList, map[schema.Gro
 		var v metav1.APIVersions
 		err = json.Unmarshal(body, &v)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		apiGroup := metav1.APIGroup{}
 		if len(v.Versions) != 0 {
@@ -258,20 +275,25 @@ func (d *DiscoveryClient) downloadLegacy() (*metav1.APIGroupList, map[schema.Gro
 		var aggregatedDiscovery apidiscovery.APIGroupDiscoveryList
 		err = json.Unmarshal(body, &aggregatedDiscovery)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
-		apiGroupList, resourcesByGV = SplitGroupsAndResources(aggregatedDiscovery)
+		apiGroupList, resourcesByGV, failedGVs = SplitGroupsAndResources(aggregatedDiscovery)
 	default:
-		return nil, nil, fmt.Errorf("Unknown discovery response content-type: %s", responseContentType)
+		return nil, nil, nil, fmt.Errorf("Unknown discovery response content-type: %s", responseContentType)
 	}

-	return apiGroupList, resourcesByGV, nil
+	return apiGroupList, resourcesByGV, failedGVs, nil
 }

 // downloadAPIs returns the discovery groups and (if aggregated format) the
 // discovery resources. The returned groups will always exist, but the
-// resources map may be nil.
-func (d *DiscoveryClient) downloadAPIs() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, error) {
+// resources map may be nil. Returned "failedGVs" can be empty, but will
+// only be nil in the case of a returned error.
+func (d *DiscoveryClient) downloadAPIs() (
+	*metav1.APIGroupList,
+	map[schema.GroupVersion]*metav1.APIResourceList,
+	map[schema.GroupVersion]error,
+	error) {
 	accept := acceptDiscoveryFormats
 	if d.UseLegacyDiscovery {
 		accept = AcceptV1
@@ -283,42 +305,38 @@ func (d *DiscoveryClient) downloadAPIs() (*metav1.APIGroupList, map[schema.Group
 		Do(context.TODO()).
 		ContentType(&responseContentType).
 		Raw()
-	// Special error handling for 403 or 404 to be compatible with older v1.0 servers.
-	// Return empty group list to be merged with /api.
-	if err != nil && !errors.IsNotFound(err) && !errors.IsForbidden(err) {
-		return nil, nil, err
-	}
-	if err != nil && (errors.IsNotFound(err) || errors.IsForbidden(err)) {
-		return &metav1.APIGroupList{}, nil, nil
+	if err != nil {
+		return nil, nil, nil, err
 	}

 	apiGroupList := &metav1.APIGroupList{}
+	failedGVs := map[schema.GroupVersion]error{}
 	var resourcesByGV map[schema.GroupVersion]*metav1.APIResourceList
 	// Switch on content-type server responded with: aggregated or unaggregated.
 	switch responseContentType {
 	case AcceptV1:
 		err = json.Unmarshal(body, apiGroupList)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 	case AcceptV2Beta1:
 		var aggregatedDiscovery apidiscovery.APIGroupDiscoveryList
 		err = json.Unmarshal(body, &aggregatedDiscovery)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
-		apiGroupList, resourcesByGV = SplitGroupsAndResources(aggregatedDiscovery)
+		apiGroupList, resourcesByGV, failedGVs = SplitGroupsAndResources(aggregatedDiscovery)
 	default:
-		return nil, nil, fmt.Errorf("Unknown discovery response content-type: %s", responseContentType)
+		return nil, nil, nil, fmt.Errorf("Unknown discovery response content-type: %s", responseContentType)
 	}

-	return apiGroupList, resourcesByGV, nil
+	return apiGroupList, resourcesByGV, failedGVs, nil
 }

 // ServerGroups returns the supported groups, with information like supported versions and the
 // preferred version.
 func (d *DiscoveryClient) ServerGroups() (*metav1.APIGroupList, error) {
-	groups, _, err := d.GroupsAndMaybeResources()
+	groups, _, _, err := d.GroupsAndMaybeResources()
 	if err != nil {
 		return nil, err
 	}
@@ -341,8 +359,10 @@ func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (r
 	}
 	err = d.restClient.Get().AbsPath(url.String()).Do(context.TODO()).Into(resources)
 	if err != nil {
-		// ignore 403 or 404 error to be compatible with an v1.0 server.
-		if groupVersion == "v1" && (errors.IsNotFound(err) || errors.IsForbidden(err)) {
+		// Tolerate core/v1 not found response by returning empty resource list;
+		// this probably should not happen. But we should verify all callers are
+		// not depending on this toleration before removal.
+		if groupVersion == "v1" && errors.IsNotFound(err) {
 			return resources, nil
 		}
 		return nil, err
@@ -383,13 +403,14 @@ func IsGroupDiscoveryFailedError(err error) bool {
 func ServerGroupsAndResources(d DiscoveryInterface) ([]*metav1.APIGroup, []*metav1.APIResourceList, error) {
 	var sgs *metav1.APIGroupList
 	var resources []*metav1.APIResourceList
+	var failedGVs map[schema.GroupVersion]error
 	var err error

 	// If the passed discovery object implements the wider AggregatedDiscoveryInterface,
 	// then attempt to retrieve aggregated discovery with both groups and the resources.
 	if ad, ok := d.(AggregatedDiscoveryInterface); ok {
 		var resourcesByGV map[schema.GroupVersion]*metav1.APIResourceList
-		sgs, resourcesByGV, err = ad.GroupsAndMaybeResources()
+		sgs, resourcesByGV, failedGVs, err = ad.GroupsAndMaybeResources()
 		for _, resourceList := range resourcesByGV {
 			resources = append(resources, resourceList)
 		}
@@ -404,8 +425,15 @@ func ServerGroupsAndResources(d DiscoveryInterface) ([]*metav1.APIGroup, []*meta
 	for i := range sgs.Groups {
 		resultGroups = append(resultGroups, &sgs.Groups[i])
 	}
+	// resources is non-nil if aggregated discovery succeeded.
 	if resources != nil {
-		return resultGroups, resources, nil
+		// Any stale Group/Versions returned by aggregated discovery
+		// must be surfaced to the caller as failed Group/Versions.
+		var ferr error
+		if len(failedGVs) > 0 {
+			ferr = &ErrGroupDiscoveryFailed{Groups: failedGVs}
+		}
+		return resultGroups, resources, ferr
 	}

 	groupVersionResources, failedGroups := fetchGroupVersionResources(d, sgs)
@@ -436,16 +464,18 @@ func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList,
 	var err error

 	// If the passed discovery object implements the wider AggregatedDiscoveryInterface,
-	// then it is attempt to retrieve both the groups and the resources.
+	// then it is attempt to retrieve both the groups and the resources. "failedGroups"
+	// are Group/Versions returned as stale in AggregatedDiscovery format.
 	ad, ok := d.(AggregatedDiscoveryInterface)
 	if ok {
-		serverGroupList, groupVersionResources, err = ad.GroupsAndMaybeResources()
+		serverGroupList, groupVersionResources, failedGroups, err = ad.GroupsAndMaybeResources()
 	} else {
 		serverGroupList, err = d.ServerGroups()
 	}
 	if err != nil {
 		return nil, err
 	}
+	// Non-aggregated discovery must fetch resources from Groups.
 	if groupVersionResources == nil {
 		groupVersionResources, failedGroups = fetchGroupVersionResources(d, serverGroupList)
 	}
--- a/discovery/discovery_client_test.go
+++ b/discovery/discovery_client_test.go
@@ -110,7 +110,6 @@ func TestGetServerGroupsWithV1Server(t *testing.T) {
 	}))
 	defer server.Close()
 	client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
-	// ServerGroups should not return an error even if server returns error at /api and /apis
 	apiGroupList, err := client.ServerGroups()
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
@@ -121,32 +120,49 @@ func TestGetServerGroupsWithV1Server(t *testing.T) {
 	}
 }

-func TestGetServerGroupsWithBrokenServer(t *testing.T) {
-	for _, statusCode := range []int{http.StatusNotFound, http.StatusForbidden} {
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-			w.WriteHeader(statusCode)
-		}))
-		defer server.Close()
-		client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
-		// ServerGroups should not return an error even if server returns Not Found or Forbidden error at all end points
-		apiGroupList, err := client.ServerGroups()
+func TestDiscoveryToleratesMissingCoreGroup(t *testing.T) {
+	// Discovery tolerates 404 from /api. Aggregated api servers can do this.
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		var obj interface{}
+		switch req.URL.Path {
+		case "/api":
+			w.WriteHeader(http.StatusNotFound)
+		case "/apis":
+			obj = &metav1.APIGroupList{
+				Groups: []metav1.APIGroup{
+					{
+						Name: "extensions",
+						Versions: []metav1.GroupVersionForDiscovery{
+							{GroupVersion: "extensions/v1beta1"},
+						},
+					},
+				},
+			}
+		}
+		output, err := json.Marshal(obj)
 		if err != nil {
-			t.Fatalf("unexpected error: %v", err)
-		}
-		groupVersions := metav1.ExtractGroupVersions(apiGroupList)
-		if len(groupVersions) != 0 {
-			t.Errorf("expected empty list, got: %q", groupVersions)
+			t.Fatalf("unexpected encoding error: %v", err)
+			return
 		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		w.Write(output)
+	}))
+	defer server.Close()
+	client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
+	// ServerGroups should not return an error even if server returns 404 at /api.
+	apiGroupList, err := client.ServerGroups()
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	groupVersions := metav1.ExtractGroupVersions(apiGroupList)
+	if !reflect.DeepEqual(groupVersions, []string{"extensions/v1beta1"}) {
+		t.Errorf("expected: %q, got: %q", []string{"extensions/v1beta1"}, groupVersions)
 	}
 }

-func TestTimeoutIsSet(t *testing.T) {
-	cfg := &restclient.Config{}
-	setDiscoveryDefaults(cfg)
-	assert.Equal(t, defaultTimeout, cfg.Timeout)
-}
-
-func TestGetServerResourcesWithV1Server(t *testing.T) {
+func TestDiscoveryFailsWhenNonCoreGroupsMissing(t *testing.T) {
+	// Discovery fails when /apis returns 404.
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		var obj interface{}
 		switch req.URL.Path {
@@ -156,13 +172,12 @@ func TestGetServerResourcesWithV1Server(t *testing.T) {
 					"v1",
 				},
 			}
-		default:
+		case "/apis":
 			w.WriteHeader(http.StatusNotFound)
-			return
 		}
 		output, err := json.Marshal(obj)
 		if err != nil {
-			t.Errorf("unexpected encoding error: %v", err)
+			t.Fatalf("unexpected encoding error: %v", err)
 			return
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -171,17 +186,34 @@ func TestGetServerResourcesWithV1Server(t *testing.T) {
 	}))
 	defer server.Close()
 	client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
-	// ServerResources should not return an error even if server returns error at /api/v1.
-	_, serverResources, err := client.ServerGroupsAndResources()
-	if err != nil {
-		t.Errorf("unexpected error: %v", err)
+	_, err := client.ServerGroups()
+	if err == nil {
+		t.Fatal("expected error, received none")
 	}
-	gvs := groupVersions(serverResources)
-	if !sets.NewString(gvs...).Has("v1") {
-		t.Errorf("missing v1 in resource list: %v", serverResources)
+}
+
+func TestGetServerGroupsWithBrokenServer(t *testing.T) {
+	// 404 Not Found errors because discovery at /apis returns an error.
+	// 403 Forbidden errors because discovery at both /api and /apis returns error.
+	for _, statusCode := range []int{http.StatusNotFound, http.StatusForbidden} {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			w.WriteHeader(statusCode)
+		}))
+		defer server.Close()
+		client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
+		_, err := client.ServerGroups()
+		if err == nil {
+			t.Fatal("expected error, received none")
+		}
 	}
 }

+func TestTimeoutIsSet(t *testing.T) {
+	cfg := &restclient.Config{}
+	setDiscoveryDefaults(cfg)
+	assert.Equal(t, defaultTimeout, cfg.Timeout)
+}
+
 func TestGetServerResourcesForGroupVersion(t *testing.T) {
 	stable := metav1.APIResourceList{
 		GroupVersion: "v1",
@@ -964,17 +996,33 @@ func TestServerPreferredNamespacedResources(t *testing.T) {
 		expected map[schema.GroupVersionResource]struct{}
 	}{
 		{
+			// Combines discovery for /api and /apis.
 			response: func(w http.ResponseWriter, req *http.Request) {
 				var list interface{}
 				switch req.URL.Path {
-				case "/api/v1":
-					list = &stable
 				case "/api":
 					list = &metav1.APIVersions{
 						Versions: []string{
 							"v1",
 						},
 					}
+				case "/api/v1":
+					list = &stable
+				case "/apis":
+					list = &metav1.APIGroupList{
+						Groups: []metav1.APIGroup{
+							{
+								Name: "batch",
+								Versions: []metav1.GroupVersionForDiscovery{
+									{GroupVersion: "batch/v1", Version: "v1"},
+								},
+								PreferredVersion: metav1.GroupVersionForDiscovery{GroupVersion: "batch/v1", Version: "v1"},
+							},
+						},
+					}
+				case "/apis/batch/v1":
+					list = &batchv1
+
 				default:
 					t.Logf("unexpected request: %s", req.URL.Path)
 					w.WriteHeader(http.StatusNotFound)
@@ -990,11 +1038,14 @@ func TestServerPreferredNamespacedResources(t *testing.T) {
 				w.Write(output)
 			},
 			expected: map[schema.GroupVersionResource]struct{}{
-				{Group: "", Version: "v1", Resource: "pods"}:     {},
-				{Group: "", Version: "v1", Resource: "services"}: {},
+				{Group: "", Version: "v1", Resource: "pods"}:      {},
+				{Group: "", Version: "v1", Resource: "services"}:  {},
+				{Group: "batch", Version: "v1", Resource: "jobs"}: {},
 			},
 		},
 		{
+			// Only return /apis (not legacy /api); does not error. 404 for legacy
+			// core/v1 at /api is tolerated.
 			response: func(w http.ResponseWriter, req *http.Request) {
 				var list interface{}
 				switch req.URL.Path {
@@ -1384,6 +1435,7 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 		expectedGroupNames    []string
 		expectedGroupVersions []string
 		expectedGVKs          []string
+		expectedFailedGVs     []string
 	}{
 		{
 			name: "Aggregated discovery: 1 group/1 resources at /api, 1 group/1 resources at /apis",
@@ -1512,6 +1564,78 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 				"apps/v2/Deployment",
 			},
 		},
+		{
+			name: "Aggregated discovery: 1 group/1 resources at /api, 1 group/2 versions/1 resources at /apis",
+			corev1: &apidiscovery.APIGroupDiscoveryList{
+				Items: []apidiscovery.APIGroupDiscovery{
+					{
+						Versions: []apidiscovery.APIVersionDiscovery{
+							{
+								Version: "v1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "pods",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "",
+											Version: "v1",
+											Kind:    "Pod",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			apis: &apidiscovery.APIGroupDiscoveryList{
+				Items: []apidiscovery.APIGroupDiscovery{
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "apps",
+						},
+						Versions: []apidiscovery.APIVersionDiscovery{
+							{
+								Version: "v1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+							},
+							{
+								Version: "v2",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v2",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
+						},
+					},
+				},
+			},
+			expectedGroupNames:    []string{"", "apps"},
+			expectedGroupVersions: []string{"v1", "apps/v1"},
+			expectedGVKs: []string{
+				"/v1/Pod",
+				"apps/v1/Deployment",
+			},
+			expectedFailedGVs: []string{"apps/v2"},
+		},
 		{
 			name: "Aggregated discovery: 1 group/2 resources at /api, 1 group/2 resources at /apis",
 			corev1: &apidiscovery.APIGroupDiscoveryList{
@@ -1552,6 +1676,31 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 							Name: "apps",
 						},
 						Versions: []apidiscovery.APIVersionDiscovery{
+							// Stale "v2" version not included.
+							{
+								Version: "v2",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v2",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "statefulsets",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v2",
+											Kind:    "StatefulSet",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
 							{
 								Version: "v1",
 								Resources: []apidiscovery.APIResourceDiscovery{
@@ -1587,9 +1736,10 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 				"apps/v1/Deployment",
 				"apps/v1/StatefulSet",
 			},
+			expectedFailedGVs: []string{"apps/v2"},
 		},
 		{
-			name: "Aggregated discovery: 1 group/2 resources at /api, 2 group/2 resources at /apis",
+			name: "Aggregated discovery: 1 group/2 resources at /api, 2 group/2 resources/1 stale GV at /apis",
 			corev1: &apidiscovery.APIGroupDiscoveryList{
 				Items: []apidiscovery.APIGroupDiscovery{
 					{
@@ -1658,6 +1808,7 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 							Name: "batch",
 						},
 						Versions: []apidiscovery.APIVersionDiscovery{
+							// Stale Group/Version is not included
 							{
 								Version: "v1",
 								Resources: []apidiscovery.APIResourceDiscovery{
@@ -1680,21 +1831,46 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 										Scope: apidiscovery.ScopeNamespace,
 									},
 								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
+							{
+								Version: "v1beta1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "jobs",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "batch",
+											Version: "v1beta1",
+											Kind:    "Job",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "cronjobs",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "batch",
+											Version: "v1beta1",
+											Kind:    "CronJob",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
 							},
 						},
 					},
 				},
 			},
 			expectedGroupNames:    []string{"", "apps", "batch"},
-			expectedGroupVersions: []string{"v1", "apps/v1", "batch/v1"},
+			expectedGroupVersions: []string{"v1", "apps/v1", "batch/v1beta1"},
 			expectedGVKs: []string{
 				"/v1/Pod",
 				"/v1/Service",
 				"apps/v1/Deployment",
 				"apps/v1/StatefulSet",
-				"batch/v1/Job",
-				"batch/v1/CronJob",
+				"batch/v1beta1/Job",
+				"batch/v1beta1/CronJob",
 			},
+			expectedFailedGVs: []string{"batch/v1"},
 		},
 		{
 			name:   "Aggregated discovery: /api returns nothing, 2 groups/2 resources at /apis",
@@ -1759,6 +1935,31 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 									},
 								},
 							},
+							{
+								// Stale "v1beta1" not included.
+								Version: "v1beta1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "jobs",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "batch",
+											Version: "v1beta1",
+											Kind:    "Job",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "cronjobs",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "batch",
+											Version: "v1beta1",
+											Kind:    "CronJob",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
 						},
 					},
 				},
@@ -1771,6 +1972,7 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 				"batch/v1/Job",
 				"batch/v1/CronJob",
 			},
+			expectedFailedGVs: []string{"batch/v1beta1"},
 		},
 	}

@@ -1796,7 +1998,15 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 		defer server.Close()
 		client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
 		apiGroups, resources, err := client.ServerGroupsAndResources()
-		require.NoError(t, err)
+		if len(test.expectedFailedGVs) > 0 {
+			require.Error(t, err)
+			expectedFailedGVs := sets.NewString(test.expectedFailedGVs...)
+			actualFailedGVs := sets.NewString(failedGroupVersions(err)...)
+			assert.True(t, expectedFailedGVs.Equal(actualFailedGVs),
+				"%s: Expected Failed GVs (%s), got (%s)", test.name, expectedFailedGVs, actualFailedGVs)
+		} else {
+			require.NoError(t, err)
+		}
 		// Test the expected groups are returned for the aggregated format.
 		expectedGroupNames := sets.NewString(test.expectedGroupNames...)
 		actualGroupNames := sets.NewString(groupNames(apiGroups)...)
@@ -1823,12 +2033,138 @@ func TestAggregatedServerGroupsAndResources(t *testing.T) {
 	}
 }

+func TestAggregatedServerGroupsAndResourcesWithErrors(t *testing.T) {
+	tests := []struct {
+		name              string
+		corev1            *apidiscovery.APIGroupDiscoveryList
+		coreHttpStatus    int
+		apis              *apidiscovery.APIGroupDiscoveryList
+		apisHttpStatus    int
+		expectedGroups    []string
+		expectedResources []string
+		expectedErr       bool
+	}{
+		{
+			name:           "Aggregated Discovery: 404 for core/v1 is tolerated",
+			corev1:         &apidiscovery.APIGroupDiscoveryList{},
+			coreHttpStatus: http.StatusNotFound,
+			apis: &apidiscovery.APIGroupDiscoveryList{
+				Items: []apidiscovery.APIGroupDiscovery{
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "apps",
+						},
+						Versions: []apidiscovery.APIVersionDiscovery{
+							{
+								Version: "v1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "daemonsets",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1",
+											Kind:    "DaemonSet",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			apisHttpStatus:    http.StatusOK,
+			expectedGroups:    []string{"apps"},
+			expectedResources: []string{"apps/v1/Deployment", "apps/v1/DaemonSet"},
+			expectedErr:       false,
+		},
+		{
+			name:           "Aggregated Discovery: 403 for core/v1 causes error",
+			corev1:         &apidiscovery.APIGroupDiscoveryList{},
+			coreHttpStatus: http.StatusForbidden,
+			apis:           &apidiscovery.APIGroupDiscoveryList{},
+			apisHttpStatus: http.StatusOK,
+			expectedErr:    true,
+		},
+		{
+			name:           "Aggregated Discovery: 404 for /apis causes error",
+			corev1:         &apidiscovery.APIGroupDiscoveryList{},
+			coreHttpStatus: http.StatusOK,
+			apis:           &apidiscovery.APIGroupDiscoveryList{},
+			apisHttpStatus: http.StatusNotFound,
+			expectedErr:    true,
+		},
+		{
+			name:           "Aggregated Discovery: 403 for /apis causes error",
+			corev1:         &apidiscovery.APIGroupDiscoveryList{},
+			coreHttpStatus: http.StatusOK,
+			apis:           &apidiscovery.APIGroupDiscoveryList{},
+			apisHttpStatus: http.StatusForbidden,
+			expectedErr:    true,
+		},
+	}
+
+	for _, test := range tests {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			var agg *apidiscovery.APIGroupDiscoveryList
+			var status int
+			switch req.URL.Path {
+			case "/api":
+				agg = test.corev1
+				status = test.coreHttpStatus
+			case "/apis":
+				agg = test.apis
+				status = test.apisHttpStatus
+			default:
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			output, err := json.Marshal(agg)
+			require.NoError(t, err)
+			// Content-type is "aggregated" discovery format.
+			w.Header().Set("Content-Type", AcceptV2Beta1)
+			w.WriteHeader(status)
+			w.Write(output)
+		}))
+		defer server.Close()
+		client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
+		apiGroups, resources, err := client.ServerGroupsAndResources()
+		if test.expectedErr {
+			require.Error(t, err)
+			require.Nil(t, apiGroups)
+			require.Nil(t, resources)
+			continue
+		}
+		require.NoError(t, err)
+		// First check the returned groups
+		expectedGroups := sets.NewString(test.expectedGroups...)
+		actualGroups := sets.NewString(groupNames(apiGroups)...)
+		assert.True(t, expectedGroups.Equal(actualGroups),
+			"%s: Expected GVKs (%s), got (%s)", test.name, expectedGroups.List(), actualGroups.List())
+		// Next check the returned resources
+		expectedGVKs := sets.NewString(test.expectedResources...)
+		actualGVKs := sets.NewString(groupVersionKinds(resources)...)
+		assert.True(t, expectedGVKs.Equal(actualGVKs),
+			"%s: Expected GVKs (%s), got (%s)", test.name, expectedGVKs.List(), actualGVKs.List())
+	}
+}
+
 func TestAggregatedServerPreferredResources(t *testing.T) {
 	tests := []struct {
-		name         string
-		corev1       *apidiscovery.APIGroupDiscoveryList
-		apis         *apidiscovery.APIGroupDiscoveryList
-		expectedGVKs []string
+		name              string
+		corev1            *apidiscovery.APIGroupDiscoveryList
+		apis              *apidiscovery.APIGroupDiscoveryList
+		expectedGVKs      []string
+		expectedFailedGVs []string
 	}{
 		{
 			name: "Aggregated discovery: basic corev1 and apps/v1 preferred resources returned",
@@ -1954,6 +2290,78 @@ func TestAggregatedServerPreferredResources(t *testing.T) {
 				"apps/v2/Deployment",
 			},
 		},
+		{
+			name: "Aggregated discovery: stale Group/Version can not produce preferred version",
+			corev1: &apidiscovery.APIGroupDiscoveryList{
+				Items: []apidiscovery.APIGroupDiscovery{
+					{
+						Versions: []apidiscovery.APIVersionDiscovery{
+							{
+								Version: "v1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "pods",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "",
+											Version: "v1",
+											Kind:    "Pod",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			apis: &apidiscovery.APIGroupDiscoveryList{
+				Items: []apidiscovery.APIGroupDiscovery{
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "apps",
+						},
+						Versions: []apidiscovery.APIVersionDiscovery{
+							// v2 is "stale", so it can not be "preferred".
+							{
+								Version: "v2",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v2",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
+							{
+								Version: "v1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			// Only v1 resources from apps group; v2 would be preferred but it is "stale".
+			expectedGVKs: []string{
+				"/v1/Pod",
+				"apps/v1/Deployment",
+			},
+			expectedFailedGVs: []string{"apps/v2"},
+		},
 		{
 			name: "Aggregated discovery: preferred multiple resources from multiple group/versions",
 			corev1: &apidiscovery.APIGroupDiscoveryList{
@@ -2017,6 +2425,30 @@ func TestAggregatedServerPreferredResources(t *testing.T) {
 									},
 								},
 							},
+							{
+								Version: "v1beta1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1beta1",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "statefulsets",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1beta1",
+											Kind:    "StatefulSet",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+								Freshness: apidiscovery.DiscoveryFreshnessStale,
+							},
 						},
 					},
 				},
@@ -2027,6 +2459,7 @@ func TestAggregatedServerPreferredResources(t *testing.T) {
 				"apps/v1/Deployment",
 				"apps/v1/StatefulSet",
 			},
+			expectedFailedGVs: []string{"apps/v1beta1"},
 		},
 		{
 			name: "Aggregated discovery: resources from multiple preferred group versions at /apis",
@@ -2091,6 +2524,30 @@ func TestAggregatedServerPreferredResources(t *testing.T) {
 									},
 								},
 							},
+							{
+								// Not included because "v1" is preferred.
+								Version: "v1beta1",
+								Resources: []apidiscovery.APIResourceDiscovery{
+									{
+										Resource: "deployments",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1beta1",
+											Kind:    "Deployment",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+									{
+										Resource: "statefulsets",
+										ResponseKind: &metav1.GroupVersionKind{
+											Group:   "apps",
+											Version: "v1beta1",
+											Kind:    "StatefulSet",
+										},
+										Scope: apidiscovery.ScopeNamespace,
+									},
+								},
+							},
 						},
 					},
 					{
@@ -2228,6 +2685,7 @@ func TestAggregatedServerPreferredResources(t *testing.T) {
 								},
 							},
 							{
+								// Not included, since "v1" is preferred.
 								Version: "v1beta1",
 								Resources: []apidiscovery.APIResourceDiscovery{
 									{
@@ -2288,7 +2746,15 @@ func TestAggregatedServerPreferredResources(t *testing.T) {
 		defer server.Close()
 		client := NewDiscoveryClientForConfigOrDie(&restclient.Config{Host: server.URL})
 		resources, err := client.ServerPreferredResources()
-		require.NoError(t, err)
+		if len(test.expectedFailedGVs) > 0 {
+			require.Error(t, err)
+			expectedFailedGVs := sets.NewString(test.expectedFailedGVs...)
+			actualFailedGVs := sets.NewString(failedGroupVersions(err)...)
+			assert.True(t, expectedFailedGVs.Equal(actualFailedGVs),
+				"%s: Expected Failed GVs (%s), got (%s)", test.name, expectedFailedGVs, actualFailedGVs)
+		} else {
+			require.NoError(t, err)
+		}
 		// Test the expected preferred GVKs are returned from the aggregated discovery.
 		expectedGVKs := sets.NewString(test.expectedGVKs...)
 		actualGVKs := sets.NewString(groupVersionKinds(resources)...)
@@ -2370,3 +2836,15 @@ func groupVersionKinds(resources []*metav1.APIResourceList) []string {
 	}
 	return result
 }
+
+func failedGroupVersions(err error) []string {
+	result := []string{}
+	ferr, ok := err.(*ErrGroupDiscoveryFailed)
+	if !ok {
+		return result
+	}
+	for gv := range ferr.Groups {
+		result = append(result, gv.String())
+	}
+	return result
+}
--- a/discovery/helper_blackbox_test.go
+++ b/discovery/helper_blackbox_test.go
@@ -65,6 +65,22 @@ func TestServerSupportsVersion(t *testing.T) {
 			expectErr:       func(err error) bool { return strings.Contains(err.Error(), `server does not support API version "v1"`) },
 			statusCode:      http.StatusOK,
 		},
+		{
+			name:            "Status 403 Forbidden for core/v1 group returns error and is unsupported",
+			requiredVersion: schema.GroupVersion{Version: "v1"},
+			serverVersions:  []string{"/version1", v1.SchemeGroupVersion.String()},
+			expectErr:       func(err error) bool { return strings.Contains(err.Error(), "unknown") },
+			statusCode:      http.StatusForbidden,
+		},
+		{
+			name:            "Status 404 Not Found for core/v1 group returns empty and is unsupported",
+			requiredVersion: schema.GroupVersion{Version: "v1"},
+			serverVersions:  []string{"/version1", v1.SchemeGroupVersion.String()},
+			expectErr: func(err error) bool {
+				return strings.Contains(err.Error(), "server could not find the requested resource")
+			},
+			statusCode: http.StatusNotFound,
+		},
 		{
 			name:           "connection refused error",
 			serverVersions: []string{"version1"},
@@ -72,11 +88,6 @@ func TestServerSupportsVersion(t *testing.T) {
 			expectErr:      func(err error) bool { return strings.Contains(err.Error(), "connection refused") },
 			statusCode:     http.StatusOK,
 		},
-		{
-			name:            "discovery fails due to 404 Not Found errors and thus serverVersions is empty, use requested GroupVersion",
-			requiredVersion: schema.GroupVersion{Version: "version1"},
-			statusCode:      http.StatusNotFound,
-		},
 	}

 	for _, test := range tests {
--- a/go.mod
+++ b/go.mod
@@ -19,13 +19,13 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.0
-	golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10
+	golang.org/x/net v0.7.0
 	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
-	golang.org/x/term v0.3.0
+	golang.org/x/term v0.5.0
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8
 	google.golang.org/protobuf v1.28.1
-	k8s.io/api v0.0.0-20221208060720-07ac8fe9ce91
-	k8s.io/apimachinery v0.0.0-20221208055745-5d4cdd22b0f7
+	k8s.io/api v0.26.3
+	k8s.io/apimachinery v0.26.3
 	k8s.io/klog/v2 v2.80.1
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280
 	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d
@@ -49,8 +49,8 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/text v0.5.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -59,6 +59,6 @@ require (
 )

 replace (
-	k8s.io/api => k8s.io/api v0.0.0-20221208060720-07ac8fe9ce91
-	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20221208055745-5d4cdd22b0f7
+	k8s.io/api => k8s.io/api v0.26.3
+	k8s.io/apimachinery => k8s.io/apimachinery v0.26.3
 )
--- a/go.sum
+++ b/go.sum
@@ -263,8 +263,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 h1:Frnccbp+ok2GkUS2tC84yAq/U9Vg+0sIO7aRL3T4Xnc=
-golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -309,19 +309,19 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
-golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -476,10 +476,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.0.0-20221208060720-07ac8fe9ce91 h1:xVuXb+iGNa+jCsqfhp0u1N4L+9vf7fF1gZJ6WZRnJU8=
-k8s.io/api v0.0.0-20221208060720-07ac8fe9ce91/go.mod h1:GetGjgKCUeHLs1GMrltnUDUYPvhhro0e4BozgWsM0Jk=
-k8s.io/apimachinery v0.0.0-20221208055745-5d4cdd22b0f7 h1:6WftyyXbOkaz1aO+YA8Z5774ZvYeQOUk1JUkvPaOTvE=
-k8s.io/apimachinery v0.0.0-20221208055745-5d4cdd22b0f7/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
+k8s.io/api v0.26.3 h1:emf74GIQMTik01Aum9dPP0gAypL8JTLl/lHa4V9RFSU=
+k8s.io/api v0.26.3/go.mod h1:PXsqwPMXBSBcL1lJ9CYDKy7kIReUydukS5JiRlxC3qE=
+k8s.io/apimachinery v0.26.3 h1:dQx6PNETJ7nODU3XPtrwkfuubs6w7sX0M8n61zHIV/k=
+k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I=
 k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
 k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
--- a/rest/request.go
+++ b/rest/request.go
@@ -34,6 +34,7 @@ import (
 	"time"

 	"golang.org/x/net/http2"
+
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -116,8 +117,11 @@ type Request struct {
 	subresource  string

 	// output
-	err  error
-	body io.Reader
+	err error
+
+	// only one of body / bodyBytes may be set. requests using body are not retriable.
+	body      io.Reader
+	bodyBytes []byte

 	retryFn requestRetryFunc
 }
@@ -443,12 +447,15 @@ func (r *Request) Body(obj interface{}) *Request {
 			return r
 		}
 		glogBody("Request Body", data)
-		r.body = bytes.NewReader(data)
+		r.body = nil
+		r.bodyBytes = data
 	case []byte:
 		glogBody("Request Body", t)
-		r.body = bytes.NewReader(t)
+		r.body = nil
+		r.bodyBytes = t
 	case io.Reader:
 		r.body = t
+		r.bodyBytes = nil
 	case runtime.Object:
 		// callers may pass typed interface pointers, therefore we must check nil with reflection
 		if reflect.ValueOf(t).IsNil() {
@@ -465,7 +472,8 @@ func (r *Request) Body(obj interface{}) *Request {
 			return r
 		}
 		glogBody("Request Body", data)
-		r.body = bytes.NewReader(data)
+		r.body = nil
+		r.bodyBytes = data
 		r.SetHeader("Content-Type", r.c.content.ContentType)
 	default:
 		r.err = fmt.Errorf("unknown type used for body: %+v", obj)
@@ -825,9 +833,6 @@ func (r *Request) Stream(ctx context.Context) (io.ReadCloser, error) {
 		if err != nil {
 			return nil, err
 		}
-		if r.body != nil {
-			req.Body = io.NopCloser(r.body)
-		}
 		resp, err := client.Do(req)
 		updateURLMetrics(ctx, r, resp, err)
 		retry.After(ctx, r, resp, err)
@@ -889,8 +894,20 @@ func (r *Request) requestPreflightCheck() error {
 }

 func (r *Request) newHTTPRequest(ctx context.Context) (*http.Request, error) {
+	var body io.Reader
+	switch {
+	case r.body != nil && r.bodyBytes != nil:
+		return nil, fmt.Errorf("cannot set both body and bodyBytes")
+	case r.body != nil:
+		body = r.body
+	case r.bodyBytes != nil:
+		// Create a new reader specifically for this request.
+		// Giving each request a dedicated reader allows retries to avoid races resetting the request body.
+		body = bytes.NewReader(r.bodyBytes)
+	}
+
 	url := r.URL().String()
-	req, err := http.NewRequest(r.verb, url, r.body)
+	req, err := http.NewRequest(r.verb, url, body)
 	if err != nil {
 		return nil, err
 	}
--- a/rest/request_test.go
+++ b/rest/request_test.go
@@ -1122,42 +1122,6 @@ func TestRequestWatch(t *testing.T) {
 			},
 			Empty: true,
 		},
-		{
-			name: "max retries 1, server returns a retry-after response, request body seek error",
-			Request: &Request{
-				body: &readSeeker{err: io.EOF},
-				c: &RESTClient{
-					base: &url.URL{},
-				},
-			},
-			maxRetries:       1,
-			attemptsExpected: 1,
-			serverReturns: []responseErr{
-				{response: retryAfterResponse(), err: nil},
-			},
-			Err: true,
-			ErrFn: func(err error) bool {
-				return !apierrors.IsInternalError(err) && strings.Contains(err.Error(), "failed to reset the request body while retrying a request: EOF")
-			},
-		},
-		{
-			name: "max retries 1, server returns a retryable error, request body seek error",
-			Request: &Request{
-				body: &readSeeker{err: io.EOF},
-				c: &RESTClient{
-					base: &url.URL{},
-				},
-			},
-			maxRetries:       1,
-			attemptsExpected: 1,
-			serverReturns: []responseErr{
-				{response: nil, err: io.EOF},
-			},
-			Err: true,
-			ErrFn: func(err error) bool {
-				return !apierrors.IsInternalError(err)
-			},
-		},
 		{
 			name: "max retries 2, server always returns a response with Retry-After header",
 			Request: &Request{
@@ -1319,7 +1283,7 @@ func TestRequestStream(t *testing.T) {
 			},
 		},
 		{
-			name: "max retries 1, server returns a retry-after response, request body seek error",
+			name: "max retries 1, server returns a retry-after response, non-bytes request, no retry",
 			Request: &Request{
 				body: &readSeeker{err: io.EOF},
 				c: &RESTClient{
@@ -1332,9 +1296,6 @@ func TestRequestStream(t *testing.T) {
 				{response: retryAfterResponse(), err: nil},
 			},
 			Err: true,
-			ErrFn: func(err error) bool {
-				return !apierrors.IsInternalError(err) && strings.Contains(err.Error(), "failed to reset the request body while retrying a request: EOF")
-			},
 		},
 		{
 			name: "max retries 2, server always returns a response with Retry-After header",
@@ -2016,20 +1977,24 @@ func TestBody(t *testing.T) {
 			}
 		}

-		if r.body == nil {
+		req, err := r.newHTTPRequest(context.Background())
+		if err != nil {
+			t.Fatal(err)
+		}
+		if req.Body == nil {
 			if len(tt.expected) != 0 {
-				t.Errorf("%d: r.body = %q; want %q", i, r.body, tt.expected)
+				t.Errorf("%d: req.Body = %q; want %q", i, req.Body, tt.expected)
 			}
 			continue
 		}
 		buf := make([]byte, len(tt.expected))
-		if _, err := r.body.Read(buf); err != nil {
-			t.Errorf("%d: r.body.Read error: %v", i, err)
+		if _, err := req.Body.Read(buf); err != nil {
+			t.Errorf("%d: req.Body.Read error: %v", i, err)
 			continue
 		}
 		body := string(buf)
 		if body != tt.expected {
-			t.Errorf("%d: r.body = %q; want %q", i, body, tt.expected)
+			t.Errorf("%d: req.Body = %q; want %q", i, body, tt.expected)
 		}
 	}
 }
@@ -2640,6 +2605,7 @@ func TestRequestWithRetry(t *testing.T) {
 	tests := []struct {
 		name                         string
 		body                         io.Reader
+		bodyBytes                    []byte
 		serverReturns                responseErr
 		errExpected                  error
 		errContains                  string
@@ -2647,53 +2613,53 @@ func TestRequestWithRetry(t *testing.T) {
 		roundTripInvokedExpected     int
 	}{
 		{
-			name:                         "server returns retry-after response, request body is not io.Seeker, retry goes ahead",
-			body:                         io.NopCloser(bytes.NewReader([]byte{})),
+			name:                         "server returns retry-after response, no request body, retry goes ahead",
+			bodyBytes:                    nil,
 			serverReturns:                responseErr{response: retryAfterResponse(), err: nil},
 			errExpected:                  nil,
 			transformFuncInvokedExpected: 1,
 			roundTripInvokedExpected:     2,
 		},
 		{
-			name:                         "server returns retry-after response, request body Seek returns error, retry aborted",
-			body:                         &readSeeker{err: io.EOF},
-			serverReturns:                responseErr{response: retryAfterResponse(), err: nil},
-			errExpected:                  nil,
-			transformFuncInvokedExpected: 0,
-			roundTripInvokedExpected:     1,
-		},
-		{
-			name:                         "server returns retry-after response, request body Seek returns no error, retry goes ahead",
-			body:                         &readSeeker{err: nil},
+			name:                         "server returns retry-after response, bytes request body, retry goes ahead",
+			bodyBytes:                    []byte{},
 			serverReturns:                responseErr{response: retryAfterResponse(), err: nil},
 			errExpected:                  nil,
 			transformFuncInvokedExpected: 1,
 			roundTripInvokedExpected:     2,
 		},
 		{
-			name:                         "server returns retryable err, request body is not io.Seek, retry goes ahead",
-			body:                         io.NopCloser(bytes.NewReader([]byte{})),
-			serverReturns:                responseErr{response: nil, err: io.ErrUnexpectedEOF},
-			errExpected:                  io.ErrUnexpectedEOF,
-			transformFuncInvokedExpected: 0,
-			roundTripInvokedExpected:     2,
-		},
-		{
-			name:                         "server returns retryable err, request body Seek returns error, retry aborted",
-			body:                         &readSeeker{err: io.EOF},
-			serverReturns:                responseErr{response: nil, err: io.ErrUnexpectedEOF},
-			errContains:                  "failed to reset the request body while retrying a request: EOF",
-			transformFuncInvokedExpected: 0,
+			name:                         "server returns retry-after response, opaque request body, retry aborted",
+			body:                         &readSeeker{},
+			serverReturns:                responseErr{response: retryAfterResponse(), err: nil},
+			errExpected:                  nil,
+			transformFuncInvokedExpected: 1,
 			roundTripInvokedExpected:     1,
 		},
 		{
-			name:                         "server returns retryable err, request body Seek returns no err, retry goes ahead",
-			body:                         &readSeeker{err: nil},
+			name:                         "server returns retryable err, no request body, retry goes ahead",
+			bodyBytes:                    nil,
 			serverReturns:                responseErr{response: nil, err: io.ErrUnexpectedEOF},
 			errExpected:                  io.ErrUnexpectedEOF,
 			transformFuncInvokedExpected: 0,
 			roundTripInvokedExpected:     2,
 		},
+		{
+			name:                         "server returns retryable err, bytes request body, retry goes ahead",
+			bodyBytes:                    []byte{},
+			serverReturns:                responseErr{response: nil, err: io.ErrUnexpectedEOF},
+			errExpected:                  io.ErrUnexpectedEOF,
+			transformFuncInvokedExpected: 0,
+			roundTripInvokedExpected:     2,
+		},
+		{
+			name:                         "server returns retryable err, opaque request body, retry aborted",
+			body:                         &readSeeker{},
+			serverReturns:                responseErr{response: nil, err: io.ErrUnexpectedEOF},
+			errExpected:                  io.ErrUnexpectedEOF,
+			transformFuncInvokedExpected: 0,
+			roundTripInvokedExpected:     1,
+		},
 	}

 	for _, test := range tests {
@@ -2864,7 +2830,8 @@ func testRequestWithRetry(t *testing.T, key string, doFunc func(ctx context.Cont
 	tests := []struct {
 		name          string
 		verb          string
-		body          func() io.Reader
+		body          io.Reader
+		bodyBytes     []byte
 		maxRetries    int
 		serverReturns []responseErr

@@ -2874,7 +2841,7 @@ func testRequestWithRetry(t *testing.T, key string, doFunc func(ctx context.Cont
 		{
 			name:       "server always returns retry-after response",
 			verb:       "GET",
-			body:       func() io.Reader { return bytes.NewReader([]byte{}) },
+			bodyBytes:  []byte{},
 			maxRetries: 2,
 			serverReturns: []responseErr{
 				{response: retryAfterResponse(), err: nil},
@@ -2902,7 +2869,7 @@ func testRequestWithRetry(t *testing.T, key string, doFunc func(ctx context.Cont
 		{
 			name:       "server always returns retryable error",
 			verb:       "GET",
-			body:       func() io.Reader { return bytes.NewReader([]byte{}) },
+			bodyBytes:  []byte{},
 			maxRetries: 2,
 			serverReturns: []responseErr{
 				{response: nil, err: io.EOF},
@@ -2931,7 +2898,7 @@ func testRequestWithRetry(t *testing.T, key string, doFunc func(ctx context.Cont
 		{
 			name:       "server returns success on the final retry",
 			verb:       "GET",
-			body:       func() io.Reader { return bytes.NewReader([]byte{}) },
+			bodyBytes:  []byte{},
 			maxRetries: 2,
 			serverReturns: []responseErr{
 				{response: retryAfterResponse(), err: nil},
@@ -2978,13 +2945,10 @@ func testRequestWithRetry(t *testing.T, key string, doFunc func(ctx context.Cont
 				return resp, test.serverReturns[attempts].err
 			})

-			reqCountGot := newCount()
-			reqRecorder := newReadTracker(reqCountGot)
-			reqRecorder.delegated = test.body()
-
 			req := &Request{
-				verb: test.verb,
-				body: reqRecorder,
+				verb:      test.verb,
+				body:      test.body,
+				bodyBytes: test.bodyBytes,
 				c: &RESTClient{
 					content: defaultContentConfig(),
 					Client:  client,
@@ -3004,9 +2968,6 @@ func testRequestWithRetry(t *testing.T, key string, doFunc func(ctx context.Cont
 				t.Errorf("Expected retries: %d, but got: %d", expected.attempts, attempts)
 			}

-			if !reflect.DeepEqual(expected.reqCount.seeks, reqCountGot.seeks) {
-				t.Errorf("Expected request body to have seek invocation: %v, but got: %v", expected.reqCount.seeks, reqCountGot.seeks)
-			}
 			if expected.respCount.closes != respCountGot.getCloseCount() {
 				t.Errorf("Expected response body Close to be invoked %d times, but got: %d", expected.respCount.closes, respCountGot.getCloseCount())
 			}
@@ -3263,8 +3224,8 @@ func testRetryWithRateLimiterBackoffAndMetrics(t *testing.T, key string, doFunc
 				t.Fatalf("Wrong test setup - did not find expected for: %s", key)
 			}
 			req := &Request{
-				verb: "GET",
-				body: bytes.NewReader([]byte{}),
+				verb:      "GET",
+				bodyBytes: []byte{},
 				c: &RESTClient{
 					base:        base,
 					content:     defaultContentConfig(),
@@ -3399,8 +3360,8 @@ func testWithRetryInvokeOrder(t *testing.T, key string, doFunc func(ctx context.
 				t.Fatalf("Wrong test setup - did not find expected for: %s", key)
 			}
 			req := &Request{
-				verb: "GET",
-				body: bytes.NewReader([]byte{}),
+				verb:      "GET",
+				bodyBytes: []byte{},
 				c: &RESTClient{
 					base:    base,
 					content: defaultContentConfig(),
@@ -3574,8 +3535,8 @@ func testWithWrapPreviousError(t *testing.T, doFunc func(ctx context.Context, r
 				t.Fatalf("Failed to create new HTTP request - %v", err)
 			}
 			req := &Request{
-				verb: "GET",
-				body: bytes.NewReader([]byte{}),
+				verb:      "GET",
+				bodyBytes: []byte{},
 				c: &RESTClient{
 					base:    base,
 					content: defaultContentConfig(),
@@ -3810,104 +3771,3 @@ func TestTransportConcurrency(t *testing.T) {
 		})
 	}
 }
-
-// TODO: see if we can consolidate the other trackers into one.
-type requestBodyTracker struct {
-	io.ReadSeeker
-	f func(string)
-}
-
-func (t *requestBodyTracker) Read(p []byte) (int, error) {
-	t.f("Request.Body.Read")
-	return t.ReadSeeker.Read(p)
-}
-
-func (t *requestBodyTracker) Seek(offset int64, whence int) (int64, error) {
-	t.f("Request.Body.Seek")
-	return t.ReadSeeker.Seek(offset, whence)
-}
-
-type responseBodyTracker struct {
-	io.ReadCloser
-	f func(string)
-}
-
-func (t *responseBodyTracker) Read(p []byte) (int, error) {
-	t.f("Response.Body.Read")
-	return t.ReadCloser.Read(p)
-}
-
-func (t *responseBodyTracker) Close() error {
-	t.f("Response.Body.Close")
-	return t.ReadCloser.Close()
-}
-
-type recorder struct {
-	order []string
-}
-
-func (r *recorder) record(call string) {
-	r.order = append(r.order, call)
-}
-
-func TestRequestBodyResetOrder(t *testing.T) {
-	recorder := &recorder{}
-	respBodyTracker := &responseBodyTracker{
-		ReadCloser: nil, // the server will fill it
-		f:          recorder.record,
-	}
-
-	var attempts int
-	client := clientForFunc(func(req *http.Request) (*http.Response, error) {
-		defer func() {
-			attempts++
-		}()
-
-		// read the request body.
-		io.ReadAll(req.Body)
-
-		// first attempt, we send a retry-after
-		if attempts == 0 {
-			resp := retryAfterResponse()
-			respBodyTracker.ReadCloser = io.NopCloser(bytes.NewReader([]byte{}))
-			resp.Body = respBodyTracker
-			return resp, nil
-		}
-
-		return &http.Response{StatusCode: http.StatusOK}, nil
-	})
-
-	reqBodyTracker := &requestBodyTracker{
-		ReadSeeker: bytes.NewReader([]byte{}), // empty body ensures one Read operation at most.
-		f:          recorder.record,
-	}
-	req := &Request{
-		verb: "POST",
-		body: reqBodyTracker,
-		c: &RESTClient{
-			content: defaultContentConfig(),
-			Client:  client,
-		},
-		backoff:    &noSleepBackOff{},
-		maxRetries: 1,
-		retryFn:    defaultRequestRetryFn,
-	}
-
-	req.Do(context.Background())
-
-	expected := []string{
-		// 1st attempt: the server handler reads the request body
-		"Request.Body.Read",
-		// the server sends a retry-after, client reads the
-		// response body, and closes it
-		"Response.Body.Read",
-		"Response.Body.Close",
-		// client retry logic seeks to the beginning of the request body
-		"Request.Body.Seek",
-		// 2nd attempt: the server reads the request body
-		"Request.Body.Read",
-	}
-	if !reflect.DeepEqual(expected, recorder.order) {
-		t.Errorf("Expected invocation request and response body operations for retry do not match: %s", cmp.Diff(expected, recorder.order))
-	}
-}
--- a/rest/with_retry.go
+++ b/rest/with_retry.go
@@ -153,6 +153,11 @@ func (r *withRetry) IsNextRetry(ctx context.Context, restReq *Request, httpReq *
 		return false
 	}

+	if restReq.body != nil {
+		// we have an opaque reader, we can't safely reset it
+		return false
+	}
+
 	r.attempts++
 	r.retryAfter = &RetryAfter{Attempt: r.attempts}
 	if r.attempts > r.maxRetries {
@@ -209,18 +214,6 @@ func (r *withRetry) Before(ctx context.Context, request *Request) error {
 		return nil
 	}

-	// At this point we've made atleast one attempt, post which the response
-	// body should have been fully read and closed in order for it to be safe
-	// to reset the request body before we reconnect, in order for us to reuse
-	// the same TCP connection.
-	if seeker, ok := request.body.(io.Seeker); ok && request.body != nil {
-		if _, err := seeker.Seek(0, io.SeekStart); err != nil {
-			err = fmt.Errorf("failed to reset the request body while retrying a request: %v", err)
-			r.trackPreviousError(err)
-			return err
-		}
-	}
-
 	// if we are here, we have made attempt(s) at least once before.
 	if request.backoff != nil {
 		delay := request.backoff.CalculateBackoff(url)
--- a/rest/with_retry_test.go
+++ b/rest/with_retry_test.go
@@ -17,7 +17,6 @@ limitations under the License.
 package rest

 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -212,7 +211,7 @@ func TestIsNextRetry(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			restReq := &Request{
-				body: bytes.NewReader([]byte{}),
+				bodyBytes: []byte{},
 				c: &RESTClient{
 					base: &url.URL{},
 				},
--- a/transport/cache.go
+++ b/transport/cache.go
@@ -109,7 +109,7 @@ func (c *tlsTransportCache) get(config *Config) (http.RoundTripper, error) {

 	// If we use are reloading files, we need to handle certificate rotation properly
 	// TODO(jackkleeman): We can also add rotation here when config.HasCertCallback() is true
-	if config.TLS.ReloadTLSFiles {
+	if config.TLS.ReloadTLSFiles && tlsConfig != nil && tlsConfig.GetClientCertificate != nil {
 		dynamicCertDialer := certRotatingDialer(tlsConfig.GetClientCertificate, dial)
 		tlsConfig.GetClientCertificate = dynamicCertDialer.GetClientCertificate
 		dial = dynamicCertDialer.connDialer.DialContext
Author	SHA1	Message	Date
Kubernetes Publisher	8cbca742ae	Update dependencies to v0.26.3 tag	2023-03-17 21:01:42 +00:00
Kubernetes Publisher	87720b3719	Merge pull request #116437 from seans3/automated-cherry-pick-of-#116145-#115865-origin-release-1.26 Automated cherry pick of #116145: Plumb stale GroupVersions through aggregated discovery #115865: Removes old discovery hack ignoring 403 and 404 Kubernetes-commit: 052842af4ddb7f01ed8c7d248b59e80ad28175ad	2023-03-10 14:34:48 -08:00
Sean Sullivan	fc13749c0d	Removes old discovery hack ignoring 403 and 404 Kubernetes-commit: 6968f56567c90ea4329448a9185445bb1f114295	2023-02-17 12:47:05 -08:00
Sean Sullivan	f39ba12dc2	Plumb stale GroupVersions through aggregated discovery Kubernetes-commit: 363bcdd815c051e52954b1aab1cd503dfc19bff7	2023-02-28 19:44:34 +00:00
Kubernetes Publisher	f538edfed7	Merge pull request #116352 from seans3/automated-cherry-pick-of-#115978-origin-release-1.26 Automated cherry pick of #115978: Tolerate empty discovery response in memcache client Kubernetes-commit: 1e8562fd21d70ae49454fcda83dc9c92250e6350	2023-03-08 23:24:00 -08:00
Sean Sullivan	5dbbc58c50	Tolerate empty discovery response in memcache client Kubernetes-commit: 6e8addd9a0a9e2983e3040e337b1b7ba6df83d87	2023-02-22 23:59:37 +00:00
Kubernetes Publisher	62133a9b18	Merge pull request #115787 from liggitt/net-0.7.0-1.26 [1.26] Update golang.org/x/net to v0.7.0 Kubernetes-commit: 0fbdfcec8d02bc9737ea9916efad04d626928036	2023-02-15 11:37:35 +00:00
Jordan Liggitt	8ce239ff60	Update golang.org/x/net to v0.7.0 Kubernetes-commit: 26d4675e15df0fcc9a749e66fb3bdb82871f84d1	2023-02-14 23:18:23 -05:00
Kubernetes Publisher	e6bc0bccc2	Merge pull request #115566 from enj/automated-cherry-pick-of-#115315-upstream-release-1.26 Automated cherry pick of #115315: kubelet/client: collapse transport wiring onto standard Kubernetes-commit: 1802182ca3f88c0d0db79e46757e9158c9b187dc	2023-02-10 22:54:13 +00:00
Kubernetes Publisher	9112e1916f	Merge pull request #115400 from pohly/automated-cherry-pick-of-#115354-origin-release-1.26 Automated cherry pick of #115354: dynamic resource allocation: avoid apiserver complaint about Kubernetes-commit: 41137e4de44ad12b86152359abc46414692906d1	2023-02-10 15:36:48 +00:00
Kubernetes Publisher	2e3434888b	Merge pull request #115642 from nckturner/pin-golang.org/x/net-to-v0.4.0-in-1.26 Pin golang.org/x/net to v0.4.0 in 1.26 Kubernetes-commit: 5ae2e75abe3a631e770b7136f8e3bd2a6051ba18	2023-02-10 15:36:45 +00:00
Nick Turner	4968c4a2f5	Pin golang.org/x/net to v0.4.0 in 1.26 Kubernetes-commit: c5fe6226c4aee24c100cf35fdf105ad3abf82026	2023-02-08 18:06:10 -08:00
Monis Khan	0519b53357	kubelet/client: collapse transport wiring onto standard approach Signed-off-by: Monis Khan <mok@microsoft.com> Kubernetes-commit: c651e4f7da1c43ddd956fbba303e990d6f27130a	2023-02-05 20:51:54 -05:00
Patrick Ohly	7be38cd631	dynamic resource allocation: avoid apiserver complaint about list content This fixes the following warning (error?) in the apiserver: E0126 18:10:38.665239 16370 fieldmanager.go:210] "[SHOULD NOT HAPPEN] failed to update managedFields" err="failed to convert new object (test/claim-84; resource.k8s.io/v1alpha1, Kind=ResourceClaim) to smd typed: .status.reservedFor: element 0: associative list without keys has an element that's a map type" VersionKind="/, Kind=" namespace="test" name="claim-84" The root cause is the same as in e50e8a0c919c0e02dc9a0ffaebb685d5348027b4: nothing in Kubernetes outright complains about a list of items where the item type is comparable in Go, but not a simple type. This nonetheless isn't supposed to be done in the API and can causes problems elsewhere. For the ReservedFor field, everything seems to work okay except for the warning. However, it's better to follow conventions and use a map. This is possible in this case because UID is guaranteed to be a unique key. Validation is now stricter than before, which is a good thing: previously, two entries with the same UID were allowed as long as some other field was different, which wasn't a situation that should have been allowed. Kubernetes-commit: b10dce49c3cb782404e09f50547120a736c03969	2023-01-26 20:37:00 +01:00
Kubernetes Publisher	0c34939c9b	Merge pull request #114617 from JoelSpeed/automated-cherry-pick-of-#114585-upstream-release-1.26 Automated cherry pick of #114585: Resource claims should be a map type Kubernetes-commit: c090810c4c96e0c5acc05ab2094a5d46669cae86	2022-12-27 21:42:33 +00:00
Joel Speed	04b098b4ef	Resource claims should be a map type Kubernetes-commit: 5e22175b1d9b623c3db4d7f61ab881bb17b2795c	2022-12-19 16:02:02 +00:00
Kubernetes Publisher	b3fff46496	Merge pull request #114415 from hoskeri/automated-cherry-pick-of-#114404-upstream-release-1.26 Automated cherry pick of #114404: Check the correct error in d.downloadAPIs Kubernetes-commit: dd0b0c00e5a10352fa74a09fb32aca509c7f0c48	2022-12-13 17:06:47 +00:00
Kubernetes Publisher	236db3c56e	Merge pull request #113988 from liggitt/automated-cherry-pick-of-#113933-upstream-release-1.26 [1.26.1] Automated cherry pick of #113933: Limit request retrying to []byte request bodies Kubernetes-commit: 7c0eb3be77cb388bc98a9ddc33371e0b6d3c27da	2022-12-13 00:11:16 +00:00
Abhijit Hoskeri	a2ef32442a	Check the correct error in d.downloadAPIs The error result of `d.downloadAPIs()` is set in `aerr`, not `err`. This prevents a nil-ptr dereference of apiGroups in the next step. Signed-off-by: Abhijit Hoskeri <abhijithoskeri@gmail.com> Kubernetes-commit: f8b99b1f09fb5d4d10b15e326c4b242cc705f007	2022-12-10 16:32:02 -08:00
Jordan Liggitt	ebb499fa8a	Limit request retrying to []byte request bodies Kubernetes-commit: 40f01d0c811c49a22b7557f7d8d06e3af6b4cabd	2022-11-15 17:47:35 -05:00