github/client-go
1
0
Fork 0
mirror of https://github.com/kubernetes/client-go.git synced 2025-06-20 12:23:22 +00:00
Code Issues Projects Releases Wiki Activity

Run resource discovery in parallel

Browse Source 
Kubernetes-commit: d037b0893fa9313b9b5b111c22d49bf3c859a9d1
This commit is contained in:
Jordan Liggitt 2018-05-07 15:54:25 -04:00 committed by Kubernetes Publisher
parent 62c4633c17
commit 334b50415a
2 changed files with 89 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
discovery/discovery_client.go
View File

@ -22,6 +22,7 @@ import (
"net/url"
"sort"
"strings"
"sync"
"time"
"github.com/golang/protobuf/proto"
@ -32,6 +33,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/version"
"k8s.io/client-go/kubernetes/scheme"
restclient "k8s.io/client-go/rest"
@ -191,33 +193,7 @@ func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (r
// serverResources returns the supported resources for all groups and versions.
func (d *DiscoveryClient) serverResources() ([]*metav1.APIResourceList, error) {
apiGroups, err := d.ServerGroups()
if err != nil {
return nil, err
}
result := []*metav1.APIResourceList{}
failedGroups := make(map[schema.GroupVersion]error)
for _, apiGroup := range apiGroups.Groups {
for _, version := range apiGroup.Versions {
gv := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}
resources, err := d.ServerResourcesForGroupVersion(version.GroupVersion)
if err != nil {
// TODO: maybe restrict this to NotFound errors
failedGroups[gv] = err
continue
}
result = append(result, resources)
}
}
if len(failedGroups) == 0 {
return result, nil
}
return result, &ErrGroupDiscoveryFailed{Groups: failedGroups}
return ServerResources(d)
}
// ServerResources returns the supported resources for all groups and versions.
@ -253,6 +229,33 @@ func (d *DiscoveryClient) serverPreferredResources() ([]*metav1.APIResourceList,
return ServerPreferredResources(d)
}
// ServerResources uses the provided discovery interface to look up supported resources for all groups and versions.
func ServerResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) {
apiGroups, err := d.ServerGroups()
if err != nil {
return nil, err
}
groupVersionResources, failedGroups := fetchGroupVersionResources(d, apiGroups)
// order results by group/version discovery order
result := []*metav1.APIResourceList{}
for _, apiGroup := range apiGroups.Groups {
for _, version := range apiGroup.Versions {
gv := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}
if resources, ok := groupVersionResources[gv]; ok {
result = append(result, resources)
}
}
}
if len(failedGroups) == 0 {
return result, nil
}
return result, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
// ServerPreferredResources uses the provided discovery interface to look up preferred resources
func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList, error) {
serverGroupList, err := d.ServerGroups()
@ -260,9 +263,9 @@ func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList,
return nil, err
}
result := []*metav1.APIResourceList{}
failedGroups := make(map[schema.GroupVersion]error)
groupVersionResources, failedGroups := fetchGroupVersionResources(d, serverGroupList)
result := []*metav1.APIResourceList{}
grVersions := map[schema.GroupResource]string{} // selected version of a GroupResource
grApiResources := map[schema.GroupResource]*metav1.APIResource{} // selected APIResource for a GroupResource
gvApiResourceLists := map[schema.GroupVersion]*metav1.APIResourceList{} // blueprint for a APIResourceList for later grouping
@ -270,10 +273,9 @@ func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList,
for _, apiGroup := range serverGroupList.Groups {
for _, version := range apiGroup.Versions {
groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}
apiResourceList, err := d.ServerResourcesForGroupVersion(version.GroupVersion)
if err != nil {
// TODO: maybe restrict this to NotFound errors
failedGroups[groupVersion] = err
apiResourceList, ok := groupVersionResources[groupVersion]
if !ok {
continue
}
@ -315,6 +317,41 @@ func ServerPreferredResources(d DiscoveryInterface) ([]*metav1.APIResourceList,
return result, &ErrGroupDiscoveryFailed{Groups: failedGroups}
}
// fetchServerResourcesForGroupVersions uses the discovery client to fetch the resources for the specified groups in parallel
func fetchGroupVersionResources(d DiscoveryInterface, apiGroups *metav1.APIGroupList) (map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error) {
groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList)
failedGroups := make(map[schema.GroupVersion]error)
wg := &sync.WaitGroup{}
resultLock := &sync.Mutex{}
for _, apiGroup := range apiGroups.Groups {
for _, version := range apiGroup.Versions {
groupVersion := schema.GroupVersion{Group: apiGroup.Name, Version: version.Version}
wg.Add(1)
go func() {
defer wg.Done()
defer utilruntime.HandleCrash()
apiResourceList, err := d.ServerResourcesForGroupVersion(groupVersion.String())
// lock to record results
resultLock.Lock()
defer resultLock.Unlock()
if err != nil {
// TODO: maybe restrict this to NotFound errors
failedGroups[groupVersion] = err
} else {
groupVersionResources[groupVersion] = apiResourceList
}
}()
}
}
wg.Wait()
return groupVersionResources, failedGroups
}
// ServerPreferredResources returns the supported resources with the version preferred by the
// server.
func (d *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, error) {

26
discovery/discovery_client_test.go
View File

@ -32,6 +32,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/diff"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/version"
restclient "k8s.io/client-go/rest"
@ -204,6 +205,14 @@ func TestGetServerResources(t *testing.T) {
{Name: "jobs", Namespaced: true, Kind: "Job"},
},
}
beta2 := metav1.APIResourceList{
GroupVersion: "extensions/v1beta2",
APIResources: []metav1.APIResource{
{Name: "deployments", Namespaced: true, Kind: "Deployment"},
{Name: "ingresses", Namespaced: true, Kind: "Ingress"},
{Name: "jobs", Namespaced: true, Kind: "Job"},
},
}
tests := []struct {
resourcesList *metav1.APIResourceList
path string
@ -236,6 +245,8 @@ func TestGetServerResources(t *testing.T) {
list = &stable
case "/apis/extensions/v1beta1":
list = &beta
case "/apis/extensions/v1beta2":
list = &beta2
case "/api":
list = &metav1.APIVersions{
Versions: []string{
@ -246,8 +257,10 @@ func TestGetServerResources(t *testing.T) {
list = &metav1.APIGroupList{
Groups: []metav1.APIGroup{
{
Name: "extensions",
Versions: []metav1.GroupVersionForDiscovery{
{GroupVersion: "extensions/v1beta1"},
{GroupVersion: "extensions/v1beta1", Version: "v1beta1"},
{GroupVersion: "extensions/v1beta2", Version: "v1beta2"},
},
},
},
@ -289,11 +302,10 @@ func TestGetServerResources(t *testing.T) {
if err != nil {
t.Errorf("unexpected error: %v", err)
}
serverGroupVersions := sets.NewString(groupVersions(serverResources)...)
for _, api := range []string{"v1", "extensions/v1beta1"} {
if !serverGroupVersions.Has(api) {
t.Errorf("missing expected api %q in %v", api, serverResources)
}
serverGroupVersions := groupVersions(serverResources)
expectedGroupVersions := []string{"v1", "extensions/v1beta1", "extensions/v1beta2"}
if !reflect.DeepEqual(expectedGroupVersions, serverGroupVersions) {
t.Errorf("unexpected group versions: %v", diff.ObjectReflectDiff(expectedGroupVersions, serverGroupVersions))
}
}
@ -639,7 +651,7 @@ func TestServerPreferredResourcesRetries(t *testing.T) {
{
Name: "extensions",
Versions: []metav1.GroupVersionForDiscovery{
{GroupVersion: "extensions/v1beta1"},
{GroupVersion: "extensions/v1beta1", Version: "v1beta1"},
},
PreferredVersion: metav1.GroupVersionForDiscovery{
GroupVersion: "extensions/v1beta1",