From e0ca8abba8454e76c5e7096f8c4f212b6b873c3e Mon Sep 17 00:00:00 2001 From: "Madhusudan.C.S" Date: Mon, 17 Apr 2017 18:31:58 -0700 Subject: [PATCH] Update Google Cloud DNS List implementation to perform a paged walk of lists to aggregate all the DNS records. The current `List()` implementation just lists the DNS resorce records in a given managed zone once and retruns the list. It neither performs a paged walk nor does it consider the `page_token` in the returned response. This change walks all the pages and aggregates the records in the pages and returns the aggregated list. This is potentially dangerous as it can blow up memory if there are a huge number of records in the given managed zone. But this is the best we can do without changing the provider interface too much. Next step is to define a new paged list interface and implement it. --- .../internal/interfaces/interfaces.go | 4 +++- .../clouddns/internal/rrsets_list_call.go | 8 +++++++ .../internal/stubs/rrsets_list_call.go | 6 +++++ .../providers/google/clouddns/rrsets.go | 23 +++++++++++++++---- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/federation/pkg/dnsprovider/providers/google/clouddns/internal/interfaces/interfaces.go b/federation/pkg/dnsprovider/providers/google/clouddns/internal/interfaces/interfaces.go index c508f4c1116..7d9f1663984 100644 --- a/federation/pkg/dnsprovider/providers/google/clouddns/internal/interfaces/interfaces.go +++ b/federation/pkg/dnsprovider/providers/google/clouddns/internal/interfaces/interfaces.go @@ -17,6 +17,8 @@ limitations under the License. package interfaces import ( + "context" + "google.golang.org/api/googleapi" "k8s.io/kubernetes/federation/pkg/dnsprovider/rrstype" ) @@ -172,8 +174,8 @@ type ( ResourceRecordSetsListCall interface { // Context(ctx context.Context) *ResourceRecordSetsListCall // TODO: Add as needed - // Do(opts ...googleapi.CallOption) (*ResourceRecordSetsListResponse, error) // TODO: Add as needed Do(opts ...googleapi.CallOption) (ResourceRecordSetsListResponse, error) + Pages(ctx context.Context, f func(ResourceRecordSetsListResponse) error) error // Fields(s ...googleapi.Field) *ResourceRecordSetsListCall // TODO: Add as needed // IfNoneMatch(entityTag string) *ResourceRecordSetsListCall // TODO: Add as needed // MaxResults(maxResults int64) *ResourceRecordSetsListCall // TODO: Add as needed diff --git a/federation/pkg/dnsprovider/providers/google/clouddns/internal/rrsets_list_call.go b/federation/pkg/dnsprovider/providers/google/clouddns/internal/rrsets_list_call.go index 32c17011fe4..f497ebcfbce 100644 --- a/federation/pkg/dnsprovider/providers/google/clouddns/internal/rrsets_list_call.go +++ b/federation/pkg/dnsprovider/providers/google/clouddns/internal/rrsets_list_call.go @@ -17,6 +17,8 @@ limitations under the License. package internal import ( + "context" + dns "google.golang.org/api/dns/v1" "google.golang.org/api/googleapi" "k8s.io/kubernetes/federation/pkg/dnsprovider/providers/google/clouddns/internal/interfaces" @@ -34,6 +36,12 @@ func (call *ResourceRecordSetsListCall) Do(opts ...googleapi.CallOption) (interf return &ResourceRecordSetsListResponse{response}, err } +func (call *ResourceRecordSetsListCall) Pages(ctx context.Context, f func(interfaces.ResourceRecordSetsListResponse) error) error { + return call.impl.Pages(ctx, func(page *dns.ResourceRecordSetsListResponse) error { + return f(&ResourceRecordSetsListResponse{page}) + }) +} + func (call *ResourceRecordSetsListCall) Name(name string) interfaces.ResourceRecordSetsListCall { call.impl.Name(name) return call diff --git a/federation/pkg/dnsprovider/providers/google/clouddns/internal/stubs/rrsets_list_call.go b/federation/pkg/dnsprovider/providers/google/clouddns/internal/stubs/rrsets_list_call.go index 5b298b767a6..8695142bff5 100644 --- a/federation/pkg/dnsprovider/providers/google/clouddns/internal/stubs/rrsets_list_call.go +++ b/federation/pkg/dnsprovider/providers/google/clouddns/internal/stubs/rrsets_list_call.go @@ -17,6 +17,8 @@ limitations under the License. package stubs import ( + "context" + "google.golang.org/api/googleapi" "k8s.io/kubernetes/federation/pkg/dnsprovider/providers/google/clouddns/internal/interfaces" ) @@ -35,6 +37,10 @@ func (call *ResourceRecordSetsListCall) Do(opts ...googleapi.CallOption) (interf return call.Response_, call.Err_ } +func (call *ResourceRecordSetsListCall) Pages(ctx context.Context, f func(interfaces.ResourceRecordSetsListResponse) error) error { + return f(call.Response_) +} + func (call *ResourceRecordSetsListCall) Name(name string) interfaces.ResourceRecordSetsListCall { call.Name_ = name return call diff --git a/federation/pkg/dnsprovider/providers/google/clouddns/rrsets.go b/federation/pkg/dnsprovider/providers/google/clouddns/rrsets.go index c7cfa9731dd..db9411c7661 100644 --- a/federation/pkg/dnsprovider/providers/google/clouddns/rrsets.go +++ b/federation/pkg/dnsprovider/providers/google/clouddns/rrsets.go @@ -17,6 +17,8 @@ limitations under the License. package clouddns import ( + "context" + "k8s.io/kubernetes/federation/pkg/dnsprovider" "k8s.io/kubernetes/federation/pkg/dnsprovider/providers/google/clouddns/internal/interfaces" "k8s.io/kubernetes/federation/pkg/dnsprovider/rrstype" @@ -30,15 +32,26 @@ type ResourceRecordSets struct { impl interfaces.ResourceRecordSetsService } +// List returns a list of resource records in the given project and +// managed zone. +// !!CAUTION!! Your memory might explode if you have a huge number of +// records in your managed zone. func (rrsets ResourceRecordSets) List() ([]dnsprovider.ResourceRecordSet, error) { - response, err := rrsets.impl.List(rrsets.project(), rrsets.zone.impl.Name()).Do() + var list []dnsprovider.ResourceRecordSet + + ctx := context.Background() + + call := rrsets.impl.List(rrsets.project(), rrsets.zone.impl.Name()) + err := call.Pages(ctx, func(page interfaces.ResourceRecordSetsListResponse) error { + for _, rrset := range page.Rrsets() { + list = append(list, ResourceRecordSet{rrset, &rrsets}) + } + return nil + }) if err != nil { return nil, err } - list := make([]dnsprovider.ResourceRecordSet, len(response.Rrsets())) - for i, rrset := range response.Rrsets() { - list[i] = ResourceRecordSet{rrset, &rrsets} - } + return list, nil }