From de05bd811e2187bcc6785e4868528ddceaa49fe9 Mon Sep 17 00:00:00 2001 From: wojtekt Date: Thu, 16 Apr 2020 18:59:14 +0200 Subject: [PATCH] Lazy initialization of network urls for GCE provider --- .../k8s.io/legacy-cloud-providers/gce/gce.go | 98 ++++++++++++------- .../gce/gce_loadbalancer_internal.go | 2 +- 2 files changed, 66 insertions(+), 34 deletions(-) diff --git a/staging/src/k8s.io/legacy-cloud-providers/gce/gce.go b/staging/src/k8s.io/legacy-cloud-providers/gce/gce.go index c4c33e446ba..cae825c1203 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/gce/gce.go +++ b/staging/src/k8s.io/legacy-cloud-providers/gce/gce.go @@ -99,6 +99,13 @@ type Cloud struct { // for the cloudprovider to start watching the configmap. ClusterID ClusterID + // initializer is used for lazy initialization of subnetworkURL + // and isLegacyNetwork fields if they are not passed via the config. + // The reason is to avoid GCE API calls to initialize them if they + // will never be used. This is especially important when + // it is run from Kubelets, as there can be thousands of them. + subnetworkURLAndIsLegacyNetworkInitializer sync.Once + service *compute.Service serviceBeta *computebeta.Service serviceAlpha *computealpha.Service @@ -115,10 +122,14 @@ type Cloud struct { // managedZones will be set to the 1 zone if running a single zone cluster // it will be set to ALL zones in region for any multi-zone cluster // Use GetAllCurrentZones to get only zones that contain nodes - managedZones []string - networkURL string - isLegacyNetwork bool - subnetworkURL string + managedZones []string + networkURL string + // unsafeIsLegacyNetwork should be used only via IsLegacyNetwork() accessor, + // to ensure it was properly initialized. + unsafeIsLegacyNetwork bool + // unsafeSubnetworkURL should be used only via SubnetworkURL() accessor, + // to ensure it was properly initialized. + unsafeSubnetworkURL string secondaryRangeName string networkProjectID string onXPN bool @@ -465,32 +476,12 @@ func CreateGCECloud(config *CloudConfig) (*Cloud, error) { subnetURL = config.SubnetworkURL } else if config.SubnetworkName != "" { subnetURL = gceSubnetworkURL(config.APIEndpoint, netProjID, config.Region, config.SubnetworkName) - } else { - // Determine the type of network and attempt to discover the correct subnet for AUTO mode. - // Gracefully fail because kubelet calls CreateGCECloud without any config, and minions - // lack the proper credentials for API calls. - if networkName := lastComponent(networkURL); networkName != "" { - var n *compute.Network - if n, err = getNetwork(service, netProjID, networkName); err != nil { - klog.Warningf("Could not retrieve network %q; err: %v", networkName, err) - } else { - switch typeOfNetwork(n) { - case netTypeLegacy: - klog.Infof("Network %q is type legacy - no subnetwork", networkName) - isLegacyNetwork = true - case netTypeCustom: - klog.Warningf("Network %q is type custom - cannot auto select a subnetwork", networkName) - case netTypeAuto: - subnetURL, err = determineSubnetURL(service, netProjID, networkName, config.Region) - if err != nil { - klog.Warningf("Could not determine subnetwork for network %q and region %v; err: %v", networkName, config.Region, err) - } else { - klog.Infof("Auto selecting subnetwork %q", subnetURL) - } - } - } - } } + // If neither SubnetworkURL nor SubnetworkName are provided, defer to + // lazy initialization. Determining subnetURL and isLegacyNetwork requires + // GCE API call. Given that it's not used in many cases and the fact that + // the provider is initialized also for Kubelets (and there can be thousands + // of them) we defer to lazy initialization here. if len(config.ManagedZones) == 0 { config.ManagedZones, err = getZonesForRegion(service, config.ProjectID, config.Region) @@ -518,8 +509,8 @@ func CreateGCECloud(config *CloudConfig) (*Cloud, error) { localZone: config.Zone, managedZones: config.ManagedZones, networkURL: networkURL, - isLegacyNetwork: isLegacyNetwork, - subnetworkURL: subnetURL, + unsafeIsLegacyNetwork: isLegacyNetwork, + unsafeSubnetworkURL: subnetURL, secondaryRangeName: config.SecondaryRangeName, nodeTags: config.NodeTags, nodeInstancePrefix: config.NodeInstancePrefix, @@ -542,6 +533,45 @@ func CreateGCECloud(config *CloudConfig) (*Cloud, error) { return gce, nil } +// initializeNetworkConfig() is supposed to be called under sync.Once() +// for accessors to subnetworkURL and isLegacyNetwork fields. +func (g *Cloud) initializeSubnetworkURLAndIsLegacyNetwork() { + if g.unsafeSubnetworkURL != "" { + // This has already been initialized via the config. + return + } + + var subnetURL string + var isLegacyNetwork bool + + // Determine the type of network and attempt to discover the correct subnet for AUTO mode. + // Gracefully fail because kubelet calls CreateGCECloud without any config, and minions + // lack the proper credentials for API calls. + if networkName := lastComponent(g.NetworkURL()); networkName != "" { + if n, err := getNetwork(g.service, g.NetworkProjectID(), networkName); err != nil { + klog.Warningf("Could not retrieve network %q; err: %v", networkName, err) + } else { + switch typeOfNetwork(n) { + case netTypeLegacy: + klog.Infof("Network %q is type legacy - no subnetwork", networkName) + isLegacyNetwork = true + case netTypeCustom: + klog.Warningf("Network %q is type custom - cannot auto select a subnetwork", networkName) + case netTypeAuto: + subnetURL, err = determineSubnetURL(g.service, g.NetworkProjectID(), networkName, g.Region()) + if err != nil { + klog.Warningf("Could not determine subnetwork for network %q and region %v; err: %v", networkName, g.Region(), err) + } else { + klog.Infof("Auto selecting subnetwork %q", subnetURL) + } + } + } + } + + g.unsafeSubnetworkURL = subnetURL + g.unsafeIsLegacyNetwork = isLegacyNetwork +} + // SetRateLimiter adds a custom cloud.RateLimiter implementation. // WARNING: Calling this could have unexpected behavior if you have in-flight // requests. It is best to use this immediately after creating a Cloud. @@ -672,12 +702,14 @@ func (g *Cloud) NetworkURL() string { // SubnetworkURL returns the subnetwork url func (g *Cloud) SubnetworkURL() string { - return g.subnetworkURL + g.subnetworkURLAndIsLegacyNetworkInitializer.Do(g.initializeSubnetworkURLAndIsLegacyNetwork) + return g.unsafeSubnetworkURL } // IsLegacyNetwork returns true if the cluster is still running a legacy network configuration. func (g *Cloud) IsLegacyNetwork() bool { - return g.isLegacyNetwork + g.subnetworkURLAndIsLegacyNetworkInitializer.Do(g.initializeSubnetworkURLAndIsLegacyNetwork) + return g.unsafeIsLegacyNetwork } // SetInformers sets up the zone handlers we need watching for node changes. diff --git a/staging/src/k8s.io/legacy-cloud-providers/gce/gce_loadbalancer_internal.go b/staging/src/k8s.io/legacy-cloud-providers/gce/gce_loadbalancer_internal.go index 0aaea68b4f2..d2f56a05095 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/gce/gce_loadbalancer_internal.go +++ b/staging/src/k8s.io/legacy-cloud-providers/gce/gce_loadbalancer_internal.go @@ -75,7 +75,7 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v } scheme := cloud.SchemeInternal options := getILBOptions(svc) - if g.isLegacyNetwork { + if g.IsLegacyNetwork() { g.eventRecorder.Event(svc, v1.EventTypeWarning, "ILBOptionsIgnored", "Internal LoadBalancer options are not supported with Legacy Networks.") options = ILBOptions{} }