Merge pull request #51410 from nicksardo/gce-consume-netproj

Automatic merge from submit-queue (batch tested with PRs 50919, 51410, 50099, 51300, 50296)

GCE: Read networkProjectID param

Fixes #48515 

/assign bowei

The first commit is the original PR cherrypicked. The master's kubelet isn't provided a cloud config path, so the project is retrieved via instance metadata. In the GKE case, this project cannot be retrieved by the master and caused an error.

**Release note**:
```release-note
NONE
```

pkg/cloudprovider/providers/gce/gce.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"net/http"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -144,42 +145,51 @@ type GCEServiceManager struct {
 	gce *GCECloud
 }
 
+type ConfigGlobal struct {
+	TokenURL  string `gcfg:"token-url"`
+	TokenBody string `gcfg:"token-body"`
+	// ProjectID and NetworkProjectID can either be the numeric or string-based
+	// unique identifier that starts with [a-z].
+	ProjectID string `gcfg:"project-id"`
+	// NetworkProjectID refers to the project which owns the network being used.
+	NetworkProjectID string `gcfg:"network-project-id"`
+	NetworkName      string `gcfg:"network-name"`
+	SubnetworkName   string `gcfg:"subnetwork-name"`
+	// SecondaryRangeName is the name of the secondary range to allocate IP
+	// aliases. The secondary range must be present on the subnetwork the
+	// cluster is attached to.
+	SecondaryRangeName string   `gcfg:"secondary-range-name"`
+	NodeTags           []string `gcfg:"node-tags"`
+	NodeInstancePrefix string   `gcfg:"node-instance-prefix"`
+	Multizone          bool     `gcfg:"multizone"`
+	// ApiEndpoint is the GCE compute API endpoint to use. If this is blank,
+	// then the default endpoint is used.
+	ApiEndpoint string `gcfg:"api-endpoint"`
+	// LocalZone specifies the GCE zone that gce cloud client instance is
+	// located in (i.e. where the controller will be running). If this is
+	// blank, then the local zone will be discovered via the metadata server.
+	LocalZone string `gcfg:"local-zone"`
+	// Possible values: List of api names separated by comma. Default to none.
+	// For example: MyFeatureFlag
+	AlphaFeatures []string `gcfg:"alpha-features"`
+}
+
 // ConfigFile is the struct used to parse the /etc/gce.conf configuration file.
 type ConfigFile struct {
-	Global struct {
+	Global ConfigGlobal `gcfg:"global"`
-		TokenURL       string `gcfg:"token-url"`
-		TokenBody      string `gcfg:"token-body"`
-		ProjectID      string `gcfg:"project-id"`
-		NetworkName    string `gcfg:"network-name"`
-		SubnetworkName string `gcfg:"subnetwork-name"`
-		// SecondaryRangeName is the name of the secondary range to allocate IP
-		// aliases. The secondary range must be present on the subnetwork the
-		// cluster is attached to.
-		SecondaryRangeName string   `gcfg:"secondary-range-name"`
-		NodeTags           []string `gcfg:"node-tags"`
-		NodeInstancePrefix string   `gcfg:"node-instance-prefix"`
-		Multizone          bool     `gcfg:"multizone"`
-		// ApiEndpoint is the GCE compute API endpoint to use. If this is blank,
-		// then the default endpoint is used.
-		ApiEndpoint string `gcfg:"api-endpoint"`
-		// LocalZone specifies the GCE zone that gce cloud client instance is
-		// located in (i.e. where the controller will be running). If this is
-		// blank, then the local zone will be discovered via the metadata server.
-		LocalZone string `gcfg:"local-zone"`
-		// AlphaFeatures is a list of API flags to be enabled. Defaults to none.
-		// Example API name format: "MyFeatureFlag"
-		AlphaFeatures []string `gcfg:"alpha-features"`
-	}
 }
 
 // CloudConfig includes all the necessary configuration for creating GCECloud
 type CloudConfig struct {
 	ApiEndpoint        string
 	ProjectID          string
+	NetworkProjectID   string
 	Region             string
 	Zone               string
 	ManagedZones       []string
+	NetworkName        string
 	NetworkURL         string
+	SubnetworkName     string
 	SubnetworkURL      string
 	SecondaryRangeName string
 	NodeTags           []string
@@ -207,11 +217,6 @@ func (g *GCECloud) GetKMSService() *cloudkms.Service {
 	return g.cloudkmsService
 }
 
-// Returns the ProjectID corresponding to the project this cloud is in.
-func (g *GCECloud) GetProjectID() string {
-	return g.projectID
-}
-
 // newGCECloud creates a new instance of GCECloud.
 func newGCECloud(config io.Reader) (gceCloud *GCECloud, err error) {
 	var cloudConfig *CloudConfig
@@ -280,6 +285,7 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
 			return nil, err
 		}
 	}
+
 	if configFile != nil {
 		if configFile.Global.ProjectID != "" {
 			cloudConfig.ProjectID = configFile.Global.ProjectID
@@ -287,6 +293,9 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
 		if configFile.Global.LocalZone != "" {
 			cloudConfig.Zone = configFile.Global.LocalZone
 		}
+		if configFile.Global.NetworkProjectID != "" {
+			cloudConfig.NetworkProjectID = configFile.Global.NetworkProjectID
+		}
 	}
 
 	// retrieve region
@@ -301,27 +310,27 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
 		cloudConfig.ManagedZones = nil // Use all zones in region
 	}
 
-	// generate networkURL
+	// Determine if network parameter is URL or Name
 	if configFile != nil && configFile.Global.NetworkName != "" {
 		if strings.Contains(configFile.Global.NetworkName, "/") {
 			cloudConfig.NetworkURL = configFile.Global.NetworkName
 		} else {
-			cloudConfig.NetworkURL = gceNetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, configFile.Global.NetworkName)
+			cloudConfig.NetworkName = configFile.Global.NetworkName
 		}
 	} else {
-		networkName, err := getNetworkNameViaMetadata()
+		cloudConfig.NetworkName, err = getNetworkNameViaMetadata()
 		if err != nil {
 			return nil, err
 		}
-		cloudConfig.NetworkURL = gceNetworkURL("", cloudConfig.ProjectID, networkName)
 	}
 
-	// generate subnetworkURL
+	// Determine if subnetwork parameter is URL or Name
+	// If cluster is on a GCP network of mode=custom, then `SubnetName` must be specified in config file.
 	if configFile != nil && configFile.Global.SubnetworkName != "" {
 		if strings.Contains(configFile.Global.SubnetworkName, "/") {
 			cloudConfig.SubnetworkURL = configFile.Global.SubnetworkName
 		} else {
-			cloudConfig.SubnetworkURL = gceSubnetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, cloudConfig.Region, configFile.Global.SubnetworkName)
+			cloudConfig.SubnetworkName = configFile.Global.SubnetworkName
 		}
 	}
 
@@ -332,11 +341,15 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
 	return cloudConfig, err
 }
 
-// Creates a GCECloud object using the specified parameters.
+// CreateGCECloud creates a GCECloud object using the specified parameters.
 // If no networkUrl is specified, loads networkName via rest call.
 // If no tokenSource is specified, uses oauth2.DefaultTokenSource.
 // If managedZones is nil / empty all zones in the region will be managed.
 func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
+	// Use ProjectID for NetworkProjectID, if it wasn't explicitly set.
+	if config.NetworkProjectID == "" {
+		config.NetworkProjectID = config.ProjectID
+	}
 
 	client, err := newOauthClient(config.TokenSource)
 	if err != nil {
@@ -385,19 +398,34 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 		return nil, err
 	}
 
-	if config.NetworkURL == "" {
+	// ProjectID and.NetworkProjectID may be project number or name.
-		networkName, err := getNetworkNameViaAPICall(service, config.ProjectID)
+	projID, netProjID := tryConvertToProjectNames(config.ProjectID, config.NetworkProjectID, service)
+
+	onXPN := projID != netProjID
+
+	var networkURL string
+	var subnetURL string
+
+	if config.NetworkName == "" && config.NetworkURL == "" {
+		// TODO: Stop using this call and return an error.
+		// This function returns the first network in a list of networks for a project. The project
+		// should be set via configuration instead of randomly taking the first.
+		networkName, err := getNetworkNameViaAPICall(service, config.NetworkProjectID)
 		if err != nil {
 			return nil, err
 		}
-		config.NetworkURL = gceNetworkURL(config.ApiEndpoint, config.ProjectID, networkName)
+		networkURL = gceNetworkURL(config.ApiEndpoint, netProjID, networkName)
+	} else if config.NetworkURL != "" {
+		networkURL = config.NetworkURL
+	} else {
+		networkURL = gceNetworkURL(config.ApiEndpoint, netProjID, config.NetworkName)
 	}
 
-	networkProjectID, err := getProjectIDInURL(config.NetworkURL)
+	if config.SubnetworkURL != "" {
-	if err != nil {
+		subnetURL = config.SubnetworkURL
-		return nil, err
+	} else if config.SubnetworkName != "" {
+		subnetURL = gceSubnetworkURL(config.ApiEndpoint, netProjID, config.Region, config.SubnetworkName)
 	}
-	onXPN := networkProjectID != config.ProjectID
 
 	if len(config.ManagedZones) == 0 {
 		config.ManagedZones, err = getZonesForRegion(service, config.ProjectID, config.Region)
@@ -417,14 +445,14 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 		serviceBeta:              serviceBeta,
 		containerService:         containerService,
 		cloudkmsService:          cloudkmsService,
-		projectID:                config.ProjectID,
+		projectID:                projID,
-		networkProjectID:         networkProjectID,
+		networkProjectID:         netProjID,
 		onXPN:                    onXPN,
 		region:                   config.Region,
 		localZone:                config.Zone,
 		managedZones:             config.ManagedZones,
-		networkURL:               config.NetworkURL,
+		networkURL:               networkURL,
-		subnetworkURL:            config.SubnetworkURL,
+		subnetworkURL:            subnetURL,
 		secondaryRangeName:       config.SecondaryRangeName,
 		nodeTags:                 config.NodeTags,
 		nodeInstancePrefix:       config.NodeInstancePrefix,
@@ -437,6 +465,33 @@ func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 	return gce, nil
 }
 
+func tryConvertToProjectNames(configProject, configNetworkProject string, service *compute.Service) (projID, netProjID string) {
+	projID = configProject
+	if isProjectNumber(projID) {
+		projName, err := getProjectID(service, projID)
+		if err != nil {
+			glog.Warningf("Failed to retrieve project %v while trying to retrieve its name. err %v", projID, err)
+		} else {
+			projID = projName
+		}
+	}
+
+	netProjID = projID
+	if configNetworkProject != configProject {
+		netProjID = configNetworkProject
+	}
+	if isProjectNumber(netProjID) {
+		netProjName, err := getProjectID(service, netProjID)
+		if err != nil {
+			glog.Warningf("Failed to retrieve network project %v while trying to retrieve its name. err %v", netProjID, err)
+		} else {
+			netProjID = netProjName
+		}
+	}
+
+	return projID, netProjID
+}
+
 // Initialize takes in a clientBuilder and spawns a goroutine for watching the clusterid configmap.
 // This must be called before utilizing the funcs of gce.ClusterID
 func (gce *GCECloud) Initialize(clientBuilder controller.ControllerClientBuilder) {
@@ -473,6 +528,16 @@ func (gce *GCECloud) ProviderName() string {
 	return ProviderName
 }
 
+// ProjectID returns the ProjectID corresponding to the project this cloud is in.
+func (g *GCECloud) ProjectID() string {
+	return g.projectID
+}
+
+// NetworkProjectID returns the ProjectID corresponding to the project this cluster's network is in.
+func (g *GCECloud) NetworkProjectID() string {
+	return g.networkProjectID
+}
+
 // Region returns the region
 func (gce *GCECloud) Region() string {
 	return gce.region
@@ -512,6 +577,13 @@ func (gce *GCECloud) HasClusterID() bool {
 	return true
 }
 
+// Project IDs cannot have a digit for the first characeter. If the id contains a digit,
+// then it must be a project number.
+func isProjectNumber(idOrNumber string) bool {
+	_, err := strconv.ParseUint(idOrNumber, 10, 64)
+	return err == nil
+}
+
 // GCECloud implements cloudprovider.Interface.
 var _ cloudprovider.Interface = (*GCECloud)(nil)
 
@@ -569,6 +641,16 @@ func getNetworkNameViaAPICall(svc *compute.Service, projectID string) (string, e
 	return networkList.Items[0].Name, nil
 }
 
+// getProjectID returns the project's string ID given a project number or string
+func getProjectID(svc *compute.Service, projectNumberOrID string) (string, error) {
+	proj, err := svc.Projects.Get(projectNumberOrID).Do()
+	if err != nil {
+		return "", err
+	}
+
+	return proj.Name, nil
+}
+
 func getZonesForRegion(svc *compute.Service, projectID, region string) ([]string, error) {
 	// TODO: use PageToken to list all not just the first 500
 	listCall := svc.Zones.List(projectID)

pkg/cloudprovider/providers/gce/gce_firewall.go

@@ -27,38 +27,38 @@ func newFirewallMetricContext(request string) *metricContext {
 // GetFirewall returns the Firewall by name.
 func (gce *GCECloud) GetFirewall(name string) (*compute.Firewall, error) {
 	mc := newFirewallMetricContext("get")
-	v, err := gce.service.Firewalls.Get(gce.projectID, name).Do()
+	v, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), name).Do()
 	return v, mc.Observe(err)
 }
 
 // CreateFirewall creates the passed firewall
 func (gce *GCECloud) CreateFirewall(f *compute.Firewall) error {
 	mc := newFirewallMetricContext("create")
-	op, err := gce.service.Firewalls.Insert(gce.projectID, f).Do()
+	op, err := gce.service.Firewalls.Insert(gce.NetworkProjectID(), f).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
 
-	return gce.waitForGlobalOp(op, mc)
+	return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc)
 }
 
 // DeleteFirewall deletes the given firewall rule.
 func (gce *GCECloud) DeleteFirewall(name string) error {
 	mc := newFirewallMetricContext("delete")
-	op, err := gce.service.Firewalls.Delete(gce.projectID, name).Do()
+	op, err := gce.service.Firewalls.Delete(gce.NetworkProjectID(), name).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
-	return gce.waitForGlobalOp(op, mc)
+	return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc)
 }
 
 // UpdateFirewall applies the given firewall as an update to an existing service.
 func (gce *GCECloud) UpdateFirewall(f *compute.Firewall) error {
 	mc := newFirewallMetricContext("update")
-	op, err := gce.service.Firewalls.Update(gce.projectID, f.Name, f).Do()
+	op, err := gce.service.Firewalls.Update(gce.NetworkProjectID(), f.Name, f).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
 
-	return gce.waitForGlobalOp(op, mc)
+	return gce.waitForGlobalOpInProject(op, gce.NetworkProjectID(), mc)
 }

pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go

@@ -729,7 +729,7 @@ func (gce *GCECloud) firewallNeedsUpdate(name, serviceName, region, ipAddress st
 		return false, false, nil
 	}
 
-	fw, err := gce.service.Firewalls.Get(gce.projectID, makeFirewallName(name)).Do()
+	fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), makeFirewallName(name)).Do()
 	if err != nil {
 		if isHTTPErrorCode(err, http.StatusNotFound) {
 			return false, true, nil
@@ -776,7 +776,7 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(serviceName, ipAddress, regio
 	ports := []v1.ServicePort{{Protocol: "tcp", Port: hcPort}}
 
 	fwName := MakeHealthCheckFirewallName(clusterID, hcName, isNodesHealthCheck)
-	fw, err := gce.service.Firewalls.Get(gce.projectID, fwName).Do()
+	fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), fwName).Do()
 	if err != nil {
 		if !isHTTPErrorCode(err, http.StatusNotFound) {
 			return fmt.Errorf("error getting firewall for health checks: %v", err)

pkg/cloudprovider/providers/gce/gce_op.go

@@ -93,62 +93,74 @@ func getErrorFromOp(op *computev1.Operation) error {
 }
 
 func (gce *GCECloud) waitForGlobalOp(op gceObject, mc *metricContext) error {
-	switch v := op.(type) {
+	return gce.waitForGlobalOpInProject(op, gce.ProjectID(), mc)
-	case *computealpha.Operation:
-		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceAlpha.GlobalOperations.Get(gce.projectID, operationName).Do()
-			return convertToV1Operation(op), err
-		}, mc)
-	case *computebeta.Operation:
-		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceBeta.GlobalOperations.Get(gce.projectID, operationName).Do()
-			return convertToV1Operation(op), err
-		}, mc)
-	case *computev1.Operation:
-		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
-			return gce.service.GlobalOperations.Get(gce.projectID, operationName).Do()
-		}, mc)
-	default:
-		return fmt.Errorf("unexpected type: %T", v)
-	}
 }
 
 func (gce *GCECloud) waitForRegionOp(op gceObject, region string, mc *metricContext) error {
+	return gce.waitForRegionOpInProject(op, gce.ProjectID(), region, mc)
+}
+
+func (gce *GCECloud) waitForZoneOp(op gceObject, zone string, mc *metricContext) error {
+	return gce.waitForZoneOpInProject(op, gce.ProjectID(), zone, mc)
+}
+
+func (gce *GCECloud) waitForGlobalOpInProject(op gceObject, projectID string, mc *metricContext) error {
 	switch v := op.(type) {
 	case *computealpha.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceAlpha.RegionOperations.Get(gce.projectID, region, operationName).Do()
+			op, err := gce.serviceAlpha.GlobalOperations.Get(projectID, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computebeta.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceBeta.RegionOperations.Get(gce.projectID, region, operationName).Do()
+			op, err := gce.serviceBeta.GlobalOperations.Get(projectID, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computev1.Operation:
 		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
-			return gce.service.RegionOperations.Get(gce.projectID, region, operationName).Do()
+			return gce.service.GlobalOperations.Get(projectID, operationName).Do()
 		}, mc)
 	default:
 		return fmt.Errorf("unexpected type: %T", v)
 	}
 }
 
-func (gce *GCECloud) waitForZoneOp(op gceObject, zone string, mc *metricContext) error {
+func (gce *GCECloud) waitForRegionOpInProject(op gceObject, projectID, region string, mc *metricContext) error {
 	switch v := op.(type) {
 	case *computealpha.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceAlpha.ZoneOperations.Get(gce.projectID, zone, operationName).Do()
+			op, err := gce.serviceAlpha.RegionOperations.Get(projectID, region, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computebeta.Operation:
 		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
-			op, err := gce.serviceBeta.ZoneOperations.Get(gce.projectID, zone, operationName).Do()
+			op, err := gce.serviceBeta.RegionOperations.Get(projectID, region, operationName).Do()
 			return convertToV1Operation(op), err
 		}, mc)
 	case *computev1.Operation:
 		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
-			return gce.service.ZoneOperations.Get(gce.projectID, zone, operationName).Do()
+			return gce.service.RegionOperations.Get(projectID, region, operationName).Do()
+		}, mc)
+	default:
+		return fmt.Errorf("unexpected type: %T", v)
+	}
+}
+
+func (gce *GCECloud) waitForZoneOpInProject(op gceObject, projectID, zone string, mc *metricContext) error {
+	switch v := op.(type) {
+	case *computealpha.Operation:
+		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
+			op, err := gce.serviceAlpha.ZoneOperations.Get(projectID, zone, operationName).Do()
+			return convertToV1Operation(op), err
+		}, mc)
+	case *computebeta.Operation:
+		return gce.waitForOp(convertToV1Operation(op), func(operationName string) (*computev1.Operation, error) {
+			op, err := gce.serviceBeta.ZoneOperations.Get(projectID, zone, operationName).Do()
+			return convertToV1Operation(op), err
+		}, mc)
+	case *computev1.Operation:
+		return gce.waitForOp(op.(*computev1.Operation), func(operationName string) (*computev1.Operation, error) {
+			return gce.service.ZoneOperations.Get(projectID, zone, operationName).Do()
 		}, mc)
 	default:
 		return fmt.Errorf("unexpected type: %T", v)

pkg/cloudprovider/providers/gce/gce_routes.go

@@ -38,13 +38,13 @@ func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, err
 	page := 0
 	for ; page == 0 || (pageToken != "" && page < maxPages); page++ {
 		mc := newRoutesMetricContext("list_page")
-		listCall := gce.service.Routes.List(gce.projectID)
+		listCall := gce.service.Routes.List(gce.NetworkProjectID())
 
 		prefix := truncateClusterName(clusterName)
 		// Filter for routes starting with clustername AND belonging to the
 		// relevant gcp network AND having description = "k8s-node-route".
 		filter := "(name eq " + prefix + "-.*) "
-		filter = filter + "(network eq " + gce.networkURL + ") "
+		filter = filter + "(network eq " + gce.NetworkURL() + ") "
 		filter = filter + "(description eq " + k8sNodeRouteTag + ")"
 		listCall = listCall.Filter(filter)
 		if pageToken != "" {
@@ -80,11 +80,11 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo
 	}
 
 	mc := newRoutesMetricContext("create")
-	insertOp, err := gce.service.Routes.Insert(gce.projectID, &compute.Route{
+	insertOp, err := gce.service.Routes.Insert(gce.NetworkProjectID(), &compute.Route{
 		Name:            routeName,
 		DestRange:       route.DestinationCIDR,
 		NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", targetInstance.Zone, targetInstance.Name),
-		Network:         gce.networkURL,
+		Network:         gce.NetworkURL(),
 		Priority:        1000,
 		Description:     k8sNodeRouteTag,
 	}).Do()
@@ -96,16 +96,16 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo
 			return mc.Observe(err)
 		}
 	}
-	return gce.waitForGlobalOp(insertOp, mc)
+	return gce.waitForGlobalOpInProject(insertOp, gce.NetworkProjectID(), mc)
 }
 
 func (gce *GCECloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error {
 	mc := newRoutesMetricContext("delete")
-	deleteOp, err := gce.service.Routes.Delete(gce.projectID, route.Name).Do()
+	deleteOp, err := gce.service.Routes.Delete(gce.NetworkProjectID(), route.Name).Do()
 	if err != nil {
 		return mc.Observe(err)
 	}
-	return gce.waitForGlobalOp(deleteOp, mc)
+	return gce.waitForGlobalOpInProject(deleteOp, gce.NetworkProjectID(), mc)
 }
 
 func truncateClusterName(clusterName string) string {

pkg/cloudprovider/providers/gce/gce_test.go

@@ -29,6 +29,43 @@ import (
 	computev1 "google.golang.org/api/compute/v1"
 )
 
+func TestReadConfigFile(t *testing.T) {
+	const s = `[Global]
+token-url = my-token-url
+token-body = my-token-body
+project-id = my-project
+network-project-id = my-network-project
+network-name = my-network
+subnetwork-name = my-subnetwork
+secondary-range-name = my-secondary-range
+node-tags = my-node-tag1
+node-instance-prefix = my-prefix
+multizone = true
+   `
+	reader := strings.NewReader(s)
+	config, err := readConfig(reader)
+	if err != nil {
+		t.Fatalf("Unexpected config parsing error %v", err)
+	}
+
+	expected := &ConfigFile{Global: ConfigGlobal{
+		TokenURL:           "my-token-url",
+		TokenBody:          "my-token-body",
+		ProjectID:          "my-project",
+		NetworkProjectID:   "my-network-project",
+		NetworkName:        "my-network",
+		SubnetworkName:     "my-subnetwork",
+		SecondaryRangeName: "my-secondary-range",
+		NodeTags:           []string{"my-node-tag1"},
+		NodeInstancePrefix: "my-prefix",
+		Multizone:          true,
+	}}
+
+	if !reflect.DeepEqual(expected, config) {
+		t.Fatalf("Expected config file values to be read into ConfigFile struct.  \nExpected:\n%+v\nActual:\n%+v", expected, config)
+	}
+}
+
 func TestExtraKeyInConfig(t *testing.T) {
 	const s = `[Global]
 project-id = my-project
@@ -263,23 +300,8 @@ func TestSplitProviderID(t *testing.T) {
 	}
 }
 
-type generateConfigParams struct {
+func TestGenerateCloudConfigs(t *testing.T) {
-	TokenURL           string
+	configBoilerplate := ConfigGlobal{
-	TokenBody          string
-	ProjectID          string
-	NetworkName        string
-	SubnetworkName     string
-	SecondaryRangeName string
-	NodeTags           []string
-	NodeInstancePrefix string
-	Multizone          bool
-	ApiEndpoint        string
-	LocalZone          string
-	AlphaFeatures      []string
-}
-
-func newGenerateConfigDefaults() *generateConfigParams {
-	return &generateConfigParams{
 		TokenURL:           "",
 		TokenBody:          "",
 		ProjectID:          "project-id",
@@ -293,197 +315,147 @@ func newGenerateConfigDefaults() *generateConfigParams {
 		LocalZone:          "us-central1-a",
 		AlphaFeatures:      []string{},
 	}
-}
 
-func TestGenerateCloudConfigs(t *testing.T) {
+	cloudBoilerplate := CloudConfig{
+		ApiEndpoint:        "",
+		ProjectID:          "project-id",
+		NetworkProjectID:   "",
+		Region:             "us-central1",
+		Zone:               "us-central1-a",
+		ManagedZones:       []string{"us-central1-a"},
+		NetworkName:        "network-name",
+		SubnetworkName:     "",
+		NetworkURL:         "",
+		SubnetworkURL:      "",
+		SecondaryRangeName: "",
+		NodeTags:           []string{"node-tag"},
+		TokenSource:        google.ComputeTokenSource(""),
+		NodeInstancePrefix: "node-prefix",
+		UseMetadataServer:  true,
+		AlphaFeatureGate:   &AlphaFeatureGate{map[string]bool{}},
+	}
+
 	testCases := []struct {
-		TokenURL           string
+		name   string
-		TokenBody          string
+		config func() ConfigGlobal
-		ProjectID          string
+		cloud  func() CloudConfig
-		NetworkName        string
-		SubnetworkName     string
-		NodeTags           []string
-		NodeInstancePrefix string
-		Multizone          bool
-		ApiEndpoint        string
-		LocalZone          string
-		cloudConfig        *CloudConfig
-		AlphaFeatures      []string
 	}{
-		// default config
 		{
-			cloudConfig: &CloudConfig{
+			name:   "Empty Config",
-				ApiEndpoint:        "",
+			config: func() ConfigGlobal { return configBoilerplate },
-				ProjectID:          "project-id",
+			cloud:  func() CloudConfig { return cloudBoilerplate },
-				Region:             "us-central1",
+		},
-				Zone:               "us-central1-a",
+		{
-				ManagedZones:       []string{"us-central1-a"},
+			name: "Nil token URL",
-				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+			config: func() ConfigGlobal {
-				SubnetworkURL:      "",
+				v := configBoilerplate
-				NodeTags:           []string{"node-tag"},
+				v.TokenURL = "nil"
-				NodeInstancePrefix: "node-prefix",
+				return v
-				TokenSource:        google.ComputeTokenSource(""),
+			},
-				UseMetadataServer:  true,
+			cloud: func() CloudConfig {
-				AlphaFeatureGate:   &AlphaFeatureGate{map[string]bool{}},
+				v := cloudBoilerplate
+				v.TokenSource = nil
+				return v
 			},
 		},
-		// nil token source
 		{
-			TokenURL: "nil",
+			name: "Network Project ID",
-			cloudConfig: &CloudConfig{
+			config: func() ConfigGlobal {
-				ApiEndpoint:        "",
+				v := configBoilerplate
-				ProjectID:          "project-id",
+				v.NetworkProjectID = "my-awesome-project"
-				Region:             "us-central1",
+				return v
-				Zone:               "us-central1-a",
+			},
-				ManagedZones:       []string{"us-central1-a"},
+			cloud: func() CloudConfig {
-				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				v := cloudBoilerplate
-				SubnetworkURL:      "",
+				v.NetworkProjectID = "my-awesome-project"
-				NodeTags:           []string{"node-tag"},
+				return v
-				NodeInstancePrefix: "node-prefix",
-				TokenSource:        nil,
-				UseMetadataServer:  true,
-				AlphaFeatureGate:   &AlphaFeatureGate{map[string]bool{}},
 			},
 		},
-		// specified api endpoint
 		{
-			ApiEndpoint: "https://www.googleapis.com/compute/staging_v1/",
+			name: "Specified API Endpint",
-			cloudConfig: &CloudConfig{
+			config: func() ConfigGlobal {
-				ApiEndpoint:        "https://www.googleapis.com/compute/staging_v1/",
+				v := configBoilerplate
-				ProjectID:          "project-id",
+				v.ApiEndpoint = "https://www.googleapis.com/compute/staging_v1/"
-				Region:             "us-central1",
+				return v
-				Zone:               "us-central1-a",
+			},
-				ManagedZones:       []string{"us-central1-a"},
+			cloud: func() CloudConfig {
-				NetworkURL:         "https://www.googleapis.com/compute/staging_v1/projects/project-id/global/networks/network-name",
+				v := cloudBoilerplate
-				SubnetworkURL:      "",
+				v.ApiEndpoint = "https://www.googleapis.com/compute/staging_v1/"
-				NodeTags:           []string{"node-tag"},
+				return v
-				NodeInstancePrefix: "node-prefix",
-				TokenSource:        google.ComputeTokenSource(""),
-				UseMetadataServer:  true,
-				AlphaFeatureGate:   &AlphaFeatureGate{map[string]bool{}},
 			},
 		},
-		// fqdn subnetname
 		{
-			SubnetworkName: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+			name: "Network & Subnetwork names",
-			cloudConfig: &CloudConfig{
+			config: func() ConfigGlobal {
-				ApiEndpoint:        "",
+				v := configBoilerplate
-				ProjectID:          "project-id",
+				v.NetworkName = "my-network"
-				Region:             "us-central1",
+				v.SubnetworkName = "my-subnetwork"
-				Zone:               "us-central1-a",
+				return v
-				ManagedZones:       []string{"us-central1-a"},
+			},
-				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+			cloud: func() CloudConfig {
-				SubnetworkURL:      "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				v := cloudBoilerplate
-				NodeTags:           []string{"node-tag"},
+				v.NetworkName = "my-network"
-				NodeInstancePrefix: "node-prefix",
+				v.SubnetworkName = "my-subnetwork"
-				TokenSource:        google.ComputeTokenSource(""),
+				return v
-				UseMetadataServer:  true,
-				AlphaFeatureGate:   &AlphaFeatureGate{map[string]bool{}},
 			},
 		},
-		// subnetname
 		{
-			SubnetworkName: "subnetwork-name",
+			name: "Network & Subnetwork URLs",
-			cloudConfig: &CloudConfig{
+			config: func() ConfigGlobal {
-				ApiEndpoint:        "",
+				v := configBoilerplate
-				ProjectID:          "project-id",
+				v.NetworkName = "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/my-network"
-				Region:             "us-central1",
+				v.SubnetworkName = "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/my-subnetwork"
-				Zone:               "us-central1-a",
+				return v
-				ManagedZones:       []string{"us-central1-a"},
+			},
-				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+			cloud: func() CloudConfig {
-				SubnetworkURL:      "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				v := cloudBoilerplate
-				NodeTags:           []string{"node-tag"},
+				v.NetworkName = ""
-				NodeInstancePrefix: "node-prefix",
+				v.SubnetworkName = ""
-				TokenSource:        google.ComputeTokenSource(""),
+				v.NetworkURL = "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/my-network"
-				UseMetadataServer:  true,
+				v.SubnetworkURL = "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/my-subnetwork"
-				AlphaFeatureGate:   &AlphaFeatureGate{map[string]bool{}},
+				return v
 			},
 		},
-		// multi zone
 		{
-			Multizone: true,
+			name: "Multizone",
-			cloudConfig: &CloudConfig{
+			config: func() ConfigGlobal {
-				ApiEndpoint:        "",
+				v := configBoilerplate
-				ProjectID:          "project-id",
+				v.Multizone = true
-				Region:             "us-central1",
+				return v
-				Zone:               "us-central1-a",
+			},
-				ManagedZones:       nil,
+			cloud: func() CloudConfig {
-				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				v := cloudBoilerplate
-				SubnetworkURL:      "",
+				v.ManagedZones = nil
-				NodeTags:           []string{"node-tag"},
+				return v
-				NodeInstancePrefix: "node-prefix",
+			},
-				TokenSource:        google.ComputeTokenSource(""),
+		},
-				UseMetadataServer:  true,
+		{
-				AlphaFeatureGate:   &AlphaFeatureGate{map[string]bool{}},
+			name: "Secondary Range Name",
+			config: func() ConfigGlobal {
+				v := configBoilerplate
+				v.SecondaryRangeName = "my-secondary"
+				return v
+			},
+			cloud: func() CloudConfig {
+				v := cloudBoilerplate
+				v.SecondaryRangeName = "my-secondary"
+				return v
 			},
 		},
 	}
 
 	for _, tc := range testCases {
-		config := newGenerateConfigDefaults()
+		t.Run(tc.name, func(t *testing.T) {
-		config.Multizone = tc.Multizone
+			resultCloud, err := generateCloudConfig(&ConfigFile{Global: tc.config()})
-		config.ApiEndpoint = tc.ApiEndpoint
+			if err != nil {
-		config.AlphaFeatures = tc.AlphaFeatures
+				t.Fatalf("Unexpect error: %v", err)
-		config.TokenBody = tc.TokenBody
+			}
 
-		if tc.TokenURL != "" {
+			v := tc.cloud()
-			config.TokenURL = tc.TokenURL
+			if !reflect.DeepEqual(*resultCloud, v) {
-		}
+				t.Errorf("Got: \n%v\nWant\n%v\n", v, *resultCloud)
-		if tc.ProjectID != "" {
+			}
-			config.ProjectID = tc.ProjectID
-		}
-		if tc.NetworkName != "" {
-			config.NetworkName = tc.NetworkName
-		}
-		if tc.SubnetworkName != "" {
-			config.SubnetworkName = tc.SubnetworkName
-		}
-		if len(tc.NodeTags) > 0 {
-			config.NodeTags = tc.NodeTags
-		}
-		if tc.NodeInstancePrefix != "" {
-			config.NodeInstancePrefix = tc.NodeInstancePrefix
-		}
-		if tc.LocalZone != "" {
-			config.LocalZone = tc.LocalZone
-		}
-
-		cloudConfig, err := generateCloudConfig(&ConfigFile{
-			Global: struct {
-				TokenURL           string   `gcfg:"token-url"`
-				TokenBody          string   `gcfg:"token-body"`
-				ProjectID          string   `gcfg:"project-id"`
-				NetworkName        string   `gcfg:"network-name"`
-				SubnetworkName     string   `gcfg:"subnetwork-name"`
-				SecondaryRangeName string   `gcfg:"secondary-range-name"`
-				NodeTags           []string `gcfg:"node-tags"`
-				NodeInstancePrefix string   `gcfg:"node-instance-prefix"`
-				Multizone          bool     `gcfg:"multizone"`
-				ApiEndpoint        string   `gcfg:"api-endpoint"`
-				LocalZone          string   `gcfg:"local-zone"`
-				AlphaFeatures      []string `gcfg:"alpha-features"`
-			}{
-				TokenURL:           config.TokenURL,
-				TokenBody:          config.TokenBody,
-				ProjectID:          config.ProjectID,
-				NetworkName:        config.NetworkName,
-				SubnetworkName:     config.SubnetworkName,
-				SecondaryRangeName: config.SecondaryRangeName,
-				NodeTags:           config.NodeTags,
-				NodeInstancePrefix: config.NodeInstancePrefix,
-				Multizone:          config.Multizone,
-				ApiEndpoint:        config.ApiEndpoint,
-				LocalZone:          config.LocalZone,
-				AlphaFeatures:      config.AlphaFeatures,
-			},
 		})
-		if err != nil {
-			t.Fatalf("Unexpect error: %v", err)
-		}
-
-		if !reflect.DeepEqual(cloudConfig, tc.cloudConfig) {
-			t.Errorf("Got %v, want %v", cloudConfig, tc.cloudConfig)
-		}
 	}
 }
 

test/e2e/e2e.go

@@ -84,8 +84,8 @@ func setupProviderConfig() error {
 			Region:             region,
 			Zone:               zone,
 			ManagedZones:       managedZones,
-			NetworkURL:         "",
+			NetworkName:        "", // TODO: Change this to use framework.TestContext.CloudConfig.Network?
-			SubnetworkURL:      "",
+			SubnetworkName:     "",
 			NodeTags:           nil,
 			NodeInstancePrefix: "",
 			TokenSource:        nil,