diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index e3be7784e83..8a97d9f0416 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -1282,7 +1282,7 @@ function start-kube-addons { if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then setup-addon-manifests "addons" "calico-policy-controller" - # Configure Calico based on cluster size and image type. + # Configure Calico based on cluster size and image type. local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml" local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml" sed -i -e "s@__CALICO_CNI_DIR__@/opt/cni/bin@g" "${ds_file}" @@ -1290,7 +1290,7 @@ function start-kube-addons { sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}" sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}" else - # If not configured to use Calico, the set the typha replica count to 0, but only if the + # If not configured to use Calico, the set the typha replica count to 0, but only if the # addon is present. local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml" if [[ -e $typha_dep_file ]]; then diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 5a23b3784ce..165c1996cd5 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -951,7 +951,7 @@ function start-kubelet { fi if [[ -n "${NODE_TAINTS:-}" ]]; then flags+=" --register-with-taints=${NODE_TAINTS}" - fi + fi if [[ -n "${EVICTION_HARD:-}" ]]; then flags+=" --eviction-hard=${EVICTION_HARD}" fi diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 2a701caba38..f54df286cda 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -86,20 +86,18 @@ type GCECloud struct { // for the cloudprovider to start watching the configmap. ClusterID ClusterID - service *compute.Service - serviceBeta *computebeta.Service - serviceAlpha *computealpha.Service - containerService *container.Service - cloudkmsService *cloudkms.Service - clientBuilder controller.ControllerClientBuilder - projectID string - region string - localZone string // The zone in which we are running - managedZones []string // List of zones we are spanning (for multi-AZ clusters, primarily when running on master) - networkURL string - subnetworkURL string - // Project which contains the cluster's network. - // Used for specific network resources: firewalls, routes, and listing of zones in a region. + service *compute.Service + serviceBeta *computebeta.Service + serviceAlpha *computealpha.Service + containerService *container.Service + cloudkmsService *cloudkms.Service + clientBuilder controller.ControllerClientBuilder + projectID string + region string + localZone string // The zone in which we are running + managedZones []string // List of zones we are spanning (for multi-AZ clusters, primarily when running on master) + networkURL string + subnetworkURL string networkProjectID string onXPN bool nodeTags []string // List of tags to use on firewall rules for load balancers @@ -137,12 +135,9 @@ type GCEServiceManager struct { type Config struct { Global struct { - TokenURL string `gcfg:"token-url"` - TokenBody string `gcfg:"token-body"` - // ProjectID and NetworkProjectID can either be the numeric or string-based unique identifier that starts with [a-z] - // However, both IDs need to be the same type for controllers to recognize this cluster as an XPN cluster. + TokenURL string `gcfg:"token-url"` + TokenBody string `gcfg:"token-body"` ProjectID string `gcfg:"project-id"` - NetworkProjectID string `gcfg:"network-project-id"` // Project which contains the cluster's network. See networkProjectID in GCECloud NetworkName string `gcfg:"network-name"` SubnetworkName string `gcfg:"subnetwork-name"` NodeTags []string `gcfg:"node-tags"` @@ -171,32 +166,29 @@ func (g *GCECloud) GetKMSService() *cloudkms.Service { return g.cloudkmsService } +// Returns the ProjectID corresponding to the project this cloud is in. +func (g *GCECloud) GetProjectID() string { + return g.projectID +} + // newGCECloud creates a new instance of GCECloud. func newGCECloud(config io.Reader) (*GCECloud, error) { apiEndpoint := "" - - // projectNumber is the numeric identifier. Note: there is also a unique string-based project identifier as well (see https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects) - projectNumber, zone, err := getProjectAndZone() + projectID, zone, err := getProjectAndZone() if err != nil { return nil, err } - // Default projectID to known project number - projectID := projectNumber region, err := GetGCERegion(zone) if err != nil { return nil, err } - // networkProjectNumber is a numeric identifier similar to the projectNumber above. - networkProjectNumber, networkName, err := getNetworkProjectAndNameViaMetadata() + networkName, err := getNetworkNameViaMetadata() if err != nil { return nil, err } - // Default networkProjectID to known network project number - networkProjectID := networkProjectNumber - - networkURL := gceNetworkURL(apiEndpoint, networkProjectNumber, networkName) + networkURL := gceNetworkURL(apiEndpoint, projectID, networkName) subnetworkURL := "" // By default, Kubernetes clusters only run against one zone @@ -218,31 +210,18 @@ func newGCECloud(config io.Reader) (*GCECloud, error) { if cfg.Global.ProjectID != "" { projectID = cfg.Global.ProjectID } - if cfg.Global.NetworkProjectID != "" { - networkProjectID = cfg.Global.NetworkProjectID + + if cfg.Global.NetworkName != "" && strings.Contains(cfg.Global.NetworkName, "/") { + networkURL = cfg.Global.NetworkName + } else { + networkURL = gceNetworkURL(apiEndpoint, projectID, networkName) } - if cfg.Global.NetworkName != "" { - if strings.Contains(cfg.Global.NetworkName, "/") { - otherNetworkProjectID, _ := getProjectIDInURL(cfg.Global.NetworkName) - if networkProjectID != otherNetworkProjectID { - glog.Warningf("Different network projects (may be id vs number). %q and %q", networkProjectID, otherNetworkProjectID) - } - - networkURL = cfg.Global.NetworkName - } else { - networkURL = gceNetworkURL(apiEndpoint, networkProjectID, cfg.Global.NetworkName) - } + if cfg.Global.SubnetworkName != "" && strings.Contains(cfg.Global.SubnetworkName, "/") { + subnetworkURL = cfg.Global.SubnetworkName + } else { + subnetworkURL = gceSubnetworkURL(apiEndpoint, cfg.Global.ProjectID, region, cfg.Global.SubnetworkName) } - - if cfg.Global.SubnetworkName != "" { - if strings.Contains(cfg.Global.SubnetworkName, "/") { - subnetworkURL = cfg.Global.SubnetworkName - } else { - subnetworkURL = gceSubnetworkURL(apiEndpoint, networkProjectID, region, cfg.Global.SubnetworkName) - } - } - if cfg.Global.TokenURL != "" { tokenSource = NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody) } @@ -253,7 +232,7 @@ func newGCECloud(config io.Reader) (*GCECloud, error) { } } - return CreateGCECloud(apiEndpoint, projectID, networkProjectID, region, zone, managedZones, networkURL, subnetworkURL, + return CreateGCECloud(apiEndpoint, projectID, region, zone, managedZones, networkURL, subnetworkURL, nodeTags, nodeInstancePrefix, tokenSource, true /* useMetadataServer */) } @@ -261,13 +240,9 @@ func newGCECloud(config io.Reader) (*GCECloud, error) { // If no networkUrl is specified, loads networkName via rest call. // If no tokenSource is specified, uses oauth2.DefaultTokenSource. // If managedZones is nil / empty all zones in the region will be managed. -func CreateGCECloud(apiEndpoint, projectID, networkProjectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string, +func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string, nodeInstancePrefix string, tokenSource oauth2.TokenSource, useMetadataServer bool) (*GCECloud, error) { - // Determine if cluster is on shared VPC network - // Must assert that the IDs are the same type (ID or number) before checking inequality - onXPN := isProjectNumber(projectID) == isProjectNumber(networkProjectID) && projectID != networkProjectID - client, err := newOauthClient(tokenSource) if err != nil { return nil, err @@ -315,6 +290,20 @@ func CreateGCECloud(apiEndpoint, projectID, networkProjectID, region, zone strin return nil, err } + if networkURL == "" { + networkName, err := getNetworkNameViaAPICall(service, projectID) + if err != nil { + return nil, err + } + networkURL = gceNetworkURL(apiEndpoint, projectID, networkName) + } + + networkProjectID, err := getProjectIDInURL(networkURL) + if err != nil { + return nil, err + } + onXPN := networkProjectID != projectID + if len(managedZones) == 0 { managedZones, err = getZonesForRegion(service, projectID, region) if err != nil { @@ -391,17 +380,7 @@ func (gce *GCECloud) Region() string { return gce.region } -// ProjectID returns the project ID which owns the instances -func (gce *GCECloud) ProjectID() string { - return gce.projectID -} - -// NetworkProjectID returns the project ID which owns the network -func (gce *GCECloud) NetworkProjectID() string { - return gce.networkProjectID -} - -// OnXPN returns true if the cluster is running on a shared VPC network +// OnXPN returns true if the cluster is running on a cross project network (XPN) func (gce *GCECloud) OnXPN() bool { return gce.onXPN } @@ -447,15 +426,6 @@ func gceSubnetworkURL(apiEndpoint, project, region, subnetwork string) string { return apiEndpoint + strings.Join([]string{"projects", project, "regions", region, "subnetworks", subnetwork}, "/") } -// Project IDs cannot have a digit for the first characeter. If the id contains a digit, -// then it must be a project number. -func isProjectNumber(idOrNumber string) bool { - if len(idOrNumber) == 0 { - return false - } - return idOrNumber[0] >= '0' && idOrNumber[0] <= '9' -} - // getProjectIDInURL parses typical full resource URLS and shorter URLS // https://www.googleapis.com/compute/v1/projects/myproject/global/networks/mycustom // projects/myproject/global/networks/mycustom @@ -470,16 +440,30 @@ func getProjectIDInURL(urlStr string) (string, error) { return "", fmt.Errorf("could not find project field in url: %v", urlStr) } -func getNetworkProjectAndNameViaMetadata() (string, string, error) { +func getNetworkNameViaMetadata() (string, error) { result, err := metadata.Get("instance/network-interfaces/0/network") if err != nil { - return "", "", err + return "", err } parts := strings.Split(result, "/") if len(parts) != 4 { - return "", "", fmt.Errorf("unexpected response: %s", result) + return "", fmt.Errorf("unexpected response: %s", result) } - return parts[1], parts[3], nil + return parts[3], nil +} + +func getNetworkNameViaAPICall(svc *compute.Service, projectID string) (string, error) { + // TODO: use PageToken to list all not just the first 500 + networkList, err := svc.Networks.List(projectID).Do() + if err != nil { + return "", err + } + + if networkList == nil || len(networkList.Items) <= 0 { + return "", fmt.Errorf("GCE Network List call returned no networks for project %q", projectID) + } + + return networkList.Items[0].Name, nil } func getZonesForRegion(svc *compute.Service, projectID, region string) ([]string, error) { diff --git a/pkg/cloudprovider/providers/gce/gce_firewall.go b/pkg/cloudprovider/providers/gce/gce_firewall.go index e007dc79fd2..668f2e05522 100644 --- a/pkg/cloudprovider/providers/gce/gce_firewall.go +++ b/pkg/cloudprovider/providers/gce/gce_firewall.go @@ -32,14 +32,14 @@ func newFirewallMetricContext(request string) *metricContext { // GetFirewall returns the Firewall by name. func (gce *GCECloud) GetFirewall(name string) (*compute.Firewall, error) { mc := newFirewallMetricContext("get") - v, err := gce.service.Firewalls.Get(gce.networkProjectID, name).Do() + v, err := gce.service.Firewalls.Get(gce.projectID, name).Do() return v, mc.Observe(err) } // CreateFirewall creates the passed firewall func (gce *GCECloud) CreateFirewall(f *compute.Firewall) error { mc := newFirewallMetricContext("create") - op, err := gce.service.Firewalls.Insert(gce.networkProjectID, f).Do() + op, err := gce.service.Firewalls.Insert(gce.projectID, f).Do() if err != nil { return mc.Observe(err) } @@ -50,7 +50,7 @@ func (gce *GCECloud) CreateFirewall(f *compute.Firewall) error { // DeleteFirewall deletes the given firewall rule. func (gce *GCECloud) DeleteFirewall(name string) error { mc := newFirewallMetricContext("delete") - op, err := gce.service.Firewalls.Delete(gce.networkProjectID, name).Do() + op, err := gce.service.Firewalls.Delete(gce.projectID, name).Do() if err != nil { return mc.Observe(err) } @@ -60,7 +60,7 @@ func (gce *GCECloud) DeleteFirewall(name string) error { // UpdateFirewall applies the given firewall as an update to an existing service. func (gce *GCECloud) UpdateFirewall(f *compute.Firewall) error { mc := newFirewallMetricContext("update") - op, err := gce.service.Firewalls.Update(gce.networkProjectID, f.Name, f).Do() + op, err := gce.service.Firewalls.Update(gce.projectID, f.Name, f).Do() if err != nil { return mc.Observe(err) } diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go index 74c8607b284..713690163fb 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go @@ -726,7 +726,7 @@ func (gce *GCECloud) firewallNeedsUpdate(name, serviceName, region, ipAddress st return false, false, nil } - fw, err := gce.GetFirewall(makeFirewallName(name)) + fw, err := gce.service.Firewalls.Get(gce.projectID, makeFirewallName(name)).Do() if err != nil { if isHTTPErrorCode(err, http.StatusNotFound) { return false, true, nil @@ -773,7 +773,7 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(serviceName, ipAddress, regio ports := []v1.ServicePort{{Protocol: "tcp", Port: hcPort}} fwName := MakeHealthCheckFirewallName(clusterID, hcName, isNodesHealthCheck) - fw, err := gce.GetFirewall(fwName) + fw, err := gce.service.Firewalls.Get(gce.projectID, fwName).Do() if err != nil { if !isHTTPErrorCode(err, http.StatusNotFound) { return fmt.Errorf("error getting firewall for health checks: %v", err) diff --git a/pkg/cloudprovider/providers/gce/gce_routes.go b/pkg/cloudprovider/providers/gce/gce_routes.go index 6caeb1cad9c..c5d3a37c245 100644 --- a/pkg/cloudprovider/providers/gce/gce_routes.go +++ b/pkg/cloudprovider/providers/gce/gce_routes.go @@ -43,7 +43,7 @@ func (gce *GCECloud) ListRoutes(clusterName string) ([]*cloudprovider.Route, err page := 0 for ; page == 0 || (pageToken != "" && page < maxPages); page++ { mc := newRoutesMetricContext("list_page") - listCall := gce.service.Routes.List(gce.networkProjectID) + listCall := gce.service.Routes.List(gce.projectID) prefix := truncateClusterName(clusterName) listCall = listCall.Filter("name eq " + prefix + "-.*") @@ -92,7 +92,7 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo } mc := newRoutesMetricContext("create") - insertOp, err := gce.service.Routes.Insert(gce.networkProjectID, &compute.Route{ + insertOp, err := gce.service.Routes.Insert(gce.projectID, &compute.Route{ Name: routeName, DestRange: route.DestinationCIDR, NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", targetInstance.Zone, targetInstance.Name), @@ -113,7 +113,7 @@ func (gce *GCECloud) CreateRoute(clusterName string, nameHint string, route *clo func (gce *GCECloud) DeleteRoute(clusterName string, route *cloudprovider.Route) error { mc := newRoutesMetricContext("delete") - deleteOp, err := gce.service.Routes.Delete(gce.networkProjectID, route.Name).Do() + deleteOp, err := gce.service.Routes.Delete(gce.projectID, route.Name).Do() if err != nil { return mc.Observe(err) } diff --git a/pkg/cloudprovider/providers/gce/gce_zones.go b/pkg/cloudprovider/providers/gce/gce_zones.go index 9f261be8fa1..5ac283c5e7c 100644 --- a/pkg/cloudprovider/providers/gce/gce_zones.go +++ b/pkg/cloudprovider/providers/gce/gce_zones.go @@ -45,7 +45,7 @@ func (gce *GCECloud) GetZone() (cloudprovider.Zone, error) { func (gce *GCECloud) ListZonesInRegion(region string) ([]*compute.Zone, error) { mc := newZonesMetricContext("list", region) filter := fmt.Sprintf("region eq %v", gce.getRegionLink(region)) - list, err := gce.service.Zones.List(gce.networkProjectID).Filter(filter).Do() + list, err := gce.service.Zones.List(gce.projectID).Filter(filter).Do() if err != nil { return nil, mc.Observe(err) } @@ -53,5 +53,5 @@ func (gce *GCECloud) ListZonesInRegion(region string) ([]*compute.Zone, error) { } func (gce *GCECloud) getRegionLink(region string) string { - return gce.service.BasePath + strings.Join([]string{"projects", gce.networkProjectID, "regions", region}, "/") + return gce.service.BasePath + strings.Join([]string{"projects", gce.projectID, "regions", region}, "/") } diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index f262ecb0a6d..23af1b57313 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -80,7 +80,6 @@ func setupProviderConfig() error { managedZones = []string{zone} } cloudConfig.Provider, err = gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ApiEndpoint, - framework.TestContext.CloudConfig.ProjectID, framework.TestContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, "" /* subnetworkUrl */, nil, /* nodeTags */ "" /* nodeInstancePerfix */, nil /* tokenSource */, false /* useMetadataServer */)