Merge pull request #48642 from freehan/gce-api-endpint

Automatic merge from submit-queue

Support GCE alpha/beta api endpoint override

fixes: https://github.com/kubernetes/kubernetes/issues/48568

Godeps/Godeps.json

@@ -2772,6 +2772,10 @@
 			"ImportPath": "google.golang.org/api/cloudmonitoring/v2beta2",
 			"Rev": "e3824ed33c72bf7e81da0286772c34b987520914"
 		},
+		{
+			"ImportPath": "google.golang.org/api/compute/v0.alpha",
+			"Rev": "e3824ed33c72bf7e81da0286772c34b987520914"
+		},
 		{
 			"ImportPath": "google.golang.org/api/compute/v0.beta",
 			"Rev": "e3824ed33c72bf7e81da0286772c34b987520914"

Godeps/LICENSES

@@ -83325,6 +83325,41 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ================================================================================
 
 
+================================================================================
+= vendor/google.golang.org/api/compute/v0.alpha licensed under: =
+
+Copyright (c) 2011 Google Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+= vendor/google.golang.org/api/LICENSE a651bb3d8b1c412632e28823bb432b40  -
+================================================================================
+
+
 ================================================================================
 = vendor/google.golang.org/api/compute/v0.beta licensed under: =
 

cluster/gce/config-default.sh

@@ -19,6 +19,9 @@
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 source "${KUBE_ROOT}/cluster/gce/config-common.sh"
 
+# Specifying KUBE_GCE_API_ENDPOINT will override the default GCE Compute API endpoint (https://www.googleapis.com/compute/v1/).
+# This endpoint has to be pointing to v1 api. For example, https://www.googleapis.com/compute/staging_v1/
+GCE_API_ENDPOINT=${KUBE_GCE_API_ENDPOINT:-}
 GCLOUD=gcloud
 ZONE=${KUBE_GCE_ZONE:-us-central1-b}
 REGION=${ZONE%-*}
@@ -73,7 +76,6 @@ GCI_VERSION=${KUBE_GCI_VERSION:-cos-stable-59-9460-64-0}
 MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-}
 MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud}
 NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}
-GCE_API_ENDPOINT=${KUBE_GCE_API_ENDPOINT:-}
 NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-cos-cloud}
 CONTAINER_RUNTIME=${KUBE_CONTAINER_RUNTIME:-docker}
 RKT_VERSION=${KUBE_RKT_VERSION:-1.23.0}

cluster/gce/config-test.sh

@@ -19,6 +19,9 @@
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 source "${KUBE_ROOT}/cluster/gce/config-common.sh"
 
+# Specifying KUBE_GCE_API_ENDPOINT will override the default GCE Compute API endpoint (https://www.googleapis.com/compute/v1/).
+# This endpoint has to be pointing to v1 api. For example, https://www.googleapis.com/compute/staging_v1/
+GCE_API_ENDPOINT=${KUBE_GCE_API_ENDPOINT:-}
 GCLOUD=gcloud
 ZONE=${KUBE_GCE_ZONE:-us-central1-b}
 REGION=${ZONE%-*}

pkg/cloudprovider/providers/gce/BUILD

@@ -56,6 +56,7 @@ go_library(
         "//vendor/golang.org/x/oauth2:go_default_library",
         "//vendor/golang.org/x/oauth2/google:go_default_library",
         "//vendor/google.golang.org/api/cloudkms/v1:go_default_library",
+        "//vendor/google.golang.org/api/compute/v0.alpha:go_default_library",
         "//vendor/google.golang.org/api/compute/v0.beta:go_default_library",
         "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/google.golang.org/api/container/v1:go_default_library",

pkg/cloudprovider/providers/gce/gce.go

@@ -39,6 +39,7 @@ import (
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	cloudkms "google.golang.org/api/cloudkms/v1"
+	computealpha "google.golang.org/api/compute/v0.alpha"
 	computebeta "google.golang.org/api/compute/v0.beta"
 	compute "google.golang.org/api/compute/v1"
 	container "google.golang.org/api/container/v1"
@@ -75,6 +76,8 @@ const (
 	gceHcHealthyThreshold = int64(1)
 	// Defaults to 5 * 2 = 10 seconds before the LB will steer traffic away
 	gceHcUnhealthyThreshold = int64(5)
+
+	gceComputeAPIEndpoint = "https://www.googleapis.com/compute/v1/"
 )
 
 // GCECloud is an implementation of Interface, LoadBalancer and Instances for Google Compute Engine.
@@ -85,6 +88,7 @@ type GCECloud struct {
 
 	service          *compute.Service
 	serviceBeta      *computebeta.Service
+	serviceAlpha     *computealpha.Service
 	containerService *container.Service
 	cloudkmsService  *cloudkms.Service
 	clientBuilder    controller.ControllerClientBuilder
@@ -144,7 +148,8 @@ type Config struct {
 		NodeTags           []string `gcfg:"node-tags"`
 		NodeInstancePrefix string   `gcfg:"node-instance-prefix"`
 		Multizone          bool     `gcfg:"multizone"`
-		ApiEndpoint        string   `gcfg:"api-endpoint"`
+		// Specifying ApiEndpoint will override the default GCE compute API endpoint.
+		ApiEndpoint string `gcfg:"api-endpoint"`
 	}
 }
 
@@ -267,22 +272,39 @@ func CreateGCECloud(apiEndpoint, projectID, networkProjectID, region, zone strin
 	if err != nil {
 		return nil, err
 	}
-
 	service, err := compute.New(client)
 	if err != nil {
 		return nil, err
 	}
 
-	if apiEndpoint != "" {
-		service.BasePath = fmt.Sprintf("%sprojects/", apiEndpoint)
-	}
-
 	client, err = newOauthClient(tokenSource)
+	if err != nil {
+		return nil, err
+	}
 	serviceBeta, err := computebeta.New(client)
 	if err != nil {
 		return nil, err
 	}
 
+	client, err = newOauthClient(tokenSource)
+	if err != nil {
+		return nil, err
+	}
+	serviceAlpha, err := computealpha.New(client)
+	if err != nil {
+		return nil, err
+	}
+
+	// Expect override api endpoint to always be v1 api and follows the same pattern as prod.
+	// Generate alpha and beta api endpoints based on override v1 api endpoint.
+	// For example,
+	// staging API endpoint: https://www.googleapis.com/compute/staging_v1/
+	if apiEndpoint != "" {
+		service.BasePath = fmt.Sprintf("%sprojects/", apiEndpoint)
+		serviceBeta.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "beta", 0))
+		serviceAlpha.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "alpha", 0))
+	}
+
 	containerService, err := container.New(client)
 	if err != nil {
 		return nil, err
@@ -413,16 +435,16 @@ var _ cloudprovider.Interface = (*GCECloud)(nil)
 
 func gceNetworkURL(apiEndpoint, project, network string) string {
 	if apiEndpoint == "" {
-		apiEndpoint = "https://www.googleapis.com/compute/v1/"
+		apiEndpoint = gceComputeAPIEndpoint
 	}
-	return fmt.Sprintf("%vprojects/%s/global/networks/%s", apiEndpoint, project, network)
+	return apiEndpoint + strings.Join([]string{"projects", project, "global", "networks", network}, "/")
 }
 
 func gceSubnetworkURL(apiEndpoint, project, region, subnetwork string) string {
 	if apiEndpoint == "" {
-		apiEndpoint = "https://www.googleapis.com/compute/v1/"
+		apiEndpoint = gceComputeAPIEndpoint
 	}
-	return fmt.Sprintf("%vprojects/%s/regions/%s/subnetworks/%s", apiEndpoint, project, region, subnetwork)
+	return apiEndpoint + strings.Join([]string{"projects", project, "regions", region, "subnetworks", subnetwork}, "/")
 }
 
 // Project IDs cannot have a digit for the first characeter. If the id contains a digit,
@@ -431,7 +453,6 @@ func isProjectNumber(idOrNumber string) bool {
 	if len(idOrNumber) == 0 {
 		return false
 	}
-
 	return idOrNumber[0] >= '0' && idOrNumber[0] <= '9'
 }
 

pkg/cloudprovider/providers/gce/gce_disks.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"path"
 	"strings"
 	"time"
 
@@ -41,7 +40,7 @@ const (
 	DiskTypeStandard = "pd-standard"
 
 	diskTypeDefault     = DiskTypeStandard
-	diskTypeUriTemplate = "https://www.googleapis.com/compute/v1/projects/%s/zones/%s/diskTypes/%s"
+	diskTypeUriTemplate = "%s/zones/%s/diskTypes/%s"
 )
 
 // Disks is interface for manipulation with GCE PDs.
@@ -234,7 +233,12 @@ func (gce *GCECloud) CreateDisk(
 	default:
 		return fmt.Errorf("invalid GCE disk type %q", diskType)
 	}
-	diskTypeUri := fmt.Sprintf(diskTypeUriTemplate, gce.projectID, zone, diskType)
+
+	apiEndpoint := gceComputeAPIEndpoint + "projects/"
+	if gce.service != nil {
+		apiEndpoint = gce.service.BasePath
+	}
+	diskTypeUri := apiEndpoint + fmt.Sprintf(diskTypeUriTemplate, gce.projectID, zone, diskType)
 
 	diskToCreate := &compute.Disk{
 		Name:        name,
@@ -424,9 +428,8 @@ func (gce *GCECloud) convertDiskToAttachedDisk(disk *GCEDisk, readWrite string)
 		DeviceName: disk.Name,
 		Kind:       disk.Kind,
 		Mode:       readWrite,
-		Source: "https://" + path.Join(
-			"www.googleapis.com/compute/v1/projects/",
-			gce.projectID, "zones", disk.Zone, "disks", disk.Name),
+		Source: gce.service.BasePath + strings.Join([]string{
+			gce.projectID, "zones", disk.Zone, "disks", disk.Name}, "/"),
 		Type: "PERSISTENT",
 	}
 }

pkg/cloudprovider/providers/gce/gce_disks_test.go

@@ -43,7 +43,7 @@ func TestCreateDisk_Basic(t *testing.T) {
 	tags := make(map[string]string)
 	tags["test-tag"] = "test-value"
 
-	diskTypeUri := fmt.Sprintf(diskTypeUriTemplate, projectId, zone, diskType)
+	diskTypeUri := gceComputeAPIEndpoint + "projects/" + fmt.Sprintf(diskTypeUriTemplate, projectId, zone, diskType)
 	expectedDescription := "{\"test-tag\":\"test-value\"}"
 
 	/* Act */

pkg/cloudprovider/providers/gce/gce_instances.go

@@ -25,7 +25,7 @@ import (
 
 	"cloud.google.com/go/compute/metadata"
 	"github.com/golang/glog"
-	computealpha "google.golang.org/api/compute/v0.beta"
+	computebeta "google.golang.org/api/compute/v0.beta"
 	compute "google.golang.org/api/compute/v1"
 
 	"k8s.io/api/core/v1"
@@ -69,7 +69,7 @@ func getZone(n *v1.Node) string {
 // ToInstanceReferences returns instance references by links
 func (gce *GCECloud) ToInstanceReferences(zone string, instanceNames []string) (refs []*compute.InstanceReference) {
 	for _, ins := range instanceNames {
-		instanceLink := makeHostURL(gce.projectID, zone, ins)
+		instanceLink := makeHostURL(gce.service.BasePath, gce.projectID, zone, ins)
 		refs = append(refs, &compute.InstanceReference{Instance: instanceLink})
 	}
 	return refs
@@ -303,7 +303,7 @@ func (gce *GCECloud) AliasRanges(nodeName types.NodeName) (cidrs []string, err e
 		return
 	}
 
-	var res *computealpha.Instance
+	var res *computebeta.Instance
 	res, err = gce.serviceBeta.Instances.Get(
 		gce.projectID, instance.Zone, instance.Name).Do()
 	if err != nil {

pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go

@@ -483,7 +483,7 @@ func (gce *GCECloud) createTargetPool(name, serviceName, ipAddress, region, clus
 
 	var instances []string
 	for _, host := range hosts {
-		instances = append(instances, makeHostURL(gce.projectID, host.Zone, host.Name))
+		instances = append(instances, makeHostURL(gce.service.BasePath, gce.projectID, host.Zone, host.Name))
 	}
 	glog.Infof("Creating targetpool %v with %d healthchecks", name, len(hcLinks))
 	pool := &compute.TargetPool{
@@ -542,7 +542,7 @@ func (gce *GCECloud) updateTargetPool(loadBalancerName string, existing sets.Str
 }
 
 func (gce *GCECloud) targetPoolURL(name, region string) string {
-	return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/targetPools/%s", gce.projectID, region, name)
+	return gce.service.BasePath + strings.Join([]string{"projects", gce.projectID, "regions", region, "targetPools", name}, "/")
 }
 
 func makeHttpHealthCheck(name, path string, port int32) *compute.HttpHealthCheck {
@@ -671,10 +671,9 @@ func nodeNames(nodes []*v1.Node) []string {
 	return ret
 }
 
-func makeHostURL(projectID, zone, host string) string {
+func makeHostURL(apiEndpoint, projectID, zone, host string) string {
 	host = canonicalizeInstanceName(host)
-	return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/instances/%s",
-		projectID, zone, host)
+	return apiEndpoint + strings.Join([]string{"projects", projectID, "zones", zone, "instances", host}, "/")
 }
 
 func hostURLToComparablePath(hostURL string) string {

pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go

@@ -624,7 +624,7 @@ func getPortsAndProtocol(svcPorts []v1.ServicePort) (ports []string, protocol v1
 }
 
 func (gce *GCECloud) getBackendServiceLink(name string) string {
-	return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/backendServices/%s", gce.projectID, gce.region, name)
+	return gce.service.BasePath + strings.Join([]string{"projects", gce.projectID, "regions", gce.region, "backendServices", name}, "/")
 }
 
 func getNameFromLink(link string) string {

pkg/cloudprovider/providers/gce/gce_zones.go

@@ -23,6 +23,7 @@ import (
 	compute "google.golang.org/api/compute/v1"
 
 	"k8s.io/kubernetes/pkg/cloudprovider"
+	"strings"
 )
 
 func newZonesMetricContext(request, region string) *metricContext {
@@ -52,5 +53,5 @@ func (gce *GCECloud) ListZonesInRegion(region string) ([]*compute.Zone, error) {
 }
 
 func (gce *GCECloud) getRegionLink(region string) string {
-	return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%v/regions/%v", gce.networkProjectID, region)
+	return gce.service.BasePath + strings.Join([]string{"projects", gce.networkProjectID, "regions", region}, "/")
 }

vendor/BUILD

@@ -352,6 +352,7 @@ filegroup(
         "//vendor/golang.org/x/tools/container/intsets:all-srcs",
         "//vendor/google.golang.org/api/cloudkms/v1:all-srcs",
         "//vendor/google.golang.org/api/cloudmonitoring/v2beta2:all-srcs",
+        "//vendor/google.golang.org/api/compute/v0.alpha:all-srcs",
         "//vendor/google.golang.org/api/compute/v0.beta:all-srcs",
         "//vendor/google.golang.org/api/compute/v1:all-srcs",
         "//vendor/google.golang.org/api/container/v1:all-srcs",

vendor/google.golang.org/api/compute/v0.alpha/BUILD

@@ -0,0 +1,33 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["compute-gen.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/golang.org/x/net/context/ctxhttp:go_default_library",
+        "//vendor/google.golang.org/api/gensupport:go_default_library",
+        "//vendor/google.golang.org/api/googleapi:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)