From 65b2f71ee7faa0f85aea4a02713380a7bb8334d4 Mon Sep 17 00:00:00 2001 From: Koonwah Chen Date: Wed, 21 Jun 2017 15:27:10 -0700 Subject: [PATCH 1/5] Add ApiEndpoint support to GCE config. --- pkg/cloudprovider/providers/gce/gce.go | 51 ++++++++++++++++---------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 90e2570a429..4e60519daaf 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -137,6 +137,7 @@ type Config struct { NodeTags []string `gcfg:"node-tags"` NodeInstancePrefix string `gcfg:"node-instance-prefix"` Multizone bool `gcfg:"multizone"` + ApiEndpoint string `gcfg:"api-endpoint"` } } @@ -155,6 +156,7 @@ func (g *GCECloud) GetComputeService() *compute.Service { // newGCECloud creates a new instance of GCECloud. func newGCECloud(config io.Reader) (*GCECloud, error) { + apiEndpoint := "" projectID, zone, err := getProjectAndZone() if err != nil { return nil, err @@ -169,7 +171,7 @@ func newGCECloud(config io.Reader) (*GCECloud, error) { if err != nil { return nil, err } - networkURL := gceNetworkURL(projectID, networkName) + networkURL := gceNetworkURL(apiEndpoint, projectID, networkName) subnetworkURL := "" // By default, Kubernetes clusters only run against one zone @@ -185,22 +187,23 @@ func newGCECloud(config io.Reader) (*GCECloud, error) { return nil, err } glog.Infof("Using GCE provider config %+v", cfg) + if cfg.Global.ApiEndpoint != "" { + apiEndpoint = cfg.Global.ApiEndpoint + } if cfg.Global.ProjectID != "" { projectID = cfg.Global.ProjectID } - if cfg.Global.NetworkName != "" { - if strings.Contains(cfg.Global.NetworkName, "/") { - networkURL = cfg.Global.NetworkName - } else { - networkURL = gceNetworkURL(cfg.Global.ProjectID, cfg.Global.NetworkName) - } + + if cfg.Global.NetworkName != "" && strings.Contains(cfg.Global.NetworkName, "/"){ + networkURL = cfg.Global.NetworkName + }else { + networkURL = gceNetworkURL(apiEndpoint, projectID, networkName) } - if cfg.Global.SubnetworkName != "" { - if strings.Contains(cfg.Global.SubnetworkName, "/") { + + if cfg.Global.SubnetworkName != "" && strings.Contains(cfg.Global.SubnetworkName, "/"){ subnetworkURL = cfg.Global.SubnetworkName - } else { - subnetworkURL = gceSubnetworkURL(cfg.Global.ProjectID, region, cfg.Global.SubnetworkName) - } + }else { + subnetworkURL = gceSubnetworkURL(apiEndpoint, cfg.Global.ProjectID, region, cfg.Global.SubnetworkName) } if cfg.Global.TokenURL != "" { tokenSource = NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody) @@ -212,7 +215,7 @@ func newGCECloud(config io.Reader) (*GCECloud, error) { } } - return CreateGCECloud(projectID, region, zone, managedZones, networkURL, subnetworkURL, + return CreateGCECloud(apiEndpoint, projectID, region, zone, managedZones, networkURL, subnetworkURL, nodeTags, nodeInstancePrefix, tokenSource, true /* useMetadataServer */) } @@ -220,7 +223,7 @@ func newGCECloud(config io.Reader) (*GCECloud, error) { // If no networkUrl is specified, loads networkName via rest call. // If no tokenSource is specified, uses oauth2.DefaultTokenSource. // If managedZones is nil / empty all zones in the region will be managed. -func CreateGCECloud(projectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string, +func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string, nodeInstancePrefix string, tokenSource oauth2.TokenSource, useMetadataServer bool) (*GCECloud, error) { client, err := newOauthClient(tokenSource) @@ -233,6 +236,10 @@ func CreateGCECloud(projectID, region, zone string, managedZones []string, netwo return nil, err } + if apiEndpoint != "" { + service.BasePath = fmt.Sprintf("%sprojects/", apiEndpoint) + } + client, err = newOauthClient(tokenSource) serviceBeta, err := computebeta.New(client) if err != nil { @@ -249,7 +256,7 @@ func CreateGCECloud(projectID, region, zone string, managedZones []string, netwo if err != nil { return nil, err } - networkURL = gceNetworkURL(projectID, networkName) + networkURL = gceNetworkURL(apiEndpoint, projectID, networkName) } networkProjectID, err := getProjectIDInURL(networkURL) @@ -365,12 +372,18 @@ func (gce *GCECloud) ScrubDNS(nameservers, searches []string) (nsOut, srchOut [] // GCECloud implements cloudprovider.Interface. var _ cloudprovider.Interface = (*GCECloud)(nil) -func gceNetworkURL(project, network string) string { - return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networks/%s", project, network) +func gceNetworkURL(api_endpoint, project, network string) string { + if api_endpoint == "" { + api_endpoint = "https://www.googleapis.com/compute/v1/" + } + return fmt.Sprintf("%sprojects/%s/global/networks/%s", api_endpoint, project, network) } -func gceSubnetworkURL(project, region, subnetwork string) string { - return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/subnetworks/%s", project, region, subnetwork) +func gceSubnetworkURL(api_endpoint, project, region, subnetwork string) string { + if api_endpoint == "" { + api_endpoint = "https://www.googleapis.com/compute/v1/" + } + return fmt.Sprintf("%sprojects/%s/regions/%s/subnetworks/%s", api_endpoint, project, region, subnetwork) } // getProjectIDInURL parses typical full resource URLS and shorter URLS From 890d356d5c788f85aa542ac945c12fc78fb8846c Mon Sep 17 00:00:00 2001 From: Koonwah Chen Date: Thu, 22 Jun 2017 14:57:21 -0700 Subject: [PATCH 2/5] update e2e for GCE ApiEndpoint support --- test/e2e/e2e.go | 3 ++- test/e2e/framework/test_context.go | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index ee8d7fa76b2..7f7a46db593 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -80,7 +80,8 @@ func setupProviderConfig() error { if !framework.TestContext.CloudConfig.MultiZone { managedZones = []string{zone} } - cloudConfig.Provider, err = gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ProjectID, + cloudConfig.Provider, err = gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ApiEndpoint, + framework.TestContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, "" /* subnetworkUrl */, nil, /* nodeTags */ "" /* nodeInstancePerfix */, nil /* tokenSource */, false /* useMetadataServer */) if err != nil { diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 07ed74b2822..b89780df1b0 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -133,6 +133,7 @@ type NodeTestContextType struct { } type CloudConfig struct { + ApiEndpoint string ProjectID string Zone string MultiZone bool @@ -203,6 +204,7 @@ func RegisterClusterFlags() { // TODO: Flags per provider? Rename gce-project/gce-zone? cloudConfig := &TestContext.CloudConfig flag.StringVar(&cloudConfig.MasterName, "kube-master", "", "Name of the kubernetes master. Only required if provider is gce or gke") + flag.StringVar(&cloudConfig.ApiEndpoint, "gce-api-endpoint", "", "The GCE ApiEndpoint being used, if applicable") flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable") flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable") flag.BoolVar(&cloudConfig.MultiZone, "gce-multizone", false, "If true, start GCE cloud provider with multizone support.") From b3956a689eaa7730e3fe3c1ca253a4027f9a1b9f Mon Sep 17 00:00:00 2001 From: Koonwah Chen Date: Wed, 28 Jun 2017 16:03:18 -0700 Subject: [PATCH 3/5] Add KUBE_GCE_API_ENDPOINT for GCE API endpoint config. --- cluster/common.sh | 1 + cluster/gce/config-default.sh | 3 ++- cluster/gce/gci/configure-helper.sh | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index 502abfbb81c..951826cbe65 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -665,6 +665,7 @@ ENABLE_APISERVER_BASIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_BASIC_AUDIT:-}) ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-}) ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false}) ADVANCED_AUDIT_BACKEND: $(yaml-quote ${ADVANCED_AUDIT_BACKEND:-log}) +KUBE_GCE_API_ENDPOINT: $(yaml-quote ${KUBE_GCE_API_ENDPOINT:-}) EOF if [ -n "${KUBELET_PORT:-}" ]; then cat >>$file </etc/gce.conf [global] EOF + if [[ -n "${KUBE_GCE_API_ENDPOINT:-}" ]]; then + cat <>/etc/gce.conf +api-endpoint = ${KUBE_GCE_API_ENDPOINT} +EOF + fi if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then use_cloud_config="true" cat <>/etc/gce.conf @@ -1695,7 +1700,7 @@ function start-kube-addons { sed -i -e "s@__CALICO_TYPHA_CPU__@$(get-calico-typha-cpu)@g" "${typha_dep_file}" sed -i -e "s@__CALICO_TYPHA_REPLICAS__@$(get-calico-typha-replicas)@g" "${typha_dep_file}" else - # If not configured to use Calico, the set the typha replica count to 0, but only if the + # If not configured to use Calico, the set the typha replica count to 0, but only if the # addon is present. local -r typha_dep_file="${dst_dir}/calico-policy-controller/typha-deployment.yaml" if [[ -e $typha_dep_file ]]; then From c4e84e263c59f2976a7f1af8e949c7676b4b3877 Mon Sep 17 00:00:00 2001 From: Koonwah Chen Date: Thu, 29 Jun 2017 10:04:53 -0700 Subject: [PATCH 4/5] Change KUBE_GCE_API_ENDPOINT to GCE_API_ENDPOINT --- cluster/common.sh | 2 +- cluster/gce/config-default.sh | 2 +- cluster/gce/gci/configure-helper.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index 951826cbe65..0c793ddec6e 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -665,7 +665,7 @@ ENABLE_APISERVER_BASIC_AUDIT: $(yaml-quote ${ENABLE_APISERVER_BASIC_AUDIT:-}) ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote ${ENABLE_APISERVER_ADVANCED_AUDIT:-}) ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote ${ENABLE_CACHE_MUTATION_DETECTOR:-false}) ADVANCED_AUDIT_BACKEND: $(yaml-quote ${ADVANCED_AUDIT_BACKEND:-log}) -KUBE_GCE_API_ENDPOINT: $(yaml-quote ${KUBE_GCE_API_ENDPOINT:-}) +GCE_API_ENDPOINT: $(yaml-quote ${GCE_API_ENDPOINT:-}) EOF if [ -n "${KUBELET_PORT:-}" ]; then cat >>$file </etc/gce.conf [global] EOF - if [[ -n "${KUBE_GCE_API_ENDPOINT:-}" ]]; then + if [[ -n "${GCE_API_ENDPOINT:-}" ]]; then cat <>/etc/gce.conf -api-endpoint = ${KUBE_GCE_API_ENDPOINT} +api-endpoint = ${GCE_API_ENDPOINT} EOF fi if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then From 0db5b3716538217f0f4fce860d2fa2fe4b5f09d3 Mon Sep 17 00:00:00 2001 From: Koonwah Chen Date: Thu, 29 Jun 2017 10:42:29 -0700 Subject: [PATCH 5/5] testing fixed hack/verify-gofmt.sh and hack/verify-flags-underscore.py --- hack/verify-flags/known-flags.txt | 1 + pkg/cloudprovider/providers/gce/gce.go | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index b504324bb95..a4422ddce5e 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -297,6 +297,7 @@ gather-logs-sizes gather-metrics-at-teardown gather-resource-usage gather-suite-metrics-at-teardown +gce-api-endpoint gce-multizone gce-project gce-service-account diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 4e60519daaf..c88064fab6d 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -194,16 +194,16 @@ func newGCECloud(config io.Reader) (*GCECloud, error) { projectID = cfg.Global.ProjectID } - if cfg.Global.NetworkName != "" && strings.Contains(cfg.Global.NetworkName, "/"){ + if cfg.Global.NetworkName != "" && strings.Contains(cfg.Global.NetworkName, "/") { networkURL = cfg.Global.NetworkName - }else { + } else { networkURL = gceNetworkURL(apiEndpoint, projectID, networkName) } - if cfg.Global.SubnetworkName != "" && strings.Contains(cfg.Global.SubnetworkName, "/"){ - subnetworkURL = cfg.Global.SubnetworkName - }else { - subnetworkURL = gceSubnetworkURL(apiEndpoint, cfg.Global.ProjectID, region, cfg.Global.SubnetworkName) + if cfg.Global.SubnetworkName != "" && strings.Contains(cfg.Global.SubnetworkName, "/") { + subnetworkURL = cfg.Global.SubnetworkName + } else { + subnetworkURL = gceSubnetworkURL(apiEndpoint, cfg.Global.ProjectID, region, cfg.Global.SubnetworkName) } if cfg.Global.TokenURL != "" { tokenSource = NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody)