From 1cad829b6eb287c5c4d0296a59dfb34dea03cac5 Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Mon, 31 Jul 2017 17:39:13 -0700 Subject: [PATCH] add LocalZone into gce.conf and refactor gce cloud provider configuration to allow avoiding external communication --- pkg/cloudprovider/providers/gce/BUILD | 1 + pkg/cloudprovider/providers/gce/gce.go | 225 ++++++++++++-------- pkg/cloudprovider/providers/gce/gce_test.go | 182 ++++++++++++++++ test/e2e/e2e.go | 17 +- 4 files changed, 331 insertions(+), 94 deletions(-) diff --git a/pkg/cloudprovider/providers/gce/BUILD b/pkg/cloudprovider/providers/gce/BUILD index 62576d352a2..1c6717d3d84 100644 --- a/pkg/cloudprovider/providers/gce/BUILD +++ b/pkg/cloudprovider/providers/gce/BUILD @@ -91,6 +91,7 @@ go_test( deps = [ "//pkg/cloudprovider:go_default_library", "//pkg/kubelet/apis:go_default_library", + "//vendor/golang.org/x/oauth2/google:go_default_library", "//vendor/google.golang.org/api/compute/v1:go_default_library", "//vendor/google.golang.org/api/googleapi:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 9652fd4d65a..a6d5dccaedb 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -133,7 +133,7 @@ type GCEServiceManager struct { gce *GCECloud } -type Config struct { +type ConfigFile struct { Global struct { TokenURL string `gcfg:"token-url"` TokenBody string `gcfg:"token-body"` @@ -145,9 +145,25 @@ type Config struct { Multizone bool `gcfg:"multizone"` // Specifying ApiEndpoint will override the default GCE compute API endpoint. ApiEndpoint string `gcfg:"api-endpoint"` + LocalZone string `gcfg:"local-zone"` } } +// CloudConfig includes all the necessary configuration for creating GCECloud +type CloudConfig struct { + ApiEndpoint string + ProjectID string + Region string + Zone string + ManagedZones []string + NetworkURL string + SubnetworkURL string + NodeTags []string + NodeInstancePrefix string + TokenSource oauth2.TokenSource + UseMetadataServer bool +} + func init() { cloudprovider.RegisterCloudProvider( ProviderName, @@ -172,77 +188,28 @@ func (g *GCECloud) GetProjectID() string { } // newGCECloud creates a new instance of GCECloud. -func newGCECloud(config io.Reader) (*GCECloud, error) { - apiEndpoint := "" - projectID, zone, err := getProjectAndZone() - if err != nil { - return nil, err - } +func newGCECloud(config io.Reader) (gceCloud *GCECloud, err error) { + var cloudConfig *CloudConfig + var configFile *ConfigFile - region, err := GetGCERegion(zone) - if err != nil { - return nil, err - } - - networkName, err := getNetworkNameViaMetadata() - if err != nil { - return nil, err - } - networkURL := gceNetworkURL("", projectID, networkName) - subnetworkURL := "" - - // By default, Kubernetes clusters only run against one zone - managedZones := []string{zone} - - tokenSource := google.ComputeTokenSource("") - var nodeTags []string - var nodeInstancePrefix string if config != nil { - cfg, err := readConfig(config) + configFile, err = readConfig(config) if err != nil { return nil, err } - - glog.Infof("Using GCE provider config %+v", cfg) - if cfg.Global.ApiEndpoint != "" { - apiEndpoint = cfg.Global.ApiEndpoint - } - if cfg.Global.ProjectID != "" { - projectID = cfg.Global.ProjectID - } - - if cfg.Global.NetworkName != "" { - if strings.Contains(cfg.Global.NetworkName, "/") { - networkURL = cfg.Global.NetworkName - } else { - networkURL = gceNetworkURL(apiEndpoint, projectID, cfg.Global.NetworkName) - } - } - - if cfg.Global.SubnetworkName != "" { - if strings.Contains(cfg.Global.SubnetworkName, "/") { - subnetworkURL = cfg.Global.SubnetworkName - } else { - subnetworkURL = gceSubnetworkURL(apiEndpoint, projectID, region, cfg.Global.SubnetworkName) - } - } - - if cfg.Global.TokenURL != "" { - tokenSource = NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody) - } - nodeTags = cfg.Global.NodeTags - nodeInstancePrefix = cfg.Global.NodeInstancePrefix - if cfg.Global.Multizone { - managedZones = nil // Use all zones in region - } + glog.Infof("Using GCE provider config %+v", configFile) } - return CreateGCECloud(apiEndpoint, projectID, region, zone, managedZones, networkURL, subnetworkURL, - nodeTags, nodeInstancePrefix, tokenSource, true /* useMetadataServer */) + cloudConfig, err = generateCloudConfig(configFile) + if err != nil { + return nil, err + } + return CreateGCECloud(cloudConfig) + } -func readConfig(reader io.Reader) (*Config, error) { - cfg := &Config{} +func readConfig(reader io.Reader) (*ConfigFile, error) { + cfg := &ConfigFile{} if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil { glog.Errorf("Couldn't read config: %v", err) return nil, err @@ -250,14 +217,92 @@ func readConfig(reader io.Reader) (*Config, error) { return cfg, nil } +func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err error) { + cloudConfig = &CloudConfig{} + // By default, fetch token from GCE metadata server + cloudConfig.TokenSource = google.ComputeTokenSource("") + cloudConfig.UseMetadataServer = true + + if configFile != nil { + if configFile.Global.ApiEndpoint != "" { + cloudConfig.ApiEndpoint = configFile.Global.ApiEndpoint + } + + if configFile.Global.TokenURL != "" { + // if tokenURL is nil, set tokenSource to nil. This will force the OAuth client to fall + // back to use DefaultTokenSource. This allows running gceCloud remotely. + if configFile.Global.TokenURL == "nil" { + cloudConfig.TokenSource = nil + } else { + cloudConfig.TokenSource = NewAltTokenSource(configFile.Global.TokenURL, configFile.Global.TokenBody) + } + } + + cloudConfig.NodeTags = configFile.Global.NodeTags + cloudConfig.NodeInstancePrefix = configFile.Global.NodeInstancePrefix + } + + // retrieve projectID and zone + if configFile == nil || configFile.Global.ProjectID == "" || configFile.Global.LocalZone == "" { + cloudConfig.ProjectID, cloudConfig.Zone, err = getProjectAndZone() + if err != nil { + return nil, err + } + } + if configFile != nil { + if configFile.Global.ProjectID != "" { + cloudConfig.ProjectID = configFile.Global.ProjectID + } + if configFile.Global.LocalZone != "" { + cloudConfig.Zone = configFile.Global.LocalZone + } + } + + // retrieve region + cloudConfig.Region, err = GetGCERegion(cloudConfig.Zone) + if err != nil { + return nil, err + } + + // generate managedZones + cloudConfig.ManagedZones = []string{cloudConfig.Zone} + if configFile != nil && configFile.Global.Multizone { + cloudConfig.ManagedZones = nil // Use all zones in region + } + + // generate networkURL + if configFile != nil && configFile.Global.NetworkName != "" { + if strings.Contains(configFile.Global.NetworkName, "/") { + cloudConfig.NetworkURL = configFile.Global.NetworkName + } else { + cloudConfig.NetworkURL = gceNetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, configFile.Global.NetworkName) + } + } else { + networkName, err := getNetworkNameViaMetadata() + if err != nil { + return nil, err + } + cloudConfig.NetworkURL = gceNetworkURL("", cloudConfig.ProjectID, networkName) + } + + // generate subnetworkURL + if configFile != nil && configFile.Global.SubnetworkName != "" { + if strings.Contains(configFile.Global.SubnetworkName, "/") { + cloudConfig.SubnetworkURL = configFile.Global.SubnetworkName + } else { + cloudConfig.SubnetworkURL = gceSubnetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, cloudConfig.Region, configFile.Global.SubnetworkName) + } + } + return cloudConfig, err +} + // Creates a GCECloud object using the specified parameters. // If no networkUrl is specified, loads networkName via rest call. // If no tokenSource is specified, uses oauth2.DefaultTokenSource. // If managedZones is nil / empty all zones in the region will be managed. -func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string, - nodeInstancePrefix string, tokenSource oauth2.TokenSource, useMetadataServer bool) (*GCECloud, error) { +func CreateGCECloud(config *CloudConfig) (*GCECloud, error) { - client, err := newOauthClient(tokenSource) + client, err := newOauthClient(config.TokenSource) if err != nil { return nil, err } @@ -266,7 +311,7 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones [] return nil, err } - client, err = newOauthClient(tokenSource) + client, err = newOauthClient(config.TokenSource) if err != nil { return nil, err } @@ -275,7 +320,7 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones [] return nil, err } - client, err = newOauthClient(tokenSource) + client, err = newOauthClient(config.TokenSource) if err != nil { return nil, err } @@ -288,10 +333,10 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones [] // Generate alpha and beta api endpoints based on override v1 api endpoint. // For example, // staging API endpoint: https://www.googleapis.com/compute/staging_v1/ - if apiEndpoint != "" { - service.BasePath = fmt.Sprintf("%sprojects/", apiEndpoint) - serviceBeta.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "beta", -1)) - serviceAlpha.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "alpha", -1)) + if config.ApiEndpoint != "" { + service.BasePath = fmt.Sprintf("%sprojects/", config.ApiEndpoint) + serviceBeta.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(config.ApiEndpoint, "v1", "beta", -1)) + serviceAlpha.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(config.ApiEndpoint, "v1", "alpha", -1)) } containerService, err := container.New(client) @@ -304,28 +349,28 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones [] return nil, err } - if networkURL == "" { - networkName, err := getNetworkNameViaAPICall(service, projectID) + if config.NetworkURL == "" { + networkName, err := getNetworkNameViaAPICall(service, config.ProjectID) if err != nil { return nil, err } - networkURL = gceNetworkURL(apiEndpoint, projectID, networkName) + config.NetworkURL = gceNetworkURL(config.ApiEndpoint, config.ProjectID, networkName) } - networkProjectID, err := getProjectIDInURL(networkURL) + networkProjectID, err := getProjectIDInURL(config.NetworkURL) if err != nil { return nil, err } - onXPN := networkProjectID != projectID + onXPN := networkProjectID != config.ProjectID - if len(managedZones) == 0 { - managedZones, err = getZonesForRegion(service, projectID, region) + if len(config.ManagedZones) == 0 { + config.ManagedZones, err = getZonesForRegion(service, config.ProjectID, config.Region) if err != nil { return nil, err } } - if len(managedZones) != 1 { - glog.Infof("managing multiple zones: %v", managedZones) + if len(config.ManagedZones) != 1 { + glog.Infof("managing multiple zones: %v", config.ManagedZones) } operationPollRateLimiter := flowcontrol.NewTokenBucketRateLimiter(10, 100) // 10 qps, 100 bucket size. @@ -336,17 +381,17 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones [] serviceBeta: serviceBeta, containerService: containerService, cloudkmsService: cloudkmsService, - projectID: projectID, + projectID: config.ProjectID, networkProjectID: networkProjectID, onXPN: onXPN, - region: region, - localZone: zone, - managedZones: managedZones, - networkURL: networkURL, - subnetworkURL: subnetworkURL, - nodeTags: nodeTags, - nodeInstancePrefix: nodeInstancePrefix, - useMetadataServer: useMetadataServer, + region: config.Region, + localZone: config.Zone, + managedZones: config.ManagedZones, + networkURL: config.NetworkURL, + subnetworkURL: config.SubnetworkURL, + nodeTags: config.NodeTags, + nodeInstancePrefix: config.NodeInstancePrefix, + useMetadataServer: config.UseMetadataServer, operationPollRateLimiter: operationPollRateLimiter, } diff --git a/pkg/cloudprovider/providers/gce/gce_test.go b/pkg/cloudprovider/providers/gce/gce_test.go index 0eff53dbdf7..a4ac14734b4 100644 --- a/pkg/cloudprovider/providers/gce/gce_test.go +++ b/pkg/cloudprovider/providers/gce/gce_test.go @@ -17,6 +17,7 @@ limitations under the License. package gce import ( + "golang.org/x/oauth2/google" "reflect" "strings" "testing" @@ -255,3 +256,184 @@ func TestSplitProviderID(t *testing.T) { } } } + +func TestGenerateCloudConfigs(t *testing.T) { + testCases := []struct { + TokenURL string + TokenBody string + ProjectID string + NetworkName string + SubnetworkName string + NodeTags []string + NodeInstancePrefix string + Multizone bool + ApiEndpoint string + LocalZone string + cloudConfig *CloudConfig + }{ + { + TokenURL: "", + TokenBody: "", + ProjectID: "project-id", + NetworkName: "network-name", + SubnetworkName: "subnetwork-name", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + Multizone: false, + ApiEndpoint: "", + LocalZone: "us-central1-a", + cloudConfig: &CloudConfig{ + ApiEndpoint: "", + ProjectID: "project-id", + Region: "us-central1", + Zone: "us-central1-a", + ManagedZones: []string{"us-central1-a"}, + NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name", + SubnetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + TokenSource: google.ComputeTokenSource(""), + UseMetadataServer: true, + }, + }, + // nil token source + { + TokenURL: "nil", + TokenBody: "", + ProjectID: "project-id", + NetworkName: "network-name", + SubnetworkName: "subnetwork-name", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + Multizone: false, + ApiEndpoint: "", + LocalZone: "us-central1-a", + cloudConfig: &CloudConfig{ + ApiEndpoint: "", + ProjectID: "project-id", + Region: "us-central1", + Zone: "us-central1-a", + ManagedZones: []string{"us-central1-a"}, + NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name", + SubnetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + TokenSource: nil, + UseMetadataServer: true, + }, + }, + // specified api endpoint + { + TokenURL: "", + TokenBody: "", + ProjectID: "project-id", + NetworkName: "network-name", + SubnetworkName: "subnetwork-name", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + Multizone: false, + ApiEndpoint: "https://www.googleapis.com/compute/staging_v1/", + LocalZone: "us-central1-a", + cloudConfig: &CloudConfig{ + ApiEndpoint: "https://www.googleapis.com/compute/staging_v1/", + ProjectID: "project-id", + Region: "us-central1", + Zone: "us-central1-a", + ManagedZones: []string{"us-central1-a"}, + NetworkURL: "https://www.googleapis.com/compute/staging_v1/projects/project-id/global/networks/network-name", + SubnetworkURL: "https://www.googleapis.com/compute/staging_v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + TokenSource: google.ComputeTokenSource(""), + UseMetadataServer: true, + }, + }, + // empty subnet-name + { + TokenURL: "", + TokenBody: "", + ProjectID: "project-id", + NetworkName: "network-name", + SubnetworkName: "", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + Multizone: false, + ApiEndpoint: "", + LocalZone: "us-central1-a", + cloudConfig: &CloudConfig{ + ApiEndpoint: "", + ProjectID: "project-id", + Region: "us-central1", + Zone: "us-central1-a", + ManagedZones: []string{"us-central1-a"}, + NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name", + SubnetworkURL: "", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + TokenSource: google.ComputeTokenSource(""), + UseMetadataServer: true, + }, + }, + // multi zone + { + TokenURL: "", + TokenBody: "", + ProjectID: "project-id", + NetworkName: "network-name", + SubnetworkName: "subnetwork-name", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + Multizone: true, + ApiEndpoint: "", + LocalZone: "us-central1-a", + cloudConfig: &CloudConfig{ + ApiEndpoint: "", + ProjectID: "project-id", + Region: "us-central1", + Zone: "us-central1-a", + ManagedZones: nil, + NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name", + SubnetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name", + NodeTags: []string{"node-tag"}, + NodeInstancePrefix: "node-prefix", + TokenSource: google.ComputeTokenSource(""), + UseMetadataServer: true, + }, + }, + } + + for _, tc := range testCases { + cloudConfig, err := generateCloudConfig(&ConfigFile{ + Global: struct { + TokenURL string `gcfg:"token-url"` + TokenBody string `gcfg:"token-body"` + ProjectID string `gcfg:"project-id"` + NetworkName string `gcfg:"network-name"` + SubnetworkName string `gcfg:"subnetwork-name"` + NodeTags []string `gcfg:"node-tags"` + NodeInstancePrefix string `gcfg:"node-instance-prefix"` + Multizone bool `gcfg:"multizone"` + ApiEndpoint string `gcfg:"api-endpoint"` + LocalZone string `gcfg:"local-zone"` + }{ + TokenURL: tc.TokenURL, + TokenBody: tc.TokenBody, + ProjectID: tc.ProjectID, + NetworkName: tc.NetworkName, + SubnetworkName: tc.SubnetworkName, + NodeTags: tc.NodeTags, + NodeInstancePrefix: tc.NodeInstancePrefix, + Multizone: tc.Multizone, + ApiEndpoint: tc.ApiEndpoint, + LocalZone: tc.LocalZone, + }, + }) + if err != nil { + t.Fatalf("Unexpect error: %v", err) + } + + if !reflect.DeepEqual(cloudConfig, tc.cloudConfig) { + t.Errorf("Expecting cloud config: %v, but got %v", tc.cloudConfig, cloudConfig) + } + } +} diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index a990213fd9a..5e5008db27c 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -78,10 +78,19 @@ func setupProviderConfig() error { managedZones = []string{zone} } - gceCloud, err := gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ApiEndpoint, - framework.TestContext.CloudConfig.ProjectID, - region, zone, managedZones, "" /* networkUrl */, "" /* subnetworkUrl */, nil, /* nodeTags */ - "" /* nodeInstancePerfix */, nil /* tokenSource */, false /* useMetadataServer */) + gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{ + ApiEndpoint: framework.TestContext.CloudConfig.ApiEndpoint, + ProjectID: framework.TestContext.CloudConfig.ProjectID, + Region: region, + Zone: zone, + ManagedZones: managedZones, + NetworkURL: "", + SubnetworkURL: "", + NodeTags: nil, + NodeInstancePrefix: "", + TokenSource: nil, + UseMetadataServer: false}) + if err != nil { return fmt.Errorf("Error building GCE/GKE provider: %v", err) }