diff --git a/pkg/cloudprovider/providers/gce/BUILD b/pkg/cloudprovider/providers/gce/BUILD index 2d2197e6681..800f8b2e332 100644 --- a/pkg/cloudprovider/providers/gce/BUILD +++ b/pkg/cloudprovider/providers/gce/BUILD @@ -23,7 +23,6 @@ go_library( "gce_disks.go", "gce_firewall.go", "gce_forwardingrule.go", - "gce_forwardingrule_fakes.go", "gce_healthchecks.go", "gce_instancegroup.go", "gce_instances.go", @@ -117,6 +116,7 @@ go_test( "//vendor/google.golang.org/api/googleapi:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], ) diff --git a/pkg/cloudprovider/providers/gce/cloud/BUILD b/pkg/cloudprovider/providers/gce/cloud/BUILD index dc6fa75605b..fa46310777d 100644 --- a/pkg/cloudprovider/providers/gce/cloud/BUILD +++ b/pkg/cloudprovider/providers/gce/cloud/BUILD @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "constants.go", "doc.go", "gce_projects.go", "gen.go", diff --git a/pkg/cloudprovider/providers/gce/cloud/constants.go b/pkg/cloudprovider/providers/gce/cloud/constants.go new file mode 100644 index 00000000000..8d09c8bead8 --- /dev/null +++ b/pkg/cloudprovider/providers/gce/cloud/constants.go @@ -0,0 +1,55 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud + +import ( + "strings" +) + +// NetworkTier represents the Network Service Tier used by a resource +type NetworkTier string + +// LbScheme represents the possible types of load balancers +type LbScheme string + +const ( + NetworkTierStandard NetworkTier = "Standard" + NetworkTierPremium NetworkTier = "Premium" + NetworkTierDefault NetworkTier = NetworkTierPremium + + SchemeExternal LbScheme = "EXTERNAL" + SchemeInternal LbScheme = "INTERNAL" +) + +// ToGCEValue converts NetworkTier to a string that we can populate the +// NetworkTier field of GCE objects, including ForwardingRules and Addresses. +func (n NetworkTier) ToGCEValue() string { + return strings.ToUpper(string(n)) +} + +// NetworkTierGCEValueToType converts the value of the NetworkTier field of a +// GCE object to the NetworkTier type. +func NetworkTierGCEValueToType(s string) NetworkTier { + switch s { + case NetworkTierStandard.ToGCEValue(): + return NetworkTierStandard + case NetworkTierPremium.ToGCEValue(): + return NetworkTierPremium + default: + return NetworkTier(s) + } +} diff --git a/pkg/cloudprovider/providers/gce/cloud/mock/BUILD b/pkg/cloudprovider/providers/gce/cloud/mock/BUILD index fa07f1fb289..bceb2e84739 100644 --- a/pkg/cloudprovider/providers/gce/cloud/mock/BUILD +++ b/pkg/cloudprovider/providers/gce/cloud/mock/BUILD @@ -8,6 +8,8 @@ go_library( deps = [ "//pkg/cloudprovider/providers/gce/cloud:go_default_library", "//pkg/cloudprovider/providers/gce/cloud/meta:go_default_library", + "//vendor/google.golang.org/api/compute/v0.alpha:go_default_library", + "//vendor/google.golang.org/api/compute/v0.beta:go_default_library", "//vendor/google.golang.org/api/compute/v1:go_default_library", "//vendor/google.golang.org/api/googleapi:go_default_library", ], diff --git a/pkg/cloudprovider/providers/gce/cloud/mock/mock.go b/pkg/cloudprovider/providers/gce/cloud/mock/mock.go index 7018f352f09..06ff480d47b 100644 --- a/pkg/cloudprovider/providers/gce/cloud/mock/mock.go +++ b/pkg/cloudprovider/providers/gce/cloud/mock/mock.go @@ -25,15 +25,23 @@ package mock import ( "context" + "encoding/json" "fmt" "net/http" + alpha "google.golang.org/api/compute/v0.alpha" + beta "google.golang.org/api/compute/v0.beta" ga "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" - cloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta" ) +// gceObject is an abstraction of all GCE API object in go client +type gceObject interface { + MarshalJSON() ([]byte, error) +} + // AddInstanceHook mocks adding a Instance to MockTargetPools func AddInstanceHook(ctx context.Context, key *meta.Key, req *ga.TargetPoolsAddInstanceRequest, m *cloud.MockTargetPools) error { pool, err := m.Get(ctx, key) @@ -74,3 +82,147 @@ func RemoveInstanceHook(ctx context.Context, key *meta.Key, req *ga.TargetPoolsR return nil } + +func convertAndInsertAlphaForwardingRule(key *meta.Key, obj gceObject, mRules map[meta.Key]*cloud.MockForwardingRulesObj, version meta.Version, projectID string) (bool, error) { + if !key.Valid() { + return false, fmt.Errorf("invalid GCE key (%+v)", key) + } + + if _, ok := mRules[*key]; ok { + err := &googleapi.Error{ + Code: http.StatusConflict, + Message: fmt.Sprintf("MockForwardingRule %v exists", key), + } + return false, err + } + + enc, err := obj.MarshalJSON() + if err != nil { + return false, err + } + var fwdRule alpha.ForwardingRule + if err := json.Unmarshal(enc, &fwdRule); err != nil { + return false, err + } + // Set the default values for the Alpha fields. + if fwdRule.NetworkTier == "" { + fwdRule.NetworkTier = cloud.NetworkTierDefault.ToGCEValue() + } + + fwdRule.Name = key.Name + if fwdRule.SelfLink == "" { + fwdRule.SelfLink = cloud.SelfLink(version, projectID, "forwardingRules", key) + } + + mRules[*key] = &cloud.MockForwardingRulesObj{Obj: fwdRule} + return true, nil +} + +// InsertFwdRuleHook mocks inserting a ForwardingRule. ForwardingRules are +// expected to default to Premium tier if no NetworkTier is specified. +func InsertFwdRuleHook(ctx context.Context, key *meta.Key, obj *ga.ForwardingRule, m *cloud.MockForwardingRules) (bool, error) { + projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionGA, "forwardingRules") + return convertAndInsertAlphaForwardingRule(key, obj, m.Objects, meta.VersionGA, projectID) +} + +// InsertBetaFwdRuleHook mocks inserting a BetaForwardingRule. +func InsertBetaFwdRuleHook(ctx context.Context, key *meta.Key, obj *beta.ForwardingRule, m *cloud.MockForwardingRules) (bool, error) { + projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionBeta, "forwardingRules") + return convertAndInsertAlphaForwardingRule(key, obj, m.Objects, meta.VersionBeta, projectID) +} + +// InsertAlphaFwdRuleHook mocks inserting an AlphaForwardingRule. +func InsertAlphaFwdRuleHook(ctx context.Context, key *meta.Key, obj *alpha.ForwardingRule, m *cloud.MockForwardingRules) (bool, error) { + projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionAlpha, "forwardingRules") + return convertAndInsertAlphaForwardingRule(key, obj, m.Objects, meta.VersionAlpha, projectID) +} + +// Used to assign Addresses with no IP a unique IP address +var ipCounter = 1 + +func convertAndInsertAlphaAddress(key *meta.Key, obj gceObject, mAddrs map[meta.Key]*cloud.MockAddressesObj, version meta.Version, projectID string) (bool, error) { + if !key.Valid() { + return false, fmt.Errorf("invalid GCE key (%+v)", key) + } + + if _, ok := mAddrs[*key]; ok { + err := &googleapi.Error{ + Code: http.StatusConflict, + Message: fmt.Sprintf("MockAddresses %v exists", key), + } + return false, err + } + + enc, err := obj.MarshalJSON() + if err != nil { + return false, err + } + var addr alpha.Address + if err := json.Unmarshal(enc, &addr); err != nil { + return false, err + } + + // Set default address type if not present. + if addr.AddressType == "" { + addr.AddressType = string(cloud.SchemeExternal) + } + + var existingAddresses []*ga.Address + for _, obj := range mAddrs { + existingAddresses = append(existingAddresses, obj.ToGA()) + } + + for _, existingAddr := range existingAddresses { + if addr.Address == existingAddr.Address { + msg := fmt.Sprintf("MockAddresses IP %v in use", addr.Address) + + // When the IP is already in use, this call returns a StatusBadRequest + // if the address is an external address, and StatusConflict if an + // internal address. This is to be consistent with actual GCE API. + errorCode := http.StatusConflict + if addr.AddressType == string(cloud.SchemeExternal) { + errorCode = http.StatusBadRequest + } + + return false, &googleapi.Error{Code: errorCode, Message: msg} + } + } + + // Set default values used in tests + addr.Name = key.Name + if addr.SelfLink == "" { + addr.SelfLink = cloud.SelfLink(version, projectID, "addresses", key) + } + + if addr.Address == "" { + addr.Address = fmt.Sprintf("1.2.3.%d", ipCounter) + ipCounter++ + } + + // Set the default values for the Alpha fields. + if addr.NetworkTier == "" { + addr.NetworkTier = cloud.NetworkTierDefault.ToGCEValue() + } + + mAddrs[*key] = &cloud.MockAddressesObj{Obj: addr} + return true, nil +} + +// InsertAddressHook mocks inserting an Address. +func InsertAddressHook(ctx context.Context, key *meta.Key, obj *ga.Address, m *cloud.MockAddresses) (bool, error) { + projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionGA, "addresses") + return convertAndInsertAlphaAddress(key, obj, m.Objects, meta.VersionGA, projectID) +} + +// InsertBetaAddressHook mocks inserting a BetaAddress. +func InsertBetaAddressHook(ctx context.Context, key *meta.Key, obj *beta.Address, m *cloud.MockAddresses) (bool, error) { + projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionBeta, "addresses") + return convertAndInsertAlphaAddress(key, obj, m.Objects, meta.VersionBeta, projectID) +} + +// InsertAlphaAddressHook mocks inserting an Address. Addresses are expected to +// default to Premium tier if no NetworkTier is specified. +func InsertAlphaAddressHook(ctx context.Context, key *meta.Key, obj *alpha.Address, m *cloud.MockAlphaAddresses) (bool, error) { + projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionBeta, "addresses") + return convertAndInsertAlphaAddress(key, obj, m.Objects, meta.VersionAlpha, projectID) +} diff --git a/pkg/cloudprovider/providers/gce/gce_address_manager.go b/pkg/cloudprovider/providers/gce/gce_address_manager.go index ad7c38b89d2..4f642a92ccf 100644 --- a/pkg/cloudprovider/providers/gce/gce_address_manager.go +++ b/pkg/cloudprovider/providers/gce/gce_address_manager.go @@ -23,6 +23,7 @@ import ( compute "google.golang.org/api/compute/v1" "github.com/golang/glog" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" ) type addressManager struct { @@ -31,13 +32,13 @@ type addressManager struct { name string serviceName string targetIP string - addressType lbScheme + addressType cloud.LbScheme region string subnetURL string tryRelease bool } -func newAddressManager(svc CloudAddressService, serviceName, region, subnetURL, name, targetIP string, addressType lbScheme) *addressManager { +func newAddressManager(svc CloudAddressService, serviceName, region, subnetURL, name, targetIP string, addressType cloud.LbScheme) *addressManager { return &addressManager{ svc: svc, logPrefix: fmt.Sprintf("AddressManager(%q)", name), diff --git a/pkg/cloudprovider/providers/gce/gce_address_manager_test.go b/pkg/cloudprovider/providers/gce/gce_address_manager_test.go index 1eee47cae93..3c7d60dc564 100644 --- a/pkg/cloudprovider/providers/gce/gce_address_manager_test.go +++ b/pkg/cloudprovider/providers/gce/gce_address_manager_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" compute "google.golang.org/api/compute/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" ) const testSvcName = "my-service" @@ -34,8 +35,8 @@ func TestAddressManagerNoRequestedIP(t *testing.T) { svc := NewFakeCloudAddressService() targetIP := "" - mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal) - testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal)) + mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal) + testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal)) testReleaseAddress(t, mgr, svc, testLBName, testRegion) } @@ -44,8 +45,8 @@ func TestAddressManagerBasic(t *testing.T) { svc := NewFakeCloudAddressService() targetIP := "1.1.1.1" - mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal) - testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal)) + mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal) + testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal)) testReleaseAddress(t, mgr, svc, testLBName, testRegion) } @@ -55,12 +56,12 @@ func TestAddressManagerOrphaned(t *testing.T) { svc := NewFakeCloudAddressService() targetIP := "1.1.1.1" - addr := &compute.Address{Name: testLBName, Address: targetIP, AddressType: string(schemeInternal)} + addr := &compute.Address{Name: testLBName, Address: targetIP, AddressType: string(cloud.SchemeInternal)} err := svc.ReserveRegionAddress(addr, testRegion) require.NoError(t, err) - mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal) - testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal)) + mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal) + testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal)) testReleaseAddress(t, mgr, svc, testLBName, testRegion) } @@ -71,12 +72,12 @@ func TestAddressManagerOutdatedOrphan(t *testing.T) { previousAddress := "1.1.0.0" targetIP := "1.1.1.1" - addr := &compute.Address{Name: testLBName, Address: previousAddress, AddressType: string(schemeExternal)} + addr := &compute.Address{Name: testLBName, Address: previousAddress, AddressType: string(cloud.SchemeExternal)} err := svc.ReserveRegionAddress(addr, testRegion) require.NoError(t, err) - mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal) - testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal)) + mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal) + testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal)) testReleaseAddress(t, mgr, svc, testLBName, testRegion) } @@ -86,11 +87,11 @@ func TestAddressManagerExternallyOwned(t *testing.T) { svc := NewFakeCloudAddressService() targetIP := "1.1.1.1" - addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeInternal)} + addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(cloud.SchemeInternal)} err := svc.ReserveRegionAddress(addr, testRegion) require.NoError(t, err) - mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal) + mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal) ipToUse, err := mgr.HoldAddress() require.NoError(t, err) assert.NotEmpty(t, ipToUse) @@ -107,11 +108,11 @@ func TestAddressManagerBadExternallyOwned(t *testing.T) { svc := NewFakeCloudAddressService() targetIP := "1.1.1.1" - addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeExternal)} + addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(cloud.SchemeExternal)} err := svc.ReserveRegionAddress(addr, testRegion) require.NoError(t, err) - mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal) + mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal) _, err = mgr.HoldAddress() assert.NotNil(t, err) } diff --git a/pkg/cloudprovider/providers/gce/gce_addresses.go b/pkg/cloudprovider/providers/gce/gce_addresses.go index af91fb14174..37e2f5e9a7b 100644 --- a/pkg/cloudprovider/providers/gce/gce_addresses.go +++ b/pkg/cloudprovider/providers/gce/gce_addresses.go @@ -26,6 +26,7 @@ import ( computebeta "google.golang.org/api/compute/v0.beta" compute "google.golang.org/api/compute/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta" ) @@ -150,7 +151,7 @@ func (gce *GCECloud) GetBetaRegionAddressByIP(region, ipAddress string) (*comput // TODO(#51665): retire this function once Network Tiers becomes Beta in GCP. func (gce *GCECloud) getNetworkTierFromAddress(name, region string) (string, error) { if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) { - return NetworkTierDefault.ToGCEValue(), nil + return cloud.NetworkTierDefault.ToGCEValue(), nil } addr, err := gce.GetAlphaRegionAddress(name, region) if err != nil { diff --git a/pkg/cloudprovider/providers/gce/gce_addresses_fakes.go b/pkg/cloudprovider/providers/gce/gce_addresses_fakes.go index e9ed209e4f4..75dfa571c9c 100644 --- a/pkg/cloudprovider/providers/gce/gce_addresses_fakes.go +++ b/pkg/cloudprovider/providers/gce/gce_addresses_fakes.go @@ -25,6 +25,8 @@ import ( computealpha "google.golang.org/api/compute/v0.alpha" computebeta "google.golang.org/api/compute/v0.beta" compute "google.golang.org/api/compute/v1" + + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" ) // test @@ -68,7 +70,7 @@ func (cas *FakeCloudAddressService) ReserveAlphaRegionAddress(addr *computealpha } if addr.AddressType == "" { - addr.AddressType = string(schemeExternal) + addr.AddressType = string(cloud.SchemeExternal) } if cas.reservedAddrs[addr.Address] { @@ -76,8 +78,8 @@ func (cas *FakeCloudAddressService) ReserveAlphaRegionAddress(addr *computealpha // When the IP is already in use, this call returns an error code based // on the type (internal vs external) of the address. This is to be // consistent with actual GCE API. - switch lbScheme(addr.AddressType) { - case schemeExternal: + switch cloud.LbScheme(addr.AddressType) { + case cloud.SchemeExternal: return makeGoogleAPIError(http.StatusBadRequest, msg) default: return makeGoogleAPIError(http.StatusConflict, msg) @@ -209,7 +211,7 @@ func convertToAlphaAddress(object gceObject) *computealpha.Address { panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha address: %v", object, err)) } // Set the default values for the Alpha fields. - addr.NetworkTier = NetworkTierDefault.ToGCEValue() + addr.NetworkTier = cloud.NetworkTierDefault.ToGCEValue() return &addr } diff --git a/pkg/cloudprovider/providers/gce/gce_annotations.go b/pkg/cloudprovider/providers/gce/gce_annotations.go index 37671aeb2a9..ccc0f444118 100644 --- a/pkg/cloudprovider/providers/gce/gce_annotations.go +++ b/pkg/cloudprovider/providers/gce/gce_annotations.go @@ -18,15 +18,14 @@ package gce import ( "fmt" - "strings" "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" ) type LoadBalancerType string -type NetworkTier string const ( // ServiceAnnotationLoadBalancerType is annotated on a service with type LoadBalancer @@ -49,12 +48,8 @@ const ( // network tier a GCP LB should use. The valid values are "Standard" and // "Premium" (default). NetworkTierAnnotationKey = "cloud.google.com/network-tier" - NetworkTierAnnotationStandard = "Standard" - NetworkTierAnnotationPremium = "Premium" - - NetworkTierStandard NetworkTier = NetworkTierAnnotationStandard - NetworkTierPremium NetworkTier = NetworkTierAnnotationPremium - NetworkTierDefault NetworkTier = NetworkTierPremium + NetworkTierAnnotationStandard = cloud.NetworkTierStandard + NetworkTierAnnotationPremium = cloud.NetworkTierPremium ) // GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled. @@ -97,38 +92,19 @@ func GetLoadBalancerAnnotationBackendShare(service *v1.Service) bool { // GetServiceNetworkTier returns the network tier of GCP load balancer // which should be assembled, and an error if the specified tier is not // supported. -func GetServiceNetworkTier(service *v1.Service) (NetworkTier, error) { +func GetServiceNetworkTier(service *v1.Service) (cloud.NetworkTier, error) { l, ok := service.Annotations[NetworkTierAnnotationKey] if !ok { - return NetworkTierDefault, nil + return cloud.NetworkTierDefault, nil } - v := NetworkTier(l) + v := cloud.NetworkTier(l) switch v { - case NetworkTierStandard: + case cloud.NetworkTierStandard: fallthrough - case NetworkTierPremium: + case cloud.NetworkTierPremium: return v, nil default: - return NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v) - } -} - -// ToGCEValue converts NetworkTier to a string that we can populate the -// NetworkTier field of GCE objects. -func (n NetworkTier) ToGCEValue() string { - return strings.ToUpper(string(n)) -} - -// NetworkTierGCEValueToType converts the value of the NetworkTier field of a -// GCE object to the NetworkTier type. -func NetworkTierGCEValueToType(s string) NetworkTier { - switch s { - case "STANDARD": - return NetworkTierStandard - case "PREMIUM": - return NetworkTierPremium - default: - return NetworkTier(s) + return cloud.NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v) } } diff --git a/pkg/cloudprovider/providers/gce/gce_annotations_test.go b/pkg/cloudprovider/providers/gce/gce_annotations_test.go index 62aad6f4180..a3b2dfb86bf 100644 --- a/pkg/cloudprovider/providers/gce/gce_annotations_test.go +++ b/pkg/cloudprovider/providers/gce/gce_annotations_test.go @@ -21,6 +21,7 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "github.com/stretchr/testify/assert" ) @@ -38,24 +39,24 @@ func TestServiceNetworkTierAnnotationKey(t *testing.T) { for testName, testCase := range map[string]struct { annotations map[string]string - expectedTier NetworkTier + expectedTier cloud.NetworkTier expectErr bool }{ "Use the default when the annotation does not exist": { annotations: nil, - expectedTier: NetworkTierDefault, + expectedTier: cloud.NetworkTierDefault, }, "Standard tier": { annotations: map[string]string{NetworkTierAnnotationKey: "Standard"}, - expectedTier: NetworkTierStandard, + expectedTier: cloud.NetworkTierStandard, }, "Premium tier": { annotations: map[string]string{NetworkTierAnnotationKey: "Premium"}, - expectedTier: NetworkTierPremium, + expectedTier: cloud.NetworkTierPremium, }, "Report an error on invalid network tier value": { annotations: map[string]string{NetworkTierAnnotationKey: "Unknown-tier"}, - expectedTier: NetworkTierPremium, + expectedTier: cloud.NetworkTierPremium, expectErr: true, }, } { diff --git a/pkg/cloudprovider/providers/gce/gce_forwardingrule.go b/pkg/cloudprovider/providers/gce/gce_forwardingrule.go index 57e7b85f3be..9fe0da9f7cc 100644 --- a/pkg/cloudprovider/providers/gce/gce_forwardingrule.go +++ b/pkg/cloudprovider/providers/gce/gce_forwardingrule.go @@ -21,6 +21,7 @@ import ( computealpha "google.golang.org/api/compute/v0.alpha" compute "google.golang.org/api/compute/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta" ) @@ -117,7 +118,7 @@ func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error { // TODO(#51665): retire this function once Network Tiers becomes Beta in GCP. func (gce *GCECloud) getNetworkTierFromForwardingRule(name, region string) (string, error) { if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) { - return NetworkTierDefault.ToGCEValue(), nil + return cloud.NetworkTierDefault.ToGCEValue(), nil } fwdRule, err := gce.GetAlphaRegionForwardingRule(name, region) if err != nil { diff --git a/pkg/cloudprovider/providers/gce/gce_forwardingrule_fakes.go b/pkg/cloudprovider/providers/gce/gce_forwardingrule_fakes.go deleted file mode 100644 index 9c612952503..00000000000 --- a/pkg/cloudprovider/providers/gce/gce_forwardingrule_fakes.go +++ /dev/null @@ -1,138 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gce - -import ( - "encoding/json" - "fmt" - "net/http" - - computealpha "google.golang.org/api/compute/v0.alpha" - compute "google.golang.org/api/compute/v1" - "google.golang.org/api/googleapi" -) - -type FakeCloudForwardingRuleService struct { - // fwdRulesByRegionAndName - // Outer key is for region string; inner key is for fwdRuleess name. - fwdRulesByRegionAndName map[string]map[string]*computealpha.ForwardingRule -} - -// FakeCloudForwardingRuleService Implements CloudForwardingRuleService -var _ CloudForwardingRuleService = &FakeCloudForwardingRuleService{} - -func NewFakeCloudForwardingRuleService() *FakeCloudForwardingRuleService { - return &FakeCloudForwardingRuleService{ - fwdRulesByRegionAndName: make(map[string]map[string]*computealpha.ForwardingRule), - } -} - -// SetRegionalForwardingRulees sets the fwdRuleesses of there region. This is used for -// setting the test environment. -func (f *FakeCloudForwardingRuleService) SetRegionalForwardingRulees(region string, fwdRules []*computealpha.ForwardingRule) { - // Reset fwdRuleesses in the region. - f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule) - - for _, fwdRule := range fwdRules { - f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule - } -} - -func (f *FakeCloudForwardingRuleService) CreateAlphaRegionForwardingRule(fwdRule *computealpha.ForwardingRule, region string) error { - if _, exists := f.fwdRulesByRegionAndName[region]; !exists { - f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule) - } - - if _, exists := f.fwdRulesByRegionAndName[region][fwdRule.Name]; exists { - return &googleapi.Error{Code: http.StatusConflict} - } - - f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule - return nil -} - -func (f *FakeCloudForwardingRuleService) CreateRegionForwardingRule(fwdRule *compute.ForwardingRule, region string) error { - alphafwdRule := convertToAlphaForwardingRule(fwdRule) - return f.CreateAlphaRegionForwardingRule(alphafwdRule, region) -} - -func (f *FakeCloudForwardingRuleService) DeleteRegionForwardingRule(name, region string) error { - if _, exists := f.fwdRulesByRegionAndName[region]; !exists { - return makeGoogleAPINotFoundError("") - } - - if _, exists := f.fwdRulesByRegionAndName[region][name]; !exists { - return makeGoogleAPINotFoundError("") - } - delete(f.fwdRulesByRegionAndName[region], name) - return nil -} - -func (f *FakeCloudForwardingRuleService) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) { - if _, exists := f.fwdRulesByRegionAndName[region]; !exists { - return nil, makeGoogleAPINotFoundError("") - } - - if fwdRule, exists := f.fwdRulesByRegionAndName[region][name]; !exists { - return nil, makeGoogleAPINotFoundError("") - } else { - return fwdRule, nil - } -} - -func (f *FakeCloudForwardingRuleService) GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error) { - fwdRule, err := f.GetAlphaRegionForwardingRule(name, region) - if fwdRule != nil { - return convertToV1ForwardingRule(fwdRule), err - } - return nil, err -} - -func (f *FakeCloudForwardingRuleService) getNetworkTierFromForwardingRule(name, region string) (string, error) { - fwdRule, err := f.GetAlphaRegionForwardingRule(name, region) - if err != nil { - return "", err - } - return fwdRule.NetworkTier, nil -} - -func convertToV1ForwardingRule(object gceObject) *compute.ForwardingRule { - enc, err := object.MarshalJSON() - if err != nil { - panic(fmt.Sprintf("Failed to encode to json: %v", err)) - } - var fwdRule compute.ForwardingRule - if err := json.Unmarshal(enc, &fwdRule); err != nil { - panic(fmt.Sprintf("Failed to convert GCE apiObject %v to v1 fwdRuleess: %v", object, err)) - } - return &fwdRule -} - -func convertToAlphaForwardingRule(object gceObject) *computealpha.ForwardingRule { - enc, err := object.MarshalJSON() - if err != nil { - panic(fmt.Sprintf("Failed to encode to json: %v", err)) - } - var fwdRule computealpha.ForwardingRule - if err := json.Unmarshal(enc, &fwdRule); err != nil { - panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha fwdRuleess: %v", object, err)) - } - // Set the default values for the Alpha fields. - fwdRule.NetworkTier = NetworkTierDefault.ToGCEValue() - - return &fwdRule -} diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer.go index 34f22de34d9..4cb66234661 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer.go @@ -28,6 +28,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" netsets "k8s.io/kubernetes/pkg/util/net/sets" ) @@ -44,13 +45,6 @@ func newLoadBalancerMetricContext(request, region string) *metricContext { return newGenericMetricContext("loadbalancer", request, region, unusedMetricLabel, computeV1Version) } -type lbScheme string - -const ( - schemeExternal lbScheme = "EXTERNAL" - schemeInternal lbScheme = "INTERNAL" -) - func init() { var err error // LB L7 proxies and all L3/4/7 health checkers have client addresses within these known CIDRs. @@ -126,13 +120,13 @@ func (gce *GCECloud) EnsureLoadBalancer(ctx context.Context, clusterName string, } if existingFwdRule != nil { - existingScheme := lbScheme(strings.ToUpper(existingFwdRule.LoadBalancingScheme)) + existingScheme := cloud.LbScheme(strings.ToUpper(existingFwdRule.LoadBalancingScheme)) // If the loadbalancer type changes between INTERNAL and EXTERNAL, the old load balancer should be deleted. if existingScheme != desiredScheme { glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): deleting existing %v loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, gce.region, existingScheme) switch existingScheme { - case schemeInternal: + case cloud.SchemeInternal: err = gce.ensureInternalLoadBalancerDeleted(clusterName, clusterID, svc) default: err = gce.ensureExternalLoadBalancerDeleted(clusterName, clusterID, svc) @@ -149,7 +143,7 @@ func (gce *GCECloud) EnsureLoadBalancer(ctx context.Context, clusterName string, var status *v1.LoadBalancerStatus switch desiredScheme { - case schemeInternal: + case cloud.SchemeInternal: status, err = gce.ensureInternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes) default: status, err = gce.ensureExternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes) @@ -170,7 +164,7 @@ func (gce *GCECloud) UpdateLoadBalancer(ctx context.Context, clusterName string, glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v, %v): updating with %d nodes", clusterName, svc.Namespace, svc.Name, loadBalancerName, gce.region, len(nodes)) switch scheme { - case schemeInternal: + case cloud.SchemeInternal: err = gce.updateInternalLoadBalancer(clusterName, clusterID, svc, nodes) default: err = gce.updateExternalLoadBalancer(clusterName, svc, nodes) @@ -191,7 +185,7 @@ func (gce *GCECloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName glog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v, %v, %v): deleting loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, gce.region) switch scheme { - case schemeInternal: + case cloud.SchemeInternal: err = gce.ensureInternalLoadBalancerDeleted(clusterName, clusterID, svc) default: err = gce.ensureExternalLoadBalancerDeleted(clusterName, clusterID, svc) @@ -200,9 +194,9 @@ func (gce *GCECloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName return err } -func getSvcScheme(svc *v1.Service) lbScheme { +func getSvcScheme(svc *v1.Service) cloud.LbScheme { if typ, ok := GetLoadBalancerAnnotationType(svc); ok && typ == LBTypeInternal { - return schemeInternal + return cloud.SchemeInternal } - return schemeExternal + return cloud.SchemeExternal } diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go index 8cce4e2fe51..4fd60b969da 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" apiservice "k8s.io/kubernetes/pkg/api/v1/service" "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" netsets "k8s.io/kubernetes/pkg/util/net/sets" "github.com/golang/glog" @@ -416,7 +417,7 @@ func (gce *GCECloud) DeleteExternalTargetPoolAndChecks(service *v1.Service, name // the verification failed. It also returns a boolean to indicate whether the // IP address is considered owned by the user (i.e., not managed by the // controller. -func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string, desiredNetTier NetworkTier) (isUserOwnedIP bool, err error) { +func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string, desiredNetTier cloud.NetworkTier) (isUserOwnedIP bool, err error) { if requestedIP == "" { return false, nil } @@ -439,7 +440,7 @@ func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP if err != nil { return false, fmt.Errorf("failed to check the network tier of the IP %q: %v", requestedIP, err) } - netTier := NetworkTierGCEValueToType(netTierStr) + netTier := cloud.NetworkTierGCEValueToType(netTierStr) if netTier != desiredNetTier { glog.Errorf("verifyUserRequestedIP: requested static IP %q (name: %s) for LB %s has network tier %s, need %s.", requestedIP, existingAddress.Name, lbRef, netTier, desiredNetTier) return false, fmt.Errorf("requrested IP %q belongs to the %s network tier; expected %s", requestedIP, netTier, desiredNetTier) @@ -852,7 +853,7 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(svc *v1.Service, serviceName, return nil } -func createForwardingRule(s CloudForwardingRuleService, name, serviceName, region, ipAddress, target string, ports []v1.ServicePort, netTier NetworkTier) error { +func createForwardingRule(s CloudForwardingRuleService, name, serviceName, region, ipAddress, target string, ports []v1.ServicePort, netTier cloud.NetworkTier) error { portRange, err := loadBalancerPortRange(ports) if err != nil { return err @@ -861,7 +862,7 @@ func createForwardingRule(s CloudForwardingRuleService, name, serviceName, regio ipProtocol := string(ports[0].Protocol) switch netTier { - case NetworkTierPremium: + case cloud.NetworkTierPremium: rule := &compute.ForwardingRule{ Name: name, Description: desc, @@ -964,7 +965,7 @@ func (gce *GCECloud) firewallObject(name, region, desc string, sourceRanges nets return firewall, nil } -func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string, netTier NetworkTier) (ipAddress string, existing bool, err error) { +func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string, netTier cloud.NetworkTier) (ipAddress string, existing bool, err error) { // If the address doesn't exist, this will create it. // If the existingIP exists but is ephemeral, this will promote it to static. // If the address already exists, this will harmlessly return a StatusConflict @@ -974,7 +975,7 @@ func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP var creationErr error switch netTier { - case NetworkTierPremium: + case cloud.NetworkTierPremium: addressObj := &compute.Address{ Name: name, Description: desc, @@ -1012,19 +1013,19 @@ func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP return addr.Address, existed, nil } -func (gce *GCECloud) getServiceNetworkTier(svc *v1.Service) (NetworkTier, error) { +func (gce *GCECloud) getServiceNetworkTier(svc *v1.Service) (cloud.NetworkTier, error) { if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) { - return NetworkTierDefault, nil + return cloud.NetworkTierDefault, nil } tier, err := GetServiceNetworkTier(svc) if err != nil { // Returns an error if the annotation is invalid. - return NetworkTier(""), err + return cloud.NetworkTier(""), err } return tier, nil } -func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, desiredNetTier NetworkTier) error { +func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, desiredNetTier cloud.NetworkTier) error { logPrefix := fmt.Sprintf("deleteWrongNetworkTieredResources:(%s)", lbRef) if err := deleteFWDRuleWithWrongTier(gce, gce.region, lbName, logPrefix, desiredNetTier); err != nil { return err @@ -1037,14 +1038,14 @@ func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, des // deleteFWDRuleWithWrongTier checks the network tier of existing forwarding // rule and delete the rule if the tier does not matched the desired tier. -func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logPrefix string, desiredNetTier NetworkTier) error { +func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logPrefix string, desiredNetTier cloud.NetworkTier) error { tierStr, err := s.getNetworkTierFromForwardingRule(name, region) if isNotFound(err) { return nil } else if err != nil { return err } - existingTier := NetworkTierGCEValueToType(tierStr) + existingTier := cloud.NetworkTierGCEValueToType(tierStr) if existingTier == desiredNetTier { return nil } @@ -1056,7 +1057,7 @@ func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logP // deleteAddressWithWrongTier checks the network tier of existing address // and delete the address if the tier does not matched the desired tier. -func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix string, desiredNetTier NetworkTier) error { +func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix string, desiredNetTier cloud.NetworkTier) error { // We only check the IP address matching the reserved name that the // controller assigned to the LB. We make the assumption that an address of // such name is owned by the controller and is safe to release. Whether an @@ -1072,7 +1073,7 @@ func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix s } else if err != nil { return err } - existingTier := NetworkTierGCEValueToType(tierStr) + existingTier := cloud.NetworkTierGCEValueToType(tierStr) if existingTier == desiredNetTier { return nil } diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go index ed9bd6dcaa8..587de352404 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go @@ -28,230 +28,21 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/mock" kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" ) -func TestEnsureStaticIP(t *testing.T) { - fcas := NewFakeCloudAddressService() - ipName := "some-static-ip" - serviceName := "" - region := "us-central1" - - // First ensure call - ip, existed, err := ensureStaticIP(fcas, ipName, serviceName, region, "", NetworkTierDefault) - if err != nil || existed || ip == "" { - t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, "") = %v, %v, %v; want valid ip, false, nil`, fcas, ipName, serviceName, region, ip, existed, err) - } - - // Second ensure call - var ipPrime string - ipPrime, existed, err = ensureStaticIP(fcas, ipName, serviceName, region, ip, NetworkTierDefault) - if err != nil || !existed || ip != ipPrime { - t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, fcas, ipName, serviceName, region, ip, ipPrime, existed, err, ip) - } -} - -func TestEnsureStaticIPWithTier(t *testing.T) { - s := NewFakeCloudAddressService() - serviceName := "" - region := "us-east1" - - for desc, tc := range map[string]struct { - name string - netTier NetworkTier - expected string - }{ - "Premium (default)": { - name: "foo-1", - netTier: NetworkTierPremium, - expected: "PREMIUM", - }, - "Standard": { - name: "foo-2", - netTier: NetworkTierStandard, - expected: "STANDARD", - }, - } { - t.Run(desc, func(t *testing.T) { - ip, existed, err := ensureStaticIP(s, tc.name, serviceName, region, "", tc.netTier) - assert.NoError(t, err) - assert.False(t, existed) - assert.NotEqual(t, "", ip) - // Get the Address from the fake address service and verify that the tier - // is set correctly. - alphaAddr, err := s.GetAlphaRegionAddress(tc.name, region) - require.NoError(t, err) - assert.Equal(t, tc.expected, alphaAddr.NetworkTier) - }) - } -} - -func TestVerifyRequestedIP(t *testing.T) { - region := "test-region" - lbRef := "test-lb" - s := NewFakeCloudAddressService() - - for desc, tc := range map[string]struct { - requestedIP string - fwdRuleIP string - netTier NetworkTier - addrList []*computealpha.Address - expectErr bool - expectUserOwned bool - }{ - "requested IP exists": { - requestedIP: "1.1.1.1", - netTier: NetworkTierPremium, - addrList: []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}}, - expectErr: false, - expectUserOwned: true, - }, - "requested IP is not static, but is in use by the fwd rule": { - requestedIP: "1.1.1.1", - fwdRuleIP: "1.1.1.1", - netTier: NetworkTierPremium, - expectErr: false, - }, - "requested IP is not static and is not used by the fwd rule": { - requestedIP: "1.1.1.1", - fwdRuleIP: "2.2.2.2", - netTier: NetworkTierPremium, - expectErr: true, - }, - "no requested IP": { - netTier: NetworkTierPremium, - expectErr: false, - }, - "requested IP exists, but network tier does not match": { - requestedIP: "1.1.1.1", - netTier: NetworkTierStandard, - addrList: []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}}, - expectErr: true, - }, - } { - t.Run(desc, func(t *testing.T) { - s.SetRegionalAddresses(region, tc.addrList) - isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef, tc.netTier) - assert.Equal(t, tc.expectErr, err != nil, fmt.Sprintf("err: %v", err)) - assert.Equal(t, tc.expectUserOwned, isUserOwnedIP) - }) - } -} - -func TestCreateForwardingRuleWithTier(t *testing.T) { - s := NewFakeCloudForwardingRuleService() - // Common variables among the tests. - ports := []v1.ServicePort{{Name: "foo", Protocol: v1.ProtocolTCP, Port: int32(123)}} - region := "test-region" - target := "test-target-pool" - svcName := "foo-svc" - - for desc, tc := range map[string]struct { - netTier NetworkTier - expectedRule *computealpha.ForwardingRule - }{ - "Premium tier": { - netTier: NetworkTierPremium, - expectedRule: &computealpha.ForwardingRule{ - Name: "lb-1", - Description: `{"kubernetes.io/service-name":"foo-svc"}`, - IPAddress: "1.1.1.1", - IPProtocol: "TCP", - PortRange: "123-123", - Target: target, - NetworkTier: "PREMIUM", - }, - }, - "Standard tier": { - netTier: NetworkTierStandard, - expectedRule: &computealpha.ForwardingRule{ - Name: "lb-2", - Description: `{"kubernetes.io/service-name":"foo-svc"}`, - IPAddress: "2.2.2.2", - IPProtocol: "TCP", - PortRange: "123-123", - Target: target, - NetworkTier: "STANDARD", - }, - }, - } { - t.Run(desc, func(t *testing.T) { - lbName := tc.expectedRule.Name - ipAddr := tc.expectedRule.IPAddress - - err := createForwardingRule(s, lbName, svcName, region, ipAddr, target, ports, tc.netTier) - assert.NoError(t, err) - - alphaRule, err := s.GetAlphaRegionForwardingRule(lbName, region) - assert.NoError(t, err) - assert.Equal(t, tc.expectedRule, alphaRule) - }) - } -} - -func TestDeleteAddressWithWrongTier(t *testing.T) { - region := "test-region" - lbRef := "test-lb" - s := NewFakeCloudAddressService() - - for desc, tc := range map[string]struct { - addrName string - netTier NetworkTier - addrList []*computealpha.Address - expectDelete bool - }{ - "Network tiers (premium) match; do nothing": { - addrName: "foo1", - netTier: NetworkTierPremium, - addrList: []*computealpha.Address{{Name: "foo1", Address: "1.1.1.1", NetworkTier: "PREMIUM"}}, - }, - "Network tiers (standard) match; do nothing": { - addrName: "foo2", - netTier: NetworkTierStandard, - addrList: []*computealpha.Address{{Name: "foo2", Address: "1.1.1.2", NetworkTier: "STANDARD"}}, - }, - "Wrong network tier (standard); delete address": { - addrName: "foo3", - netTier: NetworkTierPremium, - addrList: []*computealpha.Address{{Name: "foo3", Address: "1.1.1.3", NetworkTier: "STANDARD"}}, - expectDelete: true, - }, - "Wrong network tier (preimium); delete address": { - addrName: "foo4", - netTier: NetworkTierStandard, - addrList: []*computealpha.Address{{Name: "foo4", Address: "1.1.1.4", NetworkTier: "PREMIUM"}}, - expectDelete: true, - }, - } { - t.Run(desc, func(t *testing.T) { - s.SetRegionalAddresses(region, tc.addrList) - // Sanity check to ensure we inject the right address. - _, err := s.GetRegionAddress(tc.addrName, region) - require.NoError(t, err) - - err = deleteAddressWithWrongTier(s, region, tc.addrName, lbRef, tc.netTier) - assert.NoError(t, err) - // Check whether the address still exists. - _, err = s.GetRegionAddress(tc.addrName, region) - if tc.expectDelete { - assert.True(t, isNotFound(err)) - } else { - assert.NoError(t, err) - } - }) - } -} - const ( - gceProjectId = "test-project" - gceRegion = "us-central1" - zoneName = "us-central1-b" - nodeName = "test-node-1" - clusterName = "Test Cluster Name" - clusterID = "test-cluster-id" + projectID = "test-project" + region = "us-central1" + zoneName = "us-central1-b" + nodeName = "test-node-1" + clusterName = "Test Cluster Name" + clusterID = "test-cluster-id" + serviceName = "" ) var apiService = &v1.Service{ @@ -277,7 +68,7 @@ func fakeGCECloud() (*GCECloud, error) { } // Used in disk unit tests - fakeManager := newFakeManager(gceProjectId, gceRegion) + fakeManager := newFakeManager(projectID, region) zonesWithNodes := createNodeZones([]string{zoneName}) alphaFeatureGate, err := NewAlphaFeatureGate([]string{}) @@ -286,12 +77,12 @@ func fakeGCECloud() (*GCECloud, error) { } gce := &GCECloud{ - region: gceRegion, + region: region, service: service, manager: fakeManager, managedZones: []string{zoneName}, - projectID: gceProjectId, - networkProjectID: gceProjectId, + projectID: projectID, + networkProjectID: projectID, AlphaFeatureGate: alphaFeatureGate, nodeZones: zonesWithNodes, nodeInformerSynced: func() bool { return true }, @@ -300,12 +91,239 @@ func fakeGCECloud() (*GCECloud, error) { cloud := cloud.NewMockGCE(&gceProjectRouter{gce}) cloud.MockTargetPools.AddInstanceHook = mock.AddInstanceHook cloud.MockTargetPools.RemoveInstanceHook = mock.RemoveInstanceHook + cloud.MockForwardingRules.InsertHook = mock.InsertFwdRuleHook + cloud.MockAddresses.InsertHook = mock.InsertAddressHook + cloud.MockAlphaAddresses.InsertHook = mock.InsertAlphaAddressHook gce.c = cloud return gce, nil } +func TestEnsureStaticIP(t *testing.T) { + gce, err := fakeGCECloud() + require.NoError(t, err) + + ipName := "some-static-ip" + + // First ensure call + ip, existed, err := ensureStaticIP(gce, ipName, serviceName, region, "", cloud.NetworkTierDefault) + if err != nil || existed { + t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, "") = %v, %v, %v; want valid ip, false, nil`, gce, ipName, serviceName, region, ip, existed, err) + } + + // Second ensure call + var ipPrime string + ipPrime, existed, err = ensureStaticIP(gce, ipName, serviceName, region, ip, cloud.NetworkTierDefault) + if err != nil || !existed || ip != ipPrime { + t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, gce, ipName, serviceName, region, ip, ipPrime, existed, err, ip) + } +} + +func TestEnsureStaticIPWithTier(t *testing.T) { + s, err := fakeGCECloud() + require.NoError(t, err) + + for desc, tc := range map[string]struct { + name string + netTier cloud.NetworkTier + expected string + }{ + "Premium (default)": { + name: "foo-1", + netTier: cloud.NetworkTierPremium, + expected: "PREMIUM", + }, + "Standard": { + name: "foo-2", + netTier: cloud.NetworkTierStandard, + expected: "STANDARD", + }, + } { + t.Run(desc, func(t *testing.T) { + ip, existed, err := ensureStaticIP(s, tc.name, serviceName, region, "", tc.netTier) + assert.NoError(t, err) + assert.False(t, existed) + assert.NotEqual(t, ip, "") + // Get the Address from the fake address service and verify that the tier + // is set correctly. + alphaAddr, err := s.GetAlphaRegionAddress(tc.name, region) + require.NoError(t, err) + assert.Equal(t, tc.expected, alphaAddr.NetworkTier) + }) + } +} + +func TestVerifyRequestedIP(t *testing.T) { + lbRef := "test-lb" + + for desc, tc := range map[string]struct { + requestedIP string + fwdRuleIP string + netTier cloud.NetworkTier + addrList []*computealpha.Address + expectErr bool + expectUserOwned bool + }{ + "requested IP exists": { + requestedIP: "1.1.1.1", + netTier: cloud.NetworkTierPremium, + addrList: []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}}, + expectErr: false, + expectUserOwned: true, + }, + "requested IP is not static, but is in use by the fwd rule": { + requestedIP: "1.1.1.1", + fwdRuleIP: "1.1.1.1", + netTier: cloud.NetworkTierPremium, + expectErr: false, + }, + "requested IP is not static and is not used by the fwd rule": { + requestedIP: "1.1.1.1", + fwdRuleIP: "2.2.2.2", + netTier: cloud.NetworkTierPremium, + expectErr: true, + }, + "no requested IP": { + netTier: cloud.NetworkTierPremium, + expectErr: false, + }, + "requested IP exists, but network tier does not match": { + requestedIP: "1.1.1.1", + netTier: cloud.NetworkTierStandard, + addrList: []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}}, + expectErr: true, + }, + } { + t.Run(desc, func(t *testing.T) { + s, err := fakeGCECloud() + require.NoError(t, err) + + for _, addr := range tc.addrList { + s.ReserveAlphaRegionAddress(addr, region) + } + isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef, tc.netTier) + assert.Equal(t, tc.expectErr, err != nil, fmt.Sprintf("err: %v", err)) + assert.Equal(t, tc.expectUserOwned, isUserOwnedIP) + }) + } +} + +func TestCreateForwardingRuleWithTier(t *testing.T) { + // Common variables among the tests. + ports := []v1.ServicePort{{Name: "foo", Protocol: v1.ProtocolTCP, Port: int32(123)}} + target := "test-target-pool" + svcName := "foo-svc" + baseLinkUrl := "https://www.googleapis.com/compute/%v/projects/%v/regions/%v/forwardingRules/%v" + + for desc, tc := range map[string]struct { + netTier cloud.NetworkTier + expectedRule *computealpha.ForwardingRule + }{ + "Premium tier": { + netTier: cloud.NetworkTierPremium, + expectedRule: &computealpha.ForwardingRule{ + Name: "lb-1", + Description: `{"kubernetes.io/service-name":"foo-svc"}`, + IPAddress: "1.1.1.1", + IPProtocol: "TCP", + PortRange: "123-123", + Target: target, + NetworkTier: "PREMIUM", + SelfLink: fmt.Sprintf(baseLinkUrl, "v1", projectID, region, "lb-1"), + }, + }, + "Standard tier": { + netTier: cloud.NetworkTierStandard, + expectedRule: &computealpha.ForwardingRule{ + Name: "lb-2", + Description: `{"kubernetes.io/service-name":"foo-svc"}`, + IPAddress: "2.2.2.2", + IPProtocol: "TCP", + PortRange: "123-123", + Target: target, + NetworkTier: "STANDARD", + SelfLink: fmt.Sprintf(baseLinkUrl, "alpha", projectID, region, "lb-2"), + }, + }, + } { + t.Run(desc, func(t *testing.T) { + s, err := fakeGCECloud() + require.NoError(t, err) + + lbName := tc.expectedRule.Name + ipAddr := tc.expectedRule.IPAddress + + err = createForwardingRule(s, lbName, svcName, region, ipAddr, target, ports, tc.netTier) + assert.NoError(t, err) + + alphaRule, err := s.GetAlphaRegionForwardingRule(lbName, region) + assert.NoError(t, err) + assert.Equal(t, tc.expectedRule, alphaRule) + }) + } +} + +func TestDeleteAddressWithWrongTier(t *testing.T) { + lbRef := "test-lb" + + s, err := fakeGCECloud() + require.NoError(t, err) + + // Enable the cloud.NetworkTiers feature + s.AlphaFeatureGate.features[AlphaFeatureNetworkTiers] = true + + for desc, tc := range map[string]struct { + addrName string + netTier cloud.NetworkTier + addrList []*computealpha.Address + expectDelete bool + }{ + "Network tiers (premium) match; do nothing": { + addrName: "foo1", + netTier: cloud.NetworkTierPremium, + addrList: []*computealpha.Address{{Name: "foo1", Address: "1.1.1.1", NetworkTier: "PREMIUM"}}, + }, + "Network tiers (standard) match; do nothing": { + addrName: "foo2", + netTier: cloud.NetworkTierStandard, + addrList: []*computealpha.Address{{Name: "foo2", Address: "1.1.1.2", NetworkTier: "STANDARD"}}, + }, + "Wrong network tier (standard); delete address": { + addrName: "foo3", + netTier: cloud.NetworkTierPremium, + addrList: []*computealpha.Address{{Name: "foo3", Address: "1.1.1.3", NetworkTier: "STANDARD"}}, + expectDelete: true, + }, + "Wrong network tier (premium); delete address": { + addrName: "foo4", + netTier: cloud.NetworkTierStandard, + addrList: []*computealpha.Address{{Name: "foo4", Address: "1.1.1.4", NetworkTier: "PREMIUM"}}, + expectDelete: true, + }, + } { + t.Run(desc, func(t *testing.T) { + for _, addr := range tc.addrList { + s.ReserveAlphaRegionAddress(addr, region) + } + + // Sanity check to ensure we inject the right address. + _, err = s.GetRegionAddress(tc.addrName, region) + require.NoError(t, err) + + err = deleteAddressWithWrongTier(s, region, tc.addrName, lbRef, tc.netTier) + assert.NoError(t, err) + // Check whether the address still exists. + _, err = s.GetRegionAddress(tc.addrName, region) + if tc.expectDelete { + assert.True(t, isNotFound(err)) + } else { + assert.NoError(t, err) + } + }) + } +} + func createAndInsertNodes(gce *GCECloud, nodeNames []string) ([]*v1.Node, error) { nodes := []*v1.Node{} @@ -318,7 +336,7 @@ func createAndInsertNodes(gce *GCECloud, nodeNames []string) ([]*v1.Node, error) if instance == nil { err := gce.InsertInstance( - gceProjectId, + projectID, zoneName, &compute.Instance{ Name: name, @@ -394,7 +412,7 @@ func TestEnsureExternalLoadBalancer(t *testing.T) { } // Check that TargetPool is Created - pool, err := gce.GetTargetPool(lbName, gceRegion) + pool, err := gce.GetTargetPool(lbName, region) require.NoError(t, err) assert.Equal(t, lbName, pool.Name) assert.NotEmpty(t, pool.HealthChecks) @@ -406,7 +424,7 @@ func TestEnsureExternalLoadBalancer(t *testing.T) { assert.Equal(t, hcName, healthcheck.Name) // Check that ForwardingRule is created - fwdRule, err := gce.GetRegionForwardingRule(lbName, gceRegion) + fwdRule, err := gce.GetRegionForwardingRule(lbName, region) require.NoError(t, err) assert.Equal(t, lbName, fwdRule.Name) assert.Equal(t, "TCP", fwdRule.IPProtocol) @@ -430,7 +448,7 @@ func TestUpdateExternalLoadBalancer(t *testing.T) { lbName := cloudprovider.GetLoadBalancerName(apiService) - pool, err := gce.GetTargetPool(lbName, gceRegion) + pool, err := gce.GetTargetPool(lbName, region) require.NoError(t, err) // TODO: when testify is updated to v1.2.0+, use ElementsMatch instead @@ -455,7 +473,7 @@ func TestUpdateExternalLoadBalancer(t *testing.T) { err = gce.updateExternalLoadBalancer(clusterName, apiService, newNodes) assert.NoError(t, err) - pool, err = gce.GetTargetPool(lbName, gceRegion) + pool, err = gce.GetTargetPool(lbName, region) require.NoError(t, err) assert.Equal( @@ -491,7 +509,7 @@ func TestEnsureExternalLoadBalancerDeleted(t *testing.T) { } // Check that TargetPool is deleted - pool, err := gce.GetTargetPool(lbName, gceRegion) + pool, err := gce.GetTargetPool(lbName, region) require.Error(t, err) assert.Nil(t, pool) @@ -501,7 +519,58 @@ func TestEnsureExternalLoadBalancerDeleted(t *testing.T) { assert.Nil(t, healthcheck) // Check forwarding rule is deleted - fwdRule, err := gce.GetRegionForwardingRule(lbName, gceRegion) + fwdRule, err := gce.GetRegionForwardingRule(lbName, region) require.Error(t, err) assert.Nil(t, fwdRule) } + +func TestLoadBalancerWrongTierResourceDeletion(t *testing.T) { + gce, err := fakeGCECloud() + require.NoError(t, err) + + // Enable the cloud.NetworkTiers feature + gce.AlphaFeatureGate.features[AlphaFeatureNetworkTiers] = true + apiService.Annotations = map[string]string{NetworkTierAnnotationKey: "Premium"} + + // cloud.NetworkTier defaults to Premium + desiredTier, err := gce.getServiceNetworkTier(apiService) + require.NoError(t, err) + assert.Equal(t, cloud.NetworkTierPremium, desiredTier) + + lbName := cloudprovider.GetLoadBalancerName(apiService) + serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name} + + // create ForwardingRule and Address with the wrong tier + err = createForwardingRule( + gce, + lbName, + serviceName.String(), + region, + "", + gce.targetPoolURL(lbName), + apiService.Spec.Ports, + cloud.NetworkTierStandard, + ) + require.NoError(t, err) + + addressObj := &computealpha.Address{ + Name: lbName, + Description: serviceName.String(), + NetworkTier: cloud.NetworkTierStandard.ToGCEValue(), + } + + err = gce.ReserveAlphaRegionAddress(addressObj, region) + require.NoError(t, err) + + _, err = createExternalLoadBalancer(gce) + require.NoError(t, err) + + // Expect forwarding rule tier to not be Standard + tier, err := gce.getNetworkTierFromForwardingRule(lbName, region) + assert.NoError(t, err) + assert.Equal(t, cloud.NetworkTierDefault.ToGCEValue(), tier) + + // Expect address to be deleted + _, err = gce.GetRegionAddress(lbName, region) + assert.True(t, isNotFound(err)) +} diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go index 64f9e2e9ae4..51251fe26d2 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" v1_service "k8s.io/kubernetes/pkg/api/v1/service" "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" ) const ( @@ -37,7 +38,7 @@ const ( func (gce *GCECloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v1.Service, existingFwdRule *compute.ForwardingRule, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace} ports, protocol := getPortsAndProtocol(svc.Spec.Ports) - scheme := schemeInternal + scheme := cloud.SchemeInternal loadBalancerName := cloudprovider.GetLoadBalancerName(svc) sharedBackend := shareBackendService(svc) backendServiceName := makeBackendServiceName(loadBalancerName, clusterID, sharedBackend, scheme, protocol, svc.Spec.SessionAffinity) @@ -93,7 +94,7 @@ func (gce *GCECloud) ensureInternalLoadBalancer(clusterName, clusterID string, s var addrMgr *addressManager // If the network is not a legacy network, use the address manager if !gce.IsLegacyNetwork() { - addrMgr = newAddressManager(gce, nm.String(), gce.Region(), subnetworkURL, loadBalancerName, requestedIP, schemeInternal) + addrMgr = newAddressManager(gce, nm.String(), gce.Region(), subnetworkURL, loadBalancerName, requestedIP, cloud.SchemeInternal) ipToUse, err = addrMgr.HoldAddress() if err != nil { return nil, err @@ -208,7 +209,7 @@ func (gce *GCECloud) updateInternalLoadBalancer(clusterName, clusterID string, s // Generate the backend service name _, protocol := getPortsAndProtocol(svc.Spec.Ports) - scheme := schemeInternal + scheme := cloud.SchemeInternal loadBalancerName := cloudprovider.GetLoadBalancerName(svc) backendServiceName := makeBackendServiceName(loadBalancerName, clusterID, shareBackendService(svc), scheme, protocol, svc.Spec.SessionAffinity) // Ensure the backend service has the proper backend/instance-group links @@ -218,7 +219,7 @@ func (gce *GCECloud) updateInternalLoadBalancer(clusterName, clusterID string, s func (gce *GCECloud) ensureInternalLoadBalancerDeleted(clusterName, clusterID string, svc *v1.Service) error { loadBalancerName := cloudprovider.GetLoadBalancerName(svc) _, protocol := getPortsAndProtocol(svc.Spec.Ports) - scheme := schemeInternal + scheme := cloud.SchemeInternal sharedBackend := shareBackendService(svc) sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc) @@ -506,7 +507,7 @@ func (gce *GCECloud) ensureInternalInstanceGroupsDeleted(name string) error { return nil } -func (gce *GCECloud) ensureInternalBackendService(name, description string, affinityType v1.ServiceAffinity, scheme lbScheme, protocol v1.Protocol, igLinks []string, hcLink string) error { +func (gce *GCECloud) ensureInternalBackendService(name, description string, affinityType v1.ServiceAffinity, scheme cloud.LbScheme, protocol v1.Protocol, igLinks []string, hcLink string) error { glog.V(2).Infof("ensureInternalBackendService(%v, %v, %v): checking existing backend service with %d groups", name, scheme, protocol, len(igLinks)) bs, err := gce.GetRegionBackendService(name, gce.region) if err != nil && !isNotFound(err) { diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_naming.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_naming.go index 7fee340a629..1e1feeff53b 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_naming.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_naming.go @@ -24,6 +24,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" ) // Internal Load Balancer @@ -33,7 +34,7 @@ func makeInstanceGroupName(clusterID string) string { return fmt.Sprintf("k8s-ig--%s", clusterID) } -func makeBackendServiceName(loadBalancerName, clusterID string, shared bool, scheme lbScheme, protocol v1.Protocol, svcAffinity v1.ServiceAffinity) string { +func makeBackendServiceName(loadBalancerName, clusterID string, shared bool, scheme cloud.LbScheme, protocol v1.Protocol, svcAffinity v1.ServiceAffinity) string { if shared { hash := sha1.New() diff --git a/pkg/cloudprovider/providers/gce/gce_util.go b/pkg/cloudprovider/providers/gce/gce_util.go index 6b31c4fa1c2..363b5173abe 100644 --- a/pkg/cloudprovider/providers/gce/gce_util.go +++ b/pkg/cloudprovider/providers/gce/gce_util.go @@ -28,6 +28,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "cloud.google.com/go/compute/metadata" compute "google.golang.org/api/compute/v1" @@ -220,7 +221,7 @@ func handleAlphaNetworkTierGetError(err error) (string, error) { // Network tier is still an Alpha feature in GCP, and not every project // is whitelisted to access the API. If we cannot access the API, just // assume the tier is premium. - return NetworkTierDefault.ToGCEValue(), nil + return cloud.NetworkTierDefault.ToGCEValue(), nil } // Can't get the network tier, just return an error. return "", err diff --git a/test/e2e/network/BUILD b/test/e2e/network/BUILD index 3695b6691dc..4000cee776e 100644 --- a/test/e2e/network/BUILD +++ b/test/e2e/network/BUILD @@ -36,6 +36,7 @@ go_library( "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library", + "//pkg/cloudprovider/providers/gce/cloud:go_default_library", "//pkg/controller/endpoint:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/master/ports:go_default_library", diff --git a/test/e2e/network/network_tiers.go b/test/e2e/network/network_tiers.go index ec284e8c252..9e66a3a1e8a 100644 --- a/test/e2e/network/network_tiers.go +++ b/test/e2e/network/network_tiers.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/cloudprovider" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" @@ -75,12 +76,12 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() { By("creating a Service of type LoadBalancer using the standard network tier") svc := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer - setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard) + setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard.ToGCEValue()) }) // Verify that service has been updated properly. svcTier, err := gcecloud.GetServiceNetworkTier(svc) Expect(err).NotTo(HaveOccurred()) - Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard)) + Expect(svcTier).To(Equal(cloud.NetworkTierStandard)) // Record the LB name for test cleanup. serviceLBNames = append(serviceLBNames, cloudprovider.GetLoadBalancerName(svc)) @@ -95,7 +96,7 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() { // Verify that service has been updated properly. svcTier, err = gcecloud.GetServiceNetworkTier(svc) Expect(err).NotTo(HaveOccurred()) - Expect(svcTier).To(Equal(gcecloud.NetworkTierDefault)) + Expect(svcTier).To(Equal(cloud.NetworkTierDefault)) // Wait until the ingress IP changes. Each tier has its own pool of // IPs, so changing tiers implies changing IPs. @@ -106,7 +107,7 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() { requestedAddrName := fmt.Sprintf("e2e-ext-lb-net-tier-%s", framework.RunId) gceCloud, err := framework.GetGCECloud() Expect(err).NotTo(HaveOccurred()) - requestedIP, err := reserveAlphaRegionalAddress(gceCloud, requestedAddrName, gcecloud.NetworkTierStandard) + requestedIP, err := reserveAlphaRegionalAddress(gceCloud, requestedAddrName, cloud.NetworkTierStandard) Expect(err).NotTo(HaveOccurred(), "failed to reserve a STANDARD tiered address") defer func() { if requestedAddrName != "" { @@ -122,13 +123,13 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() { By("updating the Service to use the standard tier with a requested IP") svc = jig.UpdateServiceOrFail(ns, svc.Name, func(svc *v1.Service) { svc.Spec.LoadBalancerIP = requestedIP - setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard) + setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard.ToGCEValue()) }) // Verify that service has been updated properly. Expect(svc.Spec.LoadBalancerIP).To(Equal(requestedIP)) svcTier, err = gcecloud.GetServiceNetworkTier(svc) Expect(err).NotTo(HaveOccurred()) - Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard)) + Expect(svcTier).To(Equal(cloud.NetworkTierStandard)) // Wait until the ingress IP changes and verifies the LB. ingressIP = waitAndVerifyLBWithTier(jig, ns, svcName, ingressIP, createTimeout, lagTimeout) @@ -171,7 +172,7 @@ func waitAndVerifyLBWithTier(jig *framework.ServiceTestJig, ns, svcName, existin return ingressIP } -func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) { +func getLBNetworkTierByIP(ip string) (cloud.NetworkTier, error) { var rule *computealpha.ForwardingRule // Retry a few times to tolerate flakes. err := wait.PollImmediate(5*time.Second, 15*time.Second, func() (bool, error) { @@ -185,7 +186,7 @@ func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) { if err != nil { return "", err } - return gcecloud.NetworkTierGCEValueToType(rule.NetworkTier), nil + return cloud.NetworkTierGCEValueToType(rule.NetworkTier), nil } func getGCEForwardingRuleByIP(ip string) (*computealpha.ForwardingRule, error) { @@ -223,7 +224,7 @@ func clearNetworkTier(svc *v1.Service) { // TODO: add retries if this turns out to be flaky. // TODO(#51665): remove this helper function once Network Tiers becomes beta. -func reserveAlphaRegionalAddress(cloud *gcecloud.GCECloud, name string, netTier gcecloud.NetworkTier) (string, error) { +func reserveAlphaRegionalAddress(cloud *gcecloud.GCECloud, name string, netTier cloud.NetworkTier) (string, error) { alphaAddr := &computealpha.Address{ Name: name, NetworkTier: netTier.ToGCEValue(),