Merge pull request #59941 from agau4779/gce-external-lb-tests

Automatic merge from submit-queue (batch tested with PRs 60324, 60269, 59771, 60314, 59941). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

[GCE] Refactor ExternalLoadBalancer Tests

**What this PR does / why we need it**:

- Refactors the ExternalLoadBalancer tests to use the generated GCE mock instead of FakeCloudAddressService, FakeCloudForwardingRuleService.
- Adds hooks to populate NetworkTier on Alpha resources
- Moves shared code to top of the external loadbalancer test file
- Moves NetworkTier into a constants file at the cloud level, so it is more easily called in subpackages

**Special notes for your reviewer**:

```release-note
NONE
```

pkg/cloudprovider/providers/gce/BUILD

@@ -23,7 +23,6 @@ go_library(
         "gce_disks.go",
         "gce_firewall.go",
         "gce_forwardingrule.go",
-        "gce_forwardingrule_fakes.go",
         "gce_healthchecks.go",
         "gce_instancegroup.go",
         "gce_instances.go",
@@ -117,6 +116,7 @@ go_test(
         "//vendor/google.golang.org/api/googleapi:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
     ],
 )

pkg/cloudprovider/providers/gce/cloud/BUILD

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "constants.go",
         "doc.go",
         "gce_projects.go",
         "gen.go",

pkg/cloudprovider/providers/gce/cloud/constants.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloud
+
+import (
+	"strings"
+)
+
+// NetworkTier represents the Network Service Tier used by a resource
+type NetworkTier string
+
+// LbScheme represents the possible types of load balancers
+type LbScheme string
+
+const (
+	NetworkTierStandard NetworkTier = "Standard"
+	NetworkTierPremium  NetworkTier = "Premium"
+	NetworkTierDefault  NetworkTier = NetworkTierPremium
+
+	SchemeExternal LbScheme = "EXTERNAL"
+	SchemeInternal LbScheme = "INTERNAL"
+)
+
+// ToGCEValue converts NetworkTier to a string that we can populate the
+// NetworkTier field of GCE objects, including ForwardingRules and Addresses.
+func (n NetworkTier) ToGCEValue() string {
+	return strings.ToUpper(string(n))
+}
+
+// NetworkTierGCEValueToType converts the value of the NetworkTier field of a
+// GCE object to the NetworkTier type.
+func NetworkTierGCEValueToType(s string) NetworkTier {
+	switch s {
+	case NetworkTierStandard.ToGCEValue():
+		return NetworkTierStandard
+	case NetworkTierPremium.ToGCEValue():
+		return NetworkTierPremium
+	default:
+		return NetworkTier(s)
+	}
+}

pkg/cloudprovider/providers/gce/cloud/mock/BUILD

@@ -8,6 +8,8 @@ go_library(
     deps = [
         "//pkg/cloudprovider/providers/gce/cloud:go_default_library",
         "//pkg/cloudprovider/providers/gce/cloud/meta:go_default_library",
+        "//vendor/google.golang.org/api/compute/v0.alpha:go_default_library",
+        "//vendor/google.golang.org/api/compute/v0.beta:go_default_library",
         "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/google.golang.org/api/googleapi:go_default_library",
     ],

pkg/cloudprovider/providers/gce/cloud/mock/mock.go

@@ -25,15 +25,23 @@ package mock
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 
+	alpha "google.golang.org/api/compute/v0.alpha"
+	beta "google.golang.org/api/compute/v0.beta"
 	ga "google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
-	cloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta"
 )
 
+// gceObject is an abstraction of all GCE API object in go client
+type gceObject interface {
+	MarshalJSON() ([]byte, error)
+}
+
 // AddInstanceHook mocks adding a Instance to MockTargetPools
 func AddInstanceHook(ctx context.Context, key *meta.Key, req *ga.TargetPoolsAddInstanceRequest, m *cloud.MockTargetPools) error {
 	pool, err := m.Get(ctx, key)
@@ -74,3 +82,147 @@ func RemoveInstanceHook(ctx context.Context, key *meta.Key, req *ga.TargetPoolsR
 
 	return nil
 }
+
+func convertAndInsertAlphaForwardingRule(key *meta.Key, obj gceObject, mRules map[meta.Key]*cloud.MockForwardingRulesObj, version meta.Version, projectID string) (bool, error) {
+	if !key.Valid() {
+		return false, fmt.Errorf("invalid GCE key (%+v)", key)
+	}
+
+	if _, ok := mRules[*key]; ok {
+		err := &googleapi.Error{
+			Code:    http.StatusConflict,
+			Message: fmt.Sprintf("MockForwardingRule %v exists", key),
+		}
+		return false, err
+	}
+
+	enc, err := obj.MarshalJSON()
+	if err != nil {
+		return false, err
+	}
+	var fwdRule alpha.ForwardingRule
+	if err := json.Unmarshal(enc, &fwdRule); err != nil {
+		return false, err
+	}
+	// Set the default values for the Alpha fields.
+	if fwdRule.NetworkTier == "" {
+		fwdRule.NetworkTier = cloud.NetworkTierDefault.ToGCEValue()
+	}
+
+	fwdRule.Name = key.Name
+	if fwdRule.SelfLink == "" {
+		fwdRule.SelfLink = cloud.SelfLink(version, projectID, "forwardingRules", key)
+	}
+
+	mRules[*key] = &cloud.MockForwardingRulesObj{Obj: fwdRule}
+	return true, nil
+}
+
+// InsertFwdRuleHook mocks inserting a ForwardingRule. ForwardingRules are
+// expected to default to Premium tier if no NetworkTier is specified.
+func InsertFwdRuleHook(ctx context.Context, key *meta.Key, obj *ga.ForwardingRule, m *cloud.MockForwardingRules) (bool, error) {
+	projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionGA, "forwardingRules")
+	return convertAndInsertAlphaForwardingRule(key, obj, m.Objects, meta.VersionGA, projectID)
+}
+
+// InsertBetaFwdRuleHook mocks inserting a BetaForwardingRule.
+func InsertBetaFwdRuleHook(ctx context.Context, key *meta.Key, obj *beta.ForwardingRule, m *cloud.MockForwardingRules) (bool, error) {
+	projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionBeta, "forwardingRules")
+	return convertAndInsertAlphaForwardingRule(key, obj, m.Objects, meta.VersionBeta, projectID)
+}
+
+// InsertAlphaFwdRuleHook mocks inserting an AlphaForwardingRule.
+func InsertAlphaFwdRuleHook(ctx context.Context, key *meta.Key, obj *alpha.ForwardingRule, m *cloud.MockForwardingRules) (bool, error) {
+	projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionAlpha, "forwardingRules")
+	return convertAndInsertAlphaForwardingRule(key, obj, m.Objects, meta.VersionAlpha, projectID)
+}
+
+// Used to assign Addresses with no IP a unique IP address
+var ipCounter = 1
+
+func convertAndInsertAlphaAddress(key *meta.Key, obj gceObject, mAddrs map[meta.Key]*cloud.MockAddressesObj, version meta.Version, projectID string) (bool, error) {
+	if !key.Valid() {
+		return false, fmt.Errorf("invalid GCE key (%+v)", key)
+	}
+
+	if _, ok := mAddrs[*key]; ok {
+		err := &googleapi.Error{
+			Code:    http.StatusConflict,
+			Message: fmt.Sprintf("MockAddresses %v exists", key),
+		}
+		return false, err
+	}
+
+	enc, err := obj.MarshalJSON()
+	if err != nil {
+		return false, err
+	}
+	var addr alpha.Address
+	if err := json.Unmarshal(enc, &addr); err != nil {
+		return false, err
+	}
+
+	// Set default address type if not present.
+	if addr.AddressType == "" {
+		addr.AddressType = string(cloud.SchemeExternal)
+	}
+
+	var existingAddresses []*ga.Address
+	for _, obj := range mAddrs {
+		existingAddresses = append(existingAddresses, obj.ToGA())
+	}
+
+	for _, existingAddr := range existingAddresses {
+		if addr.Address == existingAddr.Address {
+			msg := fmt.Sprintf("MockAddresses IP %v in use", addr.Address)
+
+			// When the IP is already in use, this call returns a StatusBadRequest
+			// if the address is an external address, and StatusConflict if an
+			// internal address. This is to be consistent with actual GCE API.
+			errorCode := http.StatusConflict
+			if addr.AddressType == string(cloud.SchemeExternal) {
+				errorCode = http.StatusBadRequest
+			}
+
+			return false, &googleapi.Error{Code: errorCode, Message: msg}
+		}
+	}
+
+	// Set default values used in tests
+	addr.Name = key.Name
+	if addr.SelfLink == "" {
+		addr.SelfLink = cloud.SelfLink(version, projectID, "addresses", key)
+	}
+
+	if addr.Address == "" {
+		addr.Address = fmt.Sprintf("1.2.3.%d", ipCounter)
+		ipCounter++
+	}
+
+	// Set the default values for the Alpha fields.
+	if addr.NetworkTier == "" {
+		addr.NetworkTier = cloud.NetworkTierDefault.ToGCEValue()
+	}
+
+	mAddrs[*key] = &cloud.MockAddressesObj{Obj: addr}
+	return true, nil
+}
+
+// InsertAddressHook mocks inserting an Address.
+func InsertAddressHook(ctx context.Context, key *meta.Key, obj *ga.Address, m *cloud.MockAddresses) (bool, error) {
+	projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionGA, "addresses")
+	return convertAndInsertAlphaAddress(key, obj, m.Objects, meta.VersionGA, projectID)
+}
+
+// InsertBetaAddressHook mocks inserting a BetaAddress.
+func InsertBetaAddressHook(ctx context.Context, key *meta.Key, obj *beta.Address, m *cloud.MockAddresses) (bool, error) {
+	projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionBeta, "addresses")
+	return convertAndInsertAlphaAddress(key, obj, m.Objects, meta.VersionBeta, projectID)
+}
+
+// InsertAlphaAddressHook mocks inserting an Address. Addresses are expected to
+// default to Premium tier if no NetworkTier is specified.
+func InsertAlphaAddressHook(ctx context.Context, key *meta.Key, obj *alpha.Address, m *cloud.MockAlphaAddresses) (bool, error) {
+	projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionBeta, "addresses")
+	return convertAndInsertAlphaAddress(key, obj, m.Objects, meta.VersionAlpha, projectID)
+}

pkg/cloudprovider/providers/gce/gce_address_manager.go

@@ -23,6 +23,7 @@ import (
 	compute "google.golang.org/api/compute/v1"
 
 	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 )
 
 type addressManager struct {
@@ -31,13 +32,13 @@ type addressManager struct {
 	name        string
 	serviceName string
 	targetIP    string
-	addressType lbScheme
+	addressType cloud.LbScheme
 	region      string
 	subnetURL   string
 	tryRelease  bool
 }
 
-func newAddressManager(svc CloudAddressService, serviceName, region, subnetURL, name, targetIP string, addressType lbScheme) *addressManager {
+func newAddressManager(svc CloudAddressService, serviceName, region, subnetURL, name, targetIP string, addressType cloud.LbScheme) *addressManager {
 	return &addressManager{
 		svc:         svc,
 		logPrefix:   fmt.Sprintf("AddressManager(%q)", name),

pkg/cloudprovider/providers/gce/gce_address_manager_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	compute "google.golang.org/api/compute/v1"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 )
 
 const testSvcName = "my-service"
@@ -34,8 +35,8 @@ func TestAddressManagerNoRequestedIP(t *testing.T) {
 	svc := NewFakeCloudAddressService()
 	targetIP := ""
 
-	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
-	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
+	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal))
 	testReleaseAddress(t, mgr, svc, testLBName, testRegion)
 }
 
@@ -44,8 +45,8 @@ func TestAddressManagerBasic(t *testing.T) {
 	svc := NewFakeCloudAddressService()
 	targetIP := "1.1.1.1"
 
-	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
-	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
+	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal))
 	testReleaseAddress(t, mgr, svc, testLBName, testRegion)
 }
 
@@ -55,12 +56,12 @@ func TestAddressManagerOrphaned(t *testing.T) {
 	svc := NewFakeCloudAddressService()
 	targetIP := "1.1.1.1"
 
-	addr := &compute.Address{Name: testLBName, Address: targetIP, AddressType: string(schemeInternal)}
+	addr := &compute.Address{Name: testLBName, Address: targetIP, AddressType: string(cloud.SchemeInternal)}
 	err := svc.ReserveRegionAddress(addr, testRegion)
 	require.NoError(t, err)
 
-	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
-	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
+	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal))
 	testReleaseAddress(t, mgr, svc, testLBName, testRegion)
 }
 
@@ -71,12 +72,12 @@ func TestAddressManagerOutdatedOrphan(t *testing.T) {
 	previousAddress := "1.1.0.0"
 	targetIP := "1.1.1.1"
 
-	addr := &compute.Address{Name: testLBName, Address: previousAddress, AddressType: string(schemeExternal)}
+	addr := &compute.Address{Name: testLBName, Address: previousAddress, AddressType: string(cloud.SchemeExternal)}
 	err := svc.ReserveRegionAddress(addr, testRegion)
 	require.NoError(t, err)
 
-	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
-	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
+	testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal))
 	testReleaseAddress(t, mgr, svc, testLBName, testRegion)
 }
 
@@ -86,11 +87,11 @@ func TestAddressManagerExternallyOwned(t *testing.T) {
 	svc := NewFakeCloudAddressService()
 	targetIP := "1.1.1.1"
 
-	addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeInternal)}
+	addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(cloud.SchemeInternal)}
 	err := svc.ReserveRegionAddress(addr, testRegion)
 	require.NoError(t, err)
 
-	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
 	ipToUse, err := mgr.HoldAddress()
 	require.NoError(t, err)
 	assert.NotEmpty(t, ipToUse)
@@ -107,11 +108,11 @@ func TestAddressManagerBadExternallyOwned(t *testing.T) {
 	svc := NewFakeCloudAddressService()
 	targetIP := "1.1.1.1"
 
-	addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeExternal)}
+	addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(cloud.SchemeExternal)}
 	err := svc.ReserveRegionAddress(addr, testRegion)
 	require.NoError(t, err)
 
-	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
+	mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
 	_, err = mgr.HoldAddress()
 	assert.NotNil(t, err)
 }

pkg/cloudprovider/providers/gce/gce_addresses.go

@@ -26,6 +26,7 @@ import (
 	computebeta "google.golang.org/api/compute/v0.beta"
 	compute "google.golang.org/api/compute/v1"
 
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta"
 )
@@ -150,7 +151,7 @@ func (gce *GCECloud) GetBetaRegionAddressByIP(region, ipAddress string) (*comput
 // TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
 func (gce *GCECloud) getNetworkTierFromAddress(name, region string) (string, error) {
 	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
-		return NetworkTierDefault.ToGCEValue(), nil
+		return cloud.NetworkTierDefault.ToGCEValue(), nil
 	}
 	addr, err := gce.GetAlphaRegionAddress(name, region)
 	if err != nil {

pkg/cloudprovider/providers/gce/gce_addresses_fakes.go

@@ -25,6 +25,8 @@ import (
 	computealpha "google.golang.org/api/compute/v0.alpha"
 	computebeta "google.golang.org/api/compute/v0.beta"
 	compute "google.golang.org/api/compute/v1"
+
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 )
 
 // test
@@ -68,7 +70,7 @@ func (cas *FakeCloudAddressService) ReserveAlphaRegionAddress(addr *computealpha
 	}
 
 	if addr.AddressType == "" {
-		addr.AddressType = string(schemeExternal)
+		addr.AddressType = string(cloud.SchemeExternal)
 	}
 
 	if cas.reservedAddrs[addr.Address] {
@@ -76,8 +78,8 @@ func (cas *FakeCloudAddressService) ReserveAlphaRegionAddress(addr *computealpha
 		// When the IP is already in use, this call returns an error code based
 		// on the type (internal vs external) of the address. This is to be
 		// consistent with actual GCE API.
-		switch lbScheme(addr.AddressType) {
+		switch cloud.LbScheme(addr.AddressType) {
-		case schemeExternal:
+		case cloud.SchemeExternal:
 			return makeGoogleAPIError(http.StatusBadRequest, msg)
 		default:
 			return makeGoogleAPIError(http.StatusConflict, msg)
@@ -209,7 +211,7 @@ func convertToAlphaAddress(object gceObject) *computealpha.Address {
 		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha address: %v", object, err))
 	}
 	// Set the default values for the Alpha fields.
-	addr.NetworkTier = NetworkTierDefault.ToGCEValue()
+	addr.NetworkTier = cloud.NetworkTierDefault.ToGCEValue()
 	return &addr
 }
 

pkg/cloudprovider/providers/gce/gce_annotations.go

@@ -18,15 +18,14 @@ package gce
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/golang/glog"
 
 	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 )
 
 type LoadBalancerType string
-type NetworkTier string
 
 const (
 	// ServiceAnnotationLoadBalancerType is annotated on a service with type LoadBalancer
@@ -49,12 +48,8 @@ const (
 	// network tier a GCP LB should use. The valid values are "Standard" and
 	// "Premium" (default).
 	NetworkTierAnnotationKey      = "cloud.google.com/network-tier"
-	NetworkTierAnnotationStandard = "Standard"
+	NetworkTierAnnotationStandard = cloud.NetworkTierStandard
-	NetworkTierAnnotationPremium  = "Premium"
+	NetworkTierAnnotationPremium  = cloud.NetworkTierPremium
-
-	NetworkTierStandard NetworkTier = NetworkTierAnnotationStandard
-	NetworkTierPremium  NetworkTier = NetworkTierAnnotationPremium
-	NetworkTierDefault  NetworkTier = NetworkTierPremium
 )
 
 // GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled.
@@ -97,38 +92,19 @@ func GetLoadBalancerAnnotationBackendShare(service *v1.Service) bool {
 // GetServiceNetworkTier returns the network tier of GCP load balancer
 // which should be assembled, and an error if the specified tier is not
 // supported.
-func GetServiceNetworkTier(service *v1.Service) (NetworkTier, error) {
+func GetServiceNetworkTier(service *v1.Service) (cloud.NetworkTier, error) {
 	l, ok := service.Annotations[NetworkTierAnnotationKey]
 	if !ok {
-		return NetworkTierDefault, nil
+		return cloud.NetworkTierDefault, nil
 	}
 
-	v := NetworkTier(l)
+	v := cloud.NetworkTier(l)
 	switch v {
-	case NetworkTierStandard:
+	case cloud.NetworkTierStandard:
 		fallthrough
-	case NetworkTierPremium:
+	case cloud.NetworkTierPremium:
 		return v, nil
 	default:
-		return NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v)
+		return cloud.NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v)
-	}
-}
-
-// ToGCEValue converts NetworkTier to a string that we can populate the
-// NetworkTier field of GCE objects.
-func (n NetworkTier) ToGCEValue() string {
-	return strings.ToUpper(string(n))
-}
-
-// NetworkTierGCEValueToType converts the value of the NetworkTier field of a
-// GCE object to the NetworkTier type.
-func NetworkTierGCEValueToType(s string) NetworkTier {
-	switch s {
-	case "STANDARD":
-		return NetworkTierStandard
-	case "PREMIUM":
-		return NetworkTierPremium
-	default:
-		return NetworkTier(s)
 	}
 }

pkg/cloudprovider/providers/gce/gce_annotations_test.go

@@ -21,6 +21,7 @@ import (
 
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -38,24 +39,24 @@ func TestServiceNetworkTierAnnotationKey(t *testing.T) {
 
 	for testName, testCase := range map[string]struct {
 		annotations  map[string]string
-		expectedTier NetworkTier
+		expectedTier cloud.NetworkTier
 		expectErr    bool
 	}{
 		"Use the default when the annotation does not exist": {
 			annotations:  nil,
-			expectedTier: NetworkTierDefault,
+			expectedTier: cloud.NetworkTierDefault,
 		},
 		"Standard tier": {
 			annotations:  map[string]string{NetworkTierAnnotationKey: "Standard"},
-			expectedTier: NetworkTierStandard,
+			expectedTier: cloud.NetworkTierStandard,
 		},
 		"Premium tier": {
 			annotations:  map[string]string{NetworkTierAnnotationKey: "Premium"},
-			expectedTier: NetworkTierPremium,
+			expectedTier: cloud.NetworkTierPremium,
 		},
 		"Report an error on invalid network tier value": {
 			annotations:  map[string]string{NetworkTierAnnotationKey: "Unknown-tier"},
-			expectedTier: NetworkTierPremium,
+			expectedTier: cloud.NetworkTierPremium,
 			expectErr:    true,
 		},
 	} {

pkg/cloudprovider/providers/gce/gce_forwardingrule.go

@@ -21,6 +21,7 @@ import (
 
 	computealpha "google.golang.org/api/compute/v0.alpha"
 	compute "google.golang.org/api/compute/v1"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta"
 )
@@ -117,7 +118,7 @@ func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error {
 // TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
 func (gce *GCECloud) getNetworkTierFromForwardingRule(name, region string) (string, error) {
 	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
-		return NetworkTierDefault.ToGCEValue(), nil
+		return cloud.NetworkTierDefault.ToGCEValue(), nil
 	}
 	fwdRule, err := gce.GetAlphaRegionForwardingRule(name, region)
 	if err != nil {

pkg/cloudprovider/providers/gce/gce_forwardingrule_fakes.go

@@ -1,138 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package gce
-
-import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-
-	computealpha "google.golang.org/api/compute/v0.alpha"
-	compute "google.golang.org/api/compute/v1"
-	"google.golang.org/api/googleapi"
-)
-
-type FakeCloudForwardingRuleService struct {
-	// fwdRulesByRegionAndName
-	// Outer key is for region string; inner key is for fwdRuleess name.
-	fwdRulesByRegionAndName map[string]map[string]*computealpha.ForwardingRule
-}
-
-// FakeCloudForwardingRuleService Implements CloudForwardingRuleService
-var _ CloudForwardingRuleService = &FakeCloudForwardingRuleService{}
-
-func NewFakeCloudForwardingRuleService() *FakeCloudForwardingRuleService {
-	return &FakeCloudForwardingRuleService{
-		fwdRulesByRegionAndName: make(map[string]map[string]*computealpha.ForwardingRule),
-	}
-}
-
-// SetRegionalForwardingRulees sets the fwdRuleesses of there region. This is used for
-// setting the test environment.
-func (f *FakeCloudForwardingRuleService) SetRegionalForwardingRulees(region string, fwdRules []*computealpha.ForwardingRule) {
-	// Reset fwdRuleesses in the region.
-	f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule)
-
-	for _, fwdRule := range fwdRules {
-		f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule
-	}
-}
-
-func (f *FakeCloudForwardingRuleService) CreateAlphaRegionForwardingRule(fwdRule *computealpha.ForwardingRule, region string) error {
-	if _, exists := f.fwdRulesByRegionAndName[region]; !exists {
-		f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule)
-	}
-
-	if _, exists := f.fwdRulesByRegionAndName[region][fwdRule.Name]; exists {
-		return &googleapi.Error{Code: http.StatusConflict}
-	}
-
-	f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule
-	return nil
-}
-
-func (f *FakeCloudForwardingRuleService) CreateRegionForwardingRule(fwdRule *compute.ForwardingRule, region string) error {
-	alphafwdRule := convertToAlphaForwardingRule(fwdRule)
-	return f.CreateAlphaRegionForwardingRule(alphafwdRule, region)
-}
-
-func (f *FakeCloudForwardingRuleService) DeleteRegionForwardingRule(name, region string) error {
-	if _, exists := f.fwdRulesByRegionAndName[region]; !exists {
-		return makeGoogleAPINotFoundError("")
-	}
-
-	if _, exists := f.fwdRulesByRegionAndName[region][name]; !exists {
-		return makeGoogleAPINotFoundError("")
-	}
-	delete(f.fwdRulesByRegionAndName[region], name)
-	return nil
-}
-
-func (f *FakeCloudForwardingRuleService) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) {
-	if _, exists := f.fwdRulesByRegionAndName[region]; !exists {
-		return nil, makeGoogleAPINotFoundError("")
-	}
-
-	if fwdRule, exists := f.fwdRulesByRegionAndName[region][name]; !exists {
-		return nil, makeGoogleAPINotFoundError("")
-	} else {
-		return fwdRule, nil
-	}
-}
-
-func (f *FakeCloudForwardingRuleService) GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error) {
-	fwdRule, err := f.GetAlphaRegionForwardingRule(name, region)
-	if fwdRule != nil {
-		return convertToV1ForwardingRule(fwdRule), err
-	}
-	return nil, err
-}
-
-func (f *FakeCloudForwardingRuleService) getNetworkTierFromForwardingRule(name, region string) (string, error) {
-	fwdRule, err := f.GetAlphaRegionForwardingRule(name, region)
-	if err != nil {
-		return "", err
-	}
-	return fwdRule.NetworkTier, nil
-}
-
-func convertToV1ForwardingRule(object gceObject) *compute.ForwardingRule {
-	enc, err := object.MarshalJSON()
-	if err != nil {
-		panic(fmt.Sprintf("Failed to encode to json: %v", err))
-	}
-	var fwdRule compute.ForwardingRule
-	if err := json.Unmarshal(enc, &fwdRule); err != nil {
-		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to v1 fwdRuleess: %v", object, err))
-	}
-	return &fwdRule
-}
-
-func convertToAlphaForwardingRule(object gceObject) *computealpha.ForwardingRule {
-	enc, err := object.MarshalJSON()
-	if err != nil {
-		panic(fmt.Sprintf("Failed to encode to json: %v", err))
-	}
-	var fwdRule computealpha.ForwardingRule
-	if err := json.Unmarshal(enc, &fwdRule); err != nil {
-		panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha fwdRuleess: %v", object, err))
-	}
-	// Set the default values for the Alpha fields.
-	fwdRule.NetworkTier = NetworkTierDefault.ToGCEValue()
-
-	return &fwdRule
-}

pkg/cloudprovider/providers/gce/gce_loadbalancer.go

@@ -28,6 +28,7 @@ import (
 
 	"k8s.io/api/core/v1"
 	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 	netsets "k8s.io/kubernetes/pkg/util/net/sets"
 )
 
@@ -44,13 +45,6 @@ func newLoadBalancerMetricContext(request, region string) *metricContext {
 	return newGenericMetricContext("loadbalancer", request, region, unusedMetricLabel, computeV1Version)
 }
 
-type lbScheme string
-
-const (
-	schemeExternal lbScheme = "EXTERNAL"
-	schemeInternal lbScheme = "INTERNAL"
-)
-
 func init() {
 	var err error
 	// LB L7 proxies and all L3/4/7 health checkers have client addresses within these known CIDRs.
@@ -126,13 +120,13 @@ func (gce *GCECloud) EnsureLoadBalancer(ctx context.Context, clusterName string,
 	}
 
 	if existingFwdRule != nil {
-		existingScheme := lbScheme(strings.ToUpper(existingFwdRule.LoadBalancingScheme))
+		existingScheme := cloud.LbScheme(strings.ToUpper(existingFwdRule.LoadBalancingScheme))
 
 		// If the loadbalancer type changes between INTERNAL and EXTERNAL, the old load balancer should be deleted.
 		if existingScheme != desiredScheme {
 			glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): deleting existing %v loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, gce.region, existingScheme)
 			switch existingScheme {
-			case schemeInternal:
+			case cloud.SchemeInternal:
 				err = gce.ensureInternalLoadBalancerDeleted(clusterName, clusterID, svc)
 			default:
 				err = gce.ensureExternalLoadBalancerDeleted(clusterName, clusterID, svc)
@@ -149,7 +143,7 @@ func (gce *GCECloud) EnsureLoadBalancer(ctx context.Context, clusterName string,
 
 	var status *v1.LoadBalancerStatus
 	switch desiredScheme {
-	case schemeInternal:
+	case cloud.SchemeInternal:
 		status, err = gce.ensureInternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes)
 	default:
 		status, err = gce.ensureExternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes)
@@ -170,7 +164,7 @@ func (gce *GCECloud) UpdateLoadBalancer(ctx context.Context, clusterName string,
 	glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v, %v): updating with %d nodes", clusterName, svc.Namespace, svc.Name, loadBalancerName, gce.region, len(nodes))
 
 	switch scheme {
-	case schemeInternal:
+	case cloud.SchemeInternal:
 		err = gce.updateInternalLoadBalancer(clusterName, clusterID, svc, nodes)
 	default:
 		err = gce.updateExternalLoadBalancer(clusterName, svc, nodes)
@@ -191,7 +185,7 @@ func (gce *GCECloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName
 	glog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v, %v, %v): deleting loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, gce.region)
 
 	switch scheme {
-	case schemeInternal:
+	case cloud.SchemeInternal:
 		err = gce.ensureInternalLoadBalancerDeleted(clusterName, clusterID, svc)
 	default:
 		err = gce.ensureExternalLoadBalancerDeleted(clusterName, clusterID, svc)
@@ -200,9 +194,9 @@ func (gce *GCECloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName
 	return err
 }
 
-func getSvcScheme(svc *v1.Service) lbScheme {
+func getSvcScheme(svc *v1.Service) cloud.LbScheme {
 	if typ, ok := GetLoadBalancerAnnotationType(svc); ok && typ == LBTypeInternal {
-		return schemeInternal
+		return cloud.SchemeInternal
 	}
-	return schemeExternal
+	return cloud.SchemeExternal
 }

pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	apiservice "k8s.io/kubernetes/pkg/api/v1/service"
 	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 	netsets "k8s.io/kubernetes/pkg/util/net/sets"
 
 	"github.com/golang/glog"
@@ -416,7 +417,7 @@ func (gce *GCECloud) DeleteExternalTargetPoolAndChecks(service *v1.Service, name
 // the verification failed. It also returns a boolean to indicate whether the
 // IP address is considered owned by the user (i.e., not managed by the
 // controller.
-func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string, desiredNetTier NetworkTier) (isUserOwnedIP bool, err error) {
+func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string, desiredNetTier cloud.NetworkTier) (isUserOwnedIP bool, err error) {
 	if requestedIP == "" {
 		return false, nil
 	}
@@ -439,7 +440,7 @@ func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP
 		if err != nil {
 			return false, fmt.Errorf("failed to check the network tier of the IP %q: %v", requestedIP, err)
 		}
-		netTier := NetworkTierGCEValueToType(netTierStr)
+		netTier := cloud.NetworkTierGCEValueToType(netTierStr)
 		if netTier != desiredNetTier {
 			glog.Errorf("verifyUserRequestedIP: requested static IP %q (name: %s) for LB %s has network tier %s, need %s.", requestedIP, existingAddress.Name, lbRef, netTier, desiredNetTier)
 			return false, fmt.Errorf("requrested IP %q belongs to the %s network tier; expected %s", requestedIP, netTier, desiredNetTier)
@@ -852,7 +853,7 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(svc *v1.Service, serviceName,
 	return nil
 }
 
-func createForwardingRule(s CloudForwardingRuleService, name, serviceName, region, ipAddress, target string, ports []v1.ServicePort, netTier NetworkTier) error {
+func createForwardingRule(s CloudForwardingRuleService, name, serviceName, region, ipAddress, target string, ports []v1.ServicePort, netTier cloud.NetworkTier) error {
 	portRange, err := loadBalancerPortRange(ports)
 	if err != nil {
 		return err
@@ -861,7 +862,7 @@ func createForwardingRule(s CloudForwardingRuleService, name, serviceName, regio
 	ipProtocol := string(ports[0].Protocol)
 
 	switch netTier {
-	case NetworkTierPremium:
+	case cloud.NetworkTierPremium:
 		rule := &compute.ForwardingRule{
 			Name:        name,
 			Description: desc,
@@ -964,7 +965,7 @@ func (gce *GCECloud) firewallObject(name, region, desc string, sourceRanges nets
 	return firewall, nil
 }
 
-func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string, netTier NetworkTier) (ipAddress string, existing bool, err error) {
+func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string, netTier cloud.NetworkTier) (ipAddress string, existing bool, err error) {
 	// If the address doesn't exist, this will create it.
 	// If the existingIP exists but is ephemeral, this will promote it to static.
 	// If the address already exists, this will harmlessly return a StatusConflict
@@ -974,7 +975,7 @@ func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP
 
 	var creationErr error
 	switch netTier {
-	case NetworkTierPremium:
+	case cloud.NetworkTierPremium:
 		addressObj := &compute.Address{
 			Name:        name,
 			Description: desc,
@@ -1012,19 +1013,19 @@ func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP
 	return addr.Address, existed, nil
 }
 
-func (gce *GCECloud) getServiceNetworkTier(svc *v1.Service) (NetworkTier, error) {
+func (gce *GCECloud) getServiceNetworkTier(svc *v1.Service) (cloud.NetworkTier, error) {
 	if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
-		return NetworkTierDefault, nil
+		return cloud.NetworkTierDefault, nil
 	}
 	tier, err := GetServiceNetworkTier(svc)
 	if err != nil {
 		// Returns an error if the annotation is invalid.
-		return NetworkTier(""), err
+		return cloud.NetworkTier(""), err
 	}
 	return tier, nil
 }
 
-func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, desiredNetTier NetworkTier) error {
+func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, desiredNetTier cloud.NetworkTier) error {
 	logPrefix := fmt.Sprintf("deleteWrongNetworkTieredResources:(%s)", lbRef)
 	if err := deleteFWDRuleWithWrongTier(gce, gce.region, lbName, logPrefix, desiredNetTier); err != nil {
 		return err
@@ -1037,14 +1038,14 @@ func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, des
 
 // deleteFWDRuleWithWrongTier checks the network tier of existing forwarding
 // rule and delete the rule if the tier does not matched the desired tier.
-func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logPrefix string, desiredNetTier NetworkTier) error {
+func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logPrefix string, desiredNetTier cloud.NetworkTier) error {
 	tierStr, err := s.getNetworkTierFromForwardingRule(name, region)
 	if isNotFound(err) {
 		return nil
 	} else if err != nil {
 		return err
 	}
-	existingTier := NetworkTierGCEValueToType(tierStr)
+	existingTier := cloud.NetworkTierGCEValueToType(tierStr)
 	if existingTier == desiredNetTier {
 		return nil
 	}
@@ -1056,7 +1057,7 @@ func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logP
 
 // deleteAddressWithWrongTier checks the network tier of existing address
 // and delete the address if the tier does not matched the desired tier.
-func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix string, desiredNetTier NetworkTier) error {
+func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix string, desiredNetTier cloud.NetworkTier) error {
 	// We only check the IP address matching the reserved name that the
 	// controller assigned to the LB. We make the assumption that an address of
 	// such name is owned by the controller and is safe to release. Whether an
@@ -1072,7 +1073,7 @@ func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix s
 	} else if err != nil {
 		return err
 	}
-	existingTier := NetworkTierGCEValueToType(tierStr)
+	existingTier := cloud.NetworkTierGCEValueToType(tierStr)
 	if existingTier == desiredNetTier {
 		return nil
 	}

pkg/cloudprovider/providers/gce/gce_loadbalancer_external_test.go

@@ -28,230 +28,21 @@ import (
 
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/mock"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 )
 
-func TestEnsureStaticIP(t *testing.T) {
-	fcas := NewFakeCloudAddressService()
-	ipName := "some-static-ip"
-	serviceName := ""
-	region := "us-central1"
-
-	// First ensure call
-	ip, existed, err := ensureStaticIP(fcas, ipName, serviceName, region, "", NetworkTierDefault)
-	if err != nil || existed || ip == "" {
-		t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, "") = %v, %v, %v; want valid ip, false, nil`, fcas, ipName, serviceName, region, ip, existed, err)
-	}
-
-	// Second ensure call
-	var ipPrime string
-	ipPrime, existed, err = ensureStaticIP(fcas, ipName, serviceName, region, ip, NetworkTierDefault)
-	if err != nil || !existed || ip != ipPrime {
-		t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, fcas, ipName, serviceName, region, ip, ipPrime, existed, err, ip)
-	}
-}
-
-func TestEnsureStaticIPWithTier(t *testing.T) {
-	s := NewFakeCloudAddressService()
-	serviceName := ""
-	region := "us-east1"
-
-	for desc, tc := range map[string]struct {
-		name     string
-		netTier  NetworkTier
-		expected string
-	}{
-		"Premium (default)": {
-			name:     "foo-1",
-			netTier:  NetworkTierPremium,
-			expected: "PREMIUM",
-		},
-		"Standard": {
-			name:     "foo-2",
-			netTier:  NetworkTierStandard,
-			expected: "STANDARD",
-		},
-	} {
-		t.Run(desc, func(t *testing.T) {
-			ip, existed, err := ensureStaticIP(s, tc.name, serviceName, region, "", tc.netTier)
-			assert.NoError(t, err)
-			assert.False(t, existed)
-			assert.NotEqual(t, "", ip)
-			// Get the Address from the fake address service and verify that the tier
-			// is set correctly.
-			alphaAddr, err := s.GetAlphaRegionAddress(tc.name, region)
-			require.NoError(t, err)
-			assert.Equal(t, tc.expected, alphaAddr.NetworkTier)
-		})
-	}
-}
-
-func TestVerifyRequestedIP(t *testing.T) {
-	region := "test-region"
-	lbRef := "test-lb"
-	s := NewFakeCloudAddressService()
-
-	for desc, tc := range map[string]struct {
-		requestedIP     string
-		fwdRuleIP       string
-		netTier         NetworkTier
-		addrList        []*computealpha.Address
-		expectErr       bool
-		expectUserOwned bool
-	}{
-		"requested IP exists": {
-			requestedIP:     "1.1.1.1",
-			netTier:         NetworkTierPremium,
-			addrList:        []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
-			expectErr:       false,
-			expectUserOwned: true,
-		},
-		"requested IP is not static, but is in use by the fwd rule": {
-			requestedIP: "1.1.1.1",
-			fwdRuleIP:   "1.1.1.1",
-			netTier:     NetworkTierPremium,
-			expectErr:   false,
-		},
-		"requested IP is not static and is not used by the fwd rule": {
-			requestedIP: "1.1.1.1",
-			fwdRuleIP:   "2.2.2.2",
-			netTier:     NetworkTierPremium,
-			expectErr:   true,
-		},
-		"no requested IP": {
-			netTier:   NetworkTierPremium,
-			expectErr: false,
-		},
-		"requested IP exists, but network tier does not match": {
-			requestedIP: "1.1.1.1",
-			netTier:     NetworkTierStandard,
-			addrList:    []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
-			expectErr:   true,
-		},
-	} {
-		t.Run(desc, func(t *testing.T) {
-			s.SetRegionalAddresses(region, tc.addrList)
-			isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef, tc.netTier)
-			assert.Equal(t, tc.expectErr, err != nil, fmt.Sprintf("err: %v", err))
-			assert.Equal(t, tc.expectUserOwned, isUserOwnedIP)
-		})
-	}
-}
-
-func TestCreateForwardingRuleWithTier(t *testing.T) {
-	s := NewFakeCloudForwardingRuleService()
-	// Common variables among the tests.
-	ports := []v1.ServicePort{{Name: "foo", Protocol: v1.ProtocolTCP, Port: int32(123)}}
-	region := "test-region"
-	target := "test-target-pool"
-	svcName := "foo-svc"
-
-	for desc, tc := range map[string]struct {
-		netTier      NetworkTier
-		expectedRule *computealpha.ForwardingRule
-	}{
-		"Premium tier": {
-			netTier: NetworkTierPremium,
-			expectedRule: &computealpha.ForwardingRule{
-				Name:        "lb-1",
-				Description: `{"kubernetes.io/service-name":"foo-svc"}`,
-				IPAddress:   "1.1.1.1",
-				IPProtocol:  "TCP",
-				PortRange:   "123-123",
-				Target:      target,
-				NetworkTier: "PREMIUM",
-			},
-		},
-		"Standard tier": {
-			netTier: NetworkTierStandard,
-			expectedRule: &computealpha.ForwardingRule{
-				Name:        "lb-2",
-				Description: `{"kubernetes.io/service-name":"foo-svc"}`,
-				IPAddress:   "2.2.2.2",
-				IPProtocol:  "TCP",
-				PortRange:   "123-123",
-				Target:      target,
-				NetworkTier: "STANDARD",
-			},
-		},
-	} {
-		t.Run(desc, func(t *testing.T) {
-			lbName := tc.expectedRule.Name
-			ipAddr := tc.expectedRule.IPAddress
-
-			err := createForwardingRule(s, lbName, svcName, region, ipAddr, target, ports, tc.netTier)
-			assert.NoError(t, err)
-
-			alphaRule, err := s.GetAlphaRegionForwardingRule(lbName, region)
-			assert.NoError(t, err)
-			assert.Equal(t, tc.expectedRule, alphaRule)
-		})
-	}
-}
-
-func TestDeleteAddressWithWrongTier(t *testing.T) {
-	region := "test-region"
-	lbRef := "test-lb"
-	s := NewFakeCloudAddressService()
-
-	for desc, tc := range map[string]struct {
-		addrName     string
-		netTier      NetworkTier
-		addrList     []*computealpha.Address
-		expectDelete bool
-	}{
-		"Network tiers (premium) match; do nothing": {
-			addrName: "foo1",
-			netTier:  NetworkTierPremium,
-			addrList: []*computealpha.Address{{Name: "foo1", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
-		},
-		"Network tiers (standard) match; do nothing": {
-			addrName: "foo2",
-			netTier:  NetworkTierStandard,
-			addrList: []*computealpha.Address{{Name: "foo2", Address: "1.1.1.2", NetworkTier: "STANDARD"}},
-		},
-		"Wrong network tier (standard); delete address": {
-			addrName:     "foo3",
-			netTier:      NetworkTierPremium,
-			addrList:     []*computealpha.Address{{Name: "foo3", Address: "1.1.1.3", NetworkTier: "STANDARD"}},
-			expectDelete: true,
-		},
-		"Wrong network tier (preimium); delete address": {
-			addrName:     "foo4",
-			netTier:      NetworkTierStandard,
-			addrList:     []*computealpha.Address{{Name: "foo4", Address: "1.1.1.4", NetworkTier: "PREMIUM"}},
-			expectDelete: true,
-		},
-	} {
-		t.Run(desc, func(t *testing.T) {
-			s.SetRegionalAddresses(region, tc.addrList)
-			// Sanity check to ensure we inject the right address.
-			_, err := s.GetRegionAddress(tc.addrName, region)
-			require.NoError(t, err)
-
-			err = deleteAddressWithWrongTier(s, region, tc.addrName, lbRef, tc.netTier)
-			assert.NoError(t, err)
-			// Check whether the address still exists.
-			_, err = s.GetRegionAddress(tc.addrName, region)
-			if tc.expectDelete {
-				assert.True(t, isNotFound(err))
-			} else {
-				assert.NoError(t, err)
-			}
-		})
-	}
-}
-
 const (
-	gceProjectId = "test-project"
+	projectID   = "test-project"
-	gceRegion    = "us-central1"
+	region      = "us-central1"
-	zoneName     = "us-central1-b"
+	zoneName    = "us-central1-b"
-	nodeName     = "test-node-1"
+	nodeName    = "test-node-1"
-	clusterName  = "Test Cluster Name"
+	clusterName = "Test Cluster Name"
-	clusterID    = "test-cluster-id"
+	clusterID   = "test-cluster-id"
+	serviceName = ""
 )
 
 var apiService = &v1.Service{
@@ -277,7 +68,7 @@ func fakeGCECloud() (*GCECloud, error) {
 	}
 
 	// Used in disk unit tests
-	fakeManager := newFakeManager(gceProjectId, gceRegion)
+	fakeManager := newFakeManager(projectID, region)
 	zonesWithNodes := createNodeZones([]string{zoneName})
 
 	alphaFeatureGate, err := NewAlphaFeatureGate([]string{})
@@ -286,12 +77,12 @@ func fakeGCECloud() (*GCECloud, error) {
 	}
 
 	gce := &GCECloud{
-		region:             gceRegion,
+		region:             region,
 		service:            service,
 		manager:            fakeManager,
 		managedZones:       []string{zoneName},
-		projectID:          gceProjectId,
+		projectID:          projectID,
-		networkProjectID:   gceProjectId,
+		networkProjectID:   projectID,
 		AlphaFeatureGate:   alphaFeatureGate,
 		nodeZones:          zonesWithNodes,
 		nodeInformerSynced: func() bool { return true },
@@ -300,12 +91,239 @@ func fakeGCECloud() (*GCECloud, error) {
 	cloud := cloud.NewMockGCE(&gceProjectRouter{gce})
 	cloud.MockTargetPools.AddInstanceHook = mock.AddInstanceHook
 	cloud.MockTargetPools.RemoveInstanceHook = mock.RemoveInstanceHook
+	cloud.MockForwardingRules.InsertHook = mock.InsertFwdRuleHook
+	cloud.MockAddresses.InsertHook = mock.InsertAddressHook
+	cloud.MockAlphaAddresses.InsertHook = mock.InsertAlphaAddressHook
 
 	gce.c = cloud
 
 	return gce, nil
 }
 
+func TestEnsureStaticIP(t *testing.T) {
+	gce, err := fakeGCECloud()
+	require.NoError(t, err)
+
+	ipName := "some-static-ip"
+
+	// First ensure call
+	ip, existed, err := ensureStaticIP(gce, ipName, serviceName, region, "", cloud.NetworkTierDefault)
+	if err != nil || existed {
+		t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, "") = %v, %v, %v; want valid ip, false, nil`, gce, ipName, serviceName, region, ip, existed, err)
+	}
+
+	// Second ensure call
+	var ipPrime string
+	ipPrime, existed, err = ensureStaticIP(gce, ipName, serviceName, region, ip, cloud.NetworkTierDefault)
+	if err != nil || !existed || ip != ipPrime {
+		t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, gce, ipName, serviceName, region, ip, ipPrime, existed, err, ip)
+	}
+}
+
+func TestEnsureStaticIPWithTier(t *testing.T) {
+	s, err := fakeGCECloud()
+	require.NoError(t, err)
+
+	for desc, tc := range map[string]struct {
+		name     string
+		netTier  cloud.NetworkTier
+		expected string
+	}{
+		"Premium (default)": {
+			name:     "foo-1",
+			netTier:  cloud.NetworkTierPremium,
+			expected: "PREMIUM",
+		},
+		"Standard": {
+			name:     "foo-2",
+			netTier:  cloud.NetworkTierStandard,
+			expected: "STANDARD",
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			ip, existed, err := ensureStaticIP(s, tc.name, serviceName, region, "", tc.netTier)
+			assert.NoError(t, err)
+			assert.False(t, existed)
+			assert.NotEqual(t, ip, "")
+			// Get the Address from the fake address service and verify that the tier
+			// is set correctly.
+			alphaAddr, err := s.GetAlphaRegionAddress(tc.name, region)
+			require.NoError(t, err)
+			assert.Equal(t, tc.expected, alphaAddr.NetworkTier)
+		})
+	}
+}
+
+func TestVerifyRequestedIP(t *testing.T) {
+	lbRef := "test-lb"
+
+	for desc, tc := range map[string]struct {
+		requestedIP     string
+		fwdRuleIP       string
+		netTier         cloud.NetworkTier
+		addrList        []*computealpha.Address
+		expectErr       bool
+		expectUserOwned bool
+	}{
+		"requested IP exists": {
+			requestedIP:     "1.1.1.1",
+			netTier:         cloud.NetworkTierPremium,
+			addrList:        []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
+			expectErr:       false,
+			expectUserOwned: true,
+		},
+		"requested IP is not static, but is in use by the fwd rule": {
+			requestedIP: "1.1.1.1",
+			fwdRuleIP:   "1.1.1.1",
+			netTier:     cloud.NetworkTierPremium,
+			expectErr:   false,
+		},
+		"requested IP is not static and is not used by the fwd rule": {
+			requestedIP: "1.1.1.1",
+			fwdRuleIP:   "2.2.2.2",
+			netTier:     cloud.NetworkTierPremium,
+			expectErr:   true,
+		},
+		"no requested IP": {
+			netTier:   cloud.NetworkTierPremium,
+			expectErr: false,
+		},
+		"requested IP exists, but network tier does not match": {
+			requestedIP: "1.1.1.1",
+			netTier:     cloud.NetworkTierStandard,
+			addrList:    []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
+			expectErr:   true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			s, err := fakeGCECloud()
+			require.NoError(t, err)
+
+			for _, addr := range tc.addrList {
+				s.ReserveAlphaRegionAddress(addr, region)
+			}
+			isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef, tc.netTier)
+			assert.Equal(t, tc.expectErr, err != nil, fmt.Sprintf("err: %v", err))
+			assert.Equal(t, tc.expectUserOwned, isUserOwnedIP)
+		})
+	}
+}
+
+func TestCreateForwardingRuleWithTier(t *testing.T) {
+	// Common variables among the tests.
+	ports := []v1.ServicePort{{Name: "foo", Protocol: v1.ProtocolTCP, Port: int32(123)}}
+	target := "test-target-pool"
+	svcName := "foo-svc"
+	baseLinkUrl := "https://www.googleapis.com/compute/%v/projects/%v/regions/%v/forwardingRules/%v"
+
+	for desc, tc := range map[string]struct {
+		netTier      cloud.NetworkTier
+		expectedRule *computealpha.ForwardingRule
+	}{
+		"Premium tier": {
+			netTier: cloud.NetworkTierPremium,
+			expectedRule: &computealpha.ForwardingRule{
+				Name:        "lb-1",
+				Description: `{"kubernetes.io/service-name":"foo-svc"}`,
+				IPAddress:   "1.1.1.1",
+				IPProtocol:  "TCP",
+				PortRange:   "123-123",
+				Target:      target,
+				NetworkTier: "PREMIUM",
+				SelfLink:    fmt.Sprintf(baseLinkUrl, "v1", projectID, region, "lb-1"),
+			},
+		},
+		"Standard tier": {
+			netTier: cloud.NetworkTierStandard,
+			expectedRule: &computealpha.ForwardingRule{
+				Name:        "lb-2",
+				Description: `{"kubernetes.io/service-name":"foo-svc"}`,
+				IPAddress:   "2.2.2.2",
+				IPProtocol:  "TCP",
+				PortRange:   "123-123",
+				Target:      target,
+				NetworkTier: "STANDARD",
+				SelfLink:    fmt.Sprintf(baseLinkUrl, "alpha", projectID, region, "lb-2"),
+			},
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			s, err := fakeGCECloud()
+			require.NoError(t, err)
+
+			lbName := tc.expectedRule.Name
+			ipAddr := tc.expectedRule.IPAddress
+
+			err = createForwardingRule(s, lbName, svcName, region, ipAddr, target, ports, tc.netTier)
+			assert.NoError(t, err)
+
+			alphaRule, err := s.GetAlphaRegionForwardingRule(lbName, region)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expectedRule, alphaRule)
+		})
+	}
+}
+
+func TestDeleteAddressWithWrongTier(t *testing.T) {
+	lbRef := "test-lb"
+
+	s, err := fakeGCECloud()
+	require.NoError(t, err)
+
+	// Enable the cloud.NetworkTiers feature
+	s.AlphaFeatureGate.features[AlphaFeatureNetworkTiers] = true
+
+	for desc, tc := range map[string]struct {
+		addrName     string
+		netTier      cloud.NetworkTier
+		addrList     []*computealpha.Address
+		expectDelete bool
+	}{
+		"Network tiers (premium) match; do nothing": {
+			addrName: "foo1",
+			netTier:  cloud.NetworkTierPremium,
+			addrList: []*computealpha.Address{{Name: "foo1", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
+		},
+		"Network tiers (standard) match; do nothing": {
+			addrName: "foo2",
+			netTier:  cloud.NetworkTierStandard,
+			addrList: []*computealpha.Address{{Name: "foo2", Address: "1.1.1.2", NetworkTier: "STANDARD"}},
+		},
+		"Wrong network tier (standard); delete address": {
+			addrName:     "foo3",
+			netTier:      cloud.NetworkTierPremium,
+			addrList:     []*computealpha.Address{{Name: "foo3", Address: "1.1.1.3", NetworkTier: "STANDARD"}},
+			expectDelete: true,
+		},
+		"Wrong network tier (premium); delete address": {
+			addrName:     "foo4",
+			netTier:      cloud.NetworkTierStandard,
+			addrList:     []*computealpha.Address{{Name: "foo4", Address: "1.1.1.4", NetworkTier: "PREMIUM"}},
+			expectDelete: true,
+		},
+	} {
+		t.Run(desc, func(t *testing.T) {
+			for _, addr := range tc.addrList {
+				s.ReserveAlphaRegionAddress(addr, region)
+			}
+
+			// Sanity check to ensure we inject the right address.
+			_, err = s.GetRegionAddress(tc.addrName, region)
+			require.NoError(t, err)
+
+			err = deleteAddressWithWrongTier(s, region, tc.addrName, lbRef, tc.netTier)
+			assert.NoError(t, err)
+			// Check whether the address still exists.
+			_, err = s.GetRegionAddress(tc.addrName, region)
+			if tc.expectDelete {
+				assert.True(t, isNotFound(err))
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
 func createAndInsertNodes(gce *GCECloud, nodeNames []string) ([]*v1.Node, error) {
 	nodes := []*v1.Node{}
 
@@ -318,7 +336,7 @@ func createAndInsertNodes(gce *GCECloud, nodeNames []string) ([]*v1.Node, error)
 
 		if instance == nil {
 			err := gce.InsertInstance(
-				gceProjectId,
+				projectID,
 				zoneName,
 				&compute.Instance{
 					Name: name,
@@ -394,7 +412,7 @@ func TestEnsureExternalLoadBalancer(t *testing.T) {
 	}
 
 	// Check that TargetPool is Created
-	pool, err := gce.GetTargetPool(lbName, gceRegion)
+	pool, err := gce.GetTargetPool(lbName, region)
 	require.NoError(t, err)
 	assert.Equal(t, lbName, pool.Name)
 	assert.NotEmpty(t, pool.HealthChecks)
@@ -406,7 +424,7 @@ func TestEnsureExternalLoadBalancer(t *testing.T) {
 	assert.Equal(t, hcName, healthcheck.Name)
 
 	// Check that ForwardingRule is created
-	fwdRule, err := gce.GetRegionForwardingRule(lbName, gceRegion)
+	fwdRule, err := gce.GetRegionForwardingRule(lbName, region)
 	require.NoError(t, err)
 	assert.Equal(t, lbName, fwdRule.Name)
 	assert.Equal(t, "TCP", fwdRule.IPProtocol)
@@ -430,7 +448,7 @@ func TestUpdateExternalLoadBalancer(t *testing.T) {
 
 	lbName := cloudprovider.GetLoadBalancerName(apiService)
 
-	pool, err := gce.GetTargetPool(lbName, gceRegion)
+	pool, err := gce.GetTargetPool(lbName, region)
 	require.NoError(t, err)
 
 	// TODO: when testify is updated to v1.2.0+, use ElementsMatch instead
@@ -455,7 +473,7 @@ func TestUpdateExternalLoadBalancer(t *testing.T) {
 	err = gce.updateExternalLoadBalancer(clusterName, apiService, newNodes)
 	assert.NoError(t, err)
 
-	pool, err = gce.GetTargetPool(lbName, gceRegion)
+	pool, err = gce.GetTargetPool(lbName, region)
 	require.NoError(t, err)
 
 	assert.Equal(
@@ -491,7 +509,7 @@ func TestEnsureExternalLoadBalancerDeleted(t *testing.T) {
 	}
 
 	// Check that TargetPool is deleted
-	pool, err := gce.GetTargetPool(lbName, gceRegion)
+	pool, err := gce.GetTargetPool(lbName, region)
 	require.Error(t, err)
 	assert.Nil(t, pool)
 
@@ -501,7 +519,58 @@ func TestEnsureExternalLoadBalancerDeleted(t *testing.T) {
 	assert.Nil(t, healthcheck)
 
 	// Check forwarding rule is deleted
-	fwdRule, err := gce.GetRegionForwardingRule(lbName, gceRegion)
+	fwdRule, err := gce.GetRegionForwardingRule(lbName, region)
 	require.Error(t, err)
 	assert.Nil(t, fwdRule)
 }
+
+func TestLoadBalancerWrongTierResourceDeletion(t *testing.T) {
+	gce, err := fakeGCECloud()
+	require.NoError(t, err)
+
+	// Enable the cloud.NetworkTiers feature
+	gce.AlphaFeatureGate.features[AlphaFeatureNetworkTiers] = true
+	apiService.Annotations = map[string]string{NetworkTierAnnotationKey: "Premium"}
+
+	// cloud.NetworkTier defaults to Premium
+	desiredTier, err := gce.getServiceNetworkTier(apiService)
+	require.NoError(t, err)
+	assert.Equal(t, cloud.NetworkTierPremium, desiredTier)
+
+	lbName := cloudprovider.GetLoadBalancerName(apiService)
+	serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name}
+
+	// create ForwardingRule and Address with the wrong tier
+	err = createForwardingRule(
+		gce,
+		lbName,
+		serviceName.String(),
+		region,
+		"",
+		gce.targetPoolURL(lbName),
+		apiService.Spec.Ports,
+		cloud.NetworkTierStandard,
+	)
+	require.NoError(t, err)
+
+	addressObj := &computealpha.Address{
+		Name:        lbName,
+		Description: serviceName.String(),
+		NetworkTier: cloud.NetworkTierStandard.ToGCEValue(),
+	}
+
+	err = gce.ReserveAlphaRegionAddress(addressObj, region)
+	require.NoError(t, err)
+
+	_, err = createExternalLoadBalancer(gce)
+	require.NoError(t, err)
+
+	// Expect forwarding rule tier to not be Standard
+	tier, err := gce.getNetworkTierFromForwardingRule(lbName, region)
+	assert.NoError(t, err)
+	assert.Equal(t, cloud.NetworkTierDefault.ToGCEValue(), tier)
+
+	// Expect address to be deleted
+	_, err = gce.GetRegionAddress(lbName, region)
+	assert.True(t, isNotFound(err))
+}

pkg/cloudprovider/providers/gce/gce_loadbalancer_internal.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	v1_service "k8s.io/kubernetes/pkg/api/v1/service"
 	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 )
 
 const (
@@ -37,7 +38,7 @@ const (
 func (gce *GCECloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v1.Service, existingFwdRule *compute.ForwardingRule, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
 	nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
 	ports, protocol := getPortsAndProtocol(svc.Spec.Ports)
-	scheme := schemeInternal
+	scheme := cloud.SchemeInternal
 	loadBalancerName := cloudprovider.GetLoadBalancerName(svc)
 	sharedBackend := shareBackendService(svc)
 	backendServiceName := makeBackendServiceName(loadBalancerName, clusterID, sharedBackend, scheme, protocol, svc.Spec.SessionAffinity)
@@ -93,7 +94,7 @@ func (gce *GCECloud) ensureInternalLoadBalancer(clusterName, clusterID string, s
 	var addrMgr *addressManager
 	// If the network is not a legacy network, use the address manager
 	if !gce.IsLegacyNetwork() {
-		addrMgr = newAddressManager(gce, nm.String(), gce.Region(), subnetworkURL, loadBalancerName, requestedIP, schemeInternal)
+		addrMgr = newAddressManager(gce, nm.String(), gce.Region(), subnetworkURL, loadBalancerName, requestedIP, cloud.SchemeInternal)
 		ipToUse, err = addrMgr.HoldAddress()
 		if err != nil {
 			return nil, err
@@ -208,7 +209,7 @@ func (gce *GCECloud) updateInternalLoadBalancer(clusterName, clusterID string, s
 
 	// Generate the backend service name
 	_, protocol := getPortsAndProtocol(svc.Spec.Ports)
-	scheme := schemeInternal
+	scheme := cloud.SchemeInternal
 	loadBalancerName := cloudprovider.GetLoadBalancerName(svc)
 	backendServiceName := makeBackendServiceName(loadBalancerName, clusterID, shareBackendService(svc), scheme, protocol, svc.Spec.SessionAffinity)
 	// Ensure the backend service has the proper backend/instance-group links
@@ -218,7 +219,7 @@ func (gce *GCECloud) updateInternalLoadBalancer(clusterName, clusterID string, s
 func (gce *GCECloud) ensureInternalLoadBalancerDeleted(clusterName, clusterID string, svc *v1.Service) error {
 	loadBalancerName := cloudprovider.GetLoadBalancerName(svc)
 	_, protocol := getPortsAndProtocol(svc.Spec.Ports)
-	scheme := schemeInternal
+	scheme := cloud.SchemeInternal
 	sharedBackend := shareBackendService(svc)
 	sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc)
 
@@ -506,7 +507,7 @@ func (gce *GCECloud) ensureInternalInstanceGroupsDeleted(name string) error {
 	return nil
 }
 
-func (gce *GCECloud) ensureInternalBackendService(name, description string, affinityType v1.ServiceAffinity, scheme lbScheme, protocol v1.Protocol, igLinks []string, hcLink string) error {
+func (gce *GCECloud) ensureInternalBackendService(name, description string, affinityType v1.ServiceAffinity, scheme cloud.LbScheme, protocol v1.Protocol, igLinks []string, hcLink string) error {
 	glog.V(2).Infof("ensureInternalBackendService(%v, %v, %v): checking existing backend service with %d groups", name, scheme, protocol, len(igLinks))
 	bs, err := gce.GetRegionBackendService(name, gce.region)
 	if err != nil && !isNotFound(err) {

pkg/cloudprovider/providers/gce/gce_loadbalancer_naming.go

@@ -24,6 +24,7 @@ import (
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 )
 
 // Internal Load Balancer
@@ -33,7 +34,7 @@ func makeInstanceGroupName(clusterID string) string {
 	return fmt.Sprintf("k8s-ig--%s", clusterID)
 }
 
-func makeBackendServiceName(loadBalancerName, clusterID string, shared bool, scheme lbScheme, protocol v1.Protocol, svcAffinity v1.ServiceAffinity) string {
+func makeBackendServiceName(loadBalancerName, clusterID string, shared bool, scheme cloud.LbScheme, protocol v1.Protocol, svcAffinity v1.ServiceAffinity) string {
 	if shared {
 		hash := sha1.New()
 

pkg/cloudprovider/providers/gce/gce_util.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 
 	"cloud.google.com/go/compute/metadata"
 	compute "google.golang.org/api/compute/v1"
@@ -220,7 +221,7 @@ func handleAlphaNetworkTierGetError(err error) (string, error) {
 		// Network tier is still an Alpha feature in GCP, and not every project
 		// is whitelisted to access the API. If we cannot access the API, just
 		// assume the tier is premium.
-		return NetworkTierDefault.ToGCEValue(), nil
+		return cloud.NetworkTierDefault.ToGCEValue(), nil
 	}
 	// Can't get the network tier, just return an error.
 	return "", err

test/e2e/network/BUILD

@@ -36,6 +36,7 @@ go_library(
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/cloudprovider/providers/gce:go_default_library",
+        "//pkg/cloudprovider/providers/gce/cloud:go_default_library",
         "//pkg/controller/endpoint:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
         "//pkg/master/ports:go_default_library",

test/e2e/network/network_tiers.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+	"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
@@ -75,12 +76,12 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
 		By("creating a Service of type LoadBalancer using the standard network tier")
 		svc := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) {
 			svc.Spec.Type = v1.ServiceTypeLoadBalancer
-			setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard)
+			setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard.ToGCEValue())
 		})
 		// Verify that service has been updated properly.
 		svcTier, err := gcecloud.GetServiceNetworkTier(svc)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard))
+		Expect(svcTier).To(Equal(cloud.NetworkTierStandard))
 		// Record the LB name for test cleanup.
 		serviceLBNames = append(serviceLBNames, cloudprovider.GetLoadBalancerName(svc))
 
@@ -95,7 +96,7 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
 		// Verify that service has been updated properly.
 		svcTier, err = gcecloud.GetServiceNetworkTier(svc)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(svcTier).To(Equal(gcecloud.NetworkTierDefault))
+		Expect(svcTier).To(Equal(cloud.NetworkTierDefault))
 
 		// Wait until the ingress IP changes. Each tier has its own pool of
 		// IPs, so changing tiers implies changing IPs.
@@ -106,7 +107,7 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
 		requestedAddrName := fmt.Sprintf("e2e-ext-lb-net-tier-%s", framework.RunId)
 		gceCloud, err := framework.GetGCECloud()
 		Expect(err).NotTo(HaveOccurred())
-		requestedIP, err := reserveAlphaRegionalAddress(gceCloud, requestedAddrName, gcecloud.NetworkTierStandard)
+		requestedIP, err := reserveAlphaRegionalAddress(gceCloud, requestedAddrName, cloud.NetworkTierStandard)
 		Expect(err).NotTo(HaveOccurred(), "failed to reserve a STANDARD tiered address")
 		defer func() {
 			if requestedAddrName != "" {
@@ -122,13 +123,13 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
 		By("updating the Service to use the standard tier with a requested IP")
 		svc = jig.UpdateServiceOrFail(ns, svc.Name, func(svc *v1.Service) {
 			svc.Spec.LoadBalancerIP = requestedIP
-			setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard)
+			setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard.ToGCEValue())
 		})
 		// Verify that service has been updated properly.
 		Expect(svc.Spec.LoadBalancerIP).To(Equal(requestedIP))
 		svcTier, err = gcecloud.GetServiceNetworkTier(svc)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard))
+		Expect(svcTier).To(Equal(cloud.NetworkTierStandard))
 
 		// Wait until the ingress IP changes and verifies the LB.
 		ingressIP = waitAndVerifyLBWithTier(jig, ns, svcName, ingressIP, createTimeout, lagTimeout)
@@ -171,7 +172,7 @@ func waitAndVerifyLBWithTier(jig *framework.ServiceTestJig, ns, svcName, existin
 	return ingressIP
 }
 
-func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) {
+func getLBNetworkTierByIP(ip string) (cloud.NetworkTier, error) {
 	var rule *computealpha.ForwardingRule
 	// Retry a few times to tolerate flakes.
 	err := wait.PollImmediate(5*time.Second, 15*time.Second, func() (bool, error) {
@@ -185,7 +186,7 @@ func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) {
 	if err != nil {
 		return "", err
 	}
-	return gcecloud.NetworkTierGCEValueToType(rule.NetworkTier), nil
+	return cloud.NetworkTierGCEValueToType(rule.NetworkTier), nil
 }
 
 func getGCEForwardingRuleByIP(ip string) (*computealpha.ForwardingRule, error) {
@@ -223,7 +224,7 @@ func clearNetworkTier(svc *v1.Service) {
 
 // TODO: add retries if this turns out to be flaky.
 // TODO(#51665): remove this helper function once Network Tiers becomes beta.
-func reserveAlphaRegionalAddress(cloud *gcecloud.GCECloud, name string, netTier gcecloud.NetworkTier) (string, error) {
+func reserveAlphaRegionalAddress(cloud *gcecloud.GCECloud, name string, netTier cloud.NetworkTier) (string, error) {
 	alphaAddr := &computealpha.Address{
 		Name:        name,
 		NetworkTier: netTier.ToGCEValue(),