Merge pull request #59941 from agau4779/gce-external-lb-tests

Automatic merge from submit-queue (batch tested with PRs 60324, 60269, 59771, 60314, 59941). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

[GCE] Refactor ExternalLoadBalancer Tests

**What this PR does / why we need it**:

- Refactors the ExternalLoadBalancer tests to use the generated GCE mock instead of FakeCloudAddressService, FakeCloudForwardingRuleService.
- Adds hooks to populate NetworkTier on Alpha resources
- Moves shared code to top of the external loadbalancer test file
- Moves NetworkTier into a constants file at the cloud level, so it is more easily called in subpackages

**Special notes for your reviewer**:

```release-note
NONE
```
This commit is contained in:
Kubernetes Submit Queue 2018-02-24 20:01:50 -08:00 committed by GitHub
commit f49f799dbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 597 additions and 473 deletions

View File

@ -23,7 +23,6 @@ go_library(
"gce_disks.go",
"gce_firewall.go",
"gce_forwardingrule.go",
"gce_forwardingrule_fakes.go",
"gce_healthchecks.go",
"gce_instancegroup.go",
"gce_instances.go",
@ -117,6 +116,7 @@ go_test(
"//vendor/google.golang.org/api/googleapi:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
],
)

View File

@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"constants.go",
"doc.go",
"gce_projects.go",
"gen.go",

View File

@ -0,0 +1,55 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cloud
import (
"strings"
)
// NetworkTier represents the Network Service Tier used by a resource
type NetworkTier string
// LbScheme represents the possible types of load balancers
type LbScheme string
const (
NetworkTierStandard NetworkTier = "Standard"
NetworkTierPremium NetworkTier = "Premium"
NetworkTierDefault NetworkTier = NetworkTierPremium
SchemeExternal LbScheme = "EXTERNAL"
SchemeInternal LbScheme = "INTERNAL"
)
// ToGCEValue converts NetworkTier to a string that we can populate the
// NetworkTier field of GCE objects, including ForwardingRules and Addresses.
func (n NetworkTier) ToGCEValue() string {
return strings.ToUpper(string(n))
}
// NetworkTierGCEValueToType converts the value of the NetworkTier field of a
// GCE object to the NetworkTier type.
func NetworkTierGCEValueToType(s string) NetworkTier {
switch s {
case NetworkTierStandard.ToGCEValue():
return NetworkTierStandard
case NetworkTierPremium.ToGCEValue():
return NetworkTierPremium
default:
return NetworkTier(s)
}
}

View File

@ -8,6 +8,8 @@ go_library(
deps = [
"//pkg/cloudprovider/providers/gce/cloud:go_default_library",
"//pkg/cloudprovider/providers/gce/cloud/meta:go_default_library",
"//vendor/google.golang.org/api/compute/v0.alpha:go_default_library",
"//vendor/google.golang.org/api/compute/v0.beta:go_default_library",
"//vendor/google.golang.org/api/compute/v1:go_default_library",
"//vendor/google.golang.org/api/googleapi:go_default_library",
],

View File

@ -25,15 +25,23 @@ package mock
import (
"context"
"encoding/json"
"fmt"
"net/http"
alpha "google.golang.org/api/compute/v0.alpha"
beta "google.golang.org/api/compute/v0.beta"
ga "google.golang.org/api/compute/v1"
"google.golang.org/api/googleapi"
cloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta"
)
// gceObject is an abstraction of all GCE API object in go client
type gceObject interface {
MarshalJSON() ([]byte, error)
}
// AddInstanceHook mocks adding a Instance to MockTargetPools
func AddInstanceHook(ctx context.Context, key *meta.Key, req *ga.TargetPoolsAddInstanceRequest, m *cloud.MockTargetPools) error {
pool, err := m.Get(ctx, key)
@ -74,3 +82,147 @@ func RemoveInstanceHook(ctx context.Context, key *meta.Key, req *ga.TargetPoolsR
return nil
}
func convertAndInsertAlphaForwardingRule(key *meta.Key, obj gceObject, mRules map[meta.Key]*cloud.MockForwardingRulesObj, version meta.Version, projectID string) (bool, error) {
if !key.Valid() {
return false, fmt.Errorf("invalid GCE key (%+v)", key)
}
if _, ok := mRules[*key]; ok {
err := &googleapi.Error{
Code: http.StatusConflict,
Message: fmt.Sprintf("MockForwardingRule %v exists", key),
}
return false, err
}
enc, err := obj.MarshalJSON()
if err != nil {
return false, err
}
var fwdRule alpha.ForwardingRule
if err := json.Unmarshal(enc, &fwdRule); err != nil {
return false, err
}
// Set the default values for the Alpha fields.
if fwdRule.NetworkTier == "" {
fwdRule.NetworkTier = cloud.NetworkTierDefault.ToGCEValue()
}
fwdRule.Name = key.Name
if fwdRule.SelfLink == "" {
fwdRule.SelfLink = cloud.SelfLink(version, projectID, "forwardingRules", key)
}
mRules[*key] = &cloud.MockForwardingRulesObj{Obj: fwdRule}
return true, nil
}
// InsertFwdRuleHook mocks inserting a ForwardingRule. ForwardingRules are
// expected to default to Premium tier if no NetworkTier is specified.
func InsertFwdRuleHook(ctx context.Context, key *meta.Key, obj *ga.ForwardingRule, m *cloud.MockForwardingRules) (bool, error) {
projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionGA, "forwardingRules")
return convertAndInsertAlphaForwardingRule(key, obj, m.Objects, meta.VersionGA, projectID)
}
// InsertBetaFwdRuleHook mocks inserting a BetaForwardingRule.
func InsertBetaFwdRuleHook(ctx context.Context, key *meta.Key, obj *beta.ForwardingRule, m *cloud.MockForwardingRules) (bool, error) {
projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionBeta, "forwardingRules")
return convertAndInsertAlphaForwardingRule(key, obj, m.Objects, meta.VersionBeta, projectID)
}
// InsertAlphaFwdRuleHook mocks inserting an AlphaForwardingRule.
func InsertAlphaFwdRuleHook(ctx context.Context, key *meta.Key, obj *alpha.ForwardingRule, m *cloud.MockForwardingRules) (bool, error) {
projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionAlpha, "forwardingRules")
return convertAndInsertAlphaForwardingRule(key, obj, m.Objects, meta.VersionAlpha, projectID)
}
// Used to assign Addresses with no IP a unique IP address
var ipCounter = 1
func convertAndInsertAlphaAddress(key *meta.Key, obj gceObject, mAddrs map[meta.Key]*cloud.MockAddressesObj, version meta.Version, projectID string) (bool, error) {
if !key.Valid() {
return false, fmt.Errorf("invalid GCE key (%+v)", key)
}
if _, ok := mAddrs[*key]; ok {
err := &googleapi.Error{
Code: http.StatusConflict,
Message: fmt.Sprintf("MockAddresses %v exists", key),
}
return false, err
}
enc, err := obj.MarshalJSON()
if err != nil {
return false, err
}
var addr alpha.Address
if err := json.Unmarshal(enc, &addr); err != nil {
return false, err
}
// Set default address type if not present.
if addr.AddressType == "" {
addr.AddressType = string(cloud.SchemeExternal)
}
var existingAddresses []*ga.Address
for _, obj := range mAddrs {
existingAddresses = append(existingAddresses, obj.ToGA())
}
for _, existingAddr := range existingAddresses {
if addr.Address == existingAddr.Address {
msg := fmt.Sprintf("MockAddresses IP %v in use", addr.Address)
// When the IP is already in use, this call returns a StatusBadRequest
// if the address is an external address, and StatusConflict if an
// internal address. This is to be consistent with actual GCE API.
errorCode := http.StatusConflict
if addr.AddressType == string(cloud.SchemeExternal) {
errorCode = http.StatusBadRequest
}
return false, &googleapi.Error{Code: errorCode, Message: msg}
}
}
// Set default values used in tests
addr.Name = key.Name
if addr.SelfLink == "" {
addr.SelfLink = cloud.SelfLink(version, projectID, "addresses", key)
}
if addr.Address == "" {
addr.Address = fmt.Sprintf("1.2.3.%d", ipCounter)
ipCounter++
}
// Set the default values for the Alpha fields.
if addr.NetworkTier == "" {
addr.NetworkTier = cloud.NetworkTierDefault.ToGCEValue()
}
mAddrs[*key] = &cloud.MockAddressesObj{Obj: addr}
return true, nil
}
// InsertAddressHook mocks inserting an Address.
func InsertAddressHook(ctx context.Context, key *meta.Key, obj *ga.Address, m *cloud.MockAddresses) (bool, error) {
projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionGA, "addresses")
return convertAndInsertAlphaAddress(key, obj, m.Objects, meta.VersionGA, projectID)
}
// InsertBetaAddressHook mocks inserting a BetaAddress.
func InsertBetaAddressHook(ctx context.Context, key *meta.Key, obj *beta.Address, m *cloud.MockAddresses) (bool, error) {
projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionBeta, "addresses")
return convertAndInsertAlphaAddress(key, obj, m.Objects, meta.VersionBeta, projectID)
}
// InsertAlphaAddressHook mocks inserting an Address. Addresses are expected to
// default to Premium tier if no NetworkTier is specified.
func InsertAlphaAddressHook(ctx context.Context, key *meta.Key, obj *alpha.Address, m *cloud.MockAlphaAddresses) (bool, error) {
projectID := m.ProjectRouter.ProjectID(ctx, meta.VersionBeta, "addresses")
return convertAndInsertAlphaAddress(key, obj, m.Objects, meta.VersionAlpha, projectID)
}

View File

@ -23,6 +23,7 @@ import (
compute "google.golang.org/api/compute/v1"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
)
type addressManager struct {
@ -31,13 +32,13 @@ type addressManager struct {
name string
serviceName string
targetIP string
addressType lbScheme
addressType cloud.LbScheme
region string
subnetURL string
tryRelease bool
}
func newAddressManager(svc CloudAddressService, serviceName, region, subnetURL, name, targetIP string, addressType lbScheme) *addressManager {
func newAddressManager(svc CloudAddressService, serviceName, region, subnetURL, name, targetIP string, addressType cloud.LbScheme) *addressManager {
return &addressManager{
svc: svc,
logPrefix: fmt.Sprintf("AddressManager(%q)", name),

View File

@ -22,6 +22,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
compute "google.golang.org/api/compute/v1"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
)
const testSvcName = "my-service"
@ -34,8 +35,8 @@ func TestAddressManagerNoRequestedIP(t *testing.T) {
svc := NewFakeCloudAddressService()
targetIP := ""
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal))
testReleaseAddress(t, mgr, svc, testLBName, testRegion)
}
@ -44,8 +45,8 @@ func TestAddressManagerBasic(t *testing.T) {
svc := NewFakeCloudAddressService()
targetIP := "1.1.1.1"
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal))
testReleaseAddress(t, mgr, svc, testLBName, testRegion)
}
@ -55,12 +56,12 @@ func TestAddressManagerOrphaned(t *testing.T) {
svc := NewFakeCloudAddressService()
targetIP := "1.1.1.1"
addr := &compute.Address{Name: testLBName, Address: targetIP, AddressType: string(schemeInternal)}
addr := &compute.Address{Name: testLBName, Address: targetIP, AddressType: string(cloud.SchemeInternal)}
err := svc.ReserveRegionAddress(addr, testRegion)
require.NoError(t, err)
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal))
testReleaseAddress(t, mgr, svc, testLBName, testRegion)
}
@ -71,12 +72,12 @@ func TestAddressManagerOutdatedOrphan(t *testing.T) {
previousAddress := "1.1.0.0"
targetIP := "1.1.1.1"
addr := &compute.Address{Name: testLBName, Address: previousAddress, AddressType: string(schemeExternal)}
addr := &compute.Address{Name: testLBName, Address: previousAddress, AddressType: string(cloud.SchemeExternal)}
err := svc.ReserveRegionAddress(addr, testRegion)
require.NoError(t, err)
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(schemeInternal))
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
testHoldAddress(t, mgr, svc, testLBName, testRegion, targetIP, string(cloud.SchemeInternal))
testReleaseAddress(t, mgr, svc, testLBName, testRegion)
}
@ -86,11 +87,11 @@ func TestAddressManagerExternallyOwned(t *testing.T) {
svc := NewFakeCloudAddressService()
targetIP := "1.1.1.1"
addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeInternal)}
addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(cloud.SchemeInternal)}
err := svc.ReserveRegionAddress(addr, testRegion)
require.NoError(t, err)
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
ipToUse, err := mgr.HoldAddress()
require.NoError(t, err)
assert.NotEmpty(t, ipToUse)
@ -107,11 +108,11 @@ func TestAddressManagerBadExternallyOwned(t *testing.T) {
svc := NewFakeCloudAddressService()
targetIP := "1.1.1.1"
addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(schemeExternal)}
addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(cloud.SchemeExternal)}
err := svc.ReserveRegionAddress(addr, testRegion)
require.NoError(t, err)
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, schemeInternal)
mgr := newAddressManager(svc, testSvcName, testRegion, testSubnet, testLBName, targetIP, cloud.SchemeInternal)
_, err = mgr.HoldAddress()
assert.NotNil(t, err)
}

View File

@ -26,6 +26,7 @@ import (
computebeta "google.golang.org/api/compute/v0.beta"
compute "google.golang.org/api/compute/v1"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta"
)
@ -150,7 +151,7 @@ func (gce *GCECloud) GetBetaRegionAddressByIP(region, ipAddress string) (*comput
// TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
func (gce *GCECloud) getNetworkTierFromAddress(name, region string) (string, error) {
if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
return NetworkTierDefault.ToGCEValue(), nil
return cloud.NetworkTierDefault.ToGCEValue(), nil
}
addr, err := gce.GetAlphaRegionAddress(name, region)
if err != nil {

View File

@ -25,6 +25,8 @@ import (
computealpha "google.golang.org/api/compute/v0.alpha"
computebeta "google.golang.org/api/compute/v0.beta"
compute "google.golang.org/api/compute/v1"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
)
// test
@ -68,7 +70,7 @@ func (cas *FakeCloudAddressService) ReserveAlphaRegionAddress(addr *computealpha
}
if addr.AddressType == "" {
addr.AddressType = string(schemeExternal)
addr.AddressType = string(cloud.SchemeExternal)
}
if cas.reservedAddrs[addr.Address] {
@ -76,8 +78,8 @@ func (cas *FakeCloudAddressService) ReserveAlphaRegionAddress(addr *computealpha
// When the IP is already in use, this call returns an error code based
// on the type (internal vs external) of the address. This is to be
// consistent with actual GCE API.
switch lbScheme(addr.AddressType) {
case schemeExternal:
switch cloud.LbScheme(addr.AddressType) {
case cloud.SchemeExternal:
return makeGoogleAPIError(http.StatusBadRequest, msg)
default:
return makeGoogleAPIError(http.StatusConflict, msg)
@ -209,7 +211,7 @@ func convertToAlphaAddress(object gceObject) *computealpha.Address {
panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha address: %v", object, err))
}
// Set the default values for the Alpha fields.
addr.NetworkTier = NetworkTierDefault.ToGCEValue()
addr.NetworkTier = cloud.NetworkTierDefault.ToGCEValue()
return &addr
}

View File

@ -18,15 +18,14 @@ package gce
import (
"fmt"
"strings"
"github.com/golang/glog"
"k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
)
type LoadBalancerType string
type NetworkTier string
const (
// ServiceAnnotationLoadBalancerType is annotated on a service with type LoadBalancer
@ -49,12 +48,8 @@ const (
// network tier a GCP LB should use. The valid values are "Standard" and
// "Premium" (default).
NetworkTierAnnotationKey = "cloud.google.com/network-tier"
NetworkTierAnnotationStandard = "Standard"
NetworkTierAnnotationPremium = "Premium"
NetworkTierStandard NetworkTier = NetworkTierAnnotationStandard
NetworkTierPremium NetworkTier = NetworkTierAnnotationPremium
NetworkTierDefault NetworkTier = NetworkTierPremium
NetworkTierAnnotationStandard = cloud.NetworkTierStandard
NetworkTierAnnotationPremium = cloud.NetworkTierPremium
)
// GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled.
@ -97,38 +92,19 @@ func GetLoadBalancerAnnotationBackendShare(service *v1.Service) bool {
// GetServiceNetworkTier returns the network tier of GCP load balancer
// which should be assembled, and an error if the specified tier is not
// supported.
func GetServiceNetworkTier(service *v1.Service) (NetworkTier, error) {
func GetServiceNetworkTier(service *v1.Service) (cloud.NetworkTier, error) {
l, ok := service.Annotations[NetworkTierAnnotationKey]
if !ok {
return NetworkTierDefault, nil
return cloud.NetworkTierDefault, nil
}
v := NetworkTier(l)
v := cloud.NetworkTier(l)
switch v {
case NetworkTierStandard:
case cloud.NetworkTierStandard:
fallthrough
case NetworkTierPremium:
case cloud.NetworkTierPremium:
return v, nil
default:
return NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v)
}
}
// ToGCEValue converts NetworkTier to a string that we can populate the
// NetworkTier field of GCE objects.
func (n NetworkTier) ToGCEValue() string {
return strings.ToUpper(string(n))
}
// NetworkTierGCEValueToType converts the value of the NetworkTier field of a
// GCE object to the NetworkTier type.
func NetworkTierGCEValueToType(s string) NetworkTier {
switch s {
case "STANDARD":
return NetworkTierStandard
case "PREMIUM":
return NetworkTierPremium
default:
return NetworkTier(s)
return cloud.NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v)
}
}

View File

@ -21,6 +21,7 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
"github.com/stretchr/testify/assert"
)
@ -38,24 +39,24 @@ func TestServiceNetworkTierAnnotationKey(t *testing.T) {
for testName, testCase := range map[string]struct {
annotations map[string]string
expectedTier NetworkTier
expectedTier cloud.NetworkTier
expectErr bool
}{
"Use the default when the annotation does not exist": {
annotations: nil,
expectedTier: NetworkTierDefault,
expectedTier: cloud.NetworkTierDefault,
},
"Standard tier": {
annotations: map[string]string{NetworkTierAnnotationKey: "Standard"},
expectedTier: NetworkTierStandard,
expectedTier: cloud.NetworkTierStandard,
},
"Premium tier": {
annotations: map[string]string{NetworkTierAnnotationKey: "Premium"},
expectedTier: NetworkTierPremium,
expectedTier: cloud.NetworkTierPremium,
},
"Report an error on invalid network tier value": {
annotations: map[string]string{NetworkTierAnnotationKey: "Unknown-tier"},
expectedTier: NetworkTierPremium,
expectedTier: cloud.NetworkTierPremium,
expectErr: true,
},
} {

View File

@ -21,6 +21,7 @@ import (
computealpha "google.golang.org/api/compute/v0.alpha"
compute "google.golang.org/api/compute/v1"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta"
)
@ -117,7 +118,7 @@ func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error {
// TODO(#51665): retire this function once Network Tiers becomes Beta in GCP.
func (gce *GCECloud) getNetworkTierFromForwardingRule(name, region string) (string, error) {
if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
return NetworkTierDefault.ToGCEValue(), nil
return cloud.NetworkTierDefault.ToGCEValue(), nil
}
fwdRule, err := gce.GetAlphaRegionForwardingRule(name, region)
if err != nil {

View File

@ -1,138 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gce
import (
"encoding/json"
"fmt"
"net/http"
computealpha "google.golang.org/api/compute/v0.alpha"
compute "google.golang.org/api/compute/v1"
"google.golang.org/api/googleapi"
)
type FakeCloudForwardingRuleService struct {
// fwdRulesByRegionAndName
// Outer key is for region string; inner key is for fwdRuleess name.
fwdRulesByRegionAndName map[string]map[string]*computealpha.ForwardingRule
}
// FakeCloudForwardingRuleService Implements CloudForwardingRuleService
var _ CloudForwardingRuleService = &FakeCloudForwardingRuleService{}
func NewFakeCloudForwardingRuleService() *FakeCloudForwardingRuleService {
return &FakeCloudForwardingRuleService{
fwdRulesByRegionAndName: make(map[string]map[string]*computealpha.ForwardingRule),
}
}
// SetRegionalForwardingRulees sets the fwdRuleesses of there region. This is used for
// setting the test environment.
func (f *FakeCloudForwardingRuleService) SetRegionalForwardingRulees(region string, fwdRules []*computealpha.ForwardingRule) {
// Reset fwdRuleesses in the region.
f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule)
for _, fwdRule := range fwdRules {
f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule
}
}
func (f *FakeCloudForwardingRuleService) CreateAlphaRegionForwardingRule(fwdRule *computealpha.ForwardingRule, region string) error {
if _, exists := f.fwdRulesByRegionAndName[region]; !exists {
f.fwdRulesByRegionAndName[region] = make(map[string]*computealpha.ForwardingRule)
}
if _, exists := f.fwdRulesByRegionAndName[region][fwdRule.Name]; exists {
return &googleapi.Error{Code: http.StatusConflict}
}
f.fwdRulesByRegionAndName[region][fwdRule.Name] = fwdRule
return nil
}
func (f *FakeCloudForwardingRuleService) CreateRegionForwardingRule(fwdRule *compute.ForwardingRule, region string) error {
alphafwdRule := convertToAlphaForwardingRule(fwdRule)
return f.CreateAlphaRegionForwardingRule(alphafwdRule, region)
}
func (f *FakeCloudForwardingRuleService) DeleteRegionForwardingRule(name, region string) error {
if _, exists := f.fwdRulesByRegionAndName[region]; !exists {
return makeGoogleAPINotFoundError("")
}
if _, exists := f.fwdRulesByRegionAndName[region][name]; !exists {
return makeGoogleAPINotFoundError("")
}
delete(f.fwdRulesByRegionAndName[region], name)
return nil
}
func (f *FakeCloudForwardingRuleService) GetAlphaRegionForwardingRule(name, region string) (*computealpha.ForwardingRule, error) {
if _, exists := f.fwdRulesByRegionAndName[region]; !exists {
return nil, makeGoogleAPINotFoundError("")
}
if fwdRule, exists := f.fwdRulesByRegionAndName[region][name]; !exists {
return nil, makeGoogleAPINotFoundError("")
} else {
return fwdRule, nil
}
}
func (f *FakeCloudForwardingRuleService) GetRegionForwardingRule(name, region string) (*compute.ForwardingRule, error) {
fwdRule, err := f.GetAlphaRegionForwardingRule(name, region)
if fwdRule != nil {
return convertToV1ForwardingRule(fwdRule), err
}
return nil, err
}
func (f *FakeCloudForwardingRuleService) getNetworkTierFromForwardingRule(name, region string) (string, error) {
fwdRule, err := f.GetAlphaRegionForwardingRule(name, region)
if err != nil {
return "", err
}
return fwdRule.NetworkTier, nil
}
func convertToV1ForwardingRule(object gceObject) *compute.ForwardingRule {
enc, err := object.MarshalJSON()
if err != nil {
panic(fmt.Sprintf("Failed to encode to json: %v", err))
}
var fwdRule compute.ForwardingRule
if err := json.Unmarshal(enc, &fwdRule); err != nil {
panic(fmt.Sprintf("Failed to convert GCE apiObject %v to v1 fwdRuleess: %v", object, err))
}
return &fwdRule
}
func convertToAlphaForwardingRule(object gceObject) *computealpha.ForwardingRule {
enc, err := object.MarshalJSON()
if err != nil {
panic(fmt.Sprintf("Failed to encode to json: %v", err))
}
var fwdRule computealpha.ForwardingRule
if err := json.Unmarshal(enc, &fwdRule); err != nil {
panic(fmt.Sprintf("Failed to convert GCE apiObject %v to alpha fwdRuleess: %v", object, err))
}
// Set the default values for the Alpha fields.
fwdRule.NetworkTier = NetworkTierDefault.ToGCEValue()
return &fwdRule
}

View File

@ -28,6 +28,7 @@ import (
"k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
netsets "k8s.io/kubernetes/pkg/util/net/sets"
)
@ -44,13 +45,6 @@ func newLoadBalancerMetricContext(request, region string) *metricContext {
return newGenericMetricContext("loadbalancer", request, region, unusedMetricLabel, computeV1Version)
}
type lbScheme string
const (
schemeExternal lbScheme = "EXTERNAL"
schemeInternal lbScheme = "INTERNAL"
)
func init() {
var err error
// LB L7 proxies and all L3/4/7 health checkers have client addresses within these known CIDRs.
@ -126,13 +120,13 @@ func (gce *GCECloud) EnsureLoadBalancer(ctx context.Context, clusterName string,
}
if existingFwdRule != nil {
existingScheme := lbScheme(strings.ToUpper(existingFwdRule.LoadBalancingScheme))
existingScheme := cloud.LbScheme(strings.ToUpper(existingFwdRule.LoadBalancingScheme))
// If the loadbalancer type changes between INTERNAL and EXTERNAL, the old load balancer should be deleted.
if existingScheme != desiredScheme {
glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v): deleting existing %v loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, gce.region, existingScheme)
switch existingScheme {
case schemeInternal:
case cloud.SchemeInternal:
err = gce.ensureInternalLoadBalancerDeleted(clusterName, clusterID, svc)
default:
err = gce.ensureExternalLoadBalancerDeleted(clusterName, clusterID, svc)
@ -149,7 +143,7 @@ func (gce *GCECloud) EnsureLoadBalancer(ctx context.Context, clusterName string,
var status *v1.LoadBalancerStatus
switch desiredScheme {
case schemeInternal:
case cloud.SchemeInternal:
status, err = gce.ensureInternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes)
default:
status, err = gce.ensureExternalLoadBalancer(clusterName, clusterID, svc, existingFwdRule, nodes)
@ -170,7 +164,7 @@ func (gce *GCECloud) UpdateLoadBalancer(ctx context.Context, clusterName string,
glog.V(4).Infof("UpdateLoadBalancer(%v, %v, %v, %v, %v): updating with %d nodes", clusterName, svc.Namespace, svc.Name, loadBalancerName, gce.region, len(nodes))
switch scheme {
case schemeInternal:
case cloud.SchemeInternal:
err = gce.updateInternalLoadBalancer(clusterName, clusterID, svc, nodes)
default:
err = gce.updateExternalLoadBalancer(clusterName, svc, nodes)
@ -191,7 +185,7 @@ func (gce *GCECloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName
glog.V(4).Infof("EnsureLoadBalancerDeleted(%v, %v, %v, %v, %v): deleting loadbalancer", clusterName, svc.Namespace, svc.Name, loadBalancerName, gce.region)
switch scheme {
case schemeInternal:
case cloud.SchemeInternal:
err = gce.ensureInternalLoadBalancerDeleted(clusterName, clusterID, svc)
default:
err = gce.ensureExternalLoadBalancerDeleted(clusterName, clusterID, svc)
@ -200,9 +194,9 @@ func (gce *GCECloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName
return err
}
func getSvcScheme(svc *v1.Service) lbScheme {
func getSvcScheme(svc *v1.Service) cloud.LbScheme {
if typ, ok := GetLoadBalancerAnnotationType(svc); ok && typ == LBTypeInternal {
return schemeInternal
return cloud.SchemeInternal
}
return schemeExternal
return cloud.SchemeExternal
}

View File

@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
apiservice "k8s.io/kubernetes/pkg/api/v1/service"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
netsets "k8s.io/kubernetes/pkg/util/net/sets"
"github.com/golang/glog"
@ -416,7 +417,7 @@ func (gce *GCECloud) DeleteExternalTargetPoolAndChecks(service *v1.Service, name
// the verification failed. It also returns a boolean to indicate whether the
// IP address is considered owned by the user (i.e., not managed by the
// controller.
func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string, desiredNetTier NetworkTier) (isUserOwnedIP bool, err error) {
func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string, desiredNetTier cloud.NetworkTier) (isUserOwnedIP bool, err error) {
if requestedIP == "" {
return false, nil
}
@ -439,7 +440,7 @@ func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP
if err != nil {
return false, fmt.Errorf("failed to check the network tier of the IP %q: %v", requestedIP, err)
}
netTier := NetworkTierGCEValueToType(netTierStr)
netTier := cloud.NetworkTierGCEValueToType(netTierStr)
if netTier != desiredNetTier {
glog.Errorf("verifyUserRequestedIP: requested static IP %q (name: %s) for LB %s has network tier %s, need %s.", requestedIP, existingAddress.Name, lbRef, netTier, desiredNetTier)
return false, fmt.Errorf("requrested IP %q belongs to the %s network tier; expected %s", requestedIP, netTier, desiredNetTier)
@ -852,7 +853,7 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(svc *v1.Service, serviceName,
return nil
}
func createForwardingRule(s CloudForwardingRuleService, name, serviceName, region, ipAddress, target string, ports []v1.ServicePort, netTier NetworkTier) error {
func createForwardingRule(s CloudForwardingRuleService, name, serviceName, region, ipAddress, target string, ports []v1.ServicePort, netTier cloud.NetworkTier) error {
portRange, err := loadBalancerPortRange(ports)
if err != nil {
return err
@ -861,7 +862,7 @@ func createForwardingRule(s CloudForwardingRuleService, name, serviceName, regio
ipProtocol := string(ports[0].Protocol)
switch netTier {
case NetworkTierPremium:
case cloud.NetworkTierPremium:
rule := &compute.ForwardingRule{
Name: name,
Description: desc,
@ -964,7 +965,7 @@ func (gce *GCECloud) firewallObject(name, region, desc string, sourceRanges nets
return firewall, nil
}
func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string, netTier NetworkTier) (ipAddress string, existing bool, err error) {
func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string, netTier cloud.NetworkTier) (ipAddress string, existing bool, err error) {
// If the address doesn't exist, this will create it.
// If the existingIP exists but is ephemeral, this will promote it to static.
// If the address already exists, this will harmlessly return a StatusConflict
@ -974,7 +975,7 @@ func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP
var creationErr error
switch netTier {
case NetworkTierPremium:
case cloud.NetworkTierPremium:
addressObj := &compute.Address{
Name: name,
Description: desc,
@ -1012,19 +1013,19 @@ func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP
return addr.Address, existed, nil
}
func (gce *GCECloud) getServiceNetworkTier(svc *v1.Service) (NetworkTier, error) {
func (gce *GCECloud) getServiceNetworkTier(svc *v1.Service) (cloud.NetworkTier, error) {
if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) {
return NetworkTierDefault, nil
return cloud.NetworkTierDefault, nil
}
tier, err := GetServiceNetworkTier(svc)
if err != nil {
// Returns an error if the annotation is invalid.
return NetworkTier(""), err
return cloud.NetworkTier(""), err
}
return tier, nil
}
func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, desiredNetTier NetworkTier) error {
func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, desiredNetTier cloud.NetworkTier) error {
logPrefix := fmt.Sprintf("deleteWrongNetworkTieredResources:(%s)", lbRef)
if err := deleteFWDRuleWithWrongTier(gce, gce.region, lbName, logPrefix, desiredNetTier); err != nil {
return err
@ -1037,14 +1038,14 @@ func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, des
// deleteFWDRuleWithWrongTier checks the network tier of existing forwarding
// rule and delete the rule if the tier does not matched the desired tier.
func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logPrefix string, desiredNetTier NetworkTier) error {
func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logPrefix string, desiredNetTier cloud.NetworkTier) error {
tierStr, err := s.getNetworkTierFromForwardingRule(name, region)
if isNotFound(err) {
return nil
} else if err != nil {
return err
}
existingTier := NetworkTierGCEValueToType(tierStr)
existingTier := cloud.NetworkTierGCEValueToType(tierStr)
if existingTier == desiredNetTier {
return nil
}
@ -1056,7 +1057,7 @@ func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logP
// deleteAddressWithWrongTier checks the network tier of existing address
// and delete the address if the tier does not matched the desired tier.
func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix string, desiredNetTier NetworkTier) error {
func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix string, desiredNetTier cloud.NetworkTier) error {
// We only check the IP address matching the reserved name that the
// controller assigned to the LB. We make the assumption that an address of
// such name is owned by the controller and is safe to release. Whether an
@ -1072,7 +1073,7 @@ func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix s
} else if err != nil {
return err
}
existingTier := NetworkTierGCEValueToType(tierStr)
existingTier := cloud.NetworkTierGCEValueToType(tierStr)
if existingTier == desiredNetTier {
return nil
}

View File

@ -28,230 +28,21 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/mock"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
)
func TestEnsureStaticIP(t *testing.T) {
fcas := NewFakeCloudAddressService()
ipName := "some-static-ip"
serviceName := ""
region := "us-central1"
// First ensure call
ip, existed, err := ensureStaticIP(fcas, ipName, serviceName, region, "", NetworkTierDefault)
if err != nil || existed || ip == "" {
t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, "") = %v, %v, %v; want valid ip, false, nil`, fcas, ipName, serviceName, region, ip, existed, err)
}
// Second ensure call
var ipPrime string
ipPrime, existed, err = ensureStaticIP(fcas, ipName, serviceName, region, ip, NetworkTierDefault)
if err != nil || !existed || ip != ipPrime {
t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, fcas, ipName, serviceName, region, ip, ipPrime, existed, err, ip)
}
}
func TestEnsureStaticIPWithTier(t *testing.T) {
s := NewFakeCloudAddressService()
serviceName := ""
region := "us-east1"
for desc, tc := range map[string]struct {
name string
netTier NetworkTier
expected string
}{
"Premium (default)": {
name: "foo-1",
netTier: NetworkTierPremium,
expected: "PREMIUM",
},
"Standard": {
name: "foo-2",
netTier: NetworkTierStandard,
expected: "STANDARD",
},
} {
t.Run(desc, func(t *testing.T) {
ip, existed, err := ensureStaticIP(s, tc.name, serviceName, region, "", tc.netTier)
assert.NoError(t, err)
assert.False(t, existed)
assert.NotEqual(t, "", ip)
// Get the Address from the fake address service and verify that the tier
// is set correctly.
alphaAddr, err := s.GetAlphaRegionAddress(tc.name, region)
require.NoError(t, err)
assert.Equal(t, tc.expected, alphaAddr.NetworkTier)
})
}
}
func TestVerifyRequestedIP(t *testing.T) {
region := "test-region"
lbRef := "test-lb"
s := NewFakeCloudAddressService()
for desc, tc := range map[string]struct {
requestedIP string
fwdRuleIP string
netTier NetworkTier
addrList []*computealpha.Address
expectErr bool
expectUserOwned bool
}{
"requested IP exists": {
requestedIP: "1.1.1.1",
netTier: NetworkTierPremium,
addrList: []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
expectErr: false,
expectUserOwned: true,
},
"requested IP is not static, but is in use by the fwd rule": {
requestedIP: "1.1.1.1",
fwdRuleIP: "1.1.1.1",
netTier: NetworkTierPremium,
expectErr: false,
},
"requested IP is not static and is not used by the fwd rule": {
requestedIP: "1.1.1.1",
fwdRuleIP: "2.2.2.2",
netTier: NetworkTierPremium,
expectErr: true,
},
"no requested IP": {
netTier: NetworkTierPremium,
expectErr: false,
},
"requested IP exists, but network tier does not match": {
requestedIP: "1.1.1.1",
netTier: NetworkTierStandard,
addrList: []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
expectErr: true,
},
} {
t.Run(desc, func(t *testing.T) {
s.SetRegionalAddresses(region, tc.addrList)
isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef, tc.netTier)
assert.Equal(t, tc.expectErr, err != nil, fmt.Sprintf("err: %v", err))
assert.Equal(t, tc.expectUserOwned, isUserOwnedIP)
})
}
}
func TestCreateForwardingRuleWithTier(t *testing.T) {
s := NewFakeCloudForwardingRuleService()
// Common variables among the tests.
ports := []v1.ServicePort{{Name: "foo", Protocol: v1.ProtocolTCP, Port: int32(123)}}
region := "test-region"
target := "test-target-pool"
svcName := "foo-svc"
for desc, tc := range map[string]struct {
netTier NetworkTier
expectedRule *computealpha.ForwardingRule
}{
"Premium tier": {
netTier: NetworkTierPremium,
expectedRule: &computealpha.ForwardingRule{
Name: "lb-1",
Description: `{"kubernetes.io/service-name":"foo-svc"}`,
IPAddress: "1.1.1.1",
IPProtocol: "TCP",
PortRange: "123-123",
Target: target,
NetworkTier: "PREMIUM",
},
},
"Standard tier": {
netTier: NetworkTierStandard,
expectedRule: &computealpha.ForwardingRule{
Name: "lb-2",
Description: `{"kubernetes.io/service-name":"foo-svc"}`,
IPAddress: "2.2.2.2",
IPProtocol: "TCP",
PortRange: "123-123",
Target: target,
NetworkTier: "STANDARD",
},
},
} {
t.Run(desc, func(t *testing.T) {
lbName := tc.expectedRule.Name
ipAddr := tc.expectedRule.IPAddress
err := createForwardingRule(s, lbName, svcName, region, ipAddr, target, ports, tc.netTier)
assert.NoError(t, err)
alphaRule, err := s.GetAlphaRegionForwardingRule(lbName, region)
assert.NoError(t, err)
assert.Equal(t, tc.expectedRule, alphaRule)
})
}
}
func TestDeleteAddressWithWrongTier(t *testing.T) {
region := "test-region"
lbRef := "test-lb"
s := NewFakeCloudAddressService()
for desc, tc := range map[string]struct {
addrName string
netTier NetworkTier
addrList []*computealpha.Address
expectDelete bool
}{
"Network tiers (premium) match; do nothing": {
addrName: "foo1",
netTier: NetworkTierPremium,
addrList: []*computealpha.Address{{Name: "foo1", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
},
"Network tiers (standard) match; do nothing": {
addrName: "foo2",
netTier: NetworkTierStandard,
addrList: []*computealpha.Address{{Name: "foo2", Address: "1.1.1.2", NetworkTier: "STANDARD"}},
},
"Wrong network tier (standard); delete address": {
addrName: "foo3",
netTier: NetworkTierPremium,
addrList: []*computealpha.Address{{Name: "foo3", Address: "1.1.1.3", NetworkTier: "STANDARD"}},
expectDelete: true,
},
"Wrong network tier (preimium); delete address": {
addrName: "foo4",
netTier: NetworkTierStandard,
addrList: []*computealpha.Address{{Name: "foo4", Address: "1.1.1.4", NetworkTier: "PREMIUM"}},
expectDelete: true,
},
} {
t.Run(desc, func(t *testing.T) {
s.SetRegionalAddresses(region, tc.addrList)
// Sanity check to ensure we inject the right address.
_, err := s.GetRegionAddress(tc.addrName, region)
require.NoError(t, err)
err = deleteAddressWithWrongTier(s, region, tc.addrName, lbRef, tc.netTier)
assert.NoError(t, err)
// Check whether the address still exists.
_, err = s.GetRegionAddress(tc.addrName, region)
if tc.expectDelete {
assert.True(t, isNotFound(err))
} else {
assert.NoError(t, err)
}
})
}
}
const (
gceProjectId = "test-project"
gceRegion = "us-central1"
projectID = "test-project"
region = "us-central1"
zoneName = "us-central1-b"
nodeName = "test-node-1"
clusterName = "Test Cluster Name"
clusterID = "test-cluster-id"
serviceName = ""
)
var apiService = &v1.Service{
@ -277,7 +68,7 @@ func fakeGCECloud() (*GCECloud, error) {
}
// Used in disk unit tests
fakeManager := newFakeManager(gceProjectId, gceRegion)
fakeManager := newFakeManager(projectID, region)
zonesWithNodes := createNodeZones([]string{zoneName})
alphaFeatureGate, err := NewAlphaFeatureGate([]string{})
@ -286,12 +77,12 @@ func fakeGCECloud() (*GCECloud, error) {
}
gce := &GCECloud{
region: gceRegion,
region: region,
service: service,
manager: fakeManager,
managedZones: []string{zoneName},
projectID: gceProjectId,
networkProjectID: gceProjectId,
projectID: projectID,
networkProjectID: projectID,
AlphaFeatureGate: alphaFeatureGate,
nodeZones: zonesWithNodes,
nodeInformerSynced: func() bool { return true },
@ -300,12 +91,239 @@ func fakeGCECloud() (*GCECloud, error) {
cloud := cloud.NewMockGCE(&gceProjectRouter{gce})
cloud.MockTargetPools.AddInstanceHook = mock.AddInstanceHook
cloud.MockTargetPools.RemoveInstanceHook = mock.RemoveInstanceHook
cloud.MockForwardingRules.InsertHook = mock.InsertFwdRuleHook
cloud.MockAddresses.InsertHook = mock.InsertAddressHook
cloud.MockAlphaAddresses.InsertHook = mock.InsertAlphaAddressHook
gce.c = cloud
return gce, nil
}
func TestEnsureStaticIP(t *testing.T) {
gce, err := fakeGCECloud()
require.NoError(t, err)
ipName := "some-static-ip"
// First ensure call
ip, existed, err := ensureStaticIP(gce, ipName, serviceName, region, "", cloud.NetworkTierDefault)
if err != nil || existed {
t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, "") = %v, %v, %v; want valid ip, false, nil`, gce, ipName, serviceName, region, ip, existed, err)
}
// Second ensure call
var ipPrime string
ipPrime, existed, err = ensureStaticIP(gce, ipName, serviceName, region, ip, cloud.NetworkTierDefault)
if err != nil || !existed || ip != ipPrime {
t.Fatalf(`ensureStaticIP(%v, %v, %v, %v, %v) = %v, %v, %v; want %v, true, nil`, gce, ipName, serviceName, region, ip, ipPrime, existed, err, ip)
}
}
func TestEnsureStaticIPWithTier(t *testing.T) {
s, err := fakeGCECloud()
require.NoError(t, err)
for desc, tc := range map[string]struct {
name string
netTier cloud.NetworkTier
expected string
}{
"Premium (default)": {
name: "foo-1",
netTier: cloud.NetworkTierPremium,
expected: "PREMIUM",
},
"Standard": {
name: "foo-2",
netTier: cloud.NetworkTierStandard,
expected: "STANDARD",
},
} {
t.Run(desc, func(t *testing.T) {
ip, existed, err := ensureStaticIP(s, tc.name, serviceName, region, "", tc.netTier)
assert.NoError(t, err)
assert.False(t, existed)
assert.NotEqual(t, ip, "")
// Get the Address from the fake address service and verify that the tier
// is set correctly.
alphaAddr, err := s.GetAlphaRegionAddress(tc.name, region)
require.NoError(t, err)
assert.Equal(t, tc.expected, alphaAddr.NetworkTier)
})
}
}
func TestVerifyRequestedIP(t *testing.T) {
lbRef := "test-lb"
for desc, tc := range map[string]struct {
requestedIP string
fwdRuleIP string
netTier cloud.NetworkTier
addrList []*computealpha.Address
expectErr bool
expectUserOwned bool
}{
"requested IP exists": {
requestedIP: "1.1.1.1",
netTier: cloud.NetworkTierPremium,
addrList: []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
expectErr: false,
expectUserOwned: true,
},
"requested IP is not static, but is in use by the fwd rule": {
requestedIP: "1.1.1.1",
fwdRuleIP: "1.1.1.1",
netTier: cloud.NetworkTierPremium,
expectErr: false,
},
"requested IP is not static and is not used by the fwd rule": {
requestedIP: "1.1.1.1",
fwdRuleIP: "2.2.2.2",
netTier: cloud.NetworkTierPremium,
expectErr: true,
},
"no requested IP": {
netTier: cloud.NetworkTierPremium,
expectErr: false,
},
"requested IP exists, but network tier does not match": {
requestedIP: "1.1.1.1",
netTier: cloud.NetworkTierStandard,
addrList: []*computealpha.Address{{Name: "foo", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
expectErr: true,
},
} {
t.Run(desc, func(t *testing.T) {
s, err := fakeGCECloud()
require.NoError(t, err)
for _, addr := range tc.addrList {
s.ReserveAlphaRegionAddress(addr, region)
}
isUserOwnedIP, err := verifyUserRequestedIP(s, region, tc.requestedIP, tc.fwdRuleIP, lbRef, tc.netTier)
assert.Equal(t, tc.expectErr, err != nil, fmt.Sprintf("err: %v", err))
assert.Equal(t, tc.expectUserOwned, isUserOwnedIP)
})
}
}
func TestCreateForwardingRuleWithTier(t *testing.T) {
// Common variables among the tests.
ports := []v1.ServicePort{{Name: "foo", Protocol: v1.ProtocolTCP, Port: int32(123)}}
target := "test-target-pool"
svcName := "foo-svc"
baseLinkUrl := "https://www.googleapis.com/compute/%v/projects/%v/regions/%v/forwardingRules/%v"
for desc, tc := range map[string]struct {
netTier cloud.NetworkTier
expectedRule *computealpha.ForwardingRule
}{
"Premium tier": {
netTier: cloud.NetworkTierPremium,
expectedRule: &computealpha.ForwardingRule{
Name: "lb-1",
Description: `{"kubernetes.io/service-name":"foo-svc"}`,
IPAddress: "1.1.1.1",
IPProtocol: "TCP",
PortRange: "123-123",
Target: target,
NetworkTier: "PREMIUM",
SelfLink: fmt.Sprintf(baseLinkUrl, "v1", projectID, region, "lb-1"),
},
},
"Standard tier": {
netTier: cloud.NetworkTierStandard,
expectedRule: &computealpha.ForwardingRule{
Name: "lb-2",
Description: `{"kubernetes.io/service-name":"foo-svc"}`,
IPAddress: "2.2.2.2",
IPProtocol: "TCP",
PortRange: "123-123",
Target: target,
NetworkTier: "STANDARD",
SelfLink: fmt.Sprintf(baseLinkUrl, "alpha", projectID, region, "lb-2"),
},
},
} {
t.Run(desc, func(t *testing.T) {
s, err := fakeGCECloud()
require.NoError(t, err)
lbName := tc.expectedRule.Name
ipAddr := tc.expectedRule.IPAddress
err = createForwardingRule(s, lbName, svcName, region, ipAddr, target, ports, tc.netTier)
assert.NoError(t, err)
alphaRule, err := s.GetAlphaRegionForwardingRule(lbName, region)
assert.NoError(t, err)
assert.Equal(t, tc.expectedRule, alphaRule)
})
}
}
func TestDeleteAddressWithWrongTier(t *testing.T) {
lbRef := "test-lb"
s, err := fakeGCECloud()
require.NoError(t, err)
// Enable the cloud.NetworkTiers feature
s.AlphaFeatureGate.features[AlphaFeatureNetworkTiers] = true
for desc, tc := range map[string]struct {
addrName string
netTier cloud.NetworkTier
addrList []*computealpha.Address
expectDelete bool
}{
"Network tiers (premium) match; do nothing": {
addrName: "foo1",
netTier: cloud.NetworkTierPremium,
addrList: []*computealpha.Address{{Name: "foo1", Address: "1.1.1.1", NetworkTier: "PREMIUM"}},
},
"Network tiers (standard) match; do nothing": {
addrName: "foo2",
netTier: cloud.NetworkTierStandard,
addrList: []*computealpha.Address{{Name: "foo2", Address: "1.1.1.2", NetworkTier: "STANDARD"}},
},
"Wrong network tier (standard); delete address": {
addrName: "foo3",
netTier: cloud.NetworkTierPremium,
addrList: []*computealpha.Address{{Name: "foo3", Address: "1.1.1.3", NetworkTier: "STANDARD"}},
expectDelete: true,
},
"Wrong network tier (premium); delete address": {
addrName: "foo4",
netTier: cloud.NetworkTierStandard,
addrList: []*computealpha.Address{{Name: "foo4", Address: "1.1.1.4", NetworkTier: "PREMIUM"}},
expectDelete: true,
},
} {
t.Run(desc, func(t *testing.T) {
for _, addr := range tc.addrList {
s.ReserveAlphaRegionAddress(addr, region)
}
// Sanity check to ensure we inject the right address.
_, err = s.GetRegionAddress(tc.addrName, region)
require.NoError(t, err)
err = deleteAddressWithWrongTier(s, region, tc.addrName, lbRef, tc.netTier)
assert.NoError(t, err)
// Check whether the address still exists.
_, err = s.GetRegionAddress(tc.addrName, region)
if tc.expectDelete {
assert.True(t, isNotFound(err))
} else {
assert.NoError(t, err)
}
})
}
}
func createAndInsertNodes(gce *GCECloud, nodeNames []string) ([]*v1.Node, error) {
nodes := []*v1.Node{}
@ -318,7 +336,7 @@ func createAndInsertNodes(gce *GCECloud, nodeNames []string) ([]*v1.Node, error)
if instance == nil {
err := gce.InsertInstance(
gceProjectId,
projectID,
zoneName,
&compute.Instance{
Name: name,
@ -394,7 +412,7 @@ func TestEnsureExternalLoadBalancer(t *testing.T) {
}
// Check that TargetPool is Created
pool, err := gce.GetTargetPool(lbName, gceRegion)
pool, err := gce.GetTargetPool(lbName, region)
require.NoError(t, err)
assert.Equal(t, lbName, pool.Name)
assert.NotEmpty(t, pool.HealthChecks)
@ -406,7 +424,7 @@ func TestEnsureExternalLoadBalancer(t *testing.T) {
assert.Equal(t, hcName, healthcheck.Name)
// Check that ForwardingRule is created
fwdRule, err := gce.GetRegionForwardingRule(lbName, gceRegion)
fwdRule, err := gce.GetRegionForwardingRule(lbName, region)
require.NoError(t, err)
assert.Equal(t, lbName, fwdRule.Name)
assert.Equal(t, "TCP", fwdRule.IPProtocol)
@ -430,7 +448,7 @@ func TestUpdateExternalLoadBalancer(t *testing.T) {
lbName := cloudprovider.GetLoadBalancerName(apiService)
pool, err := gce.GetTargetPool(lbName, gceRegion)
pool, err := gce.GetTargetPool(lbName, region)
require.NoError(t, err)
// TODO: when testify is updated to v1.2.0+, use ElementsMatch instead
@ -455,7 +473,7 @@ func TestUpdateExternalLoadBalancer(t *testing.T) {
err = gce.updateExternalLoadBalancer(clusterName, apiService, newNodes)
assert.NoError(t, err)
pool, err = gce.GetTargetPool(lbName, gceRegion)
pool, err = gce.GetTargetPool(lbName, region)
require.NoError(t, err)
assert.Equal(
@ -491,7 +509,7 @@ func TestEnsureExternalLoadBalancerDeleted(t *testing.T) {
}
// Check that TargetPool is deleted
pool, err := gce.GetTargetPool(lbName, gceRegion)
pool, err := gce.GetTargetPool(lbName, region)
require.Error(t, err)
assert.Nil(t, pool)
@ -501,7 +519,58 @@ func TestEnsureExternalLoadBalancerDeleted(t *testing.T) {
assert.Nil(t, healthcheck)
// Check forwarding rule is deleted
fwdRule, err := gce.GetRegionForwardingRule(lbName, gceRegion)
fwdRule, err := gce.GetRegionForwardingRule(lbName, region)
require.Error(t, err)
assert.Nil(t, fwdRule)
}
func TestLoadBalancerWrongTierResourceDeletion(t *testing.T) {
gce, err := fakeGCECloud()
require.NoError(t, err)
// Enable the cloud.NetworkTiers feature
gce.AlphaFeatureGate.features[AlphaFeatureNetworkTiers] = true
apiService.Annotations = map[string]string{NetworkTierAnnotationKey: "Premium"}
// cloud.NetworkTier defaults to Premium
desiredTier, err := gce.getServiceNetworkTier(apiService)
require.NoError(t, err)
assert.Equal(t, cloud.NetworkTierPremium, desiredTier)
lbName := cloudprovider.GetLoadBalancerName(apiService)
serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name}
// create ForwardingRule and Address with the wrong tier
err = createForwardingRule(
gce,
lbName,
serviceName.String(),
region,
"",
gce.targetPoolURL(lbName),
apiService.Spec.Ports,
cloud.NetworkTierStandard,
)
require.NoError(t, err)
addressObj := &computealpha.Address{
Name: lbName,
Description: serviceName.String(),
NetworkTier: cloud.NetworkTierStandard.ToGCEValue(),
}
err = gce.ReserveAlphaRegionAddress(addressObj, region)
require.NoError(t, err)
_, err = createExternalLoadBalancer(gce)
require.NoError(t, err)
// Expect forwarding rule tier to not be Standard
tier, err := gce.getNetworkTierFromForwardingRule(lbName, region)
assert.NoError(t, err)
assert.Equal(t, cloud.NetworkTierDefault.ToGCEValue(), tier)
// Expect address to be deleted
_, err = gce.GetRegionAddress(lbName, region)
assert.True(t, isNotFound(err))
}

View File

@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
v1_service "k8s.io/kubernetes/pkg/api/v1/service"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
)
const (
@ -37,7 +38,7 @@ const (
func (gce *GCECloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v1.Service, existingFwdRule *compute.ForwardingRule, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
ports, protocol := getPortsAndProtocol(svc.Spec.Ports)
scheme := schemeInternal
scheme := cloud.SchemeInternal
loadBalancerName := cloudprovider.GetLoadBalancerName(svc)
sharedBackend := shareBackendService(svc)
backendServiceName := makeBackendServiceName(loadBalancerName, clusterID, sharedBackend, scheme, protocol, svc.Spec.SessionAffinity)
@ -93,7 +94,7 @@ func (gce *GCECloud) ensureInternalLoadBalancer(clusterName, clusterID string, s
var addrMgr *addressManager
// If the network is not a legacy network, use the address manager
if !gce.IsLegacyNetwork() {
addrMgr = newAddressManager(gce, nm.String(), gce.Region(), subnetworkURL, loadBalancerName, requestedIP, schemeInternal)
addrMgr = newAddressManager(gce, nm.String(), gce.Region(), subnetworkURL, loadBalancerName, requestedIP, cloud.SchemeInternal)
ipToUse, err = addrMgr.HoldAddress()
if err != nil {
return nil, err
@ -208,7 +209,7 @@ func (gce *GCECloud) updateInternalLoadBalancer(clusterName, clusterID string, s
// Generate the backend service name
_, protocol := getPortsAndProtocol(svc.Spec.Ports)
scheme := schemeInternal
scheme := cloud.SchemeInternal
loadBalancerName := cloudprovider.GetLoadBalancerName(svc)
backendServiceName := makeBackendServiceName(loadBalancerName, clusterID, shareBackendService(svc), scheme, protocol, svc.Spec.SessionAffinity)
// Ensure the backend service has the proper backend/instance-group links
@ -218,7 +219,7 @@ func (gce *GCECloud) updateInternalLoadBalancer(clusterName, clusterID string, s
func (gce *GCECloud) ensureInternalLoadBalancerDeleted(clusterName, clusterID string, svc *v1.Service) error {
loadBalancerName := cloudprovider.GetLoadBalancerName(svc)
_, protocol := getPortsAndProtocol(svc.Spec.Ports)
scheme := schemeInternal
scheme := cloud.SchemeInternal
sharedBackend := shareBackendService(svc)
sharedHealthCheck := !v1_service.RequestsOnlyLocalTraffic(svc)
@ -506,7 +507,7 @@ func (gce *GCECloud) ensureInternalInstanceGroupsDeleted(name string) error {
return nil
}
func (gce *GCECloud) ensureInternalBackendService(name, description string, affinityType v1.ServiceAffinity, scheme lbScheme, protocol v1.Protocol, igLinks []string, hcLink string) error {
func (gce *GCECloud) ensureInternalBackendService(name, description string, affinityType v1.ServiceAffinity, scheme cloud.LbScheme, protocol v1.Protocol, igLinks []string, hcLink string) error {
glog.V(2).Infof("ensureInternalBackendService(%v, %v, %v): checking existing backend service with %d groups", name, scheme, protocol, len(igLinks))
bs, err := gce.GetRegionBackendService(name, gce.region)
if err != nil && !isNotFound(err) {

View File

@ -24,6 +24,7 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
)
// Internal Load Balancer
@ -33,7 +34,7 @@ func makeInstanceGroupName(clusterID string) string {
return fmt.Sprintf("k8s-ig--%s", clusterID)
}
func makeBackendServiceName(loadBalancerName, clusterID string, shared bool, scheme lbScheme, protocol v1.Protocol, svcAffinity v1.ServiceAffinity) string {
func makeBackendServiceName(loadBalancerName, clusterID string, shared bool, scheme cloud.LbScheme, protocol v1.Protocol, svcAffinity v1.ServiceAffinity) string {
if shared {
hash := sha1.New()

View File

@ -28,6 +28,7 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
"cloud.google.com/go/compute/metadata"
compute "google.golang.org/api/compute/v1"
@ -220,7 +221,7 @@ func handleAlphaNetworkTierGetError(err error) (string, error) {
// Network tier is still an Alpha feature in GCP, and not every project
// is whitelisted to access the API. If we cannot access the API, just
// assume the tier is premium.
return NetworkTierDefault.ToGCEValue(), nil
return cloud.NetworkTierDefault.ToGCEValue(), nil
}
// Can't get the network tier, just return an error.
return "", err

View File

@ -36,6 +36,7 @@ go_library(
"//pkg/client/clientset_generated/internalclientset:go_default_library",
"//pkg/cloudprovider:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/cloudprovider/providers/gce/cloud:go_default_library",
"//pkg/controller/endpoint:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//pkg/master/ports:go_default_library",

View File

@ -29,6 +29,7 @@ import (
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/cloudprovider"
gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
"k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
@ -75,12 +76,12 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
By("creating a Service of type LoadBalancer using the standard network tier")
svc := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) {
svc.Spec.Type = v1.ServiceTypeLoadBalancer
setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard)
setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard.ToGCEValue())
})
// Verify that service has been updated properly.
svcTier, err := gcecloud.GetServiceNetworkTier(svc)
Expect(err).NotTo(HaveOccurred())
Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard))
Expect(svcTier).To(Equal(cloud.NetworkTierStandard))
// Record the LB name for test cleanup.
serviceLBNames = append(serviceLBNames, cloudprovider.GetLoadBalancerName(svc))
@ -95,7 +96,7 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
// Verify that service has been updated properly.
svcTier, err = gcecloud.GetServiceNetworkTier(svc)
Expect(err).NotTo(HaveOccurred())
Expect(svcTier).To(Equal(gcecloud.NetworkTierDefault))
Expect(svcTier).To(Equal(cloud.NetworkTierDefault))
// Wait until the ingress IP changes. Each tier has its own pool of
// IPs, so changing tiers implies changing IPs.
@ -106,7 +107,7 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
requestedAddrName := fmt.Sprintf("e2e-ext-lb-net-tier-%s", framework.RunId)
gceCloud, err := framework.GetGCECloud()
Expect(err).NotTo(HaveOccurred())
requestedIP, err := reserveAlphaRegionalAddress(gceCloud, requestedAddrName, gcecloud.NetworkTierStandard)
requestedIP, err := reserveAlphaRegionalAddress(gceCloud, requestedAddrName, cloud.NetworkTierStandard)
Expect(err).NotTo(HaveOccurred(), "failed to reserve a STANDARD tiered address")
defer func() {
if requestedAddrName != "" {
@ -122,13 +123,13 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
By("updating the Service to use the standard tier with a requested IP")
svc = jig.UpdateServiceOrFail(ns, svc.Name, func(svc *v1.Service) {
svc.Spec.LoadBalancerIP = requestedIP
setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard)
setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard.ToGCEValue())
})
// Verify that service has been updated properly.
Expect(svc.Spec.LoadBalancerIP).To(Equal(requestedIP))
svcTier, err = gcecloud.GetServiceNetworkTier(svc)
Expect(err).NotTo(HaveOccurred())
Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard))
Expect(svcTier).To(Equal(cloud.NetworkTierStandard))
// Wait until the ingress IP changes and verifies the LB.
ingressIP = waitAndVerifyLBWithTier(jig, ns, svcName, ingressIP, createTimeout, lagTimeout)
@ -171,7 +172,7 @@ func waitAndVerifyLBWithTier(jig *framework.ServiceTestJig, ns, svcName, existin
return ingressIP
}
func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) {
func getLBNetworkTierByIP(ip string) (cloud.NetworkTier, error) {
var rule *computealpha.ForwardingRule
// Retry a few times to tolerate flakes.
err := wait.PollImmediate(5*time.Second, 15*time.Second, func() (bool, error) {
@ -185,7 +186,7 @@ func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) {
if err != nil {
return "", err
}
return gcecloud.NetworkTierGCEValueToType(rule.NetworkTier), nil
return cloud.NetworkTierGCEValueToType(rule.NetworkTier), nil
}
func getGCEForwardingRuleByIP(ip string) (*computealpha.ForwardingRule, error) {
@ -223,7 +224,7 @@ func clearNetworkTier(svc *v1.Service) {
// TODO: add retries if this turns out to be flaky.
// TODO(#51665): remove this helper function once Network Tiers becomes beta.
func reserveAlphaRegionalAddress(cloud *gcecloud.GCECloud, name string, netTier gcecloud.NetworkTier) (string, error) {
func reserveAlphaRegionalAddress(cloud *gcecloud.GCECloud, name string, netTier cloud.NetworkTier) (string, error) {
alphaAddr := &computealpha.Address{
Name: name,
NetworkTier: netTier.ToGCEValue(),