diff --git a/pkg/cloudprovider/providers/gce/cloud/BUILD b/pkg/cloudprovider/providers/gce/cloud/BUILD index dc6fa75605b..fa46310777d 100644 --- a/pkg/cloudprovider/providers/gce/cloud/BUILD +++ b/pkg/cloudprovider/providers/gce/cloud/BUILD @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "constants.go", "doc.go", "gce_projects.go", "gen.go", diff --git a/pkg/cloudprovider/providers/gce/cloud/constants.go b/pkg/cloudprovider/providers/gce/cloud/constants.go new file mode 100644 index 00000000000..3dc6f423d56 --- /dev/null +++ b/pkg/cloudprovider/providers/gce/cloud/constants.go @@ -0,0 +1,48 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud + +import ( + "strings" +) + +type NetworkTier string + +const ( + NetworkTierStandard NetworkTier = "Standard" + NetworkTierPremium NetworkTier = "Premium" + NetworkTierDefault NetworkTier = NetworkTierPremium +) + +// ToGCEValue converts NetworkTier to a string that we can populate the +// NetworkTier field of GCE objects. +func (n NetworkTier) ToGCEValue() string { + return strings.ToUpper(string(n)) +} + +// NetworkTierGCEValueToType converts the value of the NetworkTier field of a +// GCE object to the NetworkTier type. +func NetworkTierGCEValueToType(s string) NetworkTier { + switch s { + case NetworkTierStandard.ToGCEValue(): + return NetworkTierStandard + case NetworkTierPremium.ToGCEValue(): + return NetworkTierPremium + default: + return NetworkTier(s) + } +} diff --git a/pkg/cloudprovider/providers/gce/gce_addresses.go b/pkg/cloudprovider/providers/gce/gce_addresses.go index af91fb14174..37e2f5e9a7b 100644 --- a/pkg/cloudprovider/providers/gce/gce_addresses.go +++ b/pkg/cloudprovider/providers/gce/gce_addresses.go @@ -26,6 +26,7 @@ import ( computebeta "google.golang.org/api/compute/v0.beta" compute "google.golang.org/api/compute/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta" ) @@ -150,7 +151,7 @@ func (gce *GCECloud) GetBetaRegionAddressByIP(region, ipAddress string) (*comput // TODO(#51665): retire this function once Network Tiers becomes Beta in GCP. func (gce *GCECloud) getNetworkTierFromAddress(name, region string) (string, error) { if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) { - return NetworkTierDefault.ToGCEValue(), nil + return cloud.NetworkTierDefault.ToGCEValue(), nil } addr, err := gce.GetAlphaRegionAddress(name, region) if err != nil { diff --git a/pkg/cloudprovider/providers/gce/gce_annotations.go b/pkg/cloudprovider/providers/gce/gce_annotations.go index 37671aeb2a9..ccc0f444118 100644 --- a/pkg/cloudprovider/providers/gce/gce_annotations.go +++ b/pkg/cloudprovider/providers/gce/gce_annotations.go @@ -18,15 +18,14 @@ package gce import ( "fmt" - "strings" "github.com/golang/glog" "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" ) type LoadBalancerType string -type NetworkTier string const ( // ServiceAnnotationLoadBalancerType is annotated on a service with type LoadBalancer @@ -49,12 +48,8 @@ const ( // network tier a GCP LB should use. The valid values are "Standard" and // "Premium" (default). NetworkTierAnnotationKey = "cloud.google.com/network-tier" - NetworkTierAnnotationStandard = "Standard" - NetworkTierAnnotationPremium = "Premium" - - NetworkTierStandard NetworkTier = NetworkTierAnnotationStandard - NetworkTierPremium NetworkTier = NetworkTierAnnotationPremium - NetworkTierDefault NetworkTier = NetworkTierPremium + NetworkTierAnnotationStandard = cloud.NetworkTierStandard + NetworkTierAnnotationPremium = cloud.NetworkTierPremium ) // GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled. @@ -97,38 +92,19 @@ func GetLoadBalancerAnnotationBackendShare(service *v1.Service) bool { // GetServiceNetworkTier returns the network tier of GCP load balancer // which should be assembled, and an error if the specified tier is not // supported. -func GetServiceNetworkTier(service *v1.Service) (NetworkTier, error) { +func GetServiceNetworkTier(service *v1.Service) (cloud.NetworkTier, error) { l, ok := service.Annotations[NetworkTierAnnotationKey] if !ok { - return NetworkTierDefault, nil + return cloud.NetworkTierDefault, nil } - v := NetworkTier(l) + v := cloud.NetworkTier(l) switch v { - case NetworkTierStandard: + case cloud.NetworkTierStandard: fallthrough - case NetworkTierPremium: + case cloud.NetworkTierPremium: return v, nil default: - return NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v) - } -} - -// ToGCEValue converts NetworkTier to a string that we can populate the -// NetworkTier field of GCE objects. -func (n NetworkTier) ToGCEValue() string { - return strings.ToUpper(string(n)) -} - -// NetworkTierGCEValueToType converts the value of the NetworkTier field of a -// GCE object to the NetworkTier type. -func NetworkTierGCEValueToType(s string) NetworkTier { - switch s { - case "STANDARD": - return NetworkTierStandard - case "PREMIUM": - return NetworkTierPremium - default: - return NetworkTier(s) + return cloud.NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v) } } diff --git a/pkg/cloudprovider/providers/gce/gce_annotations_test.go b/pkg/cloudprovider/providers/gce/gce_annotations_test.go index 62aad6f4180..a3b2dfb86bf 100644 --- a/pkg/cloudprovider/providers/gce/gce_annotations_test.go +++ b/pkg/cloudprovider/providers/gce/gce_annotations_test.go @@ -21,6 +21,7 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "github.com/stretchr/testify/assert" ) @@ -38,24 +39,24 @@ func TestServiceNetworkTierAnnotationKey(t *testing.T) { for testName, testCase := range map[string]struct { annotations map[string]string - expectedTier NetworkTier + expectedTier cloud.NetworkTier expectErr bool }{ "Use the default when the annotation does not exist": { annotations: nil, - expectedTier: NetworkTierDefault, + expectedTier: cloud.NetworkTierDefault, }, "Standard tier": { annotations: map[string]string{NetworkTierAnnotationKey: "Standard"}, - expectedTier: NetworkTierStandard, + expectedTier: cloud.NetworkTierStandard, }, "Premium tier": { annotations: map[string]string{NetworkTierAnnotationKey: "Premium"}, - expectedTier: NetworkTierPremium, + expectedTier: cloud.NetworkTierPremium, }, "Report an error on invalid network tier value": { annotations: map[string]string{NetworkTierAnnotationKey: "Unknown-tier"}, - expectedTier: NetworkTierPremium, + expectedTier: cloud.NetworkTierPremium, expectErr: true, }, } { diff --git a/pkg/cloudprovider/providers/gce/gce_forwardingrule.go b/pkg/cloudprovider/providers/gce/gce_forwardingrule.go index 57e7b85f3be..9fe0da9f7cc 100644 --- a/pkg/cloudprovider/providers/gce/gce_forwardingrule.go +++ b/pkg/cloudprovider/providers/gce/gce_forwardingrule.go @@ -21,6 +21,7 @@ import ( computealpha "google.golang.org/api/compute/v0.alpha" compute "google.golang.org/api/compute/v1" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/filter" "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud/meta" ) @@ -117,7 +118,7 @@ func (gce *GCECloud) DeleteRegionForwardingRule(name, region string) error { // TODO(#51665): retire this function once Network Tiers becomes Beta in GCP. func (gce *GCECloud) getNetworkTierFromForwardingRule(name, region string) (string, error) { if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) { - return NetworkTierDefault.ToGCEValue(), nil + return cloud.NetworkTierDefault.ToGCEValue(), nil } fwdRule, err := gce.GetAlphaRegionForwardingRule(name, region) if err != nil { diff --git a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go index 8cce4e2fe51..4fd60b969da 100644 --- a/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go +++ b/pkg/cloudprovider/providers/gce/gce_loadbalancer_external.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" apiservice "k8s.io/kubernetes/pkg/api/v1/service" "k8s.io/kubernetes/pkg/cloudprovider" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" netsets "k8s.io/kubernetes/pkg/util/net/sets" "github.com/golang/glog" @@ -416,7 +417,7 @@ func (gce *GCECloud) DeleteExternalTargetPoolAndChecks(service *v1.Service, name // the verification failed. It also returns a boolean to indicate whether the // IP address is considered owned by the user (i.e., not managed by the // controller. -func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string, desiredNetTier NetworkTier) (isUserOwnedIP bool, err error) { +func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP, lbRef string, desiredNetTier cloud.NetworkTier) (isUserOwnedIP bool, err error) { if requestedIP == "" { return false, nil } @@ -439,7 +440,7 @@ func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP if err != nil { return false, fmt.Errorf("failed to check the network tier of the IP %q: %v", requestedIP, err) } - netTier := NetworkTierGCEValueToType(netTierStr) + netTier := cloud.NetworkTierGCEValueToType(netTierStr) if netTier != desiredNetTier { glog.Errorf("verifyUserRequestedIP: requested static IP %q (name: %s) for LB %s has network tier %s, need %s.", requestedIP, existingAddress.Name, lbRef, netTier, desiredNetTier) return false, fmt.Errorf("requrested IP %q belongs to the %s network tier; expected %s", requestedIP, netTier, desiredNetTier) @@ -852,7 +853,7 @@ func (gce *GCECloud) ensureHttpHealthCheckFirewall(svc *v1.Service, serviceName, return nil } -func createForwardingRule(s CloudForwardingRuleService, name, serviceName, region, ipAddress, target string, ports []v1.ServicePort, netTier NetworkTier) error { +func createForwardingRule(s CloudForwardingRuleService, name, serviceName, region, ipAddress, target string, ports []v1.ServicePort, netTier cloud.NetworkTier) error { portRange, err := loadBalancerPortRange(ports) if err != nil { return err @@ -861,7 +862,7 @@ func createForwardingRule(s CloudForwardingRuleService, name, serviceName, regio ipProtocol := string(ports[0].Protocol) switch netTier { - case NetworkTierPremium: + case cloud.NetworkTierPremium: rule := &compute.ForwardingRule{ Name: name, Description: desc, @@ -964,7 +965,7 @@ func (gce *GCECloud) firewallObject(name, region, desc string, sourceRanges nets return firewall, nil } -func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string, netTier NetworkTier) (ipAddress string, existing bool, err error) { +func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP string, netTier cloud.NetworkTier) (ipAddress string, existing bool, err error) { // If the address doesn't exist, this will create it. // If the existingIP exists but is ephemeral, this will promote it to static. // If the address already exists, this will harmlessly return a StatusConflict @@ -974,7 +975,7 @@ func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP var creationErr error switch netTier { - case NetworkTierPremium: + case cloud.NetworkTierPremium: addressObj := &compute.Address{ Name: name, Description: desc, @@ -1012,19 +1013,19 @@ func ensureStaticIP(s CloudAddressService, name, serviceName, region, existingIP return addr.Address, existed, nil } -func (gce *GCECloud) getServiceNetworkTier(svc *v1.Service) (NetworkTier, error) { +func (gce *GCECloud) getServiceNetworkTier(svc *v1.Service) (cloud.NetworkTier, error) { if !gce.AlphaFeatureGate.Enabled(AlphaFeatureNetworkTiers) { - return NetworkTierDefault, nil + return cloud.NetworkTierDefault, nil } tier, err := GetServiceNetworkTier(svc) if err != nil { // Returns an error if the annotation is invalid. - return NetworkTier(""), err + return cloud.NetworkTier(""), err } return tier, nil } -func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, desiredNetTier NetworkTier) error { +func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, desiredNetTier cloud.NetworkTier) error { logPrefix := fmt.Sprintf("deleteWrongNetworkTieredResources:(%s)", lbRef) if err := deleteFWDRuleWithWrongTier(gce, gce.region, lbName, logPrefix, desiredNetTier); err != nil { return err @@ -1037,14 +1038,14 @@ func (gce *GCECloud) deleteWrongNetworkTieredResources(lbName, lbRef string, des // deleteFWDRuleWithWrongTier checks the network tier of existing forwarding // rule and delete the rule if the tier does not matched the desired tier. -func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logPrefix string, desiredNetTier NetworkTier) error { +func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logPrefix string, desiredNetTier cloud.NetworkTier) error { tierStr, err := s.getNetworkTierFromForwardingRule(name, region) if isNotFound(err) { return nil } else if err != nil { return err } - existingTier := NetworkTierGCEValueToType(tierStr) + existingTier := cloud.NetworkTierGCEValueToType(tierStr) if existingTier == desiredNetTier { return nil } @@ -1056,7 +1057,7 @@ func deleteFWDRuleWithWrongTier(s CloudForwardingRuleService, region, name, logP // deleteAddressWithWrongTier checks the network tier of existing address // and delete the address if the tier does not matched the desired tier. -func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix string, desiredNetTier NetworkTier) error { +func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix string, desiredNetTier cloud.NetworkTier) error { // We only check the IP address matching the reserved name that the // controller assigned to the LB. We make the assumption that an address of // such name is owned by the controller and is safe to release. Whether an @@ -1072,7 +1073,7 @@ func deleteAddressWithWrongTier(s CloudAddressService, region, name, logPrefix s } else if err != nil { return err } - existingTier := NetworkTierGCEValueToType(tierStr) + existingTier := cloud.NetworkTierGCEValueToType(tierStr) if existingTier == desiredNetTier { return nil } diff --git a/pkg/cloudprovider/providers/gce/gce_util.go b/pkg/cloudprovider/providers/gce/gce_util.go index 6b31c4fa1c2..363b5173abe 100644 --- a/pkg/cloudprovider/providers/gce/gce_util.go +++ b/pkg/cloudprovider/providers/gce/gce_util.go @@ -28,6 +28,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "cloud.google.com/go/compute/metadata" compute "google.golang.org/api/compute/v1" @@ -220,7 +221,7 @@ func handleAlphaNetworkTierGetError(err error) (string, error) { // Network tier is still an Alpha feature in GCP, and not every project // is whitelisted to access the API. If we cannot access the API, just // assume the tier is premium. - return NetworkTierDefault.ToGCEValue(), nil + return cloud.NetworkTierDefault.ToGCEValue(), nil } // Can't get the network tier, just return an error. return "", err diff --git a/test/e2e/network/BUILD b/test/e2e/network/BUILD index 3695b6691dc..4000cee776e 100644 --- a/test/e2e/network/BUILD +++ b/test/e2e/network/BUILD @@ -36,6 +36,7 @@ go_library( "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider/providers/gce:go_default_library", + "//pkg/cloudprovider/providers/gce/cloud:go_default_library", "//pkg/controller/endpoint:go_default_library", "//pkg/kubelet/apis:go_default_library", "//pkg/master/ports:go_default_library", diff --git a/test/e2e/network/network_tiers.go b/test/e2e/network/network_tiers.go index ec284e8c252..9e66a3a1e8a 100644 --- a/test/e2e/network/network_tiers.go +++ b/test/e2e/network/network_tiers.go @@ -29,6 +29,7 @@ import ( "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/cloudprovider" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" + "k8s.io/kubernetes/pkg/cloudprovider/providers/gce/cloud" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" @@ -75,12 +76,12 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() { By("creating a Service of type LoadBalancer using the standard network tier") svc := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer - setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard) + setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard.ToGCEValue()) }) // Verify that service has been updated properly. svcTier, err := gcecloud.GetServiceNetworkTier(svc) Expect(err).NotTo(HaveOccurred()) - Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard)) + Expect(svcTier).To(Equal(cloud.NetworkTierStandard)) // Record the LB name for test cleanup. serviceLBNames = append(serviceLBNames, cloudprovider.GetLoadBalancerName(svc)) @@ -95,7 +96,7 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() { // Verify that service has been updated properly. svcTier, err = gcecloud.GetServiceNetworkTier(svc) Expect(err).NotTo(HaveOccurred()) - Expect(svcTier).To(Equal(gcecloud.NetworkTierDefault)) + Expect(svcTier).To(Equal(cloud.NetworkTierDefault)) // Wait until the ingress IP changes. Each tier has its own pool of // IPs, so changing tiers implies changing IPs. @@ -106,7 +107,7 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() { requestedAddrName := fmt.Sprintf("e2e-ext-lb-net-tier-%s", framework.RunId) gceCloud, err := framework.GetGCECloud() Expect(err).NotTo(HaveOccurred()) - requestedIP, err := reserveAlphaRegionalAddress(gceCloud, requestedAddrName, gcecloud.NetworkTierStandard) + requestedIP, err := reserveAlphaRegionalAddress(gceCloud, requestedAddrName, cloud.NetworkTierStandard) Expect(err).NotTo(HaveOccurred(), "failed to reserve a STANDARD tiered address") defer func() { if requestedAddrName != "" { @@ -122,13 +123,13 @@ var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() { By("updating the Service to use the standard tier with a requested IP") svc = jig.UpdateServiceOrFail(ns, svc.Name, func(svc *v1.Service) { svc.Spec.LoadBalancerIP = requestedIP - setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard) + setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard.ToGCEValue()) }) // Verify that service has been updated properly. Expect(svc.Spec.LoadBalancerIP).To(Equal(requestedIP)) svcTier, err = gcecloud.GetServiceNetworkTier(svc) Expect(err).NotTo(HaveOccurred()) - Expect(svcTier).To(Equal(gcecloud.NetworkTierStandard)) + Expect(svcTier).To(Equal(cloud.NetworkTierStandard)) // Wait until the ingress IP changes and verifies the LB. ingressIP = waitAndVerifyLBWithTier(jig, ns, svcName, ingressIP, createTimeout, lagTimeout) @@ -171,7 +172,7 @@ func waitAndVerifyLBWithTier(jig *framework.ServiceTestJig, ns, svcName, existin return ingressIP } -func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) { +func getLBNetworkTierByIP(ip string) (cloud.NetworkTier, error) { var rule *computealpha.ForwardingRule // Retry a few times to tolerate flakes. err := wait.PollImmediate(5*time.Second, 15*time.Second, func() (bool, error) { @@ -185,7 +186,7 @@ func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) { if err != nil { return "", err } - return gcecloud.NetworkTierGCEValueToType(rule.NetworkTier), nil + return cloud.NetworkTierGCEValueToType(rule.NetworkTier), nil } func getGCEForwardingRuleByIP(ip string) (*computealpha.ForwardingRule, error) { @@ -223,7 +224,7 @@ func clearNetworkTier(svc *v1.Service) { // TODO: add retries if this turns out to be flaky. // TODO(#51665): remove this helper function once Network Tiers becomes beta. -func reserveAlphaRegionalAddress(cloud *gcecloud.GCECloud, name string, netTier gcecloud.NetworkTier) (string, error) { +func reserveAlphaRegionalAddress(cloud *gcecloud.GCECloud, name string, netTier cloud.NetworkTier) (string, error) { alphaAddr := &computealpha.Address{ Name: name, NetworkTier: netTier.ToGCEValue(),