Merge pull request #51483 from yujuhong/e2e-net-tiers

Automatic merge from submit-queue

e2e: Add tests for network tiers in GCE

This test depends on #51301, which adds the new feature. Only the `e2e: Add tests for network tiers in GCE` commit is new.
#51301 should pass this new test.

pkg/cloudprovider/providers/gce/BUILD

@@ -83,6 +83,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "gce_annotations_test.go",
         "gce_disks_test.go",
         "gce_healthchecks_test.go",
         "gce_loadbalancer_external_test.go",
@@ -100,6 +101,7 @@ go_test(
         "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/google.golang.org/api/googleapi:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
     ],
 )
 

pkg/cloudprovider/providers/gce/gce_annotations.go

@@ -17,11 +17,16 @@ limitations under the License.
 package gce
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/golang/glog"
+
 	"k8s.io/api/core/v1"
 )
 
 type LoadBalancerType string
+type NetworkTier string
 
 const (
 	// ServiceAnnotationLoadBalancerType is annotated on a service with type LoadBalancer
@@ -39,6 +44,17 @@ const (
 	ServiceAnnotationILBBackendShare = "alpha.cloud.google.com/load-balancer-backend-share"
 	// This annotation did not correctly specify "alpha", so both annotations will be checked.
 	deprecatedServiceAnnotationILBBackendShare = "cloud.google.com/load-balancer-backend-share"
+
+	// NetworkTierAnnotationKey is annotated on a Service object to indicate which
+	// network tier a GCP LB should use. The valid values are "Standard" and
+	// "Premium" (default).
+	NetworkTierAnnotationKey      = "cloud.google.com/network-tier"
+	NetworkTierAnnotationStandard = "Standard"
+	NetworkTierAnnotationPremium  = "Premium"
+
+	NetworkTierStandard NetworkTier = NetworkTierAnnotationStandard
+	NetworkTierPremium  NetworkTier = NetworkTierAnnotationPremium
+	NetworkTierDefault  NetworkTier = NetworkTierPremium
 )
 
 // GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled.
@@ -77,3 +93,42 @@ func GetLoadBalancerAnnotationBackendShare(service *v1.Service) bool {
 
 	return false
 }
+
+// GetServiceNetworkTier returns the network tier of GCP load balancer
+// which should be assembled, and an error if the specified tier is not
+// supported.
+func GetServiceNetworkTier(service *v1.Service) (NetworkTier, error) {
+	l, ok := service.Annotations[NetworkTierAnnotationKey]
+	if !ok {
+		return NetworkTierDefault, nil
+	}
+
+	v := NetworkTier(l)
+	switch v {
+	case NetworkTierStandard:
+		fallthrough
+	case NetworkTierPremium:
+		return v, nil
+	default:
+		return NetworkTierDefault, fmt.Errorf("unsupported network tier: %q", v)
+	}
+}
+
+// ToGCEValue converts NetworkTier to a string that we can populate the
+// NetworkTier field of GCE objects.
+func (n NetworkTier) ToGCEValue() string {
+	return strings.ToUpper(string(n))
+}
+
+// NetworkTierGCEValueToType converts the value of the NetworkTier field of a
+// GCE object to the NetworkTier type.
+func NetworkTierGCEValueToType(s string) NetworkTier {
+	switch s {
+	case "STANDARD":
+		return NetworkTierStandard
+	case "PREMIUM":
+		return NetworkTierPremium
+	default:
+		return NetworkTier(s)
+	}
+}

pkg/cloudprovider/providers/gce/gce_annotations_test.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestServiceNetworkTierAnnotationKey(t *testing.T) {
+	createTestService := func() *v1.Service {
+		return &v1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				UID:       "randome-uid",
+				Name:      "test-svc",
+				Namespace: "test-ns",
+			},
+		}
+	}
+
+	for testName, testCase := range map[string]struct {
+		annotations  map[string]string
+		expectedTier NetworkTier
+		expectErr    bool
+	}{
+		"Use the default when the annotation does not exist": {
+			annotations:  nil,
+			expectedTier: NetworkTierDefault,
+		},
+		"Standard tier": {
+			annotations:  map[string]string{NetworkTierAnnotationKey: "Standard"},
+			expectedTier: NetworkTierStandard,
+		},
+		"Premium tier": {
+			annotations:  map[string]string{NetworkTierAnnotationKey: "Premium"},
+			expectedTier: NetworkTierPremium,
+		},
+		"Report an error on invalid network tier value": {
+			annotations:  map[string]string{NetworkTierAnnotationKey: "Unknown-tier"},
+			expectedTier: NetworkTierPremium,
+			expectErr:    true,
+		},
+	} {
+		t.Run(testName, func(t *testing.T) {
+			svc := createTestService()
+			svc.Annotations = testCase.annotations
+			actualTier, err := GetServiceNetworkTier(svc)
+			assert.Equal(t, testCase.expectedTier, actualTier)
+			assert.Equal(t, testCase.expectErr, err != nil)
+		})
+	}
+}

pkg/cloudprovider/providers/gce/gce_forwardingrule.go

@@ -99,6 +99,14 @@ func (gce *GCECloud) ListRegionForwardingRules(region string) (*compute.Forwardi
 	return v, mc.Observe(err)
 }
 
+// ListRegionForwardingRules lists all RegionalForwardingRules in the project & region.
+func (gce *GCECloud) ListAlphaRegionForwardingRules(region string) (*computealpha.ForwardingRuleList, error) {
+	mc := newForwardingRuleMetricContextWithVersion("list", region, computeAlphaVersion)
+	// TODO: use PageToken to list all not just the first 500
+	v, err := gce.serviceAlpha.ForwardingRules.List(gce.projectID, region).Do()
+	return v, mc.Observe(err)
+}
+
 // CreateRegionForwardingRule creates and returns a
 // RegionalForwardingRule that points to the given BackendService
 func (gce *GCECloud) CreateRegionForwardingRule(rule *compute.ForwardingRule, region string) error {

test/e2e/framework/service_util.go

@@ -483,6 +483,31 @@ func (j *ServiceTestJig) UpdateServiceOrFail(namespace, name string, update func
 	return svc
 }
 
+func (j *ServiceTestJig) WaitForNewIngressIPOrFail(namespace, name, existingIP string, timeout time.Duration) *v1.Service {
+	var service *v1.Service
+	Logf("Waiting up to %v for service %q to get a new ingress IP", timeout, name)
+	pollFunc := func() (bool, error) {
+		svc, err := j.Client.Core().Services(namespace).Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		if len(svc.Status.LoadBalancer.Ingress) == 0 {
+			return false, nil
+		}
+		ip := svc.Status.LoadBalancer.Ingress[0].IP
+		if ip == "" || ip == existingIP {
+			return false, nil
+		}
+		// Got a new IP.
+		service = svc
+		return true, nil
+	}
+	if err := wait.PollImmediate(Poll, timeout, pollFunc); err != nil {
+		Failf("Timeout waiting for service %q to have a new ingress IP", name)
+	}
+	return service
+}
+
 func (j *ServiceTestJig) ChangeServiceNodePortOrFail(namespace, name string, initial int) *v1.Service {
 	var err error
 	var service *v1.Service
@@ -1421,3 +1446,10 @@ func EnableAndDisableInternalLB() (enable func(svc *v1.Service), disable func(sv
 
 	return
 }
+
+func GetServiceLoadBalancerCreationTimeout(cs clientset.Interface) time.Duration {
+	if nodes := GetReadySchedulableNodesOrDie(cs); len(nodes.Items) > LargeClusterMinNodesNumber {
+		return LoadBalancerCreateTimeoutLarge
+	}
+	return LoadBalancerCreateTimeoutDefault
+}

test/e2e/network/BUILD

@@ -17,6 +17,7 @@ go_library(
         "ingress.go",
         "kube_proxy.go",
         "network_policy.go",
+        "network_tiers.go",
         "networking.go",
         "networking_perf.go",
         "no_snat.go",
@@ -42,6 +43,7 @@ go_library(
         "//test/utils:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/google.golang.org/api/compute/v0.alpha:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/rbac/v1beta1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",

test/e2e/network/network_tiers.go

@@ -0,0 +1,165 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+import (
+	"fmt"
+	"time"
+
+	computealpha "google.golang.org/api/compute/v0.alpha"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() {
+	f := framework.NewDefaultFramework("services")
+
+	var cs clientset.Interface
+	var internalClientset internalclientset.Interface
+	serviceLBNames := []string{}
+
+	BeforeEach(func() {
+		// This test suite requires the GCE environment.
+		framework.SkipUnlessProviderIs("gce")
+		cs = f.ClientSet
+		internalClientset = f.InternalClientset
+	})
+
+	AfterEach(func() {
+		if CurrentGinkgoTestDescription().Failed {
+			framework.DescribeSvc(f.Namespace.Name)
+		}
+		for _, lb := range serviceLBNames {
+			framework.Logf("cleaning gce resource for %s", lb)
+			framework.CleanupServiceGCEResources(cs, lb, framework.TestContext.CloudConfig.Zone)
+		}
+		//reset serviceLBNames
+		serviceLBNames = []string{}
+	})
+	It("should be able to create and tear down a standard-tier load balancer [Slow]", func() {
+		lagTimeout := framework.LoadBalancerLagTimeoutDefault
+		createTimeout := framework.GetServiceLoadBalancerCreationTimeout(cs)
+
+		svcName := "net-tiers-svc"
+		ns := f.Namespace.Name
+		jig := framework.NewServiceTestJig(cs, svcName)
+
+		By("creating a pod to be part of the service " + svcName)
+		jig.RunOrFail(ns, nil)
+
+		By("creating a Service of type LoadBalancer using the standard network tier")
+		svc := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) {
+			svc.Spec.Type = v1.ServiceTypeLoadBalancer
+			setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard)
+		})
+		// Record the LB name for test cleanup.
+		serviceLBNames = append(serviceLBNames, cloudprovider.GetLoadBalancerName(svc))
+
+		svc = jig.WaitForLoadBalancerOrFail(ns, svcName, createTimeout)
+		lbIngress := &svc.Status.LoadBalancer.Ingress[0]
+		svcPort := int(svc.Spec.Ports[0].Port)
+		ingressIP := framework.GetIngressPoint(lbIngress)
+
+		By("running sanity and reachability checks")
+		jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
+		jig.TestReachableHTTP(ingressIP, svcPort, lagTimeout)
+		// Check the network tier of the forwarding rule.
+		netTier, err := getLBNetworkTierByIP(ingressIP)
+		Expect(err).NotTo(HaveOccurred(), "failed to get the network tier of the load balancer")
+		Expect(netTier).To(Equal(gcecloud.NetworkTierStandard))
+
+		By("updating the Service to use the premium (default) tier")
+		existingIP := ingressIP
+		svc = jig.UpdateServiceOrFail(ns, svcName, func(svc *v1.Service) {
+			clearNetworkTier(svc)
+		})
+		// Wait until the ingress IP changes. Each tier has its own pool of
+		// IPs, so changing tiers implies changing IPs.
+		svc = jig.WaitForNewIngressIPOrFail(ns, svcName, existingIP, createTimeout)
+		lbIngress = &svc.Status.LoadBalancer.Ingress[0]
+		ingressIP = framework.GetIngressPoint(lbIngress)
+
+		By("running sanity and reachability checks")
+		jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
+		jig.TestReachableHTTP(ingressIP, svcPort, lagTimeout)
+		// Check the network tier of the forwarding rule.
+		netTier, err = getLBNetworkTierByIP(ingressIP)
+		Expect(err).NotTo(HaveOccurred(), "failed to get the network tier of the load balancer")
+		Expect(netTier).To(Equal(gcecloud.NetworkTierPremium))
+
+		// TODO: Add tests for user-requested IPs.
+	})
+})
+
+func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) {
+	var rule *computealpha.ForwardingRule
+	// Retry a few times to tolerate flakes.
+	err := wait.PollImmediate(5*time.Second, 15*time.Second, func() (bool, error) {
+		obj, err := getGCEForwardingRuleByIP(ip)
+		if err != nil {
+			return false, err
+		}
+		rule = obj
+		return true, nil
+	})
+	if err != nil {
+		return "", err
+	}
+	return gcecloud.NetworkTierGCEValueToType(rule.NetworkTier), nil
+}
+
+func getGCEForwardingRuleByIP(ip string) (*computealpha.ForwardingRule, error) {
+	cloud, err := framework.GetGCECloud()
+	if err != nil {
+		return nil, err
+	}
+	ruleList, err := cloud.ListAlphaRegionForwardingRules(cloud.Region())
+	if err != nil {
+		return nil, err
+	}
+	for _, rule := range ruleList.Items {
+		if rule.IPAddress == ip {
+			return rule, nil
+		}
+	}
+	return nil, fmt.Errorf("forwarding rule with ip %q not found", ip)
+}
+
+func setNetworkTier(svc *v1.Service, tier string) {
+	key := gcecloud.NetworkTierAnnotationKey
+	if svc.ObjectMeta.Annotations == nil {
+		svc.ObjectMeta.Annotations = map[string]string{}
+	}
+	svc.ObjectMeta.Annotations[key] = tier
+}
+
+func clearNetworkTier(svc *v1.Service) {
+	key := gcecloud.NetworkTierAnnotationKey
+	if svc.ObjectMeta.Annotations == nil {
+		return
+	}
+	delete(svc.ObjectMeta.Annotations, key)
+}