diff --git a/pkg/cloudprovider/providers/gce/gce_forwardingrule.go b/pkg/cloudprovider/providers/gce/gce_forwardingrule.go index 0473c44266d..c747ed1ddb8 100644 --- a/pkg/cloudprovider/providers/gce/gce_forwardingrule.go +++ b/pkg/cloudprovider/providers/gce/gce_forwardingrule.go @@ -99,6 +99,14 @@ func (gce *GCECloud) ListRegionForwardingRules(region string) (*compute.Forwardi return v, mc.Observe(err) } +// ListRegionForwardingRules lists all RegionalForwardingRules in the project & region. +func (gce *GCECloud) ListAlphaRegionForwardingRules(region string) (*computealpha.ForwardingRuleList, error) { + mc := newForwardingRuleMetricContextWithVersion("list", region, computeAlphaVersion) + // TODO: use PageToken to list all not just the first 500 + v, err := gce.serviceAlpha.ForwardingRules.List(gce.projectID, region).Do() + return v, mc.Observe(err) +} + // CreateRegionForwardingRule creates and returns a // RegionalForwardingRule that points to the given BackendService func (gce *GCECloud) CreateRegionForwardingRule(rule *compute.ForwardingRule, region string) error { diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index 66cbc3f3657..d25511f260c 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -483,6 +483,31 @@ func (j *ServiceTestJig) UpdateServiceOrFail(namespace, name string, update func return svc } +func (j *ServiceTestJig) WaitForNewIngressIPOrFail(namespace, name, existingIP string, timeout time.Duration) *v1.Service { + var service *v1.Service + Logf("Waiting up to %v for service %q to get a new ingress IP", timeout, name) + pollFunc := func() (bool, error) { + svc, err := j.Client.Core().Services(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if len(svc.Status.LoadBalancer.Ingress) == 0 { + return false, nil + } + ip := svc.Status.LoadBalancer.Ingress[0].IP + if ip == "" || ip == existingIP { + return false, nil + } + // Got a new IP. + service = svc + return true, nil + } + if err := wait.PollImmediate(Poll, timeout, pollFunc); err != nil { + Failf("Timeout waiting for service %q to have a new ingress IP", name) + } + return service +} + func (j *ServiceTestJig) ChangeServiceNodePortOrFail(namespace, name string, initial int) *v1.Service { var err error var service *v1.Service @@ -1421,3 +1446,10 @@ func EnableAndDisableInternalLB() (enable func(svc *v1.Service), disable func(sv return } + +func GetServiceLoadBalancerCreationTimeout(cs clientset.Interface) time.Duration { + if nodes := GetReadySchedulableNodesOrDie(cs); len(nodes.Items) > LargeClusterMinNodesNumber { + return LoadBalancerCreateTimeoutLarge + } + return LoadBalancerCreateTimeoutDefault +} diff --git a/test/e2e/network/BUILD b/test/e2e/network/BUILD index 6080e1ed433..4d658d76a3f 100644 --- a/test/e2e/network/BUILD +++ b/test/e2e/network/BUILD @@ -17,6 +17,7 @@ go_library( "ingress.go", "kube_proxy.go", "network_policy.go", + "network_tiers.go", "networking.go", "networking_perf.go", "no_snat.go", @@ -42,6 +43,7 @@ go_library( "//test/utils:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", + "//vendor/google.golang.org/api/compute/v0.alpha:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/rbac/v1beta1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/test/e2e/network/network_tiers.go b/test/e2e/network/network_tiers.go new file mode 100644 index 00000000000..5c2307a94ec --- /dev/null +++ b/test/e2e/network/network_tiers.go @@ -0,0 +1,165 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "fmt" + "time" + + computealpha "google.golang.org/api/compute/v0.alpha" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/cloudprovider" + gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = SIGDescribe("Services [Feature:GCEAlphaFeature][Slow]", func() { + f := framework.NewDefaultFramework("services") + + var cs clientset.Interface + var internalClientset internalclientset.Interface + serviceLBNames := []string{} + + BeforeEach(func() { + // This test suite requires the GCE environment. + framework.SkipUnlessProviderIs("gce") + cs = f.ClientSet + internalClientset = f.InternalClientset + }) + + AfterEach(func() { + if CurrentGinkgoTestDescription().Failed { + framework.DescribeSvc(f.Namespace.Name) + } + for _, lb := range serviceLBNames { + framework.Logf("cleaning gce resource for %s", lb) + framework.CleanupServiceGCEResources(cs, lb, framework.TestContext.CloudConfig.Zone) + } + //reset serviceLBNames + serviceLBNames = []string{} + }) + It("should be able to create and tear down a standard-tier load balancer [Slow]", func() { + lagTimeout := framework.LoadBalancerLagTimeoutDefault + createTimeout := framework.GetServiceLoadBalancerCreationTimeout(cs) + + svcName := "net-tiers-svc" + ns := f.Namespace.Name + jig := framework.NewServiceTestJig(cs, svcName) + + By("creating a pod to be part of the service " + svcName) + jig.RunOrFail(ns, nil) + + By("creating a Service of type LoadBalancer using the standard network tier") + svc := jig.CreateTCPServiceOrFail(ns, func(svc *v1.Service) { + svc.Spec.Type = v1.ServiceTypeLoadBalancer + setNetworkTier(svc, gcecloud.NetworkTierAnnotationStandard) + }) + // Record the LB name for test cleanup. + serviceLBNames = append(serviceLBNames, cloudprovider.GetLoadBalancerName(svc)) + + svc = jig.WaitForLoadBalancerOrFail(ns, svcName, createTimeout) + lbIngress := &svc.Status.LoadBalancer.Ingress[0] + svcPort := int(svc.Spec.Ports[0].Port) + ingressIP := framework.GetIngressPoint(lbIngress) + + By("running sanity and reachability checks") + jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer) + jig.TestReachableHTTP(ingressIP, svcPort, lagTimeout) + // Check the network tier of the forwarding rule. + netTier, err := getLBNetworkTierByIP(ingressIP) + Expect(err).NotTo(HaveOccurred(), "failed to get the network tier of the load balancer") + Expect(netTier).To(Equal(gcecloud.NetworkTierStandard)) + + By("updating the Service to use the premium (default) tier") + existingIP := ingressIP + svc = jig.UpdateServiceOrFail(ns, svcName, func(svc *v1.Service) { + clearNetworkTier(svc) + }) + // Wait until the ingress IP changes. Each tier has its own pool of + // IPs, so changing tiers implies changing IPs. + svc = jig.WaitForNewIngressIPOrFail(ns, svcName, existingIP, createTimeout) + lbIngress = &svc.Status.LoadBalancer.Ingress[0] + ingressIP = framework.GetIngressPoint(lbIngress) + + By("running sanity and reachability checks") + jig.SanityCheckService(svc, v1.ServiceTypeLoadBalancer) + jig.TestReachableHTTP(ingressIP, svcPort, lagTimeout) + // Check the network tier of the forwarding rule. + netTier, err = getLBNetworkTierByIP(ingressIP) + Expect(err).NotTo(HaveOccurred(), "failed to get the network tier of the load balancer") + Expect(netTier).To(Equal(gcecloud.NetworkTierPremium)) + + // TODO: Add tests for user-requested IPs. + }) +}) + +func getLBNetworkTierByIP(ip string) (gcecloud.NetworkTier, error) { + var rule *computealpha.ForwardingRule + // Retry a few times to tolerate flakes. + err := wait.PollImmediate(5*time.Second, 15*time.Second, func() (bool, error) { + obj, err := getGCEForwardingRuleByIP(ip) + if err != nil { + return false, err + } + rule = obj + return true, nil + }) + if err != nil { + return "", err + } + return gcecloud.NetworkTierGCEValueToType(rule.NetworkTier), nil +} + +func getGCEForwardingRuleByIP(ip string) (*computealpha.ForwardingRule, error) { + cloud, err := framework.GetGCECloud() + if err != nil { + return nil, err + } + ruleList, err := cloud.ListAlphaRegionForwardingRules(cloud.Region()) + if err != nil { + return nil, err + } + for _, rule := range ruleList.Items { + if rule.IPAddress == ip { + return rule, nil + } + } + return nil, fmt.Errorf("forwarding rule with ip %q not found", ip) +} + +func setNetworkTier(svc *v1.Service, tier string) { + key := gcecloud.NetworkTierAnnotationKey + if svc.ObjectMeta.Annotations == nil { + svc.ObjectMeta.Annotations = map[string]string{} + } + svc.ObjectMeta.Annotations[key] = tier +} + +func clearNetworkTier(svc *v1.Service) { + key := gcecloud.NetworkTierAnnotationKey + if svc.ObjectMeta.Annotations == nil { + return + } + delete(svc.ObjectMeta.Annotations, key) +}