diff --git a/test/e2e/firewall.go b/test/e2e/firewall.go index d6640e18084..0030dfa57a5 100644 --- a/test/e2e/firewall.go +++ b/test/e2e/firewall.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/api/v1" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/pkg/cloudprovider" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" "k8s.io/kubernetes/pkg/master/ports" "k8s.io/kubernetes/test/e2e/framework" @@ -45,13 +46,18 @@ var _ = framework.KubeDescribe("Firewall rule", func() { gceCloud = cloudConfig.Provider.(*gcecloud.GCECloud) }) - // This test takes around 4 minutes to run + // This test takes around 6 minutes to run It("[Slow] [Serial] should create valid firewall rules for LoadBalancer type service", func() { ns := f.Namespace.Name // This source ranges is just used to examine we have exact same things on LB firewall rules firewallTestSourceRanges := []string{"0.0.0.0/1", "128.0.0.0/1"} serviceName := "firewall-test-loadbalancer" + By("Getting cluster ID") + clusterID, err := framework.GetClusterID(cs) + Expect(err).NotTo(HaveOccurred()) + framework.Logf("Got cluster ID: %v", clusterID) + jig := framework.NewServiceTestJig(cs, serviceName) nodesNames := jig.GetNodesNames(framework.MaxNodesForEndpointsTests) if len(nodesNames) <= 0 { @@ -59,28 +65,52 @@ var _ = framework.KubeDescribe("Firewall rule", func() { } nodesSet := sets.NewString(nodesNames...) - // OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE - By("Creating a LoadBalancer type service with ExternalTrafficPolicy=Local") - svc := jig.CreateOnlyLocalLoadBalancerService(ns, serviceName, - framework.LoadBalancerCreateTimeoutDefault, false, func(svc *v1.Service) { - svc.Spec.Ports = []v1.ServicePort{{Protocol: "TCP", Port: framework.FirewallTestHttpPort}} - svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges - }) + By("Creating a LoadBalancer type service with ExternalTrafficPolicy=Global") + svc := jig.CreateLoadBalancerService(ns, serviceName, framework.LoadBalancerCreateTimeoutDefault, func(svc *v1.Service) { + svc.Spec.Ports = []v1.ServicePort{{Protocol: "TCP", Port: framework.FirewallTestHttpPort}} + svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges + }) defer func() { jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeNodePort svc.Spec.LoadBalancerSourceRanges = nil }) Expect(cs.Core().Services(svc.Namespace).Delete(svc.Name, nil)).NotTo(HaveOccurred()) + By("Waiting for the local traffic health check firewall rule to be deleted") + localHCFwName := framework.MakeHealthCheckFirewallNameForLBService(clusterID, cloudprovider.GetLoadBalancerName(svc), false) + _, err := framework.WaitForFirewallRule(gceCloud, localHCFwName, false, framework.LoadBalancerCleanupTimeout) + Expect(err).NotTo(HaveOccurred()) }() svcExternalIP := svc.Status.LoadBalancer.Ingress[0].IP - By("Checking if service's firewall rules are correct") + By("Checking if service's firewall rule is correct") nodeTags := framework.GetInstanceTags(cloudConfig, nodesNames[0]) - expFw := framework.ConstructFirewallForLBService(svc, nodeTags.Items) - fw, err := gceCloud.GetFirewall(expFw.Name) + lbFw := framework.ConstructFirewallForLBService(svc, nodeTags.Items) + fw, err := gceCloud.GetFirewall(lbFw.Name) Expect(err).NotTo(HaveOccurred()) - Expect(framework.VerifyFirewallRule(fw, expFw, cloudConfig.Network, false)).NotTo(HaveOccurred()) + Expect(framework.VerifyFirewallRule(fw, lbFw, cloudConfig.Network, false)).NotTo(HaveOccurred()) + + By("Checking if service's nodes health check firewall rule is correct") + nodesHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, nodeTags.Items, true) + fw, err = gceCloud.GetFirewall(nodesHCFw.Name) + Expect(err).NotTo(HaveOccurred()) + Expect(framework.VerifyFirewallRule(fw, nodesHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred()) + + // OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE + By("Updating LoadBalancer service to ExternalTrafficPolicy=Local") + svc = jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) { + svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal + }) + + By("Waiting for the nodes health check firewall rule to be deleted") + _, err = framework.WaitForFirewallRule(gceCloud, nodesHCFw.Name, false, framework.LoadBalancerCleanupTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the correct local traffic health check firewall rule to be created") + localHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, nodeTags.Items, false) + fw, err = framework.WaitForFirewallRule(gceCloud, localHCFw.Name, true, framework.LoadBalancerCreateTimeoutDefault) + Expect(err).NotTo(HaveOccurred()) + Expect(framework.VerifyFirewallRule(fw, localHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred()) By(fmt.Sprintf("Creating netexec pods on at most %v nodes", framework.MaxNodesForEndpointsTests)) for i, nodeName := range nodesNames { @@ -100,7 +130,7 @@ var _ = framework.KubeDescribe("Firewall rule", func() { // by removing the tag on one vm and make sure it doesn't get any traffic. This is an imperfect // simulation, we really want to check that traffic doesn't reach a vm outside the GKE cluster, but // that's much harder to do in the current e2e framework. - By("Removing tags from one of the nodes") + By(fmt.Sprintf("Removing tags from one of the nodes: %v", nodesNames[0])) nodesSet.Delete(nodesNames[0]) removedTags := framework.SetInstanceTags(cloudConfig, nodesNames[0], []string{}) defer func() { diff --git a/test/e2e/framework/firewall_util.go b/test/e2e/framework/firewall_util.go index fe6ede0a57d..2de5860777a 100644 --- a/test/e2e/framework/firewall_util.go +++ b/test/e2e/framework/firewall_util.go @@ -18,12 +18,16 @@ package framework import ( "fmt" + "net/http" "strconv" "strings" "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/api/v1" + apiservice "k8s.io/kubernetes/pkg/api/v1/service" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/cloudprovider" gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce" @@ -41,7 +45,7 @@ const ( ) // MakeFirewallNameForLBService return the expected firewall name for a LB service. -// This should match the formatting of makeFirewallName() in pkg/cloudprovider/providers/gce/gce.go +// This should match the formatting of makeFirewallName() in pkg/cloudprovider/providers/gce/gce_loadbalancer.go func MakeFirewallNameForLBService(name string) string { return fmt.Sprintf("k8s-fw-%s", name) } @@ -68,6 +72,32 @@ func ConstructFirewallForLBService(svc *v1.Service, nodesTags []string) *compute return &fw } +func MakeHealthCheckFirewallNameForLBService(clusterID, name string, isNodesHealthCheck bool) string { + return gcecloud.MakeHealthCheckFirewallName(clusterID, name, isNodesHealthCheck) +} + +// ConstructHealthCheckFirewallForLBService returns the expected GCE firewall rule for a loadbalancer type service +func ConstructHealthCheckFirewallForLBService(clusterID string, svc *v1.Service, nodesTags []string, isNodesHealthCheck bool) *compute.Firewall { + if svc.Spec.Type != v1.ServiceTypeLoadBalancer { + Failf("can not construct firewall rule for non-loadbalancer type service") + } + fw := compute.Firewall{} + fw.Name = MakeHealthCheckFirewallNameForLBService(clusterID, cloudprovider.GetLoadBalancerName(svc), isNodesHealthCheck) + fw.TargetTags = nodesTags + fw.SourceRanges = gcecloud.LoadBalancerSrcRanges() + healthCheckPort := gcecloud.GetNodesHealthCheckPort() + if !isNodesHealthCheck { + healthCheckPort = apiservice.GetServiceHealthCheckNodePort(svc) + } + fw.Allowed = []*compute.FirewallAllowed{ + { + IPProtocol: "tcp", + Ports: []string{fmt.Sprintf("%d", healthCheckPort)}, + }, + } + return &fw +} + // GetNodeTags gets tags from one of the Kubernetes nodes func GetNodeTags(c clientset.Interface, cloudConfig CloudConfig) *compute.Tags { nodes := GetReadySchedulableNodesOrDie(c) @@ -303,6 +333,9 @@ func SameStringArray(result, expected []string, include bool) error { // VerifyFirewallRule verifies whether the result firewall is consistent with the expected firewall. // When `portsSubset` is false, match given ports exactly. Otherwise, only check ports are included. func VerifyFirewallRule(res, exp *compute.Firewall, network string, portsSubset bool) error { + if res == nil || exp == nil { + return fmt.Errorf("res and exp must not be nil") + } if res.Name != exp.Name { return fmt.Errorf("incorrect name: %v, expected %v", res.Name, exp.Name) } @@ -325,3 +358,40 @@ func VerifyFirewallRule(res, exp *compute.Firewall, network string, portsSubset } return nil } + +func WaitForFirewallRule(gceCloud *gcecloud.GCECloud, fwName string, exist bool, timeout time.Duration) (*compute.Firewall, error) { + Logf("Waiting up to %v for firewall %v exist=%v", timeout, fwName, exist) + var fw *compute.Firewall + var err error + + condition := func() (bool, error) { + fw, err = gceCloud.GetFirewall(fwName) + if err != nil && exist || + err == nil && !exist || + err != nil && !exist && !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { + return false, nil + } + return true, nil + } + + if err := wait.PollImmediate(5*time.Second, timeout, condition); err != nil { + return nil, fmt.Errorf("error waiting for firewall %v exist=%v", fwName, exist) + } + return fw, nil +} + +func GetClusterID(c clientset.Interface) (string, error) { + cm, err := c.Core().ConfigMaps(metav1.NamespaceSystem).Get(gcecloud.UIDConfigMapName, metav1.GetOptions{}) + if err != nil || cm == nil { + return "", fmt.Errorf("error getting cluster ID: %v", err) + } + clusterID, clusterIDExists := cm.Data[gcecloud.UIDCluster] + providerID, providerIDExists := cm.Data[gcecloud.UIDProvider] + if !clusterIDExists { + return "", fmt.Errorf("cluster ID not set") + } + if providerIDExists { + return providerID, nil + } + return clusterID, nil +} diff --git a/test/e2e/framework/service_util.go b/test/e2e/framework/service_util.go index e91cff78b07..238b7ae1e12 100644 --- a/test/e2e/framework/service_util.go +++ b/test/e2e/framework/service_util.go @@ -235,6 +235,25 @@ func (j *ServiceTestJig) CreateOnlyLocalLoadBalancerService(namespace, serviceNa return svc } +// CreateLoadBalancerService creates a loadbalancer service and waits +// for it to acquire an ingress IP. +func (j *ServiceTestJig) CreateLoadBalancerService(namespace, serviceName string, timeout time.Duration, tweak func(svc *v1.Service)) *v1.Service { + By("creating a service " + namespace + "/" + serviceName + " with type=LoadBalancer") + svc := j.CreateTCPServiceOrFail(namespace, func(svc *v1.Service) { + svc.Spec.Type = v1.ServiceTypeLoadBalancer + // We need to turn affinity off for our LB distribution tests + svc.Spec.SessionAffinity = v1.ServiceAffinityNone + if tweak != nil { + tweak(svc) + } + }) + + By("waiting for loadbalancer for service " + namespace + "/" + serviceName) + svc = j.WaitForLoadBalancerOrFail(namespace, serviceName, timeout) + j.SanityCheckService(svc, v1.ServiceTypeLoadBalancer) + return svc +} + func GetNodeAddresses(node *v1.Node, addressType v1.NodeAddressType) (ips []string) { for j := range node.Status.Addresses { nodeAddress := &node.Status.Addresses[j] diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 55986ae2f3d..f4271f07038 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -5183,12 +5183,16 @@ func CleanupGCEResources(loadBalancerName string) (retErr error) { if err := DeleteGCEStaticIP(loadBalancerName); err != nil { Logf("%v", err) } + var hcNames []string hc, getErr := gceCloud.GetHttpHealthCheck(loadBalancerName) if getErr != nil && !IsGoogleAPIHTTPErrorCode(getErr, http.StatusNotFound) { retErr = fmt.Errorf("%v\n%v", retErr, getErr) return } - if err := gceCloud.DeleteTargetPool(loadBalancerName, hc); err != nil && + if hc != nil { + hcNames = append(hcNames, hc.Name) + } + if err := gceCloud.DeleteTargetPool(loadBalancerName, hcNames...); err != nil && !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { retErr = fmt.Errorf("%v\n%v", retErr, err) }