Update firewall e2e test for LB healthcheck firewall

test/e2e/firewall.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	"k8s.io/kubernetes/pkg/cloudprovider"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
 	"k8s.io/kubernetes/pkg/master/ports"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -45,13 +46,18 @@ var _ = framework.KubeDescribe("Firewall rule", func() {
 		gceCloud = cloudConfig.Provider.(*gcecloud.GCECloud)
 	})
 
-	// This test takes around 4 minutes to run
+	// This test takes around 6 minutes to run
 	It("[Slow] [Serial] should create valid firewall rules for LoadBalancer type service", func() {
 		ns := f.Namespace.Name
 		// This source ranges is just used to examine we have exact same things on LB firewall rules
 		firewallTestSourceRanges := []string{"0.0.0.0/1", "128.0.0.0/1"}
 		serviceName := "firewall-test-loadbalancer"
 
+		By("Getting cluster ID")
+		clusterID, err := framework.GetClusterID(cs)
+		Expect(err).NotTo(HaveOccurred())
+		framework.Logf("Got cluster ID: %v", clusterID)
+
 		jig := framework.NewServiceTestJig(cs, serviceName)
 		nodesNames := jig.GetNodesNames(framework.MaxNodesForEndpointsTests)
 		if len(nodesNames) <= 0 {
@@ -59,28 +65,52 @@ var _ = framework.KubeDescribe("Firewall rule", func() {
 		}
 		nodesSet := sets.NewString(nodesNames...)
 
-		// OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE
-		By("Creating a LoadBalancer type service with ExternalTrafficPolicy=Local")
-		svc := jig.CreateOnlyLocalLoadBalancerService(ns, serviceName,
-			framework.LoadBalancerCreateTimeoutDefault, false, func(svc *v1.Service) {
-				svc.Spec.Ports = []v1.ServicePort{{Protocol: "TCP", Port: framework.FirewallTestHttpPort}}
-				svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges
-			})
+		By("Creating a LoadBalancer type service with ExternalTrafficPolicy=Global")
+		svc := jig.CreateLoadBalancerService(ns, serviceName, framework.LoadBalancerCreateTimeoutDefault, func(svc *v1.Service) {
+			svc.Spec.Ports = []v1.ServicePort{{Protocol: "TCP", Port: framework.FirewallTestHttpPort}}
+			svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges
+		})
 		defer func() {
 			jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) {
 				svc.Spec.Type = v1.ServiceTypeNodePort
 				svc.Spec.LoadBalancerSourceRanges = nil
 			})
 			Expect(cs.Core().Services(svc.Namespace).Delete(svc.Name, nil)).NotTo(HaveOccurred())
+			By("Waiting for the local traffic health check firewall rule to be deleted")
+			localHCFwName := framework.MakeHealthCheckFirewallNameForLBService(clusterID, cloudprovider.GetLoadBalancerName(svc), false)
+			_, err := framework.WaitForFirewallRule(gceCloud, localHCFwName, false, framework.LoadBalancerCleanupTimeout)
+			Expect(err).NotTo(HaveOccurred())
 		}()
 		svcExternalIP := svc.Status.LoadBalancer.Ingress[0].IP
 
-		By("Checking if service's firewall rules are correct")
+		By("Checking if service's firewall rule is correct")
 		nodeTags := framework.GetInstanceTags(cloudConfig, nodesNames[0])
-		expFw := framework.ConstructFirewallForLBService(svc, nodeTags.Items)
-		fw, err := gceCloud.GetFirewall(expFw.Name)
+		lbFw := framework.ConstructFirewallForLBService(svc, nodeTags.Items)
+		fw, err := gceCloud.GetFirewall(lbFw.Name)
 		Expect(err).NotTo(HaveOccurred())
-		Expect(framework.VerifyFirewallRule(fw, expFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+		Expect(framework.VerifyFirewallRule(fw, lbFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+
+		By("Checking if service's nodes health check firewall rule is correct")
+		nodesHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, nodeTags.Items, true)
+		fw, err = gceCloud.GetFirewall(nodesHCFw.Name)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(framework.VerifyFirewallRule(fw, nodesHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
+
+		// OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE
+		By("Updating LoadBalancer service to ExternalTrafficPolicy=Local")
+		svc = jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) {
+			svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
+		})
+
+		By("Waiting for the nodes health check firewall rule to be deleted")
+		_, err = framework.WaitForFirewallRule(gceCloud, nodesHCFw.Name, false, framework.LoadBalancerCleanupTimeout)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Waiting for the correct local traffic health check firewall rule to be created")
+		localHCFw := framework.ConstructHealthCheckFirewallForLBService(clusterID, svc, nodeTags.Items, false)
+		fw, err = framework.WaitForFirewallRule(gceCloud, localHCFw.Name, true, framework.LoadBalancerCreateTimeoutDefault)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(framework.VerifyFirewallRule(fw, localHCFw, cloudConfig.Network, false)).NotTo(HaveOccurred())
 
 		By(fmt.Sprintf("Creating netexec pods on at most %v nodes", framework.MaxNodesForEndpointsTests))
 		for i, nodeName := range nodesNames {
@@ -100,7 +130,7 @@ var _ = framework.KubeDescribe("Firewall rule", func() {
 		// by removing the tag on one vm and make sure it doesn't get any traffic. This is an imperfect
 		// simulation, we really want to check that traffic doesn't reach a vm outside the GKE cluster, but
 		// that's much harder to do in the current e2e framework.
-		By("Removing tags from one of the nodes")
+		By(fmt.Sprintf("Removing tags from one of the nodes: %v", nodesNames[0]))
 		nodesSet.Delete(nodesNames[0])
 		removedTags := framework.SetInstanceTags(cloudConfig, nodesNames[0], []string{})
 		defer func() {

test/e2e/framework/firewall_util.go

@@ -18,12 +18,16 @@ package framework
 
 import (
 	"fmt"
+	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/api/v1"
+	apiservice "k8s.io/kubernetes/pkg/api/v1/service"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
@@ -41,7 +45,7 @@ const (
 )
 
 // MakeFirewallNameForLBService return the expected firewall name for a LB service.
-// This should match the formatting of makeFirewallName() in pkg/cloudprovider/providers/gce/gce.go
+// This should match the formatting of makeFirewallName() in pkg/cloudprovider/providers/gce/gce_loadbalancer.go
 func MakeFirewallNameForLBService(name string) string {
 	return fmt.Sprintf("k8s-fw-%s", name)
 }
@@ -68,6 +72,32 @@ func ConstructFirewallForLBService(svc *v1.Service, nodesTags []string) *compute
 	return &fw
 }
 
+func MakeHealthCheckFirewallNameForLBService(clusterID, name string, isNodesHealthCheck bool) string {
+	return gcecloud.MakeHealthCheckFirewallName(clusterID, name, isNodesHealthCheck)
+}
+
+// ConstructHealthCheckFirewallForLBService returns the expected GCE firewall rule for a loadbalancer type service
+func ConstructHealthCheckFirewallForLBService(clusterID string, svc *v1.Service, nodesTags []string, isNodesHealthCheck bool) *compute.Firewall {
+	if svc.Spec.Type != v1.ServiceTypeLoadBalancer {
+		Failf("can not construct firewall rule for non-loadbalancer type service")
+	}
+	fw := compute.Firewall{}
+	fw.Name = MakeHealthCheckFirewallNameForLBService(clusterID, cloudprovider.GetLoadBalancerName(svc), isNodesHealthCheck)
+	fw.TargetTags = nodesTags
+	fw.SourceRanges = gcecloud.LoadBalancerSrcRanges()
+	healthCheckPort := gcecloud.GetNodesHealthCheckPort()
+	if !isNodesHealthCheck {
+		healthCheckPort = apiservice.GetServiceHealthCheckNodePort(svc)
+	}
+	fw.Allowed = []*compute.FirewallAllowed{
+		{
+			IPProtocol: "tcp",
+			Ports:      []string{fmt.Sprintf("%d", healthCheckPort)},
+		},
+	}
+	return &fw
+}
+
 // GetNodeTags gets tags from one of the Kubernetes nodes
 func GetNodeTags(c clientset.Interface, cloudConfig CloudConfig) *compute.Tags {
 	nodes := GetReadySchedulableNodesOrDie(c)
@@ -303,6 +333,9 @@ func SameStringArray(result, expected []string, include bool) error {
 // VerifyFirewallRule verifies whether the result firewall is consistent with the expected firewall.
 // When `portsSubset` is false, match given ports exactly. Otherwise, only check ports are included.
 func VerifyFirewallRule(res, exp *compute.Firewall, network string, portsSubset bool) error {
+	if res == nil || exp == nil {
+		return fmt.Errorf("res and exp must not be nil")
+	}
 	if res.Name != exp.Name {
 		return fmt.Errorf("incorrect name: %v, expected %v", res.Name, exp.Name)
 	}
@@ -325,3 +358,40 @@ func VerifyFirewallRule(res, exp *compute.Firewall, network string, portsSubset
 	}
 	return nil
 }
+
+func WaitForFirewallRule(gceCloud *gcecloud.GCECloud, fwName string, exist bool, timeout time.Duration) (*compute.Firewall, error) {
+	Logf("Waiting up to %v for firewall %v exist=%v", timeout, fwName, exist)
+	var fw *compute.Firewall
+	var err error
+
+	condition := func() (bool, error) {
+		fw, err = gceCloud.GetFirewall(fwName)
+		if err != nil && exist ||
+			err == nil && !exist ||
+			err != nil && !exist && !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
+			return false, nil
+		}
+		return true, nil
+	}
+
+	if err := wait.PollImmediate(5*time.Second, timeout, condition); err != nil {
+		return nil, fmt.Errorf("error waiting for firewall %v exist=%v", fwName, exist)
+	}
+	return fw, nil
+}
+
+func GetClusterID(c clientset.Interface) (string, error) {
+	cm, err := c.Core().ConfigMaps(metav1.NamespaceSystem).Get(gcecloud.UIDConfigMapName, metav1.GetOptions{})
+	if err != nil || cm == nil {
+		return "", fmt.Errorf("error getting cluster ID: %v", err)
+	}
+	clusterID, clusterIDExists := cm.Data[gcecloud.UIDCluster]
+	providerID, providerIDExists := cm.Data[gcecloud.UIDProvider]
+	if !clusterIDExists {
+		return "", fmt.Errorf("cluster ID not set")
+	}
+	if providerIDExists {
+		return providerID, nil
+	}
+	return clusterID, nil
+}

test/e2e/framework/service_util.go

@@ -235,6 +235,25 @@ func (j *ServiceTestJig) CreateOnlyLocalLoadBalancerService(namespace, serviceNa
 	return svc
 }
 
+// CreateLoadBalancerService creates a loadbalancer service and waits
+// for it to acquire an ingress IP.
+func (j *ServiceTestJig) CreateLoadBalancerService(namespace, serviceName string, timeout time.Duration, tweak func(svc *v1.Service)) *v1.Service {
+	By("creating a service " + namespace + "/" + serviceName + " with type=LoadBalancer")
+	svc := j.CreateTCPServiceOrFail(namespace, func(svc *v1.Service) {
+		svc.Spec.Type = v1.ServiceTypeLoadBalancer
+		// We need to turn affinity off for our LB distribution tests
+		svc.Spec.SessionAffinity = v1.ServiceAffinityNone
+		if tweak != nil {
+			tweak(svc)
+		}
+	})
+
+	By("waiting for loadbalancer for service " + namespace + "/" + serviceName)
+	svc = j.WaitForLoadBalancerOrFail(namespace, serviceName, timeout)
+	j.SanityCheckService(svc, v1.ServiceTypeLoadBalancer)
+	return svc
+}
+
 func GetNodeAddresses(node *v1.Node, addressType v1.NodeAddressType) (ips []string) {
 	for j := range node.Status.Addresses {
 		nodeAddress := &node.Status.Addresses[j]

test/e2e/framework/util.go

@@ -5183,12 +5183,16 @@ func CleanupGCEResources(loadBalancerName string) (retErr error) {
 	if err := DeleteGCEStaticIP(loadBalancerName); err != nil {
 		Logf("%v", err)
 	}
+	var hcNames []string
 	hc, getErr := gceCloud.GetHttpHealthCheck(loadBalancerName)
 	if getErr != nil && !IsGoogleAPIHTTPErrorCode(getErr, http.StatusNotFound) {
 		retErr = fmt.Errorf("%v\n%v", retErr, getErr)
 		return
 	}
-	if err := gceCloud.DeleteTargetPool(loadBalancerName, hc); err != nil &&
+	if hc != nil {
+		hcNames = append(hcNames, hc.Name)
+	}
+	if err := gceCloud.DeleteTargetPool(loadBalancerName, hcNames...); err != nil &&
 		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {
 		retErr = fmt.Errorf("%v\n%v", retErr, err)
 	}