[e2e] Add tests for service load balancer finalizer

test/e2e/network/service.go

@@ -29,6 +29,7 @@ import (
 	compute "google.golang.org/api/compute/v1"
 
 	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -1852,8 +1853,112 @@ var _ = SIGDescribe("Services", func() {
 		}
 	})
 
+	// This test verifies if service load balancer cleanup finalizer can be removed
+	// when feature gate isn't enabled on the cluster.
+	// This ensures downgrading from higher version cluster will not break LoadBalancer
+	// type service.
+	ginkgo.It("should remove load balancer cleanup finalizer when service is deleted [Slow]", func() {
+		jig := framework.NewServiceTestJig(cs, "lb-remove-finalizer")
+
+		ginkgo.By("Create load balancer service")
+		svc := jig.CreateTCPServiceOrFail(f.Namespace.Name, func(svc *v1.Service) {
+			svc.Spec.Type = v1.ServiceTypeLoadBalancer
+		})
+
+		defer func() {
+			waitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
+		}()
+
+		ginkgo.By("Wait for load balancer to serve traffic")
+		svc = jig.WaitForLoadBalancerOrFail(svc.Namespace, svc.Name, framework.GetServiceLoadBalancerCreationTimeout(cs))
+
+		ginkgo.By("Manually add load balancer cleanup finalizer to service")
+		svc.Finalizers = append(svc.Finalizers, "service.kubernetes.io/load-balancer-cleanup")
+		if _, err := cs.CoreV1().Services(svc.Namespace).Update(svc); err != nil {
+			framework.Failf("Failed to add finalizer to service %s/%s: %v", svc.Namespace, svc.Name, err)
+		}
+	})
+
+	// This test verifies if service load balancer cleanup finalizer is properly
+	// handled during service lifecycle.
+	// 1. Create service with type=LoadBalancer. Finalizer should be added.
+	// 2. Update service to type=ClusterIP. Finalizer should be removed.
+	// 3. Update service to type=LoadBalancer. Finalizer should be added.
+	// 4. Delete service with type=LoadBalancer. Finalizer should be removed.
+	ginkgo.It("should handle load balancer cleanup finalizer for service [Slow] [Feature:ServiceFinalizer]", func() {
+		jig := framework.NewServiceTestJig(cs, "lb-finalizer")
+
+		ginkgo.By("Create load balancer service")
+		svc := jig.CreateTCPServiceOrFail(f.Namespace.Name, func(svc *v1.Service) {
+			svc.Spec.Type = v1.ServiceTypeLoadBalancer
+		})
+
+		defer func() {
+			waitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
+		}()
+
+		ginkgo.By("Wait for load balancer to serve traffic")
+		svc = jig.WaitForLoadBalancerOrFail(svc.Namespace, svc.Name, framework.GetServiceLoadBalancerCreationTimeout(cs))
+
+		ginkgo.By("Check if finalizer presents on service with type=LoadBalancer")
+		waitForServiceUpdatedWithFinalizer(cs, svc.Namespace, svc.Name, true)
+
+		ginkgo.By("Check if finalizer is removed on service after changed to type=ClusterIP")
+		jig.ChangeServiceType(svc.Namespace, svc.Name, v1.ServiceTypeClusterIP, framework.GetServiceLoadBalancerCreationTimeout(cs))
+		waitForServiceUpdatedWithFinalizer(cs, svc.Namespace, svc.Name, false)
+
+		ginkgo.By("Check if finalizer is added back to service after changed to type=LoadBalancer")
+		jig.ChangeServiceType(svc.Namespace, svc.Name, v1.ServiceTypeLoadBalancer, framework.GetServiceLoadBalancerCreationTimeout(cs))
+		waitForServiceUpdatedWithFinalizer(cs, svc.Namespace, svc.Name, true)
+	})
 })
 
+func waitForServiceDeletedWithFinalizer(cs clientset.Interface, namespace, name string) {
+	ginkgo.By("Delete service with finalizer")
+	if err := cs.CoreV1().Services(namespace).Delete(name, nil); err != nil {
+		framework.Failf("Failed to delete service %s/%s", namespace, name)
+	}
+
+	ginkgo.By("Wait for service to disappear")
+	if pollErr := wait.PollImmediate(framework.LoadBalancerPollInterval, framework.GetServiceLoadBalancerCreationTimeout(cs), func() (bool, error) {
+		svc, err := cs.CoreV1().Services(namespace).Get(name, metav1.GetOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				e2elog.Logf("Service %s/%s is gone.", namespace, name)
+				return true, nil
+			}
+			return false, err
+		}
+		e2elog.Logf("Service %s/%s still exists with finalizers: %v", namespace, name, svc.Finalizers)
+		return false, nil
+	}); pollErr != nil {
+		framework.Failf("Failed to wait for service to disappear: %v", pollErr)
+	}
+}
+
+func waitForServiceUpdatedWithFinalizer(cs clientset.Interface, namespace, name string, hasFinalizer bool) {
+	ginkgo.By(fmt.Sprintf("Wait for service to hasFinalizer=%t", hasFinalizer))
+	if pollErr := wait.PollImmediate(framework.LoadBalancerPollInterval, framework.GetServiceLoadBalancerCreationTimeout(cs), func() (bool, error) {
+		svc, err := cs.CoreV1().Services(namespace).Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		foundFinalizer := false
+		for _, finalizer := range svc.Finalizers {
+			if finalizer == "service.kubernetes.io/load-balancer-cleanup" {
+				foundFinalizer = true
+			}
+		}
+		if foundFinalizer != hasFinalizer {
+			e2elog.Logf("Service %s/%s hasFinalizer=%t, want %t", namespace, name, foundFinalizer, hasFinalizer)
+			return false, nil
+		}
+		return true, nil
+	}); pollErr != nil {
+		framework.Failf("Failed to wait for service to hasFinalizer=%t: %v", hasFinalizer, pollErr)
+	}
+}
+
 // TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed.
 var _ = SIGDescribe("ESIPP [Slow] [DisabledForLargeClusters]", func() {
 	f := framework.NewDefaultFramework("esipp")