From a9ed5702be3738e034aff731dd32aeb6d0ced4bc Mon Sep 17 00:00:00 2001 From: Zihong Zheng Date: Mon, 27 May 2019 17:45:58 -0700 Subject: [PATCH] [e2e] Add tests for service load balancer finalizer --- test/e2e/network/service.go | 105 ++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 97ca2675c41..93a4ce8e627 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -29,6 +29,7 @@ import ( compute "google.golang.org/api/compute/v1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" @@ -1852,8 +1853,112 @@ var _ = SIGDescribe("Services", func() { } }) + // This test verifies if service load balancer cleanup finalizer can be removed + // when feature gate isn't enabled on the cluster. + // This ensures downgrading from higher version cluster will not break LoadBalancer + // type service. + ginkgo.It("should remove load balancer cleanup finalizer when service is deleted [Slow]", func() { + jig := framework.NewServiceTestJig(cs, "lb-remove-finalizer") + + ginkgo.By("Create load balancer service") + svc := jig.CreateTCPServiceOrFail(f.Namespace.Name, func(svc *v1.Service) { + svc.Spec.Type = v1.ServiceTypeLoadBalancer + }) + + defer func() { + waitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name) + }() + + ginkgo.By("Wait for load balancer to serve traffic") + svc = jig.WaitForLoadBalancerOrFail(svc.Namespace, svc.Name, framework.GetServiceLoadBalancerCreationTimeout(cs)) + + ginkgo.By("Manually add load balancer cleanup finalizer to service") + svc.Finalizers = append(svc.Finalizers, "service.kubernetes.io/load-balancer-cleanup") + if _, err := cs.CoreV1().Services(svc.Namespace).Update(svc); err != nil { + framework.Failf("Failed to add finalizer to service %s/%s: %v", svc.Namespace, svc.Name, err) + } + }) + + // This test verifies if service load balancer cleanup finalizer is properly + // handled during service lifecycle. + // 1. Create service with type=LoadBalancer. Finalizer should be added. + // 2. Update service to type=ClusterIP. Finalizer should be removed. + // 3. Update service to type=LoadBalancer. Finalizer should be added. + // 4. Delete service with type=LoadBalancer. Finalizer should be removed. + ginkgo.It("should handle load balancer cleanup finalizer for service [Slow] [Feature:ServiceFinalizer]", func() { + jig := framework.NewServiceTestJig(cs, "lb-finalizer") + + ginkgo.By("Create load balancer service") + svc := jig.CreateTCPServiceOrFail(f.Namespace.Name, func(svc *v1.Service) { + svc.Spec.Type = v1.ServiceTypeLoadBalancer + }) + + defer func() { + waitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name) + }() + + ginkgo.By("Wait for load balancer to serve traffic") + svc = jig.WaitForLoadBalancerOrFail(svc.Namespace, svc.Name, framework.GetServiceLoadBalancerCreationTimeout(cs)) + + ginkgo.By("Check if finalizer presents on service with type=LoadBalancer") + waitForServiceUpdatedWithFinalizer(cs, svc.Namespace, svc.Name, true) + + ginkgo.By("Check if finalizer is removed on service after changed to type=ClusterIP") + jig.ChangeServiceType(svc.Namespace, svc.Name, v1.ServiceTypeClusterIP, framework.GetServiceLoadBalancerCreationTimeout(cs)) + waitForServiceUpdatedWithFinalizer(cs, svc.Namespace, svc.Name, false) + + ginkgo.By("Check if finalizer is added back to service after changed to type=LoadBalancer") + jig.ChangeServiceType(svc.Namespace, svc.Name, v1.ServiceTypeLoadBalancer, framework.GetServiceLoadBalancerCreationTimeout(cs)) + waitForServiceUpdatedWithFinalizer(cs, svc.Namespace, svc.Name, true) + }) }) +func waitForServiceDeletedWithFinalizer(cs clientset.Interface, namespace, name string) { + ginkgo.By("Delete service with finalizer") + if err := cs.CoreV1().Services(namespace).Delete(name, nil); err != nil { + framework.Failf("Failed to delete service %s/%s", namespace, name) + } + + ginkgo.By("Wait for service to disappear") + if pollErr := wait.PollImmediate(framework.LoadBalancerPollInterval, framework.GetServiceLoadBalancerCreationTimeout(cs), func() (bool, error) { + svc, err := cs.CoreV1().Services(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + e2elog.Logf("Service %s/%s is gone.", namespace, name) + return true, nil + } + return false, err + } + e2elog.Logf("Service %s/%s still exists with finalizers: %v", namespace, name, svc.Finalizers) + return false, nil + }); pollErr != nil { + framework.Failf("Failed to wait for service to disappear: %v", pollErr) + } +} + +func waitForServiceUpdatedWithFinalizer(cs clientset.Interface, namespace, name string, hasFinalizer bool) { + ginkgo.By(fmt.Sprintf("Wait for service to hasFinalizer=%t", hasFinalizer)) + if pollErr := wait.PollImmediate(framework.LoadBalancerPollInterval, framework.GetServiceLoadBalancerCreationTimeout(cs), func() (bool, error) { + svc, err := cs.CoreV1().Services(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return false, err + } + foundFinalizer := false + for _, finalizer := range svc.Finalizers { + if finalizer == "service.kubernetes.io/load-balancer-cleanup" { + foundFinalizer = true + } + } + if foundFinalizer != hasFinalizer { + e2elog.Logf("Service %s/%s hasFinalizer=%t, want %t", namespace, name, foundFinalizer, hasFinalizer) + return false, nil + } + return true, nil + }); pollErr != nil { + framework.Failf("Failed to wait for service to hasFinalizer=%t: %v", hasFinalizer, pollErr) + } +} + // TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed. var _ = SIGDescribe("ESIPP [Slow] [DisabledForLargeClusters]", func() { f := framework.NewDefaultFramework("esipp")