From bf1597f6653a170cf8b4f13399f08d231d381c88 Mon Sep 17 00:00:00 2001 From: Mayank Gaikwad <8110509+mgdevstack@users.noreply.github.com> Date: Thu, 1 Aug 2019 08:32:30 +0530 Subject: [PATCH] Fix failing service e2e due to execPod IP unavailability --- test/e2e/network/service.go | 47 +++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go index 06c17ce863e..40f93be442f 100644 --- a/test/e2e/network/service.go +++ b/test/e2e/network/service.go @@ -1474,6 +1474,13 @@ var _ = SIGDescribe("Services", func() { // This container is an nginx container listening on port 80 // See kubernetes/contrib/ingress/echoheaders/nginx.conf for content of response jig.RunOrFail(namespace, nil) + var err error + // Make sure acceptPod is running. There are certain chances that pod might be teminated due to unexpected reasons. + acceptPod, err = cs.CoreV1().Pods(namespace).Get(acceptPod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "Unable to get pod %s", acceptPod.Name) + framework.ExpectEqual(acceptPod.Status.Phase, v1.PodRunning) + framework.ExpectNotEqual(acceptPod.Status.PodIP, "") + // Create loadbalancer service with source range from node[0] and podAccept svc := jig.CreateTCPServiceOrFail(namespace, func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer @@ -1503,6 +1510,12 @@ var _ = SIGDescribe("Services", func() { framework.CheckReachabilityFromPod(true, loadBalancerLagTimeout, namespace, acceptPod.Name, svcIP) framework.CheckReachabilityFromPod(false, normalReachabilityTimeout, namespace, dropPod.Name, svcIP) + // Make sure dropPod is running. There are certain chances that the pod might be teminated due to unexpected reasons. dropPod, err = cs.CoreV1().Pods(namespace).Get(dropPod.Name, metav1.GetOptions{}) + dropPod, err = cs.CoreV1().Pods(namespace).Get(dropPod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "Unable to get pod %s", dropPod.Name) + framework.ExpectEqual(acceptPod.Status.Phase, v1.PodRunning) + framework.ExpectNotEqual(acceptPod.Status.PodIP, "") + ginkgo.By("Update service LoadBalancerSourceRange and check reachability") jig.UpdateServiceOrFail(svc.Namespace, svc.Name, func(svc *v1.Service) { // only allow access from dropPod @@ -2159,10 +2172,10 @@ var _ = SIGDescribe("ESIPP [Slow] [DisabledForLargeClusters]", func() { }) ginkgo.It("should work from pods", func() { + var err error namespace := f.Namespace.Name serviceName := "external-local-pods" jig := e2eservice.NewTestJig(cs, serviceName) - nodes := jig.GetNodes(e2eservice.MaxNodesForEndpointsTests) svc := jig.CreateOnlyLocalLoadBalancerService(namespace, serviceName, loadBalancerCreateTimeout, true, nil) serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc)) @@ -2176,33 +2189,41 @@ var _ = SIGDescribe("ESIPP [Slow] [DisabledForLargeClusters]", func() { port := strconv.Itoa(int(svc.Spec.Ports[0].Port)) ipPort := net.JoinHostPort(ingressIP, port) path := fmt.Sprintf("%s/clientip", ipPort) - nodeName := nodes.Items[0].Name - podName := "execpod-sourceip" - ginkgo.By(fmt.Sprintf("Creating %v on node %v", podName, nodeName)) - execPod := e2epod.CreateExecPodOrFail(f.ClientSet, namespace, podName, func(pod *v1.Pod) { - pod.Spec.NodeName = nodeName - }) + ginkgo.By("Creating pause pod deployment to make sure, pausePods are in desired state") + deployment := jig.CreatePausePodDeployment("pause-pod-deployment", namespace, int32(1)) + framework.ExpectNoError(e2edeploy.WaitForDeploymentComplete(cs, deployment), "Failed to complete pause pod deployment") + defer func() { - err := cs.CoreV1().Pods(namespace).Delete(execPod.Name, nil) - framework.ExpectNoError(err, "failed to delete pod: %s", execPod.Name) + e2elog.Logf("Deleting deployment") + err = cs.AppsV1().Deployments(namespace).Delete(deployment.Name, &metav1.DeleteOptions{}) + framework.ExpectNoError(err, "Failed to delete deployment %s", deployment.Name) }() + deployment, err = cs.AppsV1().Deployments(namespace).Get(deployment.Name, metav1.GetOptions{}) + framework.ExpectNoError(err, "Error in retriving pause pod deployment") + labelSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) + framework.ExpectNoError(err, "Error in setting LabelSelector as selector from deployment") + + pausePods, err := cs.CoreV1().Pods(namespace).List(metav1.ListOptions{LabelSelector: labelSelector.String()}) + framework.ExpectNoError(err, "Error in listing pods associated with pause pod deployments") + + pausePod := pausePods.Items[0] e2elog.Logf("Waiting up to %v curl %v", e2eservice.KubeProxyLagTimeout, path) cmd := fmt.Sprintf(`curl -q -s --connect-timeout 30 %v`, path) var srcIP string - ginkgo.By(fmt.Sprintf("Hitting external lb %v from pod %v on node %v", ingressIP, podName, nodeName)) + ginkgo.By(fmt.Sprintf("Hitting external lb %v from pod %v on node %v", ingressIP, pausePod.Name, pausePod.Spec.NodeName)) if pollErr := wait.PollImmediate(framework.Poll, e2eservice.LoadBalancerCreateTimeoutDefault, func() (bool, error) { - stdout, err := framework.RunHostCmd(execPod.Namespace, execPod.Name, cmd) + stdout, err := framework.RunHostCmd(pausePod.Namespace, pausePod.Name, cmd) if err != nil { e2elog.Logf("got err: %v, retry until timeout", err) return false, nil } srcIP = strings.TrimSpace(strings.Split(stdout, ":")[0]) - return srcIP == execPod.Status.PodIP, nil + return srcIP == pausePod.Status.PodIP, nil }); pollErr != nil { - e2elog.Failf("Source IP not preserved from %v, expected '%v' got '%v'", podName, execPod.Status.PodIP, srcIP) + e2elog.Failf("Source IP not preserved from %v, expected '%v' got '%v'", pausePod.Name, pausePod.Status.PodIP, srcIP) } })