github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 03:11:40 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #92304 from wojtek-t/reduce_flakes_probability

Browse Source 
Attempt to deflake conntract e2e tests
This commit is contained in:
Kubernetes Prow Robot 2020-06-20 05:27:07 -07:00 committed by GitHub
commit 8e820aa785
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 57 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
test/e2e/network/conntrack.go
View File

@ -25,6 +25,7 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
e2enode "k8s.io/kubernetes/test/e2e/framework/node"
@ -75,6 +76,20 @@ var _ = SIGDescribe("Conntrack", func() {
clientNodeInfo, serverNodeInfo nodeInfo
)
logContainsFn := func(text string) wait.ConditionFunc {
return func() (bool, error) {
logs, err := e2epod.GetPodLogs(cs, ns, podClient, podClient)
if err != nil {
// Retry the error next time.
return false, nil
}
if !strings.Contains(string(logs), text) {
return false, nil
}
return true, nil
}
}
ginkgo.BeforeEach(func() {
cs = fr.ClientSet
ns = fr.Namespace.Name
@ -117,7 +132,7 @@ var _ = SIGDescribe("Conntrack", func() {
ginkgo.By("creating a client pod for probing the service " + serviceName)
clientPod := newAgnhostPod(podClient, "")
clientPod.Spec.NodeName = clientNodeInfo.name
cmd := fmt.Sprintf(`date; for i in $(seq 1 300); do echo "$(date) Try: ${i}"; echo hostname | nc -u -w 5 -p %d %s %d; echo; done`, srcPort, serverNodeInfo.nodeIP, udpService.Spec.Ports[0].NodePort)
cmd := fmt.Sprintf(`date; for i in $(seq 1 3000); do echo "$(date) Try: ${i}"; echo hostname | nc -u -w 5 -p %d %s %d; echo; done`, srcPort, serverNodeInfo.nodeIP, udpService.Spec.Ports[0].NodePort)
clientPod.Spec.Containers[0].Command = []string{"/bin/sh", "-c", cmd}
clientPod.Spec.Containers[0].Name = podClient
fr.PodClient().CreateSync(clientPod)
@ -138,15 +153,17 @@ var _ = SIGDescribe("Conntrack", func() {
err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{podBackend1: {80}})
framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns)
// Check that the pod receives the traffic
// UDP conntrack entries timeout is 30 sec by default
// Note that the fact that Endpoints object already exists, does NOT mean
// that iptables (or whatever else is used) was already programmed.
// Additionally take into account that UDP conntract entries timeout is
// 30 seconds by default.
// Based on the above check if the pod receives the traffic.
ginkgo.By("checking client pod connected to the backend 1 on Node IP " + serverNodeInfo.nodeIP)
time.Sleep(30 * time.Second)
logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient)
framework.ExpectNoError(err)
framework.Logf("Pod client logs: %s", logs)
if !strings.Contains(string(logs), podBackend1) {
framework.Failf("Failed to connecto to backend 1")
if err := wait.PollImmediate(5*time.Second, time.Minute, logContainsFn(podBackend1)); err != nil {
logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient)
framework.ExpectNoError(err)
framework.Logf("Pod client logs: %s", logs)
framework.Failf("Failed to connect to backend 1")
}
// Create a second pod
@ -160,21 +177,24 @@ var _ = SIGDescribe("Conntrack", func() {
framework.Logf("Cleaning up %s pod", podBackend1)
fr.PodClient().DeleteSync(podBackend1, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
// Waiting for service to expose endpoint.
err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{podBackend2: {80}})
framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns)
// Check that the second pod keeps receiving traffic
// UDP conntrack entries timeout is 30 sec by default
ginkgo.By("checking client pod connected to the backend 2 on Node IP " + serverNodeInfo.nodeIP)
time.Sleep(30 * time.Second)
logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient)
framework.ExpectNoError(err)
framework.Logf("Pod client logs: %s", logs)
if !strings.Contains(string(logs), podBackend2) {
framework.Failf("Failed to connecto to backend 2")
if err := wait.PollImmediate(5*time.Second, time.Minute, logContainsFn(podBackend2)); err != nil {
logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient)
framework.ExpectNoError(err)
framework.Logf("Pod client logs: %s", logs)
framework.Failf("Failed to connect to backend 2")
}
})
ginkgo.It("should be able to preserve UDP traffic when server pod cycles for a ClusterIP service", func() {
// Create a NodePort service
// Create a ClusterIP service
udpJig := e2eservice.NewTestJig(cs, ns, serviceName)
ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in " + ns)
udpService, err := udpJig.CreateUDPService(func(svc *v1.Service) {
@ -185,11 +205,11 @@ var _ = SIGDescribe("Conntrack", func() {
})
framework.ExpectNoError(err)
// Create a pod in one node to create the UDP traffic against the NodePort service every 5 seconds
// Create a pod in one node to create the UDP traffic against the ClusterIP service every 5 seconds
ginkgo.By("creating a client pod for probing the service " + serviceName)
clientPod := newAgnhostPod(podClient, "")
clientPod.Spec.NodeName = clientNodeInfo.name
cmd := fmt.Sprintf(`date; for i in $(seq 1 300); do echo "$(date) Try: ${i}"; echo hostname | nc -u -w 5 -p %d %s %d; echo; done`, srcPort, udpService.Spec.ClusterIP, udpService.Spec.Ports[0].Port)
cmd := fmt.Sprintf(`date; for i in $(seq 1 3000); do echo "$(date) Try: ${i}"; echo hostname | nc -u -w 5 -p %d %s %d; echo; done`, srcPort, udpService.Spec.ClusterIP, udpService.Spec.Ports[0].Port)
clientPod.Spec.Containers[0].Command = []string{"/bin/sh", "-c", cmd}
clientPod.Spec.Containers[0].Name = podClient
fr.PodClient().CreateSync(clientPod)
@ -210,15 +230,17 @@ var _ = SIGDescribe("Conntrack", func() {
err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{podBackend1: {80}})
framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns)
// Check that the pod receives the traffic
// UDP conntrack entries timeout is 30 sec by default
// Note that the fact that Endpoints object already exists, does NOT mean
// that iptables (or whatever else is used) was already programmed.
// Additionally take into account that UDP conntract entries timeout is
// 30 seconds by default.
// Based on the above check if the pod receives the traffic.
ginkgo.By("checking client pod connected to the backend 1 on Node IP " + serverNodeInfo.nodeIP)
time.Sleep(30 * time.Second)
logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient)
framework.ExpectNoError(err)
framework.Logf("Pod client logs: %s", logs)
if !strings.Contains(string(logs), podBackend1) {
framework.Failf("Failed to connecto to backend 1")
if err := wait.PollImmediate(5*time.Second, time.Minute, logContainsFn(podBackend1)); err != nil {
logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient)
framework.ExpectNoError(err)
framework.Logf("Pod client logs: %s", logs)
framework.Failf("Failed to connect to backend 1")
}
// Create a second pod
@ -232,15 +254,18 @@ var _ = SIGDescribe("Conntrack", func() {
framework.Logf("Cleaning up %s pod", podBackend1)
fr.PodClient().DeleteSync(podBackend1, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
// Waiting for service to expose endpoint.
err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{podBackend2: {80}})
framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns)
// Check that the second pod keeps receiving traffic
// UDP conntrack entries timeout is 30 sec by default
ginkgo.By("checking client pod connected to the backend 2 on Node IP " + serverNodeInfo.nodeIP)
time.Sleep(30 * time.Second)
logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient)
framework.ExpectNoError(err)
framework.Logf("Pod client logs: %s", logs)
if !strings.Contains(string(logs), podBackend2) {
framework.Failf("Failed to connecto to backend 2")
if err := wait.PollImmediate(5*time.Second, time.Minute, logContainsFn(podBackend2)); err != nil {
logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient)
framework.ExpectNoError(err)
framework.Logf("Pod client logs: %s", logs)
framework.Failf("Failed to connect to backend 2")
}
})
})