diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go
index d74c0ce9cbf..20c8cc83977 100644
--- a/test/e2e/network/service.go
+++ b/test/e2e/network/service.go
@@ -3227,6 +3227,18 @@ func execAffinityTestForSessionAffinityTimeout(f *framework.Framework, cs client
 	serviceType := svc.Spec.Type
 	// set an affinity timeout equal to the number of connection requests
 	svcSessionAffinityTimeout := int32(AffinityConfirmCount)
+	if proxyMode, err := proxyMode(f); err == nil {
+		if proxyMode == "ipvs" {
+			// session affinity timeout must be greater than 120 in ipvs mode,
+			// because IPVS module has a hardcoded TIME_WAIT timeout of 120s,
+			// and that value can't be sysctl'ed now.
+			// Ref: https://github.com/torvalds/linux/blob/master/net/netfilter/ipvs/ip_vs_proto_tcp.c
+			// TODO: remove this to speed up testing when IPVS does really respect session affinity timeout
+			svcSessionAffinityTimeout = int32(125)
+		}
+	} else {
+		framework.Logf("Couldn't detect KubeProxy mode - test failure may be expected: %v", err)
+	}
 	svc.Spec.SessionAffinity = v1.ServiceAffinityClientIP
 	svc.Spec.SessionAffinityConfig = &v1.SessionAffinityConfig{
 		ClientIP: &v1.ClientIPConfig{TimeoutSeconds: &svcSessionAffinityTimeout},