Handle the case of a pod IP being reused while the old Pod still exists

If the pod network reuses a pod IP while the old pod is still
terminating, then we may temporarily see two Endpoints for that IP. In
that case, prefer the non-terminating one.

pkg/proxy/endpointslicecache.go

@@ -237,7 +237,14 @@ func (cache *EndpointSliceCache) addEndpoints(svcPortName *ServicePortName, port
 		// more-recently-updated EndpointSlice, since it may have newer
 		// conditions. But we can't easily figure that out, and the situation will
 		// resolve itself once we receive the second EndpointSlice update anyway.
-		if _, exists := endpointSet[endpointInfo.String()]; !exists {
+		//
+		// On the other hand, there maybe also be two *different* Endpoints (i.e.,
+		// with different targetRefs) that point to the same IP, if the pod
+		// network reuses the IP from a terminating pod before the Pod object is
+		// fully deleted. In this case we want to prefer the running pod over the
+		// terminating one. (If there are multiple non-terminating pods with the
+		// same podIP, then the result is undefined.)
+		if _, exists := endpointSet[endpointInfo.String()]; !exists || !terminating {
 			endpointSet[endpointInfo.String()] = cache.makeEndpointInfo(endpointInfo, svcPortName)
 		}
 	}

pkg/proxy/endpointslicecache_test.go

@@ -311,6 +311,48 @@ func TestEndpointInfoByServicePort(t *testing.T) {
 				},
 			},
 		},
+		"with duplicate Endpoints, prefer non-terminating": {
+			namespacedName: types.NamespacedName{Name: "svc1", Namespace: "ns1"},
+			hostname:       "host1",
+			endpointSlices: []*discovery.EndpointSlice{
+				func() *discovery.EndpointSlice {
+					es := generateEndpointSliceWithOffset("svc1", "ns1", 1, 1, 1, 999, 999, []string{"host1"}, []*int32{ptr.To[int32](80)})
+
+					// Duplicate the endpoint
+					ep1 := es.Endpoints[0]
+					ep1.Conditions.Ready = ptr.To(false)
+					ep1.Conditions.Serving = ptr.To(true)
+					ep1.Conditions.Terminating = ptr.To(true)
+
+					ep2 := es.Endpoints[0]
+					ep2.Conditions.Ready = ptr.To(true)
+					ep2.Conditions.Serving = ptr.To(true)
+					ep2.Conditions.Terminating = ptr.To(false)
+
+					ep3 := es.Endpoints[0]
+					ep3.Conditions.Ready = ptr.To(false)
+					ep3.Conditions.Serving = ptr.To(false)
+					ep3.Conditions.Terminating = ptr.To(true)
+
+					//  ep2 should win since it's the non-terminating one
+					es.Endpoints = []discovery.Endpoint{ep1, ep2, ep3}
+					return es
+				}(),
+			},
+			expectedMap: spToEndpointMap{
+				makeServicePortName("ns1", "svc1", "port-0", v1.ProtocolTCP): {
+					"10.0.1.1:80": &BaseEndpointInfo{
+						ip:          "10.0.1.1",
+						port:        80,
+						endpoint:    "10.0.1.1:80",
+						isLocal:     true,
+						ready:       true,
+						serving:     true,
+						terminating: false,
+					},
+				},
+			},
+		},
 	}
 
 	for name, tc := range testCases {