github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-31 15:25:57 +00:00
Code Issues Projects Releases Wiki Activity

Fixes #3640 by shuffling endpoints in the round-robin load balancer

Browse Source
This commit is contained in:
Steve Reed 2015-01-20 16:01:01 -08:00
parent 2863fa9fe6
commit 79a6bfb95f
2 changed files with 150 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/proxy/roundrobin.go
View File

@ -18,8 +18,10 @@ package proxy
import (
"errors"
"math/rand"
"net"
"reflect"
"sort"
"strconv"
"sync"
"time"
@ -170,6 +172,15 @@ func filterValidEndpoints(endpoints []string) []string {
return result
}
func shuffleEndpoints(endpoints []string) []string {
shuffled := make([]string, len(endpoints))
perm := rand.Perm(len(endpoints))
for i, v := range perm {
shuffled[v] = endpoints[i]
}
return shuffled
}
//remove any session affinity records associated to a particular endpoint (for example when a pod goes down).
func removeSessionAffinityByEndpoint(lb *LoadBalancerRR, service string, endpoint string) {
for _, affinityDetail := range lb.serviceDtlMap[service].sessionAffinityMap {
@ -210,6 +221,10 @@ func (lb *LoadBalancerRR) OnUpdate(endpoints []api.Endpoints) {
for _, endpoint := range endpoints {
existingEndpoints, exists := lb.endpointsMap[endpoint.Name]
validEndpoints := filterValidEndpoints(endpoint.Endpoints)
// Need to compare sorted endpoints here, since they are shuffled below
// before being put into endpointsMap
sort.Strings(existingEndpoints)
sort.Strings(validEndpoints)
if !exists || !reflect.DeepEqual(existingEndpoints, validEndpoints) {
glog.V(3).Infof("LoadBalancerRR: Setting endpoints for %s to %+v", endpoint.Name, endpoint.Endpoints)
updateServiceDetailMap(lb, endpoint.Name, validEndpoints)
@ -217,7 +232,7 @@ func (lb *LoadBalancerRR) OnUpdate(endpoints []api.Endpoints) {
// to be safe we will call it here. A new service will only be created
// if one does not already exist.
lb.NewService(endpoint.Name, api.AffinityTypeNone, 0)
lb.endpointsMap[endpoint.Name] = validEndpoints
lb.endpointsMap[endpoint.Name] = shuffleEndpoints(validEndpoints)
// Reset the round-robin index.
lb.rrIndex[endpoint.Name] = 0

222
pkg/proxy/roundrobin_test.go
View File

@ -18,6 +18,7 @@ package proxy
import (
"net"
"sort"
"testing"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@ -56,6 +57,26 @@ func TestFilterWorks(t *testing.T) {
}
}
func TestShuffleWorks(t *testing.T) {
endpoints := []string{"foobar:1", "foobar:2", "foobar:3"}
shuffled := shuffleEndpoints(endpoints)
if len(shuffled) != 3 {
t.Errorf("Failed to shuffle to the correct size")
}
sort.Strings(shuffled)
if shuffled[0] != "foobar:1" {
t.Errorf("Index zero is not foobar:1")
}
if shuffled[1] != "foobar:2" {
t.Errorf("Index one is not foobar:2")
}
if shuffled[2] != "foobar:3" {
t.Errorf("Index two is not foobar:3")
}
}
func TestLoadBalanceFailsWithNoEndpoints(t *testing.T) {
loadBalancer := NewLoadBalancerRR()
var endpoints []api.Endpoints
@ -75,7 +96,7 @@ func expectEndpoint(t *testing.T, loadBalancer *LoadBalancerRR, service string,
t.Errorf("Didn't find a service for %s, expected %s, failed with: %v", service, expected, err)
}
if endpoint != expected {
t.Errorf("Didn't get expected endpoint for service %s, expected %s, got: %s", service, expected, endpoint)
t.Errorf("Didn't get expected endpoint for service %s client %v, expected %s, got: %s", service, netaddr, expected, endpoint)
}
}
@ -109,10 +130,11 @@ func TestLoadBalanceWorksWithMultipleEndpoints(t *testing.T) {
Endpoints: []string{"endpoint:1", "endpoint:2", "endpoint:3"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", nil)
shuffledEndpoints := loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[2], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], nil)
}
func TestLoadBalanceWorksWithMultipleEndpointsAndUpdates(t *testing.T) {
@ -127,21 +149,23 @@ func TestLoadBalanceWorksWithMultipleEndpointsAndUpdates(t *testing.T) {
Endpoints: []string{"endpoint:1", "endpoint:2", "endpoint:3"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", nil)
shuffledEndpoints := loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[2], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], nil)
// Then update the configuration with one fewer endpoints, make sure
// we start in the beginning again
endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"},
Endpoints: []string{"endpoint:8", "endpoint:9"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:8", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:9", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:8", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:9", nil)
shuffledEndpoints = loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], nil)
// Clear endpoints
endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Endpoints: []string{}}
loadBalancer.OnUpdate(endpoints)
@ -168,17 +192,19 @@ func TestLoadBalanceWorksWithServiceRemoval(t *testing.T) {
Endpoints: []string{"endpoint:4", "endpoint:5"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", nil)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", nil)
shuffledFooEndpoints := loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[1], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[2], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[1], nil)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", nil)
expectEndpoint(t, loadBalancer, "bar", "endpoint:5", nil)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", nil)
expectEndpoint(t, loadBalancer, "bar", "endpoint:5", nil)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", nil)
shuffledBarEndpoints := loadBalancer.endpointsMap["bar"]
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[1], nil)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[1], nil)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[0], nil)
// Then update the configuration by removing foo
loadBalancer.OnUpdate(endpoints[1:])
@ -188,10 +214,10 @@ func TestLoadBalanceWorksWithServiceRemoval(t *testing.T) {
}
// but bar is still there, and we continue RR from where we left off.
expectEndpoint(t, loadBalancer, "bar", "endpoint:5", nil)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", nil)
expectEndpoint(t, loadBalancer, "bar", "endpoint:5", nil)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", nil)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[1], nil)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[0], nil)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[1], nil)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[0], nil)
}
func TestStickyLoadBalanceWorksWithSingleEndpoint(t *testing.T) {
@ -232,14 +258,15 @@ func TestStickyLoadBalanaceWorksWithMultipleEndpoints(t *testing.T) {
Endpoints: []string{"endpoint:1", "endpoint:2", "endpoint:3"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", client3)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", client3)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
shuffledEndpoints := loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[2], client3)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[2], client3)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
}
func TestStickyLoadBalanaceWorksWithMultipleEndpointsStickyNone(t *testing.T) {
@ -259,14 +286,15 @@ func TestStickyLoadBalanaceWorksWithMultipleEndpointsStickyNone(t *testing.T) {
Endpoints: []string{"endpoint:1", "endpoint:2", "endpoint:3"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client3)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", client3)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client1)
shuffledEndpoints := loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[2], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client3)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[2], client3)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client1)
}
func TestStickyLoadBalanaceWorksWithMultipleEndpointsRemoveOne(t *testing.T) {
@ -289,32 +317,45 @@ func TestStickyLoadBalanaceWorksWithMultipleEndpointsRemoveOne(t *testing.T) {
Endpoints: []string{"endpoint:1", "endpoint:2", "endpoint:3"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", client3)
shuffledEndpoints := loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
client1Endpoint := shuffledEndpoints[0]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
client2Endpoint := shuffledEndpoints[1]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[2], client3)
client3Endpoint := shuffledEndpoints[2]
endpoints[0] = api.Endpoints{
ObjectMeta: api.ObjectMeta{Name: "foo"},
Endpoints: []string{"endpoint:1", "endpoint:2"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client3)
shuffledEndpoints = loadBalancer.endpointsMap["foo"]
if client1Endpoint == "endpoint:3" {
client1Endpoint = shuffledEndpoints[0]
} else if client2Endpoint == "endpoint:3" {
client2Endpoint = shuffledEndpoints[0]
} else if client3Endpoint == "endpoint:3" {
client3Endpoint = shuffledEndpoints[0]
}
expectEndpoint(t, loadBalancer, "foo", client1Endpoint, client1)
expectEndpoint(t, loadBalancer, "foo", client2Endpoint, client2)
expectEndpoint(t, loadBalancer, "foo", client3Endpoint, client3)
endpoints[0] = api.Endpoints{
ObjectMeta: api.ObjectMeta{Name: "foo"},
Endpoints: []string{"endpoint:1", "endpoint:2", "endpoint:4"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client3)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client4)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client5)
expectEndpoint(t, loadBalancer, "foo", "endpoint:4", client6)
shuffledEndpoints = loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", client1Endpoint, client1)
expectEndpoint(t, loadBalancer, "foo", client2Endpoint, client2)
expectEndpoint(t, loadBalancer, "foo", client3Endpoint, client3)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client4)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client5)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[2], client6)
}
func TestStickyLoadBalanceWorksWithMultipleEndpointsAndUpdates(t *testing.T) {
@ -334,24 +375,26 @@ func TestStickyLoadBalanceWorksWithMultipleEndpointsAndUpdates(t *testing.T) {
Endpoints: []string{"endpoint:1", "endpoint:2", "endpoint:3"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", client3)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
shuffledEndpoints := loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[2], client3)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
// Then update the configuration with one fewer endpoints, make sure
// we start in the beginning again
endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"},
Endpoints: []string{"endpoint:4", "endpoint:5"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:4", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:5", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:4", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:5", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:5", client2)
shuffledEndpoints = loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledEndpoints[1], client2)
// Clear endpoints
endpoints[0] = api.Endpoints{ObjectMeta: api.ObjectMeta{Name: "foo"}, Endpoints: []string{}}
@ -384,19 +427,21 @@ func TestStickyLoadBalanceWorksWithServiceRemoval(t *testing.T) {
Endpoints: []string{"endpoint:4", "endpoint:5"},
}
loadBalancer.OnUpdate(endpoints)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", client3)
expectEndpoint(t, loadBalancer, "foo", "endpoint:3", client3)
expectEndpoint(t, loadBalancer, "foo", "endpoint:1", client1)
expectEndpoint(t, loadBalancer, "foo", "endpoint:2", client2)
shuffledFooEndpoints := loadBalancer.endpointsMap["foo"]
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[2], client3)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[2], client3)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", client1)
expectEndpoint(t, loadBalancer, "bar", "endpoint:5", client2)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", client1)
expectEndpoint(t, loadBalancer, "bar", "endpoint:5", client2)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", client1)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", client1)
shuffledBarEndpoints := loadBalancer.endpointsMap["bar"]
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "foo", shuffledFooEndpoints[0], client1)
// Then update the configuration by removing foo
loadBalancer.OnUpdate(endpoints[1:])
@ -406,10 +451,11 @@ func TestStickyLoadBalanceWorksWithServiceRemoval(t *testing.T) {
}
// but bar is still there, and we continue RR from where we left off.
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", client1)
expectEndpoint(t, loadBalancer, "bar", "endpoint:5", client2)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", client1)
expectEndpoint(t, loadBalancer, "bar", "endpoint:5", client2)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", client1)
expectEndpoint(t, loadBalancer, "bar", "endpoint:4", client1)
shuffledBarEndpoints = loadBalancer.endpointsMap["bar"]
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[1], client2)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[0], client1)
expectEndpoint(t, loadBalancer, "bar", shuffledBarEndpoints[0], client1)
}