github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-13 13:14:05 +00:00
Code Issues Projects Releases Wiki Activity

Implement multi-port endpoints

Browse Source 
Instead of endpoints being a flat list, it is now a list of "subsets"
where each is a struct of {Addresses, Ports}.  To generate the list of
endpoints you need to take union of the Cartesian products of the
subsets.  This is compact in the vast majority of cases, yet still
represents named ports and corner cases (e.g. each pod has a different
port number).

This also stores subsets in a deterministic order (sorted by hash) to
avoid spurious updates and comparison problems.

This is a fully compatible change - old objects and clients will
keepworking as long as they don't need the new functionality.

This is the prep for multi-port Services, which will add API to produce
endpoints in this new structure.
This commit is contained in:
Tim Hockin
2015-03-20 14:24:43 -07:00
parent 3eda80b3c5
commit 8ae203825b
43 changed files with 2090 additions and 1030 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
pkg/master/publish.go
View File

@@ -18,6 +18,7 @@ package master
import (
"net"
"reflect"
"time"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@@ -40,7 +41,7 @@ func (m *Master) serviceWriterLoop(stop chan struct{}) {
if err := m.createMasterServiceIfNeeded("kubernetes", m.serviceReadWriteIP, m.serviceReadWritePort); err != nil {
glog.Errorf("Can't create rw service: %v", err)
}
if err := m.ensureEndpointsContain("kubernetes", m.clusterIP, m.publicReadWritePort); err != nil {
if err := m.setEndpoints("kubernetes", m.clusterIP, m.publicReadWritePort); err != nil {
glog.Errorf("Can't create rw endpoints: %v", err)
}
}
@@ -65,7 +66,7 @@ func (m *Master) roServiceWriterLoop(stop chan struct{}) {
if err := m.createMasterServiceIfNeeded("kubernetes-ro", m.serviceReadOnlyIP, m.serviceReadOnlyPort); err != nil {
glog.Errorf("Can't create ro service: %v", err)
}
if err := m.ensureEndpointsContain("kubernetes-ro", m.clusterIP, m.publicReadOnlyPort); err != nil {
if err := m.setEndpoints("kubernetes-ro", m.clusterIP, m.publicReadOnlyPort); err != nil {
glog.Errorf("Can't create ro endpoints: %v", err)
}
}
@@ -128,37 +129,28 @@ func (m *Master) createMasterServiceIfNeeded(serviceName string, serviceIP net.I
return err
}
// ensureEndpointsContain sets the endpoints for the given service. Also removes
// excess endpoints (as determined by m.masterCount). Extra endpoints could appear
// in the list if, for example, the master starts running on a different machine,
// changing IP addresses.
func (m *Master) ensureEndpointsContain(serviceName string, ip net.IP, port int) error {
// setEndpoints sets the endpoints for the given service.
// TODO: in a multi-master scenario this needs to consider all masters.
func (m *Master) setEndpoints(serviceName string, ip net.IP, port int) error {
// The setting we want to find.
want := []api.EndpointSubset{{
Addresses: []api.EndpointAddress{{IP: ip.String()}},
Ports: []api.EndpointPort{{Port: port, Protocol: api.ProtocolTCP}},
}}
ctx := api.NewDefaultContext()
e, err := m.endpointRegistry.GetEndpoints(ctx, serviceName)
if err != nil || e.Protocol != api.ProtocolTCP {
if err != nil {
e = &api.Endpoints{
ObjectMeta: api.ObjectMeta{
Name: serviceName,
Namespace: api.NamespaceDefault,
},
Protocol: api.ProtocolTCP,
}
}
found := false
for i := range e.Endpoints {
ep := &e.Endpoints[i]
if ep.IP == ip.String() && ep.Port == port {
found = true
break
}
}
if !found {
e.Endpoints = append(e.Endpoints, api.Endpoint{IP: ip.String(), Port: port})
if len(e.Endpoints) > m.masterCount {
// We append to the end and remove from the beginning, so this should
// converge rapidly with all masters performing this operation.
e.Endpoints = e.Endpoints[len(e.Endpoints)-m.masterCount:]
}
if !reflect.DeepEqual(e.Subsets, want) {
e.Subsets = want
glog.Infof("setting endpoints for master service %q to %v", serviceName, e)
return m.endpointRegistry.UpdateEndpoints(ctx, e)
}
// We didn't make any changes, no need to actually call update.

298
pkg/master/publish_test.go
View File

@@ -19,245 +19,149 @@ package master
import (
"net"
"reflect"
"sync"
"testing"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/registry/registrytest"
)
func TestEnsureEndpointsContain(t *testing.T) {
func TestSetEndpoints(t *testing.T) {
tests := []struct {
serviceName string
ip string
port int
expectError bool
expectUpdate bool
endpoints *api.EndpointsList
expectedEndpoints []api.Endpoint
err error
masterCount int
testName string
serviceName string
ip string
port int
endpoints *api.EndpointsList
expectUpdate bool
}{
{
testName: "no existing endpoints",
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
expectError: false,
endpoints: nil,
expectUpdate: true,
masterCount: 1,
},
{
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
expectError: false,
testName: "existing endpoints satisfy",
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
endpoints: &api.EndpointsList{
Items: []api.Endpoints{{
ObjectMeta: api.ObjectMeta{Name: "foo"},
Subsets: []api.EndpointSubset{{
Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
Ports: []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
}},
}},
},
expectUpdate: false,
endpoints: &api.EndpointsList{
Items: []api.Endpoints{
{
ObjectMeta: api.ObjectMeta{
Name: "foo",
},
Endpoints: []api.Endpoint{
{
IP: "1.2.3.4",
Port: 8080,
},
},
Protocol: api.ProtocolTCP,
},
},
},
masterCount: 1,
expectedEndpoints: []api.Endpoint{{"1.2.3.4", 8080, nil}},
},
{
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
expectError: false,
expectUpdate: true,
testName: "existing endpoints satisfy but too many",
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
endpoints: &api.EndpointsList{
Items: []api.Endpoints{
{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: api.NamespaceDefault,
},
Endpoints: []api.Endpoint{
{
IP: "4.3.2.1",
Port: 8080,
},
},
Protocol: api.ProtocolTCP,
},
},
Items: []api.Endpoints{{
ObjectMeta: api.ObjectMeta{Name: "foo"},
Subsets: []api.EndpointSubset{{
Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}, {IP: "4.3.2.1"}},
Ports: []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
}},
}},
},
masterCount: 1,
expectUpdate: true,
},
{
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
expectError: false,
expectUpdate: true,
testName: "existing endpoints wrong name",
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
endpoints: &api.EndpointsList{
Items: []api.Endpoints{
{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: api.NamespaceDefault,
},
Endpoints: []api.Endpoint{
{
IP: "4.3.2.1",
Port: 9090,
},
},
Protocol: api.ProtocolTCP,
},
},
Items: []api.Endpoints{{
ObjectMeta: api.ObjectMeta{Name: "bar"},
Subsets: []api.EndpointSubset{{
Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
Ports: []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
}},
}},
},
masterCount: 2,
expectedEndpoints: []api.Endpoint{{"4.3.2.1", 9090, nil}, {"1.2.3.4", 8080, nil}},
expectUpdate: true,
},
{
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
expectError: false,
expectUpdate: true,
testName: "existing endpoints wrong IP",
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
endpoints: &api.EndpointsList{
Items: []api.Endpoints{
{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: api.NamespaceDefault,
},
Endpoints: []api.Endpoint{
{
IP: "4.3.2.1",
Port: 9090,
},
{
IP: "1.2.3.4",
Port: 8000,
},
},
Protocol: api.ProtocolTCP,
},
},
Items: []api.Endpoints{{
ObjectMeta: api.ObjectMeta{Name: "foo"},
Subsets: []api.EndpointSubset{{
Addresses: []api.EndpointAddress{{IP: "4.3.2.1"}},
Ports: []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
}},
}},
},
masterCount: 2,
expectedEndpoints: []api.Endpoint{{"1.2.3.4", 8000, nil}, {"1.2.3.4", 8080, nil}},
expectUpdate: true,
},
{
testName: "existing endpoints wrong port",
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
endpoints: &api.EndpointsList{
Items: []api.Endpoints{{
ObjectMeta: api.ObjectMeta{Name: "foo"},
Subsets: []api.EndpointSubset{{
Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
Ports: []api.EndpointPort{{Port: 9090, Protocol: "TCP"}},
}},
}},
},
expectUpdate: true,
},
{
testName: "existing endpoints wrong protocol",
serviceName: "foo",
ip: "1.2.3.4",
port: 8080,
endpoints: &api.EndpointsList{
Items: []api.Endpoints{{
ObjectMeta: api.ObjectMeta{Name: "foo"},
Subsets: []api.EndpointSubset{{
Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
Ports: []api.EndpointPort{{Port: 8080, Protocol: "UDP"}},
}},
}},
},
expectUpdate: true,
},
}
for _, test := range tests {
master := Master{}
registry := &registrytest.EndpointRegistry{
Endpoints: test.endpoints,
Err: test.err,
}
master.endpointRegistry = registry
master.masterCount = test.masterCount
err := master.ensureEndpointsContain(test.serviceName, net.ParseIP(test.ip), test.port)
if test.expectError && err == nil {
t.Errorf("unexpected non-error")
}
if !test.expectError && err != nil {
t.Errorf("unexpected error: %v", err)
err := master.setEndpoints(test.serviceName, net.ParseIP(test.ip), test.port)
if err != nil {
t.Errorf("case %q: unexpected error: %v", test.testName, err)
}
if test.expectUpdate {
if test.expectedEndpoints == nil {
test.expectedEndpoints = []api.Endpoint{{test.ip, test.port, nil}}
}
expectedUpdate := api.Endpoints{
ObjectMeta: api.ObjectMeta{
Name: test.serviceName,
Namespace: "default",
},
Endpoints: test.expectedEndpoints,
Protocol: "TCP",
}
expectedSubsets := []api.EndpointSubset{{
Addresses: []api.EndpointAddress{{IP: "1.2.3.4"}},
Ports: []api.EndpointPort{{Port: 8080, Protocol: "TCP"}},
}}
if len(registry.Updates) != 1 {
t.Errorf("unexpected updates: %v", registry.Updates)
} else if !reflect.DeepEqual(expectedUpdate, registry.Updates[0]) {
t.Errorf("expected update:\n%#v\ngot:\n%#v\n", expectedUpdate, registry.Updates[0])
t.Errorf("case %q: unexpected updates: %v", test.testName, registry.Updates)
} else if !reflect.DeepEqual(expectedSubsets, registry.Updates[0].Subsets) {
t.Errorf("case %q: expected update:\n%#v\ngot:\n%#v\n", test.testName, expectedSubsets, registry.Updates[0].Subsets)
}
}
if !test.expectUpdate && len(registry.Updates) > 0 {
t.Errorf("no update expected, yet saw: %v", registry.Updates)
}
}
}
func TestEnsureEndpointsContainConverges(t *testing.T) {
master := Master{}
registry := &registrytest.EndpointRegistry{
Endpoints: &api.EndpointsList{
Items: []api.Endpoints{
{
ObjectMeta: api.ObjectMeta{
Name: "foo",
Namespace: api.NamespaceDefault,
},
Endpoints: []api.Endpoint{
{
IP: "4.3.2.1",
Port: 9000,
},
{
IP: "1.2.3.4",
Port: 8000,
},
},
Protocol: api.ProtocolTCP,
},
},
},
}
master.endpointRegistry = registry
master.masterCount = 2
// This is purposefully racy, it shouldn't matter the order that these things arrive,
// we should still converge on the right answer.
wg := sync.WaitGroup{}
wg.Add(2)
go func() {
for i := 0; i < 10; i++ {
if err := master.ensureEndpointsContain("foo", net.ParseIP("4.3.2.1"), 9090); err != nil {
t.Errorf("unexpected error: %v", err)
t.Fail()
}
}
wg.Done()
}()
go func() {
for i := 0; i < 10; i++ {
if err := master.ensureEndpointsContain("foo", net.ParseIP("1.2.3.4"), 8080); err != nil {
t.Errorf("unexpected error: %v", err)
t.Fail()
}
}
wg.Done()
}()
wg.Wait()
// We should see at least two updates.
if len(registry.Updates) > 2 {
t.Errorf("unexpected updates: %v", registry.Updates)
}
// Pick up the last update and validate.
endpoints := registry.Updates[len(registry.Updates)-1]
if len(endpoints.Endpoints) != 2 {
t.Errorf("unexpected update: %v", endpoints)
}
for _, endpoint := range endpoints.Endpoints {
if endpoint.IP == "4.3.2.1" && endpoint.Port != 9090 {
t.Errorf("unexpected endpoint state: %v", endpoint)
}
if endpoint.IP == "1.2.3.4" && endpoint.Port != 8080 {
t.Errorf("unexpected endpoint state: %v", endpoint)
t.Errorf("case %q: no update expected, yet saw: %v", test.testName, registry.Updates)
}
}
}