From 04313ff1734ca23ebf690ca150389e588ce7d1c8 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Thu, 6 Nov 2014 21:26:26 -0800 Subject: [PATCH] Create ip_allocator that copes with IPv6 It uses a set (via a map) of allocated IPs --- pkg/registry/service/ip_allocator.go | 195 ++++++++++++++-------- pkg/registry/service/ip_allocator_test.go | 66 ++------ pkg/registry/service/rest_test.go | 8 + 3 files changed, 145 insertions(+), 124 deletions(-) diff --git a/pkg/registry/service/ip_allocator.go b/pkg/registry/service/ip_allocator.go index 947c0a64534..21f475b3061 100644 --- a/pkg/registry/service/ip_allocator.go +++ b/pkg/registry/service/ip_allocator.go @@ -18,17 +18,67 @@ package service import ( "fmt" + math_rand "math/rand" "net" "sync" + "time" "github.com/golang/glog" ) type ipAllocator struct { - subnet *net.IPNet - // TODO: This could be smarter, but for now a bitmap will suffice. lock sync.Mutex // protects 'used' - used []byte // a bitmap of allocated IPs + + subnet net.IPNet + ipSpaceSize int64 // Size of subnet, or -1 if it does not fit in an int64 + used ipAddrSet + randomAttempts int + + random *math_rand.Rand +} + +type ipAddrSet struct { + // We are pretty severely restricted in the types of things we can use as a key + ips map[string]bool +} + +func (s *ipAddrSet) Init() { + s.ips = map[string]bool{} +} + +// Adds to the ipAddrSet; returns true iff it was added (was not already in set) +func (s *ipAddrSet) Size() int { + return len(s.ips) +} + +func (s *ipAddrSet) Contains(ip net.IP) bool { + key := ip.String() + exists := s.ips[key] + return exists +} + +// Adds to the ipAddrSet; returns true iff it was added (was not already in set) +func (s *ipAddrSet) Add(ip net.IP) bool { + key := ip.String() + exists := s.ips[key] + if exists { + return false + } + s.ips[key] = true + return true +} + +// Removes from the ipAddrSet; returns true iff it was removed (was already in set) +func (s *ipAddrSet) Remove(ip net.IP) bool { + key := ip.String() + exists := s.ips[key] + if !exists { + return false + } + delete(s.ips, key) + // TODO: We probably should add this IP to an 'embargo' list for a limited amount of time + + return true } // The smallest number of IPs we accept. @@ -40,20 +90,42 @@ func newIPAllocator(subnet *net.IPNet) *ipAllocator { return nil } + seed := time.Now().UTC().UnixNano() + r := math_rand.New(math_rand.NewSource(seed)) + + ipSpaceSize := int64(-1) ones, bits := subnet.Mask.Size() - // TODO: some settings with IPv6 address could cause this to take - // an excessive amount of memory. - numIps := 1 << uint(bits-ones) - if numIps < minIPSpace { - glog.Errorf("IPAllocator requires at least %d IPs", minIPSpace) - return nil + if (bits-ones) < 63 { + ipSpaceSize = int64(1)<>= 8 - offset += result / 256 // carry + offset += result/256 // carry } return out } -// Subtract two IPs, returning the difference as an offset - used or splitting an IP into -// network addr and host addr parts. -func ipSub(lhs, rhs net.IP) int { - // If they are not the same length, normalize them. Make copies because net.IP is - // a slice underneath. Sneaky sneaky. - lhs = simplifyIP(lhs) - rhs = simplifyIP(rhs) - if len(lhs) != len(rhs) { - lhs = copyIP(lhs).To16() - rhs = copyIP(rhs).To16() - } - offset := 0 - borrow := 0 - // Loop from most-significant to least. - for i := range lhs { - offset <<= 8 - result := (int(lhs[i]) - borrow) - int(rhs[i]) - if result < 0 { - borrow = 1 - } else { - borrow = 0 - } - offset += result - } - return offset -} - // Get the optimal slice for an IP. IPv4 addresses will come back in a 4 byte slice. IPv6 // addresses will come back in a 16 byte slice. Non-IP arguments will produce nil. func simplifyIP(in net.IP) net.IP { @@ -176,9 +234,6 @@ func (ipa *ipAllocator) Release(ip net.IP) error { if !ipa.subnet.Contains(ip) { return fmt.Errorf("IP %s does not fall within subnet %s", ip, ipa.subnet) } - offset := ipSub(ip, ipa.subnet.IP) - i := offset / 8 - m := byte(1 << byte(offset%8)) - ipa.used[i] &^= m + ipa.used.Remove(ip) return nil } diff --git a/pkg/registry/service/ip_allocator_test.go b/pkg/registry/service/ip_allocator_test.go index db5147806f1..677015de31c 100644 --- a/pkg/registry/service/ip_allocator_test.go +++ b/pkg/registry/service/ip_allocator_test.go @@ -36,13 +36,16 @@ func TestNew(t *testing.T) { if ipa == nil { t.Errorf("expected non-nil") } - if len(ipa.used) != 128 { // a /22 has 1024 IPs, 8 per byte = 128 - t.Errorf("wrong size for ipa.used") + if ipa.ipSpaceSize != 1024 { + t.Errorf("wrong size for ipa.ipSpaceSize") } - if ipa.used[0] != 0x01 { + if ipa.used.Size() != 2 { + t.Errorf("wrong size() for ipa.used") + } + if !ipa.used.Contains(net.ParseIP("93.76.0.0")) { t.Errorf("network address was not reserved") } - if ipa.used[127] != 0x80 { + if !ipa.used.Contains(net.ParseIP("93.76.3.255")) { t.Errorf("broadcast address was not reserved") } } @@ -68,6 +71,9 @@ func TestAllocateNext(t *testing.T) { _, ipnet, _ := net.ParseCIDR("93.76.0.0/22") ipa := newIPAllocator(ipnet) + // Turn off random allocation attempts, so we just allocate in sequence + ipa.randomAttempts = 0 + ip1, err := ipa.AllocateNext() if err != nil { t.Error(err) @@ -128,6 +134,8 @@ func TestRelease(t *testing.T) { _, ipnet, _ := net.ParseCIDR("93.76.0.0/24") ipa := newIPAllocator(ipnet) + ipa.randomAttempts = 0 + err := ipa.Release(net.ParseIP("1.2.3.4")) if err == nil { t.Errorf("Expected an error") @@ -172,31 +180,6 @@ func TestRelease(t *testing.T) { } } -func TestFFS(t *testing.T) { - _, err := ffs(0) - if err == nil { - t.Errorf("Expected error") - } - - testCases := []struct { - value byte - expected uint - }{ - {0x01, 0}, {0x02, 1}, {0x04, 2}, {0x08, 3}, - {0x10, 4}, {0x20, 5}, {0x40, 6}, {0x80, 7}, - {0x22, 1}, {0xa0, 5}, {0xfe, 1}, {0xff, 0}, - } - for _, tc := range testCases { - r, err := ffs(tc.value) - if err != nil { - t.Error(err) - } - if r != tc.expected { - t.Errorf("Expected %d, got %d", tc.expected, r) - } - } -} - func TestIPAdd(t *testing.T) { testCases := []struct { ip string @@ -223,31 +206,6 @@ func TestIPAdd(t *testing.T) { } } -func TestIPSub(t *testing.T) { - testCases := []struct { - lhs string - rhs string - expected int - }{ - {"1.2.3.0", "1.2.3.0", 0}, - {"1.2.3.1", "1.2.3.0", 1}, - {"1.2.3.255", "1.2.3.0", 255}, - {"1.2.4.0", "1.2.3.0", 256}, - {"1.2.4.0", "1.2.3.255", 1}, - {"1.2.4.1", "1.2.3.0", 257}, - {"1.3.3.0", "1.2.3.0", 65536}, - {"1.2.3.5", "1.2.3.4", 1}, - {"0.0.0.0", "0.0.0.1", -1}, - {"0.0.1.0", "0.0.0.1", 255}, - } - for _, tc := range testCases { - r := ipSub(net.ParseIP(tc.lhs), net.ParseIP(tc.rhs)) - if r != tc.expected { - t.Errorf("Expected %v, got %v", tc.expected, r) - } - } -} - func TestCopyIP(t *testing.T) { ip1 := net.ParseIP("1.2.3.4") ip2 := copyIP(ip1) diff --git a/pkg/registry/service/rest_test.go b/pkg/registry/service/rest_test.go index 0ddf6435fcc..4533b53f816 100644 --- a/pkg/registry/service/rest_test.go +++ b/pkg/registry/service/rest_test.go @@ -44,6 +44,8 @@ func TestServiceRegistryCreate(t *testing.T) { fakeCloud := &cloud.FakeCloud{} machines := []string{"foo", "bar", "baz"} storage := NewREST(registry, fakeCloud, registrytest.NewMinionRegistry(machines, api.NodeResources{}), makeIPNet(t)) + storage.portalMgr.randomAttempts = 0 + svc := &api.Service{ Port: 6502, ObjectMeta: api.ObjectMeta{Name: "foo"}, @@ -414,6 +416,7 @@ func TestServiceRegistryIPAllocation(t *testing.T) { fakeCloud := &cloud.FakeCloud{} machines := []string{"foo", "bar", "baz"} rest := NewREST(registry, fakeCloud, registrytest.NewMinionRegistry(machines, api.NodeResources{}), makeIPNet(t)) + rest.portalMgr.randomAttempts = 0 svc1 := &api.Service{ Port: 6502, @@ -467,6 +470,7 @@ func TestServiceRegistryIPReallocation(t *testing.T) { fakeCloud := &cloud.FakeCloud{} machines := []string{"foo", "bar", "baz"} rest := NewREST(registry, fakeCloud, registrytest.NewMinionRegistry(machines, api.NodeResources{}), makeIPNet(t)) + rest.portalMgr.randomAttempts = 0 svc1 := &api.Service{ Port: 6502, @@ -509,6 +513,7 @@ func TestServiceRegistryIPUpdate(t *testing.T) { fakeCloud := &cloud.FakeCloud{} machines := []string{"foo", "bar", "baz"} rest := NewREST(registry, fakeCloud, registrytest.NewMinionRegistry(machines, api.NodeResources{}), makeIPNet(t)) + rest.portalMgr.randomAttempts = 0 svc := &api.Service{ Port: 6502, @@ -561,6 +566,7 @@ func TestServiceRegistryIPExternalLoadBalancer(t *testing.T) { fakeCloud := &cloud.FakeCloud{} machines := []string{"foo", "bar", "baz"} rest := NewREST(registry, fakeCloud, registrytest.NewMinionRegistry(machines, api.NodeResources{}), makeIPNet(t)) + rest.portalMgr.randomAttempts = 0 svc := &api.Service{ Port: 6502, @@ -588,6 +594,7 @@ func TestServiceRegistryIPReloadFromStorage(t *testing.T) { fakeCloud := &cloud.FakeCloud{} machines := []string{"foo", "bar", "baz"} rest1 := NewREST(registry, fakeCloud, registrytest.NewMinionRegistry(machines, api.NodeResources{}), makeIPNet(t)) + rest1.portalMgr.randomAttempts = 0 svc := &api.Service{ Port: 6502, @@ -607,6 +614,7 @@ func TestServiceRegistryIPReloadFromStorage(t *testing.T) { // This will reload from storage, finding the previous 2 rest2 := NewREST(registry, fakeCloud, registrytest.NewMinionRegistry(machines, api.NodeResources{}), makeIPNet(t)) + rest2.portalMgr.randomAttempts = 0 svc = &api.Service{ Port: 6502,