From 96d71f01ebbd16c63a60d3652a6aa066774eb5d5 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Tue, 11 Jan 2022 09:23:13 +0100 Subject: [PATCH] new bitmap allocator with offset Implement a new bitmap allocator using an allocation strategy that accepts and offset and subdivides the range based on this offset, prioritizing the upper range for dynamic allocation. --- pkg/registry/core/service/allocator/bitmap.go | 56 +- .../core/service/allocator/bitmap_test.go | 599 +++++++++++++++--- .../core/service/allocator/interfaces.go | 2 + 3 files changed, 558 insertions(+), 99 deletions(-) diff --git a/pkg/registry/core/service/allocator/bitmap.go b/pkg/registry/core/service/allocator/bitmap.go index c5dad0467f0..3c12864c114 100644 --- a/pkg/registry/core/service/allocator/bitmap.go +++ b/pkg/registry/core/service/allocator/bitmap.go @@ -18,6 +18,7 @@ package allocator import ( "errors" + "fmt" "math/big" "math/rand" "sync" @@ -60,15 +61,25 @@ type bitAllocator interface { // NewAllocationMap creates an allocation bitmap using the random scan strategy. func NewAllocationMap(max int, rangeSpec string) *AllocationBitmap { + return NewAllocationMapWithOffset(max, rangeSpec, 0) +} + +// NewAllocationMapWithOffset creates an allocation bitmap using a random scan strategy that +// allows to pass an offset that divides the allocation bitmap in two blocks. +// The first block of values will not be used for random value assigned by the AllocateNext() +// method until the second block of values has been exhausted. +func NewAllocationMapWithOffset(max int, rangeSpec string, offset int) *AllocationBitmap { a := AllocationBitmap{ - strategy: randomScanStrategy{ - rand: rand.New(rand.NewSource(time.Now().UnixNano())), + strategy: randomScanStrategyWithOffset{ + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + offset: offset, }, allocated: big.NewInt(0), count: 0, max: max, rangeSpec: rangeSpec, } + return &a } @@ -78,6 +89,10 @@ func (r *AllocationBitmap) Allocate(offset int) (bool, error) { r.lock.Lock() defer r.lock.Unlock() + // max is the maximum size of the usable items in the range + if offset < 0 || offset >= r.max { + return false, fmt.Errorf("offset %d out of range [0,%d]", offset, r.max) + } if r.allocated.Bit(offset) == 1 { return false, nil } @@ -205,3 +220,40 @@ func (rss randomScanStrategy) AllocateBit(allocated *big.Int, max, count int) (i } var _ bitAllocator = randomScanStrategy{} + +// randomScanStrategyWithOffset choose a random address from the provided big.Int and then scans +// forward looking for the next available address. The big.Int range is subdivided so it will try +// to allocate first from the reserved upper range of addresses (it will wrap the upper subrange if necessary). +// If there is no free address it will try to allocate one from the lower range too. +type randomScanStrategyWithOffset struct { + rand *rand.Rand + offset int +} + +func (rss randomScanStrategyWithOffset) AllocateBit(allocated *big.Int, max, count int) (int, bool) { + if count >= max { + return 0, false + } + // size of the upper subrange, prioritized for random allocation + subrangeMax := max - rss.offset + // try to get a value from the upper range [rss.reserved, max] + start := rss.rand.Intn(subrangeMax) + for i := 0; i < subrangeMax; i++ { + at := rss.offset + ((start + i) % subrangeMax) + if allocated.Bit(at) == 0 { + return at, true + } + } + + start = rss.rand.Intn(rss.offset) + // subrange full, try to get the value from the first block before giving up. + for i := 0; i < rss.offset; i++ { + at := (start + i) % rss.offset + if allocated.Bit(at) == 0 { + return i, true + } + } + return 0, false +} + +var _ bitAllocator = randomScanStrategyWithOffset{} diff --git a/pkg/registry/core/service/allocator/bitmap_test.go b/pkg/registry/core/service/allocator/bitmap_test.go index b5eacc114e5..f9af6a1aebc 100644 --- a/pkg/registry/core/service/allocator/bitmap_test.go +++ b/pkg/registry/core/service/allocator/bitmap_test.go @@ -23,116 +23,521 @@ import ( ) func TestAllocate(t *testing.T) { - max := 10 - m := NewAllocationMap(max, "test") + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) - if _, ok, _ := m.AllocateNext(); !ok { - t.Fatalf("unexpected error") - } - if m.count != 1 { - t.Errorf("expect to get %d, but got %d", 1, m.count) - } - if f := m.Free(); f != max-1 { - t.Errorf("expect to get %d, but got %d", max-1, f) + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + if m.count != 1 { + t.Errorf("expect to get %d, but got %d", 1, m.count) + } + if f := m.Free(); f != tc.max-1 { + t.Errorf("expect to get %d, but got %d", tc.max-1, f) + } + }) } } func TestAllocateMax(t *testing.T) { - max := 10 - m := NewAllocationMap(max, "test") - for i := 0; i < max; i++ { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + for i := 0; i < tc.max; i++ { + if ok, err := m.Allocate(i); !ok || err != nil { + t.Fatalf("unexpected error") + } + } + if _, ok, _ := m.AllocateNext(); ok { + t.Errorf("unexpected success") + } + + if ok, err := m.Allocate(tc.max); ok || err == nil { + t.Fatalf("unexpected allocation") + } + + if f := m.Free(); f != 0 { + t.Errorf("expect to get %d, but got %d", 0, f) + } + }) + } +} + +func TestAllocateNextMax(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + for i := 0; i < tc.max; i++ { + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + } + if _, ok, _ := m.AllocateNext(); ok { + t.Errorf("unexpected success") + } + if f := m.Free(); f != 0 { + t.Errorf("expect to get %d, but got %d", 0, f) + } + }) + } +} +func TestAllocateError(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + if ok, _ := m.Allocate(3); !ok { + t.Errorf("error allocate offset %v", 3) + } + if ok, _ := m.Allocate(3); ok { + t.Errorf("unexpected success") + } + }) + } +} + +func TestRelease(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + offset := 3 + if ok, _ := m.Allocate(offset); !ok { + t.Errorf("error allocate offset %v", offset) + } + + if !m.Has(offset) { + t.Errorf("expect offset %v allocated", offset) + } + + if err := m.Release(offset); err != nil { + t.Errorf("unexpected error: %v", err) + } + + if m.Has(offset) { + t.Errorf("expect offset %v not allocated", offset) + } + }) + } + +} + +func TestForEach(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + subTests := []sets.Int{ + sets.NewInt(), + sets.NewInt(0), + sets.NewInt(0, 2, 5), + sets.NewInt(0, 1, 2, 3, 4, 5, 6, 7), + } + + for i, ts := range subTests { + m := tc.allocator(tc.max, "test", tc.reserved) + for offset := range ts { + if ok, _ := m.Allocate(offset); !ok { + t.Errorf("[%d] error allocate offset %v", i, offset) + } + if !m.Has(offset) { + t.Errorf("[%d] expect offset %v allocated", i, offset) + } + } + calls := sets.NewInt() + m.ForEach(func(i int) { + calls.Insert(i) + }) + if len(calls) != len(ts) { + t.Errorf("[%d] expected %d calls, got %d", i, len(ts), len(calls)) + } + if !calls.Equal(ts) { + t.Errorf("[%d] expected calls to equal testcase: %v vs %v", i, calls.List(), ts.List()) + } + } + }) + } +} + +func TestSnapshotAndRestore(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + offset := 3 + if ok, _ := m.Allocate(offset); !ok { + t.Errorf("error allocate offset %v", offset) + } + spec, bytes := m.Snapshot() + + m2 := tc.allocator(10, "test", tc.reserved) + err := m2.Restore(spec, bytes) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if m2.count != 1 { + t.Errorf("expect count to %d, but got %d", 0, m.count) + } + if !m2.Has(offset) { + t.Errorf("expect offset %v allocated", offset) + } + }) + } + +} + +// TestAllocateMaxReserved should allocate first values greater or equal than the reserved values +func TestAllocateMax_BitmapReserved(t *testing.T) { + max := 128 + dynamicOffset := 16 + + // just to double check off by one errors + allocated := 0 + // modify if necessary + m := NewAllocationMapWithOffset(max, "test", dynamicOffset) + for i := 0; i < max-dynamicOffset; i++ { if _, ok, _ := m.AllocateNext(); !ok { t.Fatalf("unexpected error") } + allocated++ + } + + if f := m.Free(); f != dynamicOffset { + t.Errorf("expect to get %d, but got %d", dynamicOffset-1, f) + } + + for i := 0; i < dynamicOffset; i++ { + if m.Has(i) { + t.Errorf("unexpected allocated value %d", i) + } + } + // it should allocate one value of the reserved block + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + allocated++ + if allocated != m.count { + t.Errorf("expect to get %d, but got %d", allocated, m.count) + } + + if m.count != max-dynamicOffset+1 { + t.Errorf("expect to get %d, but got %d", max-dynamicOffset+1, m.count) + } + if f := m.Free(); f != max-allocated { + t.Errorf("expect to get %d, but got %d", max-allocated, f) + } +} + +func TestPreAllocateReservedFull_BitmapReserved(t *testing.T) { + max := 128 + dynamicOffset := 16 + // just to double check off by one errors + allocated := 0 + m := NewAllocationMapWithOffset(max, "test", dynamicOffset) + // Allocate all possible values except the reserved + for i := dynamicOffset; i < max; i++ { + if ok, _ := m.Allocate(i); !ok { + t.Errorf("error allocate i %v", i) + } else { + allocated++ + } + } + // Allocate all the values of the reserved block except one + for i := 0; i < dynamicOffset-1; i++ { + if ok, _ := m.Allocate(i); !ok { + t.Errorf("error allocate i %v", i) + } else { + allocated++ + } + } + + // there should be only one free value + if f := m.Free(); f != 1 { + t.Errorf("expect to get %d, but got %d", 1, f) + } + // check if the last free value is in the lower band + count := 0 + for i := 0; i < dynamicOffset; i++ { + if !m.Has(i) { + count++ + } + } + if count != 1 { + t.Errorf("expected one remaining free value, got %d", count) + } + + if _, ok, _ := m.AllocateNext(); !ok { + t.Errorf("unexpected allocation error") + } else { + allocated++ + } + if f := m.Free(); f != 0 { + t.Errorf("expect to get %d, but got %d", max-1, f) } if _, ok, _ := m.AllocateNext(); ok { t.Errorf("unexpected success") } + if m.count != allocated { + t.Errorf("expect to get %d, but got %d", max, m.count) + } if f := m.Free(); f != 0 { - t.Errorf("expect to get %d, but got %d", 0, f) - } -} - -func TestAllocateError(t *testing.T) { - m := NewAllocationMap(10, "test") - if ok, _ := m.Allocate(3); !ok { - t.Errorf("error allocate offset %v", 3) - } - if ok, _ := m.Allocate(3); ok { - t.Errorf("unexpected success") - } -} - -func TestRelease(t *testing.T) { - offset := 3 - m := NewAllocationMap(10, "test") - if ok, _ := m.Allocate(offset); !ok { - t.Errorf("error allocate offset %v", offset) - } - - if !m.Has(offset) { - t.Errorf("expect offset %v allocated", offset) - } - - if err := m.Release(offset); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if m.Has(offset) { - t.Errorf("expect offset %v not allocated", offset) - } -} - -func TestForEach(t *testing.T) { - testCases := []sets.Int{ - sets.NewInt(), - sets.NewInt(0), - sets.NewInt(0, 2, 5, 9), - sets.NewInt(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), - } - - for i, tc := range testCases { - m := NewAllocationMap(10, "test") - for offset := range tc { - if ok, _ := m.Allocate(offset); !ok { - t.Errorf("[%d] error allocate offset %v", i, offset) - } - if !m.Has(offset) { - t.Errorf("[%d] expect offset %v allocated", i, offset) - } - } - calls := sets.NewInt() - m.ForEach(func(i int) { - calls.Insert(i) - }) - if len(calls) != len(tc) { - t.Errorf("[%d] expected %d calls, got %d", i, len(tc), len(calls)) - } - if !calls.Equal(tc) { - t.Errorf("[%d] expected calls to equal testcase: %v vs %v", i, calls.List(), tc.List()) - } - } -} - -func TestSnapshotAndRestore(t *testing.T) { - offset := 3 - m := NewAllocationMap(10, "test") - if ok, _ := m.Allocate(offset); !ok { - t.Errorf("error allocate offset %v", offset) - } - spec, bytes := m.Snapshot() - - m2 := NewAllocationMap(10, "test") - err := m2.Restore(spec, bytes) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - if m2.count != 1 { - t.Errorf("expect count to %d, but got %d", 0, m.count) - } - if !m2.Has(offset) { - t.Errorf("expect offset %v allocated", offset) + t.Errorf("expect to get %d, but got %d", max-1, f) } } diff --git a/pkg/registry/core/service/allocator/interfaces.go b/pkg/registry/core/service/allocator/interfaces.go index 8f1b2f179bc..bd7ebc77b81 100644 --- a/pkg/registry/core/service/allocator/interfaces.go +++ b/pkg/registry/core/service/allocator/interfaces.go @@ -40,3 +40,5 @@ type Snapshottable interface { } type AllocatorFactory func(max int, rangeSpec string) (Interface, error) + +type AllocatorWithOffsetFactory func(max int, rangeSpec string, offset int) (Interface, error)