diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 28fba47802a..374043b782b 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -841,6 +841,12 @@ const ( // // Enable MinDomains in Pod Topology Spread. MinDomainsInPodTopologySpread featuregate.Feature = "MinDomainsInPodTopologySpread" + // owner: @aojea + // kep: http://kep.k8s.io/3070 + // alpha: v1.24 + // + // Subdivide the ClusterIP range for dynamic and static IP allocation. + ServiceIPStaticSubrange featuregate.Feature = "ServiceIPStaticSubrange" ) func init() { @@ -963,6 +969,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS GRPCContainerProbe: {Default: true, PreRelease: featuregate.Beta}, LegacyServiceAccountTokenNoAutoGeneration: {Default: true, PreRelease: featuregate.Beta}, MinDomainsInPodTopologySpread: {Default: false, PreRelease: featuregate.Alpha}, + ServiceIPStaticSubrange: {Default: false, PreRelease: featuregate.Alpha}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index 8feae659016..53cb48234cc 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -199,8 +199,9 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(apiResourceConfigSource return LegacyRESTStorage{}, genericapiserver.APIGroupInfo{}, err } - serviceClusterIPAllocator, err := ipallocator.New(&serviceClusterIPRange, func(max int, rangeSpec string) (allocator.Interface, error) { - mem := allocator.NewAllocationMap(max, rangeSpec) + serviceClusterIPAllocator, err := ipallocator.New(&serviceClusterIPRange, func(max int, rangeSpec string, offset int) (allocator.Interface, error) { + var mem allocator.Snapshottable + mem = allocator.NewAllocationMapWithOffset(max, rangeSpec, offset) // TODO etcdallocator package to return a storage interface via the storageFactory etcd, err := serviceallocator.NewEtcd(mem, "/ranges/serviceips", serviceStorageConfig.ForResource(api.Resource("serviceipallocations"))) if err != nil { @@ -218,8 +219,9 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(apiResourceConfigSource var secondaryServiceClusterIPAllocator ipallocator.Interface if c.SecondaryServiceIPRange.IP != nil { var secondaryServiceClusterIPRegistry rangeallocation.RangeRegistry - secondaryServiceClusterIPAllocator, err = ipallocator.New(&c.SecondaryServiceIPRange, func(max int, rangeSpec string) (allocator.Interface, error) { - mem := allocator.NewAllocationMap(max, rangeSpec) + secondaryServiceClusterIPAllocator, err = ipallocator.New(&c.SecondaryServiceIPRange, func(max int, rangeSpec string, offset int) (allocator.Interface, error) { + var mem allocator.Snapshottable + mem = allocator.NewAllocationMapWithOffset(max, rangeSpec, offset) // TODO etcdallocator package to return a storage interface via the storageFactory etcd, err := serviceallocator.NewEtcd(mem, "/ranges/secondaryserviceips", serviceStorageConfig.ForResource(api.Resource("serviceipallocations"))) if err != nil { diff --git a/pkg/registry/core/service/allocator/bitmap.go b/pkg/registry/core/service/allocator/bitmap.go index c5dad0467f0..3c12864c114 100644 --- a/pkg/registry/core/service/allocator/bitmap.go +++ b/pkg/registry/core/service/allocator/bitmap.go @@ -18,6 +18,7 @@ package allocator import ( "errors" + "fmt" "math/big" "math/rand" "sync" @@ -60,15 +61,25 @@ type bitAllocator interface { // NewAllocationMap creates an allocation bitmap using the random scan strategy. func NewAllocationMap(max int, rangeSpec string) *AllocationBitmap { + return NewAllocationMapWithOffset(max, rangeSpec, 0) +} + +// NewAllocationMapWithOffset creates an allocation bitmap using a random scan strategy that +// allows to pass an offset that divides the allocation bitmap in two blocks. +// The first block of values will not be used for random value assigned by the AllocateNext() +// method until the second block of values has been exhausted. +func NewAllocationMapWithOffset(max int, rangeSpec string, offset int) *AllocationBitmap { a := AllocationBitmap{ - strategy: randomScanStrategy{ - rand: rand.New(rand.NewSource(time.Now().UnixNano())), + strategy: randomScanStrategyWithOffset{ + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + offset: offset, }, allocated: big.NewInt(0), count: 0, max: max, rangeSpec: rangeSpec, } + return &a } @@ -78,6 +89,10 @@ func (r *AllocationBitmap) Allocate(offset int) (bool, error) { r.lock.Lock() defer r.lock.Unlock() + // max is the maximum size of the usable items in the range + if offset < 0 || offset >= r.max { + return false, fmt.Errorf("offset %d out of range [0,%d]", offset, r.max) + } if r.allocated.Bit(offset) == 1 { return false, nil } @@ -205,3 +220,40 @@ func (rss randomScanStrategy) AllocateBit(allocated *big.Int, max, count int) (i } var _ bitAllocator = randomScanStrategy{} + +// randomScanStrategyWithOffset choose a random address from the provided big.Int and then scans +// forward looking for the next available address. The big.Int range is subdivided so it will try +// to allocate first from the reserved upper range of addresses (it will wrap the upper subrange if necessary). +// If there is no free address it will try to allocate one from the lower range too. +type randomScanStrategyWithOffset struct { + rand *rand.Rand + offset int +} + +func (rss randomScanStrategyWithOffset) AllocateBit(allocated *big.Int, max, count int) (int, bool) { + if count >= max { + return 0, false + } + // size of the upper subrange, prioritized for random allocation + subrangeMax := max - rss.offset + // try to get a value from the upper range [rss.reserved, max] + start := rss.rand.Intn(subrangeMax) + for i := 0; i < subrangeMax; i++ { + at := rss.offset + ((start + i) % subrangeMax) + if allocated.Bit(at) == 0 { + return at, true + } + } + + start = rss.rand.Intn(rss.offset) + // subrange full, try to get the value from the first block before giving up. + for i := 0; i < rss.offset; i++ { + at := (start + i) % rss.offset + if allocated.Bit(at) == 0 { + return i, true + } + } + return 0, false +} + +var _ bitAllocator = randomScanStrategyWithOffset{} diff --git a/pkg/registry/core/service/allocator/bitmap_test.go b/pkg/registry/core/service/allocator/bitmap_test.go index b5eacc114e5..f9af6a1aebc 100644 --- a/pkg/registry/core/service/allocator/bitmap_test.go +++ b/pkg/registry/core/service/allocator/bitmap_test.go @@ -23,116 +23,521 @@ import ( ) func TestAllocate(t *testing.T) { - max := 10 - m := NewAllocationMap(max, "test") + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) - if _, ok, _ := m.AllocateNext(); !ok { - t.Fatalf("unexpected error") - } - if m.count != 1 { - t.Errorf("expect to get %d, but got %d", 1, m.count) - } - if f := m.Free(); f != max-1 { - t.Errorf("expect to get %d, but got %d", max-1, f) + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + if m.count != 1 { + t.Errorf("expect to get %d, but got %d", 1, m.count) + } + if f := m.Free(); f != tc.max-1 { + t.Errorf("expect to get %d, but got %d", tc.max-1, f) + } + }) } } func TestAllocateMax(t *testing.T) { - max := 10 - m := NewAllocationMap(max, "test") - for i := 0; i < max; i++ { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + for i := 0; i < tc.max; i++ { + if ok, err := m.Allocate(i); !ok || err != nil { + t.Fatalf("unexpected error") + } + } + if _, ok, _ := m.AllocateNext(); ok { + t.Errorf("unexpected success") + } + + if ok, err := m.Allocate(tc.max); ok || err == nil { + t.Fatalf("unexpected allocation") + } + + if f := m.Free(); f != 0 { + t.Errorf("expect to get %d, but got %d", 0, f) + } + }) + } +} + +func TestAllocateNextMax(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + for i := 0; i < tc.max; i++ { + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + } + if _, ok, _ := m.AllocateNext(); ok { + t.Errorf("unexpected success") + } + if f := m.Free(); f != 0 { + t.Errorf("expect to get %d, but got %d", 0, f) + } + }) + } +} +func TestAllocateError(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + if ok, _ := m.Allocate(3); !ok { + t.Errorf("error allocate offset %v", 3) + } + if ok, _ := m.Allocate(3); ok { + t.Errorf("unexpected success") + } + }) + } +} + +func TestRelease(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + offset := 3 + if ok, _ := m.Allocate(offset); !ok { + t.Errorf("error allocate offset %v", offset) + } + + if !m.Has(offset) { + t.Errorf("expect offset %v allocated", offset) + } + + if err := m.Release(offset); err != nil { + t.Errorf("unexpected error: %v", err) + } + + if m.Has(offset) { + t.Errorf("expect offset %v not allocated", offset) + } + }) + } + +} + +func TestForEach(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + subTests := []sets.Int{ + sets.NewInt(), + sets.NewInt(0), + sets.NewInt(0, 2, 5), + sets.NewInt(0, 1, 2, 3, 4, 5, 6, 7), + } + + for i, ts := range subTests { + m := tc.allocator(tc.max, "test", tc.reserved) + for offset := range ts { + if ok, _ := m.Allocate(offset); !ok { + t.Errorf("[%d] error allocate offset %v", i, offset) + } + if !m.Has(offset) { + t.Errorf("[%d] expect offset %v allocated", i, offset) + } + } + calls := sets.NewInt() + m.ForEach(func(i int) { + calls.Insert(i) + }) + if len(calls) != len(ts) { + t.Errorf("[%d] expected %d calls, got %d", i, len(ts), len(calls)) + } + if !calls.Equal(ts) { + t.Errorf("[%d] expected calls to equal testcase: %v vs %v", i, calls.List(), ts.List()) + } + } + }) + } +} + +func TestSnapshotAndRestore(t *testing.T) { + testCases := []struct { + name string + allocator func(max int, rangeSpec string, reserved int) *AllocationBitmap + max int + reserved int + }{ + { + name: "NewAllocationMap", + allocator: NewAllocationMapWithOffset, + max: 32, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max < 16", + allocator: NewAllocationMapWithOffset, + max: 8, + reserved: 0, + }, + { + name: "NewAllocationMapWithOffset max > 16", + allocator: NewAllocationMapWithOffset, + max: 128, + reserved: 16, + }, + { + name: "NewAllocationMapWithOffset max > 256", + allocator: NewAllocationMapWithOffset, + max: 1024, + reserved: 64, + }, + { + name: "NewAllocationMapWithOffset max value", + allocator: NewAllocationMapWithOffset, + max: 65535, + reserved: 256, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + m := tc.allocator(tc.max, "test", tc.reserved) + offset := 3 + if ok, _ := m.Allocate(offset); !ok { + t.Errorf("error allocate offset %v", offset) + } + spec, bytes := m.Snapshot() + + m2 := tc.allocator(10, "test", tc.reserved) + err := m2.Restore(spec, bytes) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if m2.count != 1 { + t.Errorf("expect count to %d, but got %d", 0, m.count) + } + if !m2.Has(offset) { + t.Errorf("expect offset %v allocated", offset) + } + }) + } + +} + +// TestAllocateMaxReserved should allocate first values greater or equal than the reserved values +func TestAllocateMax_BitmapReserved(t *testing.T) { + max := 128 + dynamicOffset := 16 + + // just to double check off by one errors + allocated := 0 + // modify if necessary + m := NewAllocationMapWithOffset(max, "test", dynamicOffset) + for i := 0; i < max-dynamicOffset; i++ { if _, ok, _ := m.AllocateNext(); !ok { t.Fatalf("unexpected error") } + allocated++ + } + + if f := m.Free(); f != dynamicOffset { + t.Errorf("expect to get %d, but got %d", dynamicOffset-1, f) + } + + for i := 0; i < dynamicOffset; i++ { + if m.Has(i) { + t.Errorf("unexpected allocated value %d", i) + } + } + // it should allocate one value of the reserved block + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + allocated++ + if allocated != m.count { + t.Errorf("expect to get %d, but got %d", allocated, m.count) + } + + if m.count != max-dynamicOffset+1 { + t.Errorf("expect to get %d, but got %d", max-dynamicOffset+1, m.count) + } + if f := m.Free(); f != max-allocated { + t.Errorf("expect to get %d, but got %d", max-allocated, f) + } +} + +func TestPreAllocateReservedFull_BitmapReserved(t *testing.T) { + max := 128 + dynamicOffset := 16 + // just to double check off by one errors + allocated := 0 + m := NewAllocationMapWithOffset(max, "test", dynamicOffset) + // Allocate all possible values except the reserved + for i := dynamicOffset; i < max; i++ { + if ok, _ := m.Allocate(i); !ok { + t.Errorf("error allocate i %v", i) + } else { + allocated++ + } + } + // Allocate all the values of the reserved block except one + for i := 0; i < dynamicOffset-1; i++ { + if ok, _ := m.Allocate(i); !ok { + t.Errorf("error allocate i %v", i) + } else { + allocated++ + } + } + + // there should be only one free value + if f := m.Free(); f != 1 { + t.Errorf("expect to get %d, but got %d", 1, f) + } + // check if the last free value is in the lower band + count := 0 + for i := 0; i < dynamicOffset; i++ { + if !m.Has(i) { + count++ + } + } + if count != 1 { + t.Errorf("expected one remaining free value, got %d", count) + } + + if _, ok, _ := m.AllocateNext(); !ok { + t.Errorf("unexpected allocation error") + } else { + allocated++ + } + if f := m.Free(); f != 0 { + t.Errorf("expect to get %d, but got %d", max-1, f) } if _, ok, _ := m.AllocateNext(); ok { t.Errorf("unexpected success") } + if m.count != allocated { + t.Errorf("expect to get %d, but got %d", max, m.count) + } if f := m.Free(); f != 0 { - t.Errorf("expect to get %d, but got %d", 0, f) - } -} - -func TestAllocateError(t *testing.T) { - m := NewAllocationMap(10, "test") - if ok, _ := m.Allocate(3); !ok { - t.Errorf("error allocate offset %v", 3) - } - if ok, _ := m.Allocate(3); ok { - t.Errorf("unexpected success") - } -} - -func TestRelease(t *testing.T) { - offset := 3 - m := NewAllocationMap(10, "test") - if ok, _ := m.Allocate(offset); !ok { - t.Errorf("error allocate offset %v", offset) - } - - if !m.Has(offset) { - t.Errorf("expect offset %v allocated", offset) - } - - if err := m.Release(offset); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if m.Has(offset) { - t.Errorf("expect offset %v not allocated", offset) - } -} - -func TestForEach(t *testing.T) { - testCases := []sets.Int{ - sets.NewInt(), - sets.NewInt(0), - sets.NewInt(0, 2, 5, 9), - sets.NewInt(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), - } - - for i, tc := range testCases { - m := NewAllocationMap(10, "test") - for offset := range tc { - if ok, _ := m.Allocate(offset); !ok { - t.Errorf("[%d] error allocate offset %v", i, offset) - } - if !m.Has(offset) { - t.Errorf("[%d] expect offset %v allocated", i, offset) - } - } - calls := sets.NewInt() - m.ForEach(func(i int) { - calls.Insert(i) - }) - if len(calls) != len(tc) { - t.Errorf("[%d] expected %d calls, got %d", i, len(tc), len(calls)) - } - if !calls.Equal(tc) { - t.Errorf("[%d] expected calls to equal testcase: %v vs %v", i, calls.List(), tc.List()) - } - } -} - -func TestSnapshotAndRestore(t *testing.T) { - offset := 3 - m := NewAllocationMap(10, "test") - if ok, _ := m.Allocate(offset); !ok { - t.Errorf("error allocate offset %v", offset) - } - spec, bytes := m.Snapshot() - - m2 := NewAllocationMap(10, "test") - err := m2.Restore(spec, bytes) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - if m2.count != 1 { - t.Errorf("expect count to %d, but got %d", 0, m.count) - } - if !m2.Has(offset) { - t.Errorf("expect offset %v allocated", offset) + t.Errorf("expect to get %d, but got %d", max-1, f) } } diff --git a/pkg/registry/core/service/allocator/interfaces.go b/pkg/registry/core/service/allocator/interfaces.go index 8f1b2f179bc..bd7ebc77b81 100644 --- a/pkg/registry/core/service/allocator/interfaces.go +++ b/pkg/registry/core/service/allocator/interfaces.go @@ -40,3 +40,5 @@ type Snapshottable interface { } type AllocatorFactory func(max int, rangeSpec string) (Interface, error) + +type AllocatorWithOffsetFactory func(max int, rangeSpec string, offset int) (Interface, error) diff --git a/pkg/registry/core/service/ipallocator/allocator.go b/pkg/registry/core/service/ipallocator/allocator.go index 820ae4a73e7..b8d8560dc79 100644 --- a/pkg/registry/core/service/ipallocator/allocator.go +++ b/pkg/registry/core/service/ipallocator/allocator.go @@ -22,7 +22,9 @@ import ( "math/big" "net" + utilfeature "k8s.io/apiserver/pkg/util/feature" api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/registry/core/service/allocator" netutils "k8s.io/utils/net" ) @@ -86,7 +88,7 @@ type Range struct { } // New creates a Range over a net.IPNet, calling allocatorFactory to construct the backing store. -func New(cidr *net.IPNet, allocatorFactory allocator.AllocatorFactory) (*Range, error) { +func New(cidr *net.IPNet, allocatorFactory allocator.AllocatorWithOffsetFactory) (*Range, error) { registerMetrics() max := netutils.RangeSize(cidr) @@ -118,16 +120,24 @@ func New(cidr *net.IPNet, allocatorFactory allocator.AllocatorFactory) (*Range, max: maximum(0, int(max)), family: family, } - var err error - r.alloc, err = allocatorFactory(r.max, rangeSpec) - return &r, err + offset := 0 + if utilfeature.DefaultFeatureGate.Enabled(features.ServiceIPStaticSubrange) { + offset = calculateRangeOffset(cidr) + } + + var err error + r.alloc, err = allocatorFactory(r.max, rangeSpec, offset) + if err != nil { + return nil, err + } + return &r, nil } // NewInMemory creates an in-memory allocator. func NewInMemory(cidr *net.IPNet) (*Range, error) { - return New(cidr, func(max int, rangeSpec string) (allocator.Interface, error) { - return allocator.NewAllocationMap(max, rangeSpec), nil + return New(cidr, func(max int, rangeSpec string, offset int) (allocator.Interface, error) { + return allocator.NewAllocationMapWithOffset(max, rangeSpec, offset), nil }) } @@ -191,7 +201,7 @@ func (r *Range) allocate(ip net.IP, dryRun bool) error { ok, offset := r.contains(ip) if !ok { // update metrics - clusterIPAllocationErrors.WithLabelValues(label.String()).Inc() + clusterIPAllocationErrors.WithLabelValues(label.String(), "static").Inc() return &ErrNotInRange{ip, r.net.String()} } if dryRun { @@ -203,18 +213,18 @@ func (r *Range) allocate(ip net.IP, dryRun bool) error { allocated, err := r.alloc.Allocate(offset) if err != nil { // update metrics - clusterIPAllocationErrors.WithLabelValues(label.String()).Inc() + clusterIPAllocationErrors.WithLabelValues(label.String(), "static").Inc() return err } if !allocated { // update metrics - clusterIPAllocationErrors.WithLabelValues(label.String()).Inc() + clusterIPAllocationErrors.WithLabelValues(label.String(), "static").Inc() return ErrAllocated } // update metrics - clusterIPAllocations.WithLabelValues(label.String()).Inc() + clusterIPAllocations.WithLabelValues(label.String(), "static").Inc() clusterIPAllocated.WithLabelValues(label.String()).Set(float64(r.Used())) clusterIPAvailable.WithLabelValues(label.String()).Set(float64(r.Free())) @@ -238,18 +248,18 @@ func (r *Range) allocateNext(dryRun bool) (net.IP, error) { offset, ok, err := r.alloc.AllocateNext() if err != nil { // update metrics - clusterIPAllocationErrors.WithLabelValues(label.String()).Inc() + clusterIPAllocationErrors.WithLabelValues(label.String(), "dynamic").Inc() return nil, err } if !ok { // update metrics - clusterIPAllocationErrors.WithLabelValues(label.String()).Inc() + clusterIPAllocationErrors.WithLabelValues(label.String(), "dynamic").Inc() return nil, ErrFull } // update metrics - clusterIPAllocations.WithLabelValues(label.String()).Inc() + clusterIPAllocations.WithLabelValues(label.String(), "dynamic").Inc() clusterIPAllocated.WithLabelValues(label.String()).Set(float64(r.Used())) clusterIPAvailable.WithLabelValues(label.String()).Set(float64(r.Free())) @@ -354,6 +364,33 @@ func calculateIPOffset(base *big.Int, ip net.IP) int { return int(big.NewInt(0).Sub(netutils.BigForIP(ip), base).Int64()) } +// calculateRangeOffset estimates the offset used on the range for statically allocation based on +// the following formula `min(max($min, cidrSize/$step), $max)`, described as ~never less than +// $min or more than $max, with a graduated step function between them~. The function returns 0 +// if any of the parameters is invalid. +func calculateRangeOffset(cidr *net.IPNet) int { + // default values for min(max($min, cidrSize/$step), $max) + const ( + min = 16 + max = 256 + step = 16 + ) + + cidrSize := netutils.RangeSize(cidr) + if cidrSize < min { + return 0 + } + + offset := cidrSize / step + if offset < min { + return min + } + if offset > max { + return max + } + return int(offset) +} + // dryRunRange is a shim to satisfy Interface without persisting state. type dryRunRange struct { real *Range diff --git a/pkg/registry/core/service/ipallocator/allocator_test.go b/pkg/registry/core/service/ipallocator/allocator_test.go index bf4dec2fa83..0853c853580 100644 --- a/pkg/registry/core/service/ipallocator/allocator_test.go +++ b/pkg/registry/core/service/ipallocator/allocator_test.go @@ -17,12 +17,16 @@ limitations under the License. package ipallocator import ( + "fmt" "net" "testing" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-base/metrics/testutil" api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" netutils "k8s.io/utils/net" ) @@ -176,6 +180,57 @@ func TestAllocateTiny(t *testing.T) { } } +func TestAllocateReserved(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceIPStaticSubrange, true)() + + _, cidr, err := netutils.ParseCIDRSloppy("192.168.1.0/25") + if err != nil { + t.Fatal(err) + } + r, err := NewInMemory(cidr) + if err != nil { + t.Fatal(err) + } + // allocate all addresses on the dynamic block + // subnet /25 = 128 ; dynamic block size is min(max(16,128/16),256) = 16 + dynamicOffset := calculateRangeOffset(cidr) + dynamicBlockSize := r.max - dynamicOffset + for i := 0; i < dynamicBlockSize; i++ { + if _, err := r.AllocateNext(); err != nil { + t.Errorf("Unexpected error trying to allocate: %v", err) + } + } + for i := dynamicOffset; i < r.max; i++ { + ip := fmt.Sprintf("192.168.1.%d", i+1) + if !r.Has(netutils.ParseIPSloppy(ip)) { + t.Errorf("IP %s expected to be allocated", ip) + } + } + if f := r.Free(); f != dynamicOffset { + t.Errorf("expected %d free addresses, got %d", dynamicOffset, f) + } + // allocate all addresses on the static block + for i := 0; i < dynamicOffset; i++ { + ip := fmt.Sprintf("192.168.1.%d", i+1) + if err := r.Allocate(netutils.ParseIPSloppy(ip)); err != nil { + t.Errorf("Unexpected error trying to allocate IP %s: %v", ip, err) + } + } + if f := r.Free(); f != 0 { + t.Errorf("expected free equal to 0 got: %d", f) + } + // release one address in the allocated block and another a new one randomly + if err := r.Release(netutils.ParseIPSloppy("192.168.1.10")); err != nil { + t.Fatalf("Unexpected error trying to release ip 192.168.1.10: %v", err) + } + if _, err := r.AllocateNext(); err != nil { + t.Error(err) + } + if f := r.Free(); f != 0 { + t.Errorf("expected free equal to 0 got: %d", f) + } +} + func TestAllocateSmall(t *testing.T) { _, cidr, err := netutils.ParseCIDRSloppy("192.168.1.240/30") if err != nil { @@ -186,7 +241,7 @@ func TestAllocateSmall(t *testing.T) { t.Fatal(err) } if f := r.Free(); f != 2 { - t.Errorf("free: %d", f) + t.Errorf("expected free equal to 2 got: %d", f) } found := sets.NewString() for i := 0; i < 2; i++ { @@ -195,7 +250,7 @@ func TestAllocateSmall(t *testing.T) { t.Fatal(err) } if found.Has(ip.String()) { - t.Fatalf("already reserved: %s", ip) + t.Fatalf("address %s has been already allocated", ip) } found.Insert(ip.String()) } @@ -213,8 +268,12 @@ func TestAllocateSmall(t *testing.T) { } } - if r.Free() != 0 && r.max != 2 { - t.Fatalf("unexpected range: %v", r) + if f := r.Free(); f != 0 { + t.Errorf("expected free equal to 0 got: %d", f) + } + + if r.max != 2 { + t.Fatalf("expected range equal to 2, got: %v", r) } t.Logf("allocated: %v", found) @@ -365,6 +424,9 @@ func TestNewFromSnapshot(t *testing.T) { } func TestClusterIPMetrics(t *testing.T) { + clearMetrics() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceIPStaticSubrange, true)() + // create IPv4 allocator cidrIPv4 := "10.0.0.0/24" _, clusterCIDRv4, _ := netutils.ParseCIDRSloppy(cidrIPv4) @@ -372,7 +434,6 @@ func TestClusterIPMetrics(t *testing.T) { if err != nil { t.Fatalf("unexpected error creating CidrSet: %v", err) } - clearMetrics(map[string]string{"cidr": cidrIPv4}) // create IPv6 allocator cidrIPv6 := "2001:db8::/112" _, clusterCIDRv6, _ := netutils.ParseCIDRSloppy(cidrIPv6) @@ -380,7 +441,6 @@ func TestClusterIPMetrics(t *testing.T) { if err != nil { t.Fatalf("unexpected error creating CidrSet: %v", err) } - clearMetrics(map[string]string{"cidr": cidrIPv6}) // Check initial state em := testMetrics{ @@ -475,12 +535,72 @@ func TestClusterIPMetrics(t *testing.T) { expectMetrics(t, cidrIPv6, em) } +func TestClusterIPAllocatedMetrics(t *testing.T) { + clearMetrics() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceIPStaticSubrange, true)() + + // create IPv4 allocator + cidrIPv4 := "10.0.0.0/25" + _, clusterCIDRv4, _ := netutils.ParseCIDRSloppy(cidrIPv4) + a, err := NewInMemory(clusterCIDRv4) + if err != nil { + t.Fatalf("unexpected error creating CidrSet: %v", err) + } + + em := testMetrics{ + free: 0, + used: 0, + allocated: 0, + errors: 0, + } + expectMetrics(t, cidrIPv4, em) + + // allocate 2 dynamic IPv4 addresses + found := sets.NewString() + for i := 0; i < 2; i++ { + ip, err := a.AllocateNext() + if err != nil { + t.Fatal(err) + } + if found.Has(ip.String()) { + t.Fatalf("already reserved: %s", ip) + } + found.Insert(ip.String()) + } + + dynamic_allocated, err := testutil.GetCounterMetricValue(clusterIPAllocations.WithLabelValues(cidrIPv4, "dynamic")) + if err != nil { + t.Errorf("failed to get %s value, err: %v", clusterIPAllocations.Name, err) + } + if dynamic_allocated != 2 { + t.Fatalf("Expected 2 received %f", dynamic_allocated) + } + + // try to allocate the same IP addresses + for s := range found { + if !a.Has(netutils.ParseIPSloppy(s)) { + t.Fatalf("missing: %s", s) + } + if err := a.Allocate(netutils.ParseIPSloppy(s)); err != ErrAllocated { + t.Fatal(err) + } + } + + static_errors, err := testutil.GetCounterMetricValue(clusterIPAllocationErrors.WithLabelValues(cidrIPv4, "static")) + if err != nil { + t.Errorf("failed to get %s value, err: %v", clusterIPAllocationErrors.Name, err) + } + if static_errors != 2 { + t.Fatalf("Expected 2 received %f", dynamic_allocated) + } +} + // Metrics helpers -func clearMetrics(labels map[string]string) { - clusterIPAllocated.Delete(labels) - clusterIPAvailable.Delete(labels) - clusterIPAllocations.Delete(labels) - clusterIPAllocationErrors.Delete(labels) +func clearMetrics() { + clusterIPAllocated.Reset() + clusterIPAvailable.Reset() + clusterIPAllocations.Reset() + clusterIPAllocationErrors.Reset() } type testMetrics struct { @@ -501,14 +621,25 @@ func expectMetrics(t *testing.T, label string, em testMetrics) { if err != nil { t.Errorf("failed to get %s value, err: %v", clusterIPAllocated.Name, err) } - m.allocated, err = testutil.GetCounterMetricValue(clusterIPAllocations.WithLabelValues(label)) + static_allocated, err := testutil.GetCounterMetricValue(clusterIPAllocations.WithLabelValues(label, "static")) if err != nil { t.Errorf("failed to get %s value, err: %v", clusterIPAllocations.Name, err) } - m.errors, err = testutil.GetCounterMetricValue(clusterIPAllocationErrors.WithLabelValues(label)) + static_errors, err := testutil.GetCounterMetricValue(clusterIPAllocationErrors.WithLabelValues(label, "static")) if err != nil { t.Errorf("failed to get %s value, err: %v", clusterIPAllocationErrors.Name, err) } + dynamic_allocated, err := testutil.GetCounterMetricValue(clusterIPAllocations.WithLabelValues(label, "dynamic")) + if err != nil { + t.Errorf("failed to get %s value, err: %v", clusterIPAllocations.Name, err) + } + dynamic_errors, err := testutil.GetCounterMetricValue(clusterIPAllocationErrors.WithLabelValues(label, "dynamic")) + if err != nil { + t.Errorf("failed to get %s value, err: %v", clusterIPAllocationErrors.Name, err) + } + + m.allocated = static_allocated + dynamic_allocated + m.errors = static_errors + dynamic_errors if m != em { t.Fatalf("metrics error: expected %v, received %v", em, m) @@ -578,3 +709,75 @@ func TestDryRun(t *testing.T) { }) } } + +func Test_calculateRangeOffset(t *testing.T) { + // default $min = 16, $max = 256 and $step = 16. + tests := []struct { + name string + cidr string + want int + }{ + { + name: "full mask IPv4", + cidr: "192.168.1.1/32", + want: 0, + }, + { + name: "full mask IPv6", + cidr: "fd00::1/128", + want: 0, + }, + { + name: "very small mask IPv4", + cidr: "192.168.1.1/30", + want: 0, + }, + { + name: "very small mask IPv6", + cidr: "fd00::1/126", + want: 0, + }, + { + name: "small mask IPv4", + cidr: "192.168.1.1/28", + want: 16, + }, + { + name: "small mask IPv6", + cidr: "fd00::1/122", + want: 16, + }, + { + name: "medium mask IPv4", + cidr: "192.168.1.1/22", + want: 64, + }, + { + name: "medium mask IPv6", + cidr: "fd00::1/118", + want: 64, + }, + { + name: "large mask IPv4", + cidr: "192.168.1.1/8", + want: 256, + }, + { + name: "large mask IPv6", + cidr: "fd00::1/12", + want: 256, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + _, cidr, err := netutils.ParseCIDRSloppy(tt.cidr) + if err != nil { + t.Fatalf("Unexpected error parsing CIDR %s: %v", tt.cidr, err) + } + if got := calculateRangeOffset(cidr); got != tt.want { + t.Errorf("DynamicRangeOffset() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/registry/core/service/ipallocator/metrics.go b/pkg/registry/core/service/ipallocator/metrics.go index ffd62f32e3b..2aa1844b32a 100644 --- a/pkg/registry/core/service/ipallocator/metrics.go +++ b/pkg/registry/core/service/ipallocator/metrics.go @@ -51,7 +51,7 @@ var ( }, []string{"cidr"}, ) - // clusterIPAllocation counts the total number of ClusterIP allocation. + // clusterIPAllocation counts the total number of ClusterIP allocation and allocation mode: static or dynamic. clusterIPAllocations = metrics.NewCounterVec( &metrics.CounterOpts{ Namespace: namespace, @@ -60,9 +60,9 @@ var ( Help: "Number of Cluster IPs allocations", StabilityLevel: metrics.ALPHA, }, - []string{"cidr"}, + []string{"cidr", "scope"}, ) - // clusterIPAllocationErrors counts the number of error trying to allocate a ClusterIP. + // clusterIPAllocationErrors counts the number of error trying to allocate a ClusterIP and allocation mode: static or dynamic. clusterIPAllocationErrors = metrics.NewCounterVec( &metrics.CounterOpts{ Namespace: namespace, @@ -71,7 +71,7 @@ var ( Help: "Number of errors trying to allocate Cluster IPs", StabilityLevel: metrics.ALPHA, }, - []string{"cidr"}, + []string{"cidr", "scope"}, ) ) diff --git a/pkg/registry/core/service/ipallocator/storage/storage_test.go b/pkg/registry/core/service/ipallocator/storage/storage_test.go index 3eed72ce198..706e7bc48b6 100644 --- a/pkg/registry/core/service/ipallocator/storage/storage_test.go +++ b/pkg/registry/core/service/ipallocator/storage/storage_test.go @@ -18,6 +18,7 @@ package storage import ( "context" + "fmt" "strings" "testing" @@ -25,8 +26,11 @@ import ( "k8s.io/apiserver/pkg/storage" etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing" "k8s.io/apiserver/pkg/storage/storagebackend/factory" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" api "k8s.io/kubernetes/pkg/apis/core" _ "k8s.io/kubernetes/pkg/apis/core/install" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/registry/core/service/allocator" allocatorstore "k8s.io/kubernetes/pkg/registry/core/service/allocator/storage" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator" @@ -43,8 +47,8 @@ func newStorage(t *testing.T) (*etcd3testing.EtcdTestServer, ipallocator.Interfa var backing allocator.Interface configForAllocations := etcdStorage.ForResource(api.Resource("serviceipallocations")) - storage, err := ipallocator.New(cidr, func(max int, rangeSpec string) (allocator.Interface, error) { - mem := allocator.NewAllocationMap(max, rangeSpec) + storage, err := ipallocator.New(cidr, func(max int, rangeSpec string, offset int) (allocator.Interface, error) { + mem := allocator.NewAllocationMapWithOffset(max, rangeSpec, offset) backing = mem etcd, err := allocatorstore.NewEtcd(mem, "/ranges/serviceips", configForAllocations) if err != nil { @@ -115,3 +119,51 @@ func TestStore(t *testing.T) { t.Fatal(err) } } + +func TestAllocateReserved(t *testing.T) { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ServiceIPStaticSubrange, true)() + + _, storage, _, si, destroyFunc := newStorage(t) + defer destroyFunc() + if err := si.Create(context.TODO(), key(), validNewRangeAllocation(), nil, 0); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // allocate all addresses on the dynamic block + // subnet /24 = 256 ; dynamic block size is min(max(16,256/16),256) = 16 + dynamicOffset := 16 + max := 254 + dynamicBlockSize := max - dynamicOffset + for i := 0; i < dynamicBlockSize; i++ { + if _, err := storage.AllocateNext(); err != nil { + t.Errorf("Unexpected error trying to allocate: %v", err) + } + } + for i := dynamicOffset; i < max; i++ { + ip := fmt.Sprintf("192.168.1.%d", i+1) + if !storage.Has(netutils.ParseIPSloppy(ip)) { + t.Errorf("IP %s expected to be allocated", ip) + } + } + + // allocate all addresses on the static block + for i := 0; i < dynamicOffset; i++ { + ip := fmt.Sprintf("192.168.1.%d", i+1) + if err := storage.Allocate(netutils.ParseIPSloppy(ip)); err != nil { + t.Errorf("Unexpected error trying to allocate IP %s: %v", ip, err) + } + } + if _, err := storage.AllocateNext(); err == nil { + t.Error("Allocator expected to be full") + } + // release one address in the allocated block and another a new one randomly + if err := storage.Release(netutils.ParseIPSloppy("192.168.1.10")); err != nil { + t.Fatalf("Unexpected error trying to release ip 192.168.1.10: %v", err) + } + if _, err := storage.AllocateNext(); err != nil { + t.Error(err) + } + if _, err := storage.AllocateNext(); err == nil { + t.Error("Allocator expected to be full") + } +}