mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-29 22:46:12 +00:00
Add shuffle sharding utils and tests
Implement several shuffle sharding functions including ShuffleAndDeal, ShuffleAndDealToSlice. Add benchmarks and tests for shuffle sharding to test performance, correctness and distribution uniformity. Signed-off-by: Bruce Ma <brucema19901024@gmail.com>
This commit is contained in:
Bruce Ma
parent
057609cf93
commit
e97eaef4f6
@ -44,6 +44,7 @@ filegroup(
|
||||
"//staging/src/k8s.io/apiserver/pkg/util/flushwriter:all-srcs",
|
||||
"//staging/src/k8s.io/apiserver/pkg/util/openapi:all-srcs",
|
||||
"//staging/src/k8s.io/apiserver/pkg/util/proxy:all-srcs",
|
||||
"//staging/src/k8s.io/apiserver/pkg/util/shufflesharding:all-srcs",
|
||||
"//staging/src/k8s.io/apiserver/pkg/util/term:all-srcs",
|
||||
"//staging/src/k8s.io/apiserver/pkg/util/webhook:all-srcs",
|
||||
"//staging/src/k8s.io/apiserver/pkg/util/wsstream:all-srcs",
|
||||
|
||||
29
staging/src/k8s.io/apiserver/pkg/util/shufflesharding/BUILD
Normal file
29
staging/src/k8s.io/apiserver/pkg/util/shufflesharding/BUILD
Normal file
@ -0,0 +1,29 @@
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["shufflesharding.go"],
|
||||
importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/util/shufflesharding",
|
||||
importpath = "k8s.io/apiserver/pkg/util/shufflesharding",
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["shufflesharding_test.go"],
|
||||
embed = [":go_default_library"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
@ -0,0 +1,97 @@
|
||||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package shufflesharding
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
)
|
||||
|
||||
const maxHashBits = 60
|
||||
|
||||
// ValidateParameters can validate parameters for shuffle sharding
|
||||
// in a fast but approximate way, including deckSize and handSize
|
||||
// Algorithm: maxHashBits >= bits(deckSize^handSize)
|
||||
func ValidateParameters(deckSize, handSize int32) bool {
|
||||
if handSize <= 0 || deckSize <= 0 || handSize > deckSize {
|
||||
return false
|
||||
}
|
||||
|
||||
return math.Log2(float64(deckSize))*float64(handSize) <= maxHashBits
|
||||
}
|
||||
|
||||
// ShuffleAndDeal can shuffle a hash value to handSize-quantity and non-redundant
|
||||
// indices of decks, with the pick function, we can get the optimal deck index
|
||||
// Eg. From deckSize=128, handSize=8, we can get an index array [12 14 73 18 119 51 117 26],
|
||||
// then pick function will choose the optimal index from these
|
||||
// Algorithm: https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190228-priority-and-fairness.md#queue-assignment-proof-of-concept
|
||||
func ShuffleAndDeal(hashValue uint64, deckSize, handSize int32, pick func(int32)) {
|
||||
remainders := make([]int32, handSize)
|
||||
|
||||
for i := int32(0); i < handSize; i++ {
|
||||
hashValueNext := hashValue / uint64(deckSize-i)
|
||||
remainders[i] = int32(hashValue - uint64(deckSize-i)*hashValueNext)
|
||||
hashValue = hashValueNext
|
||||
}
|
||||
|
||||
for i := int32(0); i < handSize; i++ {
|
||||
candidate := remainders[i]
|
||||
for j := i; j > 0; j-- {
|
||||
if candidate >= remainders[j-1] {
|
||||
candidate++
|
||||
}
|
||||
}
|
||||
pick(candidate)
|
||||
}
|
||||
}
|
||||
|
||||
// ShuffleAndDealWithValidation will do validation before ShuffleAndDeal
|
||||
func ShuffleAndDealWithValidation(hashValue uint64, deckSize, handSize int32, pick func(int32)) error {
|
||||
if !ValidateParameters(deckSize, handSize) {
|
||||
return errors.New("bad parameters")
|
||||
}
|
||||
|
||||
ShuffleAndDeal(hashValue, deckSize, handSize, pick)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShuffleAndDealToSlice will use specific pick function to return slices of indices
|
||||
// after ShuffleAndDeal
|
||||
func ShuffleAndDealToSlice(hashValue uint64, deckSize, handSize int32) []int32 {
|
||||
var (
|
||||
candidates = make([]int32, handSize)
|
||||
idx = 0
|
||||
)
|
||||
|
||||
pickToSlices := func(can int32) {
|
||||
candidates[idx] = can
|
||||
idx++
|
||||
}
|
||||
|
||||
ShuffleAndDeal(hashValue, deckSize, handSize, pickToSlices)
|
||||
|
||||
return candidates
|
||||
}
|
||||
|
||||
// ShuffleAndDealToSliceWithValidation will do validation before ShuffleAndDealToSlice
|
||||
func ShuffleAndDealToSliceWithValidation(hashValue uint64, deckSize, handSize int32) ([]int32, error) {
|
||||
if !ValidateParameters(deckSize, handSize) {
|
||||
return nil, errors.New("bad parameters")
|
||||
}
|
||||
|
||||
return ShuffleAndDealToSlice(hashValue, deckSize, handSize), nil
|
||||
}
|
||||
@ -0,0 +1,304 @@
|
||||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package shufflesharding
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateParameters(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
deckSize int32
|
||||
handSize int32
|
||||
validated bool
|
||||
}{
|
||||
{
|
||||
"deckSize is < 0",
|
||||
-100,
|
||||
8,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"handSize is < 0",
|
||||
128,
|
||||
-100,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"deckSize is 0",
|
||||
0,
|
||||
8,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"handSize is 0",
|
||||
128,
|
||||
0,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"handSize is greater than deckSize",
|
||||
128,
|
||||
129,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"deckSize: 128 handSize: 6",
|
||||
128,
|
||||
6,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize: 1024 handSize: 6",
|
||||
1024,
|
||||
6,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize: 512 handSize: 8",
|
||||
512,
|
||||
8,
|
||||
false,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if ValidateParameters(test.deckSize, test.handSize) != test.validated {
|
||||
t.Errorf("test case %s fails", test.name)
|
||||
return
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkValidateParameters(b *testing.B) {
|
||||
deckSize, handSize := int32(512), int32(8)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = ValidateParameters(deckSize, handSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShuffleAndDealWithValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
deckSize int32
|
||||
handSize int32
|
||||
pick func(int32)
|
||||
validated bool
|
||||
}{
|
||||
{
|
||||
"deckSize is < 0",
|
||||
-100,
|
||||
8,
|
||||
func(int32) {},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"handSize is < 0",
|
||||
128,
|
||||
-100,
|
||||
func(int32) {},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"deckSize is 0",
|
||||
0,
|
||||
8,
|
||||
func(int32) {},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"handSize is 0",
|
||||
128,
|
||||
0,
|
||||
func(int32) {},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"handSize is greater than deckSize",
|
||||
128,
|
||||
129,
|
||||
func(int32) {},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"deckSize: 128 handSize: 6",
|
||||
128,
|
||||
6,
|
||||
func(int32) {},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize: 1024 handSize: 6",
|
||||
1024,
|
||||
6,
|
||||
func(int32) {},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize: 512 handSize: 8",
|
||||
512,
|
||||
8,
|
||||
func(int32) {},
|
||||
false,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if (ShuffleAndDealWithValidation(rand.Uint64(), test.deckSize, test.handSize, test.pick) == nil) != test.validated {
|
||||
t.Errorf("test case %s fails", test.name)
|
||||
return
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkShuffleAndDeal(b *testing.B) {
|
||||
hashValueBase := math.MaxUint64 / uint64(b.N)
|
||||
deckSize, handSize := int32(512), int32(8)
|
||||
pick := func(int32) {}
|
||||
for i := 0; i < b.N; i++ {
|
||||
ShuffleAndDeal(hashValueBase*uint64(i), deckSize, handSize, pick)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShuffleAndDealToSliceWithValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
deckSize int32
|
||||
handSize int32
|
||||
validated bool
|
||||
}{
|
||||
{
|
||||
"validation fails",
|
||||
-100,
|
||||
-100,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"deckSize = handSize = 4",
|
||||
4,
|
||||
4,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize = handSize = 8",
|
||||
8,
|
||||
8,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize = handSize = 10",
|
||||
10,
|
||||
10,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize = handSize = 12",
|
||||
12,
|
||||
12,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize = 128, handSize = 8",
|
||||
128,
|
||||
8,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize = 256, handSize = 7",
|
||||
256,
|
||||
7,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"deckSize = 512, handSize = 6",
|
||||
512,
|
||||
6,
|
||||
true,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
hashValue := rand.Uint64()
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cards, err := ShuffleAndDealToSliceWithValidation(hashValue, test.deckSize, test.handSize)
|
||||
if (err == nil) != test.validated {
|
||||
t.Errorf("test case %s fails in validation check", test.name)
|
||||
return
|
||||
}
|
||||
|
||||
if test.validated {
|
||||
// check cards number
|
||||
if len(cards) != int(test.handSize) {
|
||||
t.Errorf("test case %s fails in cards number", test.name)
|
||||
return
|
||||
}
|
||||
|
||||
// check cards range and duplication
|
||||
cardMap := make(map[int32]struct{})
|
||||
for _, cardIdx := range cards {
|
||||
if cardIdx < 0 || cardIdx >= test.deckSize {
|
||||
t.Errorf("test case %s fails in range check", test.name)
|
||||
return
|
||||
}
|
||||
cardMap[cardIdx] = struct{}{}
|
||||
}
|
||||
if len(cardMap) != int(test.handSize) {
|
||||
t.Errorf("test case %s fails in duplication check", test.name)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkShuffleAndDealToSlice(b *testing.B) {
|
||||
hashValueBase := math.MaxUint64 / uint64(b.N)
|
||||
deckSize, handSize := int32(512), int32(8)
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = ShuffleAndDealToSlice(hashValueBase*uint64(i), deckSize, handSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUniformDistribution(t *testing.T) {
|
||||
deckSize, handSize := int32(128), int32(3)
|
||||
handCoordinateMap := make(map[int]int)
|
||||
|
||||
allCoordinateCount := 128 * 127 * 126 / 6
|
||||
|
||||
for i := 0; i < allCoordinateCount*16; i++ {
|
||||
hands := ShuffleAndDealToSlice(rand.Uint64(), deckSize, handSize)
|
||||
sort.Slice(hands, func(i, j int) bool {
|
||||
return hands[i] < hands[j]
|
||||
})
|
||||
handCoordinate := 0
|
||||
for _, hand := range hands {
|
||||
handCoordinate = handCoordinate<<7 + int(hand)
|
||||
}
|
||||
handCoordinateMap[handCoordinate]++
|
||||
}
|
||||
|
||||
// TODO: check uniform distribution
|
||||
t.Logf("%d", len(handCoordinateMap))
|
||||
t.Logf("%d", allCoordinateCount)
|
||||
|
||||
return
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user