address some comments

Clean up useless functions, only keep the basic function Deal
and the function DealIntoHand which will be used by Priority
and Fairness.

Improve some comments for constants and functions.

Introduce Dealer to combine parameters and methods into a whole.

Use fixed-size slice to improve performance.

Use math.Ceil and math.Log2 to calculate required entropy bits.

Make the given hand adaptive to handSize in DealIntoHand.

Signed-off-by: Bruce Ma <brucema19901024@gmail.com>

Kubernetes-commit: 7a3ca070cdd9804a22bf5db8a99576b09fc52484
This commit is contained in:
Bruce Ma 2019-10-11 20:06:31 +08:00 committed by Kubernetes Publisher
parent a74f0d114b
commit 87c35dc179
2 changed files with 218 additions and 273 deletions

View File

@ -17,117 +17,91 @@ limitations under the License.
package shufflesharding
import (
"errors"
"fmt"
"math"
"strings"
)
const maxHashBits = 60
// MaxHashBits is the max bit length which can be used from hash value.
// If we use all bits of hash value, the critical(last) card shuffled by
// Dealer will be uneven to 2:3 (first half:second half) at most,
// in order to reduce this unevenness to 32:33, we set MaxHashBits to 60 here.
const MaxHashBits = 60
// ValidateParameters finds errors in the parameters for shuffle
// sharding. Returns a slice for which `len()` is 0 if and only if
// there are no errors. The entropy requirement is evaluated in a
// fast but approximate way: bits(deckSize^handSize).
func ValidateParameters(deckSize, handSize int) (errs []string) {
if handSize <= 0 {
errs = append(errs, "handSize is not positive")
}
if deckSize <= 0 {
errs = append(errs, "deckSize is not positive")
}
if len(errs) > 0 {
return
}
if handSize > deckSize {
return []string{"handSize is greater than deckSize"}
}
if math.Log2(float64(deckSize))*float64(handSize) > maxHashBits {
return []string{fmt.Sprintf("more than %d bits of entropy required", maxHashBits)}
}
return
// RequiredEntropyBits makes a quick and slightly conservative estimate of the number
// of bits of hash value that are consumed in shuffle sharding a deck of the given size
// to a hand of the given size. The result is meaningful only if
// 1 <= handSize <= deckSize <= 1<<26.
func RequiredEntropyBits(deckSize, handSize int) int {
return int(math.Ceil(math.Log2(float64(deckSize)) * float64(handSize)))
}
// ShuffleAndDeal can shuffle a hash value to handSize-quantity and non-redundant
// indices of decks, with the pick function, we can get the optimal deck index
// Eg. From deckSize=128, handSize=8, we can get an index array [12 14 73 18 119 51 117 26],
// then pick function will choose the optimal index from these
// Algorithm: https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190228-priority-and-fairness.md#queue-assignment-proof-of-concept
func ShuffleAndDeal(hashValue uint64, deckSize, handSize int, pick func(int)) {
remainders := make([]int, handSize)
// Dealer contains some necessary parameters and provides some methods for shuffle sharding.
// Dealer is thread-safe.
type Dealer struct {
deckSize int
handSize int
}
for i := 0; i < handSize; i++ {
hashValueNext := hashValue / uint64(deckSize-i)
remainders[i] = int(hashValue - uint64(deckSize-i)*hashValueNext)
// NewDealer will create a Dealer with the given deckSize and handSize, will return error when
// deckSize or handSize is invalid as below.
// 1. deckSize or handSize is not positive
// 2. handSize is greater than deckSize
// 3. deckSize is impractically large (greater than 1<<26)
// 4. required entropy bits of deckSize and handSize is greater than MaxHashBits
func NewDealer(deckSize, handSize int) (*Dealer, error) {
if deckSize <= 0 || handSize <= 0 {
return nil, fmt.Errorf("deckSize %d or handSize %d is not positive", deckSize, handSize)
}
if handSize > deckSize {
return nil, fmt.Errorf("handSize %d is greater than deckSize %d", handSize, deckSize)
}
if deckSize > 1<<26 {
return nil, fmt.Errorf("deckSize %d is impractically large", deckSize)
}
if RequiredEntropyBits(deckSize, handSize) > MaxHashBits {
return nil, fmt.Errorf("required entropy bits of deckSize %d and handSize %d is greater than %d", deckSize, handSize, MaxHashBits)
}
return &Dealer{
deckSize: deckSize,
handSize: handSize,
}, nil
}
// Deal shuffles a card deck and deals a hand of cards, using the given hashValue as the source of entropy.
// The deck size and hand size are properties of the Dealer.
// This function synchronously makes sequential calls to pick, one for each dealt card.
// Each card is identified by an integer in the range [0, deckSize).
// For example, for deckSize=128 and handSize=4 this function might call pick(14); pick(73); pick(119); pick(26).
func (d *Dealer) Deal(hashValue uint64, pick func(int)) {
// 15 is the largest possible value of handSize
var remainders [15]int
for i := 0; i < d.handSize; i++ {
hashValueNext := hashValue / uint64(d.deckSize-i)
remainders[i] = int(hashValue - uint64(d.deckSize-i)*hashValueNext)
hashValue = hashValueNext
}
for i := 0; i < handSize; i++ {
candidate := remainders[i]
for i := 0; i < d.handSize; i++ {
card := remainders[i]
for j := i; j > 0; j-- {
if candidate >= remainders[j-1] {
candidate++
if card >= remainders[j-1] {
card++
}
}
pick(candidate)
pick(card)
}
}
// ShuffleAndDealWithValidation will do validation before ShuffleAndDeal
func ShuffleAndDealWithValidation(hashValue uint64, deckSize, handSize int, pick func(int)) error {
if errs := ValidateParameters(deckSize, handSize); len(errs) > 0 {
return errors.New(strings.Join(errs, ";"))
}
ShuffleAndDeal(hashValue, deckSize, handSize, pick)
return nil
}
// ShuffleAndDealToSlice will use specific pick function to return slices of indices
// after ShuffleAndDeal
func ShuffleAndDealToSlice(hashValue uint64, deckSize, handSize int) []int {
var (
candidates = make([]int, handSize)
idx = 0
)
pickToSlices := func(can int) {
candidates[idx] = int(can)
idx++
}
ShuffleAndDeal(hashValue, deckSize, handSize, pickToSlices)
return candidates
}
// ShuffleAndDealIntoHand shuffles a deck of the given size by the
// given hash value and deals cards into the given slice. The virtue
// of this function compared to ShuffleAndDealToSlice is that the
// caller provides the storage for the hand.
func ShuffleAndDealIntoHand(hashValue uint64, deckSize int, hand []int) {
handSize := len(hand)
var idx int
ShuffleAndDeal(hashValue, deckSize, handSize, func(card int) {
hand[idx] = int(card)
idx++
})
}
// ShuffleAndDealToSliceWithValidation will do validation before ShuffleAndDealToSlice
func ShuffleAndDealToSliceWithValidation(hashValue uint64, deckSize, handSize int) ([]int, error) {
if errs := ValidateParameters(deckSize, handSize); len(errs) > 0 {
return nil, errors.New(strings.Join(errs, ";"))
}
return ShuffleAndDealToSlice(hashValue, deckSize, handSize), nil
}
// ShuffleAndDealIntoHandWithValidation does validation and then ShuffleAndDealIntoHand
func ShuffleAndDealIntoHandWithValidation(hashValue uint64, deckSize int, hand []int) error {
if errs := ValidateParameters(deckSize, len(hand)); len(errs) > 0 {
return errors.New(strings.Join(errs, ";"))
}
ShuffleAndDealIntoHand(hashValue, deckSize, hand)
return nil
// DealIntoHand shuffles and deals according to the Dealer's parameters,
// using the given hashValue as the source of entropy and then
// returns the dealt cards as a slice of `int`.
// If `hand` has the correct length as Dealer's handSize, it will be used as-is and no allocations will be made.
// If `hand` is nil or too small, it will be extended (performing an allocation).
// If `hand` is too large, a sub-slice will be returned.
func (d *Dealer) DealIntoHand(hashValue uint64, hand []int) []int {
h := hand[:0]
d.Deal(hashValue, func(card int) { h = append(h, card) })
return h
}

View File

@ -17,55 +17,80 @@ limitations under the License.
package shufflesharding
import (
"fmt"
"math"
"math/rand"
"reflect"
"sort"
"strings"
"testing"
)
func TestValidateParameters(t *testing.T) {
func TestRequiredEntropyBits(t *testing.T) {
tests := []struct {
name string
deckSize int
handSize int
expectedBits int
}{
{
"deckSize: 1024 handSize: 6",
1024,
6,
60,
},
{
"deckSize: 512 handSize: 8",
512,
8,
72,
},
}
for _, test := range tests {
bits := RequiredEntropyBits(test.deckSize, test.handSize)
if bits != test.expectedBits {
t.Errorf("test %s fails: expected %v but got %v", test.name, test.expectedBits, bits)
return
}
}
}
func TestNewDealer(t *testing.T) {
tests := []struct {
name string
deckSize int
handSize int
errors []string
err error
}{
{
"deckSize is < 0",
"deckSize <= 0",
-100,
8,
[]string{"deckSize is not positive"},
fmt.Errorf("deckSize -100 or handSize 8 is not positive"),
},
{
"handSize is < 0",
128,
-100,
[]string{"handSize is not positive"},
},
{
"deckSize is 0",
"handSize <= 0",
100,
0,
8,
[]string{"deckSize is not positive"},
},
{
"handSize is 0",
128,
0,
[]string{"handSize is not positive"},
fmt.Errorf("deckSize 100 or handSize 0 is not positive"),
},
{
"handSize is greater than deckSize",
128,
129,
[]string{"handSize is greater than deckSize"},
100,
101,
fmt.Errorf("handSize 101 is greater than deckSize 100"),
},
{
"deckSize: 128 handSize: 6",
128,
6,
nil,
"deckSize is impractically large",
1 << 27,
2,
fmt.Errorf("deckSize 134217728 is impractically large"),
},
{
"required entropy bits is greater than MaxHashBits",
512,
8,
fmt.Errorf("required entropy bits of deckSize 512 and handSize 8 is greater than 60"),
},
{
"deckSize: 1024 handSize: 6",
@ -73,211 +98,92 @@ func TestValidateParameters(t *testing.T) {
6,
nil,
},
{
"deckSize: 512 handSize: 8",
512,
8,
[]string{"more than 60 bits of entropy required"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got := strings.Join(ValidateParameters(test.deckSize, test.handSize), ";")
expected := strings.Join(test.errors, ";")
if got != expected {
t.Errorf("test case %s got %q but expected %q", test.name, got, expected)
_, err := NewDealer(test.deckSize, test.handSize)
if !reflect.DeepEqual(err, test.err) {
t.Errorf("test %s fails: expected %v but got %v", test.name, test.err, err)
return
}
})
}
}
func BenchmarkValidateParameters(b *testing.B) {
deckSize, handSize := 512, 8
for i := 0; i < b.N; i++ {
_ = ValidateParameters(deckSize, handSize)
}
}
func TestShuffleAndDealWithValidation(t *testing.T) {
func TestCardDuplication(t *testing.T) {
tests := []struct {
name string
deckSize int
handSize int
pick func(int)
validated bool
name string
deckSize int
handSize int
}{
{
"deckSize is < 0",
-100,
8,
func(int) {},
false,
},
{
"handSize is < 0",
128,
-100,
func(int) {},
false,
},
{
"deckSize is 0",
0,
8,
func(int) {},
false,
},
{
"handSize is 0",
128,
0,
func(int) {},
false,
},
{
"handSize is greater than deckSize",
128,
129,
func(int) {},
false,
},
{
"deckSize: 128 handSize: 6",
128,
6,
func(int) {},
true,
},
{
"deckSize: 1024 handSize: 6",
1024,
6,
func(int) {},
true,
},
{
"deckSize: 512 handSize: 8",
512,
8,
func(int) {},
false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if (ShuffleAndDealWithValidation(rand.Uint64(), test.deckSize, test.handSize, test.pick) == nil) != test.validated {
t.Errorf("test case %s fails", test.name)
return
}
})
}
}
func BenchmarkShuffleAndDeal(b *testing.B) {
hashValueBase := math.MaxUint64 / uint64(b.N)
deckSize, handSize := 512, 8
pick := func(int) {}
for i := 0; i < b.N; i++ {
ShuffleAndDeal(hashValueBase*uint64(i), deckSize, handSize, pick)
}
}
func TestShuffleAndDealToSliceWithValidation(t *testing.T) {
tests := []struct {
name string
deckSize int
handSize int
validated bool
}{
{
"validation fails",
-100,
-100,
false,
},
{
"deckSize = handSize = 4",
4,
4,
true,
},
{
"deckSize = handSize = 8",
8,
8,
true,
},
{
"deckSize = handSize = 10",
10,
10,
true,
},
{
"deckSize = handSize = 12",
12,
12,
true,
},
{
"deckSize = 128, handSize = 8",
128,
8,
true,
},
{
"deckSize = 256, handSize = 7",
256,
7,
true,
},
{
"deckSize = 512, handSize = 6",
512,
6,
true,
},
}
for _, test := range tests {
hashValue := rand.Uint64()
t.Run(test.name, func(t *testing.T) {
cards, err := ShuffleAndDealToSliceWithValidation(hashValue, test.deckSize, test.handSize)
if (err == nil) != test.validated {
t.Errorf("test case %s fails in validation check", test.name)
dealer, err := NewDealer(test.deckSize, test.handSize)
if err != nil {
t.Errorf("fail to create Dealer: %v", err)
return
}
hand := make([]int, 0)
hand = dealer.DealIntoHand(hashValue, hand)
// check cards number
if len(hand) != int(test.handSize) {
t.Errorf("test case %s fails in cards number", test.name)
return
}
if test.validated {
// check cards number
if len(cards) != int(test.handSize) {
t.Errorf("test case %s fails in cards number", test.name)
return
}
// check cards range and duplication
cardMap := make(map[int]struct{})
for _, card := range cards {
if card < 0 || card >= int(test.deckSize) {
t.Errorf("test case %s fails in range check", test.name)
return
}
cardMap[card] = struct{}{}
}
if len(cardMap) != int(test.handSize) {
t.Errorf("test case %s fails in duplication check", test.name)
// check cards range and duplication
cardMap := make(map[int]struct{})
for _, card := range hand {
if card < 0 || card >= int(test.deckSize) {
t.Errorf("test case %s fails in range check", test.name)
return
}
cardMap[card] = struct{}{}
}
if len(cardMap) != int(test.handSize) {
t.Errorf("test case %s fails in duplication check", test.name)
return
}
})
}
}
func BenchmarkShuffleAndDealToSlice(b *testing.B) {
hashValueBase := math.MaxUint64 / uint64(b.N)
deckSize, handSize := 512, 8
for i := 0; i < b.N; i++ {
_ = ShuffleAndDealToSlice(hashValueBase*uint64(i), deckSize, handSize)
})
}
}
@ -293,7 +199,7 @@ func ff(n, m int) int {
}
func TestUniformDistribution(t *testing.T) {
const spare = 64 - maxHashBits
const spare = 64 - MaxHashBits
tests := []struct {
deckSize, handSize int
hashMax int
@ -304,6 +210,11 @@ func TestUniformDistribution(t *testing.T) {
{70, 4, ff(70, 4)},
}
for _, test := range tests {
dealer, err := NewDealer(test.deckSize, test.handSize)
if err != nil {
t.Errorf("fail to create Dealer: %v", err)
return
}
handCoordinateMap := make(map[int]int) // maps coded hand to count of times seen
fallingFactorial := ff(test.deckSize, test.handSize)
@ -314,7 +225,7 @@ func TestUniformDistribution(t *testing.T) {
maxCount := permutations * int(math.Ceil(nff))
aHand := make([]int, test.handSize)
for i := 0; i < test.hashMax; i++ {
ShuffleAndDealIntoHand(uint64(i), test.deckSize, aHand)
aHand = dealer.DealIntoHand(uint64(i), aHand)
sort.IntSlice(aHand).Sort()
handCoordinate := 0
for _, card := range aHand {
@ -345,3 +256,63 @@ func TestUniformDistribution(t *testing.T) {
}
return
}
func TestDealer_DealIntoHand(t *testing.T) {
dealer, _ := NewDealer(6, 6)
tests := []struct {
name string
hand []int
expectedSize int
}{
{
"nil slice",
nil,
6,
},
{
"empty slice",
make([]int, 0),
6,
},
{
"size: 6 cap: 6 slice",
make([]int, 6, 6),
6,
},
{
"size: 6 cap: 12 slice",
make([]int, 6, 12),
6,
},
{
"size: 4 cap: 4 slice",
make([]int, 4, 4),
6,
},
{
"size: 4 cap: 12 slice",
make([]int, 4, 12),
6,
},
{
"size: 10 cap: 10 slice",
make([]int, 10, 10),
6,
},
{
"size: 10 cap: 12 slice",
make([]int, 10, 12),
6,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
h := dealer.DealIntoHand(0, test.hand)
if len(h) != test.expectedSize {
t.Errorf("test %s fails: expetced size %d but got %d", test.name, test.expectedSize, len(h))
return
}
})
}
}