Merge pull request #53720 from shyamjvs/test-kubemark

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Optimize random string generator to avoid multiple locks & use bit-masking

Ref https://github.com/kubernetes/kubernetes/issues/53327

We recently started seeing a 50% decrease in scheduling throughput (for e.g in kubemark-500 scale job) and turns out https://github.com/kubernetes/kubernetes/pull/53135 introduced it.
The reason is [this call](2caae38d32/plugin/pkg/scheduler/algorithm/predicates/predicates.go (L272)) to create a random 32-length string.
From the code of the `rand` utility (which is being heavily used throughout the system for randomizing object names), I noticed following performance issues:
- to create an n-length string, we are making n calls to `rand.Intn()` each of which does a lock+unlock operation on the RNG.. while just 1 lock+unlock operation is enough for all
- we're choosing one character (from an alphabet of 27 chars) per each random integer.. while we can select 10 characters using a single int63 (by masking and bit-shifting) as 1 character uses just 5 bits of randomness
- the character set is defined as a global slice (mutable), so the compiler needs to fetch length of the slice on each invocation to `len()` (we're making n of those).. while we can just use a const string (immutable) which will make len directly available as a cached constant (yes, go does it!)

This PR is making the above fixes. I'll try to add some benchmarking to measure the difference (as @wojtek-t suggested).

/cc @kubernetes/sig-scalability-misc @kubernetes/sig-scheduling-bugs @kubernetes/sig-api-machinery-misc @wojtek-t @smarterclayton

pkg/controller/serviceaccount/tokens_controller_test.go

@@ -73,7 +73,7 @@ func tokenSecretReferences() []v1.ObjectReference {
 
 // addTokenSecretReference adds a reference to the ServiceAccountToken that will be created
 func addTokenSecretReference(refs []v1.ObjectReference) []v1.ObjectReference {
-	return addNamedTokenSecretReference(refs, "default-token-stdpg")
+	return addNamedTokenSecretReference(refs, "default-token-xn8fg")
 }
 
 // addNamedTokenSecretReference adds a reference to the named ServiceAccountToken
@@ -118,9 +118,9 @@ func opaqueSecret() *v1.Secret {
 }
 
 // createdTokenSecret returns the ServiceAccountToken secret posted when creating a new token secret.
-// Named "default-token-stdpg", since that is the first generated name after rand.Seed(1)
+// Named "default-token-xn8fg", since that is the first generated name after rand.Seed(1)
 func createdTokenSecret(overrideName ...string) *v1.Secret {
-	return namedCreatedTokenSecret("default-token-stdpg")
+	return namedCreatedTokenSecret("default-token-xn8fg")
 }
 
 // namedTokenSecret returns the ServiceAccountToken secret posted when creating a new token secret with the given name.
@@ -264,12 +264,12 @@ func TestTokenCreation(t *testing.T) {
 
 				// Attempt 2
 				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "serviceaccounts"}, metav1.NamespaceDefault, "default"),
-				core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, metav1.NamespaceDefault, namedCreatedTokenSecret("default-token-jk9rt")),
+				core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, metav1.NamespaceDefault, namedCreatedTokenSecret("default-token-txhzt")),
 
 				// Attempt 3
 				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "serviceaccounts"}, metav1.NamespaceDefault, "default"),
-				core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, metav1.NamespaceDefault, namedCreatedTokenSecret("default-token-684pg")),
-				core.NewUpdateAction(schema.GroupVersionResource{Version: "v1", Resource: "serviceaccounts"}, metav1.NamespaceDefault, serviceAccount(addNamedTokenSecretReference(emptySecretReferences(), "default-token-684pg"))),
+				core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, metav1.NamespaceDefault, namedCreatedTokenSecret("default-token-vnmz7")),
+				core.NewUpdateAction(schema.GroupVersionResource{Version: "v1", Resource: "serviceaccounts"}, metav1.NamespaceDefault, serviceAccount(addNamedTokenSecretReference(emptySecretReferences(), "default-token-vnmz7"))),
 			},
 		},
 		"new serviceaccount with no secrets encountering unending create error": {
@@ -293,10 +293,10 @@ func TestTokenCreation(t *testing.T) {
 				core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, metav1.NamespaceDefault, createdTokenSecret()),
 				// Retry 1
 				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "serviceaccounts"}, metav1.NamespaceDefault, "default"),
-				core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, metav1.NamespaceDefault, namedCreatedTokenSecret("default-token-jk9rt")),
+				core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, metav1.NamespaceDefault, namedCreatedTokenSecret("default-token-txhzt")),
 				// Retry 2
 				core.NewGetAction(schema.GroupVersionResource{Version: "v1", Resource: "serviceaccounts"}, metav1.NamespaceDefault, "default"),
-				core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, metav1.NamespaceDefault, namedCreatedTokenSecret("default-token-684pg")),
+				core.NewCreateAction(schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, metav1.NamespaceDefault, namedCreatedTokenSecret("default-token-vnmz7")),
 			},
 		},
 		"new serviceaccount with missing secrets": {

staging/src/k8s.io/apimachinery/pkg/util/rand/rand.go

@@ -70,16 +70,41 @@ func Perm(n int) []int {
 	return rng.rand.Perm(n)
 }
 
-// We omit vowels from the set of available characters to reduce the chances
-// of "bad words" being formed.
-var alphanums = []rune("bcdfghjklmnpqrstvwxz2456789")
+const (
+	// We omit vowels from the set of available characters to reduce the chances
+	// of "bad words" being formed.
+	alphanums = "bcdfghjklmnpqrstvwxz2456789"
+	// No. of bits required to index into alphanums string.
+	alphanumsIdxBits = 5
+	// Mask used to extract last alphanumsIdxBits of an int.
+	alphanumsIdxMask = 1<<alphanumsIdxBits - 1
+	// No. of random letters we can extract from a single int63.
+	maxAlphanumsPerInt = 63 / alphanumsIdxBits
+)
 
 // String generates a random alphanumeric string, without vowels, which is n
 // characters long.  This will panic if n is less than zero.
-func String(length int) string {
-	b := make([]rune, length)
-	for i := range b {
-		b[i] = alphanums[Intn(len(alphanums))]
+// How the random string is created:
+// - we generate random int63's
+// - from each int63, we are extracting multiple random letters by bit-shifting and masking
+// - if some index is out of range of alphanums we neglect it (unlikely to happen multiple times in a row)
+func String(n int) string {
+	b := make([]byte, n)
+	rng.Lock()
+	defer rng.Unlock()
+
+	randomInt63 := rng.rand.Int63()
+	remaining := maxAlphanumsPerInt
+	for i := 0; i < n; {
+		if remaining == 0 {
+			randomInt63, remaining = rng.rand.Int63(), maxAlphanumsPerInt
+		}
+		if idx := int(randomInt63 & alphanumsIdxMask); idx < len(alphanums) {
+			b[i] = alphanums[idx]
+			i++
+		}
+		randomInt63 >>= alphanumsIdxBits
+		remaining--
 	}
 	return string(b)
 }
@@ -87,7 +112,7 @@ func String(length int) string {
 // SafeEncodeString encodes s using the same characters as rand.String. This reduces the chances of bad words and
 // ensures that strings generated from hash functions appear consistent throughout the API.
 func SafeEncodeString(s string) string {
-	r := make([]rune, len(s))
+	r := make([]byte, len(s))
 	for i, b := range []rune(s) {
 		r[i] = alphanums[(int(b) % len(alphanums))]
 	}

staging/src/k8s.io/apimachinery/pkg/util/rand/rand_test.go

@@ -24,6 +24,7 @@ import (
 
 const (
 	maxRangeTestCount = 500
+	testStringLength  = 32
 )
 
 func TestString(t *testing.T) {
@@ -99,3 +100,15 @@ func TestInt63nRange(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkRandomStringGeneration(b *testing.B) {
+	b.ResetTimer()
+	var s string
+	for i := 0; i < b.N; i++ {
+		s = String(testStringLength)
+	}
+	b.StopTimer()
+	if len(s) == 0 {
+		b.Fatal(s)
+	}
+}