From 1657ef25ebdb4ee6a71ffb6f367efb20f786590c Mon Sep 17 00:00:00 2001 From: wojtekt Date: Tue, 10 Dec 2019 11:02:24 +0100 Subject: [PATCH] Extend authorization benchmark --- .../authorizer/node/node_authorizer_test.go | 172 +++++++++++------- 1 file changed, 102 insertions(+), 70 deletions(-) diff --git a/plugin/pkg/auth/authorizer/node/node_authorizer_test.go b/plugin/pkg/auth/authorizer/node/node_authorizer_test.go index dd708f07468..9490026b567 100644 --- a/plugin/pkg/auth/authorizer/node/node_authorizer_test.go +++ b/plugin/pkg/auth/authorizer/node/node_authorizer_test.go @@ -19,6 +19,7 @@ package node import ( "context" "fmt" + "math/rand" "runtime" "runtime/pprof" "sync/atomic" @@ -65,7 +66,7 @@ func init() { func TestAuthorizer(t *testing.T) { g := NewGraph() - opts := sampleDataOpts{ + opts := &sampleDataOpts{ nodes: 2, namespaces: 2, podsPerNode: 2, @@ -521,14 +522,25 @@ func TestAuthorizerSharedResources(t *testing.T) { } type sampleDataOpts struct { - nodes int - - namespaces int - + nodes int + namespaces int podsPerNode int attachmentsPerNode int + // sharedConfigMapsPerNamespaces defines number of shared configmaps in a given + // namespace. Each pod then mounts a random set of size `sharedConfigMapsPerPod` + // from that set. sharedConfigMapsPerPod is used if greater. + sharedConfigMapsPerNamespace int + // sharedSecretsPerNamespaces defines number of shared secrets in a given + // namespace. Each pod then mounts a random set of size `sharedSecretsPerPod` + // from that set. sharedSecretsPerPod is used if greater. + sharedSecretsPerNamespace int + // sharedPVCsPerNamespaces defines number of shared pvcs in a given + // namespace. Each pod then mounts a random set of size `sharedPVCsPerPod` + // from that set. sharedPVCsPerPod is used if greater. + sharedPVCsPerNamespace int + sharedConfigMapsPerPod int sharedSecretsPerPod int sharedPVCsPerPod int @@ -539,7 +551,7 @@ type sampleDataOpts struct { } func BenchmarkPopulationAllocation(b *testing.B) { - opts := sampleDataOpts{ + opts := &sampleDataOpts{ nodes: 500, namespaces: 200, podsPerNode: 200, @@ -570,7 +582,7 @@ func BenchmarkPopulationRetention(b *testing.B) { // go tool pprof --inuse_space node.test plugin/pkg/auth/authorizer/node/BenchmarkPopulationRetention.profile // list populate - opts := sampleDataOpts{ + opts := &sampleDataOpts{ nodes: 500, namespaces: 200, podsPerNode: 200, @@ -608,7 +620,7 @@ func BenchmarkWriteIndexMaintenance(b *testing.B) { // Run with: // go test ./plugin/pkg/auth/authorizer/node -benchmem -bench BenchmarkWriteIndexMaintenance -run None - opts := sampleDataOpts{ + opts := &sampleDataOpts{ // simulate high replication in a small number of namespaces: nodes: 5000, namespaces: 1, @@ -639,7 +651,7 @@ func BenchmarkWriteIndexMaintenance(b *testing.B) { func BenchmarkAuthorization(b *testing.B) { g := NewGraph() - opts := sampleDataOpts{ + opts := &sampleDataOpts{ // To simulate high replication in a small number of namespaces: // nodes: 5000, // namespaces: 10, @@ -732,6 +744,8 @@ func BenchmarkAuthorization(b *testing.B) { }, } + podToAdd, _ := generatePod("testwrite", "ns0", "node0", "default", opts) + b.ResetTimer() for _, testWriteContention := range []bool{false, true} { @@ -755,16 +769,7 @@ func BenchmarkAuthorization(b *testing.B) { for shouldWrite == 1 { go func() { start := time.Now() - authz.graph.AddPod(&corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "testwrite", Namespace: "ns0"}, - Spec: corev1.PodSpec{ - NodeName: "node0", - ServiceAccountName: "default", - Volumes: []corev1.Volume{ - {Name: "token", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "secret0-shared"}}}, - }, - }, - }) + authz.graph.AddPod(podToAdd) diff := time.Since(start) atomic.AddInt64(&writes, 1) switch { @@ -840,71 +845,35 @@ func populate(graph *Graph, nodes []*corev1.Node, pods []*corev1.Pod, pvs []*cor } } +func randomSubset(a, b int) []int { + if b < a { + b = a + } + return rand.Perm(b)[:a] +} + // generate creates sample pods and persistent volumes based on the provided options. // the secret/configmap/pvc/node references in the pod and pv objects are named to indicate the connections between the objects. // for example, secret0-pod0-node0 is a secret referenced by pod0 which is bound to node0. // when populated into the graph, the node authorizer should allow node0 to access that secret, but not node1. -func generate(opts sampleDataOpts) ([]*corev1.Node, []*corev1.Pod, []*corev1.PersistentVolume, []*storagev1.VolumeAttachment) { +func generate(opts *sampleDataOpts) ([]*corev1.Node, []*corev1.Pod, []*corev1.PersistentVolume, []*storagev1.VolumeAttachment) { nodes := make([]*corev1.Node, 0, opts.nodes) pods := make([]*corev1.Pod, 0, opts.nodes*opts.podsPerNode) pvs := make([]*corev1.PersistentVolume, 0, (opts.nodes*opts.podsPerNode*opts.uniquePVCsPerPod)+(opts.sharedPVCsPerPod*opts.namespaces)) attachments := make([]*storagev1.VolumeAttachment, 0, opts.nodes*opts.attachmentsPerNode) + rand.Seed(12345) + for n := 0; n < opts.nodes; n++ { nodeName := fmt.Sprintf("node%d", n) for p := 0; p < opts.podsPerNode; p++ { - pod := &corev1.Pod{} - pod.Namespace = fmt.Sprintf("ns%d", p%opts.namespaces) - pod.Name = fmt.Sprintf("pod%d-%s", p, nodeName) - pod.Spec.NodeName = nodeName - pod.Spec.ServiceAccountName = fmt.Sprintf("svcacct%d-%s", p, nodeName) - - for i := 0; i < opts.uniqueSecretsPerPod; i++ { - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{SecretName: fmt.Sprintf("secret%d-%s", i, pod.Name)}, - }}) - } - for i := 0; i < opts.sharedSecretsPerPod; i++ { - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{SecretName: fmt.Sprintf("secret%d-shared", i)}, - }}) - } - - for i := 0; i < opts.uniqueConfigMapsPerPod; i++ { - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: fmt.Sprintf("configmap%d-%s", i, pod.Name)}}, - }}) - } - for i := 0; i < opts.sharedConfigMapsPerPod; i++ { - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: fmt.Sprintf("configmap%d-shared", i)}}, - }}) - } - - for i := 0; i < opts.uniquePVCsPerPod; i++ { - pv := &corev1.PersistentVolume{} - pv.Name = fmt.Sprintf("pv%d-%s-%s", i, pod.Name, pod.Namespace) - pv.Spec.FlexVolume = &corev1.FlexPersistentVolumeSource{SecretRef: &corev1.SecretReference{Name: fmt.Sprintf("secret-%s", pv.Name)}} - pv.Spec.ClaimRef = &corev1.ObjectReference{Name: fmt.Sprintf("pvc%d-%s", i, pod.Name), Namespace: pod.Namespace} - pvs = append(pvs, pv) - - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pv.Spec.ClaimRef.Name}, - }}) - } - for i := 0; i < opts.sharedPVCsPerPod; i++ { - pv := &corev1.PersistentVolume{} - pv.Name = fmt.Sprintf("pv%d-shared-%s", i, pod.Namespace) - pv.Spec.FlexVolume = &corev1.FlexPersistentVolumeSource{SecretRef: &corev1.SecretReference{Name: fmt.Sprintf("secret-%s", pv.Name)}} - pv.Spec.ClaimRef = &corev1.ObjectReference{Name: fmt.Sprintf("pvc%d-shared", i), Namespace: pod.Namespace} - pvs = append(pvs, pv) - - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pv.Spec.ClaimRef.Name}, - }}) - } + name := fmt.Sprintf("pod%d-%s", p, nodeName) + namespace := fmt.Sprintf("ns%d", p%opts.namespaces) + svcAccountName := fmt.Sprintf("svcacct%d-%s", p, nodeName) + pod, podPVs := generatePod(name, namespace, nodeName, svcAccountName, opts) pods = append(pods, pod) + pvs = append(pvs, podPVs...) } for a := 0; a < opts.attachmentsPerNode; a++ { attachment := &storagev1.VolumeAttachment{} @@ -930,3 +899,66 @@ func generate(opts sampleDataOpts) ([]*corev1.Node, []*corev1.Pod, []*corev1.Per } return nodes, pods, pvs, attachments } + +func generatePod(name, namespace, nodeName, svcAccountName string, opts *sampleDataOpts) (*corev1.Pod, []*corev1.PersistentVolume) { + pvs := make([]*corev1.PersistentVolume, 0, opts.uniquePVCsPerPod+opts.sharedPVCsPerPod) + + pod := &corev1.Pod{} + pod.Name = name + pod.Namespace = namespace + pod.Spec.NodeName = nodeName + pod.Spec.ServiceAccountName = svcAccountName + + for i := 0; i < opts.uniqueSecretsPerPod; i++ { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: fmt.Sprintf("secret%d-%s", i, pod.Name)}, + }}) + } + // Choose shared secrets randomly from shared secrets in a namespace. + subset := randomSubset(opts.sharedSecretsPerPod, opts.sharedSecretsPerNamespace) + for _, i := range subset { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: fmt.Sprintf("secret%d-shared", i)}, + }}) + } + + for i := 0; i < opts.uniqueConfigMapsPerPod; i++ { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: fmt.Sprintf("configmap%d-%s", i, pod.Name)}}, + }}) + } + // Choose shared configmaps randomly from shared configmaps in a namespace. + subset = randomSubset(opts.sharedConfigMapsPerPod, opts.sharedConfigMapsPerNamespace) + for _, i := range subset { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: fmt.Sprintf("configmap%d-shared", i)}}, + }}) + } + + for i := 0; i < opts.uniquePVCsPerPod; i++ { + pv := &corev1.PersistentVolume{} + pv.Name = fmt.Sprintf("pv%d-%s-%s", i, pod.Name, pod.Namespace) + pv.Spec.FlexVolume = &corev1.FlexPersistentVolumeSource{SecretRef: &corev1.SecretReference{Name: fmt.Sprintf("secret-%s", pv.Name)}} + pv.Spec.ClaimRef = &corev1.ObjectReference{Name: fmt.Sprintf("pvc%d-%s", i, pod.Name), Namespace: pod.Namespace} + pvs = append(pvs, pv) + + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pv.Spec.ClaimRef.Name}, + }}) + } + // Choose shared pvcs randomly from shared pvcs in a namespace. + subset = randomSubset(opts.sharedPVCsPerPod, opts.sharedPVCsPerNamespace) + for _, i := range subset { + pv := &corev1.PersistentVolume{} + pv.Name = fmt.Sprintf("pv%d-shared-%s", i, pod.Namespace) + pv.Spec.FlexVolume = &corev1.FlexPersistentVolumeSource{SecretRef: &corev1.SecretReference{Name: fmt.Sprintf("secret-%s", pv.Name)}} + pv.Spec.ClaimRef = &corev1.ObjectReference{Name: fmt.Sprintf("pvc%d-shared", i), Namespace: pod.Namespace} + pvs = append(pvs, pv) + + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pv.Spec.ClaimRef.Name}, + }}) + } + + return pod, pvs +}