scheduler_perf: use dynamic client to create arbitrary objects

With a dynamic client and a rest mapper it is possible to load arbitrary YAML
files and create the object defined by it. This is simpler than adding specific
Go code for each supported type.

Because the version now matters, the incorrect version in the DRA YAMLs were
found and fixed.

test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml

@@ -1,4 +1,4 @@
-apiVersion: resource.k8s.io/v1alpha1
+apiVersion: resource.k8s.io/v1alpha2
 kind: ResourceClaimTemplate
 metadata:
   name: test-claim-template

test/integration/scheduler_perf/config/dra/resourceclass.yaml

@@ -1,4 +1,4 @@
-apiVersion: resource.k8s.io/v1alpha1
+apiVersion: resource.k8s.io/v1alpha2
 kind: ResourceClass
 metadata:
   name: test-class

test/integration/scheduler_perf/config/performance-config.yaml

@@ -729,16 +729,16 @@
     driverName: test-driver.cdi.k8s.io
     nodes: scheduler-perf-dra-*
     maxClaimsPerNodeParam: $maxClaimsPerNode
-  - opcode: createResourceClass
+  - opcode: createAny
     templatePath: config/dra/resourceclass.yaml
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/resourceclaimtemplate.yaml
     namespace: init
   - opcode: createPods
     namespace: init
     countParam: $initPods
     podTemplatePath: config/dra/pod-with-claim-template.yaml
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/resourceclaimtemplate.yaml
     namespace: test
   - opcode: createPods
@@ -799,24 +799,24 @@
     driverName: another-test-driver.cdi.k8s.io
     nodes: scheduler-perf-dra-*
     maxClaimsPerNodeParam: $maxClaimsPerNode
-  - opcode: createResourceClass
+  - opcode: createAny
     templatePath: config/dra/resourceclass.yaml
-  - opcode: createResourceClass
+  - opcode: createAny
     templatePath: config/dra/another-resourceclass.yaml
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/resourceclaimtemplate.yaml
     namespace: init
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/another-resourceclaimtemplate.yaml
     namespace: init
   - opcode: createPods
     namespace: init
     countParam: $initPods
     podTemplatePath: config/dra/pod-with-many-claim-templates.yaml
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/resourceclaimtemplate.yaml
     namespace: test
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/another-resourceclaimtemplate.yaml
     namespace: test
   - opcode: createPods

test/integration/scheduler_perf/create.go

@@ -19,17 +19,22 @@ package benchmark
 import (
 	"context"
 	"fmt"
+	"time"
 
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery/cached/memory"
+	"k8s.io/client-go/restmapper"
+	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/test/utils/ktesting"
 )
 
-// createOp defines an op where some object gets created from a template.
+// createAny defines an op where some object gets created from a YAML file.
-// Everything specific for that object (create call, op code, names) gets
+// The nameset can be specified.
-// provided through a type.
+type createAny struct {
-type createOp[T interface{}, P createOpType[T]] struct {
+	// Must match createAnyOpcode.
-	// Must match createOpType.Opcode().
 	Opcode operationCode
 	// Namespace the object should be created in. Must be empty for cluster-scoped objects.
 	Namespace string
@@ -37,53 +42,85 @@ type createOp[T interface{}, P createOpType[T]] struct {
 	TemplatePath string
 }
 
-func (cro *createOp[T, P]) isValid(allowParameterization bool) error {
+var _ runnableOp = &createAny{}
-	var p P
+
-	if cro.Opcode != p.Opcode() {
+func (c *createAny) isValid(allowParameterization bool) error {
-		return fmt.Errorf("invalid opcode %q; expected %q", cro.Opcode, p.Opcode())
+	if c.Opcode != createAnyOpcode {
+		return fmt.Errorf("invalid opcode %q; expected %q", c.Opcode, createAnyOpcode)
 	}
-	if p.Namespaced() && cro.Namespace == "" {
+	if c.TemplatePath == "" {
-		return fmt.Errorf("Namespace must be set")
-	}
-	if !p.Namespaced() && cro.Namespace != "" {
-		return fmt.Errorf("Namespace must not be set")
-	}
-	if cro.TemplatePath == "" {
 		return fmt.Errorf("TemplatePath must be set")
 	}
+	// The namespace can only be checked during later because we don't know yet
+	// whether the object is namespaced or cluster-scoped.
 	return nil
 }
 
-func (cro *createOp[T, P]) collectsMetrics() bool {
+func (c *createAny) collectsMetrics() bool {
 	return false
 }
 
-func (cro *createOp[T, P]) patchParams(w *workload) (realOp, error) {
+func (c *createAny) patchParams(w *workload) (realOp, error) {
-	return cro, cro.isValid(false)
+	return c, c.isValid(false)
 }
 
-func (cro *createOp[T, P]) requiredNamespaces() []string {
+func (c *createAny) requiredNamespaces() []string {
-	if cro.Namespace == "" {
+	if c.Namespace == "" {
 		return nil
 	}
-	return []string{cro.Namespace}
+	return []string{c.Namespace}
 }
 
-func (cro *createOp[T, P]) run(tCtx ktesting.TContext) {
+func (c *createAny) run(tCtx ktesting.TContext) {
-	var obj *T
+	var obj *unstructured.Unstructured
-	var p P
+	if err := getSpecFromFile(&c.TemplatePath, &obj); err != nil {
-	if err := getSpecFromFile(&cro.TemplatePath, &obj); err != nil {
+		tCtx.Fatalf("%s: parsing failed: %v", c.TemplatePath, err)
-		tCtx.Fatalf("parsing %s %q: %v", p.Name(), cro.TemplatePath, err)
 	}
-	if _, err := p.CreateCall(tCtx.Client(), cro.Namespace)(tCtx, obj, metav1.CreateOptions{}); err != nil {
-		tCtx.Fatalf("create %s: %v", p.Name(), err)
-	}
-}
 
-// createOpType provides type-specific values for the generic createOp.
+	// Not caching the discovery result isn't very efficient, but good enough when
-type createOpType[T interface{}] interface {
+	// createAny isn't done often.
-	Opcode() operationCode
+	discoveryCache := memory.NewMemCacheClient(tCtx.Client().Discovery())
-	Name() string
+	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryCache)
-	Namespaced() bool
+	gv, err := schema.ParseGroupVersion(obj.GetAPIVersion())
-	CreateCall(client clientset.Interface, namespace string) func(context.Context, *T, metav1.CreateOptions) (*T, error)
+	if err != nil {
+		tCtx.Fatalf("%s: extract group+version from object %q: %v", c.TemplatePath, klog.KObj(obj), err)
+	}
+	gk := schema.GroupKind{Group: gv.Group, Kind: obj.GetKind()}
+
+	create := func() error {
+		mapping, err := restMapper.RESTMapping(gk, gv.Version)
+		if err != nil {
+			// Cached mapping might be stale, refresh on next try.
+			restMapper.Reset()
+			return fmt.Errorf("map %q to resource: %v", gk, err)
+		}
+		resourceClient := tCtx.Dynamic().Resource(mapping.Resource)
+
+		if c.Namespace != "" {
+			if mapping.Scope.Name() != meta.RESTScopeNameNamespace {
+				return fmt.Errorf("namespace %q set for %q, but %q has scope %q", c.Namespace, c.TemplatePath, gk, mapping.Scope.Name())
+			}
+			_, err = resourceClient.Namespace(c.Namespace).Create(tCtx, obj, metav1.CreateOptions{})
+		} else {
+			if mapping.Scope.Name() != meta.RESTScopeNameRoot {
+				return fmt.Errorf("namespace not set for %q, but %q has scope %q", c.TemplatePath, gk, mapping.Scope.Name())
+			}
+			_, err = resourceClient.Create(tCtx, obj, metav1.CreateOptions{})
+		}
+		return err
+	}
+	// Retry, some errors (like CRD just created and type not ready for use yet) are temporary.
+	ctx, cancel := context.WithTimeout(tCtx, 20*time.Second)
+	defer cancel()
+	for {
+		err := create()
+		if err == nil {
+			return
+		}
+		select {
+		case <-ctx.Done():
+			tCtx.Fatalf("%s: timed out (%q) while creating %q, last error was: %v", c.TemplatePath, context.Cause(ctx), klog.KObj(obj), err)
+		case <-time.After(time.Second):
+		}
+	}
 }

test/integration/scheduler_perf/dra.go

@@ -24,7 +24,6 @@ import (
 
 	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
 	draapp "k8s.io/kubernetes/test/e2e/dra/test-driver/app"
@@ -115,28 +114,6 @@ func (op *createResourceClaimsOp) run(tCtx ktesting.TContext) {
 	}
 }
 
-// createResourceClassOpType customizes createOp for creating a ResourceClass.
-type createResourceClassOpType struct{}
-
-func (c createResourceClassOpType) Opcode() operationCode { return createResourceClassOpcode }
-func (c createResourceClassOpType) Name() string          { return "ResourceClass" }
-func (c createResourceClassOpType) Namespaced() bool      { return false }
-func (c createResourceClassOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClass, metav1.CreateOptions) (*resourcev1alpha2.ResourceClass, error) {
-	return client.ResourceV1alpha2().ResourceClasses().Create
-}
-
-// createResourceClassOpType customizes createOp for creating a ResourceClaim.
-type createResourceClaimTemplateOpType struct{}
-
-func (c createResourceClaimTemplateOpType) Opcode() operationCode {
-	return createResourceClaimTemplateOpcode
-}
-func (c createResourceClaimTemplateOpType) Name() string     { return "ResourceClaimTemplate" }
-func (c createResourceClaimTemplateOpType) Namespaced() bool { return true }
-func (c createResourceClaimTemplateOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClaimTemplate, metav1.CreateOptions) (*resourcev1alpha2.ResourceClaimTemplate, error) {
-	return client.ResourceV1alpha2().ResourceClaimTemplates(namespace).Create
-}
-
 // createResourceDriverOp defines an op where resource claims are created.
 type createResourceDriverOp struct {
 	// Must be createResourceDriverOpcode.

test/integration/scheduler_perf/scheduler_perf.go

@@ -33,7 +33,6 @@ import (
 	"github.com/google/go-cmp/cmp"
 
 	v1 "k8s.io/api/core/v1"
-	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -66,17 +65,16 @@ import (
 type operationCode string
 
 const (
-	createNodesOpcode                 operationCode = "createNodes"
+	createAnyOpcode            operationCode = "createAny"
-	createNamespacesOpcode            operationCode = "createNamespaces"
+	createNodesOpcode          operationCode = "createNodes"
-	createPodsOpcode                  operationCode = "createPods"
+	createNamespacesOpcode     operationCode = "createNamespaces"
-	createPodSetsOpcode               operationCode = "createPodSets"
+	createPodsOpcode           operationCode = "createPods"
-	createResourceClaimsOpcode        operationCode = "createResourceClaims"
+	createPodSetsOpcode        operationCode = "createPodSets"
-	createResourceClaimTemplateOpcode operationCode = "createResourceClaimTemplate"
+	createResourceClaimsOpcode operationCode = "createResourceClaims"
-	createResourceClassOpcode         operationCode = "createResourceClass"
+	createResourceDriverOpcode operationCode = "createResourceDriver"
-	createResourceDriverOpcode        operationCode = "createResourceDriver"
+	churnOpcode                operationCode = "churn"
-	churnOpcode                       operationCode = "churn"
+	barrierOpcode              operationCode = "barrier"
-	barrierOpcode                     operationCode = "barrier"
+	sleepOpcode                operationCode = "sleep"
-	sleepOpcode                       operationCode = "sleep"
 )
 
 const (
@@ -239,13 +237,12 @@ type op struct {
 // which op we're decoding at runtime.
 func (op *op) UnmarshalJSON(b []byte) error {
 	possibleOps := []realOp{
+		&createAny{},
 		&createNodesOp{},
 		&createNamespacesOp{},
 		&createPodsOp{},
 		&createPodSetsOp{},
 		&createResourceClaimsOp{},
-		&createOp[resourcev1alpha2.ResourceClaimTemplate, createResourceClaimTemplateOpType]{},
-		&createOp[resourcev1alpha2.ResourceClass, createResourceClassOpType]{},
 		&createResourceDriverOp{},
 		&churnOp{},
 		&barrierOp{},