scheduler_perf: use dynamic client to create arbitrary objects

With a dynamic client and a rest mapper it is possible to load arbitrary YAML
files and create the object defined by it. This is simpler than adding specific
Go code for each supported type.

Because the version now matters, the incorrect version in the DRA YAMLs were
found and fixed.

test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml

@@ -1,4 +1,4 @@
-apiVersion: resource.k8s.io/v1alpha1
+apiVersion: resource.k8s.io/v1alpha2
 kind: ResourceClaimTemplate
 metadata:
   name: test-claim-template

test/integration/scheduler_perf/config/dra/resourceclass.yaml

@@ -1,4 +1,4 @@
-apiVersion: resource.k8s.io/v1alpha1
+apiVersion: resource.k8s.io/v1alpha2
 kind: ResourceClass
 metadata:
   name: test-class

test/integration/scheduler_perf/config/performance-config.yaml

@@ -729,16 +729,16 @@
     driverName: test-driver.cdi.k8s.io
     nodes: scheduler-perf-dra-*
     maxClaimsPerNodeParam: $maxClaimsPerNode
-  - opcode: createResourceClass
+  - opcode: createAny
     templatePath: config/dra/resourceclass.yaml
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/resourceclaimtemplate.yaml
     namespace: init
   - opcode: createPods
     namespace: init
     countParam: $initPods
     podTemplatePath: config/dra/pod-with-claim-template.yaml
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/resourceclaimtemplate.yaml
     namespace: test
   - opcode: createPods
@@ -799,24 +799,24 @@
     driverName: another-test-driver.cdi.k8s.io
     nodes: scheduler-perf-dra-*
     maxClaimsPerNodeParam: $maxClaimsPerNode
-  - opcode: createResourceClass
+  - opcode: createAny
     templatePath: config/dra/resourceclass.yaml
-  - opcode: createResourceClass
+  - opcode: createAny
     templatePath: config/dra/another-resourceclass.yaml
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/resourceclaimtemplate.yaml
     namespace: init
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/another-resourceclaimtemplate.yaml
     namespace: init
   - opcode: createPods
     namespace: init
     countParam: $initPods
     podTemplatePath: config/dra/pod-with-many-claim-templates.yaml
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/resourceclaimtemplate.yaml
     namespace: test
-  - opcode: createResourceClaimTemplate
+  - opcode: createAny
     templatePath: config/dra/another-resourceclaimtemplate.yaml
     namespace: test
   - opcode: createPods

test/integration/scheduler_perf/create.go

@@ -19,17 +19,22 @@ package benchmark
 import (
 	"context"
 	"fmt"
+	"time"
 
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery/cached/memory"
+	"k8s.io/client-go/restmapper"
+	"k8s.io/klog/v2"
 	"k8s.io/kubernetes/test/utils/ktesting"
 )
 
-// createOp defines an op where some object gets created from a template.
-// Everything specific for that object (create call, op code, names) gets
-// provided through a type.
-type createOp[T interface{}, P createOpType[T]] struct {
-	// Must match createOpType.Opcode().
+// createAny defines an op where some object gets created from a YAML file.
+// The nameset can be specified.
+type createAny struct {
+	// Must match createAnyOpcode.
 	Opcode operationCode
 	// Namespace the object should be created in. Must be empty for cluster-scoped objects.
 	Namespace string
@@ -37,53 +42,85 @@ type createOp[T interface{}, P createOpType[T]] struct {
 	TemplatePath string
 }
 
-func (cro *createOp[T, P]) isValid(allowParameterization bool) error {
-	var p P
-	if cro.Opcode != p.Opcode() {
-		return fmt.Errorf("invalid opcode %q; expected %q", cro.Opcode, p.Opcode())
+var _ runnableOp = &createAny{}
+
+func (c *createAny) isValid(allowParameterization bool) error {
+	if c.Opcode != createAnyOpcode {
+		return fmt.Errorf("invalid opcode %q; expected %q", c.Opcode, createAnyOpcode)
 	}
-	if p.Namespaced() && cro.Namespace == "" {
-		return fmt.Errorf("Namespace must be set")
-	}
-	if !p.Namespaced() && cro.Namespace != "" {
-		return fmt.Errorf("Namespace must not be set")
-	}
-	if cro.TemplatePath == "" {
+	if c.TemplatePath == "" {
 		return fmt.Errorf("TemplatePath must be set")
 	}
+	// The namespace can only be checked during later because we don't know yet
+	// whether the object is namespaced or cluster-scoped.
 	return nil
 }
 
-func (cro *createOp[T, P]) collectsMetrics() bool {
+func (c *createAny) collectsMetrics() bool {
 	return false
 }
 
-func (cro *createOp[T, P]) patchParams(w *workload) (realOp, error) {
-	return cro, cro.isValid(false)
+func (c *createAny) patchParams(w *workload) (realOp, error) {
+	return c, c.isValid(false)
 }
 
-func (cro *createOp[T, P]) requiredNamespaces() []string {
-	if cro.Namespace == "" {
+func (c *createAny) requiredNamespaces() []string {
+	if c.Namespace == "" {
 		return nil
 	}
-	return []string{cro.Namespace}
+	return []string{c.Namespace}
 }
 
-func (cro *createOp[T, P]) run(tCtx ktesting.TContext) {
-	var obj *T
-	var p P
-	if err := getSpecFromFile(&cro.TemplatePath, &obj); err != nil {
-		tCtx.Fatalf("parsing %s %q: %v", p.Name(), cro.TemplatePath, err)
+func (c *createAny) run(tCtx ktesting.TContext) {
+	var obj *unstructured.Unstructured
+	if err := getSpecFromFile(&c.TemplatePath, &obj); err != nil {
+		tCtx.Fatalf("%s: parsing failed: %v", c.TemplatePath, err)
 	}
-	if _, err := p.CreateCall(tCtx.Client(), cro.Namespace)(tCtx, obj, metav1.CreateOptions{}); err != nil {
-		tCtx.Fatalf("create %s: %v", p.Name(), err)
-	}
-}
 
-// createOpType provides type-specific values for the generic createOp.
-type createOpType[T interface{}] interface {
-	Opcode() operationCode
-	Name() string
-	Namespaced() bool
-	CreateCall(client clientset.Interface, namespace string) func(context.Context, *T, metav1.CreateOptions) (*T, error)
+	// Not caching the discovery result isn't very efficient, but good enough when
+	// createAny isn't done often.
+	discoveryCache := memory.NewMemCacheClient(tCtx.Client().Discovery())
+	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryCache)
+	gv, err := schema.ParseGroupVersion(obj.GetAPIVersion())
+	if err != nil {
+		tCtx.Fatalf("%s: extract group+version from object %q: %v", c.TemplatePath, klog.KObj(obj), err)
+	}
+	gk := schema.GroupKind{Group: gv.Group, Kind: obj.GetKind()}
+
+	create := func() error {
+		mapping, err := restMapper.RESTMapping(gk, gv.Version)
+		if err != nil {
+			// Cached mapping might be stale, refresh on next try.
+			restMapper.Reset()
+			return fmt.Errorf("map %q to resource: %v", gk, err)
+		}
+		resourceClient := tCtx.Dynamic().Resource(mapping.Resource)
+
+		if c.Namespace != "" {
+			if mapping.Scope.Name() != meta.RESTScopeNameNamespace {
+				return fmt.Errorf("namespace %q set for %q, but %q has scope %q", c.Namespace, c.TemplatePath, gk, mapping.Scope.Name())
+			}
+			_, err = resourceClient.Namespace(c.Namespace).Create(tCtx, obj, metav1.CreateOptions{})
+		} else {
+			if mapping.Scope.Name() != meta.RESTScopeNameRoot {
+				return fmt.Errorf("namespace not set for %q, but %q has scope %q", c.TemplatePath, gk, mapping.Scope.Name())
+			}
+			_, err = resourceClient.Create(tCtx, obj, metav1.CreateOptions{})
+		}
+		return err
+	}
+	// Retry, some errors (like CRD just created and type not ready for use yet) are temporary.
+	ctx, cancel := context.WithTimeout(tCtx, 20*time.Second)
+	defer cancel()
+	for {
+		err := create()
+		if err == nil {
+			return
+		}
+		select {
+		case <-ctx.Done():
+			tCtx.Fatalf("%s: timed out (%q) while creating %q, last error was: %v", c.TemplatePath, context.Cause(ctx), klog.KObj(obj), err)
+		case <-time.After(time.Second):
+		}
+	}
 }

test/integration/scheduler_perf/dra.go

@@ -24,7 +24,6 @@ import (
 
 	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
 	draapp "k8s.io/kubernetes/test/e2e/dra/test-driver/app"
@@ -115,28 +114,6 @@ func (op *createResourceClaimsOp) run(tCtx ktesting.TContext) {
 	}
 }
 
-// createResourceClassOpType customizes createOp for creating a ResourceClass.
-type createResourceClassOpType struct{}
-
-func (c createResourceClassOpType) Opcode() operationCode { return createResourceClassOpcode }
-func (c createResourceClassOpType) Name() string          { return "ResourceClass" }
-func (c createResourceClassOpType) Namespaced() bool      { return false }
-func (c createResourceClassOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClass, metav1.CreateOptions) (*resourcev1alpha2.ResourceClass, error) {
-	return client.ResourceV1alpha2().ResourceClasses().Create
-}
-
-// createResourceClassOpType customizes createOp for creating a ResourceClaim.
-type createResourceClaimTemplateOpType struct{}
-
-func (c createResourceClaimTemplateOpType) Opcode() operationCode {
-	return createResourceClaimTemplateOpcode
-}
-func (c createResourceClaimTemplateOpType) Name() string     { return "ResourceClaimTemplate" }
-func (c createResourceClaimTemplateOpType) Namespaced() bool { return true }
-func (c createResourceClaimTemplateOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClaimTemplate, metav1.CreateOptions) (*resourcev1alpha2.ResourceClaimTemplate, error) {
-	return client.ResourceV1alpha2().ResourceClaimTemplates(namespace).Create
-}
-
 // createResourceDriverOp defines an op where resource claims are created.
 type createResourceDriverOp struct {
 	// Must be createResourceDriverOpcode.

test/integration/scheduler_perf/scheduler_perf.go

@@ -33,7 +33,6 @@ import (
 	"github.com/google/go-cmp/cmp"
 
 	v1 "k8s.io/api/core/v1"
-	resourcev1alpha2 "k8s.io/api/resource/v1alpha2"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -66,17 +65,16 @@ import (
 type operationCode string
 
 const (
-	createNodesOpcode                 operationCode = "createNodes"
-	createNamespacesOpcode            operationCode = "createNamespaces"
-	createPodsOpcode                  operationCode = "createPods"
-	createPodSetsOpcode               operationCode = "createPodSets"
-	createResourceClaimsOpcode        operationCode = "createResourceClaims"
-	createResourceClaimTemplateOpcode operationCode = "createResourceClaimTemplate"
-	createResourceClassOpcode         operationCode = "createResourceClass"
-	createResourceDriverOpcode        operationCode = "createResourceDriver"
-	churnOpcode                       operationCode = "churn"
-	barrierOpcode                     operationCode = "barrier"
-	sleepOpcode                       operationCode = "sleep"
+	createAnyOpcode            operationCode = "createAny"
+	createNodesOpcode          operationCode = "createNodes"
+	createNamespacesOpcode     operationCode = "createNamespaces"
+	createPodsOpcode           operationCode = "createPods"
+	createPodSetsOpcode        operationCode = "createPodSets"
+	createResourceClaimsOpcode operationCode = "createResourceClaims"
+	createResourceDriverOpcode operationCode = "createResourceDriver"
+	churnOpcode                operationCode = "churn"
+	barrierOpcode              operationCode = "barrier"
+	sleepOpcode                operationCode = "sleep"
 )
 
 const (
@@ -239,13 +237,12 @@ type op struct {
 // which op we're decoding at runtime.
 func (op *op) UnmarshalJSON(b []byte) error {
 	possibleOps := []realOp{
+		&createAny{},
 		&createNodesOp{},
 		&createNamespacesOp{},
 		&createPodsOp{},
 		&createPodSetsOp{},
 		&createResourceClaimsOp{},
-		&createOp[resourcev1alpha2.ResourceClaimTemplate, createResourceClaimTemplateOpType]{},
-		&createOp[resourcev1alpha2.ResourceClass, createResourceClassOpType]{},
 		&createResourceDriverOp{},
 		&churnOp{},
 		&barrierOp{},