From da0c9a93aea3415797d29335ee9cd4c406664f24 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 20 Dec 2023 11:23:09 +0100 Subject: [PATCH] scheduler_perf: use dynamic client to create arbitrary objects With a dynamic client and a rest mapper it is possible to load arbitrary YAML files and create the object defined by it. This is simpler than adding specific Go code for each supported type. Because the version now matters, the incorrect version in the DRA YAMLs were found and fixed. --- .../config/dra/resourceclaimtemplate.yaml | 2 +- .../config/dra/resourceclass.yaml | 2 +- .../config/performance-config.yaml | 18 +-- test/integration/scheduler_perf/create.go | 113 ++++++++++++------ test/integration/scheduler_perf/dra.go | 23 ---- .../scheduler_perf/scheduler_perf.go | 25 ++-- 6 files changed, 97 insertions(+), 86 deletions(-) diff --git a/test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml b/test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml index d3da6879e76..29048726caf 100644 --- a/test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml +++ b/test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml @@ -1,4 +1,4 @@ -apiVersion: resource.k8s.io/v1alpha1 +apiVersion: resource.k8s.io/v1alpha2 kind: ResourceClaimTemplate metadata: name: test-claim-template diff --git a/test/integration/scheduler_perf/config/dra/resourceclass.yaml b/test/integration/scheduler_perf/config/dra/resourceclass.yaml index d24aa82392f..8ba0fcb3a2a 100644 --- a/test/integration/scheduler_perf/config/dra/resourceclass.yaml +++ b/test/integration/scheduler_perf/config/dra/resourceclass.yaml @@ -1,4 +1,4 @@ -apiVersion: resource.k8s.io/v1alpha1 +apiVersion: resource.k8s.io/v1alpha2 kind: ResourceClass metadata: name: test-class diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml index 8e4d735baa1..c3f96fb58d2 100644 --- a/test/integration/scheduler_perf/config/performance-config.yaml +++ b/test/integration/scheduler_perf/config/performance-config.yaml @@ -729,16 +729,16 @@ driverName: test-driver.cdi.k8s.io nodes: scheduler-perf-dra-* maxClaimsPerNodeParam: $maxClaimsPerNode - - opcode: createResourceClass + - opcode: createAny templatePath: config/dra/resourceclass.yaml - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/resourceclaimtemplate.yaml namespace: init - opcode: createPods namespace: init countParam: $initPods podTemplatePath: config/dra/pod-with-claim-template.yaml - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/resourceclaimtemplate.yaml namespace: test - opcode: createPods @@ -799,24 +799,24 @@ driverName: another-test-driver.cdi.k8s.io nodes: scheduler-perf-dra-* maxClaimsPerNodeParam: $maxClaimsPerNode - - opcode: createResourceClass + - opcode: createAny templatePath: config/dra/resourceclass.yaml - - opcode: createResourceClass + - opcode: createAny templatePath: config/dra/another-resourceclass.yaml - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/resourceclaimtemplate.yaml namespace: init - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/another-resourceclaimtemplate.yaml namespace: init - opcode: createPods namespace: init countParam: $initPods podTemplatePath: config/dra/pod-with-many-claim-templates.yaml - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/resourceclaimtemplate.yaml namespace: test - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/another-resourceclaimtemplate.yaml namespace: test - opcode: createPods diff --git a/test/integration/scheduler_perf/create.go b/test/integration/scheduler_perf/create.go index b80adc18d78..ddc0e350e75 100644 --- a/test/integration/scheduler_perf/create.go +++ b/test/integration/scheduler_perf/create.go @@ -19,17 +19,22 @@ package benchmark import ( "context" "fmt" + "time" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/restmapper" + "k8s.io/klog/v2" "k8s.io/kubernetes/test/utils/ktesting" ) -// createOp defines an op where some object gets created from a template. -// Everything specific for that object (create call, op code, names) gets -// provided through a type. -type createOp[T interface{}, P createOpType[T]] struct { - // Must match createOpType.Opcode(). +// createAny defines an op where some object gets created from a YAML file. +// The nameset can be specified. +type createAny struct { + // Must match createAnyOpcode. Opcode operationCode // Namespace the object should be created in. Must be empty for cluster-scoped objects. Namespace string @@ -37,53 +42,85 @@ type createOp[T interface{}, P createOpType[T]] struct { TemplatePath string } -func (cro *createOp[T, P]) isValid(allowParameterization bool) error { - var p P - if cro.Opcode != p.Opcode() { - return fmt.Errorf("invalid opcode %q; expected %q", cro.Opcode, p.Opcode()) +var _ runnableOp = &createAny{} + +func (c *createAny) isValid(allowParameterization bool) error { + if c.Opcode != createAnyOpcode { + return fmt.Errorf("invalid opcode %q; expected %q", c.Opcode, createAnyOpcode) } - if p.Namespaced() && cro.Namespace == "" { - return fmt.Errorf("Namespace must be set") - } - if !p.Namespaced() && cro.Namespace != "" { - return fmt.Errorf("Namespace must not be set") - } - if cro.TemplatePath == "" { + if c.TemplatePath == "" { return fmt.Errorf("TemplatePath must be set") } + // The namespace can only be checked during later because we don't know yet + // whether the object is namespaced or cluster-scoped. return nil } -func (cro *createOp[T, P]) collectsMetrics() bool { +func (c *createAny) collectsMetrics() bool { return false } -func (cro *createOp[T, P]) patchParams(w *workload) (realOp, error) { - return cro, cro.isValid(false) +func (c *createAny) patchParams(w *workload) (realOp, error) { + return c, c.isValid(false) } -func (cro *createOp[T, P]) requiredNamespaces() []string { - if cro.Namespace == "" { +func (c *createAny) requiredNamespaces() []string { + if c.Namespace == "" { return nil } - return []string{cro.Namespace} + return []string{c.Namespace} } -func (cro *createOp[T, P]) run(tCtx ktesting.TContext) { - var obj *T - var p P - if err := getSpecFromFile(&cro.TemplatePath, &obj); err != nil { - tCtx.Fatalf("parsing %s %q: %v", p.Name(), cro.TemplatePath, err) +func (c *createAny) run(tCtx ktesting.TContext) { + var obj *unstructured.Unstructured + if err := getSpecFromFile(&c.TemplatePath, &obj); err != nil { + tCtx.Fatalf("%s: parsing failed: %v", c.TemplatePath, err) } - if _, err := p.CreateCall(tCtx.Client(), cro.Namespace)(tCtx, obj, metav1.CreateOptions{}); err != nil { - tCtx.Fatalf("create %s: %v", p.Name(), err) - } -} -// createOpType provides type-specific values for the generic createOp. -type createOpType[T interface{}] interface { - Opcode() operationCode - Name() string - Namespaced() bool - CreateCall(client clientset.Interface, namespace string) func(context.Context, *T, metav1.CreateOptions) (*T, error) + // Not caching the discovery result isn't very efficient, but good enough when + // createAny isn't done often. + discoveryCache := memory.NewMemCacheClient(tCtx.Client().Discovery()) + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryCache) + gv, err := schema.ParseGroupVersion(obj.GetAPIVersion()) + if err != nil { + tCtx.Fatalf("%s: extract group+version from object %q: %v", c.TemplatePath, klog.KObj(obj), err) + } + gk := schema.GroupKind{Group: gv.Group, Kind: obj.GetKind()} + + create := func() error { + mapping, err := restMapper.RESTMapping(gk, gv.Version) + if err != nil { + // Cached mapping might be stale, refresh on next try. + restMapper.Reset() + return fmt.Errorf("map %q to resource: %v", gk, err) + } + resourceClient := tCtx.Dynamic().Resource(mapping.Resource) + + if c.Namespace != "" { + if mapping.Scope.Name() != meta.RESTScopeNameNamespace { + return fmt.Errorf("namespace %q set for %q, but %q has scope %q", c.Namespace, c.TemplatePath, gk, mapping.Scope.Name()) + } + _, err = resourceClient.Namespace(c.Namespace).Create(tCtx, obj, metav1.CreateOptions{}) + } else { + if mapping.Scope.Name() != meta.RESTScopeNameRoot { + return fmt.Errorf("namespace not set for %q, but %q has scope %q", c.TemplatePath, gk, mapping.Scope.Name()) + } + _, err = resourceClient.Create(tCtx, obj, metav1.CreateOptions{}) + } + return err + } + // Retry, some errors (like CRD just created and type not ready for use yet) are temporary. + ctx, cancel := context.WithTimeout(tCtx, 20*time.Second) + defer cancel() + for { + err := create() + if err == nil { + return + } + select { + case <-ctx.Done(): + tCtx.Fatalf("%s: timed out (%q) while creating %q, last error was: %v", c.TemplatePath, context.Cause(ctx), klog.KObj(obj), err) + case <-time.After(time.Second): + } + } } diff --git a/test/integration/scheduler_perf/dra.go b/test/integration/scheduler_perf/dra.go index f8d96aedd85..3e263ace1f7 100644 --- a/test/integration/scheduler_perf/dra.go +++ b/test/integration/scheduler_perf/dra.go @@ -24,7 +24,6 @@ import ( resourcev1alpha2 "k8s.io/api/resource/v1alpha2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" draapp "k8s.io/kubernetes/test/e2e/dra/test-driver/app" @@ -115,28 +114,6 @@ func (op *createResourceClaimsOp) run(tCtx ktesting.TContext) { } } -// createResourceClassOpType customizes createOp for creating a ResourceClass. -type createResourceClassOpType struct{} - -func (c createResourceClassOpType) Opcode() operationCode { return createResourceClassOpcode } -func (c createResourceClassOpType) Name() string { return "ResourceClass" } -func (c createResourceClassOpType) Namespaced() bool { return false } -func (c createResourceClassOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClass, metav1.CreateOptions) (*resourcev1alpha2.ResourceClass, error) { - return client.ResourceV1alpha2().ResourceClasses().Create -} - -// createResourceClassOpType customizes createOp for creating a ResourceClaim. -type createResourceClaimTemplateOpType struct{} - -func (c createResourceClaimTemplateOpType) Opcode() operationCode { - return createResourceClaimTemplateOpcode -} -func (c createResourceClaimTemplateOpType) Name() string { return "ResourceClaimTemplate" } -func (c createResourceClaimTemplateOpType) Namespaced() bool { return true } -func (c createResourceClaimTemplateOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClaimTemplate, metav1.CreateOptions) (*resourcev1alpha2.ResourceClaimTemplate, error) { - return client.ResourceV1alpha2().ResourceClaimTemplates(namespace).Create -} - // createResourceDriverOp defines an op where resource claims are created. type createResourceDriverOp struct { // Must be createResourceDriverOpcode. diff --git a/test/integration/scheduler_perf/scheduler_perf.go b/test/integration/scheduler_perf/scheduler_perf.go index 37eeb5c0a62..366756c502d 100644 --- a/test/integration/scheduler_perf/scheduler_perf.go +++ b/test/integration/scheduler_perf/scheduler_perf.go @@ -33,7 +33,6 @@ import ( "github.com/google/go-cmp/cmp" v1 "k8s.io/api/core/v1" - resourcev1alpha2 "k8s.io/api/resource/v1alpha2" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -66,17 +65,16 @@ import ( type operationCode string const ( - createNodesOpcode operationCode = "createNodes" - createNamespacesOpcode operationCode = "createNamespaces" - createPodsOpcode operationCode = "createPods" - createPodSetsOpcode operationCode = "createPodSets" - createResourceClaimsOpcode operationCode = "createResourceClaims" - createResourceClaimTemplateOpcode operationCode = "createResourceClaimTemplate" - createResourceClassOpcode operationCode = "createResourceClass" - createResourceDriverOpcode operationCode = "createResourceDriver" - churnOpcode operationCode = "churn" - barrierOpcode operationCode = "barrier" - sleepOpcode operationCode = "sleep" + createAnyOpcode operationCode = "createAny" + createNodesOpcode operationCode = "createNodes" + createNamespacesOpcode operationCode = "createNamespaces" + createPodsOpcode operationCode = "createPods" + createPodSetsOpcode operationCode = "createPodSets" + createResourceClaimsOpcode operationCode = "createResourceClaims" + createResourceDriverOpcode operationCode = "createResourceDriver" + churnOpcode operationCode = "churn" + barrierOpcode operationCode = "barrier" + sleepOpcode operationCode = "sleep" ) const ( @@ -239,13 +237,12 @@ type op struct { // which op we're decoding at runtime. func (op *op) UnmarshalJSON(b []byte) error { possibleOps := []realOp{ + &createAny{}, &createNodesOp{}, &createNamespacesOp{}, &createPodsOp{}, &createPodSetsOp{}, &createResourceClaimsOp{}, - &createOp[resourcev1alpha2.ResourceClaimTemplate, createResourceClaimTemplateOpType]{}, - &createOp[resourcev1alpha2.ResourceClass, createResourceClassOpType]{}, &createResourceDriverOp{}, &churnOp{}, &barrierOp{},