diff --git a/test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml b/test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml index d3da6879e76..29048726caf 100644 --- a/test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml +++ b/test/integration/scheduler_perf/config/dra/resourceclaimtemplate.yaml @@ -1,4 +1,4 @@ -apiVersion: resource.k8s.io/v1alpha1 +apiVersion: resource.k8s.io/v1alpha2 kind: ResourceClaimTemplate metadata: name: test-claim-template diff --git a/test/integration/scheduler_perf/config/dra/resourceclass.yaml b/test/integration/scheduler_perf/config/dra/resourceclass.yaml index d24aa82392f..8ba0fcb3a2a 100644 --- a/test/integration/scheduler_perf/config/dra/resourceclass.yaml +++ b/test/integration/scheduler_perf/config/dra/resourceclass.yaml @@ -1,4 +1,4 @@ -apiVersion: resource.k8s.io/v1alpha1 +apiVersion: resource.k8s.io/v1alpha2 kind: ResourceClass metadata: name: test-class diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml index 8e4d735baa1..c3f96fb58d2 100644 --- a/test/integration/scheduler_perf/config/performance-config.yaml +++ b/test/integration/scheduler_perf/config/performance-config.yaml @@ -729,16 +729,16 @@ driverName: test-driver.cdi.k8s.io nodes: scheduler-perf-dra-* maxClaimsPerNodeParam: $maxClaimsPerNode - - opcode: createResourceClass + - opcode: createAny templatePath: config/dra/resourceclass.yaml - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/resourceclaimtemplate.yaml namespace: init - opcode: createPods namespace: init countParam: $initPods podTemplatePath: config/dra/pod-with-claim-template.yaml - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/resourceclaimtemplate.yaml namespace: test - opcode: createPods @@ -799,24 +799,24 @@ driverName: another-test-driver.cdi.k8s.io nodes: scheduler-perf-dra-* maxClaimsPerNodeParam: $maxClaimsPerNode - - opcode: createResourceClass + - opcode: createAny templatePath: config/dra/resourceclass.yaml - - opcode: createResourceClass + - opcode: createAny templatePath: config/dra/another-resourceclass.yaml - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/resourceclaimtemplate.yaml namespace: init - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/another-resourceclaimtemplate.yaml namespace: init - opcode: createPods namespace: init countParam: $initPods podTemplatePath: config/dra/pod-with-many-claim-templates.yaml - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/resourceclaimtemplate.yaml namespace: test - - opcode: createResourceClaimTemplate + - opcode: createAny templatePath: config/dra/another-resourceclaimtemplate.yaml namespace: test - opcode: createPods diff --git a/test/integration/scheduler_perf/create.go b/test/integration/scheduler_perf/create.go index b80adc18d78..ddc0e350e75 100644 --- a/test/integration/scheduler_perf/create.go +++ b/test/integration/scheduler_perf/create.go @@ -19,17 +19,22 @@ package benchmark import ( "context" "fmt" + "time" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/restmapper" + "k8s.io/klog/v2" "k8s.io/kubernetes/test/utils/ktesting" ) -// createOp defines an op where some object gets created from a template. -// Everything specific for that object (create call, op code, names) gets -// provided through a type. -type createOp[T interface{}, P createOpType[T]] struct { - // Must match createOpType.Opcode(). +// createAny defines an op where some object gets created from a YAML file. +// The nameset can be specified. +type createAny struct { + // Must match createAnyOpcode. Opcode operationCode // Namespace the object should be created in. Must be empty for cluster-scoped objects. Namespace string @@ -37,53 +42,85 @@ type createOp[T interface{}, P createOpType[T]] struct { TemplatePath string } -func (cro *createOp[T, P]) isValid(allowParameterization bool) error { - var p P - if cro.Opcode != p.Opcode() { - return fmt.Errorf("invalid opcode %q; expected %q", cro.Opcode, p.Opcode()) +var _ runnableOp = &createAny{} + +func (c *createAny) isValid(allowParameterization bool) error { + if c.Opcode != createAnyOpcode { + return fmt.Errorf("invalid opcode %q; expected %q", c.Opcode, createAnyOpcode) } - if p.Namespaced() && cro.Namespace == "" { - return fmt.Errorf("Namespace must be set") - } - if !p.Namespaced() && cro.Namespace != "" { - return fmt.Errorf("Namespace must not be set") - } - if cro.TemplatePath == "" { + if c.TemplatePath == "" { return fmt.Errorf("TemplatePath must be set") } + // The namespace can only be checked during later because we don't know yet + // whether the object is namespaced or cluster-scoped. return nil } -func (cro *createOp[T, P]) collectsMetrics() bool { +func (c *createAny) collectsMetrics() bool { return false } -func (cro *createOp[T, P]) patchParams(w *workload) (realOp, error) { - return cro, cro.isValid(false) +func (c *createAny) patchParams(w *workload) (realOp, error) { + return c, c.isValid(false) } -func (cro *createOp[T, P]) requiredNamespaces() []string { - if cro.Namespace == "" { +func (c *createAny) requiredNamespaces() []string { + if c.Namespace == "" { return nil } - return []string{cro.Namespace} + return []string{c.Namespace} } -func (cro *createOp[T, P]) run(tCtx ktesting.TContext) { - var obj *T - var p P - if err := getSpecFromFile(&cro.TemplatePath, &obj); err != nil { - tCtx.Fatalf("parsing %s %q: %v", p.Name(), cro.TemplatePath, err) +func (c *createAny) run(tCtx ktesting.TContext) { + var obj *unstructured.Unstructured + if err := getSpecFromFile(&c.TemplatePath, &obj); err != nil { + tCtx.Fatalf("%s: parsing failed: %v", c.TemplatePath, err) } - if _, err := p.CreateCall(tCtx.Client(), cro.Namespace)(tCtx, obj, metav1.CreateOptions{}); err != nil { - tCtx.Fatalf("create %s: %v", p.Name(), err) - } -} -// createOpType provides type-specific values for the generic createOp. -type createOpType[T interface{}] interface { - Opcode() operationCode - Name() string - Namespaced() bool - CreateCall(client clientset.Interface, namespace string) func(context.Context, *T, metav1.CreateOptions) (*T, error) + // Not caching the discovery result isn't very efficient, but good enough when + // createAny isn't done often. + discoveryCache := memory.NewMemCacheClient(tCtx.Client().Discovery()) + restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryCache) + gv, err := schema.ParseGroupVersion(obj.GetAPIVersion()) + if err != nil { + tCtx.Fatalf("%s: extract group+version from object %q: %v", c.TemplatePath, klog.KObj(obj), err) + } + gk := schema.GroupKind{Group: gv.Group, Kind: obj.GetKind()} + + create := func() error { + mapping, err := restMapper.RESTMapping(gk, gv.Version) + if err != nil { + // Cached mapping might be stale, refresh on next try. + restMapper.Reset() + return fmt.Errorf("map %q to resource: %v", gk, err) + } + resourceClient := tCtx.Dynamic().Resource(mapping.Resource) + + if c.Namespace != "" { + if mapping.Scope.Name() != meta.RESTScopeNameNamespace { + return fmt.Errorf("namespace %q set for %q, but %q has scope %q", c.Namespace, c.TemplatePath, gk, mapping.Scope.Name()) + } + _, err = resourceClient.Namespace(c.Namespace).Create(tCtx, obj, metav1.CreateOptions{}) + } else { + if mapping.Scope.Name() != meta.RESTScopeNameRoot { + return fmt.Errorf("namespace not set for %q, but %q has scope %q", c.TemplatePath, gk, mapping.Scope.Name()) + } + _, err = resourceClient.Create(tCtx, obj, metav1.CreateOptions{}) + } + return err + } + // Retry, some errors (like CRD just created and type not ready for use yet) are temporary. + ctx, cancel := context.WithTimeout(tCtx, 20*time.Second) + defer cancel() + for { + err := create() + if err == nil { + return + } + select { + case <-ctx.Done(): + tCtx.Fatalf("%s: timed out (%q) while creating %q, last error was: %v", c.TemplatePath, context.Cause(ctx), klog.KObj(obj), err) + case <-time.After(time.Second): + } + } } diff --git a/test/integration/scheduler_perf/dra.go b/test/integration/scheduler_perf/dra.go index f8d96aedd85..3e263ace1f7 100644 --- a/test/integration/scheduler_perf/dra.go +++ b/test/integration/scheduler_perf/dra.go @@ -24,7 +24,6 @@ import ( resourcev1alpha2 "k8s.io/api/resource/v1alpha2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" draapp "k8s.io/kubernetes/test/e2e/dra/test-driver/app" @@ -115,28 +114,6 @@ func (op *createResourceClaimsOp) run(tCtx ktesting.TContext) { } } -// createResourceClassOpType customizes createOp for creating a ResourceClass. -type createResourceClassOpType struct{} - -func (c createResourceClassOpType) Opcode() operationCode { return createResourceClassOpcode } -func (c createResourceClassOpType) Name() string { return "ResourceClass" } -func (c createResourceClassOpType) Namespaced() bool { return false } -func (c createResourceClassOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClass, metav1.CreateOptions) (*resourcev1alpha2.ResourceClass, error) { - return client.ResourceV1alpha2().ResourceClasses().Create -} - -// createResourceClassOpType customizes createOp for creating a ResourceClaim. -type createResourceClaimTemplateOpType struct{} - -func (c createResourceClaimTemplateOpType) Opcode() operationCode { - return createResourceClaimTemplateOpcode -} -func (c createResourceClaimTemplateOpType) Name() string { return "ResourceClaimTemplate" } -func (c createResourceClaimTemplateOpType) Namespaced() bool { return true } -func (c createResourceClaimTemplateOpType) CreateCall(client clientset.Interface, namespace string) func(context.Context, *resourcev1alpha2.ResourceClaimTemplate, metav1.CreateOptions) (*resourcev1alpha2.ResourceClaimTemplate, error) { - return client.ResourceV1alpha2().ResourceClaimTemplates(namespace).Create -} - // createResourceDriverOp defines an op where resource claims are created. type createResourceDriverOp struct { // Must be createResourceDriverOpcode. diff --git a/test/integration/scheduler_perf/scheduler_perf.go b/test/integration/scheduler_perf/scheduler_perf.go index 37eeb5c0a62..366756c502d 100644 --- a/test/integration/scheduler_perf/scheduler_perf.go +++ b/test/integration/scheduler_perf/scheduler_perf.go @@ -33,7 +33,6 @@ import ( "github.com/google/go-cmp/cmp" v1 "k8s.io/api/core/v1" - resourcev1alpha2 "k8s.io/api/resource/v1alpha2" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -66,17 +65,16 @@ import ( type operationCode string const ( - createNodesOpcode operationCode = "createNodes" - createNamespacesOpcode operationCode = "createNamespaces" - createPodsOpcode operationCode = "createPods" - createPodSetsOpcode operationCode = "createPodSets" - createResourceClaimsOpcode operationCode = "createResourceClaims" - createResourceClaimTemplateOpcode operationCode = "createResourceClaimTemplate" - createResourceClassOpcode operationCode = "createResourceClass" - createResourceDriverOpcode operationCode = "createResourceDriver" - churnOpcode operationCode = "churn" - barrierOpcode operationCode = "barrier" - sleepOpcode operationCode = "sleep" + createAnyOpcode operationCode = "createAny" + createNodesOpcode operationCode = "createNodes" + createNamespacesOpcode operationCode = "createNamespaces" + createPodsOpcode operationCode = "createPods" + createPodSetsOpcode operationCode = "createPodSets" + createResourceClaimsOpcode operationCode = "createResourceClaims" + createResourceDriverOpcode operationCode = "createResourceDriver" + churnOpcode operationCode = "churn" + barrierOpcode operationCode = "barrier" + sleepOpcode operationCode = "sleep" ) const ( @@ -239,13 +237,12 @@ type op struct { // which op we're decoding at runtime. func (op *op) UnmarshalJSON(b []byte) error { possibleOps := []realOp{ + &createAny{}, &createNodesOp{}, &createNamespacesOp{}, &createPodsOp{}, &createPodSetsOp{}, &createResourceClaimsOp{}, - &createOp[resourcev1alpha2.ResourceClaimTemplate, createResourceClaimTemplateOpType]{}, - &createOp[resourcev1alpha2.ResourceClass, createResourceClassOpType]{}, &createResourceDriverOp{}, &churnOp{}, &barrierOp{},