DRA: remove "classic DRA"

This removes the DRAControlPlaneController feature gate, the fields controlled
by it (claim.spec.controller, claim.status.deallocationRequested,
claim.status.allocation.controller, class.spec.suitableNodes), the
PodSchedulingContext type, and all code related to the feature.

The feature gets removed because there is no path towards beta and GA and DRA
with "structured parameters" should be able to replace it.
This commit is contained in:
Patrick Ohly
2024-10-11 16:08:54 +02:00
parent b7fb82f9d8
commit f84eb5ecf8
111 changed files with 1053 additions and 12850 deletions

View File

@@ -59,8 +59,7 @@ var resetFieldsStatusData = map[schema.GroupVersionResource]string{
gvr("storage.k8s.io", "v1", "volumeattachments"): `{"status": {"attached": false}}`,
gvr("policy", "v1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 25}}`,
gvr("policy", "v1beta1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 25}}`,
gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): `{"status": {"resourceClaims": [{"name": "my-claim", "unsuitableNodes": ["node2"]}]}}`, // Not really a conflict with status_test.go: Apply just stores both nodes. Conflict testing therefore gets disabled for podschedulingcontexts.
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"status": {"allocation": {"controller": "other.example.com"}}}`,
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"status": {"allocation": {"nodeSelector": {"nodeSelectorTerms": [{"matchExpressions": [{"key": "some-label", "operator": "In", "values": ["some-other-value"]}] }]}}}}`,
gvr("internal.apiserver.k8s.io", "v1alpha1", "storageversions"): `{"status": {"commonEncodingVersion":"v1","storageVersions":[{"apiServerID":"1","decodableVersions":["v1","v2"],"encodingVersion":"v1"}],"conditions":[{"type":"AllEncodingVersionsEqual","status":"False","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"allEncodingVersionsEqual","message":"all encoding versions are set to v1"}]}}`,
// standard for []metav1.Condition
gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies"): `{"status": {"conditions":[{"type":"Accepted","status":"True","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"RuleApplied","message":"Rule was applied"}]}}`,
@@ -90,10 +89,6 @@ var noConflicts = map[string]struct{}{
// namespaces only have a spec.finalizers field which is also skipped,
// thus it will never have a conflict.
"namespaces": {},
// podschedulingcontexts.status only has a list which contains items with a list,
// therefore apply works because it simply merges either the outer or
// the inner list.
"podschedulingcontexts": {},
}
var image2 = image.GetE2EImage(image.Etcd)
@@ -152,7 +147,6 @@ var resetFieldsSpecData = map[schema.GroupVersionResource]string{
gvr("awesome.bears.com", "v3", "pandas"): `{"spec": {"replicas": 302}}`,
gvr("apiregistration.k8s.io", "v1beta1", "apiservices"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"group": "foo2.com"}}`,
gvr("apiregistration.k8s.io", "v1", "apiservices"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"group": "foo2.com"}}`,
gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): `{"spec": {"selectedNode": "node2name"}}`,
gvr("resource.k8s.io", "v1alpha3", "deviceclasses"): `{"metadata": {"labels":{"a":"c"}}}`,
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"spec": {"devices": {"requests": [{"name": "req-0", "deviceClassName": "other-class"}]}}}`, // spec is immutable, but that doesn't matter for the test.
gvr("resource.k8s.io", "v1alpha3", "resourceclaimtemplates"): `{"spec": {"spec": {"resourceClassName": "class2name"}}}`,

View File

@@ -52,8 +52,7 @@ var statusData = map[schema.GroupVersionResource]string{
gvr("storage.k8s.io", "v1", "volumeattachments"): `{"status": {"attached": true}}`,
gvr("policy", "v1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 5}}`,
gvr("policy", "v1beta1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 5}}`,
gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): `{"status": {"resourceClaims": [{"name": "my-claim", "unsuitableNodes": ["node1"]}]}}`,
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"status": {"allocation": {"controller": "example.com"}}}`,
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"status": {"allocation": {"nodeSelector": {"nodeSelectorTerms": [{"matchExpressions": [{"key": "some-label", "operator": "In", "values": ["some-value"]}] }]}}}}`,
gvr("internal.apiserver.k8s.io", "v1alpha1", "storageversions"): `{"status": {"commonEncodingVersion":"v1","storageVersions":[{"apiServerID":"1","decodableVersions":["v1","v2"],"encodingVersion":"v1"}],"conditions":[{"type":"AllEncodingVersionsEqual","status":"True","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"allEncodingVersionsEqual","message":"all encoding versions are set to v1"}]}}`,
// standard for []metav1.Condition
gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies"): `{"status": {"conditions":[{"type":"Accepted","status":"False","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"RuleApplied","message":"Rule was applied"}]}}`,

View File

@@ -431,10 +431,6 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes
Stub: `{"metadata": {"name": "claimtemplate1name"}, "spec": {"spec": {"devices": {"requests": [{"name": "req-0", "deviceClassName": "example-class", "allocationMode": "ExactCount", "count": 1}]}}}}`,
ExpectedEtcdPath: "/registry/resourceclaimtemplates/" + namespace + "/claimtemplate1name",
},
gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): {
Stub: `{"metadata": {"name": "pod1name"}, "spec": {"selectedNode": "node1name", "potentialNodes": ["node1name", "node2name"]}}`,
ExpectedEtcdPath: "/registry/podschedulingcontexts/" + namespace + "/pod1name",
},
gvr("resource.k8s.io", "v1alpha3", "resourceslices"): {
Stub: `{"metadata": {"name": "node1slice"}, "spec": {"nodeName": "worker1", "driver": "dra.example.com", "pool": {"name": "worker1", "resourceSliceCount": 1}}}`,
ExpectedEtcdPath: "/registry/resourceslices/node1slice",

View File

@@ -21,34 +21,25 @@ package scheduler
import (
"context"
"fmt"
"net/http"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/google/go-cmp/cmp"
v1 "k8s.io/api/core/v1"
resourceapi "k8s.io/api/resource/v1alpha3"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
featuregatetesting "k8s.io/component-base/featuregate/testing"
configv1 "k8s.io/kube-scheduler/config/v1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/scheduler"
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
st "k8s.io/kubernetes/pkg/scheduler/testing"
testutils "k8s.io/kubernetes/test/integration/util"
"k8s.io/kubernetes/test/utils/format"
"k8s.io/utils/pointer"
)
@@ -647,117 +638,3 @@ func TestNodeEvents(t *testing.T) {
}
}
// TestPodSchedulingContextSSA checks that the dynamicresources plugin falls
// back to SSA successfully when the normal Update call encountered
// a conflict.
//
// This is an integration test because:
// - Unit testing does not cover RBAC rules.
// - Triggering this particular race is harder in E2E testing
// and harder to verify (needs apiserver metrics and there's
// no standard API for those).
func TestPodSchedulingContextSSA(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DynamicResourceAllocation, true)
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRAControlPlaneController, true)
testCtx := testutils.InitTestAPIServer(t, "podschedulingcontext-ssa", nil)
testCtx.DisableEventSink = true
testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, 0)
testutils.SyncSchedulerInformerFactory(testCtx)
go testCtx.Scheduler.Run(testCtx.SchedulerCtx)
// Set up enough objects that the scheduler will start trying to
// schedule the pod and create the PodSchedulingContext.
nodeRes := map[v1.ResourceName]string{
v1.ResourcePods: "32",
v1.ResourceCPU: "30m",
v1.ResourceMemory: "30",
}
for _, name := range []string{"node-a", "node-b"} {
if _, err := testutils.CreateNode(testCtx.ClientSet, st.MakeNode().Name(name).Capacity(nodeRes).Obj()); err != nil {
t.Fatalf("Failed to create node: %v", err)
}
}
claim := &resourceapi.ResourceClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "my-claim",
Namespace: testCtx.NS.Name,
},
Spec: resourceapi.ResourceClaimSpec{
Controller: "dra.example.com",
},
}
if _, err := testCtx.ClientSet.ResourceV1alpha3().ResourceClaims(claim.Namespace).Create(testCtx.Ctx, claim, metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create claim: %v", err)
}
podConf := testutils.PausePodConfig{
Name: "testpod",
Namespace: testCtx.NS.Name,
}
pod := testutils.InitPausePod(&podConf)
podClaimName := "myclaim"
pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: podClaimName}}
pod.Spec.ResourceClaims = []v1.PodResourceClaim{{Name: podClaimName, ResourceClaimName: &claim.Name}}
if _, err := testCtx.ClientSet.CoreV1().Pods(pod.Namespace).Create(testCtx.Ctx, pod, metav1.CreateOptions{}); err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
// Check that the PodSchedulingContext exists and has a selected node.
var schedulingCtx *resourceapi.PodSchedulingContext
if err := wait.PollUntilContextTimeout(testCtx.Ctx, 10*time.Microsecond, 30*time.Second, true,
func(context.Context) (bool, error) {
var err error
schedulingCtx, err = testCtx.ClientSet.ResourceV1alpha3().PodSchedulingContexts(pod.Namespace).Get(testCtx.Ctx, pod.Name, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return false, nil
}
if err == nil && schedulingCtx.Spec.SelectedNode != "" {
return true, nil
}
return false, err
}); err != nil {
t.Fatalf("Failed while waiting for PodSchedulingContext with selected node: %v\nLast PodSchedulingContext:\n%s", err, format.Object(schedulingCtx, 1))
}
// Force the plugin to use SSA.
var podSchedulingContextPatchCounter atomic.Int64
roundTrip := testutils.RoundTripWrapper(func(transport http.RoundTripper, req *http.Request) (*http.Response, error) {
if strings.HasPrefix(req.URL.Path, "/apis/resource.k8s.io/") &&
strings.HasSuffix(req.URL.Path, "/podschedulingcontexts/"+pod.Name) {
switch req.Method {
case http.MethodPut, http.MethodPost:
return &http.Response{
Status: fmt.Sprintf("%d %s", http.StatusConflict, metav1.StatusReasonConflict),
StatusCode: http.StatusConflict,
}, nil
case http.MethodPatch:
podSchedulingContextPatchCounter.Add(1)
}
}
return transport.RoundTrip(req)
})
testCtx.RoundTrip.Store(&roundTrip)
// Now force the scheduler to update the PodSchedulingContext by setting UnsuitableNodes so that
// the selected node is not suitable.
schedulingCtx.Status.ResourceClaims = []resourceapi.ResourceClaimSchedulingStatus{{
Name: podClaimName,
UnsuitableNodes: []string{schedulingCtx.Spec.SelectedNode},
}}
if _, err := testCtx.ClientSet.ResourceV1alpha3().PodSchedulingContexts(pod.Namespace).UpdateStatus(testCtx.Ctx, schedulingCtx, metav1.UpdateOptions{}); err != nil {
t.Fatalf("Unexpected PodSchedulingContext status update error: %v", err)
}
// We know that the scheduler has to use SSA because above we inject a conflict
// error whenever it tries to use a plain update. We just need to wait for it...
if err := wait.PollUntilContextTimeout(testCtx.Ctx, 10*time.Microsecond, time.Minute, true,
func(context.Context) (bool, error) {
return podSchedulingContextPatchCounter.Load() > 0, nil
}); err != nil {
t.Fatalf("Failed while waiting for PodSchedulingContext Patch: %v", err)
}
}

View File

@@ -1,8 +0,0 @@
apiVersion: resource.k8s.io/v1alpha3
kind: DeviceClass
metadata:
name: test-class
spec:
selectors:
- cel:
expression: device.driver == "test-driver.cdi.k8s.io"

View File

@@ -2,3 +2,7 @@ apiVersion: resource.k8s.io/v1alpha3
kind: DeviceClass
metadata:
name: test-class
spec:
selectors:
- cel:
expression: device.driver == "test-driver.cdi.k8s.io"

View File

@@ -1,9 +0,0 @@
apiVersion: resource.k8s.io/v1alpha3
kind: ResourceClaim
metadata:
name: test-claim-{{.Index}}
spec:
devices:
requests:
- name: req-0
deviceClassName: test-class

View File

@@ -3,7 +3,6 @@ kind: ResourceClaim
metadata:
name: test-claim-{{.Index}}
spec:
controller: test-driver.cdi.k8s.io
devices:
requests:
- name: req-0

View File

@@ -1,10 +0,0 @@
apiVersion: resource.k8s.io/v1alpha3
kind: ResourceClaimTemplate
metadata:
name: test-claim-template
spec:
spec:
devices:
requests:
- name: req-0
deviceClassName: test-class

View File

@@ -4,7 +4,6 @@ metadata:
name: test-claim-template
spec:
spec:
controller: test-driver.cdi.k8s.io
devices:
requests:
- name: req-0

View File

@@ -1025,153 +1025,9 @@
measurePods: 4000
# SchedulingWithResourceClaimTemplate uses a ResourceClaimTemplate
# and dynamically created ResourceClaim instances for each pod.
- name: SchedulingWithResourceClaimTemplate
featureGates:
DRAControlPlaneController: true
DynamicResourceAllocation: true
# SchedulerQueueingHints: true
workloadTemplate:
- opcode: createNodes
countParam: $nodesWithoutDRA
- opcode: createNodes
nodeTemplatePath: config/dra/node-with-dra-test-driver.yaml
countParam: $nodesWithDRA
- opcode: createResourceDriver
driverName: test-driver.cdi.k8s.io
nodes: scheduler-perf-dra-*
maxClaimsPerNodeParam: $maxClaimsPerNode
- opcode: createAny
templatePath: config/dra/deviceclass.yaml
- opcode: createAny
templatePath: config/dra/resourceclaimtemplate.yaml
namespace: init
- opcode: createPods
namespace: init
countParam: $initPods
podTemplatePath: config/dra/pod-with-claim-template.yaml
- opcode: createAny
templatePath: config/dra/resourceclaimtemplate.yaml
namespace: test
- opcode: createPods
namespace: test
countParam: $measurePods
podTemplatePath: config/dra/pod-with-claim-template.yaml
collectMetrics: true
workloads:
- name: fast
labels: [integration-test, performance, short]
params:
# This testcase runs through all code paths without
# taking too long overall.
nodesWithDRA: 1
nodesWithoutDRA: 1
initPods: 0
measurePods: 10
maxClaimsPerNode: 10
- name: 2000pods_100nodes
params:
# In this testcase, the number of nodes is smaller
# than the limit for the PodScheduling slices.
nodesWithDRA: 100
nodesWithoutDRA: 0
initPods: 1000
measurePods: 1000
maxClaimsPerNode: 20
- name: 2000pods_200nodes
params:
# In this testcase, the driver and scheduler must
# truncate the PotentialNodes and UnsuitableNodes
# slices.
nodesWithDRA: 200
nodesWithoutDRA: 0
initPods: 1000
measurePods: 1000
maxClaimsPerNode: 10
# This similar to SchedulingWithResourceClaimTemplate, except
# that it uses four claims per pod, from two different drivers.
# This emphasizes a bit more the complexity of collaborative
# scheduling via PodSchedulingContext.
- name: SchedulingWithMultipleResourceClaims
featureGates:
DRAControlPlaneController: true
DynamicResourceAllocation: true
# SchedulerQueueingHints: true
workloadTemplate:
- opcode: createNodes
countParam: $nodesWithoutDRA
- opcode: createNodes
nodeTemplatePath: config/dra/node-with-dra-test-driver.yaml
countParam: $nodesWithDRA
- opcode: createResourceDriver
driverName: test-driver.cdi.k8s.io
nodes: scheduler-perf-dra-*
maxClaimsPerNodeParam: $maxClaimsPerNode
- opcode: createResourceDriver
driverName: another-test-driver.cdi.k8s.io
nodes: scheduler-perf-dra-*
maxClaimsPerNodeParam: $maxClaimsPerNode
- opcode: createAny
templatePath: config/dra/deviceclass.yaml
- opcode: createAny
templatePath: config/dra/resourceclaimtemplate.yaml
namespace: init
- opcode: createAny
templatePath: config/dra/another-resourceclaimtemplate.yaml
namespace: init
- opcode: createPods
namespace: init
countParam: $initPods
podTemplatePath: config/dra/pod-with-many-claim-templates.yaml
- opcode: createAny
templatePath: config/dra/resourceclaimtemplate.yaml
namespace: test
- opcode: createAny
templatePath: config/dra/another-resourceclaimtemplate.yaml
namespace: test
- opcode: createPods
namespace: test
countParam: $measurePods
podTemplatePath: config/dra/pod-with-many-claim-templates.yaml
collectMetrics: true
workloads:
- name: fast
labels: [integration-test, performance, short]
params:
# This testcase runs through all code paths without
# taking too long overall.
nodesWithDRA: 1
nodesWithoutDRA: 1
initPods: 0
measurePods: 1
maxClaimsPerNode: 20
- name: 2000pods_100nodes
params:
# In this testcase, the number of nodes is smaller
# than the limit for the PodScheduling slices.
nodesWithDRA: 100
nodesWithoutDRA: 0
initPods: 1000
measurePods: 1000
maxClaimsPerNode: 40
- name: 2000pods_200nodes
params:
# In this testcase, the driver and scheduler must
# truncate the PotentialNodes and UnsuitableNodes
# slices.
nodesWithDRA: 200
nodesWithoutDRA: 0
initPods: 1000
measurePods: 1000
maxClaimsPerNode: 20
# SchedulingWithResourceClaimTemplateStructured uses a ResourceClaimTemplate
# and dynamically creates ResourceClaim instances for each pod. Node, pod and
# device counts are chosen so that the cluster gets filled up completely.
#
# The driver uses structured parameters.
- name: SchedulingWithResourceClaimTemplateStructured
- name: SchedulingWithResourceClaimTemplate
featureGates:
DynamicResourceAllocation: true
# SchedulerQueueingHints: true
@@ -1185,18 +1041,17 @@
driverName: test-driver.cdi.k8s.io
nodes: scheduler-perf-dra-*
maxClaimsPerNodeParam: $maxClaimsPerNode
structuredParameters: true
- opcode: createAny
templatePath: config/dra/deviceclass-structured.yaml
templatePath: config/dra/deviceclass.yaml
- opcode: createAny
templatePath: config/dra/resourceclaimtemplate-structured.yaml
templatePath: config/dra/resourceclaimtemplate.yaml
namespace: init
- opcode: createPods
namespace: init
countParam: $initPods
podTemplatePath: config/dra/pod-with-claim-template.yaml
- opcode: createAny
templatePath: config/dra/resourceclaimtemplate-structured.yaml
templatePath: config/dra/resourceclaimtemplate.yaml
namespace: test
- opcode: createPods
namespace: test
@@ -1236,16 +1091,14 @@
measurePods: 2500
maxClaimsPerNode: 10
# SteadyStateResourceClaimTemplateStructured uses a ResourceClaimTemplate and
# SteadyStateResourceClaimTemplate uses a ResourceClaimTemplate and
# dynamically creates ResourceClaim instances for each pod. It creates ten
# pods, waits for them to be scheduled, deletes them, and starts again,
# so the cluster remains at the same level of utilization.
#
# The number of already allocated claims can be varied, thus simulating
# various degrees of pre-existing resource utilization.
#
# The driver uses structured parameters.
- name: SteadyStateClusterResourceClaimTemplateStructured
- name: SteadyStateClusterResourceClaimTemplate
featureGates:
DynamicResourceAllocation: true
# SchedulerQueueingHints: true
@@ -1259,17 +1112,16 @@
driverName: test-driver.cdi.k8s.io
nodes: scheduler-perf-dra-*
maxClaimsPerNodeParam: $maxClaimsPerNode
structuredParameters: true
- opcode: createAny
templatePath: config/dra/deviceclass-structured.yaml
templatePath: config/dra/deviceclass.yaml
- opcode: createAny
templatePath: config/dra/resourceclaim-structured.yaml
templatePath: config/dra/resourceclaim.yaml
countParam: $initClaims
namespace: init
- opcode: allocResourceClaims
namespace: init
- opcode: createAny
templatePath: config/dra/resourceclaimtemplate-structured.yaml
templatePath: config/dra/resourceclaimtemplate.yaml
namespace: test
- opcode: createPods
namespace: test
@@ -1358,9 +1210,7 @@
# SchedulingWithResourceClaimTemplate uses ResourceClaims
# with deterministic names that are shared between pods.
# There is a fixed ratio of 1:5 between claims and pods.
#
# The driver uses structured parameters.
- name: SchedulingWithResourceClaimStructured
- name: SchedulingWithResourceClaim
featureGates:
DynamicResourceAllocation: true
# SchedulerQueueingHints: true
@@ -1374,11 +1224,10 @@
driverName: test-driver.cdi.k8s.io
nodes: scheduler-perf-dra-*
maxClaimsPerNodeParam: $maxClaimsPerNode
structuredParameters: true
- opcode: createAny
templatePath: config/dra/deviceclass-structured.yaml
templatePath: config/dra/deviceclass.yaml
- opcode: createAny
templatePath: config/dra/resourceclaim-structured.yaml
templatePath: config/dra/resourceclaim.yaml
namespace: init
countParam: $initClaims
- opcode: createPods
@@ -1386,7 +1235,7 @@
countParam: $initPods
podTemplatePath: config/dra/pod-with-claim-ref.yaml
- opcode: createAny
templatePath: config/dra/resourceclaim-structured.yaml
templatePath: config/dra/resourceclaim.yaml
namespace: test
countParam: $measureClaims
- opcode: createPods

View File

@@ -17,7 +17,6 @@ limitations under the License.
package benchmark
import (
"context"
"fmt"
"math/rand/v2"
"path/filepath"
@@ -34,9 +33,7 @@ import (
"k8s.io/client-go/informers"
"k8s.io/client-go/util/workqueue"
"k8s.io/dynamic-resource-allocation/structured"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/scheduler/util/assumecache"
draapp "k8s.io/kubernetes/test/e2e/dra/test-driver/app"
"k8s.io/kubernetes/test/utils/ktesting"
"k8s.io/utils/ptr"
)
@@ -134,11 +131,6 @@ type createResourceDriverOp struct {
MaxClaimsPerNodeParam string
// Nodes matching this glob pattern have resources managed by the driver.
Nodes string
// StructuredParameters is true if the controller that is built into the scheduler
// is used and the control-plane controller is not needed.
// Because we don't run the kubelet plugin, ResourceSlices must
// get created for all nodes.
StructuredParameters bool
}
var _ realOp = &createResourceDriverOp{}
@@ -176,13 +168,7 @@ func (op *createResourceDriverOp) requiredNamespaces() []string { return nil }
func (op *createResourceDriverOp) run(tCtx ktesting.TContext) {
tCtx.Logf("creating resource driver %q for nodes matching %q", op.DriverName, op.Nodes)
// Start the controller side of the DRA test driver such that it simulates
// per-node resources.
resources := draapp.Resources{
DriverName: op.DriverName,
NodeLocal: true,
MaxAllocations: op.MaxClaimsPerNode,
}
var driverNodes []string
nodes, err := tCtx.Client().CoreV1().Nodes().List(tCtx, metav1.ListOptions{})
if err != nil {
@@ -194,42 +180,21 @@ func (op *createResourceDriverOp) run(tCtx ktesting.TContext) {
tCtx.Fatalf("matching glob pattern %q against node name %q: %v", op.Nodes, node.Name, err)
}
if match {
resources.Nodes = append(resources.Nodes, node.Name)
driverNodes = append(driverNodes, node.Name)
}
}
if op.StructuredParameters {
for _, nodeName := range resources.Nodes {
slice := resourceSlice(op.DriverName, nodeName, op.MaxClaimsPerNode)
_, err := tCtx.Client().ResourceV1alpha3().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{})
tCtx.ExpectNoError(err, "create node resource slice")
}
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
err := tCtx.Client().ResourceV1alpha3().ResourceSlices().DeleteCollection(tCtx,
metav1.DeleteOptions{},
metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + op.DriverName},
)
tCtx.ExpectNoError(err, "delete node resource slices")
})
// No need for the controller.
return
for _, nodeName := range driverNodes {
slice := resourceSlice(op.DriverName, nodeName, op.MaxClaimsPerNode)
_, err := tCtx.Client().ResourceV1alpha3().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{})
tCtx.ExpectNoError(err, "create node resource slice")
}
controller := draapp.NewController(tCtx.Client(), resources)
ctx, cancel := context.WithCancel(tCtx)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
ctx := klog.NewContext(ctx, klog.LoggerWithName(klog.FromContext(ctx), op.DriverName))
controller.Run(ctx, 5 /* workers */)
}()
tCtx.Cleanup(func() {
tCtx.Logf("stopping resource driver %q", op.DriverName)
// We must cancel before waiting.
cancel()
wg.Wait()
tCtx.Logf("stopped resource driver %q", op.DriverName)
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
err := tCtx.Client().ResourceV1alpha3().ResourceSlices().DeleteCollection(tCtx,
metav1.DeleteOptions{},
metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + op.DriverName},
)
tCtx.ExpectNoError(err, "delete node resource slices")
})
}

View File

@@ -130,10 +130,9 @@ func StartScheduler(ctx context.Context, clientSet clientset.Interface, kubeConf
func CreateResourceClaimController(ctx context.Context, tb ktesting.TB, clientSet clientset.Interface, informerFactory informers.SharedInformerFactory) func() {
podInformer := informerFactory.Core().V1().Pods()
schedulingInformer := informerFactory.Resource().V1alpha3().PodSchedulingContexts()
claimInformer := informerFactory.Resource().V1alpha3().ResourceClaims()
claimTemplateInformer := informerFactory.Resource().V1alpha3().ResourceClaimTemplates()
claimController, err := resourceclaim.NewController(klog.FromContext(ctx), clientSet, podInformer, schedulingInformer, claimInformer, claimTemplateInformer)
claimController, err := resourceclaim.NewController(klog.FromContext(ctx), clientSet, podInformer, claimInformer, claimTemplateInformer)
if err != nil {
tb.Fatalf("Error creating claim controller: %v", err)
}