mirror of
https://github.com/k3s-io/kubernetes.git
synced 2026-07-26 07:14:12 +00:00
DRA: remove "classic DRA"
This removes the DRAControlPlaneController feature gate, the fields controlled by it (claim.spec.controller, claim.status.deallocationRequested, claim.status.allocation.controller, class.spec.suitableNodes), the PodSchedulingContext type, and all code related to the feature. The feature gets removed because there is no path towards beta and GA and DRA with "structured parameters" should be able to replace it.
This commit is contained in:
@@ -59,8 +59,7 @@ var resetFieldsStatusData = map[schema.GroupVersionResource]string{
|
||||
gvr("storage.k8s.io", "v1", "volumeattachments"): `{"status": {"attached": false}}`,
|
||||
gvr("policy", "v1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 25}}`,
|
||||
gvr("policy", "v1beta1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 25}}`,
|
||||
gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): `{"status": {"resourceClaims": [{"name": "my-claim", "unsuitableNodes": ["node2"]}]}}`, // Not really a conflict with status_test.go: Apply just stores both nodes. Conflict testing therefore gets disabled for podschedulingcontexts.
|
||||
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"status": {"allocation": {"controller": "other.example.com"}}}`,
|
||||
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"status": {"allocation": {"nodeSelector": {"nodeSelectorTerms": [{"matchExpressions": [{"key": "some-label", "operator": "In", "values": ["some-other-value"]}] }]}}}}`,
|
||||
gvr("internal.apiserver.k8s.io", "v1alpha1", "storageversions"): `{"status": {"commonEncodingVersion":"v1","storageVersions":[{"apiServerID":"1","decodableVersions":["v1","v2"],"encodingVersion":"v1"}],"conditions":[{"type":"AllEncodingVersionsEqual","status":"False","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"allEncodingVersionsEqual","message":"all encoding versions are set to v1"}]}}`,
|
||||
// standard for []metav1.Condition
|
||||
gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies"): `{"status": {"conditions":[{"type":"Accepted","status":"True","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"RuleApplied","message":"Rule was applied"}]}}`,
|
||||
@@ -90,10 +89,6 @@ var noConflicts = map[string]struct{}{
|
||||
// namespaces only have a spec.finalizers field which is also skipped,
|
||||
// thus it will never have a conflict.
|
||||
"namespaces": {},
|
||||
// podschedulingcontexts.status only has a list which contains items with a list,
|
||||
// therefore apply works because it simply merges either the outer or
|
||||
// the inner list.
|
||||
"podschedulingcontexts": {},
|
||||
}
|
||||
|
||||
var image2 = image.GetE2EImage(image.Etcd)
|
||||
@@ -152,7 +147,6 @@ var resetFieldsSpecData = map[schema.GroupVersionResource]string{
|
||||
gvr("awesome.bears.com", "v3", "pandas"): `{"spec": {"replicas": 302}}`,
|
||||
gvr("apiregistration.k8s.io", "v1beta1", "apiservices"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"group": "foo2.com"}}`,
|
||||
gvr("apiregistration.k8s.io", "v1", "apiservices"): `{"metadata": {"labels": {"a":"c"}}, "spec": {"group": "foo2.com"}}`,
|
||||
gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): `{"spec": {"selectedNode": "node2name"}}`,
|
||||
gvr("resource.k8s.io", "v1alpha3", "deviceclasses"): `{"metadata": {"labels":{"a":"c"}}}`,
|
||||
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"spec": {"devices": {"requests": [{"name": "req-0", "deviceClassName": "other-class"}]}}}`, // spec is immutable, but that doesn't matter for the test.
|
||||
gvr("resource.k8s.io", "v1alpha3", "resourceclaimtemplates"): `{"spec": {"spec": {"resourceClassName": "class2name"}}}`,
|
||||
|
||||
@@ -52,8 +52,7 @@ var statusData = map[schema.GroupVersionResource]string{
|
||||
gvr("storage.k8s.io", "v1", "volumeattachments"): `{"status": {"attached": true}}`,
|
||||
gvr("policy", "v1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 5}}`,
|
||||
gvr("policy", "v1beta1", "poddisruptionbudgets"): `{"status": {"currentHealthy": 5}}`,
|
||||
gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): `{"status": {"resourceClaims": [{"name": "my-claim", "unsuitableNodes": ["node1"]}]}}`,
|
||||
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"status": {"allocation": {"controller": "example.com"}}}`,
|
||||
gvr("resource.k8s.io", "v1alpha3", "resourceclaims"): `{"status": {"allocation": {"nodeSelector": {"nodeSelectorTerms": [{"matchExpressions": [{"key": "some-label", "operator": "In", "values": ["some-value"]}] }]}}}}`,
|
||||
gvr("internal.apiserver.k8s.io", "v1alpha1", "storageversions"): `{"status": {"commonEncodingVersion":"v1","storageVersions":[{"apiServerID":"1","decodableVersions":["v1","v2"],"encodingVersion":"v1"}],"conditions":[{"type":"AllEncodingVersionsEqual","status":"True","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"allEncodingVersionsEqual","message":"all encoding versions are set to v1"}]}}`,
|
||||
// standard for []metav1.Condition
|
||||
gvr("admissionregistration.k8s.io", "v1alpha1", "validatingadmissionpolicies"): `{"status": {"conditions":[{"type":"Accepted","status":"False","lastTransitionTime":"2020-01-01T00:00:00Z","reason":"RuleApplied","message":"Rule was applied"}]}}`,
|
||||
|
||||
@@ -431,10 +431,6 @@ func GetEtcdStorageDataForNamespace(namespace string) map[schema.GroupVersionRes
|
||||
Stub: `{"metadata": {"name": "claimtemplate1name"}, "spec": {"spec": {"devices": {"requests": [{"name": "req-0", "deviceClassName": "example-class", "allocationMode": "ExactCount", "count": 1}]}}}}`,
|
||||
ExpectedEtcdPath: "/registry/resourceclaimtemplates/" + namespace + "/claimtemplate1name",
|
||||
},
|
||||
gvr("resource.k8s.io", "v1alpha3", "podschedulingcontexts"): {
|
||||
Stub: `{"metadata": {"name": "pod1name"}, "spec": {"selectedNode": "node1name", "potentialNodes": ["node1name", "node2name"]}}`,
|
||||
ExpectedEtcdPath: "/registry/podschedulingcontexts/" + namespace + "/pod1name",
|
||||
},
|
||||
gvr("resource.k8s.io", "v1alpha3", "resourceslices"): {
|
||||
Stub: `{"metadata": {"name": "node1slice"}, "spec": {"nodeName": "worker1", "driver": "dra.example.com", "pool": {"name": "worker1", "resourceSliceCount": 1}}}`,
|
||||
ExpectedEtcdPath: "/registry/resourceslices/node1slice",
|
||||
|
||||
@@ -21,34 +21,25 @@ package scheduler
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
resourceapi "k8s.io/api/resource/v1alpha3"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
corelisters "k8s.io/client-go/listers/core/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
configv1 "k8s.io/kube-scheduler/config/v1"
|
||||
"k8s.io/kubernetes/pkg/features"
|
||||
"k8s.io/kubernetes/pkg/scheduler"
|
||||
configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing"
|
||||
st "k8s.io/kubernetes/pkg/scheduler/testing"
|
||||
testutils "k8s.io/kubernetes/test/integration/util"
|
||||
"k8s.io/kubernetes/test/utils/format"
|
||||
"k8s.io/utils/pointer"
|
||||
)
|
||||
|
||||
@@ -647,117 +638,3 @@ func TestNodeEvents(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// TestPodSchedulingContextSSA checks that the dynamicresources plugin falls
|
||||
// back to SSA successfully when the normal Update call encountered
|
||||
// a conflict.
|
||||
//
|
||||
// This is an integration test because:
|
||||
// - Unit testing does not cover RBAC rules.
|
||||
// - Triggering this particular race is harder in E2E testing
|
||||
// and harder to verify (needs apiserver metrics and there's
|
||||
// no standard API for those).
|
||||
func TestPodSchedulingContextSSA(t *testing.T) {
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DynamicResourceAllocation, true)
|
||||
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRAControlPlaneController, true)
|
||||
|
||||
testCtx := testutils.InitTestAPIServer(t, "podschedulingcontext-ssa", nil)
|
||||
testCtx.DisableEventSink = true
|
||||
testCtx = testutils.InitTestSchedulerWithOptions(t, testCtx, 0)
|
||||
testutils.SyncSchedulerInformerFactory(testCtx)
|
||||
go testCtx.Scheduler.Run(testCtx.SchedulerCtx)
|
||||
|
||||
// Set up enough objects that the scheduler will start trying to
|
||||
// schedule the pod and create the PodSchedulingContext.
|
||||
nodeRes := map[v1.ResourceName]string{
|
||||
v1.ResourcePods: "32",
|
||||
v1.ResourceCPU: "30m",
|
||||
v1.ResourceMemory: "30",
|
||||
}
|
||||
for _, name := range []string{"node-a", "node-b"} {
|
||||
if _, err := testutils.CreateNode(testCtx.ClientSet, st.MakeNode().Name(name).Capacity(nodeRes).Obj()); err != nil {
|
||||
t.Fatalf("Failed to create node: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
claim := &resourceapi.ResourceClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-claim",
|
||||
Namespace: testCtx.NS.Name,
|
||||
},
|
||||
Spec: resourceapi.ResourceClaimSpec{
|
||||
Controller: "dra.example.com",
|
||||
},
|
||||
}
|
||||
if _, err := testCtx.ClientSet.ResourceV1alpha3().ResourceClaims(claim.Namespace).Create(testCtx.Ctx, claim, metav1.CreateOptions{}); err != nil {
|
||||
t.Fatalf("Failed to create claim: %v", err)
|
||||
}
|
||||
|
||||
podConf := testutils.PausePodConfig{
|
||||
Name: "testpod",
|
||||
Namespace: testCtx.NS.Name,
|
||||
}
|
||||
pod := testutils.InitPausePod(&podConf)
|
||||
podClaimName := "myclaim"
|
||||
pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: podClaimName}}
|
||||
pod.Spec.ResourceClaims = []v1.PodResourceClaim{{Name: podClaimName, ResourceClaimName: &claim.Name}}
|
||||
if _, err := testCtx.ClientSet.CoreV1().Pods(pod.Namespace).Create(testCtx.Ctx, pod, metav1.CreateOptions{}); err != nil {
|
||||
t.Fatalf("Failed to create pod: %v", err)
|
||||
}
|
||||
|
||||
// Check that the PodSchedulingContext exists and has a selected node.
|
||||
var schedulingCtx *resourceapi.PodSchedulingContext
|
||||
if err := wait.PollUntilContextTimeout(testCtx.Ctx, 10*time.Microsecond, 30*time.Second, true,
|
||||
func(context.Context) (bool, error) {
|
||||
var err error
|
||||
schedulingCtx, err = testCtx.ClientSet.ResourceV1alpha3().PodSchedulingContexts(pod.Namespace).Get(testCtx.Ctx, pod.Name, metav1.GetOptions{})
|
||||
if apierrors.IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
if err == nil && schedulingCtx.Spec.SelectedNode != "" {
|
||||
return true, nil
|
||||
}
|
||||
return false, err
|
||||
}); err != nil {
|
||||
t.Fatalf("Failed while waiting for PodSchedulingContext with selected node: %v\nLast PodSchedulingContext:\n%s", err, format.Object(schedulingCtx, 1))
|
||||
}
|
||||
|
||||
// Force the plugin to use SSA.
|
||||
var podSchedulingContextPatchCounter atomic.Int64
|
||||
roundTrip := testutils.RoundTripWrapper(func(transport http.RoundTripper, req *http.Request) (*http.Response, error) {
|
||||
if strings.HasPrefix(req.URL.Path, "/apis/resource.k8s.io/") &&
|
||||
strings.HasSuffix(req.URL.Path, "/podschedulingcontexts/"+pod.Name) {
|
||||
switch req.Method {
|
||||
case http.MethodPut, http.MethodPost:
|
||||
return &http.Response{
|
||||
Status: fmt.Sprintf("%d %s", http.StatusConflict, metav1.StatusReasonConflict),
|
||||
StatusCode: http.StatusConflict,
|
||||
}, nil
|
||||
case http.MethodPatch:
|
||||
podSchedulingContextPatchCounter.Add(1)
|
||||
}
|
||||
}
|
||||
return transport.RoundTrip(req)
|
||||
})
|
||||
testCtx.RoundTrip.Store(&roundTrip)
|
||||
|
||||
// Now force the scheduler to update the PodSchedulingContext by setting UnsuitableNodes so that
|
||||
// the selected node is not suitable.
|
||||
schedulingCtx.Status.ResourceClaims = []resourceapi.ResourceClaimSchedulingStatus{{
|
||||
Name: podClaimName,
|
||||
UnsuitableNodes: []string{schedulingCtx.Spec.SelectedNode},
|
||||
}}
|
||||
|
||||
if _, err := testCtx.ClientSet.ResourceV1alpha3().PodSchedulingContexts(pod.Namespace).UpdateStatus(testCtx.Ctx, schedulingCtx, metav1.UpdateOptions{}); err != nil {
|
||||
t.Fatalf("Unexpected PodSchedulingContext status update error: %v", err)
|
||||
}
|
||||
|
||||
// We know that the scheduler has to use SSA because above we inject a conflict
|
||||
// error whenever it tries to use a plain update. We just need to wait for it...
|
||||
if err := wait.PollUntilContextTimeout(testCtx.Ctx, 10*time.Microsecond, time.Minute, true,
|
||||
func(context.Context) (bool, error) {
|
||||
return podSchedulingContextPatchCounter.Load() > 0, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Failed while waiting for PodSchedulingContext Patch: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
apiVersion: resource.k8s.io/v1alpha3
|
||||
kind: DeviceClass
|
||||
metadata:
|
||||
name: test-class
|
||||
spec:
|
||||
selectors:
|
||||
- cel:
|
||||
expression: device.driver == "test-driver.cdi.k8s.io"
|
||||
@@ -2,3 +2,7 @@ apiVersion: resource.k8s.io/v1alpha3
|
||||
kind: DeviceClass
|
||||
metadata:
|
||||
name: test-class
|
||||
spec:
|
||||
selectors:
|
||||
- cel:
|
||||
expression: device.driver == "test-driver.cdi.k8s.io"
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
apiVersion: resource.k8s.io/v1alpha3
|
||||
kind: ResourceClaim
|
||||
metadata:
|
||||
name: test-claim-{{.Index}}
|
||||
spec:
|
||||
devices:
|
||||
requests:
|
||||
- name: req-0
|
||||
deviceClassName: test-class
|
||||
@@ -3,7 +3,6 @@ kind: ResourceClaim
|
||||
metadata:
|
||||
name: test-claim-{{.Index}}
|
||||
spec:
|
||||
controller: test-driver.cdi.k8s.io
|
||||
devices:
|
||||
requests:
|
||||
- name: req-0
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
apiVersion: resource.k8s.io/v1alpha3
|
||||
kind: ResourceClaimTemplate
|
||||
metadata:
|
||||
name: test-claim-template
|
||||
spec:
|
||||
spec:
|
||||
devices:
|
||||
requests:
|
||||
- name: req-0
|
||||
deviceClassName: test-class
|
||||
@@ -4,7 +4,6 @@ metadata:
|
||||
name: test-claim-template
|
||||
spec:
|
||||
spec:
|
||||
controller: test-driver.cdi.k8s.io
|
||||
devices:
|
||||
requests:
|
||||
- name: req-0
|
||||
|
||||
@@ -1025,153 +1025,9 @@
|
||||
measurePods: 4000
|
||||
|
||||
# SchedulingWithResourceClaimTemplate uses a ResourceClaimTemplate
|
||||
# and dynamically created ResourceClaim instances for each pod.
|
||||
- name: SchedulingWithResourceClaimTemplate
|
||||
featureGates:
|
||||
DRAControlPlaneController: true
|
||||
DynamicResourceAllocation: true
|
||||
# SchedulerQueueingHints: true
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $nodesWithoutDRA
|
||||
- opcode: createNodes
|
||||
nodeTemplatePath: config/dra/node-with-dra-test-driver.yaml
|
||||
countParam: $nodesWithDRA
|
||||
- opcode: createResourceDriver
|
||||
driverName: test-driver.cdi.k8s.io
|
||||
nodes: scheduler-perf-dra-*
|
||||
maxClaimsPerNodeParam: $maxClaimsPerNode
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/deviceclass.yaml
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaimtemplate.yaml
|
||||
namespace: init
|
||||
- opcode: createPods
|
||||
namespace: init
|
||||
countParam: $initPods
|
||||
podTemplatePath: config/dra/pod-with-claim-template.yaml
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaimtemplate.yaml
|
||||
namespace: test
|
||||
- opcode: createPods
|
||||
namespace: test
|
||||
countParam: $measurePods
|
||||
podTemplatePath: config/dra/pod-with-claim-template.yaml
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: fast
|
||||
labels: [integration-test, performance, short]
|
||||
params:
|
||||
# This testcase runs through all code paths without
|
||||
# taking too long overall.
|
||||
nodesWithDRA: 1
|
||||
nodesWithoutDRA: 1
|
||||
initPods: 0
|
||||
measurePods: 10
|
||||
maxClaimsPerNode: 10
|
||||
- name: 2000pods_100nodes
|
||||
params:
|
||||
# In this testcase, the number of nodes is smaller
|
||||
# than the limit for the PodScheduling slices.
|
||||
nodesWithDRA: 100
|
||||
nodesWithoutDRA: 0
|
||||
initPods: 1000
|
||||
measurePods: 1000
|
||||
maxClaimsPerNode: 20
|
||||
- name: 2000pods_200nodes
|
||||
params:
|
||||
# In this testcase, the driver and scheduler must
|
||||
# truncate the PotentialNodes and UnsuitableNodes
|
||||
# slices.
|
||||
nodesWithDRA: 200
|
||||
nodesWithoutDRA: 0
|
||||
initPods: 1000
|
||||
measurePods: 1000
|
||||
maxClaimsPerNode: 10
|
||||
|
||||
# This similar to SchedulingWithResourceClaimTemplate, except
|
||||
# that it uses four claims per pod, from two different drivers.
|
||||
# This emphasizes a bit more the complexity of collaborative
|
||||
# scheduling via PodSchedulingContext.
|
||||
- name: SchedulingWithMultipleResourceClaims
|
||||
featureGates:
|
||||
DRAControlPlaneController: true
|
||||
DynamicResourceAllocation: true
|
||||
# SchedulerQueueingHints: true
|
||||
workloadTemplate:
|
||||
- opcode: createNodes
|
||||
countParam: $nodesWithoutDRA
|
||||
- opcode: createNodes
|
||||
nodeTemplatePath: config/dra/node-with-dra-test-driver.yaml
|
||||
countParam: $nodesWithDRA
|
||||
- opcode: createResourceDriver
|
||||
driverName: test-driver.cdi.k8s.io
|
||||
nodes: scheduler-perf-dra-*
|
||||
maxClaimsPerNodeParam: $maxClaimsPerNode
|
||||
- opcode: createResourceDriver
|
||||
driverName: another-test-driver.cdi.k8s.io
|
||||
nodes: scheduler-perf-dra-*
|
||||
maxClaimsPerNodeParam: $maxClaimsPerNode
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/deviceclass.yaml
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaimtemplate.yaml
|
||||
namespace: init
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/another-resourceclaimtemplate.yaml
|
||||
namespace: init
|
||||
- opcode: createPods
|
||||
namespace: init
|
||||
countParam: $initPods
|
||||
podTemplatePath: config/dra/pod-with-many-claim-templates.yaml
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaimtemplate.yaml
|
||||
namespace: test
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/another-resourceclaimtemplate.yaml
|
||||
namespace: test
|
||||
- opcode: createPods
|
||||
namespace: test
|
||||
countParam: $measurePods
|
||||
podTemplatePath: config/dra/pod-with-many-claim-templates.yaml
|
||||
collectMetrics: true
|
||||
workloads:
|
||||
- name: fast
|
||||
labels: [integration-test, performance, short]
|
||||
params:
|
||||
# This testcase runs through all code paths without
|
||||
# taking too long overall.
|
||||
nodesWithDRA: 1
|
||||
nodesWithoutDRA: 1
|
||||
initPods: 0
|
||||
measurePods: 1
|
||||
maxClaimsPerNode: 20
|
||||
- name: 2000pods_100nodes
|
||||
params:
|
||||
# In this testcase, the number of nodes is smaller
|
||||
# than the limit for the PodScheduling slices.
|
||||
nodesWithDRA: 100
|
||||
nodesWithoutDRA: 0
|
||||
initPods: 1000
|
||||
measurePods: 1000
|
||||
maxClaimsPerNode: 40
|
||||
- name: 2000pods_200nodes
|
||||
params:
|
||||
# In this testcase, the driver and scheduler must
|
||||
# truncate the PotentialNodes and UnsuitableNodes
|
||||
# slices.
|
||||
nodesWithDRA: 200
|
||||
nodesWithoutDRA: 0
|
||||
initPods: 1000
|
||||
measurePods: 1000
|
||||
maxClaimsPerNode: 20
|
||||
|
||||
# SchedulingWithResourceClaimTemplateStructured uses a ResourceClaimTemplate
|
||||
# and dynamically creates ResourceClaim instances for each pod. Node, pod and
|
||||
# device counts are chosen so that the cluster gets filled up completely.
|
||||
#
|
||||
# The driver uses structured parameters.
|
||||
- name: SchedulingWithResourceClaimTemplateStructured
|
||||
- name: SchedulingWithResourceClaimTemplate
|
||||
featureGates:
|
||||
DynamicResourceAllocation: true
|
||||
# SchedulerQueueingHints: true
|
||||
@@ -1185,18 +1041,17 @@
|
||||
driverName: test-driver.cdi.k8s.io
|
||||
nodes: scheduler-perf-dra-*
|
||||
maxClaimsPerNodeParam: $maxClaimsPerNode
|
||||
structuredParameters: true
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/deviceclass-structured.yaml
|
||||
templatePath: config/dra/deviceclass.yaml
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaimtemplate-structured.yaml
|
||||
templatePath: config/dra/resourceclaimtemplate.yaml
|
||||
namespace: init
|
||||
- opcode: createPods
|
||||
namespace: init
|
||||
countParam: $initPods
|
||||
podTemplatePath: config/dra/pod-with-claim-template.yaml
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaimtemplate-structured.yaml
|
||||
templatePath: config/dra/resourceclaimtemplate.yaml
|
||||
namespace: test
|
||||
- opcode: createPods
|
||||
namespace: test
|
||||
@@ -1236,16 +1091,14 @@
|
||||
measurePods: 2500
|
||||
maxClaimsPerNode: 10
|
||||
|
||||
# SteadyStateResourceClaimTemplateStructured uses a ResourceClaimTemplate and
|
||||
# SteadyStateResourceClaimTemplate uses a ResourceClaimTemplate and
|
||||
# dynamically creates ResourceClaim instances for each pod. It creates ten
|
||||
# pods, waits for them to be scheduled, deletes them, and starts again,
|
||||
# so the cluster remains at the same level of utilization.
|
||||
#
|
||||
# The number of already allocated claims can be varied, thus simulating
|
||||
# various degrees of pre-existing resource utilization.
|
||||
#
|
||||
# The driver uses structured parameters.
|
||||
- name: SteadyStateClusterResourceClaimTemplateStructured
|
||||
- name: SteadyStateClusterResourceClaimTemplate
|
||||
featureGates:
|
||||
DynamicResourceAllocation: true
|
||||
# SchedulerQueueingHints: true
|
||||
@@ -1259,17 +1112,16 @@
|
||||
driverName: test-driver.cdi.k8s.io
|
||||
nodes: scheduler-perf-dra-*
|
||||
maxClaimsPerNodeParam: $maxClaimsPerNode
|
||||
structuredParameters: true
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/deviceclass-structured.yaml
|
||||
templatePath: config/dra/deviceclass.yaml
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaim-structured.yaml
|
||||
templatePath: config/dra/resourceclaim.yaml
|
||||
countParam: $initClaims
|
||||
namespace: init
|
||||
- opcode: allocResourceClaims
|
||||
namespace: init
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaimtemplate-structured.yaml
|
||||
templatePath: config/dra/resourceclaimtemplate.yaml
|
||||
namespace: test
|
||||
- opcode: createPods
|
||||
namespace: test
|
||||
@@ -1358,9 +1210,7 @@
|
||||
# SchedulingWithResourceClaimTemplate uses ResourceClaims
|
||||
# with deterministic names that are shared between pods.
|
||||
# There is a fixed ratio of 1:5 between claims and pods.
|
||||
#
|
||||
# The driver uses structured parameters.
|
||||
- name: SchedulingWithResourceClaimStructured
|
||||
- name: SchedulingWithResourceClaim
|
||||
featureGates:
|
||||
DynamicResourceAllocation: true
|
||||
# SchedulerQueueingHints: true
|
||||
@@ -1374,11 +1224,10 @@
|
||||
driverName: test-driver.cdi.k8s.io
|
||||
nodes: scheduler-perf-dra-*
|
||||
maxClaimsPerNodeParam: $maxClaimsPerNode
|
||||
structuredParameters: true
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/deviceclass-structured.yaml
|
||||
templatePath: config/dra/deviceclass.yaml
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaim-structured.yaml
|
||||
templatePath: config/dra/resourceclaim.yaml
|
||||
namespace: init
|
||||
countParam: $initClaims
|
||||
- opcode: createPods
|
||||
@@ -1386,7 +1235,7 @@
|
||||
countParam: $initPods
|
||||
podTemplatePath: config/dra/pod-with-claim-ref.yaml
|
||||
- opcode: createAny
|
||||
templatePath: config/dra/resourceclaim-structured.yaml
|
||||
templatePath: config/dra/resourceclaim.yaml
|
||||
namespace: test
|
||||
countParam: $measureClaims
|
||||
- opcode: createPods
|
||||
|
||||
@@ -17,7 +17,6 @@ limitations under the License.
|
||||
package benchmark
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"path/filepath"
|
||||
@@ -34,9 +33,7 @@ import (
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
"k8s.io/dynamic-resource-allocation/structured"
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/kubernetes/pkg/scheduler/util/assumecache"
|
||||
draapp "k8s.io/kubernetes/test/e2e/dra/test-driver/app"
|
||||
"k8s.io/kubernetes/test/utils/ktesting"
|
||||
"k8s.io/utils/ptr"
|
||||
)
|
||||
@@ -134,11 +131,6 @@ type createResourceDriverOp struct {
|
||||
MaxClaimsPerNodeParam string
|
||||
// Nodes matching this glob pattern have resources managed by the driver.
|
||||
Nodes string
|
||||
// StructuredParameters is true if the controller that is built into the scheduler
|
||||
// is used and the control-plane controller is not needed.
|
||||
// Because we don't run the kubelet plugin, ResourceSlices must
|
||||
// get created for all nodes.
|
||||
StructuredParameters bool
|
||||
}
|
||||
|
||||
var _ realOp = &createResourceDriverOp{}
|
||||
@@ -176,13 +168,7 @@ func (op *createResourceDriverOp) requiredNamespaces() []string { return nil }
|
||||
func (op *createResourceDriverOp) run(tCtx ktesting.TContext) {
|
||||
tCtx.Logf("creating resource driver %q for nodes matching %q", op.DriverName, op.Nodes)
|
||||
|
||||
// Start the controller side of the DRA test driver such that it simulates
|
||||
// per-node resources.
|
||||
resources := draapp.Resources{
|
||||
DriverName: op.DriverName,
|
||||
NodeLocal: true,
|
||||
MaxAllocations: op.MaxClaimsPerNode,
|
||||
}
|
||||
var driverNodes []string
|
||||
|
||||
nodes, err := tCtx.Client().CoreV1().Nodes().List(tCtx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
@@ -194,42 +180,21 @@ func (op *createResourceDriverOp) run(tCtx ktesting.TContext) {
|
||||
tCtx.Fatalf("matching glob pattern %q against node name %q: %v", op.Nodes, node.Name, err)
|
||||
}
|
||||
if match {
|
||||
resources.Nodes = append(resources.Nodes, node.Name)
|
||||
driverNodes = append(driverNodes, node.Name)
|
||||
}
|
||||
}
|
||||
|
||||
if op.StructuredParameters {
|
||||
for _, nodeName := range resources.Nodes {
|
||||
slice := resourceSlice(op.DriverName, nodeName, op.MaxClaimsPerNode)
|
||||
_, err := tCtx.Client().ResourceV1alpha3().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create node resource slice")
|
||||
}
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
err := tCtx.Client().ResourceV1alpha3().ResourceSlices().DeleteCollection(tCtx,
|
||||
metav1.DeleteOptions{},
|
||||
metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + op.DriverName},
|
||||
)
|
||||
tCtx.ExpectNoError(err, "delete node resource slices")
|
||||
})
|
||||
// No need for the controller.
|
||||
return
|
||||
for _, nodeName := range driverNodes {
|
||||
slice := resourceSlice(op.DriverName, nodeName, op.MaxClaimsPerNode)
|
||||
_, err := tCtx.Client().ResourceV1alpha3().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create node resource slice")
|
||||
}
|
||||
|
||||
controller := draapp.NewController(tCtx.Client(), resources)
|
||||
ctx, cancel := context.WithCancel(tCtx)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ctx := klog.NewContext(ctx, klog.LoggerWithName(klog.FromContext(ctx), op.DriverName))
|
||||
controller.Run(ctx, 5 /* workers */)
|
||||
}()
|
||||
tCtx.Cleanup(func() {
|
||||
tCtx.Logf("stopping resource driver %q", op.DriverName)
|
||||
// We must cancel before waiting.
|
||||
cancel()
|
||||
wg.Wait()
|
||||
tCtx.Logf("stopped resource driver %q", op.DriverName)
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
err := tCtx.Client().ResourceV1alpha3().ResourceSlices().DeleteCollection(tCtx,
|
||||
metav1.DeleteOptions{},
|
||||
metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + op.DriverName},
|
||||
)
|
||||
tCtx.ExpectNoError(err, "delete node resource slices")
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -130,10 +130,9 @@ func StartScheduler(ctx context.Context, clientSet clientset.Interface, kubeConf
|
||||
|
||||
func CreateResourceClaimController(ctx context.Context, tb ktesting.TB, clientSet clientset.Interface, informerFactory informers.SharedInformerFactory) func() {
|
||||
podInformer := informerFactory.Core().V1().Pods()
|
||||
schedulingInformer := informerFactory.Resource().V1alpha3().PodSchedulingContexts()
|
||||
claimInformer := informerFactory.Resource().V1alpha3().ResourceClaims()
|
||||
claimTemplateInformer := informerFactory.Resource().V1alpha3().ResourceClaimTemplates()
|
||||
claimController, err := resourceclaim.NewController(klog.FromContext(ctx), clientSet, podInformer, schedulingInformer, claimInformer, claimTemplateInformer)
|
||||
claimController, err := resourceclaim.NewController(klog.FromContext(ctx), clientSet, podInformer, claimInformer, claimTemplateInformer)
|
||||
if err != nil {
|
||||
tb.Fatalf("Error creating claim controller: %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user