github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-10-21 22:49:31 +00:00
Code Issues Projects Releases Wiki Activity
Files
9bbb46d05f0d92868d57649994cb8b734a9c9c03
kubernetes/test/integration/scheduler_perf/misc/performance-config.yaml

667 lines
19 KiB
YAML
Raw Blame History

# The following labels are used in this file. (listed in ascending order of the number of covered test cases)
#
# - integration-test: test cases to run as the integration test, usually to spot some issues in the scheduler implementation or scheduler-perf itself.
# - performance: test cases to run in the performance test.
# - short: supplemental label for the above two labels (must not used alone), which literally means short execution time test cases.
#
# Specifically, the CIs use labels like the following:
# - `ci-kubernetes-integration-master` (`integration-test`): Test cases are chosen based on a tradeoff between code coverage and overall runtime.
# It basically covers all test cases but with their smallest workload.
# - `pull-kubernetes-integration` (`integration-test`,`short`): Test cases are chosen so that they should take less than total 5 min to complete.
# - `ci-benchmark-scheduler-perf` (`performance`): Long enough test cases are chosen (ideally, longer than 10 seconds)
# to provide meaningful samples for the pod scheduling rate.
#
# Also, `performance`+`short` isn't used in the CIs, but it's used to test the performance test locally.
# (Sometimes, the test cases with `integration-test` are too small to spot issues.)
#
# Combining `performance` and `short` selects suitable workloads for a local
# before/after comparisons with benchstat.
- name: SchedulingBasic
defaultPodTemplatePath: ../templates/pod-default.yaml
workloadTemplate:
- opcode: createNodes
countParam: $initNodes
- opcode: createPods
countParam: $initPods
- opcode: createPods
countParam: $measurePods
collectMetrics: true
workloads:
- name: 5Nodes
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
initNodes: 5
initPods: 5
measurePods: 10
- name: 5Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
initNodes: 5
initPods: 5
measurePods: 10
- name: 500Nodes
labels: [performance, short]
params:
initNodes: 500
initPods: 500
measurePods: 1000
- name: 5000Nodes
labels: [performance, short]
params:
initNodes: 5000
initPods: 1000
measurePods: 1000
- name: 5000Nodes_10000Pods
featureGates:
SchedulerQueueingHints: false
labels: [performance]
threshold: 270
params:
initNodes: 5000
initPods: 1000
measurePods: 10000
- name: 5000Nodes_10000Pods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance]
threshold: 270
params:
initNodes: 5000
initPods: 1000
measurePods: 10000
# This test case simulates the scheduling of daemonset.
# https://github.com/kubernetes/kubernetes/issues/124709
- name: SchedulingDaemonset
defaultPodTemplatePath: ../templates/daemonset-pod.yaml
workloadTemplate:
# Create one node with a specific name (scheduler-perf-node),
# which is supposed to get all Pods created in this test case.
- opcode: createNodes
count: 1
nodeTemplatePath: ../templates/node-with-name.yaml
# Create other nodes that the scheduler has to filter out with PreFilterResult from NodeAffinity plugin.
- opcode: createNodes
countParam: $initNodes
nodeTemplatePath: ../templates/node-default.yaml
# Create pods with nodeAffinity (metadata.name=scheduler-perf-node).
# This scenario doesn't schedule each Pod to each Node though,
# they go through completely the same scheduling process as daemonset pods does.
- opcode: createPods
countParam: $measurePods
collectMetrics: true
workloads:
- name: 5Nodes
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
initNodes: 5
measurePods: 10
- name: 5Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
initNodes: 5
measurePods: 10
- name: 15000Nodes
featureGates:
SchedulerQueueingHints: false
labels: [performance]
threshold: 390
params:
initNodes: 15000
measurePods: 30000
- name: 15000Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance]
threshold: 390
params:
initNodes: 15000
measurePods: 30000
- name: TopologySpreading
workloadTemplate:
- opcode: createNodes
countParam: $initNodes
nodeTemplatePath: ../templates/node-default.yaml
labelNodePrepareStrategy:
labelKey: "topology.kubernetes.io/zone"
labelValues: ["moon-1", "moon-2", "moon-3"]
- opcode: createPods
countParam: $initPods
podTemplatePath: ../templates/pod-default.yaml
- opcode: createPods
countParam: $measurePods
podTemplatePath: ../templates/pod-with-topology-spreading.yaml
collectMetrics: true
workloads:
- name: 5Nodes
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
initNodes: 5
initPods: 10
measurePods: 10
- name: 5Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
initNodes: 5
initPods: 10
measurePods: 10
- name: 500Nodes
labels: [performance, short]
params:
initNodes: 500
initPods: 1000
measurePods: 1000
- name: 5000Nodes
labels: [performance, short]
params:
initNodes: 5000
initPods: 5000
measurePods: 2000
- name: 5000Nodes_5000Pods
featureGates:
SchedulerQueueingHints: false
labels: [performance]
threshold: 85
params:
initNodes: 5000
initPods: 5000
measurePods: 5000
- name: 5000Nodes_5000Pods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance]
threshold: 85
params:
initNodes: 5000
initPods: 5000
measurePods: 5000
- name: PreferredTopologySpreading
workloadTemplate:
- opcode: createNodes
countParam: $initNodes
nodeTemplatePath: ../templates/node-default.yaml
labelNodePrepareStrategy:
labelKey: "topology.kubernetes.io/zone"
labelValues: ["moon-1", "moon-2", "moon-3"]
- opcode: createPods
countParam: $initPods
podTemplatePath: ../templates/pod-default.yaml
- opcode: createPods
countParam: $measurePods
podTemplatePath: ../templates/pod-with-preferred-topology-spreading.yaml
collectMetrics: true
workloads:
- name: 5Nodes
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
initNodes: 5
initPods: 10
measurePods: 10
- name: 5Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
initNodes: 5
initPods: 10
measurePods: 10
- name: 500Nodes
labels: [performance, short]
params:
initNodes: 500
initPods: 1000
measurePods: 1000
- name: 5000Nodes
labels: [performance]
params:
initNodes: 5000
initPods: 5000
measurePods: 2000
- name: 5000Nodes_5000Pods
featureGates:
SchedulerQueueingHints: false
labels: [performance]
threshold: 125
params:
initNodes: 5000
initPods: 5000
measurePods: 5000
- name: 5000Nodes_5000Pods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance]
threshold: 125
params:
initNodes: 5000
initPods: 5000
measurePods: 5000
- name: PreemptionBasic
workloadTemplate:
- opcode: createNodes
countParam: $initNodes
- opcode: createPods
countParam: $initPods
podTemplatePath: ../templates/pod-low-priority.yaml
- opcode: createPods
countParam: $measurePods
podTemplatePath: ../templates/pod-high-priority.yaml
collectMetrics: true
workloads:
- name: 5Nodes
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
initNodes: 5
initPods: 20
measurePods: 5
- name: 5Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
initNodes: 5
initPods: 20
measurePods: 5
- name: 500Nodes
featureGates:
SchedulerQueueingHints: false
labels: [performance, short]
threshold: 18
params:
initNodes: 500
initPods: 2000
measurePods: 500
- name: 500Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance, short]
threshold: 18
params:
initNodes: 500
initPods: 2000
measurePods: 500
# This test case always seems to fail.
# https://github.com/kubernetes/kubernetes/issues/108308
#
# - name: 5000Nodes
# params:
# initNodes: 5000
# initPods: 20000
# measurePods: 5000
# Measure throughput of regular schedulable pods that are interleaved by high priority preemption pods scheduled at 5/s rate.
# Implementation of asynchronous preemption feature https://github.com/kubernetes/kubernetes/issues/126858 is supposed to increase the throughput of the measured pods as all heavy operations (apiserver communication) will be performed asynchronously, without blocking the scheduler loop.
# How is it achieved:
# 1. There are X initial nodes with 4 low priority pods each, consuming 3.6 CPU out of the total 4 available.
# 2. High priority preemption which need to preempt 3 of 4 low priority pods to fit (require 3 CPU).
# 3. Measured pods are always schedulable, as they require 0.1 CPU only.
- name: PreemptionAsync
workloadTemplate:
- opcode: createNodes
countParam: $initNodes
- opcode: createPods
countParam: $initPods
podTemplatePath: ../templates/pod-low-priority.yaml
- opcode: churn
mode: create
templatePaths:
- ../templates/pod-high-priority.yaml
intervalMilliseconds: 200
- opcode: createPods
countParam: $measurePods
podTemplatePath: ../templates/pod-default.yaml
collectMetrics: true
workloads:
- name: 5Nodes
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
initNodes: 5
initPods: 20
measurePods: 5
- name: 5Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
initNodes: 5
initPods: 20
measurePods: 5
- name: 500Nodes
labels: [performance, short]
params:
initNodes: 500
initPods: 2000
measurePods: 500
- name: 5000Nodes
featureGates:
SchedulerQueueingHints: false
labels: [performance]
threshold: 200
params:
initNodes: 5000
initPods: 20000
measurePods: 5000
- name: 5000Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance]
threshold: 120
params:
initNodes: 5000
initPods: 20000
measurePods: 5000
# Measure throughput of regular schedulable pods that are interleaved by unschedulable pods injected at 5/s rate.
- name: Unschedulable
workloadTemplate:
- opcode: createNodes
countParam: $initNodes
- opcode: churn
mode: create
templatePaths:
- ../templates/pod-high-priority-large-cpu.yaml
intervalMilliseconds: 200
- opcode: createPods
countParam: $measurePods
podTemplatePath: ../templates/pod-default.yaml
collectMetrics: true
workloads:
- name: 5Nodes/10Pods
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
initNodes: 5
measurePods: 10
- name: 5Nodes/10Pods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
initNodes: 5
measurePods: 10
- name: 500Nodes/1kPods
labels: [performance, short]
params:
initNodes: 500
measurePods: 1000
- name: 5kNodes/1kPods
labels: [performance, short]
params:
initNodes: 5000
measurePods: 1000
- name: 5kNodes/10kPods
featureGates:
SchedulerQueueingHints: false
labels: [performance]
threshold: 200
params:
initNodes: 5000
measurePods: 10000
- name: 5kNodes/10kPods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance]
threshold: 250
params:
initNodes: 5000
measurePods: 10000
- name: SchedulingWithMixedChurn
workloadTemplate:
- opcode: createNodes
countParam: $initNodes
- opcode: churn
mode: recreate
number: 1
templatePaths:
- ../templates/churn/node-default.yaml
- ../templates/pod-high-priority-large-cpu.yaml
- ../templates/churn/service-default.yaml
intervalMilliseconds: 1000
- opcode: createPods
countParam: $measurePods
podTemplatePath: ../templates/pod-default.yaml
collectMetrics: true
workloads:
- name: 10Nodes
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
initNodes: 10
measurePods: 100
- name: 10Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
initNodes: 10
measurePods: 100
- name: 1000Nodes
labels: [performance, short]
params:
initNodes: 1000
measurePods: 1000
- name: 5000Nodes
labels: [performance, short]
params:
initNodes: 5000
measurePods: 2000
- name: 5000Nodes_10000Pods
featureGates:
SchedulerQueueingHints: false
labels: [performance]
threshold: 265
params:
initNodes: 5000
measurePods: 10000
- name: 5000Nodes_10000Pods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance]
threshold: 265
params:
initNodes: 5000
measurePods: 10000
- name: SchedulingWithNodeInclusionPolicy
featureGates:
NodeInclusionPolicyInPodTopologySpread: true
defaultPodTemplatePath: ../templates/pod-with-node-inclusion-policy.yaml
workloadTemplate:
- opcode: createNodes
countParam: $normalNodes
- opcode: createNodes
nodeTemplatePath: ../templates/node-with-taint.yaml
countParam: $taintNodes
- opcode: createPods
countParam: $measurePods
collectMetrics: true
workloads:
- name: 5Nodes
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
taintNodes: 1
normalNodes: 4
measurePods: 4
- name: 5Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
taintNodes: 1
normalNodes: 4
measurePods: 4
- name: 500Nodes
labels: [performance, short]
params:
taintNodes: 100
normalNodes: 400
measurePods: 400
- name: 5000Nodes
featureGates:
SchedulerQueueingHints: false
labels: [performance, short]
threshold: 68
params:
taintNodes: 1000
normalNodes: 4000
measurePods: 4000
- name: 5000Nodes_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance, short]
threshold: 68
params:
taintNodes: 1000
normalNodes: 4000
measurePods: 4000
# This test case simulates the scheduling when many pods are gated and others are gradually deleted.
# https://github.com/kubernetes/kubernetes/issues/124384
- name: SchedulingWhileGated
defaultPodTemplatePath: ../templates/light-pod.yaml
workloadTemplate:
- opcode: createNodes
count: 1
nodeTemplatePath: ../templates/node-with-name.yaml
# Create pods that will stay gated to the end of the test.
- opcode: createPods
countParam: $gatedPods
podTemplatePath: ../templates/gated-pod.yaml
skipWaitToCompletion: true
# Wait to make sure gated pods are enqueued in scheduler.
- opcode: barrier
stageRequirement: Attempted
# Create pods that will be gradually deleted after being scheduled.
- opcode: createPods
countParam: $deletingPods
# Delete scheduled pods, which will generate many AssignedPodDelete events.
# Each of them will be processed by the scheduling queue.
# But, the scheduling throughput should only be minimally impacted by the number of gated Pods.
- opcode: deletePods
namespace: namespace-3
deletePodsPerSecond: 50
skipWaitToCompletion: true
- opcode: createPods
countParam: $measurePods
collectMetrics: true
workloads:
- name: 1Node_10GatedPods
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
gatedPods: 10
deletingPods: 10
measurePods: 10
- name: 1Node_10GatedPods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
gatedPods: 10
deletingPods: 10
measurePods: 10
- name: 1Node_10000GatedPods
featureGates:
SchedulerQueueingHints: false
labels: [performance]
threshold: 130
params:
gatedPods: 10000
deletingPods: 20000
measurePods: 20000
- name: 1Node_10000GatedPods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance]
threshold: 130
params:
gatedPods: 10000
deletingPods: 20000
measurePods: 20000
# This test case simulates the scheduling when pods selected to schedule have deletionTimestamp set.
# There was a memory leak related to this path of code fixed in:
# https://github.com/kubernetes/kubernetes/pull/126962
# Main goal of this test case is to verify if InFlightEvents is empty after the test.
- name: SchedulingDeletedPodsWithFinalizers
featureGates:
SchedulerQueueingHints: true
defaultPodTemplatePath: ../templates/light-pod.yaml
workloadTemplate:
- opcode: createNodes
countParam: $initNodes
- opcode: createPods
# Create pods with finalizers and delete them before they are scheduled.
# DeletionTimestamp field should be populated then,
# but a few pods should still be chosen into the scheduling.
countParam: $deletingPods
podTemplatePath: ../templates/pod-with-finalizer.yaml
skipWaitToCompletion: true
- opcode: deletePods
namespace: namespace-1
deletePodsPerSecond: 100
skipWaitToCompletion: true
- opcode: createPods
countParam: $measurePods
collectMetrics: true
workloads:
- name: 10Node_100DeletingPods
featureGates:
SchedulerQueueingHints: false
labels: [integration-test, short]
params:
initNodes: 10
deletingPods: 10
measurePods: 10
- name: 10Node_100DeletingPods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [integration-test, short]
params:
initNodes: 10
deletingPods: 10
measurePods: 10
- name: 1000Node_1000DeletingPods
featureGates:
SchedulerQueueingHints: false
labels: [performance, short]
params:
initNodes: 1000
deletingPods: 1000
measurePods: 1000
- name: 1000Node_1000DeletingPods_QueueingHintsEnabled
featureGates:
SchedulerQueueingHints: true
labels: [performance, short]
params:
initNodes: 1000
deletingPods: 1000
measurePods: 1000
Reference in New Issue View Git Blame Copy Permalink