From 98e4892b84065636e88780207e15e16364e4a8e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Skocze=C5=84?= Date: Thu, 3 Oct 2024 07:02:20 +0000 Subject: [PATCH] Add scheduler_perf test case for pod update events handling --- .../podupdate-node-unschedulable.yaml | 15 +++ .../podupdate-node-with-taint.yaml | 17 +++ .../podupdate-pod-blocker-update.yaml | 13 ++ .../event_handling/podupdate-pod-blocker.yaml | 15 +++ .../podupdate-pod-interpodaffinity.yaml | 19 +++ .../podupdate-pod-noderesources.yaml | 14 ++ ...odupdate-pod-nodeunschedulable-update.yaml | 14 ++ .../podupdate-pod-nodeunschedulable.yaml | 10 ++ .../podupdate-pod-podtopologyspread.yaml | 19 +++ .../podupdate-pod-schedulinggates-update.yaml | 10 ++ .../podupdate-pod-schedulinggates.yaml | 12 ++ .../podupdate-pod-tainttoleration-update.yaml | 14 ++ .../podupdate-pod-tainttoleration.yaml | 10 ++ .../config/performance-config.yaml | 121 ++++++++++++++++++ 14 files changed, 303 insertions(+) create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-node-unschedulable.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-node-with-taint.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-blocker-update.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-blocker.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-interpodaffinity.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-noderesources.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-nodeunschedulable-update.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-nodeunschedulable.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-podtopologyspread.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-schedulinggates-update.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-schedulinggates.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-tainttoleration-update.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/podupdate-pod-tainttoleration.yaml diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-node-unschedulable.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-node-unschedulable.yaml new file mode 100644 index 00000000000..3e6aa7cc87a --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-node-unschedulable.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Node +metadata: + name: unschedulable-node +spec: + unschedulable: true +status: + capacity: + pods: "90000" + cpu: "4" + memory: 32Gi + conditions: + - status: "True" + type: Ready + phase: Running diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-node-with-taint.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-node-with-taint.yaml new file mode 100644 index 00000000000..ce3efb426a4 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-node-with-taint.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Node +metadata: + name: node-with-taint +spec: + taints: + - key: toleration + effect: NoSchedule +status: + capacity: + pods: "90000" + cpu: "4" + memory: 32Gi + conditions: + - status: "True" + type: Ready + phase: Running diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-blocker-update.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-blocker-update.yaml new file mode 100644 index 00000000000..f8209a057ca --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-blocker-update.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-blocker-{{ .Index }} +spec: + containers: + - image: registry.k8s.io/pause:3.10 + name: pause + resources: + requests: + cpu: 0.0001 + memory: 1Mi + nodeName: scheduler-perf-node diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-blocker.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-blocker.yaml new file mode 100644 index 00000000000..e484475e5ce --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-blocker.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-blocker-{{ .Index }} + labels: + color: green-{{ .Index }} + topology: blue +spec: + containers: + - image: registry.k8s.io/pause:3.10 + name: pause + resources: + requests: + cpu: 0.0001 + memory: {{ div 30000 .Count }}Mi diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-interpodaffinity.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-interpodaffinity.yaml new file mode 100644 index 00000000000..50e6f7ba7ca --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-interpodaffinity.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: pod-interpodaffinity- + labels: + color: green-{{ .Index }} + type: unsched +spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + color: green-{{ .Index }} + topologyKey: kubernetes.io/hostname + namespaces: [blocker] + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-noderesources.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-noderesources.yaml new file mode 100644 index 00000000000..428e0b3c7b1 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-noderesources.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-noderesources-{{ .Index }} + labels: + type: unsched +spec: + containers: + - image: registry.k8s.io/pause:3.10 + name: pause + resources: + requests: + cpu: 0.0001 + memory: {{ div 30000 .Count }}Mi diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-nodeunschedulable-update.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-nodeunschedulable-update.yaml new file mode 100644 index 00000000000..726f17b5aef --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-nodeunschedulable-update.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-nodeunschedulable-{{ .Index }} + labels: + type: unsched +spec: + tolerations: + - key: node.kubernetes.io/unschedulable + operator: Exists + effect: NoSchedule + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-nodeunschedulable.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-nodeunschedulable.yaml new file mode 100644 index 00000000000..204f31f3451 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-nodeunschedulable.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-nodeunschedulable-{{ .Index }} + labels: + type: unsched +spec: + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-podtopologyspread.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-podtopologyspread.yaml new file mode 100644 index 00000000000..ac6955024a3 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-podtopologyspread.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: pod-podtopologyspread- + labels: + type: unsched + topology: blue +spec: + topologySpreadConstraints: + - maxSkew: {{ .Count }} + minDomains: 100 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + topology: blue + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-schedulinggates-update.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-schedulinggates-update.yaml new file mode 100644 index 00000000000..f49138dc504 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-schedulinggates-update.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-schedulinggates-{{ .Index }} + labels: + type: unsched +spec: + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-schedulinggates.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-schedulinggates.yaml new file mode 100644 index 00000000000..a7c5f764417 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-schedulinggates.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-schedulinggates-{{ .Index }} + labels: + type: unsched +spec: + schedulingGates: + - name: test.k8s.io/hold + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-tainttoleration-update.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-tainttoleration-update.yaml new file mode 100644 index 00000000000..59e62da1faa --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-tainttoleration-update.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-tainttoleration-{{ .Index }} + labels: + type: unsched +spec: + tolerations: + - key: toleration + operator: Exists + effect: NoSchedule + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/podupdate-pod-tainttoleration.yaml b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-tainttoleration.yaml new file mode 100644 index 00000000000..ffc6f4d7689 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/podupdate-pod-tainttoleration.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: pod-tainttoleration-{{ .Index }} + labels: + type: unsched +spec: + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml index b1ec9e4bcdd..403cd61e21e 100644 --- a/test/integration/scheduler_perf/config/performance-config.yaml +++ b/test/integration/scheduler_perf/config/performance-config.yaml @@ -1663,3 +1663,124 @@ initNodes: 50 blockerPods: 480 # Must be slightly below initNodes * 10 to be stable measurePods: 500 # Must be initNodes * 10 + +# This test case is used to measure the performance of queuing hints when handling the pod update events: +# UpdatePodLabel, UpdatePodScaleDown, UpdatePodTolerations and UpdatePodSchedulingGatesEliminated. +# It has a few stages, but general idea is to make a node and block some pods on it +# or to create additional blocker pods that will prevent the other ones from being scheduled. +# Then, updating the blocker pods or the unschedulable pods themselves generate cluster events, +# that through QHints make the pods schedulable. +# Plugins covered: InterPodAffinity, NodeResources, NodeUnschedulable, PodTopologySpread, SchedulingGates and TaintToleration. +- name: EventHandlingPodUpdate + featureGates: + SchedulerQueueingHints: true + InPlacePodVerticalScaling: true + workloadTemplate: + # Collect metrics from all createPods ops that initially create unschedulable pods (type: unsched). + - opcode: startCollectingMetrics + namespaces: [nodeunschedulable, tainttoleration, blocker, interpodaffinity, noderesources, schedulinggates] + labelSelector: + type: unsched + # Create one unschedulable node. + - opcode: createNodes + count: 1 + nodeTemplatePath: config/event_handling/podupdate-node-unschedulable.yaml + # Created pods blocked using NodeUnschedulable plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/podupdate-pod-nodeunschedulable.yaml + skipWaitToCompletion: true + namespace: nodeunschedulable + # Create one node with NoSchedule taint. + - opcode: createNodes + count: 1 + nodeTemplatePath: config/event_handling/podupdate-node-with-taint.yaml + # Created pods blocked using TaintToleration plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/podupdate-pod-tainttoleration.yaml + skipWaitToCompletion: true + namespace: tainttoleration + # Wait for unschedulable pods to be processed by the scheduler. + - opcode: barrier + stageRequirement: Attempted + # Update pods blocked using NodeUnschedulable plugin to make them schedulable. + - opcode: updateAny + countParam: $measurePods + templatePath: config/event_handling/podupdate-pod-nodeunschedulable-update.yaml + updatePerSecond: 100 + namespace: nodeunschedulable + # Update pods blocked using TaintToleration plugin to make them schedulable. + - opcode: updateAny + countParam: $measurePods + templatePath: config/event_handling/podupdate-pod-tainttoleration-update.yaml + updatePerSecond: 100 + namespace: tainttoleration + # Wait for NodeUnschedulable and TaintToleration pods to be scheduled. + - opcode: barrier + # Create schedulable nodes. + - opcode: createNodes + count: 1 + nodeTemplatePath: config/templates/node-with-name.yaml + # Create pods that will block other pods from being scheduled. + # They'll block using InterPodAffinity, NodeResources and PodTopologySpread plugins. + # All blocker pods are scheduled before proceeding. + - opcode: createPods + countParam: $blockerPods + podTemplatePath: config/event_handling/podupdate-pod-blocker.yaml + namespace: blocker + # Created pods blocked using InterPodAffinity plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/podupdate-pod-interpodaffinity.yaml + skipWaitToCompletion: true + namespace: interpodaffinity + # Created pods blocked using NodeResources plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/podupdate-pod-noderesources.yaml + skipWaitToCompletion: true + namespace: noderesources + # Created pods blocked using PodTopologySpread plugin. + # Count has to match $blockerPods as pod uses it as a maxSkew value, + # that must be equal to number of blocker pods. + - opcode: createPods + countParam: $blockerPods + podTemplatePath: config/event_handling/podupdate-pod-podtopologyspread.yaml + skipWaitToCompletion: true + namespace: blocker + # Created pods blocked using SchedulingGates plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/podupdate-pod-schedulinggates.yaml + skipWaitToCompletion: true + namespace: schedulinggates + # Wait for unschedulable pods to be processed by the scheduler. + - opcode: barrier + stageRequirement: Attempted + labelSelector: + type: unsched + # Update blocker pods' labels and scale down their resource requests + # to make the unschedulable pods schedulable. + - opcode: updateAny + countParam: $blockerPods + templatePath: config/event_handling/podupdate-pod-blocker-update.yaml + updatePerSecond: 100 + namespace: blocker + # Update pods blocked by SchedulingGates by removing the gate from themselves. + - opcode: updateAny + countParam: $measurePods + templatePath: config/event_handling/podupdate-pod-schedulinggates-update.yaml + updatePerSecond: 100 + namespace: schedulinggates + # Wait for previously unschedulable pods to be scheduled. + - opcode: barrier + labelSelector: + type: unsched + - opcode: stopCollectingMetrics + workloads: + - name: 1Node_1000Pods + labels: [performance, short] + params: + blockerPods: 1000 + measurePods: 1000