From 25850caf8a691e0e960920cede8163cf6400c57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Skocze=C5=84?= Date: Tue, 24 Sep 2024 14:33:06 +0000 Subject: [PATCH] Add scheduler_perf test case for NodeAdd event handling --- .../nodeadd-node-high-capacity.yaml | 17 ++++ .../nodeadd-node-low-capacity.yaml | 17 ++++ .../nodeadd-node-unschedulable.yaml | 23 +++++ .../nodeadd-node-with-labels.yaml | 21 ++++ .../nodeadd-node-with-taint.yaml | 17 ++++ .../nodeadd-pod-interpodaffinity.yaml | 21 ++++ .../nodeadd-pod-nodeaffinity.yaml | 21 ++++ .../nodeadd-pod-noderesources.yaml | 16 +++ .../nodeadd-pod-nodeunschedulable.yaml | 12 +++ .../nodeadd-pod-podtopologyspread.yaml | 22 +++++ .../nodeadd-pod-tainttoleration.yaml | 12 +++ .../config/performance-config.yaml | 97 +++++++++++++++++++ 12 files changed, 296 insertions(+) create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-node-high-capacity.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-node-low-capacity.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-node-unschedulable.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-labels.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-taint.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-pod-interpodaffinity.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeaffinity.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-pod-noderesources.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeunschedulable.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-pod-podtopologyspread.yaml create mode 100644 test/integration/scheduler_perf/config/event_handling/nodeadd-pod-tainttoleration.yaml diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-high-capacity.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-high-capacity.yaml new file mode 100644 index 00000000000..515e6e5d3e6 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-high-capacity.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Node +metadata: + generateName: node-high-capacity- +spec: + taints: + - key: allow + effect: NoSchedule +status: + capacity: + pods: "3000" + cpu: "200" + memory: 200Gi + conditions: + - status: "True" + type: Ready + phase: Running diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-low-capacity.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-low-capacity.yaml new file mode 100644 index 00000000000..226317c0fa9 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-low-capacity.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Node +metadata: + generateName: node-low-capacity- +spec: + taints: + - key: allow + effect: NoSchedule +status: + capacity: + pods: "3000" + cpu: "0.1" + memory: 100Mi + conditions: + - status: "True" + type: Ready + phase: Running diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-unschedulable.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-unschedulable.yaml new file mode 100644 index 00000000000..97b4bc0a8b8 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-unschedulable.yaml @@ -0,0 +1,23 @@ +# Node unschedulable is similar to node unblocker but has unschedulable: true. +apiVersion: v1 +kind: Node +metadata: + generateName: node-unschedulable- + labels: + topology.kubernetes.io/zone: zone1 + affinity: allow + topology: allow +spec: + unschedulable: true + taints: + - key: allow + effect: NoSchedule +status: + capacity: + pods: "3000" + cpu: "4" + memory: 32Gi + conditions: + - status: "True" + type: Ready + phase: Running diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-labels.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-labels.yaml new file mode 100644 index 00000000000..1b3e269b650 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-labels.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Node +metadata: + generateName: node-unblocker- + labels: + topology.kubernetes.io/zone: zone1 + affinity: allow + topology: allow +spec: + taints: + - key: allow + effect: NoSchedule +status: + capacity: + pods: "3000" + cpu: "4" + memory: 32Gi + conditions: + - status: "True" + type: Ready + phase: Running diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-taint.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-taint.yaml new file mode 100644 index 00000000000..f7b1a27c97b --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-node-with-taint.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Node +metadata: + generateName: node-with-taint- +spec: + taints: + - key: toleration + effect: NoSchedule +status: + capacity: + pods: "3000" + cpu: "4" + memory: 32Gi + conditions: + - status: "True" + type: Ready + phase: Running diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-interpodaffinity.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-interpodaffinity.yaml new file mode 100644 index 00000000000..5649614ddcc --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-interpodaffinity.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: pod-unsched- + labels: + color: green +spec: + tolerations: + - key: allow + operator: Exists + effect: NoSchedule + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + color: green + topologyKey: topology.kubernetes.io/zone + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeaffinity.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeaffinity.yaml new file mode 100644 index 00000000000..8aec41c3f12 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeaffinity.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: pod-unsched- +spec: + tolerations: + - key: allow + operator: Exists + effect: NoSchedule + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: affinity + operator: In + values: + - allow + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-noderesources.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-noderesources.yaml new file mode 100644 index 00000000000..740eaf83bf7 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-noderesources.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: pod-unsched- +spec: + tolerations: + - key: allow + operator: Exists + effect: NoSchedule + containers: + - image: registry.k8s.io/pause:3.10 + name: pause + resources: + requests: + cpu: 0.2 + memory: 200Mi diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeunschedulable.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeunschedulable.yaml new file mode 100644 index 00000000000..22ecbdae450 --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-nodeunschedulable.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: pod-unsched- +spec: + tolerations: + - key: allow + operator: Exists + effect: NoSchedule + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-podtopologyspread.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-podtopologyspread.yaml new file mode 100644 index 00000000000..70f6fb486cc --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-podtopologyspread.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: pod-unsched- + labels: + color: blue +spec: + tolerations: + - key: allow + operator: Exists + effect: NoSchedule + topologySpreadConstraints: + - maxSkew: 1000 + minDomains: 10000 + topologyKey: topology + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + color: blue + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-tainttoleration.yaml b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-tainttoleration.yaml new file mode 100644 index 00000000000..9f105e300fe --- /dev/null +++ b/test/integration/scheduler_perf/config/event_handling/nodeadd-pod-tainttoleration.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + generateName: pod-unsched- +spec: + tolerations: + - key: toleration + operator: Exists + effect: NoSchedule + containers: + - image: registry.k8s.io/pause:3.10 + name: pause diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml index 403cd61e21e..90c089ee7b9 100644 --- a/test/integration/scheduler_perf/config/performance-config.yaml +++ b/test/integration/scheduler_perf/config/performance-config.yaml @@ -1784,3 +1784,100 @@ params: blockerPods: 1000 measurePods: 1000 + +# This test case is used to measure the performance of queuing hints when handling the NodeAdd events. +# First, an unschedulable node is created, which prevents any pod from being scheduled on it. +# Then multiple types of pods are created, and each group is filtered by different plugin. +# Next, nodes are created where previously unscheduled pods can be scheduled. +# The test case is divided into several stages to make sure that the pods are filtered by a specific plugin. +# Plugins covered: InterPodAffinity, NodeAffinity, NodeResources, NodeUnschedulable, PodTopologySpread and TaintToleration. +- name: EventHandlingNodeAdd + featureGates: + SchedulerQueueingHints: true + workloadTemplate: + # Collect metrics from all createPods ops. + - opcode: startCollectingMetrics + name: unschedPods + namespaces: [nodeunschedulable, noderesources, interpodaffinity, nodeaffinity, podtopologyspread, tainttoleration] + # Create one unschedulable node. + - opcode: createNodes + count: 1 + nodeTemplatePath: config/event_handling/nodeadd-node-unschedulable.yaml + # Created pods blocked using NodeUnschedulable plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/nodeadd-pod-nodeunschedulable.yaml + skipWaitToCompletion: true + namespace: nodeunschedulable + # Wait for unschedulable pods to be processed by the scheduler. + - opcode: barrier + stageRequirement: Attempted + # Create schedulable node with low capacity. + - opcode: createNodes + count: 1 + nodeTemplatePath: config/event_handling/nodeadd-node-low-capacity.yaml + # Created pods blocked using NodeResources plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/nodeadd-pod-noderesources.yaml + skipWaitToCompletion: true + namespace: noderesources + # Wait for unschedulable pods to be processed by the scheduler. + - opcode: barrier + stageRequirement: Attempted + # Create nodes that will have enough resource capacity for pods blocked by NodeResources plugin. + # These nodes will still block the next pods from being scheduled. + - opcode: createNodes + countParam: $nodes + nodeTemplatePath: config/event_handling/nodeadd-node-high-capacity.yaml + # Wait on barrier for NodeUnschedulable and NodeResources pods to be scheduled. + - opcode: barrier + # Created pods blocked using InterPodAffinity plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/nodeadd-pod-interpodaffinity.yaml + skipWaitToCompletion: true + namespace: interpodaffinity + # Created pods blocked using NodeAffinity plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/nodeadd-pod-nodeaffinity.yaml + skipWaitToCompletion: true + namespace: nodeaffinity + # Created pods blocked using PodTopologySpread plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/nodeadd-pod-podtopologyspread.yaml + skipWaitToCompletion: true + namespace: podtopologyspread + # Wait for unschedulable pods to be processed by the scheduler. + - opcode: barrier + stageRequirement: Attempted + # Create nodes that will unblock most of the unschedulable pods. + - opcode: createNodes + countParam: $nodes + nodeTemplatePath: config/event_handling/nodeadd-node-with-labels.yaml + # Wait on barrier for InterPodAffinity, NodeAffinity and PodTopologySpread pods to be scheduled. + - opcode: barrier + # Created pods blocked using TaintToleration plugin. + - opcode: createPods + countParam: $measurePods + podTemplatePath: config/event_handling/nodeadd-pod-tainttoleration.yaml + skipWaitToCompletion: true + namespace: tainttoleration + # Wait for unschedulable pods to be processed by the scheduler. + - opcode: barrier + stageRequirement: Attempted + # Create nodes that will unblock pods filtered out by TaintToleration plugin. + - opcode: createNodes + countParam: $nodes + nodeTemplatePath: config/event_handling/nodeadd-node-with-taint.yaml + # Wait on barrier for TaintToleration pods to be scheduled. + - opcode: barrier + - opcode: stopCollectingMetrics + workloads: + - name: 100Nodes_500Pods + labels: [performance, short] + params: + nodes: 100 + measurePods: 1000 # Must be initNodes * 10