scheduler_perf: refactor to allow arbitrary workloads

Signed-off-by: Adhityaa Chandrasekar <adtac@google.com>

test/integration/scheduler_perf/config/performance-config.yaml

@@ -1,264 +1,426 @@
-- template:
-    desc: SchedulingBasic
-    initPods:
-    - podTemplatePath: config/pod-default.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-default.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [500]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 1000
-- template:
-    desc: SchedulingPodAntiAffinity
-    nodes:
-      uniqueNodeLabelStrategy:
-        labelKey: kubernetes.io/hostname
-    initPods:
-    - podTemplatePath: config/pod-with-pod-anti-affinity.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-with-pod-anti-affinity.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [100]
-    numPodsToSchedule: 400
-  - numNodes: 5000
-    numInitPods: [1000]
-    numPodsToSchedule: 1000
-- template:
-    desc: SchedulingSecrets
-    initPods:
-    - podTemplatePath: config/pod-with-secret-volume.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-with-secret-volume.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [500]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 1000
-- template:
-    desc: SchedulingInTreePVs
-    initPods:
-    - persistentVolumeTemplatePath: config/pv-aws.yaml
-      persistentVolumeClaimTemplatePath: config/pvc.yaml
-    podsToSchedule:
-      persistentVolumeTemplatePath: config/pv-aws.yaml
-      persistentVolumeClaimTemplatePath: config/pvc.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [500]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 1000
-- template:
-    desc: SchedulingMigratedInTreePVs
-    nodes:
-      nodeTemplatePath: config/node-default.yaml
-      nodeAllocatableStrategy:
-        nodeAllocatable:
-          attachable-volumes-csi-ebs.csi.aws.com: 39
-        csiNodeAllocatable:
-          ebs.csi.aws.com:
-            count: 39
-        migratedPlugins:
-        - "kubernetes.io/aws-ebs"
-    initPods:
-    - persistentVolumeTemplatePath: config/pv-aws.yaml
-      persistentVolumeClaimTemplatePath: config/pvc.yaml
-    podsToSchedule:
-      persistentVolumeTemplatePath: config/pv-aws.yaml
-      persistentVolumeClaimTemplatePath: config/pvc.yaml
-    featureGates:
-      CSIMigration: true
-      CSIMigrationAWS: true
-  params:
-  - numNodes: 500
-    numInitPods: [500]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 1000
-- template:
-    desc: SchedulingCSIPVs
-    nodes:
-      nodeTemplatePath: config/node-default.yaml
-      nodeAllocatableStrategy:
-        nodeAllocatable:
-          attachable-volumes-csi-ebs.csi.aws.com: 39
-        csiNodeAllocatable:
-          ebs.csi.aws.com:
-            count: 39
-    initPods:
-    - persistentVolumeTemplatePath: config/pv-csi.yaml
-      persistentVolumeClaimTemplatePath: config/pvc.yaml
-    podsToSchedule:
-      persistentVolumeTemplatePath: config/pv-csi.yaml
-      persistentVolumeClaimTemplatePath: config/pvc.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [500]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 1000
-- template:
-    desc: SchedulingPodAffinity
-    nodes:
-      nodeTemplatePath: config/node-default.yaml
-      labelNodePrepareStrategy:
-        labelKey: "failure-domain.beta.kubernetes.io/zone"
-        labelValues: ["zone1"]
-    initPods:
-    - podTemplatePath: config/pod-with-pod-affinity.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-with-pod-affinity.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [500]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 1000
-- template:
-    desc: SchedulingPreferredPodAffinity
-    nodes:
-      uniqueNodeLabelStrategy:
-        labelKey: kubernetes.io/hostname
-    initPods:
-    - podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [500]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 1000
-- template:
-    desc: SchedulingPreferredPodAntiAffinity
-    nodes:
-      uniqueNodeLabelStrategy:
-        labelKey: kubernetes.io/hostname
-    initPods:
-    - podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [500]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 1000
-- template:
-    desc: SchedulingNodeAffinity
-    nodes:
-      nodeTemplatePath: config/node-default.yaml
-      labelNodePrepareStrategy:
-        labelKey: "failure-domain.beta.kubernetes.io/zone"
-        labelValues: ["zone1"]
-    initPods:
-    - podTemplatePath: config/pod-with-node-affinity.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-with-node-affinity.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [500]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 1000
-- template:
-    desc: TopologySpreading
-    nodes:
-      nodeTemplatePath: config/node-default.yaml
-      labelNodePrepareStrategy:
-        labelKey: "topology.kubernetes.io/zone"
-        labelValues: ["moon-1", "moon-2", "moon-3"]
-    initPods:
-    - podTemplatePath: config/pod-default.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-with-topology-spreading.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [1000]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 2000
-- template:
-    desc: PreferredTopologySpreading
-    nodes:
-      nodeTemplatePath: config/node-default.yaml
-      labelNodePrepareStrategy:
-        labelKey: "topology.kubernetes.io/zone"
-        labelValues: ["moon-1", "moon-2", "moon-3"]
-    initPods:
-    - podTemplatePath: config/pod-default.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-with-preferred-topology-spreading.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [1000]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [5000]
-    numPodsToSchedule: 2000
-- template:
-    desc: MixedSchedulingBasePod
-    nodes:
-      nodeTemplatePath: config/node-default.yaml
-      labelNodePrepareStrategy:
-        labelKey: "topology.kubernetes.io/zone"
-        labelValues: ["zone1"]
-    initPods:
-    - podTemplatePath: config/pod-default.yaml
-    - podTemplatePath: config/pod-with-pod-affinity.yaml
-    - podTemplatePath: config/pod-with-pod-anti-affinity.yaml
-    - podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
-    - podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-default.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [200, 200, 200, 200, 200]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [2000, 2000, 2000, 2000, 2000]
-    numPodsToSchedule: 1000
-- template:
-    desc: Preemption
-    initPods:
-    - podTemplatePath: config/pod-low-priority.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-high-priority.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [2000]
-    numPodsToSchedule: 500
-  - numNodes: 5000
-    numInitPods: [20000]
-    numPodsToSchedule: 5000
-- template:
-    desc: Unschedulable
-    skipWaitUntilInitPodsScheduled: true
-    initPods:
-    - podTemplatePath: config/pod-large-cpu.yaml
-    podsToSchedule:
-      podTemplatePath: config/pod-default.yaml
-  params:
-  - numNodes: 500
-    numInitPods: [200]
-    numPodsToSchedule: 1000
-  - numNodes: 5000
-    numInitPods: [200]
-    numPodsToSchedule: 5000
-  - numNodes: 5000
-    numInitPods: [2000]
-    numPodsToSchedule: 5000
+- name: SchedulingBasic
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-default.yaml
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-default.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 500
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 1000
+      measurePods: 1000
+
+- name: SchedulingPodAntiAffinity
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    uniqueNodeLabelStrategy:
+      labelKey: kubernetes.io/hostname
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-pod-anti-affinity.yaml
+    namespace: sched-setup
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-with-pod-anti-affinity.yaml
+    collectMetrics: true
+    namespace: sched-test
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 100
+      measurePods: 400
+  - name: 5000Nodes
+    params:
+      initNodes: 500
+      initPods: 100
+      measurePods: 400
+
+- name: SchedulingSecrets
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-secret-volume.yaml
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-with-secret-volume.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 500
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 1000
+
+- name: SchedulingInTreePVs
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+  - opcode: createPods
+    countParam: $initPods
+    persistentVolumeTemplatePath: config/pv-aws.yaml
+    persistentVolumeClaimTemplatePath: config/pvc.yaml
+  - opcode: createPods
+    countParam: $measurePods
+    persistentVolumeTemplatePath: config/pv-aws.yaml
+    persistentVolumeClaimTemplatePath: config/pvc.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 500
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 1000
+
+- name: SchedulingMigratedInTreePVs
+  featureGates:
+    CSIMigration: true
+    CSIMigrationAWS: true
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    nodeTemplatePath: config/node-default.yaml
+    nodeAllocatableStrategy:
+      nodeAllocatable:
+        attachable-volumes-csi-ebs.csi.aws.com: "39"
+      csiNodeAllocatable:
+        ebs.csi.aws.com:
+          count: 39
+      migratedPlugins:
+      - "kubernetes.io/aws-ebs"
+  - opcode: createPods
+    countParam: $initPods
+    persistentVolumeTemplatePath: config/pv-aws.yaml
+    persistentVolumeClaimTemplatePath: config/pvc.yaml
+  - opcode: createPods
+    countParam: $measurePods
+    persistentVolumeTemplatePath: config/pv-aws.yaml
+    persistentVolumeClaimTemplatePath: config/pvc.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 500
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 1000
+
+- name: SchedulingCSIPVs
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    nodeTemplatePath: config/node-default.yaml
+    nodeAllocatableStrategy:
+      nodeAllocatable:
+        attachable-volumes-csi-ebs.csi.aws.com: "39"
+      csiNodeAllocatable:
+        ebs.csi.aws.com:
+          count: 39
+  - opcode: createPods
+    countParam: $initPods
+    persistentVolumeTemplatePath: config/pv-csi.yaml
+    persistentVolumeClaimTemplatePath: config/pvc.yaml
+  - opcode: createPods
+    countParam: $measurePods
+    persistentVolumeTemplatePath: config/pv-csi.yaml
+    persistentVolumeClaimTemplatePath: config/pvc.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 500
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 1000
+
+- name: SchedulingPodAffinity
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    nodeTemplatePath: config/node-default.yaml
+    labelNodePrepareStrategy:
+      labelKey: "failure-domain.beta.kubernetes.io/zone"
+      labelValues: ["zone1"]
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-pod-affinity.yaml
+    namespace: sched-setup
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-with-pod-affinity.yaml
+    namespace: sched-test
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 500
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 1000
+
+- name: SchedulingPreferredPodAffinity
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    uniqueNodeLabelStrategy:
+      labelKey: kubernetes.io/hostname
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
+    namespace: sched-setup
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
+    namespace: sched-test
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 500
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 1000
+
+- name: SchedulingPreferredPodAntiAffinity
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    uniqueNodeLabelStrategy:
+      labelKey: kubernetes.io/hostname
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
+    namespace: sched-setup
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
+    namespace: sched-test
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 500
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 1000
+
+- name: SchedulingNodeAffinity
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    nodeTemplatePath: config/node-default.yaml
+    labelNodePrepareStrategy:
+      labelKey: "failure-domain.beta.kubernetes.io/zone"
+      labelValues: ["zone1"]
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-node-affinity.yaml
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-with-node-affinity.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 500
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 1000
+
+- name: TopologySpreading
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    nodeTemplatePath: config/node-default.yaml
+    labelNodePrepareStrategy:
+      labelKey: "topology.kubernetes.io/zone"
+      labelValues: ["moon-1", "moon-2", "moon-3"]
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-default.yaml
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-with-topology-spreading.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 1000
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 2000
+
+- name: PreferredTopologySpreading
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    nodeTemplatePath: config/node-default.yaml
+    labelNodePrepareStrategy:
+      labelKey: "topology.kubernetes.io/zone"
+      labelValues: ["moon-1", "moon-2", "moon-3"]
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-default.yaml
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-with-preferred-topology-spreading.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 1000
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 5000
+      measurePods: 2000
+
+- name: MixedSchedulingBasePod
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    nodeTemplatePath: config/node-default.yaml
+    labelNodePrepareStrategy:
+      labelKey: "topology.kubernetes.io/zone"
+      labelValues: ["zone1"]
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-default.yaml
+    namespace: sched-setup
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-pod-affinity.yaml
+    namespace: sched-setup
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-pod-anti-affinity.yaml
+    namespace: sched-setup
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
+    namespace: sched-setup
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
+    namespace: sched-setup
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-default.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 200
+      measurePods: 1000
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 2000
+      measurePods: 1000
+
+- name: Preemption
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-low-priority.yaml
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-high-priority.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes
+    params:
+      initNodes: 500
+      initPods: 2000
+      measurePods: 500
+  - name: 5000Nodes
+    params:
+      initNodes: 5000
+      initPods: 20000
+      measurePods: 5000
+
+- name: Unschedulable
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+  - opcode: createPods
+    countParam: $initPods
+    podTemplatePath: config/pod-large-cpu.yaml
+    skipWaitToCompletion: true
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/pod-default.yaml
+    collectMetrics: true
+  workloads:
+  - name: 500Nodes/200InitPods
+    params:
+      initNodes: 500
+      initPods: 200
+      measurePods: 1000
+  - name: 5000Nodes/200InitPods
+    params:
+      initNodes: 5000
+      initPods: 200
+      measurePods: 5000
+  - name: 5000Nodes/2000InitPods
+    params:
+      initNodes: 5000
+      initPods: 2000
+      measurePods: 5000