From a2cd8aa5397b75dcc7bdc9fffbb9a846fefe2c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Skocze=C5=84?= Date: Thu, 25 Jul 2024 09:38:35 +0000 Subject: [PATCH 1/3] Make smaller workloads for scheduler_perf integration tests --- test/integration/scheduler_perf/README.md | 3 + .../config/performance-config.yaml | 205 ++++++++++++++++-- 2 files changed, 186 insertions(+), 22 deletions(-) diff --git a/test/integration/scheduler_perf/README.md b/test/integration/scheduler_perf/README.md index fd6baa58cb9..f1c4af4288e 100644 --- a/test/integration/scheduler_perf/README.md +++ b/test/integration/scheduler_perf/README.md @@ -113,3 +113,6 @@ Integration testing uses the same `config/performance-config.yaml` as benchmarking. By default, workloads labeled as `integration-test` are executed as part of integration testing. `-test-scheduling-label-filter` can be used to change that. + +We should make each test case with `integration-test` label very small, +so that all tests with the label should take less than 5 min to complete. diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml index 0e48e85b122..ac7840200c6 100644 --- a/test/integration/scheduler_perf/config/performance-config.yaml +++ b/test/integration/scheduler_perf/config/performance-config.yaml @@ -3,6 +3,8 @@ # - integration-test: used to select workloads that # run in pull-kubernetes-integration. Choosing those tests # is a tradeoff between code coverage and overall runtime. +# We should make each test case with integration-test label very small, +# so that all tests with the label should take less than 5 min to complete. # - performance: used to select workloads that run # in ci-benchmark-scheduler-perf. Such workloads # must run long enough (ideally, longer than 10 seconds) @@ -23,6 +25,12 @@ countParam: $measurePods collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 5 + measurePods: 10 - name: 500Nodes labels: [integration-test, fast] params: @@ -58,6 +66,12 @@ collectMetrics: true namespace: sched-1 workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 1 + measurePods: 4 - name: 500Nodes labels: [integration-test, fast] params: @@ -88,8 +102,14 @@ countParam: $measurePods collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 5 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [integration-test, fast] params: initNodes: 500 initPods: 500 @@ -121,13 +141,20 @@ persistentVolumeClaimTemplatePath: config/templates/pvc.yaml collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 5 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [integration-test, performance, fast] params: initNodes: 500 initPods: 500 measurePods: 1000 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPods: 5000 @@ -162,13 +189,20 @@ persistentVolumeClaimTemplatePath: config/templates/pvc.yaml collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 5 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [integration-test, performance, fast] params: initNodes: 500 initPods: 500 measurePods: 1000 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPods: 5000 @@ -201,13 +235,20 @@ persistentVolumeClaimTemplatePath: config/templates/pvc.yaml collectMetrics: true workloads: - - name: 500Nodes + - name: 5Nodes labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 5 + measurePods: 10 + - name: 500Nodes + labels: [integration-test, performance, fast] params: initNodes: 500 initPods: 500 measurePods: 1000 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPods: 5000 @@ -239,8 +280,14 @@ namespace: sched-1 collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 5 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [integration-test, fast] params: initNodes: 500 initPods: 500 @@ -274,13 +321,20 @@ namespace: sched-1 collectMetrics: true workloads: - - name: 500Nodes + - name: 5Nodes labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 5 + measurePods: 10 + - name: 500Nodes + labels: [integration-test, performance, fast] params: initNodes: 500 initPods: 500 measurePods: 1000 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPods: 5000 @@ -308,8 +362,14 @@ namespace: sched-1 collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 5 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [integration-test, fast] params: initNodes: 500 initPods: 500 @@ -348,6 +408,11 @@ countParam: $measurePods collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + measurePods: 10 - name: 15000Nodes labels: [performance, fast] params: @@ -369,8 +434,14 @@ countParam: $measurePods collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 5 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [integration-test, fast] params: initNodes: 500 initPods: 500 @@ -404,6 +475,12 @@ podTemplatePath: config/templates/pod-with-topology-spreading.yaml collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 10 + measurePods: 10 - name: 500Nodes labels: [integration-test, fast] params: @@ -439,13 +516,20 @@ podTemplatePath: config/templates/pod-with-preferred-topology-spreading.yaml collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 10 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [integration-test, performance, fast] params: initNodes: 500 initPods: 1000 measurePods: 1000 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPods: 5000 @@ -492,13 +576,20 @@ countParam: $measurePods collectMetrics: true workloads: - - name: 500Nodes + - name: 5Nodes labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 2 + measurePods: 10 + - name: 500Nodes + labels: [integration-test, performance, fast] params: initNodes: 500 initPods: 200 measurePods: 1000 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPods: 2000 @@ -511,7 +602,6 @@ measurePods: 5000 - name: PreemptionBasic - labels: [performance] workloadTemplate: - opcode: createNodes countParam: $initNodes @@ -523,8 +613,14 @@ podTemplatePath: config/templates/pod-high-priority.yaml collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 20 + measurePods: 5 - name: 500Nodes - labels: [fast] + labels: [performance, fast] params: initNodes: 500 initPods: 2000 @@ -539,7 +635,6 @@ # measurePods: 5000 - name: PreemptionPVs - labels: [performance] workloadTemplate: - opcode: createNodes countParam: $initNodes @@ -553,8 +648,14 @@ persistentVolumeClaimTemplatePath: config/templates/pvc.yaml collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 20 + measurePods: 5 - name: 500Nodes - labels: [fast] + labels: [performance, fast] params: initNodes: 500 initPods: 2000 @@ -581,8 +682,14 @@ podTemplatePath: config/templates/pod-default.yaml collectMetrics: true workloads: + - name: 5Nodes/2InitPods + labels: [integration-test, fast] + params: + initNodes: 5 + initPods: 2 + measurePods: 10 - name: 500Nodes/200InitPods - labels: [fast] + labels: [integration-test, fast] params: initNodes: 500 initPods: 200 @@ -622,6 +729,11 @@ podTemplatePath: config/templates//pod-default.yaml collectMetrics: true workloads: + - name: 10Nodes + labels: [integration-test, fast] + params: + initNodes: 10 + measurePods: 100 - name: 1000Nodes labels: [integration-test, fast] params: @@ -662,14 +774,22 @@ collectMetrics: true namespace: measure-ns-0 workloads: + - name: 10Nodes + labels: [integration-test, fast] + params: + initNodes: 10 + initPodsPerNamespace: 2 + initNamespaces: 2 + measurePods: 6 - name: 500Nodes - labels: [fast] + labels: [performance, fast] params: initNodes: 500 initPodsPerNamespace: 4 initNamespaces: 10 measurePods: 100 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPodsPerNamespace: 40 @@ -707,14 +827,22 @@ collectMetrics: true namespace: measure-ns-0 workloads: + - name: 10Nodes + labels: [integration-test, fast] + params: + initNodes: 10 + initPodsPerNamespace: 2 + initNamespaces: 2 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [performance, fast] params: initNodes: 500 initPodsPerNamespace: 4 initNamespaces: 10 measurePods: 100 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPodsPerNamespace: 40 @@ -755,14 +883,22 @@ collectMetrics: true namespace: measure-ns-0 workloads: + - name: 10Nodes + labels: [integration-test, fast] + params: + initNodes: 10 + initPodsPerNamespace: 2 + initNamespaces: 2 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [performance, fast] params: initNodes: 500 initPodsPerNamespace: 4 initNamespaces: 10 measurePods: 100 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPodsPerNamespace: 50 @@ -800,14 +936,22 @@ collectMetrics: true namespace: measure-ns-0 workloads: + - name: 10Nodes + labels: [integration-test, fast] + params: + initNodes: 10 + initPodsPerNamespace: 2 + initNamespaces: 2 + measurePods: 10 - name: 500Nodes - labels: [fast] + labels: [performance, fast] params: initNodes: 500 initPodsPerNamespace: 4 initNamespaces: 10 measurePods: 100 - name: 5000Nodes + labels: [performance] params: initNodes: 5000 initPodsPerNamespace: 50 @@ -835,8 +979,14 @@ countParam: $measurePods collectMetrics: true workloads: + - name: 5Nodes + labels: [integration-test, fast] + params: + taintNodes: 1 + normalNodes: 4 + measurePods: 4 - name: 500Nodes - labels: [fast] + labels: [integration-test, fast] params: taintNodes: 100 normalNodes: 400 @@ -1165,7 +1315,13 @@ countParam: $measurePods collectMetrics: true workloads: - - name: 1Node + - name: 1Node_10GatedPods + labels: [integration-test, fast] + params: + gatedPods: 10 + deletingPods: 100 + measurePods: 100 + - name: 1Node_10000GatedPods labels: [performance, fast] params: gatedPods: 10000 @@ -1192,7 +1348,12 @@ countParam: $measurePods collectMetrics: true workloads: - - name: 1000Node + - name: 1Node_10GatedPods + labels: [integration-test, fast] + params: + gatedPods: 10 + measurePods: 10 + - name: 1Node_10000GatedPods labels: [performance, fast] params: gatedPods: 10000 From 09fc3998371652f35876692edbfa32266a3495c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Skocze=C5=84?= Date: Thu, 25 Jul 2024 15:41:38 +0000 Subject: [PATCH 2/3] Add label to select short workloads in scheduler_perf tests --- test/integration/scheduler_perf/README.md | 11 +++++++---- .../scheduler_perf/config/performance-config.yaml | 7 +++++-- test/integration/scheduler_perf/scheduler_test.go | 6 ++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/test/integration/scheduler_perf/README.md b/test/integration/scheduler_perf/README.md index f1c4af4288e..0b2c07f2b26 100644 --- a/test/integration/scheduler_perf/README.md +++ b/test/integration/scheduler_perf/README.md @@ -110,9 +110,12 @@ make test-integration WHAT=./test/integration/scheduler_perf KUBE_TEST_ARGS=-use ``` Integration testing uses the same `config/performance-config.yaml` as -benchmarking. By default, workloads labeled as `integration-test` are executed -as part of integration testing. `-test-scheduling-label-filter` can be used to -change that. +benchmarking. By default, workloads labeled as `integration-test` +are executed as part of integration testing. `-test-scheduling-label-filter` can be used to +change that. -We should make each test case with `integration-test` label very small, +Running the integration tests as above will only execute the workloads labeled as `short`. + `SHORT=--short=false` variable added to the command can be used to disable this filtering. + +We should make each test case with `short` label very small, so that all tests with the label should take less than 5 min to complete. diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml index ac7840200c6..ad5ba14dde3 100644 --- a/test/integration/scheduler_perf/config/performance-config.yaml +++ b/test/integration/scheduler_perf/config/performance-config.yaml @@ -1,10 +1,13 @@ # The following labels are used in this file: # - fast: short execution time, ideally less than 30 seconds # - integration-test: used to select workloads that -# run in pull-kubernetes-integration. Choosing those tests +# run in ci-kubernetes-integration-master. Choosing those tests # is a tradeoff between code coverage and overall runtime. -# We should make each test case with integration-test label very small, +# - short: used to select workloads that +# run in pull-kubernetes-integration. +# We should make each test case with short label very small, # so that all tests with the label should take less than 5 min to complete. +# They can be run using --short=true flag. # - performance: used to select workloads that run # in ci-benchmark-scheduler-perf. Such workloads # must run long enough (ideally, longer than 10 seconds) diff --git a/test/integration/scheduler_perf/scheduler_test.go b/test/integration/scheduler_perf/scheduler_test.go index a0beb47a39f..6ec8cb6212d 100644 --- a/test/integration/scheduler_perf/scheduler_test.go +++ b/test/integration/scheduler_perf/scheduler_test.go @@ -18,6 +18,8 @@ package benchmark import ( "testing" + + "k8s.io/utils/ptr" ) func TestScheduling(t *testing.T) { @@ -29,6 +31,10 @@ func TestScheduling(t *testing.T) { t.Fatal(err) } + if testing.Short() { + testSchedulingLabelFilter = ptr.To(*testSchedulingLabelFilter + ",+short") + } + for _, tc := range testCases { t.Run(tc.Name, func(t *testing.T) { for _, w := range tc.Workloads { From 77372cf3cf16124acf882df0ae113cf632fd3311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Skocze=C5=84?= Date: Fri, 26 Jul 2024 08:53:26 +0000 Subject: [PATCH 3/3] Label short workloads in scheduler_perf tests --- test/integration/scheduler_perf/README.md | 18 ++++++++++++++---- .../config/performance-config.yaml | 18 +++++++++--------- .../scheduler_perf/scheduler_test.go | 4 +--- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/test/integration/scheduler_perf/README.md b/test/integration/scheduler_perf/README.md index 0b2c07f2b26..287ed486244 100644 --- a/test/integration/scheduler_perf/README.md +++ b/test/integration/scheduler_perf/README.md @@ -111,11 +111,21 @@ make test-integration WHAT=./test/integration/scheduler_perf KUBE_TEST_ARGS=-use Integration testing uses the same `config/performance-config.yaml` as benchmarking. By default, workloads labeled as `integration-test` -are executed as part of integration testing. `-test-scheduling-label-filter` can be used to -change that. +are executed as part of integration testing (in ci-kubernetes-integration-master job). +`-test-scheduling-label-filter` can be used to change that. +All test cases should have at least one workload labeled as `integration-test`. -Running the integration tests as above will only execute the workloads labeled as `short`. - `SHORT=--short=false` variable added to the command can be used to disable this filtering. +Running the integration tests with command above will only execute the workloads labeled as `short`. +`SHORT=--short=false` variable added to the command can be used to disable this filtering. We should make each test case with `short` label very small, so that all tests with the label should take less than 5 min to complete. +The test cases labeled as `short` are executed in pull-kubernetes-integration job. + +### Labels used by CI jobs + +| CI Job | Labels | +|----------------------------------|------------------------| +| ci-kubernetes-integration-master | integration-test | +| pull-kubernetes-integration | integration-test,short | +| ci-benchmark-scheduler-perf | performance | diff --git a/test/integration/scheduler_perf/config/performance-config.yaml b/test/integration/scheduler_perf/config/performance-config.yaml index ad5ba14dde3..f41ebe34461 100644 --- a/test/integration/scheduler_perf/config/performance-config.yaml +++ b/test/integration/scheduler_perf/config/performance-config.yaml @@ -29,7 +29,7 @@ collectMetrics: true workloads: - name: 5Nodes - labels: [integration-test, fast] + labels: [integration-test, fast, short] params: initNodes: 5 initPods: 5 @@ -70,7 +70,7 @@ namespace: sched-1 workloads: - name: 5Nodes - labels: [integration-test, fast] + labels: [integration-test, fast, short] params: initNodes: 5 initPods: 1 @@ -106,7 +106,7 @@ collectMetrics: true workloads: - name: 5Nodes - labels: [integration-test, fast] + labels: [integration-test, fast, short] params: initNodes: 5 initPods: 5 @@ -239,7 +239,7 @@ collectMetrics: true workloads: - name: 5Nodes - labels: [integration-test, fast] + labels: [integration-test, fast, short] params: initNodes: 5 initPods: 5 @@ -438,7 +438,7 @@ collectMetrics: true workloads: - name: 5Nodes - labels: [integration-test, fast] + labels: [integration-test, fast, short] params: initNodes: 5 initPods: 5 @@ -479,7 +479,7 @@ collectMetrics: true workloads: - name: 5Nodes - labels: [integration-test, fast] + labels: [integration-test, fast, short] params: initNodes: 5 initPods: 10 @@ -617,7 +617,7 @@ collectMetrics: true workloads: - name: 5Nodes - labels: [integration-test, fast] + labels: [integration-test, fast, short] params: initNodes: 5 initPods: 20 @@ -686,7 +686,7 @@ collectMetrics: true workloads: - name: 5Nodes/2InitPods - labels: [integration-test, fast] + labels: [integration-test, fast, short] params: initNodes: 5 initPods: 2 @@ -1181,7 +1181,7 @@ collectMetrics: true workloads: - name: fast - labels: [integration-test, fast] + labels: [integration-test, fast, short] params: # This testcase runs through all code paths without # taking too long overall. diff --git a/test/integration/scheduler_perf/scheduler_test.go b/test/integration/scheduler_perf/scheduler_test.go index 6ec8cb6212d..fc1dd5cb89e 100644 --- a/test/integration/scheduler_perf/scheduler_test.go +++ b/test/integration/scheduler_perf/scheduler_test.go @@ -18,8 +18,6 @@ package benchmark import ( "testing" - - "k8s.io/utils/ptr" ) func TestScheduling(t *testing.T) { @@ -32,7 +30,7 @@ func TestScheduling(t *testing.T) { } if testing.Short() { - testSchedulingLabelFilter = ptr.To(*testSchedulingLabelFilter + ",+short") + *testSchedulingLabelFilter += ",+short" } for _, tc := range testCases {