diff --git a/test/integration/scheduler_perf/scheduler_perf_test.go b/test/integration/scheduler_perf/scheduler_perf_test.go index e141c227366..e49df2979ee 100644 --- a/test/integration/scheduler_perf/scheduler_perf_test.go +++ b/test/integration/scheduler_perf/scheduler_perf_test.go @@ -52,23 +52,30 @@ import ( "sigs.k8s.io/yaml" ) -const ( - configFile = "config/performance-config.yaml" - createNodesOpcode = "createNodes" - createNamespacesOpcode = "createNamespaces" - createPodsOpcode = "createPods" - createPodSetsOpcode = "createPodSets" - churnOpcode = "churn" - barrierOpcode = "barrier" - sleepOpcode = "sleep" - extensionPointsLabelName = "extension_point" +type operationCode string +const ( + createNodesOpcode operationCode = "createNodes" + createNamespacesOpcode operationCode = "createNamespaces" + createPodsOpcode operationCode = "createPods" + createPodSetsOpcode operationCode = "createPodSets" + churnOpcode operationCode = "churn" + barrierOpcode operationCode = "barrier" + sleepOpcode operationCode = "sleep" +) + +const ( // Two modes supported in "churn" operator. - // Recreate creates a number of API objects and then delete them, and repeat the iteration. - Recreate = "recreate" // Create continuously create API objects without deleting them. Create = "create" + // Recreate creates a number of API objects and then delete them, and repeat the iteration. + Recreate = "recreate" +) + +const ( + configFile = "config/performance-config.yaml" + extensionPointsLabelName = "extension_point" ) var ( @@ -84,16 +91,18 @@ var ( } ) -// testCase defines a set of test cases that intend to test the performance of +// testCase defines a set of test cases that intends to test the performance of // similar workloads of varying sizes with shared overall settings such as // feature gates and metrics collected. type testCase struct { // Name of the testCase. Name string - // Feature gates to set before running the test. Optional. + // Feature gates to set before running the test. + // Optional FeatureGates map[featuregate.Feature]bool - // List of metrics to collect. Optional, defaults to + // List of metrics to collect. Defaults to // defaultMetricsCollectorConfig if unspecified. + // Optional MetricsCollectorConfig *metricsCollectorConfig // Template for sequence of ops that each workload must follow. Each op will // be executed serially one after another. Each element of the list must be @@ -101,10 +110,12 @@ type testCase struct { WorkloadTemplate []op // List of workloads to run under this testCase. Workloads []*workload - // SchedulerConfigFile is the path of scheduler configuration - SchedulerConfigFile string - // Default path to spec file describing the pods to create. Optional. + // SchedulerConfigPath is the path of scheduler configuration + // Optional + SchedulerConfigPath *string + // Default path to spec file describing the pods to create. // This path can be overridden in createPodsOp by setting PodTemplatePath . + // Optional DefaultPodTemplatePath *string } @@ -253,15 +264,17 @@ func isValidParameterizable(val string) bool { // createNodesOp defines an op where nodes are created as a part of a workload. type createNodesOp struct { // Must be "createNodes". - Opcode string + Opcode operationCode // Number of nodes to create. Parameterizable through CountParam. Count int // Template parameter for Count. CountParam string - // Path to spec file describing the nodes to create. Optional. + // Path to spec file describing the nodes to create. + // Optional NodeTemplatePath *string - // At most one of the following strategies can be defined. Optional, defaults + // At most one of the following strategies can be defined. Defaults // to TrivialNodePrepareStrategy if unspecified. + // Optional NodeAllocatableStrategy *testutils.NodeAllocatableStrategy LabelNodePrepareStrategy *testutils.LabelNodePrepareStrategy UniqueNodeLabelStrategy *testutils.UniqueNodeLabelStrategy @@ -297,7 +310,7 @@ func (cno createNodesOp) patchParams(w *workload) (realOp, error) { // createNamespacesOp defines an op for creating namespaces type createNamespacesOp struct { // Must be "createNamespaces". - Opcode string + Opcode operationCode // Name prefix of the Namespace. The format is "-", where number is // between 0 and count-1. Prefix string @@ -305,7 +318,8 @@ type createNamespacesOp struct { Count int // Template parameter for Count. Takes precedence over Count if both set. CountParam string - // Path to spec file describing the Namespaces to create. Optional. + // Path to spec file describing the Namespaces to create. + // Optional NamespaceTemplatePath *string } @@ -341,7 +355,7 @@ func (cmo createNamespacesOp) patchParams(w *workload) (realOp, error) { // continue asynchronously. type createPodsOp struct { // Must be "createPods". - Opcode string + Opcode operationCode // Number of pods to schedule. Parameterizable through CountParam. Count int // Template parameter for Count. @@ -350,16 +364,20 @@ type createPodsOp struct { // Optional. Both CollectMetrics and SkipWaitToCompletion cannot be true at // the same time for a particular createPodsOp. CollectMetrics bool - // Namespace the pods should be created in. Optional, defaults to a unique + // Namespace the pods should be created in. Defaults to a unique // namespace of the format "namespace-". + // Optional Namespace *string - // Path to spec file describing the pods to schedule. Optional. + // Path to spec file describing the pods to schedule. // If nil, DefaultPodTemplatePath will be used. + // Optional PodTemplatePath *string - // Whether or not to wait for all pods in this op to get scheduled. Optional, - // defaults to false. + // Whether or not to wait for all pods in this op to get scheduled. + // Defaults to false if not specified. + // Optional SkipWaitToCompletion bool - // Persistent volume settings for the pods to be scheduled. Optional. + // Persistent volume settings for the pods to be scheduled. + // Optional PersistentVolumeTemplatePath *string PersistentVolumeClaimTemplatePath *string } @@ -397,10 +415,10 @@ func (cpo createPodsOp) patchParams(w *workload) (realOp, error) { return &cpo, (&cpo).isValid(false) } -// createPodSetsOp defines an op where a set of createPodsOp is created each in a unique namespace. +// createPodSetsOp defines an op where a set of createPodsOps is created in each unique namespace. type createPodSetsOp struct { // Must be "createPodSets". - Opcode string + Opcode operationCode // Number of sets to create. Count int // Template parameter for Count. @@ -442,7 +460,7 @@ func (cpso createPodSetsOp) patchParams(w *workload) (realOp, error) { // churnOp defines an op where services are created as a part of a workload. type churnOp struct { // Must be "churnOp". - Opcode string + Opcode operationCode // Value must be one of the followings: // - recreate. In this mode, API objects will be created for N cycles, and then // deleted in the next N cycles. N is specified by the "Number" field. @@ -454,8 +472,9 @@ type churnOp struct { Number int // Intervals of churning. Defaults to 500 millisecond. IntervalMilliseconds int64 - // Namespace the churning objects should be created in. Optional, defaults to a unique + // Namespace the churning objects should be created in. Defaults to a unique // namespace of the format "namespace-". + // Optional Namespace *string // Path of API spec files. TemplatePaths []string @@ -493,7 +512,7 @@ func (co churnOp) patchParams(w *workload) (realOp, error) { // were scheduled with SkipWaitToCompletion set to true. type barrierOp struct { // Must be "barrier". - Opcode string + Opcode operationCode // Namespaces to block on. Empty array or not specifying this field signifies // that the barrier should block on all namespaces. Namespaces []string @@ -518,14 +537,14 @@ func (bo barrierOp) patchParams(w *workload) (realOp, error) { // This is useful in simulating workloads that require some sort of time-based synchronisation. type sleepOp struct { // Must be "sleep". - Opcode string + Opcode operationCode // duration of sleep. Duration time.Duration } func (so *sleepOp) UnmarshalJSON(data []byte) (err error) { var tmp struct { - Opcode string + Opcode operationCode Duration string } if err = json.Unmarshal(data, &tmp); err != nil { @@ -627,8 +646,8 @@ func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem { defer cancel() var cfg *config.KubeSchedulerConfiguration var err error - if len(tc.SchedulerConfigFile) != 0 { - cfg, err = loadSchedulerConfig(tc.SchedulerConfigFile) + if tc.SchedulerConfigPath != nil { + cfg, err = loadSchedulerConfig(*tc.SchedulerConfigPath) if err != nil { b.Fatalf("error loading scheduler config file: %v", err) } @@ -815,7 +834,26 @@ func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem { ticker := time.NewTicker(time.Duration(interval) * time.Millisecond) defer ticker.Stop() - if concreteOp.Mode == Recreate { + switch concreteOp.Mode { + case Create: + go func() { + count, threshold := 0, concreteOp.Number + if threshold == 0 { + threshold = math.MaxInt32 + } + for count < threshold { + select { + case <-ticker.C: + for i := range churnFns { + churnFns[i]("") + } + count++ + case <-ctx.Done(): + return + } + } + }() + case Recreate: go func() { retVals := make([][]string, len(churnFns)) // For each churn function, instantiate a slice of strings with length "concreteOp.Number". @@ -836,24 +874,6 @@ func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem { } } }() - } else if concreteOp.Mode == Create { - go func() { - count, threshold := 0, concreteOp.Number - if threshold == 0 { - threshold = math.MaxInt32 - } - for count < threshold { - select { - case <-ticker.C: - for i := range churnFns { - churnFns[i]("") - } - count++ - case <-ctx.Done(): - return - } - } - }() } case *barrierOp: @@ -1032,7 +1052,7 @@ func getUnstructuredFromFile(path string) (*unstructured.Unstructured, *schema.G func getTestCases(path string) ([]*testCase, error) { testCases := make([]*testCase, 0) if err := getSpecFromFile(&path, &testCases); err != nil { - return nil, fmt.Errorf("parsing test cases: %w", err) + return nil, fmt.Errorf("parsing test cases error: %w", err) } return testCases, nil }