Merge pull request #127828 from macsko/add_template_parameters_to_createnodesop_in_scheduler_perf

Add template parameters to createNodesOp in scheduler_perf

test/integration/framework/util.go

@@ -22,25 +22,14 @@ import (
 	"context"
 	"fmt"
 	"testing"
-	"time"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 	nodectlr "k8s.io/kubernetes/pkg/controller/nodelifecycle"
 )
 
-const (
-	// poll is how often to Poll pods, nodes and claims.
-	poll = 2 * time.Second
-
-	// singleCallTimeout is how long to try single API calls (like 'get' or 'list'). Used to prevent
-	// transient failures from failing tests.
-	singleCallTimeout = 5 * time.Minute
-)
-
 // CreateNamespaceOrDie creates a namespace.
 func CreateNamespaceOrDie(c clientset.Interface, baseName string, t testing.TB) *v1.Namespace {
 	ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: baseName}}
@@ -59,22 +48,6 @@ func DeleteNamespaceOrDie(c clientset.Interface, ns *v1.Namespace, t testing.TB)
 	}
 }
 
-// waitListAllNodes is a wrapper around listing nodes supporting retries.
-func waitListAllNodes(c clientset.Interface) (*v1.NodeList, error) {
-	var nodes *v1.NodeList
-	var err error
-	if wait.PollImmediate(poll, singleCallTimeout, func() (bool, error) {
-		nodes, err = c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
-		if err != nil {
-			return false, err
-		}
-		return true, nil
-	}) != nil {
-		return nodes, err
-	}
-	return nodes, nil
-}
-
 // Filter filters nodes in NodeList in place, removing nodes that do not
 // satisfy the given condition
 func Filter(nodeList *v1.NodeList, fn func(node v1.Node) bool) {

test/integration/scheduler_perf/node_util.go

@@ -14,48 +14,70 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package framework
+package benchmark
 
 import (
 	"context"
 	"fmt"
+	"time"
 
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/rand"
+	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 	testutils "k8s.io/kubernetes/test/utils"
 )
 
 const (
-	retries = 5
+	// createNodeRetries defines number of retries when creating nodes.
+	createNodeRetries = 5
+
+	// pollingInterval defines how often to poll when waiting for nodes to be created.
+	pollingInterval = 2 * time.Second
+
+	// singleCallTimeout is how long to try single API calls (like 'get' or 'list'). Used to prevent
+	// transient failures from failing tests.
+	singleCallTimeout = 5 * time.Minute
 )
 
+// NodeTemplate is responsible for creating a v1.Node instance that is ready
+// to be sent to the API server.
+type NodeTemplate interface {
+	// GetNodeTemplate returns a node template for one out of many different nodes.
+	// It gets called multiple times with an increasing index and a fixed count parameters.
+	// This number can, but doesn't have to be, used to modify parts of the node spec like
+	// for example a named reference to some other object.
+	GetNodeTemplate(index, count int) (*v1.Node, error)
+}
+
+// StaticNodeTemplate returns an implementation of NodeTemplate for a fixed node that is the same regardless of the index.
+func StaticNodeTemplate(node *v1.Node) NodeTemplate {
+	return (*staticNodeTemplate)(node)
+}
+
+type staticNodeTemplate v1.Node
+
+// GetNodeTemplate implements [NodeTemplate.GetNodeTemplate] by returning the same node
+// for each call.
+func (s *staticNodeTemplate) GetNodeTemplate(index, count int) (*v1.Node, error) {
+	return (*v1.Node)(s), nil
+}
+
 // IntegrationTestNodePreparer holds configuration information for the test node preparer.
 type IntegrationTestNodePreparer struct {
 	client          clientset.Interface
 	countToStrategy []testutils.CountToStrategy
-	nodeNamePrefix  string
+	nodeTemplate    NodeTemplate
-	nodeSpec        *v1.Node
 }
 
-// NewIntegrationTestNodePreparer creates an IntegrationTestNodePreparer configured with defaults.
+// NewIntegrationTestNodePreparer creates an IntegrationTestNodePreparer with a given nodeTemplate.
-func NewIntegrationTestNodePreparer(client clientset.Interface, countToStrategy []testutils.CountToStrategy, nodeNamePrefix string) testutils.TestNodePreparer {
+func NewIntegrationTestNodePreparer(client clientset.Interface, countToStrategy []testutils.CountToStrategy, nodeTemplate NodeTemplate) testutils.TestNodePreparer {
 	return &IntegrationTestNodePreparer{
 		client:          client,
 		countToStrategy: countToStrategy,
-		nodeNamePrefix:  nodeNamePrefix,
+		nodeTemplate:    nodeTemplate,
-	}
-}
-
-// NewIntegrationTestNodePreparerWithNodeSpec creates an IntegrationTestNodePreparer configured with nodespec.
-func NewIntegrationTestNodePreparerWithNodeSpec(client clientset.Interface, countToStrategy []testutils.CountToStrategy, nodeSpec *v1.Node) testutils.TestNodePreparer {
-	return &IntegrationTestNodePreparer{
-		client:          client,
-		countToStrategy: countToStrategy,
-		nodeSpec:        nodeSpec,
 	}
 }
 
@@ -67,30 +89,13 @@ func (p *IntegrationTestNodePreparer) PrepareNodes(ctx context.Context, nextNode
 	}
 
 	klog.Infof("Making %d nodes", numNodes)
-	baseNode := &v1.Node{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: p.nodeNamePrefix,
-		},
-		Status: v1.NodeStatus{
-			Capacity: v1.ResourceList{
-				v1.ResourcePods:   *resource.NewQuantity(110, resource.DecimalSI),
-				v1.ResourceCPU:    resource.MustParse("4"),
-				v1.ResourceMemory: resource.MustParse("32Gi"),
-			},
-			Phase: v1.NodeRunning,
-			Conditions: []v1.NodeCondition{
-				{Type: v1.NodeReady, Status: v1.ConditionTrue},
-			},
-		},
-	}
-
-	if p.nodeSpec != nil {
-		baseNode = p.nodeSpec
-	}
 
 	for i := 0; i < numNodes; i++ {
-		var err error
+		baseNode, err := p.nodeTemplate.GetNodeTemplate(i, numNodes)
-		for retry := 0; retry < retries; retry++ {
+		if err != nil {
+			return fmt.Errorf("failed to get node template: %w", err)
+		}
+		for retry := 0; retry < createNodeRetries; retry++ {
 			// Create nodes with the usual kubernetes.io/hostname label.
 			// For that we need to know the name in advance, if we want to
 			// do it in one request.
@@ -114,7 +119,7 @@ func (p *IntegrationTestNodePreparer) PrepareNodes(ctx context.Context, nextNode
 		}
 	}
 
-	nodes, err := waitListAllNodes(p.client)
+	nodes, err := waitListAllNodes(ctx, p.client)
 	if err != nil {
 		return fmt.Errorf("listing nodes: %w", err)
 	}
@@ -133,7 +138,7 @@ func (p *IntegrationTestNodePreparer) PrepareNodes(ctx context.Context, nextNode
 func (p *IntegrationTestNodePreparer) CleanupNodes(ctx context.Context) error {
 	// TODO(#93794): make CleanupNodes only clean up the nodes created by this
 	// IntegrationTestNodePreparer to make this more intuitive.
-	nodes, err := waitListAllNodes(p.client)
+	nodes, err := waitListAllNodes(ctx, p.client)
 	if err != nil {
 		klog.Fatalf("Error listing nodes: %v", err)
 	}
@@ -146,3 +151,19 @@ func (p *IntegrationTestNodePreparer) CleanupNodes(ctx context.Context) error {
 	}
 	return errRet
 }
+
+// waitListAllNodes is a wrapper around listing nodes supporting retries.
+func waitListAllNodes(ctx context.Context, c clientset.Interface) (*v1.NodeList, error) {
+	var nodes *v1.NodeList
+	var err error
+	if wait.PollUntilContextTimeout(ctx, pollingInterval, singleCallTimeout, true, func(ctx context.Context) (bool, error) {
+		nodes, err = c.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	}) != nil {
+		return nodes, err
+	}
+	return nodes, nil
+}

test/integration/scheduler_perf/scheduler_perf.go

@@ -1702,21 +1702,15 @@ func getNodePreparer(prefix string, cno *createNodesOp, clientset clientset.Inte
 		nodeStrategy = cno.UniqueNodeLabelStrategy
 	}
 
+	nodeTemplate := StaticNodeTemplate(makeBaseNode(prefix))
 	if cno.NodeTemplatePath != nil {
-		node, err := getNodeSpecFromFile(cno.NodeTemplatePath)
+		nodeTemplate = nodeTemplateFromFile(*cno.NodeTemplatePath)
-		if err != nil {
-			return nil, err
-		}
-		return framework.NewIntegrationTestNodePreparerWithNodeSpec(
-			clientset,
-			[]testutils.CountToStrategy{{Count: cno.Count, Strategy: nodeStrategy}},
-			node,
-		), nil
 	}
-	return framework.NewIntegrationTestNodePreparer(
+
+	return NewIntegrationTestNodePreparer(
 		clientset,
 		[]testutils.CountToStrategy{{Count: cno.Count, Strategy: nodeStrategy}},
-		prefix,
+		nodeTemplate,
 	), nil
 }
 
@@ -2103,9 +2097,11 @@ func getPodStrategy(cpo *createPodsOp) (testutils.TestPodCreateStrategy, error)
 	return testutils.NewCreatePodWithPersistentVolumeStrategy(pvcTemplate, getCustomVolumeFactory(pvTemplate), podTemplate), nil
 }
 
-func getNodeSpecFromFile(path *string) (*v1.Node, error) {
+type nodeTemplateFromFile string
+
+func (f nodeTemplateFromFile) GetNodeTemplate(index, count int) (*v1.Node, error) {
 	nodeSpec := &v1.Node{}
-	if err := getSpecFromFile(path, nodeSpec); err != nil {
+	if err := getSpecFromTextTemplateFile(string(f), map[string]any{"Index": index, "Count": count}, nodeSpec); err != nil {
 		return nil, fmt.Errorf("parsing Node: %w", err)
 	}
 	return nodeSpec, nil

test/integration/scheduler_perf/util.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -227,6 +228,27 @@ func makeBasePod() *v1.Pod {
 	return basePod
 }
 
+// makeBaseNode creates a Node object with given nodeNamePrefix to be used as a template.
+func makeBaseNode(nodeNamePrefix string) *v1.Node {
+	return &v1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: nodeNamePrefix,
+		},
+		Status: v1.NodeStatus{
+			Capacity: v1.ResourceList{
+				v1.ResourcePods:   *resource.NewQuantity(110, resource.DecimalSI),
+				v1.ResourceCPU:    resource.MustParse("4"),
+				v1.ResourceMemory: resource.MustParse("32Gi"),
+			},
+			Phase: v1.NodeRunning,
+			Conditions: []v1.NodeCondition{
+				{Type: v1.NodeReady, Status: v1.ConditionTrue},
+			},
+		},
+	}
+
+}
+
 func dataItems2JSONFile(dataItems DataItems, namePrefix string) error {
 	// perfdash expects all data items to have the same set of labels.  It
 	// then renders drop-down buttons for each label with all values found