diff --git a/test/integration/framework/util.go b/test/integration/framework/util.go index ac792a55323..878151e8bc1 100644 --- a/test/integration/framework/util.go +++ b/test/integration/framework/util.go @@ -22,25 +22,14 @@ import ( "context" "fmt" "testing" - "time" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" nodectlr "k8s.io/kubernetes/pkg/controller/nodelifecycle" ) -const ( - // poll is how often to Poll pods, nodes and claims. - poll = 2 * time.Second - - // singleCallTimeout is how long to try single API calls (like 'get' or 'list'). Used to prevent - // transient failures from failing tests. - singleCallTimeout = 5 * time.Minute -) - // CreateNamespaceOrDie creates a namespace. func CreateNamespaceOrDie(c clientset.Interface, baseName string, t testing.TB) *v1.Namespace { ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: baseName}} @@ -59,22 +48,6 @@ func DeleteNamespaceOrDie(c clientset.Interface, ns *v1.Namespace, t testing.TB) } } -// waitListAllNodes is a wrapper around listing nodes supporting retries. -func waitListAllNodes(c clientset.Interface) (*v1.NodeList, error) { - var nodes *v1.NodeList - var err error - if wait.PollImmediate(poll, singleCallTimeout, func() (bool, error) { - nodes, err = c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return false, err - } - return true, nil - }) != nil { - return nodes, err - } - return nodes, nil -} - // Filter filters nodes in NodeList in place, removing nodes that do not // satisfy the given condition func Filter(nodeList *v1.NodeList, fn func(node v1.Node) bool) { diff --git a/test/integration/framework/perf_utils.go b/test/integration/scheduler_perf/node_util.go similarity index 56% rename from test/integration/framework/perf_utils.go rename to test/integration/scheduler_perf/node_util.go index 566308355fc..262c46661c3 100644 --- a/test/integration/framework/perf_utils.go +++ b/test/integration/scheduler_perf/node_util.go @@ -14,48 +14,70 @@ See the License for the specific language governing permissions and limitations under the License. */ -package framework +package benchmark import ( "context" "fmt" + "time" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" testutils "k8s.io/kubernetes/test/utils" ) const ( - retries = 5 + // createNodeRetries defines number of retries when creating nodes. + createNodeRetries = 5 + + // pollingInterval defines how often to poll when waiting for nodes to be created. + pollingInterval = 2 * time.Second + + // singleCallTimeout is how long to try single API calls (like 'get' or 'list'). Used to prevent + // transient failures from failing tests. + singleCallTimeout = 5 * time.Minute ) +// NodeTemplate is responsible for creating a v1.Node instance that is ready +// to be sent to the API server. +type NodeTemplate interface { + // GetNodeTemplate returns a node template for one out of many different nodes. + // It gets called multiple times with an increasing index and a fixed count parameters. + // This number can, but doesn't have to be, used to modify parts of the node spec like + // for example a named reference to some other object. + GetNodeTemplate(index, count int) (*v1.Node, error) +} + +// StaticNodeTemplate returns an implementation of NodeTemplate for a fixed node that is the same regardless of the index. +func StaticNodeTemplate(node *v1.Node) NodeTemplate { + return (*staticNodeTemplate)(node) +} + +type staticNodeTemplate v1.Node + +// GetNodeTemplate implements [NodeTemplate.GetNodeTemplate] by returning the same node +// for each call. +func (s *staticNodeTemplate) GetNodeTemplate(index, count int) (*v1.Node, error) { + return (*v1.Node)(s), nil +} + // IntegrationTestNodePreparer holds configuration information for the test node preparer. type IntegrationTestNodePreparer struct { client clientset.Interface countToStrategy []testutils.CountToStrategy - nodeNamePrefix string - nodeSpec *v1.Node + nodeTemplate NodeTemplate } -// NewIntegrationTestNodePreparer creates an IntegrationTestNodePreparer configured with defaults. -func NewIntegrationTestNodePreparer(client clientset.Interface, countToStrategy []testutils.CountToStrategy, nodeNamePrefix string) testutils.TestNodePreparer { +// NewIntegrationTestNodePreparer creates an IntegrationTestNodePreparer with a given nodeTemplate. +func NewIntegrationTestNodePreparer(client clientset.Interface, countToStrategy []testutils.CountToStrategy, nodeTemplate NodeTemplate) testutils.TestNodePreparer { return &IntegrationTestNodePreparer{ client: client, countToStrategy: countToStrategy, - nodeNamePrefix: nodeNamePrefix, - } -} - -// NewIntegrationTestNodePreparerWithNodeSpec creates an IntegrationTestNodePreparer configured with nodespec. -func NewIntegrationTestNodePreparerWithNodeSpec(client clientset.Interface, countToStrategy []testutils.CountToStrategy, nodeSpec *v1.Node) testutils.TestNodePreparer { - return &IntegrationTestNodePreparer{ - client: client, - countToStrategy: countToStrategy, - nodeSpec: nodeSpec, + nodeTemplate: nodeTemplate, } } @@ -67,30 +89,13 @@ func (p *IntegrationTestNodePreparer) PrepareNodes(ctx context.Context, nextNode } klog.Infof("Making %d nodes", numNodes) - baseNode := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: p.nodeNamePrefix, - }, - Status: v1.NodeStatus{ - Capacity: v1.ResourceList{ - v1.ResourcePods: *resource.NewQuantity(110, resource.DecimalSI), - v1.ResourceCPU: resource.MustParse("4"), - v1.ResourceMemory: resource.MustParse("32Gi"), - }, - Phase: v1.NodeRunning, - Conditions: []v1.NodeCondition{ - {Type: v1.NodeReady, Status: v1.ConditionTrue}, - }, - }, - } - - if p.nodeSpec != nil { - baseNode = p.nodeSpec - } for i := 0; i < numNodes; i++ { - var err error - for retry := 0; retry < retries; retry++ { + baseNode, err := p.nodeTemplate.GetNodeTemplate(i, numNodes) + if err != nil { + return fmt.Errorf("failed to get node template: %w", err) + } + for retry := 0; retry < createNodeRetries; retry++ { // Create nodes with the usual kubernetes.io/hostname label. // For that we need to know the name in advance, if we want to // do it in one request. @@ -114,7 +119,7 @@ func (p *IntegrationTestNodePreparer) PrepareNodes(ctx context.Context, nextNode } } - nodes, err := waitListAllNodes(p.client) + nodes, err := waitListAllNodes(ctx, p.client) if err != nil { return fmt.Errorf("listing nodes: %w", err) } @@ -133,7 +138,7 @@ func (p *IntegrationTestNodePreparer) PrepareNodes(ctx context.Context, nextNode func (p *IntegrationTestNodePreparer) CleanupNodes(ctx context.Context) error { // TODO(#93794): make CleanupNodes only clean up the nodes created by this // IntegrationTestNodePreparer to make this more intuitive. - nodes, err := waitListAllNodes(p.client) + nodes, err := waitListAllNodes(ctx, p.client) if err != nil { klog.Fatalf("Error listing nodes: %v", err) } @@ -146,3 +151,19 @@ func (p *IntegrationTestNodePreparer) CleanupNodes(ctx context.Context) error { } return errRet } + +// waitListAllNodes is a wrapper around listing nodes supporting retries. +func waitListAllNodes(ctx context.Context, c clientset.Interface) (*v1.NodeList, error) { + var nodes *v1.NodeList + var err error + if wait.PollUntilContextTimeout(ctx, pollingInterval, singleCallTimeout, true, func(ctx context.Context) (bool, error) { + nodes, err = c.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + return true, nil + }) != nil { + return nodes, err + } + return nodes, nil +} diff --git a/test/integration/scheduler_perf/scheduler_perf.go b/test/integration/scheduler_perf/scheduler_perf.go index e41b2cc148e..8bcb8f58c91 100644 --- a/test/integration/scheduler_perf/scheduler_perf.go +++ b/test/integration/scheduler_perf/scheduler_perf.go @@ -1702,21 +1702,15 @@ func getNodePreparer(prefix string, cno *createNodesOp, clientset clientset.Inte nodeStrategy = cno.UniqueNodeLabelStrategy } + nodeTemplate := StaticNodeTemplate(makeBaseNode(prefix)) if cno.NodeTemplatePath != nil { - node, err := getNodeSpecFromFile(cno.NodeTemplatePath) - if err != nil { - return nil, err - } - return framework.NewIntegrationTestNodePreparerWithNodeSpec( - clientset, - []testutils.CountToStrategy{{Count: cno.Count, Strategy: nodeStrategy}}, - node, - ), nil + nodeTemplate = nodeTemplateFromFile(*cno.NodeTemplatePath) } - return framework.NewIntegrationTestNodePreparer( + + return NewIntegrationTestNodePreparer( clientset, []testutils.CountToStrategy{{Count: cno.Count, Strategy: nodeStrategy}}, - prefix, + nodeTemplate, ), nil } @@ -2103,9 +2097,11 @@ func getPodStrategy(cpo *createPodsOp) (testutils.TestPodCreateStrategy, error) return testutils.NewCreatePodWithPersistentVolumeStrategy(pvcTemplate, getCustomVolumeFactory(pvTemplate), podTemplate), nil } -func getNodeSpecFromFile(path *string) (*v1.Node, error) { +type nodeTemplateFromFile string + +func (f nodeTemplateFromFile) GetNodeTemplate(index, count int) (*v1.Node, error) { nodeSpec := &v1.Node{} - if err := getSpecFromFile(path, nodeSpec); err != nil { + if err := getSpecFromTextTemplateFile(string(f), map[string]any{"Index": index, "Count": count}, nodeSpec); err != nil { return nil, fmt.Errorf("parsing Node: %w", err) } return nodeSpec, nil diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index 879372af6f1..a4e45f069a9 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -30,6 +30,7 @@ import ( "time" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" @@ -227,6 +228,27 @@ func makeBasePod() *v1.Pod { return basePod } +// makeBaseNode creates a Node object with given nodeNamePrefix to be used as a template. +func makeBaseNode(nodeNamePrefix string) *v1.Node { + return &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: nodeNamePrefix, + }, + Status: v1.NodeStatus{ + Capacity: v1.ResourceList{ + v1.ResourcePods: *resource.NewQuantity(110, resource.DecimalSI), + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("32Gi"), + }, + Phase: v1.NodeRunning, + Conditions: []v1.NodeCondition{ + {Type: v1.NodeReady, Status: v1.ConditionTrue}, + }, + }, + } + +} + func dataItems2JSONFile(dataItems DataItems, namePrefix string) error { // perfdash expects all data items to have the same set of labels. It // then renders drop-down buttons for each label with all values found