github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-09-14 13:45:06 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #127828 from macsko/add_template_parameters_to_createnodesop_in_scheduler_perf

Browse Source 
Add template parameters to createNodesOp in scheduler_perf
This commit is contained in:
Kubernetes Prow Robot
2024-10-15 20:43:04 +01:00
committed by GitHub
parent 3694a02bb9 6dbb5d84b3
commit 9872b17ccc
4 changed files with 93 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
test/integration/framework/util.go
View File

@@ -22,25 +22,14 @@ import (
"context"
"fmt"
"testing"
"time"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
nodectlr "k8s.io/kubernetes/pkg/controller/nodelifecycle"
)
const (
// poll is how often to Poll pods, nodes and claims.
poll = 2 * time.Second
// singleCallTimeout is how long to try single API calls (like 'get' or 'list'). Used to prevent
// transient failures from failing tests.
singleCallTimeout = 5 * time.Minute
)
// CreateNamespaceOrDie creates a namespace.
func CreateNamespaceOrDie(c clientset.Interface, baseName string, t testing.TB) *v1.Namespace {
ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: baseName}}
@@ -59,22 +48,6 @@ func DeleteNamespaceOrDie(c clientset.Interface, ns *v1.Namespace, t testing.TB)
}
}
// waitListAllNodes is a wrapper around listing nodes supporting retries.
func waitListAllNodes(c clientset.Interface) (*v1.NodeList, error) {
var nodes *v1.NodeList
var err error
if wait.PollImmediate(poll, singleCallTimeout, func() (bool, error) {
nodes, err = c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if err != nil {
return false, err
}
return true, nil
}) != nil {
return nodes, err
}
return nodes, nil
}
// Filter filters nodes in NodeList in place, removing nodes that do not
// satisfy the given condition
func Filter(nodeList *v1.NodeList, fn func(node v1.Node) bool) {

103
test/integration/framework/perf_utils.go → test/integration/scheduler_perf/node_util.go
View File

@@ -14,48 +14,70 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package framework
package benchmark
import (
"context"
"fmt"
"time"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
testutils "k8s.io/kubernetes/test/utils"
)
const (
retries = 5
// createNodeRetries defines number of retries when creating nodes.
createNodeRetries = 5
// pollingInterval defines how often to poll when waiting for nodes to be created.
pollingInterval = 2 * time.Second
// singleCallTimeout is how long to try single API calls (like 'get' or 'list'). Used to prevent
// transient failures from failing tests.
singleCallTimeout = 5 * time.Minute
)
// NodeTemplate is responsible for creating a v1.Node instance that is ready
// to be sent to the API server.
type NodeTemplate interface {
// GetNodeTemplate returns a node template for one out of many different nodes.
// It gets called multiple times with an increasing index and a fixed count parameters.
// This number can, but doesn't have to be, used to modify parts of the node spec like
// for example a named reference to some other object.
GetNodeTemplate(index, count int) (*v1.Node, error)
}
// StaticNodeTemplate returns an implementation of NodeTemplate for a fixed node that is the same regardless of the index.
func StaticNodeTemplate(node *v1.Node) NodeTemplate {
return (*staticNodeTemplate)(node)
}
type staticNodeTemplate v1.Node
// GetNodeTemplate implements [NodeTemplate.GetNodeTemplate] by returning the same node
// for each call.
func (s *staticNodeTemplate) GetNodeTemplate(index, count int) (*v1.Node, error) {
return (*v1.Node)(s), nil
}
// IntegrationTestNodePreparer holds configuration information for the test node preparer.
type IntegrationTestNodePreparer struct {
client clientset.Interface
countToStrategy []testutils.CountToStrategy
nodeNamePrefix string
nodeSpec *v1.Node
nodeTemplate NodeTemplate
}
// NewIntegrationTestNodePreparer creates an IntegrationTestNodePreparer configured with defaults.
func NewIntegrationTestNodePreparer(client clientset.Interface, countToStrategy []testutils.CountToStrategy, nodeNamePrefix string) testutils.TestNodePreparer {
// NewIntegrationTestNodePreparer creates an IntegrationTestNodePreparer with a given nodeTemplate.
func NewIntegrationTestNodePreparer(client clientset.Interface, countToStrategy []testutils.CountToStrategy, nodeTemplate NodeTemplate) testutils.TestNodePreparer {
return &IntegrationTestNodePreparer{
client: client,
countToStrategy: countToStrategy,
nodeNamePrefix: nodeNamePrefix,
}
}
// NewIntegrationTestNodePreparerWithNodeSpec creates an IntegrationTestNodePreparer configured with nodespec.
func NewIntegrationTestNodePreparerWithNodeSpec(client clientset.Interface, countToStrategy []testutils.CountToStrategy, nodeSpec *v1.Node) testutils.TestNodePreparer {
return &IntegrationTestNodePreparer{
client: client,
countToStrategy: countToStrategy,
nodeSpec: nodeSpec,
nodeTemplate: nodeTemplate,
}
}
@@ -67,30 +89,13 @@ func (p *IntegrationTestNodePreparer) PrepareNodes(ctx context.Context, nextNode
}
klog.Infof("Making %d nodes", numNodes)
baseNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
GenerateName: p.nodeNamePrefix,
},
Status: v1.NodeStatus{
Capacity: v1.ResourceList{
v1.ResourcePods: *resource.NewQuantity(110, resource.DecimalSI),
v1.ResourceCPU: resource.MustParse("4"),
v1.ResourceMemory: resource.MustParse("32Gi"),
},
Phase: v1.NodeRunning,
Conditions: []v1.NodeCondition{
{Type: v1.NodeReady, Status: v1.ConditionTrue},
},
},
}
if p.nodeSpec != nil {
baseNode = p.nodeSpec
}
for i := 0; i < numNodes; i++ {
var err error
for retry := 0; retry < retries; retry++ {
baseNode, err := p.nodeTemplate.GetNodeTemplate(i, numNodes)
if err != nil {
return fmt.Errorf("failed to get node template: %w", err)
}
for retry := 0; retry < createNodeRetries; retry++ {
// Create nodes with the usual kubernetes.io/hostname label.
// For that we need to know the name in advance, if we want to
// do it in one request.
@@ -114,7 +119,7 @@ func (p *IntegrationTestNodePreparer) PrepareNodes(ctx context.Context, nextNode
}
}
nodes, err := waitListAllNodes(p.client)
nodes, err := waitListAllNodes(ctx, p.client)
if err != nil {
return fmt.Errorf("listing nodes: %w", err)
}
@@ -133,7 +138,7 @@ func (p *IntegrationTestNodePreparer) PrepareNodes(ctx context.Context, nextNode
func (p *IntegrationTestNodePreparer) CleanupNodes(ctx context.Context) error {
// TODO(#93794): make CleanupNodes only clean up the nodes created by this
// IntegrationTestNodePreparer to make this more intuitive.
nodes, err := waitListAllNodes(p.client)
nodes, err := waitListAllNodes(ctx, p.client)
if err != nil {
klog.Fatalf("Error listing nodes: %v", err)
}
@@ -146,3 +151,19 @@ func (p *IntegrationTestNodePreparer) CleanupNodes(ctx context.Context) error {
}
return errRet
}
// waitListAllNodes is a wrapper around listing nodes supporting retries.
func waitListAllNodes(ctx context.Context, c clientset.Interface) (*v1.NodeList, error) {
var nodes *v1.NodeList
var err error
if wait.PollUntilContextTimeout(ctx, pollingInterval, singleCallTimeout, true, func(ctx context.Context) (bool, error) {
nodes, err = c.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
return false, err
}
return true, nil
}) != nil {
return nodes, err
}
return nodes, nil
}

22
test/integration/scheduler_perf/scheduler_perf.go
View File

@@ -1702,21 +1702,15 @@ func getNodePreparer(prefix string, cno *createNodesOp, clientset clientset.Inte
nodeStrategy = cno.UniqueNodeLabelStrategy
}
nodeTemplate := StaticNodeTemplate(makeBaseNode(prefix))
if cno.NodeTemplatePath != nil {
node, err := getNodeSpecFromFile(cno.NodeTemplatePath)
if err != nil {
return nil, err
}
return framework.NewIntegrationTestNodePreparerWithNodeSpec(
clientset,
[]testutils.CountToStrategy{{Count: cno.Count, Strategy: nodeStrategy}},
node,
), nil
nodeTemplate = nodeTemplateFromFile(*cno.NodeTemplatePath)
}
return framework.NewIntegrationTestNodePreparer(
return NewIntegrationTestNodePreparer(
clientset,
[]testutils.CountToStrategy{{Count: cno.Count, Strategy: nodeStrategy}},
prefix,
nodeTemplate,
), nil
}
@@ -2103,9 +2097,11 @@ func getPodStrategy(cpo *createPodsOp) (testutils.TestPodCreateStrategy, error)
return testutils.NewCreatePodWithPersistentVolumeStrategy(pvcTemplate, getCustomVolumeFactory(pvTemplate), podTemplate), nil
}
func getNodeSpecFromFile(path *string) (*v1.Node, error) {
type nodeTemplateFromFile string
func (f nodeTemplateFromFile) GetNodeTemplate(index, count int) (*v1.Node, error) {
nodeSpec := &v1.Node{}
if err := getSpecFromFile(path, nodeSpec); err != nil {
if err := getSpecFromTextTemplateFile(string(f), map[string]any{"Index": index, "Count": count}, nodeSpec); err != nil {
return nil, fmt.Errorf("parsing Node: %w", err)
}
return nodeSpec, nil

22
test/integration/scheduler_perf/util.go
View File

@@ -30,6 +30,7 @@ import (
"time"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
@@ -227,6 +228,27 @@ func makeBasePod() *v1.Pod {
return basePod
}
// makeBaseNode creates a Node object with given nodeNamePrefix to be used as a template.
func makeBaseNode(nodeNamePrefix string) *v1.Node {
return &v1.Node{
ObjectMeta: metav1.ObjectMeta{
GenerateName: nodeNamePrefix,
},
Status: v1.NodeStatus{
Capacity: v1.ResourceList{
v1.ResourcePods: *resource.NewQuantity(110, resource.DecimalSI),
v1.ResourceCPU: resource.MustParse("4"),
v1.ResourceMemory: resource.MustParse("32Gi"),
},
Phase: v1.NodeRunning,
Conditions: []v1.NodeCondition{
{Type: v1.NodeReady, Status: v1.ConditionTrue},
},
},
}
}
func dataItems2JSONFile(dataItems DataItems, namePrefix string) error {
// perfdash expects all data items to have the same set of labels. It
// then renders drop-down buttons for each label with all values found