From fbb3d6bf882663cb137fe1959ad141163638c536 Mon Sep 17 00:00:00 2001 From: gmarek Date: Wed, 19 Oct 2016 17:15:55 +0200 Subject: [PATCH] Generalize Node preparation for e2e and integration tests --- test/e2e/framework/util.go | 60 ++++++++++ test/integration/framework/perf_utils.go | 103 ++++++++++++++++++ .../scheduler_perf/scheduler_bench_test.go | 16 ++- .../scheduler_perf/scheduler_test.go | 15 ++- test/integration/scheduler_perf/util.go | 28 ----- test/utils/runners.go | 62 +++++++++++ 6 files changed, 254 insertions(+), 30 deletions(-) create mode 100644 test/integration/framework/perf_utils.go diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 7f0fbf96bb5..83473a4a306 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -4514,3 +4514,63 @@ func ListNamespaceEvents(c *client.Client, ns string) error { } return nil } + +// E2ETestNodePreparer implements testutils.TestNodePreparer interface, which is used +// to create/modify Nodes before running a test. +type E2ETestNodePreparer struct { + client clientset.Interface + // Specifies how many nodes should be modified using the given strategy. + // Only one strategy can be applied to a single Node, so there needs to + // be at least Nodes in the cluster. + countToStrategy map[int]testutils.PrepareNodeStrategy + nodeToAppliedStrategy map[string]testutils.PrepareNodeStrategy +} + +func NewE2ETestNodePreparer(client clientset.Interface, countToStrategy map[int]testutils.PrepareNodeStrategy) testutils.TestNodePreparer { + return &E2ETestNodePreparer{ + client: client, + countToStrategy: countToStrategy, + nodeToAppliedStrategy: make(map[string]testutils.PrepareNodeStrategy), + } +} + +func (p *E2ETestNodePreparer) PrepareNodes() error { + nodes := GetReadySchedulableNodesOrDie(p.client) + numTemplates := 0 + for k := range p.countToStrategy { + numTemplates += k + } + if numTemplates > len(nodes.Items) { + return fmt.Errorf("Can't prepare Nodes. Got more templates than existing Nodes.") + } + index := 0 + sum := 0 + for k, strategy := range p.countToStrategy { + sum += k + for ; index < sum; index++ { + if err := testutils.DoPrepareNode(p.client, &nodes.Items[index], strategy); err != nil { + glog.Errorf("Aborting node preparation: %v", err) + return err + } + p.nodeToAppliedStrategy[nodes.Items[index].Name] = strategy + } + } + return nil +} + +func (p *E2ETestNodePreparer) CleanupNodes() error { + var encounteredError error + nodes := GetReadySchedulableNodesOrDie(p.client) + for i := range nodes.Items { + var err error + name := nodes.Items[i].Name + strategy, found := p.nodeToAppliedStrategy[name] + if found { + if err = testutils.DoCleanupNode(p.client, name, strategy); err != nil { + glog.Errorf("Skipping cleanup of Node: failed update of %v: %v", name, err) + encounteredError = err + } + } + } + return encounteredError +} diff --git a/test/integration/framework/perf_utils.go b/test/integration/framework/perf_utils.go new file mode 100644 index 00000000000..37a40bdaec7 --- /dev/null +++ b/test/integration/framework/perf_utils.go @@ -0,0 +1,103 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + e2eframework "k8s.io/kubernetes/test/e2e/framework" + testutils "k8s.io/kubernetes/test/utils" + + "github.com/golang/glog" +) + +const ( + retries = 5 +) + +type IntegrationTestNodePreparer struct { + client clientset.Interface + countToStrategy map[int]testutils.PrepareNodeStrategy + nodeNamePrefix string +} + +func NewIntegrationTestNodePreparer(client clientset.Interface, countToStrategy map[int]testutils.PrepareNodeStrategy, nodeNamePrefix string) testutils.TestNodePreparer { + return &IntegrationTestNodePreparer{ + client: client, + countToStrategy: countToStrategy, + nodeNamePrefix: nodeNamePrefix, + } +} + +func (p *IntegrationTestNodePreparer) PrepareNodes() error { + numNodes := 0 + for k := range p.countToStrategy { + numNodes += k + } + + glog.Infof("Making %d nodes", numNodes) + baseNode := &api.Node{ + ObjectMeta: api.ObjectMeta{ + GenerateName: p.nodeNamePrefix, + }, + Spec: api.NodeSpec{ + // TODO: investigate why this is needed. + ExternalID: "foo", + }, + Status: api.NodeStatus{ + Capacity: api.ResourceList{ + api.ResourcePods: *resource.NewQuantity(110, resource.DecimalSI), + api.ResourceCPU: resource.MustParse("4"), + api.ResourceMemory: resource.MustParse("32Gi"), + }, + Phase: api.NodeRunning, + Conditions: []api.NodeCondition{ + {Type: api.NodeReady, Status: api.ConditionTrue}, + }, + }, + } + for i := 0; i < numNodes; i++ { + if _, err := p.client.Core().Nodes().Create(baseNode); err != nil { + glog.Fatalf("Error creating node: %v", err) + } + } + + nodes := e2eframework.GetReadySchedulableNodesOrDie(p.client) + index := 0 + sum := 0 + for k, strategy := range p.countToStrategy { + sum += k + for ; index < sum; index++ { + if err := testutils.DoPrepareNode(p.client, &nodes.Items[index], strategy); err != nil { + glog.Errorf("Aborting node preparation: %v", err) + return err + } + } + } + return nil +} + +func (p *IntegrationTestNodePreparer) CleanupNodes() error { + nodes := e2eframework.GetReadySchedulableNodesOrDie(p.client) + for i := range nodes.Items { + if err := p.client.Core().Nodes().Delete(nodes.Items[i].Name, &api.DeleteOptions{}); err != nil { + glog.Errorf("Error while deleting Node: %v", err) + } + } + return nil +} diff --git a/test/integration/scheduler_perf/scheduler_bench_test.go b/test/integration/scheduler_perf/scheduler_bench_test.go index 0ce6b8b3431..2cd4bcb7dc5 100644 --- a/test/integration/scheduler_perf/scheduler_bench_test.go +++ b/test/integration/scheduler_perf/scheduler_bench_test.go @@ -19,6 +19,11 @@ package benchmark import ( "testing" "time" + + "k8s.io/kubernetes/test/integration/framework" + testutils "k8s.io/kubernetes/test/utils" + + "github.com/golang/glog" ) // BenchmarkScheduling100Nodes0Pods benchmarks the scheduling rate @@ -53,8 +58,17 @@ func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) { defer finalFunc() c := schedulerConfigFactory.Client - makeNodes(c, numNodes) + nodePreparer := framework.NewIntegrationTestNodePreparer( + c, + map[int]testutils.PrepareNodeStrategy{numNodes: &testutils.TrivialNodePrepareStrategy{}}, + "scheduler-perf-", + ) + if err := nodePreparer.PrepareNodes(); err != nil { + glog.Fatalf("%v", err) + } + defer nodePreparer.CleanupNodes() makePodsFromRC(c, "rc1", numScheduledPods) + for { scheduled := schedulerConfigFactory.ScheduledPodLister.Indexer.List() if len(scheduled) >= numScheduledPods { diff --git a/test/integration/scheduler_perf/scheduler_test.go b/test/integration/scheduler_perf/scheduler_test.go index e8d3081e203..1a9d1780c12 100644 --- a/test/integration/scheduler_perf/scheduler_test.go +++ b/test/integration/scheduler_perf/scheduler_test.go @@ -21,6 +21,11 @@ import ( "math" "testing" "time" + + "k8s.io/kubernetes/test/integration/framework" + testutils "k8s.io/kubernetes/test/utils" + + "github.com/golang/glog" ) const ( @@ -76,7 +81,15 @@ func schedulePods(numNodes, numPods int) int32 { defer destroyFunc() c := schedulerConfigFactory.Client - makeNodes(c, numNodes) + nodePreparer := framework.NewIntegrationTestNodePreparer( + c, + map[int]testutils.PrepareNodeStrategy{numNodes: &testutils.TrivialNodePrepareStrategy{}}, + "scheduler-perf-", + ) + if err := nodePreparer.PrepareNodes(); err != nil { + glog.Fatalf("%v", err) + } + defer nodePreparer.CleanupNodes() makePodsFromRC(c, "rc1", numPods) prev := 0 diff --git a/test/integration/scheduler_perf/util.go b/test/integration/scheduler_perf/util.go index e1e6171abec..dbc60cb1b69 100644 --- a/test/integration/scheduler_perf/util.go +++ b/test/integration/scheduler_perf/util.go @@ -80,34 +80,6 @@ func mustSetupScheduler() (schedulerConfigFactory *factory.ConfigFactory, destro return } -func makeNodes(c clientset.Interface, nodeCount int) { - glog.Infof("making %d nodes", nodeCount) - baseNode := &api.Node{ - ObjectMeta: api.ObjectMeta{ - GenerateName: "scheduler-test-node-", - }, - Spec: api.NodeSpec{ - ExternalID: "foobar", - }, - Status: api.NodeStatus{ - Capacity: api.ResourceList{ - api.ResourcePods: *resource.NewQuantity(110, resource.DecimalSI), - api.ResourceCPU: resource.MustParse("4"), - api.ResourceMemory: resource.MustParse("32Gi"), - }, - Phase: api.NodeRunning, - Conditions: []api.NodeCondition{ - {Type: api.NodeReady, Status: api.ConditionTrue}, - }, - }, - } - for i := 0; i < nodeCount; i++ { - if _, err := c.Core().Nodes().Create(baseNode); err != nil { - panic("error creating node: " + err.Error()) - } - } -} - func makePodSpec() api.PodSpec { return api.PodSpec{ Containers: []api.Container{{ diff --git a/test/utils/runners.go b/test/utils/runners.go index b9efbef6e53..a7bf2c30c74 100644 --- a/test/utils/runners.go +++ b/test/utils/runners.go @@ -23,9 +23,11 @@ import ( "time" "k8s.io/kubernetes/pkg/api" + apierrs "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/apis/extensions" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/labels" @@ -605,3 +607,63 @@ waitLoop: } return nil } + +type TestNodePreparer interface { + PrepareNodes() error + CleanupNodes() error +} + +type PrepareNodeStrategy interface { + PreparePatch(node *api.Node) []byte + CleanupNode(node *api.Node) *api.Node +} + +type TrivialNodePrepareStrategy struct{} + +func (*TrivialNodePrepareStrategy) PreparePatch(*api.Node) []byte { + return []byte{} +} + +func (*TrivialNodePrepareStrategy) CleanupNode(node *api.Node) *api.Node { + nodeCopy := *node + return &nodeCopy +} + +func DoPrepareNode(client clientset.Interface, node *api.Node, strategy PrepareNodeStrategy) error { + var err error + patch := strategy.PreparePatch(node) + if len(patch) == 0 { + return nil + } + for attempt := 0; attempt < retries; attempt++ { + if _, err = client.Core().Nodes().Patch(node.Name, api.MergePatchType, []byte(patch)); err == nil { + return nil + } + if !apierrs.IsConflict(err) { + return fmt.Errorf("Error while applying patch %v to Node %v: %v", string(patch), node.Name, err) + } + time.Sleep(100 * time.Millisecond) + } + return fmt.Errorf("To many conflicts when applying patch %v to Node %v", string(patch), node.Name) +} + +func DoCleanupNode(client clientset.Interface, nodeName string, strategy PrepareNodeStrategy) error { + for attempt := 0; attempt < retries; attempt++ { + node, err := client.Core().Nodes().Get(nodeName) + if err != nil { + return fmt.Errorf("Skipping cleanup of Node: failed to get Node %v: %v", nodeName, err) + } + updatedNode := strategy.CleanupNode(node) + if api.Semantic.DeepEqual(node, updatedNode) { + return nil + } + if _, err = client.Core().Nodes().Update(updatedNode); err == nil { + return nil + } + if !apierrs.IsConflict(err) { + return fmt.Errorf("Error when updating Node %v: %v", nodeName, err) + } + time.Sleep(100 * time.Millisecond) + } + return fmt.Errorf("To many conflicts when trying to cleanup Node %v", nodeName) +}