diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index b7f7d8af389..9859ed5b539 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -19,6 +19,7 @@ package framework import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -2601,6 +2602,55 @@ func NodeHasTaint(c clientset.Interface, nodeName string, taint *v1.Taint) (bool return true, nil } +//AddOrUpdateAvoidPodOnNode adds avoidPods annotations to node, will override if it exists +func AddOrUpdateAvoidPodOnNode(c clientset.Interface, nodeName string, avoidPods v1.AvoidPods) { + err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) { + node, err := c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + ExpectNoError(err) + + taintsData, err := json.Marshal(avoidPods) + ExpectNoError(err) + + if node.Annotations == nil { + node.Annotations = make(map[string]string) + } + node.Annotations[v1.PreferAvoidPodsAnnotationKey] = string(taintsData) + _, err = c.CoreV1().Nodes().Update(node) + if err != nil { + if !apierrs.IsConflict(err) { + ExpectNoError(err) + } else { + Logf("Conflict when trying to add/update avoidPonds %v to %v", avoidPods, nodeName) + } + } + return true, nil + }) + ExpectNoError(err) +} + +//RemoveAnnotationOffNode removes AvoidPods annotations from the node. It does not fail if no such annotation exists. +func RemoveAvoidPodsOffNode(c clientset.Interface, nodeName string) { + err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) { + node, err := c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + ExpectNoError(err) + + if node.Annotations == nil { + return true, nil + } + delete(node.Annotations, v1.PreferAvoidPodsAnnotationKey) + _, err = c.CoreV1().Nodes().Update(node) + if err != nil { + if !apierrs.IsConflict(err) { + ExpectNoError(err) + } else { + Logf("Conflict when trying to remove avoidPods to %v", nodeName) + } + } + return true, nil + }) + ExpectNoError(err) +} + func getScalerForKind(internalClientset internalclientset.Interface, kind schema.GroupKind) (kubectl.Scaler, error) { return kubectl.ScalerFor(kind, internalClientset) } diff --git a/test/e2e/scheduling/BUILD b/test/e2e/scheduling/BUILD index 627c523dca7..ae5d265edad 100644 --- a/test/e2e/scheduling/BUILD +++ b/test/e2e/scheduling/BUILD @@ -13,6 +13,7 @@ go_library( "events.go", "opaque_resource.go", "predicates.go", + "priorities.go", "rescheduler.go", ], tags = ["automanaged"], @@ -20,6 +21,7 @@ go_library( "//pkg/api/v1:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", "//pkg/util/system:go_default_library", + "//plugin/pkg/scheduler/algorithm/priorities/util:go_default_library", "//test/e2e/common:go_default_library", "//test/e2e/framework:go_default_library", "//test/utils:go_default_library", diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index 42dbc8e32e3..e51b62ec2d0 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -48,6 +48,7 @@ type pausePodConfig struct { Annotations, Labels, NodeSelector map[string]string Resources *v1.ResourceRequirements Tolerations []v1.Toleration + NodeName string } var _ = framework.KubeDescribe("SchedulerPredicates [Serial]", func() { @@ -757,6 +758,7 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod { }, }, Tolerations: conf.Tolerations, + NodeName: conf.NodeName, }, } if conf.Resources != nil { diff --git a/test/e2e/scheduling/priorities.go b/test/e2e/scheduling/priorities.go new file mode 100644 index 00000000000..f61f53a1d7f --- /dev/null +++ b/test/e2e/scheduling/priorities.go @@ -0,0 +1,553 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling + +import ( + "encoding/json" + "fmt" + "math" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + _ "github.com/stretchr/testify/assert" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/kubernetes/pkg/api/v1" + "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" + priorityutil "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities/util" + "k8s.io/kubernetes/test/e2e/common" + "k8s.io/kubernetes/test/e2e/framework" + testutils "k8s.io/kubernetes/test/utils" +) + +type Resource struct { + MilliCPU int64 + Memory int64 +} + +var balancePodLabel map[string]string = map[string]string{"name": "priority-balanced-memory"} + +var podRequestedResource *v1.ResourceRequirements = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + }, +} + +// This test suite is used to verifies scheduler priority functions based on the default provider +var _ = framework.KubeDescribe("SchedulerPriorities [Serial]", func() { + var cs clientset.Interface + var nodeList *v1.NodeList + var systemPodsNo int + var ns string + var masterNodes sets.String + f := framework.NewDefaultFramework("sched-priority") + ignoreLabels := framework.ImagePullerLabels + + AfterEach(func() { + }) + + BeforeEach(func() { + cs = f.ClientSet + ns = f.Namespace.Name + nodeList = &v1.NodeList{} + + framework.WaitForAllNodesHealthy(cs, time.Minute) + masterNodes, nodeList = framework.GetMasterAndWorkerNodesOrDie(cs) + + err := framework.CheckTestingNSDeletedExcept(cs, ns) + framework.ExpectNoError(err) + + err = framework.WaitForPodsRunningReady(cs, metav1.NamespaceSystem, int32(systemPodsNo), 0, framework.PodReadyBeforeTimeout, ignoreLabels) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Pods should be scheduled to low resource use rate node", func() { + //Make sure all the schedulable nodes are balanced (have the same cpu/mem usage ratio) + By("Create pods on each node except the last one to raise cpu and memory usage to the same high level") + var expectedNodeName string + expectedNodeName = nodeList.Items[len(nodeList.Items)-1].Name + nodes := nodeList.Items[:len(nodeList.Items)-1] + // make the nodes except last have cpu,mem usage to 90% + createBalancedPodForNodes(f, cs, ns, nodes, podRequestedResource, 0.9) + By("Create a pod,pod should schedule to the least requested nodes") + createPausePod(f, pausePodConfig{ + Name: "priority-least-requested", + Labels: map[string]string{"name": "priority-least-requested"}, + Resources: podRequestedResource, + }) + By("Wait for all the pods are running") + err := f.WaitForPodRunning("priority-least-requested") + framework.ExpectNoError(err) + By("Verify the pod is scheduled to the expected node") + testPod, err := cs.CoreV1().Pods(ns).Get("priority-least-requested", metav1.GetOptions{}) + framework.ExpectNoError(err) + Expect(testPod.Spec.NodeName).Should(Equal(expectedNodeName)) + }) + + It("Pods created by ReplicationController should spread to different node", func() { + By("Create a pod for each node to make the nodes have almost same cpu/mem usage ratio") + + createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.6) + + By("Create an RC with the same number of replicas as the schedualble nodes. One pod should be scheduled on each node.") + rc := createRC(ns, "scheduler-priority-selector-spreading", int32(len(nodeList.Items)), map[string]string{"name": "scheduler-priority-selector-spreading"}, f, podRequestedResource) + // Cleanup the replication controller when we are done. + defer func() { + // Resize the replication controller to zero to get rid of pods. + if err := framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, rc.Name); err != nil { + framework.Logf("Failed to cleanup replication controller %v: %v.", rc.Name, err) + } + }() + pods, err := framework.PodsCreated(f.ClientSet, f.Namespace.Name, "scheduler-priority-selector-spreading", int32(len(nodeList.Items))) + Expect(err).NotTo(HaveOccurred()) + By("Ensuring each pod is running") + + result := map[string]bool{} + for _, pod := range pods.Items { + if pod.DeletionTimestamp != nil { + continue + } + err = f.WaitForPodRunning(pod.Name) + Expect(err).NotTo(HaveOccurred()) + result[pod.Spec.NodeName] = true + framework.PrintAllKubeletPods(cs, pod.Spec.NodeName) + } + By("Verify the pods were scheduled to each node") + if len(nodeList.Items) != len(result) { + framework.Failf("Pods are not spread to each node") + } + }) + + It("Pod should be prefer scheduled to node that satisify the NodeAffinity", func() { + nodeName := GetNodeThatCanRunPod(f) + By("Trying to apply a label on the found node.") + k := fmt.Sprintf("kubernetes.io/e2e-%s", "node-topologyKey") + v := "topologyvalue" + framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v) + framework.ExpectNodeHasLabel(cs, nodeName, k, v) + defer framework.RemoveLabelOffNode(cs, nodeName, k) + + // make the nodes have balanced cpu,mem usage ratio + createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.6) + By("Trying to relaunch the pod, now with labels.") + labelPodName := "pod-with-node-affinity" + pod := createPausePod(f, pausePodConfig{ + Name: labelPodName, + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: k, + Operator: v1.NodeSelectorOpIn, + Values: []string{v}, + }, + }, + }, + Weight: 20, + }, + }, + }, + }, + }) + By("Wait the pod becomes running.") + framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) + labelPod, err := cs.CoreV1().Pods(ns).Get(labelPodName, metav1.GetOptions{}) + framework.ExpectNoError(err) + By("Verify the pod was scheduled to the expected node.") + Expect(labelPod.Spec.NodeName).To(Equal(nodeName)) + }) + + It("Pod should be schedule to node that satisify the PodAffinity", func() { + By("Trying to launch a pod with a label to get a node which can launch it.") + pod := runPausePod(f, pausePodConfig{ + Name: "with-label-security-s1", + Labels: map[string]string{"service": "S1"}, + Resources: podRequestedResource, + }) + nodeName := pod.Spec.NodeName + + By("Trying to apply label on the found node.") + k := fmt.Sprintf("kubernetes.io/e2e-%s", "node-topologyKey") + v := "topologyvalue" + framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v) + framework.ExpectNodeHasLabel(cs, nodeName, k, v) + defer framework.RemoveLabelOffNode(cs, nodeName, k) + + // make the nodes have balanced cpu,mem usage + createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.6) + + By("Trying to launch the pod, now with podAffinity.") + labelPodName := "pod-with-podaffinity" + pod = createPausePod(f, pausePodConfig{ + Resources: podRequestedResource, + Name: labelPodName, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S1", "value2"}, + }, + { + Key: "service", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"S2"}, + }, { + Key: "service", + Operator: metav1.LabelSelectorOpExists, + }, + }, + }, + TopologyKey: k, + Namespaces: []string{ns}, + }, + Weight: 20, + }, + }, + }, + }, + }) + By("Wait the pod becomes running.") + framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) + labelPod, err := cs.CoreV1().Pods(ns).Get(labelPodName, metav1.GetOptions{}) + framework.ExpectNoError(err) + By("Verify the pod was scheduled to the expected node.") + Expect(labelPod.Spec.NodeName).To(Equal(nodeName)) + }) + + It("Pod should be schedule to node that don't match the PodAntiAffinity terms", func() { + By("Trying to launch a pod with a label to get a node which can launch it.") + pod := runPausePod(f, pausePodConfig{ + Name: "pod-with-label-security-s1", + Labels: map[string]string{"security": "S1"}, + }) + nodeName := pod.Spec.NodeName + + By("Trying to apply a label on the found node.") + k := fmt.Sprintf("kubernetes.io/e2e-%s", "node-topologyKey") + v := "topologyvalue" + framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v) + framework.ExpectNodeHasLabel(cs, nodeName, k, v) + defer framework.RemoveLabelOffNode(cs, nodeName, k) + + // make the nodes have balanced cpu,mem usage + createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.6) + By("Trying to launch the pod with podAntiAffinity.") + labelPodName := "pod-with-pod-antiaffinity" + pod = createPausePod(f, pausePodConfig{ + Resources: podRequestedResource, + Name: labelPodName, + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"S1", "value2"}, + }, + { + Key: "security", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"S2"}, + }, { + Key: "security", + Operator: metav1.LabelSelectorOpExists, + }, + }, + }, + TopologyKey: k, + Namespaces: []string{ns}, + }, + Weight: 10, + }, + }, + }, + }, + }) + By("Wait the pod becomes running") + framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) + labelPod, err := cs.CoreV1().Pods(ns).Get(labelPodName, metav1.GetOptions{}) + framework.ExpectNoError(err) + By("Verify the pod was scheduled to the expected node.") + Expect(labelPod.Spec.NodeName).NotTo(Equal(nodeName)) + }) + + It("Pod should avoid to schedule to node that have avoidPod annotation", func() { + nodeName := nodeList.Items[0].Name + // make the nodes have balanced cpu,mem usage + createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5) + By("Create a RC, with 0 replicas") + rc := createRC(ns, "scheduler-priority-avoid-pod", int32(0), map[string]string{"name": "scheduler-priority-avoid-pod"}, f, podRequestedResource) + // Cleanup the replication controller when we are done. + defer func() { + // Resize the replication controller to zero to get rid of pods. + if err := framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, rc.Name); err != nil { + framework.Logf("Failed to cleanup replication controller %v: %v.", rc.Name, err) + } + }() + + By("Trying to apply avoidPod annotations on the first node.") + avoidPod := v1.AvoidPods{ + PreferAvoidPods: []v1.PreferAvoidPodsEntry{ + { + PodSignature: v1.PodSignature{ + PodController: &metav1.OwnerReference{ + APIVersion: "v1", + Kind: "ReplicationController", + Name: rc.Name, + UID: rc.UID, + Controller: func() *bool { b := true; return &b }(), + }, + }, + Reason: "some reson", + Message: "some message", + }, + }, + } + action := func() error { + framework.AddOrUpdateAvoidPodOnNode(cs, nodeName, avoidPod) + return nil + } + predicate := func(node *v1.Node) bool { + val, err := json.Marshal(avoidPod) + if err != nil { + return false + } + return node.Annotations[v1.PreferAvoidPodsAnnotationKey] == string(val) + } + success, err := common.ObserveNodeUpdateAfterAction(f, nodeName, predicate, action) + Expect(err).NotTo(HaveOccurred()) + Expect(success).To(Equal(true)) + + defer framework.RemoveAvoidPodsOffNode(cs, nodeName) + + By(fmt.Sprintf("Scale the RC: %s to len(nodeList.Item)-1 : %v.", rc.Name, len(nodeList.Items)-1)) + + framework.ScaleRC(f.ClientSet, f.InternalClientset, ns, rc.Name, uint(len(nodeList.Items)-1), true) + testPods, err := cs.CoreV1().Pods(ns).List(metav1.ListOptions{ + LabelSelector: "name=scheduler-priority-avoid-pod", + }) + Expect(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("Verify the pods should not scheduled to the node: %s", nodeName)) + for _, pod := range testPods.Items { + Expect(pod.Spec.NodeName).NotTo(Equal(nodeName)) + } + }) + + It("Pod should perfer to scheduled to nodes pod can tolerate", func() { + // make the nodes have balanced cpu,mem usage ratio + createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5) + //we need apply more taints on a node, because one match toleration only count 1 + By("Trying to apply 10 taint on the nodes except first one.") + nodeName := nodeList.Items[0].Name + + for index, node := range nodeList.Items { + if index == 0 { + continue + } + for i := 0; i < 10; i++ { + testTaint := addRandomTaitToNode(cs, node.Name) + defer framework.RemoveTaintOffNode(cs, node.Name, *testTaint) + } + } + By("Create a pod without any tolerations") + tolerationPodName := "without-tolerations" + pod := createPausePod(f, pausePodConfig{ + Name: tolerationPodName, + }) + framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) + + By("Pod should prefer scheduled to the node don't have the taint.") + tolePod, err := cs.CoreV1().Pods(ns).Get(tolerationPodName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(tolePod.Spec.NodeName).To(Equal(nodeName)) + + By("Trying to apply 10 taint on the first node.") + var tolerations []v1.Toleration + for i := 0; i < 10; i++ { + testTaint := addRandomTaitToNode(cs, nodeName) + tolerations = append(tolerations, v1.Toleration{Key: testTaint.Key, Value: testTaint.Value, Effect: testTaint.Effect}) + defer framework.RemoveTaintOffNode(cs, nodeName, *testTaint) + } + tolerationPodName = "with-tolerations" + By("Create a pod that tolerates all the taints of the first node.") + pod = createPausePod(f, pausePodConfig{ + Name: tolerationPodName, + Tolerations: tolerations, + }) + framework.ExpectNoError(f.WaitForPodRunning(pod.Name)) + + By("Pod should prefer scheduled to the node that pod can tolerate.") + tolePod, err = cs.CoreV1().Pods(ns).Get(tolerationPodName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(tolePod.Spec.NodeName).To(Equal(nodeName)) + }) +}) + +// createBalancedPodForNodes creates a pod per node that asks for enough resources to make all nodes have the same mem/cpu usage ratio. +func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, ns string, nodes []v1.Node, requestedResource *v1.ResourceRequirements, ratio float64) { + // find the max, if the node has the max,use the one, if not,use the ratio parameter + var maxCpuFraction, maxMemFraction float64 = ratio, ratio + var cpuFractionMap = make(map[string]float64) + var memFractionMap = make(map[string]float64) + for _, node := range nodes { + cpuFraction, memFraction := computeCpuMemFraction(cs, node, requestedResource) + cpuFractionMap[node.Name] = cpuFraction + memFractionMap[node.Name] = memFraction + if cpuFraction > maxCpuFraction { + maxCpuFraction = cpuFraction + } + if memFraction > maxMemFraction { + maxMemFraction = memFraction + } + } + // we need the max one to keep the same cpu/mem use rate + ratio = math.Max(maxCpuFraction, maxMemFraction) + for _, node := range nodes { + memAllocatable, found := node.Status.Allocatable["memory"] + Expect(found).To(Equal(true)) + memAllocatableVal := memAllocatable.Value() + + cpuAllocatable, found := node.Status.Allocatable["cpu"] + Expect(found).To(Equal(true)) + cpuAllocatableMil := cpuAllocatable.MilliValue() + + needCreateResource := v1.ResourceList{} + cpuFraction := cpuFractionMap[node.Name] + memFraction := memFractionMap[node.Name] + needCreateResource["cpu"] = *resource.NewMilliQuantity(int64((ratio-cpuFraction)*float64(cpuAllocatableMil)), resource.DecimalSI) + + needCreateResource["memory"] = *resource.NewQuantity(int64((ratio-memFraction)*float64(memAllocatableVal)), resource.BinarySI) + + testutils.StartPods(cs, 1, ns, "priority-balanced-mem-"+node.Name, + *initPausePod(f, pausePodConfig{ + Name: "", + Labels: balancePodLabel, + Resources: &v1.ResourceRequirements{ + Limits: needCreateResource, + Requests: needCreateResource, + }, + NodeName: node.Name, + }), true, framework.Logf) + } +} + +func computeCpuMemFraction(cs clientset.Interface, node v1.Node, resource *v1.ResourceRequirements) (float64, float64) { + framework.Logf("ComputeCpuMemFraction for node: %v", node.Name) + totalRequestedCpuResource := resource.Requests.Cpu().MilliValue() + totalRequestedMemResource := resource.Requests.Memory().Value() + allpods, err := cs.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}) + if err != nil { + framework.Failf("Expect error of invalid, got : %v", err) + } + for _, pod := range allpods.Items { + if pod.Spec.NodeName == node.Name { + framework.Logf("Pod for on the node: %v, Cpu: %v, Mem: %v", pod.Name, getNonZeroRequests(&pod).MilliCPU, getNonZeroRequests(&pod).Memory) + totalRequestedCpuResource += getNonZeroRequests(&pod).MilliCPU + totalRequestedMemResource += getNonZeroRequests(&pod).Memory + } + } + cpuAllocatable, found := node.Status.Allocatable["cpu"] + Expect(found).To(Equal(true)) + cpuAllocatableMil := cpuAllocatable.MilliValue() + + cpuFraction := float64(totalRequestedCpuResource) / float64(cpuAllocatableMil) + memAllocatable, found := node.Status.Allocatable["memory"] + Expect(found).To(Equal(true)) + memAllocatableVal := memAllocatable.Value() + memFraction := float64(totalRequestedMemResource) / float64(memAllocatableVal) + + framework.Logf("Node: %v, totalRequestedCpuResource: %v, cpuAllocatableMil: %v, cpuFraction: %v", node.Name, totalRequestedCpuResource, cpuAllocatableMil, cpuFraction) + framework.Logf("Node: %v, totalRequestedMemResource: %v, memAllocatableVal: %v, memFraction: %v", node.Name, memAllocatableVal, cpuAllocatableMil, memFraction) + + return cpuFraction, memFraction +} + +func getNonZeroRequests(pod *v1.Pod) Resource { + result := Resource{} + for i := range pod.Spec.Containers { + container := &pod.Spec.Containers[i] + cpu, memory := priorityutil.GetNonzeroRequests(&container.Resources.Requests) + result.MilliCPU += cpu + result.Memory += memory + } + return result +} + +func createRC(ns, rsName string, replicas int32, rcPodLabels map[string]string, f *framework.Framework, resource *v1.ResourceRequirements) *v1.ReplicationController { + rc := &v1.ReplicationController{ + TypeMeta: metav1.TypeMeta{ + Kind: "ReplicationController", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: rsName, + }, + Spec: v1.ReplicationControllerSpec{ + Replicas: &replicas, + Template: &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: rcPodLabels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: rsName, + Image: framework.GetPauseImageName(f.ClientSet), + Resources: *resource, + }, + }, + }, + }, + }, + } + rc, err := f.ClientSet.CoreV1().ReplicationControllers(ns).Create(rc) + Expect(err).NotTo(HaveOccurred()) + return rc +} + +func addRandomTaitToNode(cs clientset.Interface, nodeName string) *v1.Taint { + testTaint := v1.Taint{ + Key: fmt.Sprintf("kubernetes.io/e2e-taint-key-%s", string(uuid.NewUUID())), + Value: fmt.Sprintf("testing-taint-value-%s", string(uuid.NewUUID())), + Effect: v1.TaintEffectPreferNoSchedule, + } + framework.AddOrUpdateTaintOnNode(cs, nodeName, testTaint) + framework.ExpectNodeHasTaint(cs, nodeName, &testTaint) + return &testTaint +}