mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Merge pull request #94684 from ingvagabund/e2e-priorities-avoidPods-wait-for-pods-getting-deleted
[sig-scheduling] SchedulerPriorities [Serial] Pod should avoid nodes that have avoidPod annotation: clean remaining pods
This commit is contained in:
commit
759ac2e8c8
@ -887,8 +887,8 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
|
|||||||
ObjectMeta: metav1.ObjectMeta{
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
Name: conf.Name,
|
Name: conf.Name,
|
||||||
Namespace: conf.Namespace,
|
Namespace: conf.Namespace,
|
||||||
Labels: conf.Labels,
|
Labels: map[string]string{},
|
||||||
Annotations: conf.Annotations,
|
Annotations: map[string]string{},
|
||||||
OwnerReferences: conf.OwnerReferences,
|
OwnerReferences: conf.OwnerReferences,
|
||||||
},
|
},
|
||||||
Spec: v1.PodSpec{
|
Spec: v1.PodSpec{
|
||||||
@ -908,6 +908,12 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
|
|||||||
TerminationGracePeriodSeconds: &gracePeriod,
|
TerminationGracePeriodSeconds: &gracePeriod,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
for key, value := range conf.Labels {
|
||||||
|
pod.ObjectMeta.Labels[key] = value
|
||||||
|
}
|
||||||
|
for key, value := range conf.Annotations {
|
||||||
|
pod.ObjectMeta.Annotations[key] = value
|
||||||
|
}
|
||||||
// TODO: setting the Pod's nodeAffinity instead of setting .spec.nodeName works around the
|
// TODO: setting the Pod's nodeAffinity instead of setting .spec.nodeName works around the
|
||||||
// Preemption e2e flake (#88441), but we should investigate deeper to get to the bottom of it.
|
// Preemption e2e flake (#88441), but we should investigate deeper to get to the bottom of it.
|
||||||
if len(conf.NodeName) != 0 {
|
if len(conf.NodeName) != 0 {
|
||||||
|
@ -32,6 +32,7 @@ import (
|
|||||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
"k8s.io/apimachinery/pkg/util/uuid"
|
"k8s.io/apimachinery/pkg/util/uuid"
|
||||||
"k8s.io/apimachinery/pkg/util/wait"
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
clientset "k8s.io/client-go/kubernetes"
|
||||||
@ -52,7 +53,7 @@ type Resource struct {
|
|||||||
Memory int64
|
Memory int64
|
||||||
}
|
}
|
||||||
|
|
||||||
var balancePodLabel = map[string]string{"name": "priority-balanced-memory"}
|
var balancePodLabel = map[string]string{"podname": "priority-balanced-memory"}
|
||||||
|
|
||||||
var podRequestedResource = &v1.ResourceRequirements{
|
var podRequestedResource = &v1.ResourceRequirements{
|
||||||
Limits: v1.ResourceList{
|
Limits: v1.ResourceList{
|
||||||
@ -187,7 +188,8 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// make the nodes have balanced cpu,mem usage
|
// make the nodes have balanced cpu,mem usage
|
||||||
err = createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.6)
|
cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.6)
|
||||||
|
defer cleanUp()
|
||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
ginkgo.By("Trying to launch the pod with podAntiAffinity.")
|
ginkgo.By("Trying to launch the pod with podAntiAffinity.")
|
||||||
labelPodName := "pod-with-pod-antiaffinity"
|
labelPodName := "pod-with-pod-antiaffinity"
|
||||||
@ -236,7 +238,8 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
|
|||||||
ginkgo.It("Pod should avoid nodes that have avoidPod annotation", func() {
|
ginkgo.It("Pod should avoid nodes that have avoidPod annotation", func() {
|
||||||
nodeName := nodeList.Items[0].Name
|
nodeName := nodeList.Items[0].Name
|
||||||
// make the nodes have balanced cpu,mem usage
|
// make the nodes have balanced cpu,mem usage
|
||||||
err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5)
|
cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5)
|
||||||
|
defer cleanUp()
|
||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
ginkgo.By("Create a RC, with 0 replicas")
|
ginkgo.By("Create a RC, with 0 replicas")
|
||||||
rc := createRC(ns, "scheduler-priority-avoid-pod", int32(0), map[string]string{"name": "scheduler-priority-avoid-pod"}, f, podRequestedResource)
|
rc := createRC(ns, "scheduler-priority-avoid-pod", int32(0), map[string]string{"name": "scheduler-priority-avoid-pod"}, f, podRequestedResource)
|
||||||
@ -298,7 +301,8 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
|
|||||||
|
|
||||||
ginkgo.It("Pod should be preferably scheduled to nodes pod can tolerate", func() {
|
ginkgo.It("Pod should be preferably scheduled to nodes pod can tolerate", func() {
|
||||||
// make the nodes have balanced cpu,mem usage ratio
|
// make the nodes have balanced cpu,mem usage ratio
|
||||||
err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5)
|
cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5)
|
||||||
|
defer cleanUp()
|
||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
// Apply 10 taints to first node
|
// Apply 10 taints to first node
|
||||||
nodeName := nodeList.Items[0].Name
|
nodeName := nodeList.Items[0].Name
|
||||||
@ -360,7 +364,8 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Make the nodes have balanced cpu,mem usage.
|
// Make the nodes have balanced cpu,mem usage.
|
||||||
err := createBalancedPodForNodes(f, cs, ns, nodes, podRequestedResource, 0.5)
|
cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodes, podRequestedResource, 0.5)
|
||||||
|
defer cleanUp()
|
||||||
framework.ExpectNoError(err)
|
framework.ExpectNoError(err)
|
||||||
|
|
||||||
replicas := 4
|
replicas := 4
|
||||||
@ -425,7 +430,34 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
// createBalancedPodForNodes creates a pod per node that asks for enough resources to make all nodes have the same mem/cpu usage ratio.
|
// createBalancedPodForNodes creates a pod per node that asks for enough resources to make all nodes have the same mem/cpu usage ratio.
|
||||||
func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, ns string, nodes []v1.Node, requestedResource *v1.ResourceRequirements, ratio float64) error {
|
func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, ns string, nodes []v1.Node, requestedResource *v1.ResourceRequirements, ratio float64) (func(), error) {
|
||||||
|
cleanUp := func() {
|
||||||
|
// Delete all remaining pods
|
||||||
|
err := cs.CoreV1().Pods(ns).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{
|
||||||
|
LabelSelector: labels.SelectorFromSet(labels.Set(balancePodLabel)).String(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
framework.Logf("Failed to delete memory balanced pods: %v.", err)
|
||||||
|
} else {
|
||||||
|
err := wait.PollImmediate(2*time.Second, time.Minute, func() (bool, error) {
|
||||||
|
podList, err := cs.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{
|
||||||
|
LabelSelector: labels.SelectorFromSet(labels.Set(balancePodLabel)).String(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
framework.Logf("Failed to list memory balanced pods: %v.", err)
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if len(podList.Items) > 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
framework.Logf("Failed to wait until all memory balanced pods are deleted: %v.", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// find the max, if the node has the max,use the one, if not,use the ratio parameter
|
// find the max, if the node has the max,use the one, if not,use the ratio parameter
|
||||||
var maxCPUFraction, maxMemFraction float64 = ratio, ratio
|
var maxCPUFraction, maxMemFraction float64 = ratio, ratio
|
||||||
var cpuFractionMap = make(map[string]float64)
|
var cpuFractionMap = make(map[string]float64)
|
||||||
@ -485,7 +517,7 @@ func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, n
|
|||||||
*initPausePod(f, *podConfig), true, framework.Logf)
|
*initPausePod(f, *podConfig), true, framework.Logf)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return cleanUp, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -494,7 +526,7 @@ func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, n
|
|||||||
computeCPUMemFraction(cs, node, requestedResource)
|
computeCPUMemFraction(cs, node, requestedResource)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return cleanUp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func computeCPUMemFraction(cs clientset.Interface, node v1.Node, resource *v1.ResourceRequirements) (float64, float64) {
|
func computeCPUMemFraction(cs clientset.Interface, node v1.Node, resource *v1.ResourceRequirements) (float64, float64) {
|
||||||
|
Loading…
Reference in New Issue
Block a user