Merge pull request #94684 from ingvagabund/e2e-priorities-avoidPods-wait-for-pods-getting-deleted

[sig-scheduling] SchedulerPriorities [Serial] Pod should avoid nodes that have avoidPod annotation: clean remaining pods
This commit is contained in:
Kubernetes Prow Robot 2021-01-21 09:57:01 -08:00 committed by GitHub
commit 759ac2e8c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 48 additions and 10 deletions

View File

@ -887,8 +887,8 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
Name: conf.Name, Name: conf.Name,
Namespace: conf.Namespace, Namespace: conf.Namespace,
Labels: conf.Labels, Labels: map[string]string{},
Annotations: conf.Annotations, Annotations: map[string]string{},
OwnerReferences: conf.OwnerReferences, OwnerReferences: conf.OwnerReferences,
}, },
Spec: v1.PodSpec{ Spec: v1.PodSpec{
@ -908,6 +908,12 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
TerminationGracePeriodSeconds: &gracePeriod, TerminationGracePeriodSeconds: &gracePeriod,
}, },
} }
for key, value := range conf.Labels {
pod.ObjectMeta.Labels[key] = value
}
for key, value := range conf.Annotations {
pod.ObjectMeta.Annotations[key] = value
}
// TODO: setting the Pod's nodeAffinity instead of setting .spec.nodeName works around the // TODO: setting the Pod's nodeAffinity instead of setting .spec.nodeName works around the
// Preemption e2e flake (#88441), but we should investigate deeper to get to the bottom of it. // Preemption e2e flake (#88441), but we should investigate deeper to get to the bottom of it.
if len(conf.NodeName) != 0 { if len(conf.NodeName) != 0 {

View File

@ -32,6 +32,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
@ -52,7 +53,7 @@ type Resource struct {
Memory int64 Memory int64
} }
var balancePodLabel = map[string]string{"name": "priority-balanced-memory"} var balancePodLabel = map[string]string{"podname": "priority-balanced-memory"}
var podRequestedResource = &v1.ResourceRequirements{ var podRequestedResource = &v1.ResourceRequirements{
Limits: v1.ResourceList{ Limits: v1.ResourceList{
@ -187,7 +188,8 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
} }
// make the nodes have balanced cpu,mem usage // make the nodes have balanced cpu,mem usage
err = createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.6) cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.6)
defer cleanUp()
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("Trying to launch the pod with podAntiAffinity.") ginkgo.By("Trying to launch the pod with podAntiAffinity.")
labelPodName := "pod-with-pod-antiaffinity" labelPodName := "pod-with-pod-antiaffinity"
@ -236,7 +238,8 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
ginkgo.It("Pod should avoid nodes that have avoidPod annotation", func() { ginkgo.It("Pod should avoid nodes that have avoidPod annotation", func() {
nodeName := nodeList.Items[0].Name nodeName := nodeList.Items[0].Name
// make the nodes have balanced cpu,mem usage // make the nodes have balanced cpu,mem usage
err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5) cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5)
defer cleanUp()
framework.ExpectNoError(err) framework.ExpectNoError(err)
ginkgo.By("Create a RC, with 0 replicas") ginkgo.By("Create a RC, with 0 replicas")
rc := createRC(ns, "scheduler-priority-avoid-pod", int32(0), map[string]string{"name": "scheduler-priority-avoid-pod"}, f, podRequestedResource) rc := createRC(ns, "scheduler-priority-avoid-pod", int32(0), map[string]string{"name": "scheduler-priority-avoid-pod"}, f, podRequestedResource)
@ -298,7 +301,8 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
ginkgo.It("Pod should be preferably scheduled to nodes pod can tolerate", func() { ginkgo.It("Pod should be preferably scheduled to nodes pod can tolerate", func() {
// make the nodes have balanced cpu,mem usage ratio // make the nodes have balanced cpu,mem usage ratio
err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5) cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodeList.Items, podRequestedResource, 0.5)
defer cleanUp()
framework.ExpectNoError(err) framework.ExpectNoError(err)
// Apply 10 taints to first node // Apply 10 taints to first node
nodeName := nodeList.Items[0].Name nodeName := nodeList.Items[0].Name
@ -360,7 +364,8 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
} }
// Make the nodes have balanced cpu,mem usage. // Make the nodes have balanced cpu,mem usage.
err := createBalancedPodForNodes(f, cs, ns, nodes, podRequestedResource, 0.5) cleanUp, err := createBalancedPodForNodes(f, cs, ns, nodes, podRequestedResource, 0.5)
defer cleanUp()
framework.ExpectNoError(err) framework.ExpectNoError(err)
replicas := 4 replicas := 4
@ -425,7 +430,34 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
}) })
// createBalancedPodForNodes creates a pod per node that asks for enough resources to make all nodes have the same mem/cpu usage ratio. // createBalancedPodForNodes creates a pod per node that asks for enough resources to make all nodes have the same mem/cpu usage ratio.
func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, ns string, nodes []v1.Node, requestedResource *v1.ResourceRequirements, ratio float64) error { func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, ns string, nodes []v1.Node, requestedResource *v1.ResourceRequirements, ratio float64) (func(), error) {
cleanUp := func() {
// Delete all remaining pods
err := cs.CoreV1().Pods(ns).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(labels.Set(balancePodLabel)).String(),
})
if err != nil {
framework.Logf("Failed to delete memory balanced pods: %v.", err)
} else {
err := wait.PollImmediate(2*time.Second, time.Minute, func() (bool, error) {
podList, err := cs.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{
LabelSelector: labels.SelectorFromSet(labels.Set(balancePodLabel)).String(),
})
if err != nil {
framework.Logf("Failed to list memory balanced pods: %v.", err)
return false, nil
}
if len(podList.Items) > 0 {
return false, nil
}
return true, nil
})
if err != nil {
framework.Logf("Failed to wait until all memory balanced pods are deleted: %v.", err)
}
}
}
// find the max, if the node has the max,use the one, if not,use the ratio parameter // find the max, if the node has the max,use the one, if not,use the ratio parameter
var maxCPUFraction, maxMemFraction float64 = ratio, ratio var maxCPUFraction, maxMemFraction float64 = ratio, ratio
var cpuFractionMap = make(map[string]float64) var cpuFractionMap = make(map[string]float64)
@ -485,7 +517,7 @@ func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, n
*initPausePod(f, *podConfig), true, framework.Logf) *initPausePod(f, *podConfig), true, framework.Logf)
if err != nil { if err != nil {
return err return cleanUp, err
} }
} }
@ -494,7 +526,7 @@ func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, n
computeCPUMemFraction(cs, node, requestedResource) computeCPUMemFraction(cs, node, requestedResource)
} }
return nil return cleanUp, nil
} }
func computeCPUMemFraction(cs clientset.Interface, node v1.Node, resource *v1.ResourceRequirements) (float64, float64) { func computeCPUMemFraction(cs clientset.Interface, node v1.Node, resource *v1.ResourceRequirements) (float64, float64) {