diff --git a/test/e2e/scheduling/predicates.go b/test/e2e/scheduling/predicates.go index c1fed6b4b7b..c67860d150f 100644 --- a/test/e2e/scheduling/predicates.go +++ b/test/e2e/scheduling/predicates.go @@ -383,259 +383,6 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() { Expect(labelPod.Spec.NodeName).To(Equal(nodeName)) }) - // labelSelector Operator is DoesNotExist but values are there in requiredDuringSchedulingIgnoredDuringExecution - // part of podAffinity, so validation fails. - It("validates that a pod with an invalid podAffinity is rejected because of the LabelSelectorRequirement is invalid", func() { - By("Trying to launch a pod with an invalid pod Affinity data.") - podName := "without-label-" + string(uuid.NewUUID()) - _, err := cs.CoreV1().Pods(ns).Create(initPausePod(f, pausePodConfig{ - Name: podName, - Labels: map[string]string{"name": "without-label"}, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "security", - Operator: metav1.LabelSelectorOpDoesNotExist, - Values: []string{"securityscan"}, - }, - }, - }, - TopologyKey: "kubernetes.io/hostname", - }, - }, - }, - }, - })) - - if err == nil || !errors.IsInvalid(err) { - framework.Failf("Expect error of invalid, got : %v", err) - } - }) - - // Test Nodes does not have any pod, hence it should be impossible to schedule a Pod with pod affinity. - It("validates that Inter-pod-Affinity is respected if not matching", func() { - By("Trying to schedule Pod with nonempty Pod Affinity.") - framework.WaitForStableCluster(cs, masterNodes) - podName := "without-label-" + string(uuid.NewUUID()) - conf := pausePodConfig{ - Name: podName, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"securityscan", "value2"}, - }, - }, - }, - TopologyKey: "kubernetes.io/hostname", - }, - }, - }, - }, - } - - WaitForSchedulerAfterAction(f, createPausePodAction(f, conf), podName, false) - verifyResult(cs, 0, 1, ns) - }) - - // test the pod affinity successful matching scenario. - It("validates that InterPodAffinity is respected if matching", func() { - nodeName, _ := runAndKeepPodWithLabelAndGetNodeName(f) - - By("Trying to apply a random label on the found node.") - k := "e2e.inter-pod-affinity.kubernetes.io/zone" - v := "china-e2etest" - framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v) - framework.ExpectNodeHasLabel(cs, nodeName, k, v) - defer framework.RemoveLabelOffNode(cs, nodeName, k) - - By("Trying to launch the pod, now with podAffinity.") - labelPodName := "with-podaffinity-" + string(uuid.NewUUID()) - createPausePod(f, pausePodConfig{ - Name: labelPodName, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "security", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"S1", "value2"}, - }, - }, - }, - TopologyKey: k, - Namespaces: []string{ns}, - }, - }, - }, - }, - }) - - // check that pod got scheduled. We intentionally DO NOT check that the - // pod is running because this will create a race condition with the - // kubelet and the scheduler: the scheduler might have scheduled a pod - // already when the kubelet does not know about its new label yet. The - // kubelet will then refuse to launch the pod. - framework.ExpectNoError(framework.WaitForPodNotPending(cs, ns, labelPodName)) - labelPod, err := cs.CoreV1().Pods(ns).Get(labelPodName, metav1.GetOptions{}) - framework.ExpectNoError(err) - Expect(labelPod.Spec.NodeName).To(Equal(nodeName)) - }) - - // test when the pod anti affinity rule is not satisfied, the pod would stay pending. - It("validates that InterPodAntiAffinity is respected if matching 2", func() { - // launch pods to find nodes which can launch a pod. We intentionally do - // not just take the node list and choose the first and the second of them. - // Depending on the cluster and the scheduler it might be that a "normal" pod - // cannot be scheduled onto it. - By("Launching two pods on two distinct nodes to get two node names") - CreateHostPortPods(f, "host-port", 2, true) - defer framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, ns, "host-port") - podList, err := cs.CoreV1().Pods(ns).List(metav1.ListOptions{}) - framework.ExpectNoError(err) - Expect(len(podList.Items)).To(Equal(2)) - nodeNames := []string{podList.Items[0].Spec.NodeName, podList.Items[1].Spec.NodeName} - Expect(nodeNames[0]).ToNot(Equal(nodeNames[1])) - - By("Applying a random label to both nodes.") - k := "e2e.inter-pod-affinity.kubernetes.io/zone" - v := "china-e2etest" - for _, nodeName := range nodeNames { - framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v) - framework.ExpectNodeHasLabel(cs, nodeName, k, v) - defer framework.RemoveLabelOffNode(cs, nodeName, k) - } - - By("Trying to launch another pod on the first node with the service label.") - podName := "with-label-" + string(uuid.NewUUID()) - - runPausePod(f, pausePodConfig{ - Name: podName, - Labels: map[string]string{"service": "S1"}, - NodeSelector: map[string]string{k: v}, // only launch on our two nodes - }) - - By("Trying to launch another pod, now with podAntiAffinity with same Labels.") - labelPodName := "with-podantiaffinity-" + string(uuid.NewUUID()) - conf := pausePodConfig{ - Name: labelPodName, - Labels: map[string]string{"service": "Diff"}, - NodeSelector: map[string]string{k: v}, // only launch on our two nodes, contradicting the podAntiAffinity - Affinity: &v1.Affinity{ - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "service", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"S1", "value2"}, - }, - }, - }, - TopologyKey: k, - Namespaces: []string{ns}, - }, - }, - }, - }, - } - - WaitForSchedulerAfterAction(f, createPausePodAction(f, conf), labelPodName, false) - verifyResult(cs, 3, 1, ns) - }) - - // test the pod affinity successful matching scenario with multiple Label Operators. - It("validates that InterPodAffinity is respected if matching with multiple Affinities", func() { - nodeName, _ := runAndKeepPodWithLabelAndGetNodeName(f) - - By("Trying to apply a random label on the found node.") - k := "e2e.inter-pod-affinity.kubernetes.io/zone" - v := "kubernetes-e2e" - framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v) - framework.ExpectNodeHasLabel(cs, nodeName, k, v) - defer framework.RemoveLabelOffNode(cs, nodeName, k) - - By("Trying to launch the pod, now with multiple pod affinities with diff LabelOperators.") - labelPodName := "with-podaffinity-" + string(uuid.NewUUID()) - createPausePod(f, pausePodConfig{ - Name: labelPodName, - Affinity: &v1.Affinity{ - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "security", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"S1", "value2"}, - }, { - Key: "security", - Operator: metav1.LabelSelectorOpNotIn, - Values: []string{"S2"}, - }, { - Key: "security", - Operator: metav1.LabelSelectorOpExists, - }, - }, - }, - TopologyKey: k, - }, - }, - }, - }, - }) - - // check that pod got scheduled. We intentionally DO NOT check that the - // pod is running because this will create a race condition with the - // kubelet and the scheduler: the scheduler might have scheduled a pod - // already when the kubelet does not know about its new label yet. The - // kubelet will then refuse to launch the pod. - framework.ExpectNoError(framework.WaitForPodNotPending(cs, ns, labelPodName)) - labelPod, err := cs.CoreV1().Pods(ns).Get(labelPodName, metav1.GetOptions{}) - framework.ExpectNoError(err) - Expect(labelPod.Spec.NodeName).To(Equal(nodeName)) - }) - - // test the pod affinity and anti affinity successful matching scenario. - It("validates that InterPod Affinity and AntiAffinity is respected if matching", func() { - nodeName, _ := runAndKeepPodWithLabelAndGetNodeName(f) - - By("Trying to apply a random label on the found node.") - k := "e2e.inter-pod-affinity.kubernetes.io/zone" - v := "e2e-testing" - framework.AddOrUpdateLabelOnNode(cs, nodeName, k, v) - framework.ExpectNodeHasLabel(cs, nodeName, k, v) - defer framework.RemoveLabelOffNode(cs, nodeName, k) - - By("Trying to launch the pod, now with Pod affinity and anti affinity.") - pod := createPodWithPodAffinity(f, k) - - // check that pod got scheduled. We intentionally DO NOT check that the - // pod is running because this will create a race condition with the - // kubelet and the scheduler: the scheduler might have scheduled a pod - // already when the kubelet does not know about its new label yet. The - // kubelet will then refuse to launch the pod. - framework.ExpectNoError(framework.WaitForPodNotPending(cs, ns, pod.Name)) - labelPod, err := cs.Core().Pods(ns).Get(pod.Name, metav1.GetOptions{}) - framework.ExpectNoError(err) - Expect(labelPod.Spec.NodeName).To(Equal(nodeName)) - }) - // 1. Run a pod to get an available node, then delete the pod // 2. Taint the node with a random taint // 3. Try to relaunch the pod with tolerations tolerate the taints on node, diff --git a/test/integration/scheduler/BUILD b/test/integration/scheduler/BUILD index 310541a2003..f9ce9514453 100644 --- a/test/integration/scheduler/BUILD +++ b/test/integration/scheduler/BUILD @@ -14,6 +14,7 @@ go_test( srcs = [ "extender_test.go", "main_test.go", + "predicates_test.go", "priorities_test.go", "scheduler_test.go", ], @@ -37,6 +38,7 @@ go_test( "//test/integration/framework:go_default_library", "//test/utils:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", diff --git a/test/integration/scheduler/predicates_test.go b/test/integration/scheduler/predicates_test.go new file mode 100644 index 00000000000..f7d673e52a2 --- /dev/null +++ b/test/integration/scheduler/predicates_test.go @@ -0,0 +1,868 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduler + +import ( + "testing" + "time" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/integration/framework" + testutils "k8s.io/kubernetes/test/utils" +) + +// This file tests the scheduler predicates functionality. + +// TestInterPodAffinity verifies that scheduler's inter pod affinity and +// anti-affinity predicate functions works correctly. +func TestInterPodAffinity(t *testing.T) { + context := initTest(t, "inter-pod-affinity") + defer cleanupTest(t, context) + // Add a few nodes. + nodes, err := createNodes(context.clientSet, "testnode", nil, 2) + if err != nil { + t.Fatalf("Cannot create nodes: %v", err) + } + // Add labels to the nodes. + labels1 := map[string]string{ + "region": "r1", + "zone": "z11", + } + for _, node := range nodes { + if err = testutils.AddLabelsToNode(context.clientSet, node.Name, labels1); err != nil { + t.Fatalf("Cannot add labels to node: %v", err) + } + if err = waitForNodeLabels(context.clientSet, node.Name, labels1); err != nil { + t.Fatalf("Adding labels to node didn't succeed: %v", err) + } + } + + cs := context.clientSet + podLabel := map[string]string{"service": "securityscan"} + podLabel2 := map[string]string{"security": "S1"} + + tests := []struct { + pod *v1.Pod + pods []*v1.Pod + node *v1.Node + fits bool + errorType string + test string + }{ + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel2, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpDoesNotExist, + Values: []string{"securityscan"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + }, + }, + node: nodes[0], + fits: false, + errorType: "invalidPod", + test: "validates that a pod with an invalid podAffinity is rejected because of the LabelSelectorRequirement is invalid", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel2, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "security", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + }, + }, + node: nodes[0], + fits: false, + test: "validates that Inter-pod-Affinity is respected if not matching", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel2, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name, + }, + }, + }, + node: nodes[0], + fits: true, + test: "validates that InterPodAffinity is respected if matching. requiredDuringSchedulingIgnoredDuringExecution in PodAffinity using In operator that matches the existing pod", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel2, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"securityscan3", "value3"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{{Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name}, + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}}}, + node: nodes[0], + fits: true, + test: "validates that InterPodAffinity is respected if matching. requiredDuringSchedulingIgnoredDuringExecution in PodAffinity using not in operator in labelSelector that matches the existing pod", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel2, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "region", + Namespaces: []string{"diff-namespace"}, + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{{Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name}, + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel, Namespace: "ns"}}}, + node: nodes[0], + fits: false, + test: "validates that inter-pod-affinity is respected when pods have different Namespaces", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"antivirusscan", "value2"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{{Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name}, ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}}}, + node: nodes[0], + fits: false, + test: "Doesn't satisfy the PodAffinity because of unmatching labelSelector with the existing pod", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel2, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpExists, + }, { + Key: "wrongkey", + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + TopologyKey: "region", + }, { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan"}, + }, { + Key: "service", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"WrongValue"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{{Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name}, ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}}}, + node: nodes[0], + fits: true, + test: "validates that InterPodAffinity is respected if matching with multiple affinities in multiple RequiredDuringSchedulingIgnoredDuringExecution ", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabel2, + Name: "fakename", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpExists, + }, { + Key: "wrongkey", + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + TopologyKey: "region", + }, { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan2"}, + }, { + Key: "service", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"WrongValue"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{{Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name}, ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}}}, + node: nodes[0], + fits: false, + test: "The labelSelector requirements(items of matchExpressions) are ANDed, the pod cannot schedule onto the node because one of the matchExpression items doesn't match.", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel2, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"antivirusscan", "value2"}, + }, + }, + }, + TopologyKey: "node", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{{Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name}, ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}}}, + node: nodes[0], + fits: true, + test: "validates that InterPod Affinity and AntiAffinity is respected if matching", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel2, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"antivirusscan", "value2"}, + }, + }, + }, + TopologyKey: "node", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{ + { + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name, + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"antivirusscan", "value2"}, + }, + }, + }, + TopologyKey: "node", + }, + }, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}, + }, + }, + node: nodes[0], + fits: true, + test: "satisfies the PodAffinity and PodAntiAffinity and PodAntiAffinity symmetry with the existing pod", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel2, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "zone", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{{Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name}, ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}}}, + node: nodes[0], + fits: false, + test: "satisfies the PodAffinity but doesn't satisfies the PodAntiAffinity with the existing pod", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"antivirusscan", "value2"}, + }, + }, + }, + TopologyKey: "node", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{ + { + Spec: v1.PodSpec{ + NodeName: nodes[0].Name, + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan", "value3"}, + }, + }, + }, + TopologyKey: "zone", + }, + }, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}, + }, + }, + node: nodes[0], + fits: false, + test: "satisfies the PodAffinity and PodAntiAffinity but doesn't satisfies PodAntiAffinity symmetry with the existing pod", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{{Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: "machine2"}, ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}}}, + node: nodes[0], + fits: false, + test: "pod matches its own Label in PodAffinity and that matches the existing pod Labels", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename", + Labels: podLabel, + }, + Spec: v1.PodSpec{Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}}, + }, + pods: []*v1.Pod{ + { + Spec: v1.PodSpec{NodeName: nodes[0].Name, + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "zone", + }, + }, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "fakename2", + Labels: podLabel}, + }, + }, + node: nodes[0], + fits: false, + test: "Verify that PodAntiAffinity of an existing pod is respected when PodAntiAffinity symmetry is not satisfied with the existing pod", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-name", + Labels: podLabel, + }, + Spec: v1.PodSpec{Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}}, + }, + pods: []*v1.Pod{ + { + Spec: v1.PodSpec{NodeName: nodes[0].Name, + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "service", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"securityscan", "value2"}, + }, + }, + }, + TopologyKey: "zone", + }, + }, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-name2", + Labels: podLabel}, + }, + }, + node: nodes[0], + fits: true, + test: "Verify that PodAntiAffinity from existing pod is respected when pod statisfies PodAntiAffinity symmetry with the existing pod", + }, + { + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "fake-name2"}, + Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeSelector: map[string]string{"region": "r1"}, + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "foo", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"abc"}, + }, + }, + }, + TopologyKey: "region", + }, + }, + }, + }, + }, + }, + pods: []*v1.Pod{ + {Spec: v1.PodSpec{ + Containers: []v1.Container{{Name: "container", Image: framework.GetPauseImageName(cs)}}, + NodeName: nodes[0].Name}, ObjectMeta: metav1.ObjectMeta{Name: "fakename", Labels: map[string]string{"foo": "abc"}}}, + }, + fits: false, + test: "nodes[0] and nodes[1] have same topologyKey and label value. nodes[0] has an existing pod that matches the inter pod affinity rule. The new pod can not be scheduled onto either of the two nodes.", + }, + } + for _, test := range tests { + for _, pod := range test.pods { + var nsName string + if pod.Namespace != "" { + nsName = pod.Namespace + } else { + nsName = context.ns.Name + } + createdPod, err := cs.Core().Pods(nsName).Create(pod) + if err != nil { + t.Fatalf("Test Failed: error, %v, while creating pod during test: %v", err, test.test) + } + err = wait.Poll(time.Second, wait.ForeverTestTimeout, podScheduled(cs, createdPod.Namespace, createdPod.Name)) + if err != nil { + t.Errorf("Test Failed: error, %v, while waiting for pod during test, %v", err, test) + } + } + testPod, err := cs.Core().Pods(context.ns.Name).Create(test.pod) + if err != nil { + if !(test.errorType == "invalidPod" && errors.IsInvalid(err)) { + t.Fatalf("Test Failed: error, %v, while creating pod during test: %v", err, test.test) + } + } else { + err = wait.Poll(time.Second, wait.ForeverTestTimeout, podScheduled(cs, testPod.Namespace, testPod.Name)) + if err != nil && err != wait.ErrWaitTimeout { + t.Errorf("Test Failed: error, %v, while waiting for pod to get scheduled, %v", err, test.test) + } + if (err == nil) != test.fits { + t.Errorf("Test Failed: %v, err %v, test.fits %v", test.test, err, test.fits) + } + + for _, pod := range test.pods { + var nsName string + if pod.Namespace != "" { + nsName = pod.Namespace + } else { + nsName = context.ns.Name + } + err = cs.Core().Pods(nsName).Delete(pod.Name, metav1.NewDeleteOptions(0)) + if err != nil { + t.Errorf("Test Failed: error, %v, while deleting pod during test: %v", err, test.test) + } + err = wait.Poll(time.Second, wait.ForeverTestTimeout, podDeleted(cs, nsName, pod.Name)) + if err != nil { + t.Errorf("Test Failed: error, %v, while waiting for pod to get deleted, %v", err, test.test) + } + } + err = cs.Core().Pods(context.ns.Name).Delete(test.pod.Name, metav1.NewDeleteOptions(0)) + if err != nil { + t.Errorf("Test Failed: error, %v, while deleting pod during test: %v", err, test.test) + } + err = wait.Poll(time.Second, wait.ForeverTestTimeout, podDeleted(cs, context.ns.Name, test.pod.Name)) + if err != nil { + t.Errorf("Test Failed: error, %v, while waiting for pod to get deleted, %v", err, test.test) + } + } + } +} diff --git a/test/integration/scheduler/util.go b/test/integration/scheduler/util.go index a018ab2e4c6..2c62bcc7404 100644 --- a/test/integration/scheduler/util.go +++ b/test/integration/scheduler/util.go @@ -282,6 +282,17 @@ func runPausePod(cs clientset.Interface, conf *pausePodConfig) (*v1.Pod, error) return pod, nil } +// podDeleted returns true if a pod is not found in the given namespace. +func podDeleted(c clientset.Interface, podNamespace, podName string) wait.ConditionFunc { + return func() (bool, error) { + _, err := c.Core().Pods(podNamespace).Get(podName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return true, nil + } + return false, nil + } +} + // podScheduled returns true if a node is assigned to the given pod. func podScheduled(c clientset.Interface, podNamespace, podName string) wait.ConditionFunc { return func() (bool, error) {