mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 04:06:03 +00:00
Merge pull request #93660 from damemi/1.19-affinity-validation
Add LabelSelector validation in Pod Affinity/AntiAffinity Filter and Score plugins
This commit is contained in:
commit
377287ad37
@ -251,6 +251,9 @@ func (pl *InterPodAffinity) PreFilter(ctx context.Context, cycleState *framework
|
|||||||
}
|
}
|
||||||
|
|
||||||
podInfo := framework.NewPodInfo(pod)
|
podInfo := framework.NewPodInfo(pod)
|
||||||
|
if podInfo.ParseError != nil {
|
||||||
|
return framework.NewStatus(framework.UnschedulableAndUnresolvable, fmt.Sprintf("parsing pod: %+v", podInfo.ParseError))
|
||||||
|
}
|
||||||
|
|
||||||
// existingPodAntiAffinityMap will be used later for efficient check on existing pods' anti-affinity
|
// existingPodAntiAffinityMap will be used later for efficient check on existing pods' anti-affinity
|
||||||
existingPodAntiAffinityMap := getTPMapMatchingExistingAntiAffinity(pod, havePodsWithAffinityNodes)
|
existingPodAntiAffinityMap := getTPMapMatchingExistingAntiAffinity(pod, havePodsWithAffinityNodes)
|
||||||
|
@ -19,6 +19,7 @@ package interpodaffinity
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
@ -772,6 +773,80 @@ func TestRequiredAffinitySingleNode(t *testing.T) {
|
|||||||
),
|
),
|
||||||
name: "PodAntiAffinity symmetry check b2: incoming pod and existing pod partially match each other on AffinityTerms",
|
name: "PodAntiAffinity symmetry check b2: incoming pod and existing pod partially match each other on AffinityTerms",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "PodAffinity fails PreFilter with an invalid affinity label syntax",
|
||||||
|
pod: createPodWithAffinityTerms(defaultNamespace, "", podLabel,
|
||||||
|
[]v1.PodAffinityTerm{
|
||||||
|
{
|
||||||
|
LabelSelector: &metav1.LabelSelector{
|
||||||
|
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||||
|
{
|
||||||
|
Key: "service",
|
||||||
|
Operator: metav1.LabelSelectorOpIn,
|
||||||
|
Values: []string{"{{.bad-value.}}"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
TopologyKey: "region",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
[]v1.PodAffinityTerm{
|
||||||
|
{
|
||||||
|
LabelSelector: &metav1.LabelSelector{
|
||||||
|
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||||
|
{
|
||||||
|
Key: "service",
|
||||||
|
Operator: metav1.LabelSelectorOpIn,
|
||||||
|
Values: []string{"antivirusscan", "value2"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
TopologyKey: "node",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
node: &node1,
|
||||||
|
wantStatus: framework.NewStatus(
|
||||||
|
framework.UnschedulableAndUnresolvable,
|
||||||
|
"invalid label value",
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PodAntiAffinity fails PreFilter with an invalid antiaffinity label syntax",
|
||||||
|
pod: createPodWithAffinityTerms(defaultNamespace, "", podLabel,
|
||||||
|
[]v1.PodAffinityTerm{
|
||||||
|
{
|
||||||
|
LabelSelector: &metav1.LabelSelector{
|
||||||
|
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||||
|
{
|
||||||
|
Key: "service",
|
||||||
|
Operator: metav1.LabelSelectorOpIn,
|
||||||
|
Values: []string{"foo"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
TopologyKey: "region",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
[]v1.PodAffinityTerm{
|
||||||
|
{
|
||||||
|
LabelSelector: &metav1.LabelSelector{
|
||||||
|
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||||
|
{
|
||||||
|
Key: "service",
|
||||||
|
Operator: metav1.LabelSelectorOpIn,
|
||||||
|
Values: []string{"{{.bad-value.}}"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
TopologyKey: "node",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
node: &node1,
|
||||||
|
wantStatus: framework.NewStatus(
|
||||||
|
framework.UnschedulableAndUnresolvable,
|
||||||
|
"invalid label value",
|
||||||
|
),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
@ -783,12 +858,15 @@ func TestRequiredAffinitySingleNode(t *testing.T) {
|
|||||||
state := framework.NewCycleState()
|
state := framework.NewCycleState()
|
||||||
preFilterStatus := p.PreFilter(context.Background(), state, test.pod)
|
preFilterStatus := p.PreFilter(context.Background(), state, test.pod)
|
||||||
if !preFilterStatus.IsSuccess() {
|
if !preFilterStatus.IsSuccess() {
|
||||||
t.Errorf("prefilter failed with status: %v", preFilterStatus)
|
if !strings.Contains(preFilterStatus.Message(), test.wantStatus.Message()) {
|
||||||
}
|
t.Errorf("prefilter failed with status: %v", preFilterStatus)
|
||||||
nodeInfo := mustGetNodeInfo(t, snapshot, test.node.Name)
|
}
|
||||||
gotStatus := p.Filter(context.Background(), state, test.pod, nodeInfo)
|
} else {
|
||||||
if !reflect.DeepEqual(gotStatus, test.wantStatus) {
|
nodeInfo := mustGetNodeInfo(t, snapshot, test.node.Name)
|
||||||
t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus)
|
gotStatus := p.Filter(context.Background(), state, test.pod, nodeInfo)
|
||||||
|
if !reflect.DeepEqual(gotStatus, test.wantStatus) {
|
||||||
|
t.Errorf("status does not match: %v, want: %v", gotStatus, test.wantStatus)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -160,9 +160,15 @@ func (pl *InterPodAffinity) PreScore(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
podInfo := framework.NewPodInfo(pod)
|
||||||
|
if podInfo.ParseError != nil {
|
||||||
|
// Ideally we never reach here, because errors will be caught by PreFilter
|
||||||
|
return framework.NewStatus(framework.Error, fmt.Sprintf("parsing pod: %+v", podInfo.ParseError))
|
||||||
|
}
|
||||||
|
|
||||||
state := &preScoreState{
|
state := &preScoreState{
|
||||||
topologyScore: make(map[string]map[string]int64),
|
topologyScore: make(map[string]map[string]int64),
|
||||||
podInfo: framework.NewPodInfo(pod),
|
podInfo: podInfo,
|
||||||
}
|
}
|
||||||
|
|
||||||
topoScores := make([]scoreMap, len(allNodes))
|
topoScores := make([]scoreMap, len(allNodes))
|
||||||
|
@ -19,6 +19,7 @@ package interpodaffinity
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
v1 "k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
@ -252,12 +253,56 @@ func TestPreferredAffinity(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
invalidAffinityLabels := &v1.Affinity{
|
||||||
|
PodAffinity: &v1.PodAffinity{
|
||||||
|
PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
|
||||||
|
{
|
||||||
|
Weight: 8,
|
||||||
|
PodAffinityTerm: v1.PodAffinityTerm{
|
||||||
|
LabelSelector: &metav1.LabelSelector{
|
||||||
|
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||||
|
{
|
||||||
|
Key: "security",
|
||||||
|
Operator: metav1.LabelSelectorOpIn,
|
||||||
|
Values: []string{"{{.bad-value.}}"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
TopologyKey: "region",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
invalidAntiAffinityLabels := &v1.Affinity{
|
||||||
|
PodAntiAffinity: &v1.PodAntiAffinity{
|
||||||
|
PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
|
||||||
|
{
|
||||||
|
Weight: 5,
|
||||||
|
PodAffinityTerm: v1.PodAffinityTerm{
|
||||||
|
LabelSelector: &metav1.LabelSelector{
|
||||||
|
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||||
|
{
|
||||||
|
Key: "security",
|
||||||
|
Operator: metav1.LabelSelectorOpIn,
|
||||||
|
Values: []string{"{{.bad-value.}}"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
TopologyKey: "az",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
pod *v1.Pod
|
pod *v1.Pod
|
||||||
pods []*v1.Pod
|
pods []*v1.Pod
|
||||||
nodes []*v1.Node
|
nodes []*v1.Node
|
||||||
expectedList framework.NodeScoreList
|
expectedList framework.NodeScoreList
|
||||||
name string
|
name string
|
||||||
|
wantStatus *framework.Status
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}},
|
pod: &v1.Pod{Spec: v1.PodSpec{NodeName: ""}, ObjectMeta: metav1.ObjectMeta{Labels: podLabelSecurityS1}},
|
||||||
@ -513,6 +558,24 @@ func TestPreferredAffinity(t *testing.T) {
|
|||||||
expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}},
|
expectedList: []framework.NodeScore{{Name: "machine1", Score: framework.MaxNodeScore}, {Name: "machine2", Score: framework.MaxNodeScore}},
|
||||||
name: "Avoid panic when partial nodes in a topology don't have pods with affinity",
|
name: "Avoid panic when partial nodes in a topology don't have pods with affinity",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "invalid Affinity fails PreScore",
|
||||||
|
pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: invalidAffinityLabels}},
|
||||||
|
wantStatus: framework.NewStatus(framework.Error, "invalid label value"),
|
||||||
|
nodes: []*v1.Node{
|
||||||
|
{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}},
|
||||||
|
{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgChina}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid AntiAffinity fails PreScore",
|
||||||
|
pod: &v1.Pod{Spec: v1.PodSpec{NodeName: "", Affinity: invalidAntiAffinityLabels}},
|
||||||
|
wantStatus: framework.NewStatus(framework.Error, "invalid label value"),
|
||||||
|
nodes: []*v1.Node{
|
||||||
|
{ObjectMeta: metav1.ObjectMeta{Name: "machine1", Labels: labelRgChina}},
|
||||||
|
{ObjectMeta: metav1.ObjectMeta{Name: "machine2", Labels: labelRgChina}},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
@ -527,25 +590,28 @@ func TestPreferredAffinity(t *testing.T) {
|
|||||||
|
|
||||||
status := p.PreScore(context.Background(), state, test.pod, test.nodes)
|
status := p.PreScore(context.Background(), state, test.pod, test.nodes)
|
||||||
if !status.IsSuccess() {
|
if !status.IsSuccess() {
|
||||||
t.Errorf("unexpected error: %v", status)
|
if !strings.Contains(status.Message(), test.wantStatus.Message()) {
|
||||||
}
|
t.Errorf("unexpected error: %v", status)
|
||||||
var gotList framework.NodeScoreList
|
}
|
||||||
for _, n := range test.nodes {
|
} else {
|
||||||
nodeName := n.ObjectMeta.Name
|
var gotList framework.NodeScoreList
|
||||||
score, status := p.Score(context.Background(), state, test.pod, nodeName)
|
for _, n := range test.nodes {
|
||||||
|
nodeName := n.ObjectMeta.Name
|
||||||
|
score, status := p.Score(context.Background(), state, test.pod, nodeName)
|
||||||
|
if !status.IsSuccess() {
|
||||||
|
t.Errorf("unexpected error: %v", status)
|
||||||
|
}
|
||||||
|
gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score})
|
||||||
|
}
|
||||||
|
|
||||||
|
status = p.ScoreExtensions().NormalizeScore(context.Background(), state, test.pod, gotList)
|
||||||
if !status.IsSuccess() {
|
if !status.IsSuccess() {
|
||||||
t.Errorf("unexpected error: %v", status)
|
t.Errorf("unexpected error: %v", status)
|
||||||
}
|
}
|
||||||
gotList = append(gotList, framework.NodeScore{Name: nodeName, Score: score})
|
|
||||||
}
|
|
||||||
|
|
||||||
status = p.ScoreExtensions().NormalizeScore(context.Background(), state, test.pod, gotList)
|
if !reflect.DeepEqual(test.expectedList, gotList) {
|
||||||
if !status.IsSuccess() {
|
t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList)
|
||||||
t.Errorf("unexpected error: %v", status)
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(test.expectedList, gotList) {
|
|
||||||
t.Errorf("expected:\n\t%+v,\ngot:\n\t%+v", test.expectedList, gotList)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
})
|
})
|
||||||
|
@ -73,6 +73,7 @@ type PodInfo struct {
|
|||||||
RequiredAntiAffinityTerms []AffinityTerm
|
RequiredAntiAffinityTerms []AffinityTerm
|
||||||
PreferredAffinityTerms []WeightedAffinityTerm
|
PreferredAffinityTerms []WeightedAffinityTerm
|
||||||
PreferredAntiAffinityTerms []WeightedAffinityTerm
|
PreferredAntiAffinityTerms []WeightedAffinityTerm
|
||||||
|
ParseError error
|
||||||
}
|
}
|
||||||
|
|
||||||
// AffinityTerm is a processed version of v1.PodAffinityTerm.
|
// AffinityTerm is a processed version of v1.PodAffinityTerm.
|
||||||
@ -88,53 +89,50 @@ type WeightedAffinityTerm struct {
|
|||||||
Weight int32
|
Weight int32
|
||||||
}
|
}
|
||||||
|
|
||||||
func newAffinityTerm(pod *v1.Pod, term *v1.PodAffinityTerm) *AffinityTerm {
|
func newAffinityTerm(pod *v1.Pod, term *v1.PodAffinityTerm) (*AffinityTerm, error) {
|
||||||
namespaces := schedutil.GetNamespacesFromPodAffinityTerm(pod, term)
|
namespaces := schedutil.GetNamespacesFromPodAffinityTerm(pod, term)
|
||||||
selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector)
|
selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Errorf("Cannot process label selector: %v", err)
|
return nil, err
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
return &AffinityTerm{Namespaces: namespaces, Selector: selector, TopologyKey: term.TopologyKey}
|
return &AffinityTerm{Namespaces: namespaces, Selector: selector, TopologyKey: term.TopologyKey}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getAffinityTerms receives a Pod and affinity terms and returns the namespaces and
|
// getAffinityTerms receives a Pod and affinity terms and returns the namespaces and
|
||||||
// selectors of the terms.
|
// selectors of the terms.
|
||||||
func getAffinityTerms(pod *v1.Pod, v1Terms []v1.PodAffinityTerm) []AffinityTerm {
|
func getAffinityTerms(pod *v1.Pod, v1Terms []v1.PodAffinityTerm) ([]AffinityTerm, error) {
|
||||||
if v1Terms == nil {
|
if v1Terms == nil {
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var terms []AffinityTerm
|
var terms []AffinityTerm
|
||||||
for _, term := range v1Terms {
|
for _, term := range v1Terms {
|
||||||
t := newAffinityTerm(pod, &term)
|
t, err := newAffinityTerm(pod, &term)
|
||||||
if t == nil {
|
if err != nil {
|
||||||
// We get here if the label selector failed to process, this is not supposed
|
// We get here if the label selector failed to process
|
||||||
// to happen because the pod should have been validated by the api server.
|
return nil, err
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
terms = append(terms, *t)
|
terms = append(terms, *t)
|
||||||
}
|
}
|
||||||
return terms
|
return terms, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getWeightedAffinityTerms returns the list of processed affinity terms.
|
// getWeightedAffinityTerms returns the list of processed affinity terms.
|
||||||
func getWeightedAffinityTerms(pod *v1.Pod, v1Terms []v1.WeightedPodAffinityTerm) []WeightedAffinityTerm {
|
func getWeightedAffinityTerms(pod *v1.Pod, v1Terms []v1.WeightedPodAffinityTerm) ([]WeightedAffinityTerm, error) {
|
||||||
if v1Terms == nil {
|
if v1Terms == nil {
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var terms []WeightedAffinityTerm
|
var terms []WeightedAffinityTerm
|
||||||
for _, term := range v1Terms {
|
for _, term := range v1Terms {
|
||||||
t := newAffinityTerm(pod, &term.PodAffinityTerm)
|
t, err := newAffinityTerm(pod, &term.PodAffinityTerm)
|
||||||
if t == nil {
|
if err != nil {
|
||||||
// We get here if the label selector failed to process, this is not supposed
|
// We get here if the label selector failed to process
|
||||||
// to happen because the pod should have been validated by the api server.
|
return nil, err
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
terms = append(terms, WeightedAffinityTerm{AffinityTerm: *t, Weight: term.Weight})
|
terms = append(terms, WeightedAffinityTerm{AffinityTerm: *t, Weight: term.Weight})
|
||||||
}
|
}
|
||||||
return terms
|
return terms, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewPodInfo return a new PodInfo
|
// NewPodInfo return a new PodInfo
|
||||||
@ -150,12 +148,32 @@ func NewPodInfo(pod *v1.Pod) *PodInfo {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Attempt to parse the affinity terms
|
||||||
|
var parseErr error
|
||||||
|
requiredAffinityTerms, err := getAffinityTerms(pod, schedutil.GetPodAffinityTerms(pod.Spec.Affinity))
|
||||||
|
if err != nil {
|
||||||
|
parseErr = fmt.Errorf("requiredAffinityTerms: %w", err)
|
||||||
|
}
|
||||||
|
requiredAntiAffinityTerms, err := getAffinityTerms(pod, schedutil.GetPodAntiAffinityTerms(pod.Spec.Affinity))
|
||||||
|
if err != nil {
|
||||||
|
parseErr = fmt.Errorf("requiredAntiAffinityTerms: %w", err)
|
||||||
|
}
|
||||||
|
weightedAffinityTerms, err := getWeightedAffinityTerms(pod, preferredAffinityTerms)
|
||||||
|
if err != nil {
|
||||||
|
parseErr = fmt.Errorf("preferredAffinityTerms: %w", err)
|
||||||
|
}
|
||||||
|
weightedAntiAffinityTerms, err := getWeightedAffinityTerms(pod, preferredAntiAffinityTerms)
|
||||||
|
if err != nil {
|
||||||
|
parseErr = fmt.Errorf("preferredAntiAffinityTerms: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return &PodInfo{
|
return &PodInfo{
|
||||||
Pod: pod,
|
Pod: pod,
|
||||||
RequiredAffinityTerms: getAffinityTerms(pod, schedutil.GetPodAffinityTerms(pod.Spec.Affinity)),
|
RequiredAffinityTerms: requiredAffinityTerms,
|
||||||
RequiredAntiAffinityTerms: getAffinityTerms(pod, schedutil.GetPodAntiAffinityTerms(pod.Spec.Affinity)),
|
RequiredAntiAffinityTerms: requiredAntiAffinityTerms,
|
||||||
PreferredAffinityTerms: getWeightedAffinityTerms(pod, preferredAffinityTerms),
|
PreferredAffinityTerms: weightedAffinityTerms,
|
||||||
PreferredAntiAffinityTerms: getWeightedAffinityTerms(pod, preferredAntiAffinityTerms),
|
PreferredAntiAffinityTerms: weightedAntiAffinityTerms,
|
||||||
|
ParseError: parseErr,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user