Merge pull request #84795 from ahg-g/ahg-custom

Convert multiple node label predicates to be a single filter plugin
This commit is contained in:
Kubernetes Prow Robot 2019-11-05 11:45:11 -08:00 committed by GitHub
commit 75aca1fe03
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 127 additions and 72 deletions

View File

@ -940,16 +940,18 @@ func PodFitsHost(pod *v1.Pod, meta PredicateMetadata, nodeInfo *schedulernodeinf
// NodeLabelChecker contains information to check node labels for a predicate. // NodeLabelChecker contains information to check node labels for a predicate.
type NodeLabelChecker struct { type NodeLabelChecker struct {
labels []string // presentLabels should be present for the node to be considered a fit for hosting the pod
presence bool presentLabels []string
// absentLabels should be absent for the node to be considered a fit for hosting the pod
absentLabels []string
} }
// NewNodeLabelPredicate creates a predicate which evaluates whether a pod can fit based on the // NewNodeLabelPredicate creates a predicate which evaluates whether a pod can fit based on the
// node labels which match a filter that it requests. // node labels which match a filter that it requests.
func NewNodeLabelPredicate(labels []string, presence bool) FitPredicate { func NewNodeLabelPredicate(presentLabels []string, absentLabels []string) FitPredicate {
labelChecker := &NodeLabelChecker{ labelChecker := &NodeLabelChecker{
labels: labels, presentLabels: presentLabels,
presence: presence, absentLabels: absentLabels,
} }
return labelChecker.CheckNodeLabelPresence return labelChecker.CheckNodeLabelPresence
} }
@ -972,15 +974,21 @@ func (n *NodeLabelChecker) CheckNodeLabelPresence(pod *v1.Pod, meta PredicateMet
return false, nil, fmt.Errorf("node not found") return false, nil, fmt.Errorf("node not found")
} }
var exists bool
nodeLabels := labels.Set(node.Labels) nodeLabels := labels.Set(node.Labels)
for _, label := range n.labels { check := func(labels []string, presence bool) bool {
exists = nodeLabels.Has(label) for _, label := range labels {
if (exists && !n.presence) || (!exists && n.presence) { exists := nodeLabels.Has(label)
return false, []PredicateFailureReason{ErrNodeLabelPresenceViolated}, nil if (exists && !presence) || (!exists && presence) {
return false
}
} }
return true
} }
return true, nil, nil if check(n.presentLabels, true) && check(n.absentLabels, false) {
return true, nil, nil
}
return false, []PredicateFailureReason{ErrNodeLabelPresenceViolated}, nil
} }
// ServiceAffinity defines a struct used for creating service affinity predicates. // ServiceAffinity defines a struct used for creating service affinity predicates.

View File

@ -1658,47 +1658,41 @@ func TestPodFitsSelector(t *testing.T) {
func TestNodeLabelPresence(t *testing.T) { func TestNodeLabelPresence(t *testing.T) {
label := map[string]string{"foo": "bar", "bar": "foo"} label := map[string]string{"foo": "bar", "bar": "foo"}
tests := []struct { tests := []struct {
pod *v1.Pod pod *v1.Pod
labels []string presentLabels []string
presence bool absentLabels []string
fits bool fits bool
name string name string
}{ }{
{ {
labels: []string{"baz"}, presentLabels: []string{"baz"},
presence: true, fits: false,
fits: false, name: "label does not match, presence true",
name: "label does not match, presence true",
}, },
{ {
labels: []string{"baz"}, absentLabels: []string{"baz"},
presence: false, fits: true,
fits: true, name: "label does not match, presence false",
name: "label does not match, presence false",
}, },
{ {
labels: []string{"foo", "baz"}, presentLabels: []string{"foo", "baz"},
presence: true, fits: false,
fits: false, name: "one label matches, presence true",
name: "one label matches, presence true",
}, },
{ {
labels: []string{"foo", "baz"}, absentLabels: []string{"foo", "baz"},
presence: false, fits: false,
fits: false, name: "one label matches, presence false",
name: "one label matches, presence false",
}, },
{ {
labels: []string{"foo", "bar"}, presentLabels: []string{"foo", "bar"},
presence: true, fits: true,
fits: true, name: "all labels match, presence true",
name: "all labels match, presence true",
}, },
{ {
labels: []string{"foo", "bar"}, absentLabels: []string{"foo", "bar"},
presence: false, fits: false,
fits: false, name: "all labels match, presence false",
name: "all labels match, presence false",
}, },
} }
expectedFailureReasons := []PredicateFailureReason{ErrNodeLabelPresenceViolated} expectedFailureReasons := []PredicateFailureReason{ErrNodeLabelPresenceViolated}
@ -1709,7 +1703,7 @@ func TestNodeLabelPresence(t *testing.T) {
nodeInfo := schedulernodeinfo.NewNodeInfo() nodeInfo := schedulernodeinfo.NewNodeInfo()
nodeInfo.SetNode(&node) nodeInfo.SetNode(&node)
labelChecker := NodeLabelChecker{test.labels, test.presence} labelChecker := NodeLabelChecker{test.presentLabels, test.absentLabels}
fits, reasons, err := labelChecker.CheckNodeLabelPresence(test.pod, GetPredicateMetadata(test.pod, nil), nodeInfo) fits, reasons, err := labelChecker.CheckNodeLabelPresence(test.pod, GetPredicateMetadata(test.pod, nil), nodeInfo)
if err != nil { if err != nil {
t.Errorf("unexpected error: %v", err) t.Errorf("unexpected error: %v", err)

View File

@ -287,17 +287,23 @@ func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy, args *plugi
} }
} else if policy.Argument.LabelsPresence != nil { } else if policy.Argument.LabelsPresence != nil {
// map LabelPresence policy to ConfigProducerArgs that's used to configure the NodeLabel plugin. // map LabelPresence policy to ConfigProducerArgs that's used to configure the NodeLabel plugin.
args.NodeLabelArgs = &nodelabel.Args{ if args.NodeLabelArgs == nil {
Labels: policy.Argument.LabelsPresence.Labels, args.NodeLabelArgs = &nodelabel.Args{}
Presence: policy.Argument.LabelsPresence.Presence,
} }
predicateFactory = func(args PluginFactoryArgs) predicates.FitPredicate { if policy.Argument.LabelsPresence.Presence {
args.NodeLabelArgs.PresentLabels = append(args.NodeLabelArgs.PresentLabels, policy.Argument.LabelsPresence.Labels...)
} else {
args.NodeLabelArgs.AbsentLabels = append(args.NodeLabelArgs.AbsentLabels, policy.Argument.LabelsPresence.Labels...)
}
predicateFactory = func(_ PluginFactoryArgs) predicates.FitPredicate {
return predicates.NewNodeLabelPredicate( return predicates.NewNodeLabelPredicate(
policy.Argument.LabelsPresence.Labels, args.NodeLabelArgs.PresentLabels,
policy.Argument.LabelsPresence.Presence, args.NodeLabelArgs.AbsentLabels,
) )
} }
// We do not allow specifying the name for custom plugins, see #83472 // We use the NodeLabel plugin name for all NodeLabel custom priorities.
// It may get called multiple times but we essentially only register one instance of NodeLabel predicate.
// This name is then used to find the registered plugin and run the plugin instead of the predicate.
name = nodelabel.Name name = nodelabel.Name
} }
} else if predicateFactory, ok = fitPredicateMap[policy.Name]; ok { } else if predicateFactory, ok = fitPredicateMap[policy.Name]; ok {

View File

@ -92,6 +92,7 @@ func TestCreateFromConfig(t *testing.T) {
"predicates" : [ "predicates" : [
{"name" : "TestZoneAffinity", "argument" : {"serviceAffinity" : {"labels" : ["zone"]}}}, {"name" : "TestZoneAffinity", "argument" : {"serviceAffinity" : {"labels" : ["zone"]}}},
{"name" : "TestRequireZone", "argument" : {"labelsPresence" : {"labels" : ["zone"], "presence" : true}}}, {"name" : "TestRequireZone", "argument" : {"labelsPresence" : {"labels" : ["zone"], "presence" : true}}},
{"name" : "TestNoFooLabel", "argument" : {"labelsPresence" : {"labels" : ["foo"], "presence" : false}}},
{"name" : "PredicateOne"}, {"name" : "PredicateOne"},
{"name" : "PredicateTwo"} {"name" : "PredicateTwo"}
], ],
@ -123,7 +124,7 @@ func TestCreateFromConfig(t *testing.T) {
if err != nil { if err != nil {
t.Errorf("Failed to marshal %+v: %v", nodeLabelConfig, err) t.Errorf("Failed to marshal %+v: %v", nodeLabelConfig, err)
} }
want := `{"Name":"NodeLabel","Args":{"labels":["zone"],"presence":true}}` want := `{"Name":"NodeLabel","Args":{"presentLabels":["zone"],"absentLabels":["foo"]}}`
if string(encoding) != want { if string(encoding) != want {
t.Errorf("Config for NodeLabel plugin mismatch. got: %v, want: %v", string(encoding), want) t.Errorf("Config for NodeLabel plugin mismatch. got: %v, want: %v", string(encoding), want)
} }

View File

@ -18,6 +18,7 @@ package nodelabel
import ( import (
"context" "context"
"fmt"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
@ -32,11 +33,24 @@ const Name = "NodeLabel"
// Args holds the args that are used to configure the plugin. // Args holds the args that are used to configure the plugin.
type Args struct { type Args struct {
// The list of labels that identify node "groups" // PresentLabels should be present for the node to be considered a fit for hosting the pod
// All of the labels should be either present (or absent) for the node to be considered a fit for hosting the pod PresentLabels []string `json:"presentLabels,omitempty"`
Labels []string `json:"labels,omitempty"` // AbsentLabels should be absent for the node to be considered a fit for hosting the pod
// The boolean flag that indicates whether the labels should be present or absent from the node AbsentLabels []string `json:"absentLabels,omitempty"`
Presence bool `json:"presence,omitempty"` }
// validateArgs validates that PresentLabels and AbsentLabels do not conflict.
func validateArgs(args *Args) error {
presentLabels := make(map[string]struct{}, len(args.PresentLabels))
for _, l := range args.PresentLabels {
presentLabels[l] = struct{}{}
}
for _, l := range args.AbsentLabels {
if _, ok := presentLabels[l]; ok {
return fmt.Errorf("detecting at least one label (e.g., %q) that exist in both the present and absent label list: %+v", l, args)
}
}
return nil
} }
// New initializes a new plugin and returns it. // New initializes a new plugin and returns it.
@ -45,8 +59,11 @@ func New(plArgs *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin
if err := framework.DecodeInto(plArgs, args); err != nil { if err := framework.DecodeInto(plArgs, args); err != nil {
return nil, err return nil, err
} }
if err := validateArgs(args); err != nil {
return nil, err
}
return &NodeLabel{ return &NodeLabel{
predicate: predicates.NewNodeLabelPredicate(args.Labels, args.Presence), predicate: predicates.NewNodeLabelPredicate(args.PresentLabels, args.AbsentLabels),
}, nil }, nil
} }

View File

@ -27,42 +27,71 @@ import (
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo" schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"
) )
func TestNodeLabelPresence(t *testing.T) { func TestValidateNodeLabelArgs(t *testing.T) {
label := map[string]string{"foo": "bar", "bar": "foo"} // "bar" exists in both present and absent labels therefore validatio should fail.
args := &runtime.Unknown{Raw: []byte(`{"presentLabels" : ["foo", "bar"], "absentLabels" : ["bar", "baz"]}`)}
_, err := New(args, nil)
if err == nil {
t.Fatal("Plugin initialization should fail.")
}
}
func TestNodeLabelFilter(t *testing.T) {
label := map[string]string{"foo": "any value", "bar": "any value"}
var pod *v1.Pod
tests := []struct { tests := []struct {
name string name string
pod *v1.Pod
rawArgs string rawArgs string
res framework.Code res framework.Code
}{ }{
{ {
name: "label does not match, presence true", name: "present label does not match",
rawArgs: `{"labels" : ["baz"], "presence" : true}`, rawArgs: `{"presentLabels" : ["baz"]}`,
res: framework.UnschedulableAndUnresolvable, res: framework.UnschedulableAndUnresolvable,
}, },
{ {
name: "label does not match, presence false", name: "absent label does not match",
rawArgs: `{"labels" : ["baz"], "presence" : false}`, rawArgs: `{"absentLabels" : ["baz"]}`,
res: framework.Success, res: framework.Success,
}, },
{ {
name: "one label matches, presence true", name: "one of two present labels matches",
rawArgs: `{"labels" : ["foo", "baz"], "presence" : true}`, rawArgs: `{"presentLabels" : ["foo", "baz"]}`,
res: framework.UnschedulableAndUnresolvable, res: framework.UnschedulableAndUnresolvable,
}, },
{ {
name: "one label matches, presence false", name: "one of two absent labels matches",
rawArgs: `{"labels" : ["foo", "baz"], "presence" : false}`, rawArgs: `{"absentLabels" : ["foo", "baz"]}`,
res: framework.UnschedulableAndUnresolvable, res: framework.UnschedulableAndUnresolvable,
}, },
{ {
name: "all labels match, presence true", name: "all present abels match",
rawArgs: `{"labels" : ["foo", "bar"], "presence" : true}`, rawArgs: `{"presentLabels" : ["foo", "bar"]}`,
res: framework.Success, res: framework.Success,
}, },
{ {
name: "all labels match, presence false", name: "all absent labels match",
rawArgs: `{"labels" : ["foo", "bar"], "presence" : false}`, rawArgs: `{"absentLabels" : ["foo", "bar"]}`,
res: framework.UnschedulableAndUnresolvable,
},
{
name: "both present and absent label matches",
rawArgs: `{"presentLabels" : ["foo"], "absentLabels" : ["bar"]}`,
res: framework.UnschedulableAndUnresolvable,
},
{
name: "neither present nor absent label matches",
rawArgs: `{"presentLabels" : ["foz"], "absentLabels" : ["baz"]}`,
res: framework.UnschedulableAndUnresolvable,
},
{
name: "present label matches and absent label doesn't match",
rawArgs: `{"presentLabels" : ["foo"], "absentLabels" : ["baz"]}`,
res: framework.Success,
},
{
name: "present label doesn't match and absent label matches",
rawArgs: `{"presentLabels" : ["foz"], "absentLabels" : ["bar"]}`,
res: framework.UnschedulableAndUnresolvable, res: framework.UnschedulableAndUnresolvable,
}, },
} }
@ -79,7 +108,7 @@ func TestNodeLabelPresence(t *testing.T) {
t.Fatalf("Failed to create plugin: %v", err) t.Fatalf("Failed to create plugin: %v", err)
} }
status := p.(framework.FilterPlugin).Filter(context.TODO(), nil, test.pod, nodeInfo) status := p.(framework.FilterPlugin).Filter(context.TODO(), nil, pod, nodeInfo)
if status.Code() != test.res { if status.Code() != test.res {
t.Errorf("Status mismatch. got: %v, want: %v", status.Code(), test.res) t.Errorf("Status mismatch. got: %v, want: %v", status.Code(), test.res)
} }

View File

@ -48,7 +48,7 @@ var (
// BenchmarkScheduling benchmarks the scheduling rate when the cluster has // BenchmarkScheduling benchmarks the scheduling rate when the cluster has
// various quantities of nodes and scheduled pods. // various quantities of nodes and scheduled pods.
func BenchmarkScheduling(b *testing.B) { func BenchmarkSchedulingV(b *testing.B) {
tests := []struct{ nodes, existingPods, minPods int }{ tests := []struct{ nodes, existingPods, minPods int }{
{nodes: 100, existingPods: 0, minPods: 100}, {nodes: 100, existingPods: 0, minPods: 100},
{nodes: 100, existingPods: 1000, minPods: 100}, {nodes: 100, existingPods: 1000, minPods: 100},