github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-13 13:55:41 +00:00
Code Issues Projects Releases Wiki Activity

[KCCM]: have providerID trigger re-sync, but not be required for nodes

Browse Source
This commit is contained in:
Alexander Constantinescu 2023-04-26 01:43:11 +02:00
parent ff1e3ee26e
commit 3c45b61b64
2 changed files with 182 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
staging/src/k8s.io/cloud-provider/controllers/service/controller.go
View File

@ -948,17 +948,14 @@ var (
nodeIncludedPredicate, nodeIncludedPredicate,
nodeUnTaintedPredicate, nodeUnTaintedPredicate,
nodeReadyPredicate, nodeReadyPredicate,
nodeHasProviderIDPredicate,
} }
etpLocalNodePredicates []NodeConditionPredicate = []NodeConditionPredicate{ etpLocalNodePredicates []NodeConditionPredicate = []NodeConditionPredicate{
nodeIncludedPredicate, nodeIncludedPredicate,
nodeUnTaintedPredicate, nodeUnTaintedPredicate,
nodeHasProviderIDPredicate,
} }
stableNodeSetPredicates []NodeConditionPredicate = []NodeConditionPredicate{ stableNodeSetPredicates []NodeConditionPredicate = []NodeConditionPredicate{
nodeNotDeletedPredicate, nodeNotDeletedPredicate,
nodeIncludedPredicate, nodeIncludedPredicate,
nodeHasProviderIDPredicate,
// This is not perfect, but probably good enough. We won't update the // This is not perfect, but probably good enough. We won't update the
// LBs just because the taint was added (see shouldSyncUpdatedNode) but // LBs just because the taint was added (see shouldSyncUpdatedNode) but
// if any other situation causes an LB sync, tainted nodes will be // if any other situation causes an LB sync, tainted nodes will be
@ -978,16 +975,19 @@ func getNodePredicatesForService(service *v1.Service) []NodeConditionPredicate {
return allNodePredicates return allNodePredicates
} }
// This predicate just validates if the providerID has been set and triggers a
// node sync. It is _not_ used when determining which nodes to use when
// configuring the load balancer's backend pool.
func nodeHasProviderIDPredicate(node *v1.Node) bool {
return node.Spec.ProviderID != ""
}
// We consider the node for load balancing only when the node is not labelled for exclusion. // We consider the node for load balancing only when the node is not labelled for exclusion.
func nodeIncludedPredicate(node *v1.Node) bool { func nodeIncludedPredicate(node *v1.Node) bool {
_, hasExcludeBalancerLabel := node.Labels[v1.LabelNodeExcludeBalancers] _, hasExcludeBalancerLabel := node.Labels[v1.LabelNodeExcludeBalancers]
return !hasExcludeBalancerLabel return !hasExcludeBalancerLabel
} }
func nodeHasProviderIDPredicate(node *v1.Node) bool {
return node.Spec.ProviderID != ""
}
// We consider the node for load balancing only when its not tainted for deletion by the cluster autoscaler. // We consider the node for load balancing only when its not tainted for deletion by the cluster autoscaler.
func nodeUnTaintedPredicate(node *v1.Node) bool { func nodeUnTaintedPredicate(node *v1.Node) bool {
for _, taint := range node.Spec.Taints { for _, taint := range node.Spec.Taints {

297
staging/src/k8s.io/cloud-provider/controllers/service/controller_test.go
View File

@ -1779,7 +1779,7 @@ func Test_respectsPredicates(t *testing.T) {
}{ }{
{want: false, input: &v1.Node{}}, {want: false, input: &v1.Node{}},
{want: true, input: &v1.Node{Spec: v1.NodeSpec{ProviderID: providerID}, Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}}}}, {want: true, input: &v1.Node{Spec: v1.NodeSpec{ProviderID: providerID}, Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}}}},
{want: false, input: &v1.Node{Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}}}}, {want: true, input: &v1.Node{Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}}}},
{want: false, input: &v1.Node{Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionFalse}}}}}, {want: false, input: &v1.Node{Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionFalse}}}}},
{want: true, input: &v1.Node{Spec: v1.NodeSpec{ProviderID: providerID}, Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}}, ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{}}}}, {want: true, input: &v1.Node{Spec: v1.NodeSpec{ProviderID: providerID}, Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}}, ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{}}}},
{want: false, input: &v1.Node{Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}}, ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1.LabelNodeExcludeBalancers: ""}}}}, {want: false, input: &v1.Node{Status: v1.NodeStatus{Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: v1.ConditionTrue}}}, ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1.LabelNodeExcludeBalancers: ""}}}},
@ -2144,143 +2144,196 @@ func Test_shouldSyncUpdatedNode_individualPredicates(t *testing.T) {
} }
func Test_shouldSyncUpdatedNode_compoundedPredicates(t *testing.T) { func Test_shouldSyncUpdatedNode_compoundedPredicates(t *testing.T) {
type testCase struct {
name string
oldNode *v1.Node
newNode *v1.Node
shouldSync bool
fgEnabled bool
}
testcases := []testCase{}
for _, fgEnabled := range []bool{true, false} { for _, fgEnabled := range []bool{true, false} {
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StableLoadBalancerNodeSet, fgEnabled)() testcases = append(testcases, []testCase{
testcases := []struct { {
name string name: "tainted T, excluded F->T",
oldNode *v1.Node oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint)),
newNode *v1.Node newNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
shouldSync bool shouldSync: true,
}{{ fgEnabled: fgEnabled,
name: "tainted T, excluded F->T", }, {
oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint)), name: "tainted T, excluded T->F",
newNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
shouldSync: true, newNode: makeNode(tweakAddTaint(ToBeDeletedTaint)),
}, { shouldSync: true,
name: "tainted T, excluded T->F", fgEnabled: fgEnabled,
oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), }, {
newNode: makeNode(tweakAddTaint(ToBeDeletedTaint)), name: "tainted T, providerID set F->T",
shouldSync: true, oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakUnsetProviderID()),
}, { newNode: makeNode(tweakAddTaint(ToBeDeletedTaint)),
name: "tainted T, providerID set F->T", shouldSync: true,
oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakUnsetProviderID()), fgEnabled: fgEnabled,
newNode: makeNode(tweakAddTaint(ToBeDeletedTaint)), }, {
shouldSync: true, name: "tainted T, providerID set T->F",
}, { oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint)),
name: "tainted T, providerID set T->F", newNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakUnsetProviderID()),
oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint)), shouldSync: true,
newNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakUnsetProviderID()), fgEnabled: fgEnabled,
shouldSync: true, }, {
}, { name: "tainted T, ready F->T",
name: "tainted T, ready F->T", oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakSetReady(false)),
oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakSetReady(false)), newNode: makeNode(tweakAddTaint(ToBeDeletedTaint)),
newNode: makeNode(tweakAddTaint(ToBeDeletedTaint)), shouldSync: false,
shouldSync: false, fgEnabled: fgEnabled,
}, { }, {
name: "tainted T, ready T->F", name: "tainted T, ready T->F",
oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint)), oldNode: makeNode(tweakAddTaint(ToBeDeletedTaint)),
newNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakSetReady(false)), newNode: makeNode(tweakAddTaint(ToBeDeletedTaint), tweakSetReady(false)),
shouldSync: false, shouldSync: false,
}, { fgEnabled: fgEnabled,
name: "excluded T, tainted F->T", }, {
oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), name: "excluded T, tainted F->T",
newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakAddTaint(ToBeDeletedTaint)), oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
shouldSync: false, newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakAddTaint(ToBeDeletedTaint)),
}, { shouldSync: false,
name: "excluded T, tainted T->F", fgEnabled: fgEnabled,
oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakAddTaint(ToBeDeletedTaint)), }, {
newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), name: "excluded T, tainted T->F",
shouldSync: false, oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakAddTaint(ToBeDeletedTaint)),
}, { newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
name: "excluded T, ready F->T", shouldSync: false,
oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakSetReady(false)), fgEnabled: fgEnabled,
newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), }, {
shouldSync: false, name: "excluded T, ready F->T",
}, { oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakSetReady(false)),
name: "excluded T, ready T->F", newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), shouldSync: false,
newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakSetReady(false)), fgEnabled: fgEnabled,
shouldSync: false, }, {
}, { name: "excluded T, ready T->F",
name: "excluded T, providerID set F->T", oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakUnsetProviderID()), newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakSetReady(false)),
newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), shouldSync: false,
shouldSync: true, fgEnabled: fgEnabled,
}, { }, {
name: "excluded T, providerID set T->F", name: "excluded T, providerID set F->T",
oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakUnsetProviderID()),
newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakUnsetProviderID()), newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
shouldSync: true, shouldSync: true,
}, { fgEnabled: fgEnabled,
name: "ready F, tainted F->T", }, {
oldNode: makeNode(tweakSetReady(false)), name: "excluded T, providerID set T->F",
newNode: makeNode(tweakSetReady(false), tweakAddTaint(ToBeDeletedTaint)), oldNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
shouldSync: false, newNode: makeNode(tweakSetLabel(v1.LabelNodeExcludeBalancers, ""), tweakUnsetProviderID()),
}, { shouldSync: true,
name: "ready F, tainted T->F", fgEnabled: fgEnabled,
oldNode: makeNode(tweakSetReady(false), tweakAddTaint(ToBeDeletedTaint)), }, {
newNode: makeNode(tweakSetReady(false)), name: "ready F, tainted F->T",
shouldSync: false, oldNode: makeNode(tweakSetReady(false)),
}, { newNode: makeNode(tweakSetReady(false), tweakAddTaint(ToBeDeletedTaint)),
name: "ready F, excluded F->T", shouldSync: false,
oldNode: makeNode(tweakSetReady(false)), fgEnabled: fgEnabled,
newNode: makeNode(tweakSetReady(false), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), }, {
shouldSync: true, name: "ready F, tainted T->F",
}, { oldNode: makeNode(tweakSetReady(false), tweakAddTaint(ToBeDeletedTaint)),
name: "ready F, excluded T->F", newNode: makeNode(tweakSetReady(false)),
oldNode: makeNode(tweakSetReady(false), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), shouldSync: false,
newNode: makeNode(tweakSetReady(false)), fgEnabled: fgEnabled,
shouldSync: true, }, {
}, { name: "ready F, excluded F->T",
name: "ready F, providerID set F->T", oldNode: makeNode(tweakSetReady(false)),
oldNode: makeNode(tweakSetReady(false), tweakUnsetProviderID()), newNode: makeNode(tweakSetReady(false), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
newNode: makeNode(tweakSetReady(false)), shouldSync: true,
shouldSync: true, fgEnabled: fgEnabled,
}, { }, {
name: "ready F, providerID set T->F", name: "ready F, excluded T->F",
oldNode: makeNode(tweakSetReady(false)), oldNode: makeNode(tweakSetReady(false), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
newNode: makeNode(tweakSetReady(false), tweakUnsetProviderID()), newNode: makeNode(tweakSetReady(false)),
shouldSync: true, shouldSync: true,
}, { fgEnabled: fgEnabled,
name: "providerID unset, tainted F->T", }, {
name: "ready F, providerID set F->T",
oldNode: makeNode(tweakSetReady(false), tweakUnsetProviderID()),
newNode: makeNode(tweakSetReady(false)),
shouldSync: true,
fgEnabled: fgEnabled,
}, {
name: "ready F, providerID set T->F",
oldNode: makeNode(tweakSetReady(false)),
newNode: makeNode(tweakSetReady(false), tweakUnsetProviderID()),
shouldSync: true,
fgEnabled: fgEnabled,
}, {
name: "providerID unset, excluded F->T",
oldNode: makeNode(tweakUnsetProviderID()),
newNode: makeNode(tweakUnsetProviderID(), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
shouldSync: true,
fgEnabled: fgEnabled,
}, {
name: "providerID unset, excluded T->F",
oldNode: makeNode(tweakUnsetProviderID(), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
newNode: makeNode(tweakUnsetProviderID()),
shouldSync: true,
fgEnabled: fgEnabled,
}, {
name: "providerID unset, ready T->F",
oldNode: makeNode(tweakUnsetProviderID()),
newNode: makeNode(tweakUnsetProviderID(), tweakSetReady(true)),
shouldSync: false,
fgEnabled: fgEnabled,
},
}...)
}
testcases = append(testcases, []testCase{
{
name: "providerID unset, ready F->T",
oldNode: makeNode(tweakUnsetProviderID()), oldNode: makeNode(tweakUnsetProviderID()),
newNode: makeNode(tweakUnsetProviderID(), tweakAddTaint(ToBeDeletedTaint)), newNode: makeNode(tweakUnsetProviderID(), tweakSetReady(false)),
shouldSync: false, shouldSync: false,
fgEnabled: true,
},
{
name: "providerID unset, ready F->T",
oldNode: makeNode(tweakUnsetProviderID()),
newNode: makeNode(tweakUnsetProviderID(), tweakSetReady(false)),
shouldSync: true,
fgEnabled: false,
}, { }, {
name: "providerID unset, tainted T->F", name: "providerID unset, tainted T->F",
oldNode: makeNode(tweakUnsetProviderID(), tweakAddTaint(ToBeDeletedTaint)), oldNode: makeNode(tweakUnsetProviderID(), tweakAddTaint(ToBeDeletedTaint)),
newNode: makeNode(tweakUnsetProviderID()), newNode: makeNode(tweakUnsetProviderID()),
shouldSync: false, shouldSync: false,
}, { fgEnabled: true,
name: "providerID unset, excluded F->T", },
oldNode: makeNode(tweakUnsetProviderID()), {
newNode: makeNode(tweakUnsetProviderID(), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")), name: "providerID unset, tainted T->F",
shouldSync: true, oldNode: makeNode(tweakUnsetProviderID(), tweakAddTaint(ToBeDeletedTaint)),
}, {
name: "providerID unset, excluded T->F",
oldNode: makeNode(tweakUnsetProviderID(), tweakSetLabel(v1.LabelNodeExcludeBalancers, "")),
newNode: makeNode(tweakUnsetProviderID()), newNode: makeNode(tweakUnsetProviderID()),
shouldSync: true, shouldSync: true,
fgEnabled: false,
}, { }, {
name: "providerID unset, ready F->T", name: "providerID unset, tainted F->T",
oldNode: makeNode(tweakUnsetProviderID()), oldNode: makeNode(tweakUnsetProviderID()),
newNode: makeNode(tweakUnsetProviderID(), tweakSetReady(false)), newNode: makeNode(tweakUnsetProviderID(), tweakAddTaint(ToBeDeletedTaint)),
shouldSync: false, shouldSync: false,
fgEnabled: true,
}, { }, {
name: "providerID unset, ready T->F", name: "providerID unset, tainted F->T",
oldNode: makeNode(tweakUnsetProviderID()), oldNode: makeNode(tweakUnsetProviderID()),
newNode: makeNode(tweakUnsetProviderID(), tweakSetReady(true)), newNode: makeNode(tweakUnsetProviderID(), tweakAddTaint(ToBeDeletedTaint)),
shouldSync: false, shouldSync: true,
}} fgEnabled: false,
for _, testcase := range testcases { },
t.Run(fmt.Sprintf("%s - StableLoadBalancerNodeSet: %v", testcase.name, fgEnabled), func(t *testing.T) { }...)
shouldSync := shouldSyncUpdatedNode(testcase.oldNode, testcase.newNode) for _, testcase := range testcases {
if shouldSync != testcase.shouldSync { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StableLoadBalancerNodeSet, testcase.fgEnabled)()
t.Errorf("unexpected result from shouldSyncNode, expected: %v, actual: %v", testcase.shouldSync, shouldSync) t.Run(fmt.Sprintf("%s - StableLoadBalancerNodeSet: %v", testcase.name, testcase.fgEnabled), func(t *testing.T) {
} shouldSync := shouldSyncUpdatedNode(testcase.oldNode, testcase.newNode)
}) if shouldSync != testcase.shouldSync {
} t.Errorf("unexpected result from shouldSyncNode, expected: %v, actual: %v", testcase.shouldSync, shouldSync)
}
})
} }
} }
type fakeNodeLister struct { type fakeNodeLister struct {