From 01ea75e36ddb1380243172be777b0feac2e1830e Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Thu, 21 Jan 2021 18:15:12 +0200 Subject: [PATCH] kubeadm: exclude control plane nodes from external LBs Apply the label: "node.kubernetes.io/exclude-from-external-load-balancers" To control plane nodes to preserve backwards compatibility with the legacy mode where "master" nodes were excluded from LBs. --- cmd/kubeadm/app/cmd/upgrade/apply.go | 7 +++++ cmd/kubeadm/app/constants/constants.go | 4 +++ .../markcontrolplane/markcontrolplane.go | 24 ++++++++++++---- .../markcontrolplane/markcontrolplane_test.go | 10 +++++-- cmd/kubeadm/app/phases/upgrade/postupgrade.go | 28 +++++++++++++++++++ 5 files changed, 64 insertions(+), 9 deletions(-) diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index 89767d8a438..1f46eef9697 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -171,6 +171,13 @@ func runApply(flags *applyFlags, args []string) error { return err } + // TODO: https://github.com/kubernetes/kubeadm/issues/2375 + fmt.Printf("[upgrade/postupgrade] Applying label %s='' to control plane Nodes\n", + kubeadmconstants.LabelExcludeFromExternalLB) + if err := upgrade.LabelControlPlaneNodesWithExcludeFromLB(client); err != nil { + return err + } + // Upgrade RBAC rules and addons. klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons") if err := upgrade.PerformPostUpgradeTasks(client, cfg, flags.dryRun); err != nil { diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 57ddbcc5764..fe1a48db7b5 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -232,6 +232,10 @@ const ( // LabelNodeRoleControlPlane specifies that a node hosts control-plane components LabelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane" + // LabelExcludeFromExternalLB can be set on a node to exclude it from external load balancers. + // This is added to control plane nodes to preserve backwards compatibility with a legacy behavior. + LabelExcludeFromExternalLB = "node.kubernetes.io/exclude-from-external-load-balancers" + // AnnotationKubeadmCRISocket specifies the annotation kubeadm uses to preserve the crisocket information given to kubeadm at // init/join time for use later. kubeadm annotates the node object with this information AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket" diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go index 63344c82ae3..ec47e402d06 100644 --- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go +++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go @@ -25,12 +25,24 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) +var labelsToAdd = []string{ + // TODO: remove this label: + // https://github.com/kubernetes/kubeadm/issues/2200 + constants.LabelNodeRoleOldControlPlane, + constants.LabelNodeRoleControlPlane, + constants.LabelExcludeFromExternalLB, +} + // MarkControlPlane taints the control-plane and sets the control-plane label func MarkControlPlane(client clientset.Interface, controlPlaneName string, taints []v1.Taint) error { + // TODO: remove this "deprecated" amend and pass "labelsToAdd" directly: + // https://github.com/kubernetes/kubeadm/issues/2200 + labels := make([]string, len(labelsToAdd)) + copy(labels, labelsToAdd) + labels[0] = constants.LabelNodeRoleOldControlPlane + "(deprecated)" - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the labels \"%s=''\" and \"%s='' (deprecated)\"\n", - controlPlaneName, constants.LabelNodeRoleOldControlPlane, constants.LabelNodeRoleControlPlane) + fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the labels: %v\n", + controlPlaneName, labels) if len(taints) > 0 { taintStrs := []string{} @@ -56,9 +68,9 @@ func taintExists(taint v1.Taint, taints []v1.Taint) bool { } func markControlPlaneNode(n *v1.Node, taints []v1.Taint) { - // TODO: https://github.com/kubernetes/kubeadm/issues/2200 - n.ObjectMeta.Labels[constants.LabelNodeRoleOldControlPlane] = "" - n.ObjectMeta.Labels[constants.LabelNodeRoleControlPlane] = "" + for _, label := range labelsToAdd { + n.ObjectMeta.Labels[label] = "" + } for _, nt := range n.Spec.Taints { if !taintExists(nt, taints) { diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go index 3233976e12e..741f83ac7a3 100644 --- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go +++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go @@ -48,27 +48,28 @@ func TestMarkControlPlane(t *testing.T) { existingLabels: []string{""}, existingTaints: nil, newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`, + expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`, }, { name: "control-plane label and taint missing but taint not wanted", existingLabels: []string{""}, existingTaints: nil, newTaints: nil, - expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}}}`, + expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`, }, { name: "control-plane label missing", existingLabels: []string{""}, existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, - expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}}}`, + expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`, }, { name: "control-plane taint missing", existingLabels: []string{ kubeadmconstants.LabelNodeRoleOldControlPlane, kubeadmconstants.LabelNodeRoleControlPlane, + kubeadmconstants.LabelExcludeFromExternalLB, }, existingTaints: nil, newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, @@ -79,6 +80,7 @@ func TestMarkControlPlane(t *testing.T) { existingLabels: []string{ kubeadmconstants.LabelNodeRoleOldControlPlane, kubeadmconstants.LabelNodeRoleControlPlane, + kubeadmconstants.LabelExcludeFromExternalLB, }, existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, @@ -89,6 +91,7 @@ func TestMarkControlPlane(t *testing.T) { existingLabels: []string{ kubeadmconstants.LabelNodeRoleOldControlPlane, kubeadmconstants.LabelNodeRoleControlPlane, + kubeadmconstants.LabelExcludeFromExternalLB, }, existingTaints: []v1.Taint{ { @@ -104,6 +107,7 @@ func TestMarkControlPlane(t *testing.T) { existingLabels: []string{ kubeadmconstants.LabelNodeRoleOldControlPlane, kubeadmconstants.LabelNodeRoleControlPlane, + kubeadmconstants.LabelExcludeFromExternalLB, }, existingTaints: []v1.Taint{ { diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 94ee2edb595..80447240cfd 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -281,3 +281,31 @@ func LabelOldControlPlaneNodes(client clientset.Interface) error { } return nil } + +// LabelControlPlaneNodesWithExcludeFromLB finds all control plane nodes and applies the LabelExcludeFromExternalLB +// label to them. +// TODO: https://github.com/kubernetes/kubeadm/issues/2375 +func LabelControlPlaneNodesWithExcludeFromLB(client clientset.Interface) error { + selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ + kubeadmconstants.LabelNodeRoleControlPlane: "", + })) + nodesWithLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: selectorControlPlane.String(), + }) + if err != nil { + return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleControlPlane) + } + + for _, n := range nodesWithLabel.Items { + if _, hasLabel := n.ObjectMeta.Labels[kubeadmconstants.LabelExcludeFromExternalLB]; hasLabel { + continue + } + err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) { + n.ObjectMeta.Labels[kubeadmconstants.LabelExcludeFromExternalLB] = "" + }) + if err != nil { + return err + } + } + return nil +}