Merge pull request #98269 from neolit123/1.21-kubeadm-add-lb-label

kubeadm: exclude control plane nodes from external LBs

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -171,6 +171,13 @@ func runApply(flags *applyFlags, args []string) error {
 		return err
 	}
 
+	// TODO: https://github.com/kubernetes/kubeadm/issues/2375
+	fmt.Printf("[upgrade/postupgrade] Applying label %s='' to control plane Nodes\n",
+		kubeadmconstants.LabelExcludeFromExternalLB)
+	if err := upgrade.LabelControlPlaneNodesWithExcludeFromLB(client); err != nil {
+		return err
+	}
+
 	// Upgrade RBAC rules and addons.
 	klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons")
 	if err := upgrade.PerformPostUpgradeTasks(client, cfg, flags.dryRun); err != nil {

cmd/kubeadm/app/constants/constants.go

@@ -232,6 +232,10 @@ const (
 	// LabelNodeRoleControlPlane specifies that a node hosts control-plane components
 	LabelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane"
 
+	// LabelExcludeFromExternalLB can be set on a node to exclude it from external load balancers.
+	// This is added to control plane nodes to preserve backwards compatibility with a legacy behavior.
+	LabelExcludeFromExternalLB = "node.kubernetes.io/exclude-from-external-load-balancers"
+
 	// AnnotationKubeadmCRISocket specifies the annotation kubeadm uses to preserve the crisocket information given to kubeadm at
 	// init/join time for use later. kubeadm annotates the node object with this information
 	AnnotationKubeadmCRISocket = "kubeadm.alpha.kubernetes.io/cri-socket"

cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go

@@ -25,12 +25,24 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 )
 
+var labelsToAdd = []string{
+	// TODO: remove this label:
+	// https://github.com/kubernetes/kubeadm/issues/2200
+	constants.LabelNodeRoleOldControlPlane,
+	constants.LabelNodeRoleControlPlane,
+	constants.LabelExcludeFromExternalLB,
+}
+
 // MarkControlPlane taints the control-plane and sets the control-plane label
 func MarkControlPlane(client clientset.Interface, controlPlaneName string, taints []v1.Taint) error {
+	// TODO: remove this "deprecated" amend and pass "labelsToAdd" directly:
+	// https://github.com/kubernetes/kubeadm/issues/2200
+	labels := make([]string, len(labelsToAdd))
+	copy(labels, labelsToAdd)
+	labels[0] = constants.LabelNodeRoleOldControlPlane + "(deprecated)"
 
-	// TODO: https://github.com/kubernetes/kubeadm/issues/2200
+	fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the labels: %v\n",
-	fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the labels \"%s=''\" and \"%s='' (deprecated)\"\n",
+		controlPlaneName, labels)
-		controlPlaneName, constants.LabelNodeRoleOldControlPlane, constants.LabelNodeRoleControlPlane)
 
 	if len(taints) > 0 {
 		taintStrs := []string{}
@@ -56,9 +68,9 @@ func taintExists(taint v1.Taint, taints []v1.Taint) bool {
 }
 
 func markControlPlaneNode(n *v1.Node, taints []v1.Taint) {
-	// TODO: https://github.com/kubernetes/kubeadm/issues/2200
+	for _, label := range labelsToAdd {
-	n.ObjectMeta.Labels[constants.LabelNodeRoleOldControlPlane] = ""
+		n.ObjectMeta.Labels[label] = ""
-	n.ObjectMeta.Labels[constants.LabelNodeRoleControlPlane] = ""
+	}
 
 	for _, nt := range n.Spec.Taints {
 		if !taintExists(nt, taints) {

cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go

@@ -48,27 +48,28 @@ func TestMarkControlPlane(t *testing.T) {
 			existingLabels: []string{""},
 			existingTaints: nil,
 			newTaints:      []v1.Taint{kubeadmconstants.OldControlPlaneTaint},
-			expectedPatch:  `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`,
+			expectedPatch:  `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`,
 		},
 		{
 			name:           "control-plane label and taint missing but taint not wanted",
 			existingLabels: []string{""},
 			existingTaints: nil,
 			newTaints:      nil,
-			expectedPatch:  `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}}}`,
+			expectedPatch:  `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`,
 		},
 		{
 			name:           "control-plane label missing",
 			existingLabels: []string{""},
 			existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint},
 			newTaints:      []v1.Taint{kubeadmconstants.OldControlPlaneTaint},
-			expectedPatch:  `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}}}`,
+			expectedPatch:  `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`,
 		},
 		{
 			name: "control-plane taint missing",
 			existingLabels: []string{
 				kubeadmconstants.LabelNodeRoleOldControlPlane,
 				kubeadmconstants.LabelNodeRoleControlPlane,
+				kubeadmconstants.LabelExcludeFromExternalLB,
 			},
 			existingTaints: nil,
 			newTaints:      []v1.Taint{kubeadmconstants.OldControlPlaneTaint},
@@ -79,6 +80,7 @@ func TestMarkControlPlane(t *testing.T) {
 			existingLabels: []string{
 				kubeadmconstants.LabelNodeRoleOldControlPlane,
 				kubeadmconstants.LabelNodeRoleControlPlane,
+				kubeadmconstants.LabelExcludeFromExternalLB,
 			},
 			existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint},
 			newTaints:      []v1.Taint{kubeadmconstants.OldControlPlaneTaint},
@@ -89,6 +91,7 @@ func TestMarkControlPlane(t *testing.T) {
 			existingLabels: []string{
 				kubeadmconstants.LabelNodeRoleOldControlPlane,
 				kubeadmconstants.LabelNodeRoleControlPlane,
+				kubeadmconstants.LabelExcludeFromExternalLB,
 			},
 			existingTaints: []v1.Taint{
 				{
@@ -104,6 +107,7 @@ func TestMarkControlPlane(t *testing.T) {
 			existingLabels: []string{
 				kubeadmconstants.LabelNodeRoleOldControlPlane,
 				kubeadmconstants.LabelNodeRoleControlPlane,
+				kubeadmconstants.LabelExcludeFromExternalLB,
 			},
 			existingTaints: []v1.Taint{
 				{

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -281,3 +281,31 @@ func LabelOldControlPlaneNodes(client clientset.Interface) error {
 	}
 	return nil
 }
+
+// LabelControlPlaneNodesWithExcludeFromLB finds all control plane nodes and applies the LabelExcludeFromExternalLB
+// label to them.
+// TODO: https://github.com/kubernetes/kubeadm/issues/2375
+func LabelControlPlaneNodesWithExcludeFromLB(client clientset.Interface) error {
+	selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{
+		kubeadmconstants.LabelNodeRoleControlPlane: "",
+	}))
+	nodesWithLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
+		LabelSelector: selectorControlPlane.String(),
+	})
+	if err != nil {
+		return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleControlPlane)
+	}
+
+	for _, n := range nodesWithLabel.Items {
+		if _, hasLabel := n.ObjectMeta.Labels[kubeadmconstants.LabelExcludeFromExternalLB]; hasLabel {
+			continue
+		}
+		err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) {
+			n.ObjectMeta.Labels[kubeadmconstants.LabelExcludeFromExternalLB] = ""
+		})
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}