From fb7ddf88e907100409b64a158fbf9f3444aa186d Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Thu, 8 Oct 2020 01:02:59 +0300 Subject: [PATCH] kubeadm: mark the "master" label/taint as deprecated - Mark the "node-role.kubernetes.io/master" key for labels and taints as deprecated. - During "kubeadm init/join" apply the label "node-role.kubernetes.io/control-plane" to new control-plane nodes, next to the existing "node-role.kubernetes.io/master" label. - During "kubeadm upgrade apply", find all Nodes with the "master" label and also apply the "control-plane" label to them (if they don't have it). - During upgrade health-checks collect Nodes labeled both "master" and "control-plane". - Rename the constants.ControlPlane{Taint|Toleraton} to constants.OldControlPlane{Taint|Toleraton} to manage the transition. - Mark constants.OldControlPlane{{Taint|Toleraton} as deprecated. - Use constants.OldControlPlane{{Taint|Toleraton} instead of constants.ControlPlane{Taint|Toleraton} everywhere. - Introduce constants.ControlPlane{Taint|Toleraton}. - Add constants.ControlPlaneToleraton to the kube-dns / CoreDNS Deployments to make them anticipate the introduction of the "node-role.kubernetes.io/control-plane:NoSchedule" taint (constants.ControlPlaneTaint) on kubeadm control-plane Nodes. --- cmd/kubeadm/app/cmd/upgrade/apply.go | 8 ++ cmd/kubeadm/app/constants/constants.go | 27 +++++-- cmd/kubeadm/app/phases/addons/dns/dns.go | 38 +++++----- cmd/kubeadm/app/phases/addons/dns/dns_test.go | 74 ++++++++++--------- .../app/phases/addons/dns/manifests.go | 4 + .../markcontrolplane/markcontrolplane.go | 8 +- .../markcontrolplane/markcontrolplane_test.go | 60 +++++++++------ .../phases/selfhosting/podspec_mutation.go | 9 ++- .../selfhosting/podspec_mutation_test.go | 22 +++--- cmd/kubeadm/app/phases/upgrade/health.go | 27 +++++-- cmd/kubeadm/app/phases/upgrade/postupgrade.go | 30 ++++++++ cmd/kubeadm/app/util/config/cluster_test.go | 6 +- .../app/util/config/initconfiguration.go | 7 +- 13 files changed, 209 insertions(+), 111 deletions(-) diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index b781310775b..89767d8a438 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -28,6 +28,7 @@ import ( "k8s.io/klog/v2" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" @@ -163,6 +164,13 @@ func runApply(flags *applyFlags, args []string) error { return errors.Wrap(err, "[upgrade/apply] FATAL") } + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 + fmt.Printf("[upgrade/postupgrade] Applying label %s='' to Nodes with label %s='' (deprecated)\n", + kubeadmconstants.LabelNodeRoleControlPlane, kubeadmconstants.LabelNodeRoleOldControlPlane) + if err := upgrade.LabelOldControlPlaneNodes(client); err != nil { + return err + } + // Upgrade RBAC rules and addons. klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons") if err := upgrade.PerformPostUpgradeTasks(client, cfg, flags.dryRun); err != nil { diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index a9313574e17..bf9f3094d1d 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -225,9 +225,12 @@ const ( // CertificateKeySize specifies the size of the key used to encrypt certificates on uploadcerts phase CertificateKeySize = 32 - // LabelNodeRoleMaster specifies that a node is a control-plane - // This is a duplicate definition of the constant in pkg/controller/service/controller.go - LabelNodeRoleMaster = "node-role.kubernetes.io/master" + // LabelNodeRoleOldControlPlane specifies that a node hosts control-plane components + // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 + LabelNodeRoleOldControlPlane = "node-role.kubernetes.io/master" + + // LabelNodeRoleControlPlane specifies that a node hosts control-plane components + LabelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane" // AnnotationKubeadmCRISocket specifies the annotation kubeadm uses to preserve the crisocket information given to kubeadm at // init/join time for use later. kubeadm annotates the node object with this information @@ -414,15 +417,29 @@ const ( ) var ( + // OldControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane + // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 + OldControlPlaneTaint = v1.Taint{ + Key: LabelNodeRoleOldControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + + // OldControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane + // DEPRECATED: https://github.com/kubernetes/kubeadm/issues/2200 + OldControlPlaneToleration = v1.Toleration{ + Key: LabelNodeRoleOldControlPlane, + Effect: v1.TaintEffectNoSchedule, + } + // ControlPlaneTaint is the taint to apply on the PodSpec for being able to run that Pod on the control-plane ControlPlaneTaint = v1.Taint{ - Key: LabelNodeRoleMaster, + Key: LabelNodeRoleControlPlane, Effect: v1.TaintEffectNoSchedule, } // ControlPlaneToleration is the toleration to apply on the PodSpec for being able to run that Pod on the control-plane ControlPlaneToleration = v1.Toleration{ - Key: LabelNodeRoleMaster, + Key: LabelNodeRoleControlPlane, Effect: v1.TaintEffectNoSchedule, } diff --git a/cmd/kubeadm/app/phases/addons/dns/dns.go b/cmd/kubeadm/app/phases/addons/dns/dns.go index a1a96e54001..1f2e2d973bb 100644 --- a/cmd/kubeadm/app/phases/addons/dns/dns.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns.go @@ -135,18 +135,20 @@ func kubeDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interfa dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment, struct { - DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, ControlPlaneTaintKey string - Replicas *int32 + DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, OldControlPlaneTaintKey, ControlPlaneTaintKey string + Replicas *int32 }{ - DeploymentName: kubeadmconstants.KubeDNSDeploymentName, - KubeDNSImage: images.GetDNSImage(cfg, kubeadmconstants.KubeDNSKubeDNSImageName), - DNSMasqImage: images.GetDNSImage(cfg, kubeadmconstants.KubeDNSDnsMasqNannyImageName), - SidecarImage: images.GetDNSImage(cfg, kubeadmconstants.KubeDNSSidecarImageName), - DNSBindAddr: dnsBindAddr, - DNSProbeAddr: dnsProbeAddr, - DNSDomain: cfg.Networking.DNSDomain, - ControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleMaster, - Replicas: replicas, + DeploymentName: kubeadmconstants.KubeDNSDeploymentName, + KubeDNSImage: images.GetDNSImage(cfg, kubeadmconstants.KubeDNSKubeDNSImageName), + DNSMasqImage: images.GetDNSImage(cfg, kubeadmconstants.KubeDNSDnsMasqNannyImageName), + SidecarImage: images.GetDNSImage(cfg, kubeadmconstants.KubeDNSSidecarImageName), + DNSBindAddr: dnsBindAddr, + DNSProbeAddr: dnsProbeAddr, + DNSDomain: cfg.Networking.DNSDomain, + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 + OldControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleOldControlPlane, + ControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleControlPlane, + Replicas: replicas, }) if err != nil { return errors.Wrap(err, "error when parsing kube-dns deployment template") @@ -196,13 +198,15 @@ func createKubeDNSAddon(deploymentBytes, serviceBytes []byte, client clientset.I func coreDNSAddon(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, replicas *int32) error { // Get the YAML manifest coreDNSDeploymentBytes, err := kubeadmutil.ParseTemplate(CoreDNSDeployment, struct { - DeploymentName, Image, ControlPlaneTaintKey string - Replicas *int32 + DeploymentName, Image, OldControlPlaneTaintKey, ControlPlaneTaintKey string + Replicas *int32 }{ - DeploymentName: kubeadmconstants.CoreDNSDeploymentName, - Image: images.GetDNSImage(cfg, kubeadmconstants.CoreDNSImageName), - ControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleMaster, - Replicas: replicas, + DeploymentName: kubeadmconstants.CoreDNSDeploymentName, + Image: images.GetDNSImage(cfg, kubeadmconstants.CoreDNSImageName), + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 + OldControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleOldControlPlane, + ControlPlaneTaintKey: kubeadmconstants.LabelNodeRoleControlPlane, + Replicas: replicas, }) if err != nil { return errors.Wrap(err, "error when parsing CoreDNS deployment template") diff --git a/cmd/kubeadm/app/phases/addons/dns/dns_test.go b/cmd/kubeadm/app/phases/addons/dns/dns_test.go index 129392e02cd..f0b58448026 100644 --- a/cmd/kubeadm/app/phases/addons/dns/dns_test.go +++ b/cmd/kubeadm/app/phases/addons/dns/dns_test.go @@ -102,18 +102,19 @@ func TestCompileManifests(t *testing.T) { name: "KubeDNSDeployment manifest", manifest: KubeDNSDeployment, data: struct { - DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, ControlPlaneTaintKey string - Replicas *int32 + DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, OldControlPlaneTaintKey, ControlPlaneTaintKey string + Replicas *int32 }{ - DeploymentName: "foo", - KubeDNSImage: "foo", - DNSMasqImage: "foo", - SidecarImage: "foo", - DNSBindAddr: "foo", - DNSProbeAddr: "foo", - DNSDomain: "foo", - ControlPlaneTaintKey: "foo", - Replicas: &replicas, + DeploymentName: "foo", + KubeDNSImage: "foo", + DNSMasqImage: "foo", + SidecarImage: "foo", + DNSBindAddr: "foo", + DNSProbeAddr: "foo", + DNSDomain: "foo", + OldControlPlaneTaintKey: "foo", + ControlPlaneTaintKey: "foo", + Replicas: &replicas, }, }, { @@ -127,13 +128,14 @@ func TestCompileManifests(t *testing.T) { name: "CoreDNSDeployment manifest", manifest: CoreDNSDeployment, data: struct { - DeploymentName, Image, ControlPlaneTaintKey string - Replicas *int32 + DeploymentName, Image, OldControlPlaneTaintKey, ControlPlaneTaintKey string + Replicas *int32 }{ - DeploymentName: "foo", - Image: "foo", - ControlPlaneTaintKey: "foo", - Replicas: &replicas, + DeploymentName: "foo", + Image: "foo", + OldControlPlaneTaintKey: "foo", + ControlPlaneTaintKey: "foo", + Replicas: &replicas, }, }, { @@ -506,32 +508,34 @@ func TestDeploymentsHaveSystemClusterCriticalPriorityClassName(t *testing.T) { name: "KubeDNSDeployment", manifest: KubeDNSDeployment, data: struct { - DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, ControlPlaneTaintKey string - Replicas *int32 + DeploymentName, KubeDNSImage, DNSMasqImage, SidecarImage, DNSBindAddr, DNSProbeAddr, DNSDomain, OldControlPlaneTaintKey, ControlPlaneTaintKey string + Replicas *int32 }{ - DeploymentName: "foo", - KubeDNSImage: "foo", - DNSMasqImage: "foo", - SidecarImage: "foo", - DNSBindAddr: "foo", - DNSProbeAddr: "foo", - DNSDomain: "foo", - ControlPlaneTaintKey: "foo", - Replicas: &replicas, + DeploymentName: "foo", + KubeDNSImage: "foo", + DNSMasqImage: "foo", + SidecarImage: "foo", + DNSBindAddr: "foo", + DNSProbeAddr: "foo", + DNSDomain: "foo", + OldControlPlaneTaintKey: "foo", + ControlPlaneTaintKey: "foo", + Replicas: &replicas, }, }, { name: "CoreDNSDeployment", manifest: CoreDNSDeployment, data: struct { - DeploymentName, Image, ControlPlaneTaintKey, CoreDNSConfigMapName string - Replicas *int32 + DeploymentName, Image, OldControlPlaneTaintKey, ControlPlaneTaintKey, CoreDNSConfigMapName string + Replicas *int32 }{ - DeploymentName: "foo", - Image: "foo", - ControlPlaneTaintKey: "foo", - CoreDNSConfigMapName: "foo", - Replicas: &replicas, + DeploymentName: "foo", + Image: "foo", + OldControlPlaneTaintKey: "foo", + ControlPlaneTaintKey: "foo", + CoreDNSConfigMapName: "foo", + Replicas: &replicas, }, }, } diff --git a/cmd/kubeadm/app/phases/addons/dns/manifests.go b/cmd/kubeadm/app/phases/addons/dns/manifests.go index cfc13edd9a3..014cbd773c2 100644 --- a/cmd/kubeadm/app/phases/addons/dns/manifests.go +++ b/cmd/kubeadm/app/phases/addons/dns/manifests.go @@ -170,6 +170,8 @@ spec: tolerations: - key: CriticalAddonsOnly operator: Exists + - key: {{ .OldControlPlaneTaintKey }} + effect: NoSchedule - key: {{ .ControlPlaneTaintKey }} effect: NoSchedule ` @@ -238,6 +240,8 @@ spec: tolerations: - key: CriticalAddonsOnly operator: Exists + - key: {{ .OldControlPlaneTaintKey }} + effect: NoSchedule - key: {{ .ControlPlaneTaintKey }} effect: NoSchedule nodeSelector: diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go index 220565aa35a..63344c82ae3 100644 --- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go +++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go @@ -28,7 +28,9 @@ import ( // MarkControlPlane taints the control-plane and sets the control-plane label func MarkControlPlane(client clientset.Interface, controlPlaneName string, taints []v1.Taint) error { - fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the label \"%s=''\"\n", controlPlaneName, constants.LabelNodeRoleMaster) + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 + fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the labels \"%s=''\" and \"%s='' (deprecated)\"\n", + controlPlaneName, constants.LabelNodeRoleOldControlPlane, constants.LabelNodeRoleControlPlane) if len(taints) > 0 { taintStrs := []string{} @@ -54,7 +56,9 @@ func taintExists(taint v1.Taint, taints []v1.Taint) bool { } func markControlPlaneNode(n *v1.Node, taints []v1.Taint) { - n.ObjectMeta.Labels[constants.LabelNodeRoleMaster] = "" + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 + n.ObjectMeta.Labels[constants.LabelNodeRoleOldControlPlane] = "" + n.ObjectMeta.Labels[constants.LabelNodeRoleControlPlane] = "" for _, nt := range n.Spec.Taints { if !taintExists(nt, taints) { diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go index 244c4d93aaa..3233976e12e 100644 --- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go +++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go @@ -38,49 +38,58 @@ func TestMarkControlPlane(t *testing.T) { // future. tests := []struct { name string - existingLabel string + existingLabels []string existingTaints []v1.Taint newTaints []v1.Taint expectedPatch string }{ { name: "control-plane label and taint missing", - existingLabel: "", + existingLabels: []string{""}, existingTaints: nil, - newTaints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, - expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/master":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`, + newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, + expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`, }, { name: "control-plane label and taint missing but taint not wanted", - existingLabel: "", + existingLabels: []string{""}, existingTaints: nil, newTaints: nil, - expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/master":""}}}`, + expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}}}`, }, { name: "control-plane label missing", - existingLabel: "", - existingTaints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, - newTaints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, - expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/master":""}}}`, + existingLabels: []string{""}, + existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, + newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, + expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":""}}}`, }, { - name: "control-plane taint missing", - existingLabel: kubeadmconstants.LabelNodeRoleMaster, + name: "control-plane taint missing", + existingLabels: []string{ + kubeadmconstants.LabelNodeRoleOldControlPlane, + kubeadmconstants.LabelNodeRoleControlPlane, + }, existingTaints: nil, - newTaints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, + newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, expectedPatch: `{"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`, }, { - name: "nothing missing", - existingLabel: kubeadmconstants.LabelNodeRoleMaster, - existingTaints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, - newTaints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, + name: "nothing missing", + existingLabels: []string{ + kubeadmconstants.LabelNodeRoleOldControlPlane, + kubeadmconstants.LabelNodeRoleControlPlane, + }, + existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, + newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, expectedPatch: `{}`, }, { - name: "has taint and no new taints wanted", - existingLabel: kubeadmconstants.LabelNodeRoleMaster, + name: "has taint and no new taints wanted", + existingLabels: []string{ + kubeadmconstants.LabelNodeRoleOldControlPlane, + kubeadmconstants.LabelNodeRoleControlPlane, + }, existingTaints: []v1.Taint{ { Key: "node.cloudprovider.kubernetes.io/uninitialized", @@ -91,15 +100,18 @@ func TestMarkControlPlane(t *testing.T) { expectedPatch: `{}`, }, { - name: "has taint and should merge with wanted taint", - existingLabel: kubeadmconstants.LabelNodeRoleMaster, + name: "has taint and should merge with wanted taint", + existingLabels: []string{ + kubeadmconstants.LabelNodeRoleOldControlPlane, + kubeadmconstants.LabelNodeRoleControlPlane, + }, existingTaints: []v1.Taint{ { Key: "node.cloudprovider.kubernetes.io/uninitialized", Effect: v1.TaintEffectNoSchedule, }, }, - newTaints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, + newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, expectedPatch: `{"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized"}]}}`, }, } @@ -116,8 +128,8 @@ func TestMarkControlPlane(t *testing.T) { }, } - if tc.existingLabel != "" { - controlPlaneNode.ObjectMeta.Labels[tc.existingLabel] = "" + for _, label := range tc.existingLabels { + controlPlaneNode.ObjectMeta.Labels[label] = "" } if tc.existingTaints != nil { diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go index 80790ffe13d..0d4090798c6 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation.go @@ -85,21 +85,22 @@ func mutatePodSpec(mutators map[string][]PodSpecMutatorFunc, name string, podSpe // addNodeSelectorToPodSpec makes Pod require to be scheduled on a node marked with the control-plane label func addNodeSelectorToPodSpec(podSpec *v1.PodSpec) { if podSpec.NodeSelector == nil { - podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleMaster: ""} + podSpec.NodeSelector = map[string]string{kubeadmconstants.LabelNodeRoleOldControlPlane: ""} return } - podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleMaster] = "" + podSpec.NodeSelector[kubeadmconstants.LabelNodeRoleOldControlPlane] = "" } // setControlPlaneTolerationOnPodSpec makes the Pod tolerate the control-plane taint func setControlPlaneTolerationOnPodSpec(podSpec *v1.PodSpec) { if podSpec.Tolerations == nil { - podSpec.Tolerations = []v1.Toleration{kubeadmconstants.ControlPlaneToleration} + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 + podSpec.Tolerations = []v1.Toleration{kubeadmconstants.OldControlPlaneToleration} return } - podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.ControlPlaneToleration) + podSpec.Tolerations = append(podSpec.Tolerations, kubeadmconstants.OldControlPlaneToleration) } // setHostIPOnPodSpec sets the environment variable HOST_IP using downward API diff --git a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go index 8edfdede036..81e69b9e225 100644 --- a/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go +++ b/cmd/kubeadm/app/phases/selfhosting/podspec_mutation_test.go @@ -66,10 +66,10 @@ func TestMutatePodSpec(t *testing.T) { }, NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleMaster: "", + kubeadmconstants.LabelNodeRoleOldControlPlane: "", }, Tolerations: []v1.Toleration{ - kubeadmconstants.ControlPlaneToleration, + kubeadmconstants.OldControlPlaneToleration, }, DNSPolicy: v1.DNSClusterFirstWithHostNet, }, @@ -80,10 +80,10 @@ func TestMutatePodSpec(t *testing.T) { podSpec: &v1.PodSpec{}, expected: v1.PodSpec{ NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleMaster: "", + kubeadmconstants.LabelNodeRoleOldControlPlane: "", }, Tolerations: []v1.Toleration{ - kubeadmconstants.ControlPlaneToleration, + kubeadmconstants.OldControlPlaneToleration, }, DNSPolicy: v1.DNSClusterFirstWithHostNet, }, @@ -94,10 +94,10 @@ func TestMutatePodSpec(t *testing.T) { podSpec: &v1.PodSpec{}, expected: v1.PodSpec{ NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleMaster: "", + kubeadmconstants.LabelNodeRoleOldControlPlane: "", }, Tolerations: []v1.Toleration{ - kubeadmconstants.ControlPlaneToleration, + kubeadmconstants.OldControlPlaneToleration, }, DNSPolicy: v1.DNSClusterFirstWithHostNet, }, @@ -126,7 +126,7 @@ func TestAddNodeSelectorToPodSpec(t *testing.T) { podSpec: &v1.PodSpec{}, expected: v1.PodSpec{ NodeSelector: map[string]string{ - kubeadmconstants.LabelNodeRoleMaster: "", + kubeadmconstants.LabelNodeRoleOldControlPlane: "", }, }, }, @@ -139,8 +139,8 @@ func TestAddNodeSelectorToPodSpec(t *testing.T) { }, expected: v1.PodSpec{ NodeSelector: map[string]string{ - "foo": "bar", - kubeadmconstants.LabelNodeRoleMaster: "", + "foo": "bar", + kubeadmconstants.LabelNodeRoleOldControlPlane: "", }, }, }, @@ -168,7 +168,7 @@ func TestSetControlPlaneTolerationOnPodSpec(t *testing.T) { podSpec: &v1.PodSpec{}, expected: v1.PodSpec{ Tolerations: []v1.Toleration{ - kubeadmconstants.ControlPlaneToleration, + kubeadmconstants.OldControlPlaneToleration, }, }, }, @@ -182,7 +182,7 @@ func TestSetControlPlaneTolerationOnPodSpec(t *testing.T) { expected: v1.PodSpec{ Tolerations: []v1.Toleration{ {Key: "foo", Value: "bar"}, - kubeadmconstants.ControlPlaneToleration, + kubeadmconstants.OldControlPlaneToleration, }, }, }, diff --git a/cmd/kubeadm/app/phases/upgrade/health.go b/cmd/kubeadm/app/phases/upgrade/health.go index dfdb975e846..3a238968d90 100644 --- a/cmd/kubeadm/app/phases/upgrade/health.go +++ b/cmd/kubeadm/app/phases/upgrade/health.go @@ -208,21 +208,34 @@ func deleteHealthCheckJob(client clientset.Interface, ns, jobName string) error // controlPlaneNodesReady checks whether all control-plane Nodes in the cluster are in the Running state func controlPlaneNodesReady(client clientset.Interface, _ *kubeadmapi.ClusterConfiguration) error { - selector := labels.SelectorFromSet(labels.Set(map[string]string{ - constants.LabelNodeRoleMaster: "", + // list nodes labeled with a "master" node-role + selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ + constants.LabelNodeRoleOldControlPlane: "", })) - controlPlanes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - LabelSelector: selector.String(), + nodesWithOldLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: selectorOldControlPlane.String(), }) if err != nil { - return errors.Wrap(err, "couldn't list control-planes in cluster") + return errors.Wrapf(err, "could not list nodes labeled with %q", constants.LabelNodeRoleOldControlPlane) } - if len(controlPlanes.Items) == 0 { + // list nodes labeled with a "control-plane" node-role + selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ + constants.LabelNodeRoleControlPlane: "", + })) + nodesControlPlane, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: selectorControlPlane.String(), + }) + if err != nil { + return errors.Wrapf(err, "could not list nodes labeled with %q", constants.LabelNodeRoleControlPlane) + } + + nodes := append(nodesWithOldLabel.Items, nodesControlPlane.Items...) + if len(nodes) == 0 { return errors.New("failed to find any nodes with a control-plane role") } - notReadyControlPlanes := getNotReadyNodes(controlPlanes.Items) + notReadyControlPlanes := getNotReadyNodes(nodes) if len(notReadyControlPlanes) != 0 { return errors.Errorf("there are NotReady control-planes in the cluster: %v", notReadyControlPlanes) } diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 886a9da1fca..1fe329edcb2 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -22,9 +22,11 @@ import ( "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" errorsutil "k8s.io/apimachinery/pkg/util/errors" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -251,3 +253,31 @@ func rollbackFiles(files map[string]string, originalErr error) error { } return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs)) } + +// LabelOldControlPlaneNodes finds all nodes with the legacy node-role label and also applies +// the "control-plane" node-role label to them. +// TODO: https://github.com/kubernetes/kubeadm/issues/2200 +func LabelOldControlPlaneNodes(client clientset.Interface) error { + selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ + kubeadmconstants.LabelNodeRoleOldControlPlane: "", + })) + nodesWithOldLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: selectorOldControlPlane.String(), + }) + if err != nil { + return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleOldControlPlane) + } + + for _, n := range nodesWithOldLabel.Items { + if _, hasNewLabel := n.ObjectMeta.Labels[kubeadmconstants.LabelNodeRoleControlPlane]; hasNewLabel { + continue + } + err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) { + n.ObjectMeta.Labels[kubeadmconstants.LabelNodeRoleControlPlane] = "" + }) + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/kubeadm/app/util/config/cluster_test.go b/cmd/kubeadm/app/util/config/cluster_test.go index a2cfde5f536..13e221ff827 100644 --- a/cmd/kubeadm/app/util/config/cluster_test.go +++ b/cmd/kubeadm/app/util/config/cluster_test.go @@ -303,7 +303,7 @@ func TestGetNodeRegistration(t *testing.T) { }, }, Spec: v1.NodeSpec{ - Taints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, + Taints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, }, }, }, @@ -583,7 +583,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) { }, }, Spec: v1.NodeSpec{ - Taints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, + Taints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, }, }, }, @@ -660,7 +660,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) { }, }, Spec: v1.NodeSpec{ - Taints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, + Taints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, }, }, }, diff --git a/cmd/kubeadm/app/util/config/initconfiguration.go b/cmd/kubeadm/app/util/config/initconfiguration.go index 9e0720bb682..d8a37124870 100644 --- a/cmd/kubeadm/app/util/config/initconfiguration.go +++ b/cmd/kubeadm/app/util/config/initconfiguration.go @@ -83,7 +83,7 @@ func SetBootstrapTokensDynamicDefaults(cfg *[]kubeadmapi.BootstrapToken) error { } // SetNodeRegistrationDynamicDefaults checks and sets configuration values for the NodeRegistration object -func SetNodeRegistrationDynamicDefaults(cfg *kubeadmapi.NodeRegistrationOptions, ControlPlaneTaint bool) error { +func SetNodeRegistrationDynamicDefaults(cfg *kubeadmapi.NodeRegistrationOptions, controlPlaneTaint bool) error { var err error cfg.Name, err = kubeadmutil.GetHostname(cfg.Name) if err != nil { @@ -91,8 +91,9 @@ func SetNodeRegistrationDynamicDefaults(cfg *kubeadmapi.NodeRegistrationOptions, } // Only if the slice is nil, we should append the control-plane taint. This allows the user to specify an empty slice for no default control-plane taint - if ControlPlaneTaint && cfg.Taints == nil { - cfg.Taints = []v1.Taint{kubeadmconstants.ControlPlaneTaint} + if controlPlaneTaint && cfg.Taints == nil { + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 + cfg.Taints = []v1.Taint{kubeadmconstants.OldControlPlaneTaint} } if cfg.CRISocket == "" {