diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go b/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go index 1f83a155ed9..3fa27fc2df5 100644 --- a/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go +++ b/cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go @@ -24,6 +24,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" ) @@ -78,6 +79,12 @@ func runControlPlane() func(c workflow.RunData) error { return errors.Wrap(err, "couldn't complete the static pod upgrade") } + if features.Enabled(cfg.FeatureGates, features.UpgradeAddonsAfterControlPlane) { + if err := upgrade.PerformAddonsUpgrade(client, cfg, data.OutputWriter()); err != nil { + return errors.Wrap(err, "failed to perform addons upgrade") + } + } + fmt.Println("[upgrade] The control plane instance for this node was successfully updated!") return nil diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index 2d4d59bf4c4..614ea8fe4ad 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -41,6 +41,7 @@ import ( // supported by this api will be exposed as a flag. type nodeOptions struct { kubeConfigPath string + isControlPlaneNode bool etcdUpgrade bool renewCerts bool dryRun bool @@ -105,11 +106,26 @@ func newCmdNode(out io.Writer) *cobra.Command { // newNodeOptions returns a struct ready for being used for creating cmd kubeadm upgrade node flags. func newNodeOptions() *nodeOptions { + kubeConfigPath := constants.GetKubeletKubeConfigPath() + + // isControlPlaneNode checks if a node is a control-plane node by looking up + // the kube-apiserver manifest file + isControlPlaneNode := true + filepath := constants.GetStaticPodFilepath(constants.KubeAPIServer, constants.GetStaticPodDirectory()) + if _, err := os.Stat(filepath); os.IsNotExist(err) { + isControlPlaneNode = false + } + + if isControlPlaneNode { + kubeConfigPath = constants.GetAdminKubeConfigPath() + } + return &nodeOptions{ - kubeConfigPath: constants.GetKubeletKubeConfigPath(), - dryRun: false, - renewCerts: true, - etcdUpgrade: true, + kubeConfigPath: kubeConfigPath, + isControlPlaneNode: isControlPlaneNode, + dryRun: false, + renewCerts: true, + etcdUpgrade: true, } } @@ -129,19 +145,10 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions, out io if err != nil { return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", options.kubeConfigPath) } - - // isControlPlane checks if a node is a control-plane node by looking up - // the kube-apiserver manifest file - isControlPlaneNode := true - filepath := constants.GetStaticPodFilepath(constants.KubeAPIServer, constants.GetStaticPodDirectory()) - if _, err := os.Stat(filepath); os.IsNotExist(err) { - isControlPlaneNode = false - } - // Fetches the cluster configuration // NB in case of control-plane node, we are reading all the info for the node; in case of NOT control-plane node // (worker node), we are not reading local API address and the CRI socket from the node object - cfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", !isControlPlaneNode, false) + cfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", !options.isControlPlaneNode, false) if err != nil { return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") } @@ -158,7 +165,7 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions, out io dryRun: options.dryRun, cfg: cfg, client: client, - isControlPlaneNode: isControlPlaneNode, + isControlPlaneNode: options.isControlPlaneNode, patchesDir: options.patchesDir, ignorePreflightErrors: ignorePreflightErrorsSet, kubeConfigPath: options.kubeConfigPath, diff --git a/cmd/kubeadm/app/features/features.go b/cmd/kubeadm/app/features/features.go index 154901da10a..04afec3f07e 100644 --- a/cmd/kubeadm/app/features/features.go +++ b/cmd/kubeadm/app/features/features.go @@ -35,13 +35,16 @@ const ( RootlessControlPlane = "RootlessControlPlane" // EtcdLearnerMode is expected to be in alpha in v1.27 EtcdLearnerMode = "EtcdLearnerMode" + // UpgradeAddonsAfterControlPlane is expected to be in alpha in v1.28 + UpgradeAddonsAfterControlPlane = "UpgradeAddonsAfterControlPlane" ) // InitFeatureGates are the default feature gates for the init command var InitFeatureGates = FeatureList{ - PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, - RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, - EtcdLearnerMode: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, + PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, + RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, + EtcdLearnerMode: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, + UpgradeAddonsAfterControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}}, } // Feature represents a feature being gated diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 36a63b74478..2d39eeba937 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -27,12 +27,15 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" errorsutil "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" @@ -103,6 +106,41 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon errs = append(errs, err) } + if err := PerformAddonsUpgrade(client, cfg, out); err != nil { + errs = append(errs, err) + } + + return errorsutil.NewAggregate(errs) +} + +// PerformAddonsUpgrade performs the upgrade of the coredns and kube-proxy addons. +// When UpgradeAddonsAfterControlPlane feature gate is disabled, the addons will be upgraded immediately. +// When UpgradeAddonsAfterControlPlane feature gate is enabled, the addons will only get updated after all the control plane instances have been upgraded. +func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, out io.Writer) error { + unupgradedControlPlanes, err := unupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name) + if err != nil { + err = errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded") + if features.Enabled(cfg.FeatureGates, features.UpgradeAddonsAfterControlPlane) { + return err + } + + // when UpgradeAddonsAfterControlPlane feature gate is disabled, just throw a warning + klog.V(1).Info(err) + } + if len(unupgradedControlPlanes) > 0 { + if features.Enabled(cfg.FeatureGates, features.UpgradeAddonsAfterControlPlane) { + fmt.Fprintf(out, "[upgrade/addons] skip upgrade addons because control plane instances %v have not been upgraded\n", unupgradedControlPlanes) + return nil + } + + // when UpgradeAddonsAfterControlPlane feature gate is disabled, just throw a warning + klog.V(1).Infof("upgrading addons when control plane instances %v have not been upgraded "+ + "may lead to incompatibility problems. You can enable the UpgradeAddonsAfterControlPlane feature gate to"+ + "ensure that the addons upgrade is executed only when all the control plane instances have been upgraded.", unupgradedControlPlanes) + } + + var errs []error + // If the coredns ConfigMap is missing, show a warning and assume that the // DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired. // @@ -156,6 +194,60 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon return errorsutil.NewAggregate(errs) } +// unupgradedControlPlaneInstances returns a list of control palne instances that have not yet been upgraded. +// +// NB. This function can only be called after the current control plane instance has been upgraded already. +// Because it determines whether the other control plane instances have been upgraded by checking whether +// the kube-apiserver image of other control plane instance is the same as that of this instance. +func unupgradedControlPlaneInstances(client clientset.Interface, nodeName string) ([]string, error) { + selector := labels.SelectorFromSet(labels.Set(map[string]string{ + "component": kubeadmconstants.KubeAPIServer, + })) + pods, err := client.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{ + LabelSelector: selector.String(), + }) + if err != nil { + return nil, errors.Wrap(err, "failed to list kube-apiserver Pod from cluster") + } + if len(pods.Items) == 0 { + return nil, errors.Errorf("cannot find kube-apiserver Pod by label selector: %v", selector.String()) + } + + nodeImageMap := map[string]string{} + + for _, pod := range pods.Items { + found := false + for _, c := range pod.Spec.Containers { + if c.Name == kubeadmconstants.KubeAPIServer { + nodeImageMap[pod.Spec.NodeName] = c.Image + found = true + break + } + } + if !found { + return nil, errors.Errorf("cannot find container by name %q for Pod %v", kubeadmconstants.KubeAPIServer, klog.KObj(&pod)) + } + } + + upgradedImage, ok := nodeImageMap[nodeName] + if !ok { + return nil, errors.Errorf("cannot find kube-apiserver image for current control plane instance %v", nodeName) + } + + unupgradedNodes := sets.New[string]() + for node, image := range nodeImageMap { + if image != upgradedImage { + unupgradedNodes.Insert(node) + } + } + + if len(unupgradedNodes) > 0 { + return sets.List(unupgradedNodes), nil + } + + return nil, nil +} + func WriteKubeletConfigFiles(cfg *kubeadmapi.InitConfiguration, patchesDir string, dryRun bool, out io.Writer) error { // Set up the kubelet directory to use. If dry-running, this will return a fake directory kubeletDir, err := GetKubeletDir(dryRun)