kubeadm: add the experimental (alpha) feature gate UpgradeAddonsAfterControlPlane that supports upgrade coredns and kube-proxy addons after all the control plane instances have been upgraded

cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go

@@ -24,6 +24,7 @@ import (
 
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 )
@@ -78,6 +79,12 @@ func runControlPlane() func(c workflow.RunData) error {
 			return errors.Wrap(err, "couldn't complete the static pod upgrade")
 		}
 
+		if features.Enabled(cfg.FeatureGates, features.UpgradeAddonsAfterControlPlane) {
+			if err := upgrade.PerformAddonsUpgrade(client, cfg, data.OutputWriter()); err != nil {
+				return errors.Wrap(err, "failed to perform addons upgrade")
+			}
+		}
+
 		fmt.Println("[upgrade] The control plane instance for this node was successfully updated!")
 
 		return nil

cmd/kubeadm/app/cmd/upgrade/node.go

@@ -41,6 +41,7 @@ import (
 // supported by this api will be exposed as a flag.
 type nodeOptions struct {
 	kubeConfigPath        string
+	isControlPlaneNode    bool
 	etcdUpgrade           bool
 	renewCerts            bool
 	dryRun                bool
@@ -105,11 +106,26 @@ func newCmdNode(out io.Writer) *cobra.Command {
 
 // newNodeOptions returns a struct ready for being used for creating cmd kubeadm upgrade node flags.
 func newNodeOptions() *nodeOptions {
+	kubeConfigPath := constants.GetKubeletKubeConfigPath()
+
+	// isControlPlaneNode checks if a node is a control-plane node by looking up
+	// the kube-apiserver manifest file
+	isControlPlaneNode := true
+	filepath := constants.GetStaticPodFilepath(constants.KubeAPIServer, constants.GetStaticPodDirectory())
+	if _, err := os.Stat(filepath); os.IsNotExist(err) {
+		isControlPlaneNode = false
+	}
+
+	if isControlPlaneNode {
+		kubeConfigPath = constants.GetAdminKubeConfigPath()
+	}
+
 	return &nodeOptions{
-		kubeConfigPath: constants.GetKubeletKubeConfigPath(),
-		dryRun:         false,
-		renewCerts:     true,
-		etcdUpgrade:    true,
+		kubeConfigPath:     kubeConfigPath,
+		isControlPlaneNode: isControlPlaneNode,
+		dryRun:             false,
+		renewCerts:         true,
+		etcdUpgrade:        true,
 	}
 }
 
@@ -129,19 +145,10 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions, out io
 	if err != nil {
 		return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", options.kubeConfigPath)
 	}
-
-	// isControlPlane checks if a node is a control-plane node by looking up
-	// the kube-apiserver manifest file
-	isControlPlaneNode := true
-	filepath := constants.GetStaticPodFilepath(constants.KubeAPIServer, constants.GetStaticPodDirectory())
-	if _, err := os.Stat(filepath); os.IsNotExist(err) {
-		isControlPlaneNode = false
-	}
-
 	// Fetches the cluster configuration
 	// NB in case of control-plane node, we are reading all the info for the node; in case of NOT control-plane node
 	//    (worker node), we are not reading local API address and the CRI socket from the node object
-	cfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", !isControlPlaneNode, false)
+	cfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", !options.isControlPlaneNode, false)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap")
 	}
@@ -158,7 +165,7 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions, out io
 		dryRun:                options.dryRun,
 		cfg:                   cfg,
 		client:                client,
-		isControlPlaneNode:    isControlPlaneNode,
+		isControlPlaneNode:    options.isControlPlaneNode,
 		patchesDir:            options.patchesDir,
 		ignorePreflightErrors: ignorePreflightErrorsSet,
 		kubeConfigPath:        options.kubeConfigPath,

cmd/kubeadm/app/features/features.go

@@ -35,13 +35,16 @@ const (
 	RootlessControlPlane = "RootlessControlPlane"
 	// EtcdLearnerMode is expected to be in alpha in v1.27
 	EtcdLearnerMode = "EtcdLearnerMode"
+	// UpgradeAddonsAfterControlPlane is expected to be in alpha in v1.28
+	UpgradeAddonsAfterControlPlane = "UpgradeAddonsAfterControlPlane"
 )
 
 // InitFeatureGates are the default feature gates for the init command
 var InitFeatureGates = FeatureList{
-	PublicKeysECDSA:      {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
-	RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
-	EtcdLearnerMode:      {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
+	PublicKeysECDSA:                {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
+	RootlessControlPlane:           {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
+	EtcdLearnerMode:                {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
+	UpgradeAddonsAfterControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
 }
 
 // Feature represents a feature being gated

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -27,12 +27,15 @@ import (
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	errorsutil "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
@@ -103,6 +106,41 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
 		errs = append(errs, err)
 	}
 
+	if err := PerformAddonsUpgrade(client, cfg, out); err != nil {
+		errs = append(errs, err)
+	}
+
+	return errorsutil.NewAggregate(errs)
+}
+
+// PerformAddonsUpgrade performs the upgrade of the coredns and kube-proxy addons.
+// When UpgradeAddonsAfterControlPlane feature gate is disabled, the addons will be upgraded immediately.
+// When UpgradeAddonsAfterControlPlane feature gate is enabled, the addons will only get updated after all the control plane instances have been upgraded.
+func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, out io.Writer) error {
+	unupgradedControlPlanes, err := unupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
+	if err != nil {
+		err = errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded")
+		if features.Enabled(cfg.FeatureGates, features.UpgradeAddonsAfterControlPlane) {
+			return err
+		}
+
+		// when UpgradeAddonsAfterControlPlane feature gate is disabled, just throw a warning
+		klog.V(1).Info(err)
+	}
+	if len(unupgradedControlPlanes) > 0 {
+		if features.Enabled(cfg.FeatureGates, features.UpgradeAddonsAfterControlPlane) {
+			fmt.Fprintf(out, "[upgrade/addons] skip upgrade addons because control plane instances %v have not been upgraded\n", unupgradedControlPlanes)
+			return nil
+		}
+
+		// when UpgradeAddonsAfterControlPlane feature gate is disabled, just throw a warning
+		klog.V(1).Infof("upgrading addons when control plane instances %v have not been upgraded "+
+			"may lead to incompatibility problems. You can enable the UpgradeAddonsAfterControlPlane feature gate to"+
+			"ensure that the addons upgrade is executed only when all the control plane instances have been upgraded.", unupgradedControlPlanes)
+	}
+
+	var errs []error
+
 	// If the coredns ConfigMap is missing, show a warning and assume that the
 	// DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
 	//
@@ -156,6 +194,60 @@ func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitCon
 	return errorsutil.NewAggregate(errs)
 }
 
+// unupgradedControlPlaneInstances returns a list of control palne instances that have not yet been upgraded.
+//
+// NB. This function can only be called after the current control plane instance has been upgraded already.
+// Because it determines whether the other control plane instances have been upgraded by checking whether
+// the kube-apiserver image of other control plane instance is the same as that of this instance.
+func unupgradedControlPlaneInstances(client clientset.Interface, nodeName string) ([]string, error) {
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{
+		"component": kubeadmconstants.KubeAPIServer,
+	}))
+	pods, err := client.CoreV1().Pods(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{
+		LabelSelector: selector.String(),
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list kube-apiserver Pod from cluster")
+	}
+	if len(pods.Items) == 0 {
+		return nil, errors.Errorf("cannot find kube-apiserver Pod by label selector: %v", selector.String())
+	}
+
+	nodeImageMap := map[string]string{}
+
+	for _, pod := range pods.Items {
+		found := false
+		for _, c := range pod.Spec.Containers {
+			if c.Name == kubeadmconstants.KubeAPIServer {
+				nodeImageMap[pod.Spec.NodeName] = c.Image
+				found = true
+				break
+			}
+		}
+		if !found {
+			return nil, errors.Errorf("cannot find container by name %q for Pod %v", kubeadmconstants.KubeAPIServer, klog.KObj(&pod))
+		}
+	}
+
+	upgradedImage, ok := nodeImageMap[nodeName]
+	if !ok {
+		return nil, errors.Errorf("cannot find kube-apiserver image for current control plane instance %v", nodeName)
+	}
+
+	unupgradedNodes := sets.New[string]()
+	for node, image := range nodeImageMap {
+		if image != upgradedImage {
+			unupgradedNodes.Insert(node)
+		}
+	}
+
+	if len(unupgradedNodes) > 0 {
+		return sets.List(unupgradedNodes), nil
+	}
+
+	return nil, nil
+}
+
 func WriteKubeletConfigFiles(cfg *kubeadmapi.InitConfiguration, patchesDir string, dryRun bool, out io.Writer) error {
 	// Set up the kubelet directory to use. If dry-running, this will return a fake directory
 	kubeletDir, err := GetKubeletDir(dryRun)