kubeadm: implement 'kubeadm upgrade apply phase'

Signed-off-by: SataQiu <shidaqiu2018@gmail.com>

cmd/kubeadm/app/cmd/phases/upgrade/apply/addons.go

@@ -0,0 +1,188 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package apply implements phases of 'kubeadm upgrade apply'.
+package apply
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pkg/errors"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	dnsaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
+	proxyaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
+)
+
+// NewAddonPhase returns the addon Cobra command
+func NewAddonPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:  "addon",
+		Short: "Install required addons for passing conformance tests",
+		Long:  cmdutil.MacroCommandLongDescription,
+		Phases: []workflow.Phase{
+			{
+				Name:           "all",
+				Short:          "Install all the addons",
+				InheritFlags:   getAddonPhaseFlags("all"),
+				RunAllSiblings: true,
+			},
+			{
+				Name:         "coredns",
+				Short:        "Install the CoreDNS addon to a Kubernetes cluster",
+				InheritFlags: getAddonPhaseFlags("coredns"),
+				Run:          runCoreDNSAddon,
+			},
+			{
+				Name:         "kube-proxy",
+				Short:        "Install the kube-proxy addon to a Kubernetes cluster",
+				InheritFlags: getAddonPhaseFlags("kube-proxy"),
+				Run:          runKubeProxyAddon,
+			},
+		},
+	}
+}
+
+func shouldUpgradeAddons(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, out io.Writer) (bool, error) {
+	unupgradedControlPlanes, err := upgrade.UnupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded")
+	}
+	if len(unupgradedControlPlanes) > 0 {
+		fmt.Fprintf(out, "[upgrade/addons] skip upgrade addons because control plane instances %v have not been upgraded\n", unupgradedControlPlanes)
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func getInitData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, string, io.Writer, bool, error) {
+	data, ok := c.(Data)
+	if !ok {
+		return nil, nil, "", nil, false, errors.New("addon phase invoked with an invalid data struct")
+	}
+	return data.InitCfg(), data.Client(), data.PatchesDir(), data.OutputWriter(), data.DryRun(), nil
+}
+
+// runCoreDNSAddon installs CoreDNS addon to a Kubernetes cluster
+func runCoreDNSAddon(c workflow.RunData) error {
+	cfg, client, patchesDir, out, dryRun, err := getInitData(c)
+	if err != nil {
+		return err
+	}
+
+	shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out)
+	if err != nil {
+		return err
+	}
+	if !shouldUpgradeAddons {
+		return nil
+	}
+
+	// If the coredns ConfigMap is missing, show a warning and assume that the
+	// DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
+	//
+	// TODO: remove this once "kubeadm upgrade apply" phases are supported:
+	//   https://github.com/kubernetes/kubeadm/issues/1318
+	if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
+		context.TODO(),
+		kubeadmconstants.CoreDNSConfigMap,
+		metav1.GetOptions{},
+	); err != nil && apierrors.IsNotFound(err) {
+		klog.Warningf("the ConfigMaps %q in the namespace %q were not found. "+
+			"Assuming that a DNS server was not deployed for this cluster. "+
+			"Note that once 'kubeadm upgrade apply' supports phases you "+
+			"will have to skip the DNS upgrade manually",
+			kubeadmconstants.CoreDNSConfigMap,
+			metav1.NamespaceSystem)
+		return nil
+	}
+
+	// Upgrade CoreDNS
+	if err := dnsaddon.EnsureDNSAddon(&cfg.ClusterConfiguration, client, patchesDir, out, dryRun); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// runKubeProxyAddon installs KubeProxy addon to a Kubernetes cluster
+func runKubeProxyAddon(c workflow.RunData) error {
+	cfg, client, _, out, dryRun, err := getInitData(c)
+	if err != nil {
+		return err
+	}
+
+	shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out)
+	if err != nil {
+		return err
+	}
+	if !shouldUpgradeAddons {
+		return nil
+	}
+
+	// If the kube-proxy ConfigMap is missing, show a warning and assume that kube-proxy
+	// was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
+	//
+	// TODO: remove this once "kubeadm upgrade apply" phases are supported:
+	//   https://github.com/kubernetes/kubeadm/issues/1318
+	if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
+		context.TODO(),
+		kubeadmconstants.KubeProxyConfigMap,
+		metav1.GetOptions{},
+	); err != nil && apierrors.IsNotFound(err) {
+		klog.Warningf("the ConfigMap %q in the namespace %q was not found. "+
+			"Assuming that kube-proxy was not deployed for this cluster. "+
+			"Note that once 'kubeadm upgrade apply' supports phases you "+
+			"will have to skip the kube-proxy upgrade manually",
+			kubeadmconstants.KubeProxyConfigMap,
+			metav1.NamespaceSystem)
+		return nil
+	}
+
+	// Upgrade kube-proxy
+	if err := proxyaddon.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, dryRun); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getAddonPhaseFlags(name string) []string {
+	flags := []string{
+		options.CfgPath,
+		options.KubeconfigPath,
+		options.DryRun,
+	}
+	if name == "all" || name == "coredns" {
+		flags = append(flags,
+			options.Patches,
+		)
+	}
+	return flags
+}

cmd/kubeadm/app/cmd/phases/upgrade/apply/bootstraptoken.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package apply implements phases of 'kubeadm upgrade apply'.
+package apply
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	errorsutil "k8s.io/apimachinery/pkg/util/errors"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
+	nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
+)
+
+// NewBootstrapTokenPhase returns the phase to bootstrapToken
+func NewBootstrapTokenPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:    "bootstrap-token",
+		Aliases: []string{"bootstraptoken"},
+		Short:   "Generates bootstrap tokens used to join a node to a cluster",
+		InheritFlags: []string{
+			options.CfgPath,
+			options.KubeconfigPath,
+			options.DryRun,
+		},
+		Run: runBootstrapToken,
+	}
+}
+
+func runBootstrapToken(c workflow.RunData) error {
+	data, ok := c.(Data)
+	if !ok {
+		return errors.New("bootstrap-token phase invoked with an invalid data struct")
+	}
+
+	if data.DryRun() {
+		fmt.Println("[dryrun] Would config cluster-info ConfigMap, RBAC Roles")
+		return nil
+	}
+
+	fmt.Println("[bootstrap-token] Configuring cluster-info ConfigMap, RBAC Roles")
+
+	client := data.Client()
+
+	var errs []error
+	// Create RBAC rules that makes the bootstrap tokens able to get nodes
+	if err := nodebootstraptoken.AllowBootstrapTokensToGetNodes(client); err != nil {
+		errs = append(errs, err)
+	}
+
+	// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
+	if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
+		errs = append(errs, err)
+	}
+
+	// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
+	if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
+		errs = append(errs, err)
+	}
+
+	// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
+	if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
+		errs = append(errs, err)
+	}
+
+	// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
+	if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil {
+		errs = append(errs, err)
+	}
+
+	return errorsutil.NewAggregate(errs)
+}

cmd/kubeadm/app/cmd/phases/upgrade/apply/controlplane.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package apply implements phases of 'kubeadm upgrade apply'.
+package apply
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+)
+
+// NewControlPlanePhase creates a kubeadm workflow phase that implements handling of control-plane upgrade.
+func NewControlPlanePhase() workflow.Phase {
+	phase := workflow.Phase{
+		Name:  "control-plane",
+		Short: "Upgrade the control plane",
+		Run:   runControlPlane,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.KubeconfigPath,
+			options.DryRun,
+			options.CertificateRenewal,
+			options.EtcdUpgrade,
+			options.Patches,
+		},
+	}
+	return phase
+}
+
+func runControlPlane(c workflow.RunData) error {
+	data, ok := c.(Data)
+	if !ok {
+		return errors.New("control-plane phase invoked with an invalid data struct")
+	}
+
+	initCfg, upgradeCfg, client, patchesDir := data.InitCfg(), data.Cfg(), data.Client(), data.PatchesDir()
+
+	if data.DryRun() {
+		fmt.Printf("[dryrun] Would upgrade your Static Pod-hosted control plane to version %q", initCfg.KubernetesVersion)
+		return upgrade.DryRunStaticPodUpgrade(patchesDir, initCfg)
+	}
+
+	fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n",
+		initCfg.KubernetesVersion, upgradeCfg.Timeouts.UpgradeManifests.Duration)
+
+	waiter := apiclient.NewKubeWaiter(client, upgradeCfg.Timeouts.UpgradeManifests.Duration, os.Stdout)
+	if err := upgrade.PerformStaticPodUpgrade(client, waiter, initCfg, data.EtcdUpgrade(), data.RenewCerts(), patchesDir); err != nil {
+		return errors.Wrap(err, "couldn't complete the static pod upgrade")
+	}
+
+	fmt.Println("[upgrade] The control plane instance for this node was successfully updated!")
+
+	return nil
+}

cmd/kubeadm/app/cmd/phases/upgrade/apply/data.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package apply implements phases of 'kubeadm upgrade apply'.
+package apply
+
+import (
+	"io"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	clientset "k8s.io/client-go/kubernetes"
+
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+)
+
+// Data is the interface to use for kubeadm upgrade apply phases.
+// The "applyData" type from "cmd/upgrade/apply.go" must satisfy this interface.
+type Data interface {
+	EtcdUpgrade() bool
+	RenewCerts() bool
+	DryRun() bool
+	Cfg() *kubeadmapi.UpgradeConfiguration
+	InitCfg() *kubeadmapi.InitConfiguration
+	Client() clientset.Interface
+	IgnorePreflightErrors() sets.Set[string]
+	PatchesDir() string
+	OutputWriter() io.Writer
+	SessionIsInteractive() bool
+	AllowExperimentalUpgrades() bool
+	AllowRCUpgrades() bool
+	ForceUpgrade() bool
+}

cmd/kubeadm/app/cmd/phases/upgrade/apply/data_test.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apply
+
+import (
+	"io"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	clientset "k8s.io/client-go/kubernetes"
+
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+)
+
+// a package local type for testing purposes.
+type testData struct{}
+
+// testData must satisfy Data.
+var _ Data = &testData{}
+
+func (t *testData) EtcdUpgrade() bool                       { return false }
+func (t *testData) RenewCerts() bool                        { return false }
+func (t *testData) DryRun() bool                            { return false }
+func (t *testData) Cfg() *kubeadmapi.UpgradeConfiguration   { return nil }
+func (t *testData) InitCfg() *kubeadmapi.InitConfiguration  { return nil }
+func (t *testData) Client() clientset.Interface             { return nil }
+func (t *testData) IgnorePreflightErrors() sets.Set[string] { return nil }
+func (t *testData) PatchesDir() string                      { return "" }
+func (t *testData) OutputWriter() io.Writer                 { return nil }
+func (t *testData) SessionIsInteractive() bool              { return false }
+func (t *testData) AllowExperimentalUpgrades() bool         { return false }
+func (t *testData) AllowRCUpgrades() bool                   { return false }
+func (t *testData) ForceUpgrade() bool                      { return false }

cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeconfig.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package apply implements phases of 'kubeadm upgrade apply'.
+package apply
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
+)
+
+// NewKubeconfigPhase creates a kubeadm workflow phase that implements handling of kubeconfig upgrade.
+func NewKubeconfigPhase() workflow.Phase {
+	phase := workflow.Phase{
+		Name:   "kubeconfig",
+		Short:  "Upgrade kubeconfig files for this node",
+		Run:    runKubeconfig(),
+		Hidden: true,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.DryRun,
+			options.KubeconfigPath,
+		},
+	}
+	return phase
+}
+
+func runKubeconfig() func(c workflow.RunData) error {
+	return func(c workflow.RunData) error {
+		data, ok := c.(Data)
+		if !ok {
+			return errors.New("kubeconfig phase invoked with an invalid data struct")
+		}
+
+		cfg := data.InitCfg()
+
+		if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
+			if err := upgrade.UpdateKubeletLocalMode(cfg, data.DryRun()); err != nil {
+				return errors.Wrap(err, "failed to update kubelet local mode")
+			}
+		}
+
+		fmt.Println("[upgrade] The kubeconfig for this node was successfully updated!")
+
+		return nil
+	}
+}

cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeletconfig.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package apply implements phases of 'kubeadm upgrade apply'.
+package apply
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
+)
+
+var (
+	kubeletConfigLongDesc = cmdutil.LongDesc(`
+		Download the kubelet configuration from the kubelet-config ConfigMap stored in the cluster
+		`)
+)
+
+// NewKubeletConfigPhase creates a kubeadm workflow phase that implements handling of kubelet-config upgrade.
+func NewKubeletConfigPhase() workflow.Phase {
+	phase := workflow.Phase{
+		Name:  "kubelet-config",
+		Short: "Upgrade the kubelet configuration for this node",
+		Long:  kubeletConfigLongDesc,
+		Run:   runKubeletConfigPhase,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.DryRun,
+			options.KubeconfigPath,
+			options.Patches,
+		},
+	}
+	return phase
+}
+
+func runKubeletConfigPhase(c workflow.RunData) error {
+	data, ok := c.(Data)
+	if !ok {
+		return errors.New("kubelet-config phase invoked with an invalid data struct")
+	}
+
+	initCfg, dryRun := data.InitCfg(), data.DryRun()
+
+	// Write the configuration for the kubelet down to disk and print the generated manifests instead if dry-running.
+	// If not dry-running, the kubelet config file will be backed up to /etc/kubernetes/tmp/ dir, so that it could be
+	// recovered if there is anything goes wrong.
+	err := upgrade.WriteKubeletConfigFiles(initCfg, data.PatchesDir(), dryRun, data.OutputWriter())
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("[upgrade] The configuration for this node was successfully updated!")
+	fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.")
+	return nil
+}

cmd/kubeadm/app/cmd/phases/upgrade/apply/preflight.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package apply implements phases of 'kubeadm upgrade apply'.
+package apply
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/klog/v2"
+	utilsexec "k8s.io/utils/exec"
+
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
+	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/output"
+)
+
+// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for kubeadm upgrade apply.
+func NewPreflightPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:  "preflight",
+		Short: "Run upgrade apply pre-flight checks",
+		Long:  "Run pre-flight checks for kubeadm upgrade apply.",
+		Run:   runPreflight,
+		InheritFlags: []string{
+			options.CfgPath,
+			options.KubeconfigPath,
+			options.DryRun,
+			options.IgnorePreflightErrors,
+			"allow-experimental-upgrades",
+			"allow-release-candidate-upgrades",
+			"force",
+			"yes",
+		},
+	}
+}
+
+// runPreflight executes preflight checks logic.
+func runPreflight(c workflow.RunData) error {
+	data, ok := c.(Data)
+	if !ok {
+		return errors.New("preflight phase invoked with an invalid data struct")
+	}
+	fmt.Println("[preflight] Running pre-flight checks")
+
+	printer := &output.TextPrinter{}
+
+	initCfg, client, ignorePreflightErrors := data.InitCfg(), data.Client(), data.IgnorePreflightErrors()
+
+	// First, check if we're root separately from the other preflight checks and fail fast
+	if err := preflight.RunRootCheckOnly(ignorePreflightErrors); err != nil {
+		return err
+	}
+
+	// Run CoreDNS migration check
+	if err := upgrade.RunCoreDNSMigrationCheck(client, ignorePreflightErrors); err != nil {
+		return err
+	}
+
+	// Run healthchecks against the cluster
+	klog.V(1).Infoln("[upgrade/apply] verifying health of cluster")
+	if err := upgrade.CheckClusterHealth(client, &initCfg.ClusterConfiguration, ignorePreflightErrors, printer); err != nil {
+		return err
+	}
+
+	// Check if feature gate flags used in the cluster are consistent with the set of features currently supported by kubeadm
+	if msg := features.CheckDeprecatedFlags(&features.InitFeatureGates, initCfg.FeatureGates); len(msg) > 0 {
+		for _, m := range msg {
+			_, _ = printer.Printf("[upgrade/config] %s\n", m)
+		}
+	}
+
+	// Validate requested and validate actual version
+	klog.V(1).Infoln("[upgrade/apply] validating requested and actual version")
+	if err := configutil.NormalizeKubernetesVersion(&initCfg.ClusterConfiguration); err != nil {
+		return err
+	}
+
+	// Use normalized version string in all following code.
+	upgradeVersion, err := version.ParseSemantic(initCfg.KubernetesVersion)
+	if err != nil {
+		return errors.Errorf("unable to parse normalized version %q as a semantic version", initCfg.KubernetesVersion)
+	}
+
+	if err := features.ValidateVersion(features.InitFeatureGates, initCfg.FeatureGates, initCfg.KubernetesVersion); err != nil {
+		return err
+	}
+
+	versionGetter := upgrade.NewOfflineVersionGetter(upgrade.NewKubeVersionGetter(client), initCfg.KubernetesVersion)
+	if err := EnforceVersionPolicies(initCfg.KubernetesVersion, upgradeVersion, data.AllowExperimentalUpgrades(), data.AllowRCUpgrades(), data.ForceUpgrade(), versionGetter); err != nil {
+		return err
+	}
+
+	if data.SessionIsInteractive() {
+		if err := cmdutil.InteractivelyConfirmAction("upgrade", "Are you sure you want to proceed?", os.Stdin); err != nil {
+			return err
+		}
+	}
+
+	if !data.DryRun() {
+		fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
+		fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
+		fmt.Println("[preflight] You can also perform this action beforehand using 'kubeadm config images pull'")
+		if err := preflight.RunPullImagesCheck(utilsexec.New(), initCfg, ignorePreflightErrors); err != nil {
+			return err
+		}
+	} else {
+		fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')")
+	}
+
+	return nil
+}
+
+// EnforceVersionPolicies makes sure that the version the user specified is valid to upgrade to
+// There are both fatal and skippable (with --force) errors
+func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Version, allowExperimentalUpgrades, allowRCUpgrades, force bool, versionGetter upgrade.VersionGetter) error {
+	fmt.Printf("[upgrade/version] You have chosen to change the cluster version to %q\n", newK8sVersionStr)
+
+	versionSkewErrs := upgrade.EnforceVersionPolicies(versionGetter, newK8sVersionStr, newK8sVersion, allowExperimentalUpgrades, allowRCUpgrades)
+	if versionSkewErrs != nil {
+
+		if len(versionSkewErrs.Mandatory) > 0 {
+			return errors.Errorf("the --version argument is invalid due to these fatal errors:\n\n%v\nPlease fix the misalignments highlighted above and try upgrading again",
+				kubeadmutil.FormatErrMsg(versionSkewErrs.Mandatory))
+		}
+
+		if len(versionSkewErrs.Skippable) > 0 {
+			// Return the error if the user hasn't specified the --force flag
+			if !force {
+				return errors.Errorf("the --version argument is invalid due to these errors:\n\n%v\nCan be bypassed if you pass the --force flag",
+					kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
+			}
+			// Soft errors found, but --force was specified
+			fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: \n\n%v", len(versionSkewErrs.Skippable), kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
+		}
+	}
+	return nil
+}

cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package apply implements phases of 'kubeadm upgrade apply'.
+package apply
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
+	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
+	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
+)
+
+// NewUploadConfigPhase returns the phase to uploadConfig
+func NewUploadConfigPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:    "upload-config",
+		Aliases: []string{"uploadconfig"},
+		Short:   "Upload the kubeadm and kubelet configuration to a ConfigMap",
+		Long:    cmdutil.MacroCommandLongDescription,
+		Phases: []workflow.Phase{
+			{
+				Name:           "all",
+				Short:          "Upload all configuration to a config map",
+				RunAllSiblings: true,
+				InheritFlags:   getUploadConfigPhaseFlags(),
+			},
+			{
+				Name:         "kubeadm",
+				Short:        "Upload the kubeadm ClusterConfiguration to a ConfigMap",
+				Run:          runUploadKubeadmConfig,
+				InheritFlags: getUploadConfigPhaseFlags(),
+			},
+			{
+				Name:         "kubelet",
+				Short:        "Upload the kubelet component config to a ConfigMap",
+				Run:          runUploadKubeletConfig,
+				InheritFlags: getUploadConfigPhaseFlags(),
+			},
+		},
+	}
+}
+
+func getUploadConfigPhaseFlags() []string {
+	return []string{
+		options.CfgPath,
+		options.KubeconfigPath,
+		options.DryRun,
+	}
+}
+
+// runUploadKubeadmConfig uploads the kubeadm configuration to a ConfigMap
+func runUploadKubeadmConfig(c workflow.RunData) error {
+	cfg, client, dryRun, err := getUploadConfigData(c)
+	if err != nil {
+		return err
+	}
+
+	if dryRun {
+		fmt.Println("[dryrun] Would upload the kubeadm ClusterConfiguration to a ConfigMap")
+		return nil
+	}
+
+	klog.V(1).Infoln("[upload-config] Uploading the kubeadm ClusterConfiguration to a ConfigMap")
+	if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
+		return errors.Wrap(err, "error uploading the kubeadm ClusterConfiguration")
+	}
+	return nil
+}
+
+// runUploadKubeletConfig uploads the kubelet configuration to a ConfigMap
+func runUploadKubeletConfig(c workflow.RunData) error {
+	cfg, client, dryRun, err := getUploadConfigData(c)
+	if err != nil {
+		return err
+	}
+
+	if dryRun {
+		fmt.Println("[dryrun] Would upload the kubelet component config to a ConfigMap")
+		fmt.Println("[dryrun] Would write the CRISocket annotation for the control-plane node")
+		return nil
+	}
+
+	klog.V(1).Infoln("[upload-config] Uploading the kubelet component config to a ConfigMap")
+	if err = kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, client); err != nil {
+		return errors.Wrap(err, "error creating kubelet configuration ConfigMap")
+	}
+
+	klog.V(1).Infoln("[upload-config] Preserving the CRISocket information for the control-plane node")
+	if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
+		return errors.Wrap(err, "Error writing Crisocket information for the control-plane node")
+	}
+
+	return nil
+}
+
+func getUploadConfigData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, bool, error) {
+	data, ok := c.(Data)
+	if !ok {
+		return nil, nil, false, errors.New("upload-config phase invoked with an invalid data struct")
+	}
+
+	return data.InitCfg(), data.Client(), data.DryRun(), nil
+}

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -18,27 +18,26 @@ package upgrade
 
 import (
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apimachinery/pkg/util/version"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
-	utilsexec "k8s.io/utils/exec"
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/apply"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
-	"k8s.io/kubernetes/cmd/kubeadm/app/features"
-	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
-	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
-	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
-	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/output"
 )
@@ -55,9 +54,26 @@ type applyFlags struct {
 	patchesDir         string
 }
 
-// sessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run)
-func (f *applyFlags) sessionIsInteractive() bool {
-	return !(f.nonInteractiveMode || f.dryRun || f.force)
+// compile-time assert that the local data object satisfies the phases data interface.
+var _ phases.Data = &applyData{}
+
+// applyData defines all the runtime information used when running the kubeadm upgrade apply workflow;
+// this data is shared across all the phases that are included in the workflow.
+type applyData struct {
+	nonInteractiveMode        bool
+	force                     bool
+	dryRun                    bool
+	etcdUpgrade               bool
+	renewCerts                bool
+	allowExperimentalUpgrades bool
+	allowRCUpgrades           bool
+	printConfig               bool
+	cfg                       *kubeadmapi.UpgradeConfiguration
+	initCfg                   *kubeadmapi.InitConfiguration
+	client                    clientset.Interface
+	patchesDir                string
+	ignorePreflightErrors     sets.Set[string]
+	outputWriter              io.Writer
 }
 
 // newCmdApply returns the cobra command for `kubeadm upgrade apply`
@@ -68,6 +84,8 @@ func newCmdApply(apf *applyPlanFlags) *cobra.Command {
 		renewCerts:     true,
 	}
 
+	applyRunner := workflow.NewRunner()
+
 	cmd := &cobra.Command{
 		Use:                   "apply [version]",
 		DisableFlagsInUseLine: true,
@@ -76,7 +94,32 @@ func newCmdApply(apf *applyPlanFlags) *cobra.Command {
 			if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil {
 				return err
 			}
-			return runApply(cmd.Flags(), flags, args)
+
+			data, err := applyRunner.InitData(args)
+			if err != nil {
+				return err
+			}
+
+			applyData, ok := data.(*applyData)
+			if !ok {
+				return errors.New("invalid data struct")
+			}
+
+			if err := applyRunner.Run(args); err != nil {
+				return err
+			}
+
+			if flags.dryRun {
+				fmt.Println("[upgrade/successful] Finished dryrunning successfully!")
+				return nil
+			}
+
+			fmt.Println("")
+			fmt.Printf("[upgrade/successful] SUCCESS! Your cluster was upgraded to %q. Enjoy!\n", applyData.InitCfg().KubernetesVersion)
+			fmt.Println("")
+			fmt.Println("[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.")
+
+			return nil
 		},
 	}
 
@@ -90,181 +133,222 @@ func newCmdApply(apf *applyPlanFlags) *cobra.Command {
 	cmd.Flags().BoolVar(&flags.renewCerts, options.CertificateRenewal, flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
 	options.AddPatchesFlag(cmd.Flags(), &flags.patchesDir)
 
+	// initialize the workflow runner with the list of phases
+	applyRunner.AppendPhase(phases.NewPreflightPhase())
+	applyRunner.AppendPhase(phases.NewControlPlanePhase())
+	applyRunner.AppendPhase(phases.NewUploadConfigPhase())
+	applyRunner.AppendPhase(phases.NewKubeconfigPhase())
+	applyRunner.AppendPhase(phases.NewKubeletConfigPhase())
+	applyRunner.AppendPhase(phases.NewBootstrapTokenPhase())
+	applyRunner.AppendPhase(phases.NewAddonPhase())
+
+	// sets the data builder function, that will be used by the runner
+	// both when running the entire workflow or single phases
+	applyRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) {
+		data, err := newApplyData(cmd, args, flags)
+		if err != nil {
+			return nil, err
+		}
+		// If the flag for skipping phases was empty, use the values from config
+		if len(applyRunner.Options.SkipPhases) == 0 {
+			applyRunner.Options.SkipPhases = data.cfg.Apply.SkipPhases
+		}
+		return data, nil
+	})
+
+	// binds the Runner to kubeadm upgrade apply command by altering
+	// command help, adding --skip-phases flag and by adding phases subcommands
+	applyRunner.BindToCommand(cmd)
+
 	return cmd
 }
 
-// runApply takes care of the actual upgrade functionality
-// It does the following things:
-// - Checks if the cluster is healthy
-// - Gets the configuration from the kubeadm-config ConfigMap in the cluster
-// - Enforces all version skew policies
-// - Asks the user if they really want to upgrade
-// - Makes sure the control plane images are available locally on the control-plane(s)
-// - Upgrades the control plane components
-// - Applies the other resources that'd be created with kubeadm init as well, like
-//   - Uploads the newly used configuration to the cluster ConfigMap
-//   - Creating the RBAC rules for the bootstrap tokens and the cluster-info ConfigMap
-//   - Applying new CoreDNS and kube-proxy manifests
-func runApply(flagSet *pflag.FlagSet, flags *applyFlags, args []string) error {
+// newApplyData returns a new applyData struct to be used for the execution of the kubeadm upgrade apply workflow.
+func newApplyData(cmd *cobra.Command, args []string, applyFlags *applyFlags) (*applyData, error) {
+	externalCfg := &v1beta4.UpgradeConfiguration{}
+	opt := configutil.LoadOrDefaultConfigurationOptions{}
+	upgradeCfg, err := configutil.LoadOrDefaultUpgradeConfiguration(applyFlags.cfgPath, externalCfg, opt)
+	if err != nil {
+		return nil, err
+	}
 
-	// Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap)
-	klog.V(1).Infoln("[upgrade/apply] verifying health of cluster")
+	upgradeVersion := upgradeCfg.Apply.KubernetesVersion
+	// The version arg is mandatory, during upgrade apply, unless it's specified in the config file
+	if upgradeVersion == "" {
+		if err := cmdutil.ValidateExactArgNumber(args, []string{"version"}); err != nil {
+			return nil, err
+		}
+	}
+
+	// If option was specified in both args and config file, args will overwrite the config file.
+	if len(args) == 1 {
+		upgradeVersion = args[0]
+	}
+
+	force, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "force", upgradeCfg.Apply.ForceUpgrade, &applyFlags.force).(*bool)
+	if !ok {
+		return nil, cmdutil.TypeMismatchErr("forceUpgrade", "bool")
+	}
+
+	dryRun, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.DryRun, upgradeCfg.Apply.DryRun, &applyFlags.dryRun).(*bool)
+	if !ok {
+		return nil, cmdutil.TypeMismatchErr("dryRun", "bool")
+	}
+
+	etcdUpgrade, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.EtcdUpgrade, upgradeCfg.Apply.EtcdUpgrade, &applyFlags.etcdUpgrade).(*bool)
+	if !ok {
+		return nil, cmdutil.TypeMismatchErr("etcdUpgrade", "bool")
+	}
+
+	renewCerts, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.CertificateRenewal, upgradeCfg.Apply.CertificateRenewal, &applyFlags.renewCerts).(*bool)
+	if !ok {
+		return nil, cmdutil.TypeMismatchErr("certificateRenewal", "bool")
+	}
+
+	allowExperimentalUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "allow-experimental-upgrades", upgradeCfg.Apply.AllowExperimentalUpgrades, &applyFlags.allowExperimentalUpgrades).(*bool)
+	if !ok {
+		return nil, cmdutil.TypeMismatchErr("allowExperimentalUpgrades", "bool")
+	}
+
+	allowRCUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "allow-release-candidate-upgrades", upgradeCfg.Apply.AllowRCUpgrades, &applyFlags.allowRCUpgrades).(*bool)
+	if !ok {
+		return nil, cmdutil.TypeMismatchErr("allowRCUpgrades", "bool")
+	}
+
+	printConfig, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "print-config", upgradeCfg.Apply.PrintConfig, &applyFlags.printConfig).(*bool)
+	if !ok {
+		return nil, cmdutil.TypeMismatchErr("printConfig", "bool")
+	}
+
+	client, err := getClient(applyFlags.kubeConfigPath, *dryRun)
+	if err != nil {
+		return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", applyFlags.kubeConfigPath)
+	}
+
+	printer := &output.TextPrinter{}
+
+	// Fetches the cluster configuration
 	klog.V(1).Infoln("[upgrade/apply] retrieving configuration from cluster")
-	client, versionGetter, initCfg, upgradeCfg, err := enforceRequirements(flagSet, flags.applyPlanFlags, args, flags.dryRun, true, &output.TextPrinter{})
+	initCfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", false, false)
 	if err != nil {
-		return err
+		if apierrors.IsNotFound(err) {
+			_, _ = printer.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %q namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
+			_, _ = printer.Printf("[upgrade/config] Use 'kubeadm init phase upload-config --config your-config.yaml' to re-upload it.\n")
+			err = errors.Errorf("the ConfigMap %q in the %q namespace was not found", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
+		}
+		return nil, errors.Wrap(err, "[upgrade/init config] FATAL")
 	}
 
-	// Validate requested and validate actual version
-	klog.V(1).Infoln("[upgrade/apply] validating requested and actual version")
-	if err := configutil.NormalizeKubernetesVersion(&initCfg.ClusterConfiguration); err != nil {
-		return err
-	}
-
-	// Use normalized version string in all following code.
-	newK8sVersion, err := version.ParseSemantic(initCfg.KubernetesVersion)
+	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(applyFlags.ignorePreflightErrors, upgradeCfg.Apply.IgnorePreflightErrors)
 	if err != nil {
-		return errors.Errorf("unable to parse normalized version %q as a semantic version", initCfg.KubernetesVersion)
+		return nil, err
 	}
+	// Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration:
+	initCfg.NodeRegistration.IgnorePreflightErrors = sets.List(ignorePreflightErrorsSet)
 
-	if err := features.ValidateVersion(features.InitFeatureGates, initCfg.FeatureGates, initCfg.KubernetesVersion); err != nil {
-		return err
-	}
+	// Set the ImagePullPolicy and ImagePullSerial from the UpgradeApplyConfiguration to the InitConfiguration.
+	// These are used by preflight.RunPullImagesCheck() when running 'apply'.
+	initCfg.NodeRegistration.ImagePullPolicy = upgradeCfg.Apply.ImagePullPolicy
+	initCfg.NodeRegistration.ImagePullSerial = upgradeCfg.Apply.ImagePullSerial
 
-	// Enforce the version skew policies
-	klog.V(1).Infoln("[upgrade/version] enforcing version skew policies")
-	allowRCUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.AllowRCUpgrades, upgradeCfg.Apply.AllowRCUpgrades, &flags.allowRCUpgrades).(*bool)
-	if ok {
-		flags.allowRCUpgrades = *allowRCUpgrades
+	// The `upgrade apply` version always overwrites the KubernetesVersion in the returned cfg with the target
+	// version. While this is not the same for `upgrade plan` where the KubernetesVersion should be the old
+	// one (because the call to getComponentConfigVersionStates requires the currently installed version).
+	// This also makes the KubernetesVersion value returned for `upgrade plan` consistent as that command
+	// allows to not specify a target version in which case KubernetesVersion will always hold the currently
+	// installed one.
+	initCfg.KubernetesVersion = upgradeVersion
+
+	var patchesDir string
+	if upgradeCfg.Apply.Patches != nil {
+		patchesDir = cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.Patches, upgradeCfg.Apply.Patches.Directory, applyFlags.patchesDir).(string)
 	} else {
-		return cmdutil.TypeMismatchErr("allowRCUpgrades", "bool")
+		patchesDir = applyFlags.patchesDir
 	}
 
-	force, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, "force", upgradeCfg.Apply.ForceUpgrade, &flags.force).(*bool)
-	if ok {
-		flags.force = *force
-	} else {
-		return cmdutil.TypeMismatchErr("force", "bool")
+	if *printConfig {
+		printConfiguration(&initCfg.ClusterConfiguration, os.Stdout, printer)
 	}
 
-	allowExperimentalUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.AllowExperimentalUpgrades, upgradeCfg.Apply.AllowExperimentalUpgrades, &flags.allowExperimentalUpgrades).(*bool)
-	if ok {
-		flags.allowExperimentalUpgrades = *allowExperimentalUpgrades
-	} else {
-		return cmdutil.TypeMismatchErr("allowExperimentalUpgrades", "bool")
+	return &applyData{
+		nonInteractiveMode:        applyFlags.nonInteractiveMode,
+		force:                     *force,
+		dryRun:                    *dryRun,
+		etcdUpgrade:               *etcdUpgrade,
+		renewCerts:                *renewCerts,
+		allowExperimentalUpgrades: *allowExperimentalUpgrades,
+		allowRCUpgrades:           *allowRCUpgrades,
+		printConfig:               *printConfig,
+		cfg:                       upgradeCfg,
+		initCfg:                   initCfg,
+		client:                    client,
+		patchesDir:                patchesDir,
+		ignorePreflightErrors:     ignorePreflightErrorsSet,
+		outputWriter:              applyFlags.out,
+	}, nil
 }
 
-	if err := EnforceVersionPolicies(initCfg.KubernetesVersion, newK8sVersion, flags, versionGetter); err != nil {
-		return errors.Wrap(err, "[upgrade/version] FATAL")
+// DryRun returns the dryRun flag.
+func (d *applyData) DryRun() bool {
+	return d.dryRun
 }
 
-	// If the current session is interactive, ask the user whether they really want to upgrade.
-	dryRun, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.DryRun, upgradeCfg.Apply.DryRun, &flags.dryRun).(*bool)
-	if ok {
-		flags.dryRun = *dryRun
-	} else {
-		return cmdutil.TypeMismatchErr("dryRun", "bool")
+// EtcdUpgrade returns the etcdUpgrade flag.
+func (d *applyData) EtcdUpgrade() bool {
+	return d.etcdUpgrade
 }
 
-	if flags.sessionIsInteractive() {
-		if err := cmdutil.InteractivelyConfirmAction("upgrade", "Are you sure you want to proceed?", os.Stdin); err != nil {
-			return err
-		}
+// RenewCerts returns the renewCerts flag.
+func (d *applyData) RenewCerts() bool {
+	return d.renewCerts
 }
 
-	if !flags.dryRun {
-		fmt.Println("[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster")
-		fmt.Println("[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection")
-		fmt.Println("[upgrade/prepull] You can also perform this action beforehand using 'kubeadm config images pull'")
-		if err := preflight.RunPullImagesCheck(utilsexec.New(), initCfg, sets.New(upgradeCfg.Apply.IgnorePreflightErrors...)); err != nil {
-			return err
-		}
-	} else {
-		fmt.Println("[upgrade/prepull] Would pull the required images (like 'kubeadm config images pull')")
+// Cfg returns upgradeConfiguration.
+func (d *applyData) Cfg() *kubeadmapi.UpgradeConfiguration {
+	return d.cfg
 }
 
-	waiter := getWaiter(flags.dryRun, client, upgradeCfg.Timeouts.UpgradeManifests.Duration)
-
-	// If the config is set by flag, just overwrite it!
-	etcdUpgrade, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.EtcdUpgrade, upgradeCfg.Apply.EtcdUpgrade, &flags.etcdUpgrade).(*bool)
-	if ok {
-		upgradeCfg.Apply.EtcdUpgrade = etcdUpgrade
-	} else {
-		return cmdutil.TypeMismatchErr("etcdUpgrade", "bool")
+// InitCfg returns the InitConfiguration.
+func (d *applyData) InitCfg() *kubeadmapi.InitConfiguration {
+	return d.initCfg
 }
 
-	renewCerts, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.CertificateRenewal, upgradeCfg.Apply.CertificateRenewal, &flags.renewCerts).(*bool)
-	if ok {
-		upgradeCfg.Apply.CertificateRenewal = renewCerts
-	} else {
-		return cmdutil.TypeMismatchErr("renewCerts", "bool")
+// Client returns a Kubernetes client to be used by kubeadm.
+func (d *applyData) Client() clientset.Interface {
+	return d.client
 }
 
-	if len(flags.patchesDir) > 0 {
-		upgradeCfg.Apply.Patches = &kubeadmapi.Patches{Directory: flags.patchesDir}
-	} else if upgradeCfg.Apply.Patches == nil {
-		upgradeCfg.Apply.Patches = &kubeadmapi.Patches{}
+// PatchesDir returns the folder where patches for components are stored.
+func (d *applyData) PatchesDir() string {
+	return d.patchesDir
 }
 
-	// Now; perform the upgrade procedure
-	if err := PerformControlPlaneUpgrade(flags, client, waiter, initCfg, upgradeCfg); err != nil {
-		return errors.Wrap(err, "[upgrade/apply] FATAL")
+// IgnorePreflightErrors returns the list of preflight errors to ignore.
+func (d *applyData) IgnorePreflightErrors() sets.Set[string] {
+	return d.ignorePreflightErrors
 }
 
-	// Upgrade RBAC rules and addons.
-	klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons")
-	if err := upgrade.PerformPostUpgradeTasks(client, initCfg, upgradeCfg.Apply.Patches.Directory, flags.dryRun, flags.applyPlanFlags.out); err != nil {
-		return errors.Wrap(err, "[upgrade/postupgrade] FATAL post-upgrade error")
+// OutputWriter returns the output writer to be used by kubeadm.
+func (d *applyData) OutputWriter() io.Writer {
+	return d.outputWriter
 }
 
-	if flags.dryRun {
-		fmt.Println("[upgrade/successful] Finished dryrunning successfully!")
-		return nil
+// SessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run).
+func (d *applyData) SessionIsInteractive() bool {
+	return !(d.nonInteractiveMode || d.dryRun || d.force)
 }
 
-	fmt.Println("")
-	fmt.Printf("[upgrade/successful] SUCCESS! Your cluster was upgraded to %q. Enjoy!\n", initCfg.KubernetesVersion)
-	fmt.Println("")
-	fmt.Println("[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.")
-
-	return nil
+// AllowExperimentalUpgrades returns true if allow upgrading to an alpha/beta/release candidate version of Kubernetes.
+func (d *applyData) AllowExperimentalUpgrades() bool {
+	return d.allowExperimentalUpgrades
 }
 
-// EnforceVersionPolicies makes sure that the version the user specified is valid to upgrade to
-// There are both fatal and skippable (with --force) errors
-func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Version, flags *applyFlags, versionGetter upgrade.VersionGetter) error {
-	fmt.Printf("[upgrade/version] You have chosen to change the cluster version to %q\n", newK8sVersionStr)
-
-	versionSkewErrs := upgrade.EnforceVersionPolicies(versionGetter, newK8sVersionStr, newK8sVersion, flags.allowExperimentalUpgrades, flags.allowRCUpgrades)
-	if versionSkewErrs != nil {
-
-		if len(versionSkewErrs.Mandatory) > 0 {
-			return errors.Errorf("the --version argument is invalid due to these fatal errors:\n\n%v\nPlease fix the misalignments highlighted above and try upgrading again",
-				kubeadmutil.FormatErrMsg(versionSkewErrs.Mandatory))
+// AllowRCUpgrades returns true if allow upgrading to a release candidate version of Kubernetes.
+func (d *applyData) AllowRCUpgrades() bool {
+	return d.allowRCUpgrades
 }
 
-		if len(versionSkewErrs.Skippable) > 0 {
-			// Return the error if the user hasn't specified the --force flag
-			if !flags.force {
-				return errors.Errorf("the --version argument is invalid due to these errors:\n\n%v\nCan be bypassed if you pass the --force flag",
-					kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
-			}
-			// Soft errors found, but --force was specified
-			fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: \n\n%v", len(versionSkewErrs.Skippable), kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
-		}
-	}
-	return nil
-}
-
-// PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted)
-func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, initCfg *kubeadmapi.InitConfiguration, upgradeCfg *kubeadmapi.UpgradeConfiguration) error {
-	// OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster
-	fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n",
-		initCfg.KubernetesVersion, upgradeCfg.Timeouts.UpgradeManifests.Duration)
-
-	if flags.dryRun {
-		return upgrade.DryRunStaticPodUpgrade(upgradeCfg.Apply.Patches.Directory, initCfg)
-	}
-
-	return upgrade.PerformStaticPodUpgrade(client, waiter, initCfg, *upgradeCfg.Apply.EtcdUpgrade, *upgradeCfg.Apply.CertificateRenewal, upgradeCfg.Apply.Patches.Directory)
+// ForceUpgrade returns true if force upgrading although some requirements might not be met.
+func (d *applyData) ForceUpgrade() bool {
+	return d.force
 }

cmd/kubeadm/app/cmd/upgrade/apply_test.go

@@ -23,39 +23,39 @@ import (
 func TestSessionIsInteractive(t *testing.T) {
 	var tcases = []struct {
 		name     string
-		flags    *applyFlags
+		data     *applyData
 		expected bool
 	}{
 		{
 			name: "Explicitly non-interactive",
-			flags: &applyFlags{
+			data: &applyData{
 				nonInteractiveMode: true,
 			},
 			expected: false,
 		},
 		{
 			name: "Implicitly non-interactive since --dryRun is used",
-			flags: &applyFlags{
+			data: &applyData{
 				dryRun: true,
 			},
 			expected: false,
 		},
 		{
 			name: "Implicitly non-interactive since --force is used",
-			flags: &applyFlags{
+			data: &applyData{
 				force: true,
 			},
 			expected: false,
 		},
 		{
 			name:     "Interactive session",
-			flags:    &applyFlags{},
+			data:     &applyData{},
 			expected: true,
 		},
 	}
 	for _, tt := range tcases {
 		t.Run(tt.name, func(t *testing.T) {
-			if tt.flags.sessionIsInteractive() != tt.expected {
+			if tt.data.SessionIsInteractive() != tt.expected {
 				t.Error("unexpected result")
 			}
 		})

cmd/kubeadm/app/cmd/upgrade/common.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"io"
 	"os"
-	"time"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
@@ -45,7 +44,6 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
-	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/output"
 )
@@ -224,11 +222,3 @@ func getClient(file string, dryRun bool) (clientset.Interface, error) {
 	}
 	return kubeconfigutil.ClientSetFromFile(file)
 }
-
-// getWaiter gets the right waiter implementation
-func getWaiter(dryRun bool, client clientset.Interface, timeout time.Duration) apiclient.Waiter {
-	if dryRun {
-		return dryrunutil.NewWaiter()
-	}
-	return apiclient.NewKubeWaiter(client, timeout, os.Stdout)
-}

cmd/kubeadm/app/phases/upgrade/postupgrade.go

@@ -36,93 +36,16 @@ import (
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
-	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
-	"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
-	nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
 	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
-	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
-	"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 )
 
-// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
-// Note that the mark-control-plane phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
-func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, patchesDir string, dryRun bool, out io.Writer) error {
-	var errs []error
-
-	// Upload currently used configuration to the cluster
-	// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
-	// depend on centralized information from this source in the future
-	if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
-		errs = append(errs, err)
-	}
-
-	// Create the new, version-branched kubelet ComponentConfig ConfigMap
-	if err := kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, client); err != nil {
-		errs = append(errs, errors.Wrap(err, "error creating kubelet configuration ConfigMap"))
-	}
-
-	// Write the new kubelet config down to disk and the env file if needed
-	if err := WriteKubeletConfigFiles(cfg, patchesDir, dryRun, out); err != nil {
-		errs = append(errs, err)
-	}
-
-	// Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or
-	// --cri-socket.
-	// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
-	if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
-		errs = append(errs, errors.Wrap(err, "error uploading crisocket"))
-	}
-
-	// Create RBAC rules that makes the bootstrap tokens able to get nodes
-	if err := nodebootstraptoken.AllowBootstrapTokensToGetNodes(client); err != nil {
-		errs = append(errs, err)
-	}
-
-	// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
-	if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
-		errs = append(errs, err)
-	}
-
-	// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
-	if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
-		errs = append(errs, err)
-	}
-
-	// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
-	if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
-		errs = append(errs, err)
-	}
-
-	// TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade
-	// Create the cluster-info ConfigMap with the associated RBAC rules
-	// if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil {
-	// 	return err
-	//}
-	// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
-	if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil {
-		errs = append(errs, err)
-	}
-
-	if err := PerformAddonsUpgrade(client, cfg, patchesDir, out); err != nil {
-		errs = append(errs, err)
-	}
-
-	if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
-		if err := UpdateKubeletLocalMode(cfg, dryRun); err != nil {
-			return errors.Wrap(err, "failed to update kubelet local mode")
-		}
-	}
-
-	return errorsutil.NewAggregate(errs)
-}
-
 // PerformAddonsUpgrade performs the upgrade of the coredns and kube-proxy addons.
 func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, patchesDir string, out io.Writer) error {
-	unupgradedControlPlanes, err := unupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
+	unupgradedControlPlanes, err := UnupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
 	if err != nil {
 		return errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded")
 	}
@@ -186,12 +109,12 @@ func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfig
 	return errorsutil.NewAggregate(errs)
 }
 
-// unupgradedControlPlaneInstances returns a list of control plane instances that have not yet been upgraded.
+// UnupgradedControlPlaneInstances returns a list of control palne instances that have not yet been upgraded.
 //
 // NB. This function can only be called after the current control plane instance has been upgraded already.
 // Because it determines whether the other control plane instances have been upgraded by checking whether
 // the kube-apiserver image of other control plane instance is the same as that of this instance.
-func unupgradedControlPlaneInstances(client clientset.Interface, nodeName string) ([]string, error) {
+func UnupgradedControlPlaneInstances(client clientset.Interface, nodeName string) ([]string, error) {
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{
 		"component": kubeadmconstants.KubeAPIServer,
 	}))