github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-11 13:02:14 +00:00
Code Issues Projects Releases Wiki Activity

kubeadm: implement 'kubeadm upgrade apply phase'

Browse Source 
Signed-off-by: SataQiu <shidaqiu2018@gmail.com>
This commit is contained in:
SataQiu 2024-08-14 16:42:28 +08:00
parent 9f01cd7b28
commit 595482d264
13 changed files with 1131 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
cmd/kubeadm/app/cmd/phases/upgrade/apply/addons.go Normal file
View File

@ -0,0 +1,188 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package apply implements phases of 'kubeadm upgrade apply'.
package apply
import (
"context"
"fmt"
"io"
"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
dnsaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
proxyaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
)
// NewAddonPhase returns the addon Cobra command
func NewAddonPhase() workflow.Phase {
return workflow.Phase{
Name: "addon",
Short: "Install required addons for passing conformance tests",
Long: cmdutil.MacroCommandLongDescription,
Phases: []workflow.Phase{
{
Name: "all",
Short: "Install all the addons",
InheritFlags: getAddonPhaseFlags("all"),
RunAllSiblings: true,
},
{
Name: "coredns",
Short: "Install the CoreDNS addon to a Kubernetes cluster",
InheritFlags: getAddonPhaseFlags("coredns"),
Run: runCoreDNSAddon,
},
{
Name: "kube-proxy",
Short: "Install the kube-proxy addon to a Kubernetes cluster",
InheritFlags: getAddonPhaseFlags("kube-proxy"),
Run: runKubeProxyAddon,
},
},
}
}
func shouldUpgradeAddons(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, out io.Writer) (bool, error) {
unupgradedControlPlanes, err := upgrade.UnupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
if err != nil {
return false, errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded")
}
if len(unupgradedControlPlanes) > 0 {
fmt.Fprintf(out, "[upgrade/addons] skip upgrade addons because control plane instances %v have not been upgraded\n", unupgradedControlPlanes)
return false, nil
}
return true, nil
}
func getInitData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, string, io.Writer, bool, error) {
data, ok := c.(Data)
if !ok {
return nil, nil, "", nil, false, errors.New("addon phase invoked with an invalid data struct")
}
return data.InitCfg(), data.Client(), data.PatchesDir(), data.OutputWriter(), data.DryRun(), nil
}
// runCoreDNSAddon installs CoreDNS addon to a Kubernetes cluster
func runCoreDNSAddon(c workflow.RunData) error {
cfg, client, patchesDir, out, dryRun, err := getInitData(c)
if err != nil {
return err
}
shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out)
if err != nil {
return err
}
if !shouldUpgradeAddons {
return nil
}
// If the coredns ConfigMap is missing, show a warning and assume that the
// DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
//
// TODO: remove this once "kubeadm upgrade apply" phases are supported:
// https://github.com/kubernetes/kubeadm/issues/1318
if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
context.TODO(),
kubeadmconstants.CoreDNSConfigMap,
metav1.GetOptions{},
); err != nil && apierrors.IsNotFound(err) {
klog.Warningf("the ConfigMaps %q in the namespace %q were not found. "+
"Assuming that a DNS server was not deployed for this cluster. "+
"Note that once 'kubeadm upgrade apply' supports phases you "+
"will have to skip the DNS upgrade manually",
kubeadmconstants.CoreDNSConfigMap,
metav1.NamespaceSystem)
return nil
}
// Upgrade CoreDNS
if err := dnsaddon.EnsureDNSAddon(&cfg.ClusterConfiguration, client, patchesDir, out, dryRun); err != nil {
return err
}
return nil
}
// runKubeProxyAddon installs KubeProxy addon to a Kubernetes cluster
func runKubeProxyAddon(c workflow.RunData) error {
cfg, client, _, out, dryRun, err := getInitData(c)
if err != nil {
return err
}
shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out)
if err != nil {
return err
}
if !shouldUpgradeAddons {
return nil
}
// If the kube-proxy ConfigMap is missing, show a warning and assume that kube-proxy
// was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
//
// TODO: remove this once "kubeadm upgrade apply" phases are supported:
// https://github.com/kubernetes/kubeadm/issues/1318
if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
context.TODO(),
kubeadmconstants.KubeProxyConfigMap,
metav1.GetOptions{},
); err != nil && apierrors.IsNotFound(err) {
klog.Warningf("the ConfigMap %q in the namespace %q was not found. "+
"Assuming that kube-proxy was not deployed for this cluster. "+
"Note that once 'kubeadm upgrade apply' supports phases you "+
"will have to skip the kube-proxy upgrade manually",
kubeadmconstants.KubeProxyConfigMap,
metav1.NamespaceSystem)
return nil
}
// Upgrade kube-proxy
if err := proxyaddon.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, dryRun); err != nil {
return err
}
return nil
}
func getAddonPhaseFlags(name string) []string {
flags := []string{
options.CfgPath,
options.KubeconfigPath,
options.DryRun,
}
if name == "all" || name == "coredns" {
flags = append(flags,
options.Patches,
)
}
return flags
}

90
cmd/kubeadm/app/cmd/phases/upgrade/apply/bootstraptoken.go Normal file
View File

@ -0,0 +1,90 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package apply implements phases of 'kubeadm upgrade apply'.
package apply
import (
"fmt"
"github.com/pkg/errors"
errorsutil "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
)
// NewBootstrapTokenPhase returns the phase to bootstrapToken
func NewBootstrapTokenPhase() workflow.Phase {
return workflow.Phase{
Name: "bootstrap-token",
Aliases: []string{"bootstraptoken"},
Short: "Generates bootstrap tokens used to join a node to a cluster",
InheritFlags: []string{
options.CfgPath,
options.KubeconfigPath,
options.DryRun,
},
Run: runBootstrapToken,
}
}
func runBootstrapToken(c workflow.RunData) error {
data, ok := c.(Data)
if !ok {
return errors.New("bootstrap-token phase invoked with an invalid data struct")
}
if data.DryRun() {
fmt.Println("[dryrun] Would config cluster-info ConfigMap, RBAC Roles")
return nil
}
fmt.Println("[bootstrap-token] Configuring cluster-info ConfigMap, RBAC Roles")
client := data.Client()
var errs []error
// Create RBAC rules that makes the bootstrap tokens able to get nodes
if err := nodebootstraptoken.AllowBootstrapTokensToGetNodes(client); err != nil {
errs = append(errs, err)
}
// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
errs = append(errs, err)
}
// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
errs = append(errs, err)
}
// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
errs = append(errs, err)
}
// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil {
errs = append(errs, err)
}
return errorsutil.NewAggregate(errs)
}

74
cmd/kubeadm/app/cmd/phases/upgrade/apply/controlplane.go Normal file
View File

@ -0,0 +1,74 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package apply implements phases of 'kubeadm upgrade apply'.
package apply
import (
"fmt"
"os"
"github.com/pkg/errors"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
// NewControlPlanePhase creates a kubeadm workflow phase that implements handling of control-plane upgrade.
func NewControlPlanePhase() workflow.Phase {
phase := workflow.Phase{
Name: "control-plane",
Short: "Upgrade the control plane",
Run: runControlPlane,
InheritFlags: []string{
options.CfgPath,
options.KubeconfigPath,
options.DryRun,
options.CertificateRenewal,
options.EtcdUpgrade,
options.Patches,
},
}
return phase
}
func runControlPlane(c workflow.RunData) error {
data, ok := c.(Data)
if !ok {
return errors.New("control-plane phase invoked with an invalid data struct")
}
initCfg, upgradeCfg, client, patchesDir := data.InitCfg(), data.Cfg(), data.Client(), data.PatchesDir()
if data.DryRun() {
fmt.Printf("[dryrun] Would upgrade your Static Pod-hosted control plane to version %q", initCfg.KubernetesVersion)
return upgrade.DryRunStaticPodUpgrade(patchesDir, initCfg)
}
fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n",
initCfg.KubernetesVersion, upgradeCfg.Timeouts.UpgradeManifests.Duration)
waiter := apiclient.NewKubeWaiter(client, upgradeCfg.Timeouts.UpgradeManifests.Duration, os.Stdout)
if err := upgrade.PerformStaticPodUpgrade(client, waiter, initCfg, data.EtcdUpgrade(), data.RenewCerts(), patchesDir); err != nil {
return errors.Wrap(err, "couldn't complete the static pod upgrade")
}
fmt.Println("[upgrade] The control plane instance for this node was successfully updated!")
return nil
}

45
cmd/kubeadm/app/cmd/phases/upgrade/apply/data.go Normal file
View File

@ -0,0 +1,45 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package apply implements phases of 'kubeadm upgrade apply'.
package apply
import (
"io"
"k8s.io/apimachinery/pkg/util/sets"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
)
// Data is the interface to use for kubeadm upgrade apply phases.
// The "applyData" type from "cmd/upgrade/apply.go" must satisfy this interface.
type Data interface {
EtcdUpgrade() bool
RenewCerts() bool
DryRun() bool
Cfg() *kubeadmapi.UpgradeConfiguration
InitCfg() *kubeadmapi.InitConfiguration
Client() clientset.Interface
IgnorePreflightErrors() sets.Set[string]
PatchesDir() string
OutputWriter() io.Writer
SessionIsInteractive() bool
AllowExperimentalUpgrades() bool
AllowRCUpgrades() bool
ForceUpgrade() bool
}

46
cmd/kubeadm/app/cmd/phases/upgrade/apply/data_test.go Normal file
View File

@ -0,0 +1,46 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apply
import (
"io"
"k8s.io/apimachinery/pkg/util/sets"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
)
// a package local type for testing purposes.
type testData struct{}
// testData must satisfy Data.
var _ Data = &testData{}
func (t *testData) EtcdUpgrade() bool { return false }
func (t *testData) RenewCerts() bool { return false }
func (t *testData) DryRun() bool { return false }
func (t *testData) Cfg() *kubeadmapi.UpgradeConfiguration { return nil }
func (t *testData) InitCfg() *kubeadmapi.InitConfiguration { return nil }
func (t *testData) Client() clientset.Interface { return nil }
func (t *testData) IgnorePreflightErrors() sets.Set[string] { return nil }
func (t *testData) PatchesDir() string { return "" }
func (t *testData) OutputWriter() io.Writer { return nil }
func (t *testData) SessionIsInteractive() bool { return false }
func (t *testData) AllowExperimentalUpgrades() bool { return false }
func (t *testData) AllowRCUpgrades() bool { return false }
func (t *testData) ForceUpgrade() bool { return false }

66
cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeconfig.go Normal file
View File

@ -0,0 +1,66 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package apply implements phases of 'kubeadm upgrade apply'.
package apply
import (
"fmt"
"github.com/pkg/errors"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
)
// NewKubeconfigPhase creates a kubeadm workflow phase that implements handling of kubeconfig upgrade.
func NewKubeconfigPhase() workflow.Phase {
phase := workflow.Phase{
Name: "kubeconfig",
Short: "Upgrade kubeconfig files for this node",
Run: runKubeconfig(),
Hidden: true,
InheritFlags: []string{
options.CfgPath,
options.DryRun,
options.KubeconfigPath,
},
}
return phase
}
func runKubeconfig() func(c workflow.RunData) error {
return func(c workflow.RunData) error {
data, ok := c.(Data)
if !ok {
return errors.New("kubeconfig phase invoked with an invalid data struct")
}
cfg := data.InitCfg()
if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
if err := upgrade.UpdateKubeletLocalMode(cfg, data.DryRun()); err != nil {
return errors.Wrap(err, "failed to update kubelet local mode")
}
}
fmt.Println("[upgrade] The kubeconfig for this node was successfully updated!")
return nil
}
}

73
cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeletconfig.go Normal file
View File

@ -0,0 +1,73 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package apply implements phases of 'kubeadm upgrade apply'.
package apply
import (
"fmt"
"github.com/pkg/errors"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
)
var (
kubeletConfigLongDesc = cmdutil.LongDesc(`
Download the kubelet configuration from the kubelet-config ConfigMap stored in the cluster
`)
)
// NewKubeletConfigPhase creates a kubeadm workflow phase that implements handling of kubelet-config upgrade.
func NewKubeletConfigPhase() workflow.Phase {
phase := workflow.Phase{
Name: "kubelet-config",
Short: "Upgrade the kubelet configuration for this node",
Long: kubeletConfigLongDesc,
Run: runKubeletConfigPhase,
InheritFlags: []string{
options.CfgPath,
options.DryRun,
options.KubeconfigPath,
options.Patches,
},
}
return phase
}
func runKubeletConfigPhase(c workflow.RunData) error {
data, ok := c.(Data)
if !ok {
return errors.New("kubelet-config phase invoked with an invalid data struct")
}
initCfg, dryRun := data.InitCfg(), data.DryRun()
// Write the configuration for the kubelet down to disk and print the generated manifests instead if dry-running.
// If not dry-running, the kubelet config file will be backed up to /etc/kubernetes/tmp/ dir, so that it could be
// recovered if there is anything goes wrong.
err := upgrade.WriteKubeletConfigFiles(initCfg, data.PatchesDir(), dryRun, data.OutputWriter())
if err != nil {
return err
}
fmt.Println("[upgrade] The configuration for this node was successfully updated!")
fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.")
return nil
}

161
cmd/kubeadm/app/cmd/phases/upgrade/apply/preflight.go Normal file
View File

@ -0,0 +1,161 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package apply implements phases of 'kubeadm upgrade apply'.
package apply
import (
"fmt"
"os"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/klog/v2"
utilsexec "k8s.io/utils/exec"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
"k8s.io/kubernetes/cmd/kubeadm/app/util/output"
)
// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for kubeadm upgrade apply.
func NewPreflightPhase() workflow.Phase {
return workflow.Phase{
Name: "preflight",
Short: "Run upgrade apply pre-flight checks",
Long: "Run pre-flight checks for kubeadm upgrade apply.",
Run: runPreflight,
InheritFlags: []string{
options.CfgPath,
options.KubeconfigPath,
options.DryRun,
options.IgnorePreflightErrors,
"allow-experimental-upgrades",
"allow-release-candidate-upgrades",
"force",
"yes",
},
}
}
// runPreflight executes preflight checks logic.
func runPreflight(c workflow.RunData) error {
data, ok := c.(Data)
if !ok {
return errors.New("preflight phase invoked with an invalid data struct")
}
fmt.Println("[preflight] Running pre-flight checks")
printer := &output.TextPrinter{}
initCfg, client, ignorePreflightErrors := data.InitCfg(), data.Client(), data.IgnorePreflightErrors()
// First, check if we're root separately from the other preflight checks and fail fast
if err := preflight.RunRootCheckOnly(ignorePreflightErrors); err != nil {
return err
}
// Run CoreDNS migration check
if err := upgrade.RunCoreDNSMigrationCheck(client, ignorePreflightErrors); err != nil {
return err
}
// Run healthchecks against the cluster
klog.V(1).Infoln("[upgrade/apply] verifying health of cluster")
if err := upgrade.CheckClusterHealth(client, &initCfg.ClusterConfiguration, ignorePreflightErrors, printer); err != nil {
return err
}
// Check if feature gate flags used in the cluster are consistent with the set of features currently supported by kubeadm
if msg := features.CheckDeprecatedFlags(&features.InitFeatureGates, initCfg.FeatureGates); len(msg) > 0 {
for _, m := range msg {
_, _ = printer.Printf("[upgrade/config] %s\n", m)
}
}
// Validate requested and validate actual version
klog.V(1).Infoln("[upgrade/apply] validating requested and actual version")
if err := configutil.NormalizeKubernetesVersion(&initCfg.ClusterConfiguration); err != nil {
return err
}
// Use normalized version string in all following code.
upgradeVersion, err := version.ParseSemantic(initCfg.KubernetesVersion)
if err != nil {
return errors.Errorf("unable to parse normalized version %q as a semantic version", initCfg.KubernetesVersion)
}
if err := features.ValidateVersion(features.InitFeatureGates, initCfg.FeatureGates, initCfg.KubernetesVersion); err != nil {
return err
}
versionGetter := upgrade.NewOfflineVersionGetter(upgrade.NewKubeVersionGetter(client), initCfg.KubernetesVersion)
if err := EnforceVersionPolicies(initCfg.KubernetesVersion, upgradeVersion, data.AllowExperimentalUpgrades(), data.AllowRCUpgrades(), data.ForceUpgrade(), versionGetter); err != nil {
return err
}
if data.SessionIsInteractive() {
if err := cmdutil.InteractivelyConfirmAction("upgrade", "Are you sure you want to proceed?", os.Stdin); err != nil {
return err
}
}
if !data.DryRun() {
fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
fmt.Println("[preflight] You can also perform this action beforehand using 'kubeadm config images pull'")
if err := preflight.RunPullImagesCheck(utilsexec.New(), initCfg, ignorePreflightErrors); err != nil {
return err
}
} else {
fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')")
}
return nil
}
// EnforceVersionPolicies makes sure that the version the user specified is valid to upgrade to
// There are both fatal and skippable (with --force) errors
func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Version, allowExperimentalUpgrades, allowRCUpgrades, force bool, versionGetter upgrade.VersionGetter) error {
fmt.Printf("[upgrade/version] You have chosen to change the cluster version to %q\n", newK8sVersionStr)
versionSkewErrs := upgrade.EnforceVersionPolicies(versionGetter, newK8sVersionStr, newK8sVersion, allowExperimentalUpgrades, allowRCUpgrades)
if versionSkewErrs != nil {
if len(versionSkewErrs.Mandatory) > 0 {
return errors.Errorf("the --version argument is invalid due to these fatal errors:\n\n%v\nPlease fix the misalignments highlighted above and try upgrading again",
kubeadmutil.FormatErrMsg(versionSkewErrs.Mandatory))
}
if len(versionSkewErrs.Skippable) > 0 {
// Return the error if the user hasn't specified the --force flag
if !force {
return errors.Errorf("the --version argument is invalid due to these errors:\n\n%v\nCan be bypassed if you pass the --force flag",
kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
}
// Soft errors found, but --force was specified
fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: \n\n%v", len(versionSkewErrs.Skippable), kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
}
}
return nil
}

127
cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go Normal file
View File

@ -0,0 +1,127 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package apply implements phases of 'kubeadm upgrade apply'.
package apply
import (
"fmt"
"github.com/pkg/errors"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
)
// NewUploadConfigPhase returns the phase to uploadConfig
func NewUploadConfigPhase() workflow.Phase {
return workflow.Phase{
Name: "upload-config",
Aliases: []string{"uploadconfig"},
Short: "Upload the kubeadm and kubelet configuration to a ConfigMap",
Long: cmdutil.MacroCommandLongDescription,
Phases: []workflow.Phase{
{
Name: "all",
Short: "Upload all configuration to a config map",
RunAllSiblings: true,
InheritFlags: getUploadConfigPhaseFlags(),
},
{
Name: "kubeadm",
Short: "Upload the kubeadm ClusterConfiguration to a ConfigMap",
Run: runUploadKubeadmConfig,
InheritFlags: getUploadConfigPhaseFlags(),
},
{
Name: "kubelet",
Short: "Upload the kubelet component config to a ConfigMap",
Run: runUploadKubeletConfig,
InheritFlags: getUploadConfigPhaseFlags(),
},
},
}
}
func getUploadConfigPhaseFlags() []string {
return []string{
options.CfgPath,
options.KubeconfigPath,
options.DryRun,
}
}
// runUploadKubeadmConfig uploads the kubeadm configuration to a ConfigMap
func runUploadKubeadmConfig(c workflow.RunData) error {
cfg, client, dryRun, err := getUploadConfigData(c)
if err != nil {
return err
}
if dryRun {
fmt.Println("[dryrun] Would upload the kubeadm ClusterConfiguration to a ConfigMap")
return nil
}
klog.V(1).Infoln("[upload-config] Uploading the kubeadm ClusterConfiguration to a ConfigMap")
if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
return errors.Wrap(err, "error uploading the kubeadm ClusterConfiguration")
}
return nil
}
// runUploadKubeletConfig uploads the kubelet configuration to a ConfigMap
func runUploadKubeletConfig(c workflow.RunData) error {
cfg, client, dryRun, err := getUploadConfigData(c)
if err != nil {
return err
}
if dryRun {
fmt.Println("[dryrun] Would upload the kubelet component config to a ConfigMap")
fmt.Println("[dryrun] Would write the CRISocket annotation for the control-plane node")
return nil
}
klog.V(1).Infoln("[upload-config] Uploading the kubelet component config to a ConfigMap")
if err = kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, client); err != nil {
return errors.Wrap(err, "error creating kubelet configuration ConfigMap")
}
klog.V(1).Infoln("[upload-config] Preserving the CRISocket information for the control-plane node")
if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
return errors.Wrap(err, "Error writing Crisocket information for the control-plane node")
}
return nil
}
func getUploadConfigData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, bool, error) {
data, ok := c.(Data)
if !ok {
return nil, nil, false, errors.New("upload-config phase invoked with an invalid data struct")
}
return data.InitCfg(), data.Client(), data.DryRun(), nil
}

420
cmd/kubeadm/app/cmd/upgrade/apply.go
View File

@ -18,27 +18,26 @@ package upgrade
import (
"fmt"
"io"
"os"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
utilsexec "k8s.io/utils/exec"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/apply"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
"k8s.io/kubernetes/cmd/kubeadm/app/util/output"
)
@ -55,9 +54,26 @@ type applyFlags struct {
patchesDir string
}
// sessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run)
func (f *applyFlags) sessionIsInteractive() bool {
return !(f.nonInteractiveMode || f.dryRun || f.force)
// compile-time assert that the local data object satisfies the phases data interface.
var _ phases.Data = &applyData{}
// applyData defines all the runtime information used when running the kubeadm upgrade apply workflow;
// this data is shared across all the phases that are included in the workflow.
type applyData struct {
nonInteractiveMode bool
force bool
dryRun bool
etcdUpgrade bool
renewCerts bool
allowExperimentalUpgrades bool
allowRCUpgrades bool
printConfig bool
cfg *kubeadmapi.UpgradeConfiguration
initCfg *kubeadmapi.InitConfiguration
client clientset.Interface
patchesDir string
ignorePreflightErrors sets.Set[string]
outputWriter io.Writer
}
// newCmdApply returns the cobra command for `kubeadm upgrade apply`
@ -68,6 +84,8 @@ func newCmdApply(apf *applyPlanFlags) *cobra.Command {
renewCerts: true,
}
applyRunner := workflow.NewRunner()
cmd := &cobra.Command{
Use: "apply [version]",
DisableFlagsInUseLine: true,
@ -76,7 +94,32 @@ func newCmdApply(apf *applyPlanFlags) *cobra.Command {
if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil {
return err
}
return runApply(cmd.Flags(), flags, args)
data, err := applyRunner.InitData(args)
if err != nil {
return err
}
applyData, ok := data.(*applyData)
if !ok {
return errors.New("invalid data struct")
}
if err := applyRunner.Run(args); err != nil {
return err
}
if flags.dryRun {
fmt.Println("[upgrade/successful] Finished dryrunning successfully!")
return nil
}
fmt.Println("")
fmt.Printf("[upgrade/successful] SUCCESS! Your cluster was upgraded to %q. Enjoy!\n", applyData.InitCfg().KubernetesVersion)
fmt.Println("")
fmt.Println("[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.")
return nil
},
}
@ -90,181 +133,222 @@ func newCmdApply(apf *applyPlanFlags) *cobra.Command {
cmd.Flags().BoolVar(&flags.renewCerts, options.CertificateRenewal, flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
options.AddPatchesFlag(cmd.Flags(), &flags.patchesDir)
// initialize the workflow runner with the list of phases
applyRunner.AppendPhase(phases.NewPreflightPhase())
applyRunner.AppendPhase(phases.NewControlPlanePhase())
applyRunner.AppendPhase(phases.NewUploadConfigPhase())
applyRunner.AppendPhase(phases.NewKubeconfigPhase())
applyRunner.AppendPhase(phases.NewKubeletConfigPhase())
applyRunner.AppendPhase(phases.NewBootstrapTokenPhase())
applyRunner.AppendPhase(phases.NewAddonPhase())
// sets the data builder function, that will be used by the runner
// both when running the entire workflow or single phases
applyRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) {
data, err := newApplyData(cmd, args, flags)
if err != nil {
return nil, err
}
// If the flag for skipping phases was empty, use the values from config
if len(applyRunner.Options.SkipPhases) == 0 {
applyRunner.Options.SkipPhases = data.cfg.Apply.SkipPhases
}
return data, nil
})
// binds the Runner to kubeadm upgrade apply command by altering
// command help, adding --skip-phases flag and by adding phases subcommands
applyRunner.BindToCommand(cmd)
return cmd
}
// runApply takes care of the actual upgrade functionality
// It does the following things:
// - Checks if the cluster is healthy
// - Gets the configuration from the kubeadm-config ConfigMap in the cluster
// - Enforces all version skew policies
// - Asks the user if they really want to upgrade
// - Makes sure the control plane images are available locally on the control-plane(s)
// - Upgrades the control plane components
// - Applies the other resources that'd be created with kubeadm init as well, like
// - Uploads the newly used configuration to the cluster ConfigMap
// - Creating the RBAC rules for the bootstrap tokens and the cluster-info ConfigMap
// - Applying new CoreDNS and kube-proxy manifests
func runApply(flagSet *pflag.FlagSet, flags *applyFlags, args []string) error {
// newApplyData returns a new applyData struct to be used for the execution of the kubeadm upgrade apply workflow.
func newApplyData(cmd *cobra.Command, args []string, applyFlags *applyFlags) (*applyData, error) {
externalCfg := &v1beta4.UpgradeConfiguration{}
opt := configutil.LoadOrDefaultConfigurationOptions{}
upgradeCfg, err := configutil.LoadOrDefaultUpgradeConfiguration(applyFlags.cfgPath, externalCfg, opt)
if err != nil {
return nil, err
}
// Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap)
klog.V(1).Infoln("[upgrade/apply] verifying health of cluster")
upgradeVersion := upgradeCfg.Apply.KubernetesVersion
// The version arg is mandatory, during upgrade apply, unless it's specified in the config file
if upgradeVersion == "" {
if err := cmdutil.ValidateExactArgNumber(args, []string{"version"}); err != nil {
return nil, err
}
}
// If option was specified in both args and config file, args will overwrite the config file.
if len(args) == 1 {
upgradeVersion = args[0]
}
force, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "force", upgradeCfg.Apply.ForceUpgrade, &applyFlags.force).(*bool)
if !ok {
return nil, cmdutil.TypeMismatchErr("forceUpgrade", "bool")
}
dryRun, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.DryRun, upgradeCfg.Apply.DryRun, &applyFlags.dryRun).(*bool)
if !ok {
return nil, cmdutil.TypeMismatchErr("dryRun", "bool")
}
etcdUpgrade, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.EtcdUpgrade, upgradeCfg.Apply.EtcdUpgrade, &applyFlags.etcdUpgrade).(*bool)
if !ok {
return nil, cmdutil.TypeMismatchErr("etcdUpgrade", "bool")
}
renewCerts, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.CertificateRenewal, upgradeCfg.Apply.CertificateRenewal, &applyFlags.renewCerts).(*bool)
if !ok {
return nil, cmdutil.TypeMismatchErr("certificateRenewal", "bool")
}
allowExperimentalUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "allow-experimental-upgrades", upgradeCfg.Apply.AllowExperimentalUpgrades, &applyFlags.allowExperimentalUpgrades).(*bool)
if !ok {
return nil, cmdutil.TypeMismatchErr("allowExperimentalUpgrades", "bool")
}
allowRCUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "allow-release-candidate-upgrades", upgradeCfg.Apply.AllowRCUpgrades, &applyFlags.allowRCUpgrades).(*bool)
if !ok {
return nil, cmdutil.TypeMismatchErr("allowRCUpgrades", "bool")
}
printConfig, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "print-config", upgradeCfg.Apply.PrintConfig, &applyFlags.printConfig).(*bool)
if !ok {
return nil, cmdutil.TypeMismatchErr("printConfig", "bool")
}
client, err := getClient(applyFlags.kubeConfigPath, *dryRun)
if err != nil {
return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", applyFlags.kubeConfigPath)
}
printer := &output.TextPrinter{}
// Fetches the cluster configuration
klog.V(1).Infoln("[upgrade/apply] retrieving configuration from cluster")
client, versionGetter, initCfg, upgradeCfg, err := enforceRequirements(flagSet, flags.applyPlanFlags, args, flags.dryRun, true, &output.TextPrinter{})
initCfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", false, false)
if err != nil {
return err
if apierrors.IsNotFound(err) {
_, _ = printer.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %q namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
_, _ = printer.Printf("[upgrade/config] Use 'kubeadm init phase upload-config --config your-config.yaml' to re-upload it.\n")
err = errors.Errorf("the ConfigMap %q in the %q namespace was not found", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem)
}
return nil, errors.Wrap(err, "[upgrade/init config] FATAL")
}
// Validate requested and validate actual version
klog.V(1).Infoln("[upgrade/apply] validating requested and actual version")
if err := configutil.NormalizeKubernetesVersion(&initCfg.ClusterConfiguration); err != nil {
return err
}
// Use normalized version string in all following code.
newK8sVersion, err := version.ParseSemantic(initCfg.KubernetesVersion)
ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(applyFlags.ignorePreflightErrors, upgradeCfg.Apply.IgnorePreflightErrors)
if err != nil {
return errors.Errorf("unable to parse normalized version %q as a semantic version", initCfg.KubernetesVersion)
return nil, err
}
// Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration:
initCfg.NodeRegistration.IgnorePreflightErrors = sets.List(ignorePreflightErrorsSet)
if err := features.ValidateVersion(features.InitFeatureGates, initCfg.FeatureGates, initCfg.KubernetesVersion); err != nil {
return err
}
// Set the ImagePullPolicy and ImagePullSerial from the UpgradeApplyConfiguration to the InitConfiguration.
// These are used by preflight.RunPullImagesCheck() when running 'apply'.
initCfg.NodeRegistration.ImagePullPolicy = upgradeCfg.Apply.ImagePullPolicy
initCfg.NodeRegistration.ImagePullSerial = upgradeCfg.Apply.ImagePullSerial
// Enforce the version skew policies
klog.V(1).Infoln("[upgrade/version] enforcing version skew policies")
allowRCUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.AllowRCUpgrades, upgradeCfg.Apply.AllowRCUpgrades, &flags.allowRCUpgrades).(*bool)
if ok {
flags.allowRCUpgrades = *allowRCUpgrades
// The `upgrade apply` version always overwrites the KubernetesVersion in the returned cfg with the target
// version. While this is not the same for `upgrade plan` where the KubernetesVersion should be the old
// one (because the call to getComponentConfigVersionStates requires the currently installed version).
// This also makes the KubernetesVersion value returned for `upgrade plan` consistent as that command
// allows to not specify a target version in which case KubernetesVersion will always hold the currently
// installed one.
initCfg.KubernetesVersion = upgradeVersion
var patchesDir string
if upgradeCfg.Apply.Patches != nil {
patchesDir = cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.Patches, upgradeCfg.Apply.Patches.Directory, applyFlags.patchesDir).(string)
} else {
return cmdutil.TypeMismatchErr("allowRCUpgrades", "bool")
patchesDir = applyFlags.patchesDir
}
force, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, "force", upgradeCfg.Apply.ForceUpgrade, &flags.force).(*bool)
if ok {
flags.force = *force
} else {
return cmdutil.TypeMismatchErr("force", "bool")
if *printConfig {
printConfiguration(&initCfg.ClusterConfiguration, os.Stdout, printer)
}
allowExperimentalUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.AllowExperimentalUpgrades, upgradeCfg.Apply.AllowExperimentalUpgrades, &flags.allowExperimentalUpgrades).(*bool)
if ok {
flags.allowExperimentalUpgrades = *allowExperimentalUpgrades
} else {
return cmdutil.TypeMismatchErr("allowExperimentalUpgrades", "bool")
}
if err := EnforceVersionPolicies(initCfg.KubernetesVersion, newK8sVersion, flags, versionGetter); err != nil {
return errors.Wrap(err, "[upgrade/version] FATAL")
}
// If the current session is interactive, ask the user whether they really want to upgrade.
dryRun, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.DryRun, upgradeCfg.Apply.DryRun, &flags.dryRun).(*bool)
if ok {
flags.dryRun = *dryRun
} else {
return cmdutil.TypeMismatchErr("dryRun", "bool")
}
if flags.sessionIsInteractive() {
if err := cmdutil.InteractivelyConfirmAction("upgrade", "Are you sure you want to proceed?", os.Stdin); err != nil {
return err
}
}
if !flags.dryRun {
fmt.Println("[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster")
fmt.Println("[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection")
fmt.Println("[upgrade/prepull] You can also perform this action beforehand using 'kubeadm config images pull'")
if err := preflight.RunPullImagesCheck(utilsexec.New(), initCfg, sets.New(upgradeCfg.Apply.IgnorePreflightErrors...)); err != nil {
return err
}
} else {
fmt.Println("[upgrade/prepull] Would pull the required images (like 'kubeadm config images pull')")
}
waiter := getWaiter(flags.dryRun, client, upgradeCfg.Timeouts.UpgradeManifests.Duration)
// If the config is set by flag, just overwrite it!
etcdUpgrade, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.EtcdUpgrade, upgradeCfg.Apply.EtcdUpgrade, &flags.etcdUpgrade).(*bool)
if ok {
upgradeCfg.Apply.EtcdUpgrade = etcdUpgrade
} else {
return cmdutil.TypeMismatchErr("etcdUpgrade", "bool")
}
renewCerts, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.CertificateRenewal, upgradeCfg.Apply.CertificateRenewal, &flags.renewCerts).(*bool)
if ok {
upgradeCfg.Apply.CertificateRenewal = renewCerts
} else {
return cmdutil.TypeMismatchErr("renewCerts", "bool")
}
if len(flags.patchesDir) > 0 {
upgradeCfg.Apply.Patches = &kubeadmapi.Patches{Directory: flags.patchesDir}
} else if upgradeCfg.Apply.Patches == nil {
upgradeCfg.Apply.Patches = &kubeadmapi.Patches{}
}
// Now; perform the upgrade procedure
if err := PerformControlPlaneUpgrade(flags, client, waiter, initCfg, upgradeCfg); err != nil {
return errors.Wrap(err, "[upgrade/apply] FATAL")
}
// Upgrade RBAC rules and addons.
klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons")
if err := upgrade.PerformPostUpgradeTasks(client, initCfg, upgradeCfg.Apply.Patches.Directory, flags.dryRun, flags.applyPlanFlags.out); err != nil {
return errors.Wrap(err, "[upgrade/postupgrade] FATAL post-upgrade error")
}
if flags.dryRun {
fmt.Println("[upgrade/successful] Finished dryrunning successfully!")
return nil
}
fmt.Println("")
fmt.Printf("[upgrade/successful] SUCCESS! Your cluster was upgraded to %q. Enjoy!\n", initCfg.KubernetesVersion)
fmt.Println("")
fmt.Println("[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.")
return nil
return &applyData{
nonInteractiveMode: applyFlags.nonInteractiveMode,
force: *force,
dryRun: *dryRun,
etcdUpgrade: *etcdUpgrade,
renewCerts: *renewCerts,
allowExperimentalUpgrades: *allowExperimentalUpgrades,
allowRCUpgrades: *allowRCUpgrades,
printConfig: *printConfig,
cfg: upgradeCfg,
initCfg: initCfg,
client: client,
patchesDir: patchesDir,
ignorePreflightErrors: ignorePreflightErrorsSet,
outputWriter: applyFlags.out,
}, nil
}
// EnforceVersionPolicies makes sure that the version the user specified is valid to upgrade to
// There are both fatal and skippable (with --force) errors
func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Version, flags *applyFlags, versionGetter upgrade.VersionGetter) error {
fmt.Printf("[upgrade/version] You have chosen to change the cluster version to %q\n", newK8sVersionStr)
versionSkewErrs := upgrade.EnforceVersionPolicies(versionGetter, newK8sVersionStr, newK8sVersion, flags.allowExperimentalUpgrades, flags.allowRCUpgrades)
if versionSkewErrs != nil {
if len(versionSkewErrs.Mandatory) > 0 {
return errors.Errorf("the --version argument is invalid due to these fatal errors:\n\n%v\nPlease fix the misalignments highlighted above and try upgrading again",
kubeadmutil.FormatErrMsg(versionSkewErrs.Mandatory))
}
if len(versionSkewErrs.Skippable) > 0 {
// Return the error if the user hasn't specified the --force flag
if !flags.force {
return errors.Errorf("the --version argument is invalid due to these errors:\n\n%v\nCan be bypassed if you pass the --force flag",
kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
}
// Soft errors found, but --force was specified
fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: \n\n%v", len(versionSkewErrs.Skippable), kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable))
}
}
return nil
// DryRun returns the dryRun flag.
func (d *applyData) DryRun() bool {
return d.dryRun
}
// PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted)
func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, initCfg *kubeadmapi.InitConfiguration, upgradeCfg *kubeadmapi.UpgradeConfiguration) error {
// OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster
fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n",
initCfg.KubernetesVersion, upgradeCfg.Timeouts.UpgradeManifests.Duration)
if flags.dryRun {
return upgrade.DryRunStaticPodUpgrade(upgradeCfg.Apply.Patches.Directory, initCfg)
}
return upgrade.PerformStaticPodUpgrade(client, waiter, initCfg, *upgradeCfg.Apply.EtcdUpgrade, *upgradeCfg.Apply.CertificateRenewal, upgradeCfg.Apply.Patches.Directory)
// EtcdUpgrade returns the etcdUpgrade flag.
func (d *applyData) EtcdUpgrade() bool {
return d.etcdUpgrade
}
// RenewCerts returns the renewCerts flag.
func (d *applyData) RenewCerts() bool {
return d.renewCerts
}
// Cfg returns upgradeConfiguration.
func (d *applyData) Cfg() *kubeadmapi.UpgradeConfiguration {
return d.cfg
}
// InitCfg returns the InitConfiguration.
func (d *applyData) InitCfg() *kubeadmapi.InitConfiguration {
return d.initCfg
}
// Client returns a Kubernetes client to be used by kubeadm.
func (d *applyData) Client() clientset.Interface {
return d.client
}
// PatchesDir returns the folder where patches for components are stored.
func (d *applyData) PatchesDir() string {
return d.patchesDir
}
// IgnorePreflightErrors returns the list of preflight errors to ignore.
func (d *applyData) IgnorePreflightErrors() sets.Set[string] {
return d.ignorePreflightErrors
}
// OutputWriter returns the output writer to be used by kubeadm.
func (d *applyData) OutputWriter() io.Writer {
return d.outputWriter
}
// SessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run).
func (d *applyData) SessionIsInteractive() bool {
return !(d.nonInteractiveMode || d.dryRun || d.force)
}
// AllowExperimentalUpgrades returns true if allow upgrading to an alpha/beta/release candidate version of Kubernetes.
func (d *applyData) AllowExperimentalUpgrades() bool {
return d.allowExperimentalUpgrades
}
// AllowRCUpgrades returns true if allow upgrading to a release candidate version of Kubernetes.
func (d *applyData) AllowRCUpgrades() bool {
return d.allowRCUpgrades
}
// ForceUpgrade returns true if force upgrading although some requirements might not be met.
func (d *applyData) ForceUpgrade() bool {
return d.force
}

12
cmd/kubeadm/app/cmd/upgrade/apply_test.go
View File

@ -23,39 +23,39 @@ import (
func TestSessionIsInteractive(t *testing.T) {
var tcases = []struct {
name string
flags *applyFlags
data *applyData
expected bool
}{
{
name: "Explicitly non-interactive",
flags: &applyFlags{
data: &applyData{
nonInteractiveMode: true,
},
expected: false,
},
{
name: "Implicitly non-interactive since --dryRun is used",
flags: &applyFlags{
data: &applyData{
dryRun: true,
},
expected: false,
},
{
name: "Implicitly non-interactive since --force is used",
flags: &applyFlags{
data: &applyData{
force: true,
},
expected: false,
},
{
name: "Interactive session",
flags: &applyFlags{},
data: &applyData{},
expected: true,
},
}
for _, tt := range tcases {
t.Run(tt.name, func(t *testing.T) {
if tt.flags.sessionIsInteractive() != tt.expected {
if tt.data.SessionIsInteractive() != tt.expected {
t.Error("unexpected result")
}
})

10
cmd/kubeadm/app/cmd/upgrade/common.go
View File

@ -21,7 +21,6 @@ import (
"bytes"
"io"
"os"
"time"
"github.com/pkg/errors"
"github.com/spf13/pflag"
@ -45,7 +44,6 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
"k8s.io/kubernetes/cmd/kubeadm/app/util/output"
)
@ -224,11 +222,3 @@ func getClient(file string, dryRun bool) (clientset.Interface, error) {
}
return kubeconfigutil.ClientSetFromFile(file)
}
// getWaiter gets the right waiter implementation
func getWaiter(dryRun bool, client clientset.Interface, timeout time.Duration) apiclient.Waiter {
if dryRun {
return dryrunutil.NewWaiter()
}
return apiclient.NewKubeWaiter(client, timeout, os.Stdout)
}

83
cmd/kubeadm/app/phases/upgrade/postupgrade.go
View File

@ -36,93 +36,16 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo"
nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node"
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
)
// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do
// Note that the mark-control-plane phase is left out, not needed, and no token is created as that doesn't belong to the upgrade
func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, patchesDir string, dryRun bool, out io.Writer) error {
var errs []error
// Upload currently used configuration to the cluster
// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
// depend on centralized information from this source in the future
if err := uploadconfig.UploadConfiguration(cfg, client); err != nil {
errs = append(errs, err)
}
// Create the new, version-branched kubelet ComponentConfig ConfigMap
if err := kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, client); err != nil {
errs = append(errs, errors.Wrap(err, "error creating kubelet configuration ConfigMap"))
}
// Write the new kubelet config down to disk and the env file if needed
if err := WriteKubeletConfigFiles(cfg, patchesDir, dryRun, out); err != nil {
errs = append(errs, err)
}
// Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or
// --cri-socket.
// TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly
if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil {
errs = append(errs, errors.Wrap(err, "error uploading crisocket"))
}
// Create RBAC rules that makes the bootstrap tokens able to get nodes
if err := nodebootstraptoken.AllowBootstrapTokensToGetNodes(client); err != nil {
errs = append(errs, err)
}
// Create/update RBAC rules that makes the bootstrap tokens able to post CSRs
if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil {
errs = append(errs, err)
}
// Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically
if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil {
errs = append(errs, err)
}
// Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically
if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil {
errs = append(errs, err)
}
// TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade
// Create the cluster-info ConfigMap with the associated RBAC rules
// if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil {
// return err
//}
// Create/update RBAC rules that makes the cluster-info ConfigMap reachable
if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil {
errs = append(errs, err)
}
if err := PerformAddonsUpgrade(client, cfg, patchesDir, out); err != nil {
errs = append(errs, err)
}
if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
if err := UpdateKubeletLocalMode(cfg, dryRun); err != nil {
return errors.Wrap(err, "failed to update kubelet local mode")
}
}
return errorsutil.NewAggregate(errs)
}
// PerformAddonsUpgrade performs the upgrade of the coredns and kube-proxy addons.
func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, patchesDir string, out io.Writer) error {
unupgradedControlPlanes, err := unupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
unupgradedControlPlanes, err := UnupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
if err != nil {
return errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded")
}
@ -186,12 +109,12 @@ func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfig
return errorsutil.NewAggregate(errs)
}
// unupgradedControlPlaneInstances returns a list of control plane instances that have not yet been upgraded.
// UnupgradedControlPlaneInstances returns a list of control palne instances that have not yet been upgraded.
//
// NB. This function can only be called after the current control plane instance has been upgraded already.
// Because it determines whether the other control plane instances have been upgraded by checking whether
// the kube-apiserver image of other control plane instance is the same as that of this instance.
func unupgradedControlPlaneInstances(client clientset.Interface, nodeName string) ([]string, error) {
func UnupgradedControlPlaneInstances(client clientset.Interface, nodeName string) ([]string, error) {
selector := labels.SelectorFromSet(labels.Set(map[string]string{
"component": kubeadmconstants.KubeAPIServer,
}))