github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-08-17 15:50:10 +00:00
Code Issues Projects Releases Wiki Activity

kubeadm: add addon and post-upgrade phase for 'kubeadm upgrade node'

Browse Source
This commit is contained in:
SataQiu 2024-09-09 19:04:23 +08:00
parent 17bae91f20
commit 8db2dd3c8b
11 changed files with 229 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeletconfig.go
View File

@ -57,16 +57,14 @@ func runKubeletConfigPhase(c workflow.RunData) error {
return errors.New("kubelet-config phase invoked with an invalid data struct")
}
initCfg, dryRun := data.InitCfg(), data.DryRun()
// Write the configuration for the kubelet down to disk and print the generated manifests instead of dry-running.
// If not dry-running, the kubelet config file will be backed up to the /etc/kubernetes/tmp/ dir, so that it could be
// recovered if anything goes wrong.
err := upgrade.WriteKubeletConfigFiles(initCfg, data.PatchesDir(), dryRun, data.OutputWriter())
err := upgrade.WriteKubeletConfigFiles(data.InitCfg(), data.PatchesDir(), data.DryRun(), data.OutputWriter())
if err != nil {
return err
}
fmt.Println("[upgrade/kubelet-config] The kubelet configuration for this node was successfully updated!")
fmt.Println("[upgrade/kubelet-config] The kubelet configuration for this node was successfully upgraded!")
return nil
}

2
cmd/kubeadm/app/cmd/phases/upgrade/apply/postupgrade.go
View File

@ -41,7 +41,7 @@ func NewPostUpgradePhase() workflow.Phase {
func runPostUpgrade(c workflow.RunData) error {
_, ok := c.(Data)
if !ok {
return errors.New("preflight phase invoked with an invalid data struct")
return errors.New("post-upgrade phase invoked with an invalid data struct")
}
// PLACEHOLDER: this phase should contain any release specific post-upgrade tasks.

144
cmd/kubeadm/app/cmd/phases/upgrade/node/addons.go Normal file
View File

@ -0,0 +1,144 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package node implements phases of 'kubeadm upgrade node'.
package node
import (
"fmt"
"io"
"github.com/pkg/errors"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
dnsaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
proxyaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
)
// NewAddonPhase returns a new addon phase.
func NewAddonPhase() workflow.Phase {
return workflow.Phase{
Name: "addon",
Short: "Upgrade the default kubeadm addons",
Long: cmdutil.MacroCommandLongDescription,
Phases: []workflow.Phase{
{
Name: "all",
Short: "Upgrade all the addons",
InheritFlags: getAddonPhaseFlags("all"),
RunAllSiblings: true,
},
{
Name: "coredns",
Short: "Upgrade the CoreDNS addon",
InheritFlags: getAddonPhaseFlags("coredns"),
Run: runCoreDNSAddon,
},
{
Name: "kube-proxy",
Short: "Upgrade the kube-proxy addon",
InheritFlags: getAddonPhaseFlags("kube-proxy"),
Run: runKubeProxyAddon,
},
},
}
}
func shouldUpgradeAddons(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, out io.Writer) (bool, error) {
unupgradedControlPlanes, err := upgrade.UnupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
if err != nil {
return false, errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded")
}
if len(unupgradedControlPlanes) > 0 {
fmt.Fprintf(out, "[upgrade/addon] Skipping upgrade of addons because control plane instances %v have not been upgraded\n", unupgradedControlPlanes)
return false, nil
}
return true, nil
}
func getInitData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, string, io.Writer, bool, error) {
data, ok := c.(Data)
if !ok {
return nil, nil, "", nil, false, errors.New("addon phase invoked with an invalid data struct")
}
return data.InitCfg(), data.Client(), data.PatchesDir(), data.OutputWriter(), data.DryRun(), nil
}
// runCoreDNSAddon upgrades the CoreDNS addon.
func runCoreDNSAddon(c workflow.RunData) error {
cfg, client, patchesDir, out, dryRun, err := getInitData(c)
if err != nil {
return err
}
shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out)
if err != nil {
return err
}
if !shouldUpgradeAddons {
return nil
}
// Upgrade CoreDNS
if err := dnsaddon.EnsureDNSAddon(&cfg.ClusterConfiguration, client, patchesDir, out, dryRun); err != nil {
return err
}
return nil
}
// runKubeProxyAddon upgrades the kube-proxy addon.
func runKubeProxyAddon(c workflow.RunData) error {
cfg, client, _, out, dryRun, err := getInitData(c)
if err != nil {
return err
}
shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out)
if err != nil {
return err
}
if !shouldUpgradeAddons {
return nil
}
// Upgrade kube-proxy
if err := proxyaddon.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, dryRun); err != nil {
return err
}
return nil
}
func getAddonPhaseFlags(name string) []string {
flags := []string{
options.CfgPath,
options.KubeconfigPath,
options.DryRun,
}
if name == "all" || name == "coredns" {
flags = append(flags,
options.Patches,
)
}
return flags
}

15
cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go
View File

@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package node implements phases of 'kubeadm upgrade node'.
package node
import (
@ -28,13 +29,14 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
)
// NewControlPlane creates a kubeadm workflow phase that implements handling of control-plane upgrade.
// NewControlPlane returns a new control-plane phase.
func NewControlPlane() workflow.Phase {
phase := workflow.Phase{
Name: "control-plane",
Short: "Upgrade the control plane instance deployed on this node, if any",
Run: runControlPlane(),
InheritFlags: []string{
options.CfgPath,
options.DryRun,
options.KubeconfigPath,
options.CertificateRenewal,
@ -54,7 +56,7 @@ func runControlPlane() func(c workflow.RunData) error {
// if this is not a control-plane node, this phase should not be executed
if !data.IsControlPlaneNode() {
fmt.Println("[upgrade] Skipping phase. Not a control plane node.")
fmt.Println("[upgrade/control-plane] Skipping phase. Not a control plane node.")
return nil
}
@ -67,8 +69,9 @@ func runControlPlane() func(c workflow.RunData) error {
patchesDir := data.PatchesDir()
// Upgrade the control plane and etcd if installed on this node
fmt.Printf("[upgrade] Upgrading your Static Pod-hosted control plane instance to version %q...\n", cfg.KubernetesVersion)
fmt.Printf("[upgrade/control-plane] Upgrading your Static Pod-hosted control plane instance to version %q...\n", cfg.KubernetesVersion)
if dryRun {
fmt.Printf("[dryrun] Would upgrade your static Pod-hosted control plane to version %q", cfg.KubernetesVersion)
return upgrade.DryRunStaticPodUpgrade(patchesDir, cfg)
}
@ -78,11 +81,7 @@ func runControlPlane() func(c workflow.RunData) error {
return errors.Wrap(err, "couldn't complete the static pod upgrade")
}
if err := upgrade.PerformAddonsUpgrade(client, cfg, data.PatchesDir(), data.OutputWriter()); err != nil {
return errors.Wrap(err, "failed to perform addons upgrade")
}
fmt.Println("[upgrade] The control plane instance for this node was successfully updated!")
fmt.Println("[upgrade/control-plane] The control plane instance for this node was successfully upgraded!")
return nil
}

1
cmd/kubeadm/app/cmd/phases/upgrade/node/data.go
View File

@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package node implements phases of 'kubeadm upgrade node'.
package node
import (

16
cmd/kubeadm/app/cmd/phases/upgrade/node/kubeconfig.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package node holds phases of "kubeadm upgrade node".
// Package node implements phases of 'kubeadm upgrade node'.
package node
import (
@ -28,7 +28,7 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
)
// NewKubeconfigPhase creates a kubeadm workflow phase that implements handling of kubeconfig upgrade.
// NewKubeconfigPhase returns a new kubeconfig phase.
func NewKubeconfigPhase() workflow.Phase {
phase := workflow.Phase{
Name: "kubeconfig",
@ -36,6 +36,7 @@ func NewKubeconfigPhase() workflow.Phase {
Run: runKubeconfig(),
Hidden: true,
InheritFlags: []string{
options.CfgPath,
options.DryRun,
options.KubeconfigPath,
},
@ -50,23 +51,22 @@ func runKubeconfig() func(c workflow.RunData) error {
return errors.New("kubeconfig phase invoked with an invalid data struct")
}
// if this is not a control-plane node, this phase should not be executed
// If this is not a control-plane node, this phase should not be executed.
if !data.IsControlPlaneNode() {
fmt.Println("[upgrade] Skipping phase. Not a control plane node.")
fmt.Println("[upgrade/kubeconfig] Skipping phase. Not a control plane node.")
return nil
}
// otherwise, retrieve all the info required for kubeconfig upgrade
// Otherwise, retrieve all the info required for kubeconfig upgrade.
cfg := data.InitCfg()
dryRun := data.DryRun()
if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) {
if err := upgrade.UpdateKubeletLocalMode(cfg, dryRun); err != nil {
if err := upgrade.UpdateKubeletLocalMode(cfg, data.DryRun()); err != nil {
return errors.Wrap(err, "failed to update kubelet local mode")
}
}
fmt.Println("[upgrade] The kubeconfig for this node was successfully updated!")
fmt.Println("[upgrade/kubeconfig] The kubeconfig files for this node were successfully upgraded!")
return nil
}

14
cmd/kubeadm/app/cmd/phases/upgrade/node/kubeletconfig.go
View File

@ -29,11 +29,11 @@ import (
var (
kubeletConfigLongDesc = cmdutil.LongDesc(`
Download the kubelet configuration from the kubelet-config ConfigMap stored in the cluster
Upgrade the kubelet configuration for this node by downloading it from the kubelet-config ConfigMap stored in the cluster
`)
)
// NewKubeletConfigPhase creates a kubeadm workflow phase that implements handling of kubelet-config upgrade.
// NewKubeletConfigPhase returns a new kubelet-config phase.
func NewKubeletConfigPhase() workflow.Phase {
phase := workflow.Phase{
Name: "kubelet-config",
@ -41,6 +41,7 @@ func NewKubeletConfigPhase() workflow.Phase {
Long: kubeletConfigLongDesc,
Run: runKubeletConfigPhase(),
InheritFlags: []string{
options.CfgPath,
options.DryRun,
options.KubeconfigPath,
options.Patches,
@ -56,20 +57,15 @@ func runKubeletConfigPhase() func(c workflow.RunData) error {
return errors.New("kubelet-config phase invoked with an invalid data struct")
}
// otherwise, retrieve all the info required for kubelet config upgrade
cfg := data.InitCfg()
dryRun := data.DryRun()
// Write the configuration for the kubelet down to disk and print the generated manifests instead if dry-running.
// If not dry-running, the kubelet config file will be backed up to /etc/kubernetes/tmp/ dir, so that it could be
// recovered if there is anything goes wrong.
err := upgrade.WriteKubeletConfigFiles(cfg, data.PatchesDir(), dryRun, data.OutputWriter())
err := upgrade.WriteKubeletConfigFiles(data.InitCfg(), data.PatchesDir(), data.DryRun(), data.OutputWriter())
if err != nil {
return err
}
fmt.Println("[upgrade] The configuration for this node was successfully updated!")
fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.")
fmt.Println("[upgrade/kubelet-config] The configuration for this node was successfully upgraded!")
return nil
}
}

49
cmd/kubeadm/app/cmd/phases/upgrade/node/postupgrade.go Normal file
View File

@ -0,0 +1,49 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package node implements phases of 'kubeadm upgrade node'.
package node
import (
"github.com/pkg/errors"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
)
// NewPostUpgradePhase returns a new post-upgrade phase.
func NewPostUpgradePhase() workflow.Phase {
return workflow.Phase{
Name: "post-upgrade",
Short: "Run post upgrade tasks",
Run: runPostUpgrade,
InheritFlags: []string{
options.CfgPath,
options.KubeconfigPath,
options.DryRun,
},
}
}
func runPostUpgrade(c workflow.RunData) error {
_, ok := c.(Data)
if !ok {
return errors.New("post-upgrade phase invoked with an invalid data struct")
}
// PLACEHOLDER: this phase should contain any release specific post-upgrade tasks.
return nil
}

19
cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go
View File

@ -28,7 +28,7 @@ import (
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
)
// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for a new node join
// NewPreflightPhase returns a new preflight phase.
func NewPreflightPhase() workflow.Phase {
return workflow.Phase{
Name: "preflight",
@ -36,6 +36,7 @@ func NewPreflightPhase() workflow.Phase {
Long: "Run pre-flight checks for kubeadm upgrade node.",
Run: runPreflight,
InheritFlags: []string{
options.CfgPath,
options.IgnorePreflightErrors,
},
}
@ -47,14 +48,14 @@ func runPreflight(c workflow.RunData) error {
if !ok {
return errors.New("preflight phase invoked with an invalid data struct")
}
fmt.Println("[preflight] Running pre-flight checks")
fmt.Println("[upgrade/preflight] Running pre-flight checks")
// First, check if we're root separately from the other preflight checks and fail fast
// First, check if we're root separately from the other preflight checks and fail fast.
if err := preflight.RunRootCheckOnly(data.IgnorePreflightErrors()); err != nil {
return err
}
// If this is a control-plane node, pull the basic images
// If this is a control-plane node, pull the basic images.
if data.IsControlPlaneNode() {
// Update the InitConfiguration used for RunPullImagesCheck with ImagePullPolicy and ImagePullSerial
// that come from UpgradeNodeConfiguration.
@ -63,17 +64,17 @@ func runPreflight(c workflow.RunData) error {
initConfig.NodeRegistration.ImagePullSerial = data.Cfg().Node.ImagePullSerial
if !data.DryRun() {
fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
fmt.Println("[preflight] You can also perform this action beforehand using 'kubeadm config images pull'")
fmt.Println("[upgrade/preflight] Pulling images required for setting up a Kubernetes cluster")
fmt.Println("[upgrade/preflight] This might take a minute or two, depending on the speed of your internet connection")
fmt.Println("[upgrade/preflight] You can also perform this action beforehand using 'kubeadm config images pull'")
if err := preflight.RunPullImagesCheck(utilsexec.New(), initConfig, data.IgnorePreflightErrors()); err != nil {
return err
}
} else {
fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')")
fmt.Println("[upgrade/preflight] Would pull the required images (like 'kubeadm config images pull')")
}
} else {
fmt.Println("[preflight] Skipping prepull. Not a control plane node.")
fmt.Println("[upgrade/preflight] Skipping prepull. Not a control plane node.")
return nil
}

2
cmd/kubeadm/app/cmd/upgrade/node.go
View File

@ -99,6 +99,8 @@ func newCmdNode(out io.Writer) *cobra.Command {
nodeRunner.AppendPhase(phases.NewControlPlane())
nodeRunner.AppendPhase(phases.NewKubeconfigPhase())
nodeRunner.AppendPhase(phases.NewKubeletConfigPhase())
nodeRunner.AppendPhase(phases.NewAddonPhase())
nodeRunner.AppendPhase(phases.NewPostUpgradePhase())
// sets the data builder function, that will be used by the runner
// both when running the entire workflow or single phases

69
cmd/kubeadm/app/phases/upgrade/postupgrade.go
View File

@ -25,7 +25,6 @@ import (
"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
errorsutil "k8s.io/apimachinery/pkg/util/errors"
@ -36,79 +35,11 @@ import (
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy"
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
)
// PerformAddonsUpgrade performs the upgrade of the coredns and kube-proxy addons.
func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, patchesDir string, out io.Writer) error {
unupgradedControlPlanes, err := UnupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name)
if err != nil {
return errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded")
}
if len(unupgradedControlPlanes) > 0 {
fmt.Fprintf(out, "[upgrade/addons] skip upgrade addons because control plane instances %v have not been upgraded\n", unupgradedControlPlanes)
return nil
}
var errs []error
// If the coredns ConfigMap is missing, show a warning and assume that the
// DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
//
// TODO: remove this once "kubeadm upgrade apply" phases are supported:
// https://github.com/kubernetes/kubeadm/issues/1318
var missingCoreDNSConfigMap bool
if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
context.TODO(),
kubeadmconstants.CoreDNSConfigMap,
metav1.GetOptions{},
); err != nil && apierrors.IsNotFound(err) {
missingCoreDNSConfigMap = true
}
if missingCoreDNSConfigMap {
klog.Warningf("the ConfigMaps %q in the namespace %q were not found. "+
"Assuming that a DNS server was not deployed for this cluster. "+
"Note that once 'kubeadm upgrade apply' supports phases you "+
"will have to skip the DNS upgrade manually",
kubeadmconstants.CoreDNSConfigMap,
metav1.NamespaceSystem)
} else {
// Upgrade CoreDNS
if err := dns.EnsureDNSAddon(&cfg.ClusterConfiguration, client, patchesDir, out, false); err != nil {
errs = append(errs, err)
}
}
// If the kube-proxy ConfigMap is missing, show a warning and assume that kube-proxy
// was skipped during "kubeadm init", and that its redeployment on upgrade is not desired.
//
// TODO: remove this once "kubeadm upgrade apply" phases are supported:
// https://github.com/kubernetes/kubeadm/issues/1318
if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(
context.TODO(),
kubeadmconstants.KubeProxyConfigMap,
metav1.GetOptions{},
); err != nil && apierrors.IsNotFound(err) {
klog.Warningf("the ConfigMap %q in the namespace %q was not found. "+
"Assuming that kube-proxy was not deployed for this cluster. "+
"Note that once 'kubeadm upgrade apply' supports phases you "+
"will have to skip the kube-proxy upgrade manually",
kubeadmconstants.KubeProxyConfigMap,
metav1.NamespaceSystem)
} else {
// Upgrade kube-proxy
if err := proxy.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, false); err != nil {
errs = append(errs, err)
}
}
return errorsutil.NewAggregate(errs)
}
// UnupgradedControlPlaneInstances returns a list of control plane instances that have not yet been upgraded.
//
// NB. This function can only be called after the current control plane instance has been upgraded already.