diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/addons.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/addons.go new file mode 100644 index 00000000000..3159b5cb287 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/addons.go @@ -0,0 +1,188 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apply implements phases of 'kubeadm upgrade apply'. +package apply + +import ( + "context" + "fmt" + "io" + + "github.com/pkg/errors" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" + dnsaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" + proxyaddon "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" +) + +// NewAddonPhase returns the addon Cobra command +func NewAddonPhase() workflow.Phase { + return workflow.Phase{ + Name: "addon", + Short: "Install required addons for passing conformance tests", + Long: cmdutil.MacroCommandLongDescription, + Phases: []workflow.Phase{ + { + Name: "all", + Short: "Install all the addons", + InheritFlags: getAddonPhaseFlags("all"), + RunAllSiblings: true, + }, + { + Name: "coredns", + Short: "Install the CoreDNS addon to a Kubernetes cluster", + InheritFlags: getAddonPhaseFlags("coredns"), + Run: runCoreDNSAddon, + }, + { + Name: "kube-proxy", + Short: "Install the kube-proxy addon to a Kubernetes cluster", + InheritFlags: getAddonPhaseFlags("kube-proxy"), + Run: runKubeProxyAddon, + }, + }, + } +} + +func shouldUpgradeAddons(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, out io.Writer) (bool, error) { + unupgradedControlPlanes, err := upgrade.UnupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name) + if err != nil { + return false, errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded") + } + if len(unupgradedControlPlanes) > 0 { + fmt.Fprintf(out, "[upgrade/addons] skip upgrade addons because control plane instances %v have not been upgraded\n", unupgradedControlPlanes) + return false, nil + } + + return true, nil +} + +func getInitData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, string, io.Writer, bool, error) { + data, ok := c.(Data) + if !ok { + return nil, nil, "", nil, false, errors.New("addon phase invoked with an invalid data struct") + } + return data.InitCfg(), data.Client(), data.PatchesDir(), data.OutputWriter(), data.DryRun(), nil +} + +// runCoreDNSAddon installs CoreDNS addon to a Kubernetes cluster +func runCoreDNSAddon(c workflow.RunData) error { + cfg, client, patchesDir, out, dryRun, err := getInitData(c) + if err != nil { + return err + } + + shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out) + if err != nil { + return err + } + if !shouldUpgradeAddons { + return nil + } + + // If the coredns ConfigMap is missing, show a warning and assume that the + // DNS addon was skipped during "kubeadm init", and that its redeployment on upgrade is not desired. + // + // TODO: remove this once "kubeadm upgrade apply" phases are supported: + // https://github.com/kubernetes/kubeadm/issues/1318 + if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get( + context.TODO(), + kubeadmconstants.CoreDNSConfigMap, + metav1.GetOptions{}, + ); err != nil && apierrors.IsNotFound(err) { + klog.Warningf("the ConfigMaps %q in the namespace %q were not found. "+ + "Assuming that a DNS server was not deployed for this cluster. "+ + "Note that once 'kubeadm upgrade apply' supports phases you "+ + "will have to skip the DNS upgrade manually", + kubeadmconstants.CoreDNSConfigMap, + metav1.NamespaceSystem) + return nil + } + + // Upgrade CoreDNS + if err := dnsaddon.EnsureDNSAddon(&cfg.ClusterConfiguration, client, patchesDir, out, dryRun); err != nil { + return err + } + + return nil +} + +// runKubeProxyAddon installs KubeProxy addon to a Kubernetes cluster +func runKubeProxyAddon(c workflow.RunData) error { + cfg, client, _, out, dryRun, err := getInitData(c) + if err != nil { + return err + } + + shouldUpgradeAddons, err := shouldUpgradeAddons(client, cfg, out) + if err != nil { + return err + } + if !shouldUpgradeAddons { + return nil + } + + // If the kube-proxy ConfigMap is missing, show a warning and assume that kube-proxy + // was skipped during "kubeadm init", and that its redeployment on upgrade is not desired. + // + // TODO: remove this once "kubeadm upgrade apply" phases are supported: + // https://github.com/kubernetes/kubeadm/issues/1318 + if _, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get( + context.TODO(), + kubeadmconstants.KubeProxyConfigMap, + metav1.GetOptions{}, + ); err != nil && apierrors.IsNotFound(err) { + klog.Warningf("the ConfigMap %q in the namespace %q was not found. "+ + "Assuming that kube-proxy was not deployed for this cluster. "+ + "Note that once 'kubeadm upgrade apply' supports phases you "+ + "will have to skip the kube-proxy upgrade manually", + kubeadmconstants.KubeProxyConfigMap, + metav1.NamespaceSystem) + return nil + } + + // Upgrade kube-proxy + if err := proxyaddon.EnsureProxyAddon(&cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, client, out, dryRun); err != nil { + return err + } + + return nil +} + +func getAddonPhaseFlags(name string) []string { + flags := []string{ + options.CfgPath, + options.KubeconfigPath, + options.DryRun, + } + if name == "all" || name == "coredns" { + flags = append(flags, + options.Patches, + ) + } + return flags +} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/bootstraptoken.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/bootstraptoken.go new file mode 100644 index 00000000000..950e0895dd2 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/bootstraptoken.go @@ -0,0 +1,90 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apply implements phases of 'kubeadm upgrade apply'. +package apply + +import ( + "fmt" + + "github.com/pkg/errors" + + errorsutil "k8s.io/apimachinery/pkg/util/errors" + + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + clusterinfophase "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" + nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" +) + +// NewBootstrapTokenPhase returns the phase to bootstrapToken +func NewBootstrapTokenPhase() workflow.Phase { + return workflow.Phase{ + Name: "bootstrap-token", + Aliases: []string{"bootstraptoken"}, + Short: "Generates bootstrap tokens used to join a node to a cluster", + InheritFlags: []string{ + options.CfgPath, + options.KubeconfigPath, + options.DryRun, + }, + Run: runBootstrapToken, + } +} + +func runBootstrapToken(c workflow.RunData) error { + data, ok := c.(Data) + if !ok { + return errors.New("bootstrap-token phase invoked with an invalid data struct") + } + + if data.DryRun() { + fmt.Println("[dryrun] Would config cluster-info ConfigMap, RBAC Roles") + return nil + } + + fmt.Println("[bootstrap-token] Configuring cluster-info ConfigMap, RBAC Roles") + + client := data.Client() + + var errs []error + // Create RBAC rules that makes the bootstrap tokens able to get nodes + if err := nodebootstraptoken.AllowBootstrapTokensToGetNodes(client); err != nil { + errs = append(errs, err) + } + + // Create/update RBAC rules that makes the bootstrap tokens able to post CSRs + if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil { + errs = append(errs, err) + } + + // Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically + if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil { + errs = append(errs, err) + } + + // Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically + if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil { + errs = append(errs, err) + } + + // Create/update RBAC rules that makes the cluster-info ConfigMap reachable + if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil { + errs = append(errs, err) + } + + return errorsutil.NewAggregate(errs) +} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/controlplane.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/controlplane.go new file mode 100644 index 00000000000..137a35b08af --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/controlplane.go @@ -0,0 +1,74 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apply implements phases of 'kubeadm upgrade apply'. +package apply + +import ( + "fmt" + "os" + + "github.com/pkg/errors" + + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" +) + +// NewControlPlanePhase creates a kubeadm workflow phase that implements handling of control-plane upgrade. +func NewControlPlanePhase() workflow.Phase { + phase := workflow.Phase{ + Name: "control-plane", + Short: "Upgrade the control plane", + Run: runControlPlane, + InheritFlags: []string{ + options.CfgPath, + options.KubeconfigPath, + options.DryRun, + options.CertificateRenewal, + options.EtcdUpgrade, + options.Patches, + }, + } + return phase +} + +func runControlPlane(c workflow.RunData) error { + data, ok := c.(Data) + if !ok { + return errors.New("control-plane phase invoked with an invalid data struct") + } + + initCfg, upgradeCfg, client, patchesDir := data.InitCfg(), data.Cfg(), data.Client(), data.PatchesDir() + + if data.DryRun() { + fmt.Printf("[dryrun] Would upgrade your Static Pod-hosted control plane to version %q", initCfg.KubernetesVersion) + return upgrade.DryRunStaticPodUpgrade(patchesDir, initCfg) + } + + fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n", + initCfg.KubernetesVersion, upgradeCfg.Timeouts.UpgradeManifests.Duration) + + waiter := apiclient.NewKubeWaiter(client, upgradeCfg.Timeouts.UpgradeManifests.Duration, os.Stdout) + if err := upgrade.PerformStaticPodUpgrade(client, waiter, initCfg, data.EtcdUpgrade(), data.RenewCerts(), patchesDir); err != nil { + return errors.Wrap(err, "couldn't complete the static pod upgrade") + } + + fmt.Println("[upgrade] The control plane instance for this node was successfully updated!") + + return nil +} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/data.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/data.go new file mode 100644 index 00000000000..dfe291e0d5d --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/data.go @@ -0,0 +1,45 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apply implements phases of 'kubeadm upgrade apply'. +package apply + +import ( + "io" + + "k8s.io/apimachinery/pkg/util/sets" + clientset "k8s.io/client-go/kubernetes" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" +) + +// Data is the interface to use for kubeadm upgrade apply phases. +// The "applyData" type from "cmd/upgrade/apply.go" must satisfy this interface. +type Data interface { + EtcdUpgrade() bool + RenewCerts() bool + DryRun() bool + Cfg() *kubeadmapi.UpgradeConfiguration + InitCfg() *kubeadmapi.InitConfiguration + Client() clientset.Interface + IgnorePreflightErrors() sets.Set[string] + PatchesDir() string + OutputWriter() io.Writer + SessionIsInteractive() bool + AllowExperimentalUpgrades() bool + AllowRCUpgrades() bool + ForceUpgrade() bool +} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/data_test.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/data_test.go new file mode 100644 index 00000000000..1016293348f --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/data_test.go @@ -0,0 +1,46 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apply + +import ( + "io" + + "k8s.io/apimachinery/pkg/util/sets" + clientset "k8s.io/client-go/kubernetes" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" +) + +// a package local type for testing purposes. +type testData struct{} + +// testData must satisfy Data. +var _ Data = &testData{} + +func (t *testData) EtcdUpgrade() bool { return false } +func (t *testData) RenewCerts() bool { return false } +func (t *testData) DryRun() bool { return false } +func (t *testData) Cfg() *kubeadmapi.UpgradeConfiguration { return nil } +func (t *testData) InitCfg() *kubeadmapi.InitConfiguration { return nil } +func (t *testData) Client() clientset.Interface { return nil } +func (t *testData) IgnorePreflightErrors() sets.Set[string] { return nil } +func (t *testData) PatchesDir() string { return "" } +func (t *testData) OutputWriter() io.Writer { return nil } +func (t *testData) SessionIsInteractive() bool { return false } +func (t *testData) AllowExperimentalUpgrades() bool { return false } +func (t *testData) AllowRCUpgrades() bool { return false } +func (t *testData) ForceUpgrade() bool { return false } diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeconfig.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeconfig.go new file mode 100644 index 00000000000..4c9420fac88 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeconfig.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apply implements phases of 'kubeadm upgrade apply'. +package apply + +import ( + "fmt" + + "github.com/pkg/errors" + + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + "k8s.io/kubernetes/cmd/kubeadm/app/features" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" +) + +// NewKubeconfigPhase creates a kubeadm workflow phase that implements handling of kubeconfig upgrade. +func NewKubeconfigPhase() workflow.Phase { + phase := workflow.Phase{ + Name: "kubeconfig", + Short: "Upgrade kubeconfig files for this node", + Run: runKubeconfig(), + Hidden: true, + InheritFlags: []string{ + options.CfgPath, + options.DryRun, + options.KubeconfigPath, + }, + } + return phase +} + +func runKubeconfig() func(c workflow.RunData) error { + return func(c workflow.RunData) error { + data, ok := c.(Data) + if !ok { + return errors.New("kubeconfig phase invoked with an invalid data struct") + } + + cfg := data.InitCfg() + + if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) { + if err := upgrade.UpdateKubeletLocalMode(cfg, data.DryRun()); err != nil { + return errors.Wrap(err, "failed to update kubelet local mode") + } + } + + fmt.Println("[upgrade] The kubeconfig for this node was successfully updated!") + + return nil + } +} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeletconfig.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeletconfig.go new file mode 100644 index 00000000000..a97596b98ea --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/kubeletconfig.go @@ -0,0 +1,73 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apply implements phases of 'kubeadm upgrade apply'. +package apply + +import ( + "fmt" + + "github.com/pkg/errors" + + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" +) + +var ( + kubeletConfigLongDesc = cmdutil.LongDesc(` + Download the kubelet configuration from the kubelet-config ConfigMap stored in the cluster + `) +) + +// NewKubeletConfigPhase creates a kubeadm workflow phase that implements handling of kubelet-config upgrade. +func NewKubeletConfigPhase() workflow.Phase { + phase := workflow.Phase{ + Name: "kubelet-config", + Short: "Upgrade the kubelet configuration for this node", + Long: kubeletConfigLongDesc, + Run: runKubeletConfigPhase, + InheritFlags: []string{ + options.CfgPath, + options.DryRun, + options.KubeconfigPath, + options.Patches, + }, + } + return phase +} + +func runKubeletConfigPhase(c workflow.RunData) error { + data, ok := c.(Data) + if !ok { + return errors.New("kubelet-config phase invoked with an invalid data struct") + } + + initCfg, dryRun := data.InitCfg(), data.DryRun() + + // Write the configuration for the kubelet down to disk and print the generated manifests instead if dry-running. + // If not dry-running, the kubelet config file will be backed up to /etc/kubernetes/tmp/ dir, so that it could be + // recovered if there is anything goes wrong. + err := upgrade.WriteKubeletConfigFiles(initCfg, data.PatchesDir(), dryRun, data.OutputWriter()) + if err != nil { + return err + } + + fmt.Println("[upgrade] The configuration for this node was successfully updated!") + fmt.Println("[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.") + return nil +} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/preflight.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/preflight.go new file mode 100644 index 00000000000..bec44f66bd3 --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/preflight.go @@ -0,0 +1,161 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apply implements phases of 'kubeadm upgrade apply'. +package apply + +import ( + "fmt" + "os" + + "github.com/pkg/errors" + + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/klog/v2" + utilsexec "k8s.io/utils/exec" + + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + "k8s.io/kubernetes/cmd/kubeadm/app/features" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + "k8s.io/kubernetes/cmd/kubeadm/app/preflight" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" +) + +// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for kubeadm upgrade apply. +func NewPreflightPhase() workflow.Phase { + return workflow.Phase{ + Name: "preflight", + Short: "Run upgrade apply pre-flight checks", + Long: "Run pre-flight checks for kubeadm upgrade apply.", + Run: runPreflight, + InheritFlags: []string{ + options.CfgPath, + options.KubeconfigPath, + options.DryRun, + options.IgnorePreflightErrors, + "allow-experimental-upgrades", + "allow-release-candidate-upgrades", + "force", + "yes", + }, + } +} + +// runPreflight executes preflight checks logic. +func runPreflight(c workflow.RunData) error { + data, ok := c.(Data) + if !ok { + return errors.New("preflight phase invoked with an invalid data struct") + } + fmt.Println("[preflight] Running pre-flight checks") + + printer := &output.TextPrinter{} + + initCfg, client, ignorePreflightErrors := data.InitCfg(), data.Client(), data.IgnorePreflightErrors() + + // First, check if we're root separately from the other preflight checks and fail fast + if err := preflight.RunRootCheckOnly(ignorePreflightErrors); err != nil { + return err + } + + // Run CoreDNS migration check + if err := upgrade.RunCoreDNSMigrationCheck(client, ignorePreflightErrors); err != nil { + return err + } + + // Run healthchecks against the cluster + klog.V(1).Infoln("[upgrade/apply] verifying health of cluster") + if err := upgrade.CheckClusterHealth(client, &initCfg.ClusterConfiguration, ignorePreflightErrors, printer); err != nil { + return err + } + + // Check if feature gate flags used in the cluster are consistent with the set of features currently supported by kubeadm + if msg := features.CheckDeprecatedFlags(&features.InitFeatureGates, initCfg.FeatureGates); len(msg) > 0 { + for _, m := range msg { + _, _ = printer.Printf("[upgrade/config] %s\n", m) + } + } + + // Validate requested and validate actual version + klog.V(1).Infoln("[upgrade/apply] validating requested and actual version") + if err := configutil.NormalizeKubernetesVersion(&initCfg.ClusterConfiguration); err != nil { + return err + } + + // Use normalized version string in all following code. + upgradeVersion, err := version.ParseSemantic(initCfg.KubernetesVersion) + if err != nil { + return errors.Errorf("unable to parse normalized version %q as a semantic version", initCfg.KubernetesVersion) + } + + if err := features.ValidateVersion(features.InitFeatureGates, initCfg.FeatureGates, initCfg.KubernetesVersion); err != nil { + return err + } + + versionGetter := upgrade.NewOfflineVersionGetter(upgrade.NewKubeVersionGetter(client), initCfg.KubernetesVersion) + if err := EnforceVersionPolicies(initCfg.KubernetesVersion, upgradeVersion, data.AllowExperimentalUpgrades(), data.AllowRCUpgrades(), data.ForceUpgrade(), versionGetter); err != nil { + return err + } + + if data.SessionIsInteractive() { + if err := cmdutil.InteractivelyConfirmAction("upgrade", "Are you sure you want to proceed?", os.Stdin); err != nil { + return err + } + } + + if !data.DryRun() { + fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster") + fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection") + fmt.Println("[preflight] You can also perform this action beforehand using 'kubeadm config images pull'") + if err := preflight.RunPullImagesCheck(utilsexec.New(), initCfg, ignorePreflightErrors); err != nil { + return err + } + } else { + fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')") + } + + return nil +} + +// EnforceVersionPolicies makes sure that the version the user specified is valid to upgrade to +// There are both fatal and skippable (with --force) errors +func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Version, allowExperimentalUpgrades, allowRCUpgrades, force bool, versionGetter upgrade.VersionGetter) error { + fmt.Printf("[upgrade/version] You have chosen to change the cluster version to %q\n", newK8sVersionStr) + + versionSkewErrs := upgrade.EnforceVersionPolicies(versionGetter, newK8sVersionStr, newK8sVersion, allowExperimentalUpgrades, allowRCUpgrades) + if versionSkewErrs != nil { + + if len(versionSkewErrs.Mandatory) > 0 { + return errors.Errorf("the --version argument is invalid due to these fatal errors:\n\n%v\nPlease fix the misalignments highlighted above and try upgrading again", + kubeadmutil.FormatErrMsg(versionSkewErrs.Mandatory)) + } + + if len(versionSkewErrs.Skippable) > 0 { + // Return the error if the user hasn't specified the --force flag + if !force { + return errors.Errorf("the --version argument is invalid due to these errors:\n\n%v\nCan be bypassed if you pass the --force flag", + kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable)) + } + // Soft errors found, but --force was specified + fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: \n\n%v", len(versionSkewErrs.Skippable), kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable)) + } + } + return nil +} diff --git a/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go b/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go new file mode 100644 index 00000000000..74cbdcbe15d --- /dev/null +++ b/cmd/kubeadm/app/cmd/phases/upgrade/apply/uploadconfig.go @@ -0,0 +1,127 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apply implements phases of 'kubeadm upgrade apply'. +package apply + +import ( + "fmt" + + "github.com/pkg/errors" + + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" + kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" + patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" +) + +// NewUploadConfigPhase returns the phase to uploadConfig +func NewUploadConfigPhase() workflow.Phase { + return workflow.Phase{ + Name: "upload-config", + Aliases: []string{"uploadconfig"}, + Short: "Upload the kubeadm and kubelet configuration to a ConfigMap", + Long: cmdutil.MacroCommandLongDescription, + Phases: []workflow.Phase{ + { + Name: "all", + Short: "Upload all configuration to a config map", + RunAllSiblings: true, + InheritFlags: getUploadConfigPhaseFlags(), + }, + { + Name: "kubeadm", + Short: "Upload the kubeadm ClusterConfiguration to a ConfigMap", + Run: runUploadKubeadmConfig, + InheritFlags: getUploadConfigPhaseFlags(), + }, + { + Name: "kubelet", + Short: "Upload the kubelet component config to a ConfigMap", + Run: runUploadKubeletConfig, + InheritFlags: getUploadConfigPhaseFlags(), + }, + }, + } +} + +func getUploadConfigPhaseFlags() []string { + return []string{ + options.CfgPath, + options.KubeconfigPath, + options.DryRun, + } +} + +// runUploadKubeadmConfig uploads the kubeadm configuration to a ConfigMap +func runUploadKubeadmConfig(c workflow.RunData) error { + cfg, client, dryRun, err := getUploadConfigData(c) + if err != nil { + return err + } + + if dryRun { + fmt.Println("[dryrun] Would upload the kubeadm ClusterConfiguration to a ConfigMap") + return nil + } + + klog.V(1).Infoln("[upload-config] Uploading the kubeadm ClusterConfiguration to a ConfigMap") + if err := uploadconfig.UploadConfiguration(cfg, client); err != nil { + return errors.Wrap(err, "error uploading the kubeadm ClusterConfiguration") + } + return nil +} + +// runUploadKubeletConfig uploads the kubelet configuration to a ConfigMap +func runUploadKubeletConfig(c workflow.RunData) error { + cfg, client, dryRun, err := getUploadConfigData(c) + if err != nil { + return err + } + + if dryRun { + fmt.Println("[dryrun] Would upload the kubelet component config to a ConfigMap") + fmt.Println("[dryrun] Would write the CRISocket annotation for the control-plane node") + return nil + } + + klog.V(1).Infoln("[upload-config] Uploading the kubelet component config to a ConfigMap") + if err = kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, client); err != nil { + return errors.Wrap(err, "error creating kubelet configuration ConfigMap") + } + + klog.V(1).Infoln("[upload-config] Preserving the CRISocket information for the control-plane node") + if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { + return errors.Wrap(err, "Error writing Crisocket information for the control-plane node") + } + + return nil +} + +func getUploadConfigData(c workflow.RunData) (*kubeadmapi.InitConfiguration, clientset.Interface, bool, error) { + data, ok := c.(Data) + if !ok { + return nil, nil, false, errors.New("upload-config phase invoked with an invalid data struct") + } + + return data.InitCfg(), data.Client(), data.DryRun(), nil +} diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index 1aaefc08617..6e7a60ec034 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -18,27 +18,26 @@ package upgrade import ( "fmt" + "io" "os" "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/spf13/pflag" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - utilsexec "k8s.io/utils/exec" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/apply" + "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" - "k8s.io/kubernetes/cmd/kubeadm/app/features" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" - "k8s.io/kubernetes/cmd/kubeadm/app/preflight" - kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" - "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) @@ -55,9 +54,26 @@ type applyFlags struct { patchesDir string } -// sessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run) -func (f *applyFlags) sessionIsInteractive() bool { - return !(f.nonInteractiveMode || f.dryRun || f.force) +// compile-time assert that the local data object satisfies the phases data interface. +var _ phases.Data = &applyData{} + +// applyData defines all the runtime information used when running the kubeadm upgrade apply workflow; +// this data is shared across all the phases that are included in the workflow. +type applyData struct { + nonInteractiveMode bool + force bool + dryRun bool + etcdUpgrade bool + renewCerts bool + allowExperimentalUpgrades bool + allowRCUpgrades bool + printConfig bool + cfg *kubeadmapi.UpgradeConfiguration + initCfg *kubeadmapi.InitConfiguration + client clientset.Interface + patchesDir string + ignorePreflightErrors sets.Set[string] + outputWriter io.Writer } // newCmdApply returns the cobra command for `kubeadm upgrade apply` @@ -68,6 +84,8 @@ func newCmdApply(apf *applyPlanFlags) *cobra.Command { renewCerts: true, } + applyRunner := workflow.NewRunner() + cmd := &cobra.Command{ Use: "apply [version]", DisableFlagsInUseLine: true, @@ -76,7 +94,32 @@ func newCmdApply(apf *applyPlanFlags) *cobra.Command { if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil { return err } - return runApply(cmd.Flags(), flags, args) + + data, err := applyRunner.InitData(args) + if err != nil { + return err + } + + applyData, ok := data.(*applyData) + if !ok { + return errors.New("invalid data struct") + } + + if err := applyRunner.Run(args); err != nil { + return err + } + + if flags.dryRun { + fmt.Println("[upgrade/successful] Finished dryrunning successfully!") + return nil + } + + fmt.Println("") + fmt.Printf("[upgrade/successful] SUCCESS! Your cluster was upgraded to %q. Enjoy!\n", applyData.InitCfg().KubernetesVersion) + fmt.Println("") + fmt.Println("[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.") + + return nil }, } @@ -90,181 +133,222 @@ func newCmdApply(apf *applyPlanFlags) *cobra.Command { cmd.Flags().BoolVar(&flags.renewCerts, options.CertificateRenewal, flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.") options.AddPatchesFlag(cmd.Flags(), &flags.patchesDir) + // initialize the workflow runner with the list of phases + applyRunner.AppendPhase(phases.NewPreflightPhase()) + applyRunner.AppendPhase(phases.NewControlPlanePhase()) + applyRunner.AppendPhase(phases.NewUploadConfigPhase()) + applyRunner.AppendPhase(phases.NewKubeconfigPhase()) + applyRunner.AppendPhase(phases.NewKubeletConfigPhase()) + applyRunner.AppendPhase(phases.NewBootstrapTokenPhase()) + applyRunner.AppendPhase(phases.NewAddonPhase()) + + // sets the data builder function, that will be used by the runner + // both when running the entire workflow or single phases + applyRunner.SetDataInitializer(func(cmd *cobra.Command, args []string) (workflow.RunData, error) { + data, err := newApplyData(cmd, args, flags) + if err != nil { + return nil, err + } + // If the flag for skipping phases was empty, use the values from config + if len(applyRunner.Options.SkipPhases) == 0 { + applyRunner.Options.SkipPhases = data.cfg.Apply.SkipPhases + } + return data, nil + }) + + // binds the Runner to kubeadm upgrade apply command by altering + // command help, adding --skip-phases flag and by adding phases subcommands + applyRunner.BindToCommand(cmd) + return cmd } -// runApply takes care of the actual upgrade functionality -// It does the following things: -// - Checks if the cluster is healthy -// - Gets the configuration from the kubeadm-config ConfigMap in the cluster -// - Enforces all version skew policies -// - Asks the user if they really want to upgrade -// - Makes sure the control plane images are available locally on the control-plane(s) -// - Upgrades the control plane components -// - Applies the other resources that'd be created with kubeadm init as well, like -// - Uploads the newly used configuration to the cluster ConfigMap -// - Creating the RBAC rules for the bootstrap tokens and the cluster-info ConfigMap -// - Applying new CoreDNS and kube-proxy manifests -func runApply(flagSet *pflag.FlagSet, flags *applyFlags, args []string) error { +// newApplyData returns a new applyData struct to be used for the execution of the kubeadm upgrade apply workflow. +func newApplyData(cmd *cobra.Command, args []string, applyFlags *applyFlags) (*applyData, error) { + externalCfg := &v1beta4.UpgradeConfiguration{} + opt := configutil.LoadOrDefaultConfigurationOptions{} + upgradeCfg, err := configutil.LoadOrDefaultUpgradeConfiguration(applyFlags.cfgPath, externalCfg, opt) + if err != nil { + return nil, err + } - // Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap) - klog.V(1).Infoln("[upgrade/apply] verifying health of cluster") + upgradeVersion := upgradeCfg.Apply.KubernetesVersion + // The version arg is mandatory, during upgrade apply, unless it's specified in the config file + if upgradeVersion == "" { + if err := cmdutil.ValidateExactArgNumber(args, []string{"version"}); err != nil { + return nil, err + } + } + + // If option was specified in both args and config file, args will overwrite the config file. + if len(args) == 1 { + upgradeVersion = args[0] + } + + force, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "force", upgradeCfg.Apply.ForceUpgrade, &applyFlags.force).(*bool) + if !ok { + return nil, cmdutil.TypeMismatchErr("forceUpgrade", "bool") + } + + dryRun, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.DryRun, upgradeCfg.Apply.DryRun, &applyFlags.dryRun).(*bool) + if !ok { + return nil, cmdutil.TypeMismatchErr("dryRun", "bool") + } + + etcdUpgrade, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.EtcdUpgrade, upgradeCfg.Apply.EtcdUpgrade, &applyFlags.etcdUpgrade).(*bool) + if !ok { + return nil, cmdutil.TypeMismatchErr("etcdUpgrade", "bool") + } + + renewCerts, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.CertificateRenewal, upgradeCfg.Apply.CertificateRenewal, &applyFlags.renewCerts).(*bool) + if !ok { + return nil, cmdutil.TypeMismatchErr("certificateRenewal", "bool") + } + + allowExperimentalUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "allow-experimental-upgrades", upgradeCfg.Apply.AllowExperimentalUpgrades, &applyFlags.allowExperimentalUpgrades).(*bool) + if !ok { + return nil, cmdutil.TypeMismatchErr("allowExperimentalUpgrades", "bool") + } + + allowRCUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "allow-release-candidate-upgrades", upgradeCfg.Apply.AllowRCUpgrades, &applyFlags.allowRCUpgrades).(*bool) + if !ok { + return nil, cmdutil.TypeMismatchErr("allowRCUpgrades", "bool") + } + + printConfig, ok := cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), "print-config", upgradeCfg.Apply.PrintConfig, &applyFlags.printConfig).(*bool) + if !ok { + return nil, cmdutil.TypeMismatchErr("printConfig", "bool") + } + + client, err := getClient(applyFlags.kubeConfigPath, *dryRun) + if err != nil { + return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", applyFlags.kubeConfigPath) + } + + printer := &output.TextPrinter{} + + // Fetches the cluster configuration klog.V(1).Infoln("[upgrade/apply] retrieving configuration from cluster") - client, versionGetter, initCfg, upgradeCfg, err := enforceRequirements(flagSet, flags.applyPlanFlags, args, flags.dryRun, true, &output.TextPrinter{}) + initCfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", false, false) if err != nil { - return err + if apierrors.IsNotFound(err) { + _, _ = printer.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %q namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) + _, _ = printer.Printf("[upgrade/config] Use 'kubeadm init phase upload-config --config your-config.yaml' to re-upload it.\n") + err = errors.Errorf("the ConfigMap %q in the %q namespace was not found", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) + } + return nil, errors.Wrap(err, "[upgrade/init config] FATAL") } - // Validate requested and validate actual version - klog.V(1).Infoln("[upgrade/apply] validating requested and actual version") - if err := configutil.NormalizeKubernetesVersion(&initCfg.ClusterConfiguration); err != nil { - return err - } - - // Use normalized version string in all following code. - newK8sVersion, err := version.ParseSemantic(initCfg.KubernetesVersion) + ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(applyFlags.ignorePreflightErrors, upgradeCfg.Apply.IgnorePreflightErrors) if err != nil { - return errors.Errorf("unable to parse normalized version %q as a semantic version", initCfg.KubernetesVersion) + return nil, err } + // Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration: + initCfg.NodeRegistration.IgnorePreflightErrors = sets.List(ignorePreflightErrorsSet) - if err := features.ValidateVersion(features.InitFeatureGates, initCfg.FeatureGates, initCfg.KubernetesVersion); err != nil { - return err - } + // Set the ImagePullPolicy and ImagePullSerial from the UpgradeApplyConfiguration to the InitConfiguration. + // These are used by preflight.RunPullImagesCheck() when running 'apply'. + initCfg.NodeRegistration.ImagePullPolicy = upgradeCfg.Apply.ImagePullPolicy + initCfg.NodeRegistration.ImagePullSerial = upgradeCfg.Apply.ImagePullSerial - // Enforce the version skew policies - klog.V(1).Infoln("[upgrade/version] enforcing version skew policies") - allowRCUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.AllowRCUpgrades, upgradeCfg.Apply.AllowRCUpgrades, &flags.allowRCUpgrades).(*bool) - if ok { - flags.allowRCUpgrades = *allowRCUpgrades + // The `upgrade apply` version always overwrites the KubernetesVersion in the returned cfg with the target + // version. While this is not the same for `upgrade plan` where the KubernetesVersion should be the old + // one (because the call to getComponentConfigVersionStates requires the currently installed version). + // This also makes the KubernetesVersion value returned for `upgrade plan` consistent as that command + // allows to not specify a target version in which case KubernetesVersion will always hold the currently + // installed one. + initCfg.KubernetesVersion = upgradeVersion + + var patchesDir string + if upgradeCfg.Apply.Patches != nil { + patchesDir = cmdutil.ValueFromFlagsOrConfig(cmd.Flags(), options.Patches, upgradeCfg.Apply.Patches.Directory, applyFlags.patchesDir).(string) } else { - return cmdutil.TypeMismatchErr("allowRCUpgrades", "bool") + patchesDir = applyFlags.patchesDir } - force, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, "force", upgradeCfg.Apply.ForceUpgrade, &flags.force).(*bool) - if ok { - flags.force = *force - } else { - return cmdutil.TypeMismatchErr("force", "bool") + if *printConfig { + printConfiguration(&initCfg.ClusterConfiguration, os.Stdout, printer) } - allowExperimentalUpgrades, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.AllowExperimentalUpgrades, upgradeCfg.Apply.AllowExperimentalUpgrades, &flags.allowExperimentalUpgrades).(*bool) - if ok { - flags.allowExperimentalUpgrades = *allowExperimentalUpgrades - } else { - return cmdutil.TypeMismatchErr("allowExperimentalUpgrades", "bool") - } - - if err := EnforceVersionPolicies(initCfg.KubernetesVersion, newK8sVersion, flags, versionGetter); err != nil { - return errors.Wrap(err, "[upgrade/version] FATAL") - } - - // If the current session is interactive, ask the user whether they really want to upgrade. - dryRun, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.DryRun, upgradeCfg.Apply.DryRun, &flags.dryRun).(*bool) - if ok { - flags.dryRun = *dryRun - } else { - return cmdutil.TypeMismatchErr("dryRun", "bool") - } - - if flags.sessionIsInteractive() { - if err := cmdutil.InteractivelyConfirmAction("upgrade", "Are you sure you want to proceed?", os.Stdin); err != nil { - return err - } - } - - if !flags.dryRun { - fmt.Println("[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster") - fmt.Println("[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection") - fmt.Println("[upgrade/prepull] You can also perform this action beforehand using 'kubeadm config images pull'") - if err := preflight.RunPullImagesCheck(utilsexec.New(), initCfg, sets.New(upgradeCfg.Apply.IgnorePreflightErrors...)); err != nil { - return err - } - } else { - fmt.Println("[upgrade/prepull] Would pull the required images (like 'kubeadm config images pull')") - } - - waiter := getWaiter(flags.dryRun, client, upgradeCfg.Timeouts.UpgradeManifests.Duration) - - // If the config is set by flag, just overwrite it! - etcdUpgrade, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.EtcdUpgrade, upgradeCfg.Apply.EtcdUpgrade, &flags.etcdUpgrade).(*bool) - if ok { - upgradeCfg.Apply.EtcdUpgrade = etcdUpgrade - } else { - return cmdutil.TypeMismatchErr("etcdUpgrade", "bool") - } - - renewCerts, ok := cmdutil.ValueFromFlagsOrConfig(flagSet, options.CertificateRenewal, upgradeCfg.Apply.CertificateRenewal, &flags.renewCerts).(*bool) - if ok { - upgradeCfg.Apply.CertificateRenewal = renewCerts - } else { - return cmdutil.TypeMismatchErr("renewCerts", "bool") - } - - if len(flags.patchesDir) > 0 { - upgradeCfg.Apply.Patches = &kubeadmapi.Patches{Directory: flags.patchesDir} - } else if upgradeCfg.Apply.Patches == nil { - upgradeCfg.Apply.Patches = &kubeadmapi.Patches{} - } - - // Now; perform the upgrade procedure - if err := PerformControlPlaneUpgrade(flags, client, waiter, initCfg, upgradeCfg); err != nil { - return errors.Wrap(err, "[upgrade/apply] FATAL") - } - - // Upgrade RBAC rules and addons. - klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons") - if err := upgrade.PerformPostUpgradeTasks(client, initCfg, upgradeCfg.Apply.Patches.Directory, flags.dryRun, flags.applyPlanFlags.out); err != nil { - return errors.Wrap(err, "[upgrade/postupgrade] FATAL post-upgrade error") - } - - if flags.dryRun { - fmt.Println("[upgrade/successful] Finished dryrunning successfully!") - return nil - } - - fmt.Println("") - fmt.Printf("[upgrade/successful] SUCCESS! Your cluster was upgraded to %q. Enjoy!\n", initCfg.KubernetesVersion) - fmt.Println("") - fmt.Println("[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.") - - return nil + return &applyData{ + nonInteractiveMode: applyFlags.nonInteractiveMode, + force: *force, + dryRun: *dryRun, + etcdUpgrade: *etcdUpgrade, + renewCerts: *renewCerts, + allowExperimentalUpgrades: *allowExperimentalUpgrades, + allowRCUpgrades: *allowRCUpgrades, + printConfig: *printConfig, + cfg: upgradeCfg, + initCfg: initCfg, + client: client, + patchesDir: patchesDir, + ignorePreflightErrors: ignorePreflightErrorsSet, + outputWriter: applyFlags.out, + }, nil } -// EnforceVersionPolicies makes sure that the version the user specified is valid to upgrade to -// There are both fatal and skippable (with --force) errors -func EnforceVersionPolicies(newK8sVersionStr string, newK8sVersion *version.Version, flags *applyFlags, versionGetter upgrade.VersionGetter) error { - fmt.Printf("[upgrade/version] You have chosen to change the cluster version to %q\n", newK8sVersionStr) - - versionSkewErrs := upgrade.EnforceVersionPolicies(versionGetter, newK8sVersionStr, newK8sVersion, flags.allowExperimentalUpgrades, flags.allowRCUpgrades) - if versionSkewErrs != nil { - - if len(versionSkewErrs.Mandatory) > 0 { - return errors.Errorf("the --version argument is invalid due to these fatal errors:\n\n%v\nPlease fix the misalignments highlighted above and try upgrading again", - kubeadmutil.FormatErrMsg(versionSkewErrs.Mandatory)) - } - - if len(versionSkewErrs.Skippable) > 0 { - // Return the error if the user hasn't specified the --force flag - if !flags.force { - return errors.Errorf("the --version argument is invalid due to these errors:\n\n%v\nCan be bypassed if you pass the --force flag", - kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable)) - } - // Soft errors found, but --force was specified - fmt.Printf("[upgrade/version] Found %d potential version compatibility errors but skipping since the --force flag is set: \n\n%v", len(versionSkewErrs.Skippable), kubeadmutil.FormatErrMsg(versionSkewErrs.Skippable)) - } - } - return nil +// DryRun returns the dryRun flag. +func (d *applyData) DryRun() bool { + return d.dryRun } -// PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted) -func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, initCfg *kubeadmapi.InitConfiguration, upgradeCfg *kubeadmapi.UpgradeConfiguration) error { - // OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster - fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q (timeout: %v)...\n", - initCfg.KubernetesVersion, upgradeCfg.Timeouts.UpgradeManifests.Duration) - - if flags.dryRun { - return upgrade.DryRunStaticPodUpgrade(upgradeCfg.Apply.Patches.Directory, initCfg) - } - - return upgrade.PerformStaticPodUpgrade(client, waiter, initCfg, *upgradeCfg.Apply.EtcdUpgrade, *upgradeCfg.Apply.CertificateRenewal, upgradeCfg.Apply.Patches.Directory) +// EtcdUpgrade returns the etcdUpgrade flag. +func (d *applyData) EtcdUpgrade() bool { + return d.etcdUpgrade +} + +// RenewCerts returns the renewCerts flag. +func (d *applyData) RenewCerts() bool { + return d.renewCerts +} + +// Cfg returns upgradeConfiguration. +func (d *applyData) Cfg() *kubeadmapi.UpgradeConfiguration { + return d.cfg +} + +// InitCfg returns the InitConfiguration. +func (d *applyData) InitCfg() *kubeadmapi.InitConfiguration { + return d.initCfg +} + +// Client returns a Kubernetes client to be used by kubeadm. +func (d *applyData) Client() clientset.Interface { + return d.client +} + +// PatchesDir returns the folder where patches for components are stored. +func (d *applyData) PatchesDir() string { + return d.patchesDir +} + +// IgnorePreflightErrors returns the list of preflight errors to ignore. +func (d *applyData) IgnorePreflightErrors() sets.Set[string] { + return d.ignorePreflightErrors +} + +// OutputWriter returns the output writer to be used by kubeadm. +func (d *applyData) OutputWriter() io.Writer { + return d.outputWriter +} + +// SessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run). +func (d *applyData) SessionIsInteractive() bool { + return !(d.nonInteractiveMode || d.dryRun || d.force) +} + +// AllowExperimentalUpgrades returns true if allow upgrading to an alpha/beta/release candidate version of Kubernetes. +func (d *applyData) AllowExperimentalUpgrades() bool { + return d.allowExperimentalUpgrades +} + +// AllowRCUpgrades returns true if allow upgrading to a release candidate version of Kubernetes. +func (d *applyData) AllowRCUpgrades() bool { + return d.allowRCUpgrades +} + +// ForceUpgrade returns true if force upgrading although some requirements might not be met. +func (d *applyData) ForceUpgrade() bool { + return d.force } diff --git a/cmd/kubeadm/app/cmd/upgrade/apply_test.go b/cmd/kubeadm/app/cmd/upgrade/apply_test.go index 5a19eaf8585..99fa48c320f 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply_test.go @@ -23,39 +23,39 @@ import ( func TestSessionIsInteractive(t *testing.T) { var tcases = []struct { name string - flags *applyFlags + data *applyData expected bool }{ { name: "Explicitly non-interactive", - flags: &applyFlags{ + data: &applyData{ nonInteractiveMode: true, }, expected: false, }, { name: "Implicitly non-interactive since --dryRun is used", - flags: &applyFlags{ + data: &applyData{ dryRun: true, }, expected: false, }, { name: "Implicitly non-interactive since --force is used", - flags: &applyFlags{ + data: &applyData{ force: true, }, expected: false, }, { name: "Interactive session", - flags: &applyFlags{}, + data: &applyData{}, expected: true, }, } for _, tt := range tcases { t.Run(tt.name, func(t *testing.T) { - if tt.flags.sessionIsInteractive() != tt.expected { + if tt.data.SessionIsInteractive() != tt.expected { t.Error("unexpected result") } }) diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 2f339676619..b59f0d8885d 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -21,7 +21,6 @@ import ( "bytes" "io" "os" - "time" "github.com/pkg/errors" "github.com/spf13/pflag" @@ -45,7 +44,6 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/preflight" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) @@ -224,11 +222,3 @@ func getClient(file string, dryRun bool) (clientset.Interface, error) { } return kubeconfigutil.ClientSetFromFile(file) } - -// getWaiter gets the right waiter implementation -func getWaiter(dryRun bool, client clientset.Interface, timeout time.Duration) apiclient.Waiter { - if dryRun { - return dryrunutil.NewWaiter() - } - return apiclient.NewKubeWaiter(client, timeout, os.Stdout) -} diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go index 0e5710924df..520335af886 100644 --- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go +++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go @@ -36,93 +36,16 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" - "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/clusterinfo" - nodebootstraptoken "k8s.io/kubernetes/cmd/kubeadm/app/phases/bootstraptoken/node" kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" - patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode" - "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" ) -// PerformPostUpgradeTasks runs nearly the same functions as 'kubeadm init' would do -// Note that the mark-control-plane phase is left out, not needed, and no token is created as that doesn't belong to the upgrade -func PerformPostUpgradeTasks(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, patchesDir string, dryRun bool, out io.Writer) error { - var errs []error - - // Upload currently used configuration to the cluster - // Note: This is done right in the beginning of cluster initialization; as we might want to make other phases - // depend on centralized information from this source in the future - if err := uploadconfig.UploadConfiguration(cfg, client); err != nil { - errs = append(errs, err) - } - - // Create the new, version-branched kubelet ComponentConfig ConfigMap - if err := kubeletphase.CreateConfigMap(&cfg.ClusterConfiguration, client); err != nil { - errs = append(errs, errors.Wrap(err, "error creating kubelet configuration ConfigMap")) - } - - // Write the new kubelet config down to disk and the env file if needed - if err := WriteKubeletConfigFiles(cfg, patchesDir, dryRun, out); err != nil { - errs = append(errs, err) - } - - // Annotate the node with the crisocket information, sourced either from the InitConfiguration struct or - // --cri-socket. - // TODO: In the future we want to use something more official like NodeStatus or similar for detecting this properly - if err := patchnodephase.AnnotateCRISocket(client, cfg.NodeRegistration.Name, cfg.NodeRegistration.CRISocket); err != nil { - errs = append(errs, errors.Wrap(err, "error uploading crisocket")) - } - - // Create RBAC rules that makes the bootstrap tokens able to get nodes - if err := nodebootstraptoken.AllowBootstrapTokensToGetNodes(client); err != nil { - errs = append(errs, err) - } - - // Create/update RBAC rules that makes the bootstrap tokens able to post CSRs - if err := nodebootstraptoken.AllowBootstrapTokensToPostCSRs(client); err != nil { - errs = append(errs, err) - } - - // Create/update RBAC rules that makes the bootstrap tokens able to get their CSRs approved automatically - if err := nodebootstraptoken.AutoApproveNodeBootstrapTokens(client); err != nil { - errs = append(errs, err) - } - - // Create/update RBAC rules that makes the nodes to rotate certificates and get their CSRs approved automatically - if err := nodebootstraptoken.AutoApproveNodeCertificateRotation(client); err != nil { - errs = append(errs, err) - } - - // TODO: Is this needed to do here? I think that updating cluster info should probably be separate from a normal upgrade - // Create the cluster-info ConfigMap with the associated RBAC rules - // if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil { - // return err - //} - // Create/update RBAC rules that makes the cluster-info ConfigMap reachable - if err := clusterinfo.CreateClusterInfoRBACRules(client); err != nil { - errs = append(errs, err) - } - - if err := PerformAddonsUpgrade(client, cfg, patchesDir, out); err != nil { - errs = append(errs, err) - } - - if features.Enabled(cfg.FeatureGates, features.ControlPlaneKubeletLocalMode) { - if err := UpdateKubeletLocalMode(cfg, dryRun); err != nil { - return errors.Wrap(err, "failed to update kubelet local mode") - } - } - - return errorsutil.NewAggregate(errs) -} - // PerformAddonsUpgrade performs the upgrade of the coredns and kube-proxy addons. func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfiguration, patchesDir string, out io.Writer) error { - unupgradedControlPlanes, err := unupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name) + unupgradedControlPlanes, err := UnupgradedControlPlaneInstances(client, cfg.NodeRegistration.Name) if err != nil { return errors.Wrapf(err, "failed to determine whether all the control plane instances have been upgraded") } @@ -186,12 +109,12 @@ func PerformAddonsUpgrade(client clientset.Interface, cfg *kubeadmapi.InitConfig return errorsutil.NewAggregate(errs) } -// unupgradedControlPlaneInstances returns a list of control plane instances that have not yet been upgraded. +// UnupgradedControlPlaneInstances returns a list of control palne instances that have not yet been upgraded. // // NB. This function can only be called after the current control plane instance has been upgraded already. // Because it determines whether the other control plane instances have been upgraded by checking whether // the kube-apiserver image of other control plane instance is the same as that of this instance. -func unupgradedControlPlaneInstances(client clientset.Interface, nodeName string) ([]string, error) { +func UnupgradedControlPlaneInstances(client clientset.Interface, nodeName string) ([]string, error) { selector := labels.SelectorFromSet(labels.Set(map[string]string{ "component": kubeadmconstants.KubeAPIServer, }))