From 93daef6e77bb0f7eca9a5ed9804c34a5840fcb63 Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Tue, 1 Mar 2022 15:11:23 +0800 Subject: [PATCH 1/3] feature(kubeadm): add support of json/yaml format for upgrade plan - cherry-pick 83941 and rebase --- cmd/kubeadm/app/apis/output/types.go | 4 + cmd/kubeadm/app/apis/output/v1alpha2/types.go | 4 + .../output/v1alpha2/zz_generated.deepcopy.go | 9 + .../app/apis/output/zz_generated.deepcopy.go | 9 + cmd/kubeadm/app/cmd/certs.go | 3 +- cmd/kubeadm/app/cmd/join.go | 3 +- cmd/kubeadm/app/cmd/reset.go | 3 +- cmd/kubeadm/app/cmd/upgrade/apply.go | 3 +- cmd/kubeadm/app/cmd/upgrade/common.go | 19 +- cmd/kubeadm/app/cmd/upgrade/common_test.go | 3 +- cmd/kubeadm/app/cmd/upgrade/diff.go | 3 +- cmd/kubeadm/app/cmd/upgrade/node.go | 3 +- cmd/kubeadm/app/cmd/upgrade/plan.go | 244 +++++++++++++++--- cmd/kubeadm/app/phases/upgrade/compute.go | 5 - cmd/kubeadm/app/util/config/cluster.go | 8 +- cmd/kubeadm/app/util/output/output.go | 44 +++- 16 files changed, 300 insertions(+), 67 deletions(-) diff --git a/cmd/kubeadm/app/apis/output/types.go b/cmd/kubeadm/app/apis/output/types.go index cd3b4a7746b..9dda9491425 100644 --- a/cmd/kubeadm/app/apis/output/types.go +++ b/cmd/kubeadm/app/apis/output/types.go @@ -40,8 +40,12 @@ type Images struct { Images []string } +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + // ComponentUpgradePlan represents information about upgrade plan for one component type ComponentUpgradePlan struct { + metav1.TypeMeta + Name string CurrentVersion string NewVersion string diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/types.go b/cmd/kubeadm/app/apis/output/v1alpha2/types.go index 71bc2fa93be..529c029dd59 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/types.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/types.go @@ -40,8 +40,12 @@ type Images struct { Images []string `json:"images"` } +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + // ComponentUpgradePlan represents information about upgrade plan for one component type ComponentUpgradePlan struct { + metav1.TypeMeta + Name string `json:"name"` CurrentVersion string `json:"currentVersion"` NewVersion string `json:"newVersion"` diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go index 54cc75256b3..73e8c32def2 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go @@ -70,6 +70,7 @@ func (in *ComponentConfigVersionState) DeepCopy() *ComponentConfigVersionState { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ComponentUpgradePlan) DeepCopyInto(out *ComponentUpgradePlan) { *out = *in + out.TypeMeta = in.TypeMeta return } @@ -83,6 +84,14 @@ func (in *ComponentUpgradePlan) DeepCopy() *ComponentUpgradePlan { return out } +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ComponentUpgradePlan) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Images) DeepCopyInto(out *Images) { *out = *in diff --git a/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go index 0d7bc006cf9..8d553d35c15 100644 --- a/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go @@ -70,6 +70,7 @@ func (in *ComponentConfigVersionState) DeepCopy() *ComponentConfigVersionState { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ComponentUpgradePlan) DeepCopyInto(out *ComponentUpgradePlan) { *out = *in + out.TypeMeta = in.TypeMeta return } @@ -83,6 +84,14 @@ func (in *ComponentUpgradePlan) DeepCopy() *ComponentUpgradePlan { return out } +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ComponentUpgradePlan) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Images) DeepCopyInto(out *Images) { *out = *in diff --git a/cmd/kubeadm/app/cmd/certs.go b/cmd/kubeadm/app/cmd/certs.go index 7506c8b6d0a..a5265b4b1c3 100644 --- a/cmd/kubeadm/app/cmd/certs.go +++ b/cmd/kubeadm/app/cmd/certs.go @@ -39,6 +39,7 @@ import ( kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) var ( @@ -332,7 +333,7 @@ func getInternalCfg(cfgPath string, kubeconfigPath string, cfg kubeadmapiv1.Clus if cfgPath == "" { client, err := kubeconfigutil.ClientSetFromFile(kubeconfigPath) if err == nil { - internalcfg, err := configutil.FetchInitConfigurationFromCluster(client, out, logPrefix, false, false) + internalcfg, err := configutil.FetchInitConfigurationFromCluster(client, &output.TextPrinter{}, logPrefix, false, false) if err == nil { fmt.Println() // add empty line to separate the FetchInitConfigurationFromCluster output from the command output return internalcfg, nil diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index fa0b07e3631..61c6c36c5b1 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -47,6 +47,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/discovery" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) var ( @@ -618,7 +619,7 @@ func fetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.I } // Fetches the init configuration - initConfiguration, err := configutil.FetchInitConfigurationFromCluster(tlsClient, os.Stdout, "preflight", true, false) + initConfiguration, err := configutil.FetchInitConfigurationFromCluster(tlsClient, &output.TextPrinter{}, "preflight", true, false) if err != nil { return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") } diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index aaccbd56cec..ef33b1ad8b8 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -37,6 +37,7 @@ import ( cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" ) @@ -98,7 +99,7 @@ func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out i client, err := getClientset(options.kubeconfigPath, false) if err == nil { klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", options.kubeconfigPath) - cfg, err = configutil.FetchInitConfigurationFromCluster(client, out, "reset", false, false) + cfg, err = configutil.FetchInitConfigurationFromCluster(client, &output.TextPrinter{}, "reset", false, false) if err != nil { klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap from cluster: %v", err) } diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index b47a7cc7eda..4b83ed65ccd 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -37,6 +37,7 @@ import ( kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) // applyFlags holds the information about the flags that can be passed to apply @@ -103,7 +104,7 @@ func runApply(flags *applyFlags, args []string) error { // Start with the basics, verify that the cluster is healthy and get the configuration from the cluster (using the ConfigMap) klog.V(1).Infoln("[upgrade/apply] verifying health of cluster") klog.V(1).Infoln("[upgrade/apply] retrieving configuration from cluster") - client, versionGetter, cfg, err := enforceRequirements(flags.applyPlanFlags, args, flags.dryRun, true) + client, versionGetter, cfg, err := enforceRequirements(flags.applyPlanFlags, args, flags.dryRun, true, &output.TextPrinter{}) if err != nil { return err } diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 6cd4ee96b6c..8fc9358e486 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -47,6 +47,7 @@ import ( configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) // isKubeadmConfigPresent checks if a kubeadm config type is found in the provided document map @@ -63,14 +64,14 @@ func isKubeadmConfigPresent(docmap kubeadmapi.DocumentMap) bool { // are loaded. This function allows the component configs to be loaded from a file that contains only them. If the file contains any kubeadm types // in it (API group "kubeadm.kubernetes.io" present), then the supplied file is treaded as a legacy reconfiguration style "--config" use and the // returned bool value is set to true (the only case to be done so). -func loadConfig(cfgPath string, client clientset.Interface, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, bool, error) { +func loadConfig(cfgPath string, client clientset.Interface, skipComponentConfigs bool, printer output.Printer) (*kubeadmapi.InitConfiguration, bool, error) { // Used for info logs here const logPrefix = "upgrade/config" // The usual case here is to not have a config file, but rather load the config from the cluster. // This is probably 90% of the time. So we handle it first. if cfgPath == "" { - cfg, err := configutil.FetchInitConfigurationFromCluster(client, os.Stdout, logPrefix, false, skipComponentConfigs) + cfg, err := configutil.FetchInitConfigurationFromCluster(client, printer, logPrefix, false, skipComponentConfigs) return cfg, false, err } @@ -98,7 +99,7 @@ func loadConfig(cfgPath string, client clientset.Interface, skipComponentConfigs // If no kubeadm config types are present, we assume that there are manually upgraded component configs in the file. // Hence, we load the kubeadm types from the cluster. - initCfg, err := configutil.FetchInitConfigurationFromCluster(client, os.Stdout, logPrefix, false, true) + initCfg, err := configutil.FetchInitConfigurationFromCluster(client, printer, logPrefix, false, true) if err != nil { return nil, false, err } @@ -121,17 +122,17 @@ func loadConfig(cfgPath string, client clientset.Interface, skipComponentConfigs } // enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure -func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgradeApply bool) (clientset.Interface, upgrade.VersionGetter, *kubeadmapi.InitConfiguration, error) { +func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgradeApply bool, printer output.Printer) (clientset.Interface, upgrade.VersionGetter, *kubeadmapi.InitConfiguration, error) { client, err := getClient(flags.kubeConfigPath, dryRun) if err != nil { return nil, nil, nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) } // Fetch the configuration from a file or ConfigMap and validate it - fmt.Println("[upgrade/config] Making sure the configuration is correct:") + printer.Printf("[upgrade/config] Making sure the configuration is correct:\n") var newK8sVersion string - cfg, legacyReconfigure, err := loadConfig(flags.cfgPath, client, !upgradeApply) + cfg, legacyReconfigure, err := loadConfig(flags.cfgPath, client, !upgradeApply, printer) if err != nil { if apierrors.IsNotFound(err) { fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) @@ -161,7 +162,7 @@ func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgr // Ensure the user is root klog.V(1).Info("running preflight checks") - if err := runPreflightChecks(client, ignorePreflightErrorsSet, &cfg.ClusterConfiguration); err != nil { + if err := runPreflightChecks(client, ignorePreflightErrorsSet, &cfg.ClusterConfiguration, printer); err != nil { return nil, nil, nil, err } @@ -235,8 +236,8 @@ func printConfiguration(clustercfg *kubeadmapi.ClusterConfiguration, w io.Writer } // runPreflightChecks runs the root preflight check -func runPreflightChecks(client clientset.Interface, ignorePreflightErrors sets.String, cfg *kubeadmapi.ClusterConfiguration) error { - fmt.Println("[preflight] Running pre-flight checks.") +func runPreflightChecks(client clientset.Interface, ignorePreflightErrors sets.String, cfg *kubeadmapi.ClusterConfiguration, printer output.Printer) error { + printer.Printf("[preflight] Running pre-flight checks.\n") err := preflight.RunRootCheckOnly(ignorePreflightErrors) if err != nil { return err diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go index ecdcf341249..357148e9afc 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/common_test.go @@ -23,6 +23,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) func TestEnforceRequirements(t *testing.T) { @@ -54,7 +55,7 @@ func TestEnforceRequirements(t *testing.T) { } for _, tt := range tcases { t.Run(tt.name, func(t *testing.T) { - _, _, _, err := enforceRequirements(&tt.flags, nil, tt.dryRun, false) + _, _, _, err := enforceRequirements(&tt.flags, nil, tt.dryRun, false, &output.TextPrinter{}) if err == nil && tt.expectedErr { t.Error("Expected error, but got success") diff --git a/cmd/kubeadm/app/cmd/upgrade/diff.go b/cmd/kubeadm/app/cmd/upgrade/diff.go index 73cdedcf17f..fe421e94811 100644 --- a/cmd/kubeadm/app/cmd/upgrade/diff.go +++ b/cmd/kubeadm/app/cmd/upgrade/diff.go @@ -37,6 +37,7 @@ import ( kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) type diffFlags struct { @@ -118,7 +119,7 @@ func runDiff(flags *diffFlags, args []string) error { if err != nil { return errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) } - cfg, err = configutil.FetchInitConfigurationFromCluster(client, flags.out, "upgrade/diff", false, false) + cfg, err = configutil.FetchInitConfigurationFromCluster(client, &output.TextPrinter{}, "upgrade/diff", false, false) } if err != nil { return err diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index 7072312a0b8..5224de86afb 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -33,6 +33,7 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" "k8s.io/kubernetes/cmd/kubeadm/app/constants" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) // nodeOptions defines all the options exposed via flags by kubeadm upgrade node. @@ -139,7 +140,7 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*node // Fetches the cluster configuration // NB in case of control-plane node, we are reading all the info for the node; in case of NOT control-plane node // (worker node), we are not reading local API address and the CRI socket from the node object - cfg, err := configutil.FetchInitConfigurationFromCluster(client, os.Stdout, "upgrade", !isControlPlaneNode, false) + cfg, err := configutil.FetchInitConfigurationFromCluster(client, &output.TextPrinter{}, "upgrade", !isControlPlaneNode, false) if err != nil { return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") } diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go index 7bd654ebbb8..92ff3d4db4b 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -28,16 +28,21 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/version" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/printers" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" outputapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" + outputapischeme "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme" + outputapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha2" "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) type planFlags struct { @@ -50,25 +55,188 @@ func newCmdPlan(apf *applyPlanFlags) *cobra.Command { applyPlanFlags: apf, } + outputFlags := newUpgradePlanPrintFlags(output.TextOutput) + cmd := &cobra.Command{ Use: "plan [version] [flags]", Short: "Check which versions are available to upgrade to and validate whether your current cluster is upgradeable. To skip the internet check, pass in the optional [version] parameter", RunE: func(_ *cobra.Command, args []string) error { - return runPlan(flags, args) + printer, err := outputFlags.ToPrinter() + if err != nil { + return err + } + + return runPlan(flags, args, printer) }, } + outputFlags.AddFlags(cmd) + // Register the common flags for apply and plan addApplyPlanFlags(cmd.Flags(), flags.applyPlanFlags) return cmd } +// newComponentUpgradePlan helper creates outputapiv1alpha1.ComponentUpgradePlan object +func newComponentUpgradePlan(name, currentVersion, newVersion string) outputapi.ComponentUpgradePlan { + return outputapi.ComponentUpgradePlan{ + Name: name, + CurrentVersion: currentVersion, + NewVersion: newVersion, + } +} + +// upgradePlanPrintFlags defines printer flag structure for the +// upgrade plan kubeadm command and provides a method +// of retrieving a known printer based on flag values provided. +type upgradePlanPrintFlags struct { + // JSONYamlPrintFlags provides default flags necessary for json/yaml printing. + JSONYamlPrintFlags *upgradePlanJSONYamlPrintFlags + // TextPrintFlags provides default flags necessary for text printing. + TextPrintFlags *upgradePlanTextPrintFlags + // TypeSetterPrinter is an implementation of ResourcePrinter that wraps another printer with types set on the objects + TypeSetterPrinter *printers.TypeSetterPrinter + // OutputFormat contains currently set output format + OutputFormat string +} + +func newUpgradePlanPrintFlags(outputFormat string) *upgradePlanPrintFlags { + return &upgradePlanPrintFlags{ + JSONYamlPrintFlags: &upgradePlanJSONYamlPrintFlags{}, + TextPrintFlags: &upgradePlanTextPrintFlags{}, + TypeSetterPrinter: printers.NewTypeSetter(outputapischeme.Scheme), + OutputFormat: strings.ToLower(outputFormat), + } +} + +// AllowedFormats returns list of allowed output formats +func (pf *upgradePlanPrintFlags) AllowedFormats() []string { + ret := pf.TextPrintFlags.AllowedFormats() + return append(ret, pf.JSONYamlPrintFlags.AllowedFormats()...) +} + +// AddFlags receives a *cobra.Command reference and binds +// flags related to Kubeadm printing to it +func (pf *upgradePlanPrintFlags) AddFlags(cmd *cobra.Command) { + pf.TextPrintFlags.AddFlags(cmd) + pf.JSONYamlPrintFlags.AddFlags(cmd) + cmd.Flags().StringVarP(&pf.OutputFormat, "experimental-output", "o", pf.OutputFormat, fmt.Sprintf("Output format. One of: %s.", strings.Join(pf.AllowedFormats(), "|"))) +} + +// ToPrinter receives an outputFormat and returns a printer capable of +// handling format printing. +// Returns error if the specified outputFormat does not match supported formats. +func (pf *upgradePlanPrintFlags) ToPrinter() (output.Printer, error) { + switch pf.OutputFormat { + case output.TextOutput: + return pf.TextPrintFlags.ToPrinter(pf.OutputFormat) + case output.JSONOutput: + return newUpgradePlanJSONYAMLPrinter(pf.TypeSetterPrinter.WrapToPrinter(pf.JSONYamlPrintFlags.ToPrinter(output.JSONOutput))) + case output.YAMLOutput: + return newUpgradePlanJSONYAMLPrinter(pf.TypeSetterPrinter.WrapToPrinter(pf.JSONYamlPrintFlags.ToPrinter(output.YAMLOutput))) + default: + return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: &pf.OutputFormat, AllowedFormats: pf.AllowedFormats()} + } +} + +type upgradePlanJSONYamlPrintFlags struct { + genericclioptions.JSONYamlPrintFlags +} + +// AllowedFormats returns list of allowed output formats +func (pf *upgradePlanJSONYamlPrintFlags) AllowedFormats() []string { + return []string{output.JSONOutput, output.YAMLOutput} +} + +// upgradePlanJSONYAMLPrinter prints upgrade plan in a JSON or YAML format +type upgradePlanJSONYAMLPrinter struct { + output.ResourcePrinterWrapper + Buffer []outputapiv1alpha2.ComponentUpgradePlan +} + +// newUpgradePlanJSONYAMLPrinter creates new upgradePlanJSONYAMLPrinter object +func newUpgradePlanJSONYAMLPrinter(resourcePrinter printers.ResourcePrinter, err error) (output.Printer, error) { + if err != nil { + return nil, err + } + return &upgradePlanJSONYAMLPrinter{ResourcePrinterWrapper: output.ResourcePrinterWrapper{Printer: resourcePrinter}}, nil +} + +// PrintObj is an implementation of ResourcePrinter.PrintObj that adds object to the buffer +func (p *upgradePlanJSONYAMLPrinter) PrintObj(obj runtime.Object, writer io.Writer) error { + item, ok := obj.(*outputapiv1alpha2.ComponentUpgradePlan) + if !ok { + return fmt.Errorf("expected ComponentUpgradePlan, but got %+v", obj) + } + p.Buffer = append(p.Buffer, *item) + return nil +} + +// Close writes any buffered data and empties list of buffered components +func (p *upgradePlanJSONYAMLPrinter) Close(writer io.Writer) { + plan := &outputapiv1alpha2.UpgradePlan{Components: p.Buffer} + p.Printer.PrintObj(plan, writer) + p.Buffer = []outputapiv1alpha2.ComponentUpgradePlan{} +} + +// upgradePlanTextPrinter prints upgrade plan in a text form +type upgradePlanTextPrinter struct { + output.TextPrinter + columns []string + tabwriter *tabwriter.Writer +} + +// Flush writes any buffered data +func (p *upgradePlanTextPrinter) Flush(writer io.Writer) { + if p.tabwriter != nil { + p.tabwriter.Flush() + p.tabwriter = nil + } +} + +// PrintObj is an implementation of ResourcePrinter.PrintObj for upgrade plan plain text output +func (p *upgradePlanTextPrinter) PrintObj(obj runtime.Object, writer io.Writer) error { + if p.tabwriter == nil { + p.tabwriter = tabwriter.NewWriter(writer, 10, 4, 3, ' ', 0) + // Print header + fmt.Fprintln(p.tabwriter, strings.Join(p.columns, "\t")) + } + + item, ok := obj.(*outputapiv1alpha2.ComponentUpgradePlan) + if !ok { + return fmt.Errorf("expected ComponentUpgradePlan, but got %+v", obj) + } + + // Print item + fmt.Fprintf(p.tabwriter, "%s\t%s\t%s\n", item.Name, item.CurrentVersion, item.NewVersion) + + return nil +} + +// upgradePlanTextPrintFlags provides flags necessary for printing upgrade plan in a text form. +type upgradePlanTextPrintFlags struct{} + +func (pf *upgradePlanTextPrintFlags) AddFlags(cmd *cobra.Command) {} + +// AllowedFormats returns list of allowed output formats +func (pf *upgradePlanTextPrintFlags) AllowedFormats() []string { + return []string{output.TextOutput} +} + +// ToPrinter returns kubeadm printer for the text output format +func (pf *upgradePlanTextPrintFlags) ToPrinter(outputFormat string) (output.Printer, error) { + if outputFormat == output.TextOutput { + return &upgradePlanTextPrinter{columns: []string{"COMPONENT", "CURRENT", "AVAILABLE"}}, nil + } + return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: &outputFormat, AllowedFormats: []string{output.TextOutput}} +} + // runPlan takes care of outputting available versions to upgrade to for the user -func runPlan(flags *planFlags, args []string) error { +func runPlan(flags *planFlags, args []string, printer output.Printer) error { // Start with the basics, verify that the cluster is healthy, build a client and a versionGetter. Never dry-run when planning. klog.V(1).Infoln("[upgrade/plan] verifying health of cluster") klog.V(1).Infoln("[upgrade/plan] retrieving configuration from cluster") - client, versionGetter, cfg, err := enforceRequirements(flags.applyPlanFlags, args, false, false) + client, versionGetter, cfg, err := enforceRequirements(flags.applyPlanFlags, args, false, false, printer) if err != nil { return err } @@ -79,6 +247,7 @@ func runPlan(flags *planFlags, args []string) error { // Compute which upgrade possibilities there are klog.V(1).Infoln("[upgrade/plan] computing upgrade possibilities") + klog.V(1).Infoln("[upgrade/plan] Fetching available versions to upgrade to") availUpgrades, err := upgrade.GetAvailableUpgrades(versionGetter, flags.allowExperimentalUpgrades, flags.allowRCUpgrades, isExternalEtcd, client, constants.GetStaticPodDirectory()) if err != nil { return errors.Wrap(err, "[upgrade/versions] FATAL") @@ -109,24 +278,17 @@ func runPlan(flags *planFlags, args []string) error { // in the human readable output if it did so plan.ConfigVersions = configVersionStates - printUpgradePlan(&up, plan, unstableVersionFlag, isExternalEtcd, os.Stdout) + printUpgradePlan(&up, plan, unstableVersionFlag, isExternalEtcd, os.Stdout, printer) } // Finally, print the component config state table - printComponentConfigVersionStates(configVersionStates, os.Stdout) + printComponentConfigVersionStates(configVersionStates, os.Stdout, printer) + // Add a newline in the end of this output to leave some space to the next output section + klog.V(1).Infoln() return nil } -// newComponentUpgradePlan helper creates outputapi.ComponentUpgradePlan object -func newComponentUpgradePlan(name, currentVersion, newVersion string) outputapi.ComponentUpgradePlan { - return outputapi.ComponentUpgradePlan{ - Name: name, - CurrentVersion: currentVersion, - NewVersion: newVersion, - } -} - // TODO There is currently no way to cleanly output upgrades that involve adding, removing, or changing components // https://github.com/kubernetes/kubeadm/issues/810 was created to track addressing this. func appendDNSComponent(components []outputapi.ComponentUpgradePlan, up *upgrade.Upgrade, name string) []outputapi.ComponentUpgradePlan { @@ -199,14 +361,14 @@ func getComponentConfigVersionStates(cfg *kubeadmapi.ClusterConfiguration, clien } // printUpgradePlan prints a UX-friendly overview of what versions are available to upgrade to -func printUpgradePlan(up *upgrade.Upgrade, plan *outputapi.UpgradePlan, unstableVersionFlag string, isExternalEtcd bool, w io.Writer) { +func printUpgradePlan(up *upgrade.Upgrade, plan *outputapi.UpgradePlan, unstableVersionFlag string, isExternalEtcd bool, w io.Writer, printer output.Printer) { // The tab writer writes to the "real" writer w tabw := tabwriter.NewWriter(w, 10, 4, 3, ' ', 0) // endOfTable helper function flashes table writer endOfTable := func() { tabw.Flush() - fmt.Fprintln(w, "") + printer.Fprintln(w, "") } printHeader := true @@ -217,41 +379,41 @@ func printUpgradePlan(up *upgrade.Upgrade, plan *outputapi.UpgradePlan, unstable continue } else if component.Name == constants.Kubelet { if printManualUpgradeHeader { - fmt.Fprintln(w, "Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':") - fmt.Fprintln(tabw, "COMPONENT\tCURRENT\tTARGET") - fmt.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) + printer.Fprintln(w, "Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':") + printer.Fprintln(tabw, "COMPONENT\tCURRENT\tTARGET") + printer.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) printManualUpgradeHeader = false } else { - fmt.Fprintf(tabw, "%s\t%s\t%s\n", "", component.CurrentVersion, component.NewVersion) + printer.Fprintf(tabw, "%s\t%s\t%s\n", "", component.CurrentVersion, component.NewVersion) } } else { if printHeader { // End of manual upgrades table endOfTable() - fmt.Fprintf(w, "Upgrade to the latest %s:\n", up.Description) - fmt.Fprintln(w, "") - fmt.Fprintln(tabw, "COMPONENT\tCURRENT\tTARGET") + printer.Fprintf(w, "Upgrade to the latest %s:\n", up.Description) + printer.Fprintln(w, "") + printer.Fprintln(tabw, "COMPONENT\tCURRENT\tTARGET") printHeader = false } - fmt.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) + printer.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) } } // End of control plane table endOfTable() //fmt.Fprintln(w, "") - fmt.Fprintln(w, "You can now apply the upgrade by executing the following command:") - fmt.Fprintln(w, "") - fmt.Fprintf(w, "\tkubeadm upgrade apply %s%s\n", up.After.KubeVersion, unstableVersionFlag) - fmt.Fprintln(w, "") + printer.Fprintln(w, "You can now apply the upgrade by executing the following command:") + printer.Fprintln(w, "") + printer.Fprintf(w, "\tkubeadm upgrade apply %s%s\n", up.After.KubeVersion, unstableVersionFlag) + printer.Fprintln(w, "") if up.Before.KubeadmVersion != up.After.KubeadmVersion { - fmt.Fprintf(w, "Note: Before you can perform this upgrade, you have to update kubeadm to %s.\n", up.After.KubeadmVersion) - fmt.Fprintln(w, "") + printer.Fprintf(w, "Note: Before you can perform this upgrade, you have to update kubeadm to %s.\n", up.After.KubeadmVersion) + printer.Fprintln(w, "") } - printLineSeparator(w) + printLineSeparator(w, printer) } // sortedSliceFromStringIntMap returns a slice of the keys in the map sorted alphabetically @@ -278,18 +440,18 @@ func yesOrNo(b bool) string { return "no" } -func printLineSeparator(w io.Writer) { - fmt.Fprintln(w, "_____________________________________________________________________") - fmt.Fprintln(w, "") +func printLineSeparator(w io.Writer, printer output.Printer) { + printer.Fprintln(w, "_____________________________________________________________________") + printer.Fprintln(w, "") } -func printComponentConfigVersionStates(versionStates []outputapi.ComponentConfigVersionState, w io.Writer) { +func printComponentConfigVersionStates(versionStates []outputapi.ComponentConfigVersionState, w io.Writer, printer output.Printer) { if len(versionStates) == 0 { - fmt.Fprintln(w, "No information available on component configs.") + printer.Fprintln(w, "No information available on component configs.") return } - fmt.Fprintln(w, dedent.Dedent(` + printer.Fprintln(w, dedent.Dedent(` The table below shows the current state of component configs as understood by this version of kubeadm. Configs that have a "yes" mark in the "MANUAL UPGRADE REQUIRED" column require manual config upgrade or resetting to kubeadm defaults before a successful upgrade can be performed. The version to manually @@ -297,10 +459,10 @@ func printComponentConfigVersionStates(versionStates []outputapi.ComponentConfig `)) tabw := tabwriter.NewWriter(w, 10, 4, 3, ' ', 0) - fmt.Fprintln(tabw, "API GROUP\tCURRENT VERSION\tPREFERRED VERSION\tMANUAL UPGRADE REQUIRED") + printer.Fprintln(tabw, "API GROUP\tCURRENT VERSION\tPREFERRED VERSION\tMANUAL UPGRADE REQUIRED") for _, state := range versionStates { - fmt.Fprintf(tabw, + printer.Fprintf(tabw, "%s\t%s\t%s\t%s\n", state.Group, strOrDash(state.CurrentVersion), @@ -310,5 +472,5 @@ func printComponentConfigVersionStates(versionStates []outputapi.ComponentConfig } tabw.Flush() - printLineSeparator(w) + printLineSeparator(w, printer) } diff --git a/cmd/kubeadm/app/phases/upgrade/compute.go b/cmd/kubeadm/app/phases/upgrade/compute.go index f46cecef7ab..52008b30c6a 100644 --- a/cmd/kubeadm/app/phases/upgrade/compute.go +++ b/cmd/kubeadm/app/phases/upgrade/compute.go @@ -73,8 +73,6 @@ type ClusterState struct { // GetAvailableUpgrades fetches all versions from the specified VersionGetter and computes which // kinds of upgrades can be performed func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed, externalEtcd bool, client clientset.Interface, manifestsDir string) ([]Upgrade, error) { - fmt.Println("[upgrade] Fetching available versions to upgrade to") - // Collect the upgrades kubeadm can do in this list upgrades := []Upgrade{} @@ -266,9 +264,6 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA } } - // Add a newline in the end of this output to leave some space to the next output section - fmt.Println("") - return upgrades, nil } diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go index 3728c293261..ed9648f61e0 100644 --- a/cmd/kubeadm/app/util/config/cluster.go +++ b/cmd/kubeadm/app/util/config/cluster.go @@ -20,7 +20,6 @@ import ( "context" "crypto/x509" "fmt" - "io" "path/filepath" "strings" @@ -42,12 +41,13 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) // FetchInitConfigurationFromCluster fetches configuration from a ConfigMap in the cluster -func FetchInitConfigurationFromCluster(client clientset.Interface, w io.Writer, logPrefix string, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { - fmt.Fprintf(w, "[%s] Reading configuration from the cluster...\n", logPrefix) - fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -o yaml'\n", logPrefix, metav1.NamespaceSystem, constants.KubeadmConfigConfigMap) +func FetchInitConfigurationFromCluster(client clientset.Interface, printer output.Printer, logPrefix string, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { + printer.Printf("[%s] Reading configuration from the cluster...\n", logPrefix) + printer.Printf("[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -o yaml'\n", logPrefix, metav1.NamespaceSystem, constants.KubeadmConfigConfigMap) // Fetch the actual config from cluster cfg, err := getInitConfigurationFromCluster(constants.KubernetesDir, client, newControlPlane, skipComponentConfigs) diff --git a/cmd/kubeadm/app/util/output/output.go b/cmd/kubeadm/app/util/output/output.go index a9776657784..b99f51eaac7 100644 --- a/cmd/kubeadm/app/util/output/output.go +++ b/cmd/kubeadm/app/util/output/output.go @@ -31,6 +31,12 @@ import ( // TextOutput describes the plain text output const TextOutput = "text" +// JSONOutput describes the JSON output +const JSONOutput = "json" + +// YAMLOutput describes the YAML output +const YAMLOutput = "yaml" + // TextPrintFlags is an interface to handle custom text output type TextPrintFlags interface { ToPrinter(outputFormat string) (Printer, error) @@ -44,7 +50,7 @@ type PrintFlags struct { JSONYamlPrintFlags *genericclioptions.JSONYamlPrintFlags // KubeTemplatePrintFlags composes print flags that provide both a JSONPath and a go-template printer. KubeTemplatePrintFlags *genericclioptions.KubeTemplatePrintFlags - // JSONYamlPrintFlags provides default flags necessary for kubeadm text printing. + // TextPrintFlags provides default flags necessary for kubeadm text printing. TextPrintFlags TextPrintFlags // TypeSetterPrinter is an implementation of ResourcePrinter that wraps another printer with types set on the objects TypeSetterPrinter *printers.TypeSetterPrinter @@ -133,13 +139,27 @@ func NewOutputFlags(textPrintFlags TextPrintFlags) *PrintFlags { type Printer interface { PrintObj(obj runtime.Object, writer io.Writer) error Fprintf(writer io.Writer, format string, args ...interface{}) (n int, err error) + Fprintln(writer io.Writer, args ...interface{}) (n int, err error) Printf(format string, args ...interface{}) (n int, err error) + + Flush(writer io.Writer) + Close(writer io.Writer) } // TextPrinter implements Printer interface for generic text output type TextPrinter struct { } +// Flush writes any buffered data +func (tp *TextPrinter) Flush(writer io.Writer) { + return +} + +// Close flushes any buffered data and closes the printer +func (tp *TextPrinter) Close(writer io.Writer) { + return +} + // PrintObj is an implementation of ResourcePrinter.PrintObj that prints object func (tp *TextPrinter) PrintObj(obj runtime.Object, writer io.Writer) error { _, err := fmt.Fprintf(writer, "%+v\n", obj) @@ -151,6 +171,11 @@ func (tp *TextPrinter) Fprintf(writer io.Writer, format string, args ...interfac return fmt.Fprintf(writer, format, args...) } +// Fprintln is a wrapper around fmt.Fprintln +func (tp *TextPrinter) Fprintln(writer io.Writer, args ...interface{}) (n int, err error) { + return fmt.Fprintln(writer, args...) +} + // Printf is a wrapper around fmt.Printf func (tp *TextPrinter) Printf(format string, args ...interface{}) (n int, err error) { return fmt.Printf(format, args...) @@ -169,6 +194,16 @@ func NewResourcePrinterWrapper(resourcePrinter printers.ResourcePrinter, err err return &ResourcePrinterWrapper{Printer: resourcePrinter}, nil } +// Flush writes any buffered data +func (rpw *ResourcePrinterWrapper) Flush(writer io.Writer) { + return +} + +// Close flushes any buffered data and closes the printer +func (rpw *ResourcePrinterWrapper) Close(writer io.Writer) { + return +} + // PrintObj is an implementation of ResourcePrinter.PrintObj that calls underlying printer API func (rpw *ResourcePrinterWrapper) PrintObj(obj runtime.Object, writer io.Writer) error { return rpw.Printer.PrintObj(obj, writer) @@ -181,6 +216,13 @@ func (rpw *ResourcePrinterWrapper) Fprintf(writer io.Writer, format string, args return 0, nil } +// Fprintln is an empty method to satisfy Printer interface +// and silent info printing for structured output +// This method is usually redefined for the text output +func (rpw *ResourcePrinterWrapper) Fprintln(writer io.Writer, args ...interface{}) (n int, err error) { + return 0, nil +} + // Printf is an empty method to satisfy Printer interface // and silent info printing for structured output // This method is usually redefined for the text output From 79ecd60208afbc40b1426ed2a3955a2e9d49cc5e Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Wed, 2 Mar 2022 19:28:42 +0800 Subject: [PATCH 2/3] bugfix(kubeadm): Fix UT and log optmize --- cmd/kubeadm/app/apis/output/types.go | 2 +- cmd/kubeadm/app/apis/output/v1alpha2/types.go | 2 +- .../v1alpha2/zz_generated.conversion.go | 4 +- .../output/v1alpha2/zz_generated.deepcopy.go | 10 +- .../app/apis/output/zz_generated.deepcopy.go | 10 +- cmd/kubeadm/app/cmd/upgrade/common.go | 26 +-- cmd/kubeadm/app/cmd/upgrade/common_test.go | 2 +- cmd/kubeadm/app/cmd/upgrade/plan.go | 81 ++++----- cmd/kubeadm/app/cmd/upgrade/plan_test.go | 170 +++++++++++++++++- cmd/kubeadm/app/componentconfigs/configset.go | 12 +- .../app/componentconfigs/fakeconfig_test.go | 10 +- cmd/kubeadm/app/phases/upgrade/compute.go | 16 +- .../app/phases/upgrade/compute_test.go | 3 +- 13 files changed, 262 insertions(+), 86 deletions(-) diff --git a/cmd/kubeadm/app/apis/output/types.go b/cmd/kubeadm/app/apis/output/types.go index 9dda9491425..d0229a28e04 100644 --- a/cmd/kubeadm/app/apis/output/types.go +++ b/cmd/kubeadm/app/apis/output/types.go @@ -78,7 +78,7 @@ type ComponentConfigVersionState struct { type UpgradePlan struct { metav1.TypeMeta - Components []ComponentUpgradePlan + Components []*ComponentUpgradePlan ConfigVersions []ComponentConfigVersionState } diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/types.go b/cmd/kubeadm/app/apis/output/v1alpha2/types.go index 529c029dd59..7938849a356 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/types.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/types.go @@ -78,7 +78,7 @@ type ComponentConfigVersionState struct { type UpgradePlan struct { metav1.TypeMeta - Components []ComponentUpgradePlan `json:"components"` + Components []*ComponentUpgradePlan `json:"components"` ConfigVersions []ComponentConfigVersionState `json:"configVersions"` } diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go index 5893ba6c309..005b39773fb 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go @@ -180,7 +180,7 @@ func Convert_output_Images_To_v1alpha2_Images(in *output.Images, out *Images, s } func autoConvert_v1alpha2_UpgradePlan_To_output_UpgradePlan(in *UpgradePlan, out *output.UpgradePlan, s conversion.Scope) error { - out.Components = *(*[]output.ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) + out.Components = *(*[]*output.ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) out.ConfigVersions = *(*[]output.ComponentConfigVersionState)(unsafe.Pointer(&in.ConfigVersions)) return nil } @@ -191,7 +191,7 @@ func Convert_v1alpha2_UpgradePlan_To_output_UpgradePlan(in *UpgradePlan, out *ou } func autoConvert_output_UpgradePlan_To_v1alpha2_UpgradePlan(in *output.UpgradePlan, out *UpgradePlan, s conversion.Scope) error { - out.Components = *(*[]ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) + out.Components = *(*[]*ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) out.ConfigVersions = *(*[]ComponentConfigVersionState)(unsafe.Pointer(&in.ConfigVersions)) return nil } diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go index 73e8c32def2..dec35b0d537 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go @@ -128,8 +128,14 @@ func (in *UpgradePlan) DeepCopyInto(out *UpgradePlan) { out.TypeMeta = in.TypeMeta if in.Components != nil { in, out := &in.Components, &out.Components - *out = make([]ComponentUpgradePlan, len(*in)) - copy(*out, *in) + *out = make([]*ComponentUpgradePlan, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ComponentUpgradePlan) + **out = **in + } + } } if in.ConfigVersions != nil { in, out := &in.ConfigVersions, &out.ConfigVersions diff --git a/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go index 8d553d35c15..d09de881e1c 100644 --- a/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go @@ -128,8 +128,14 @@ func (in *UpgradePlan) DeepCopyInto(out *UpgradePlan) { out.TypeMeta = in.TypeMeta if in.Components != nil { in, out := &in.Components, &out.Components - *out = make([]ComponentUpgradePlan, len(*in)) - copy(*out, *in) + *out = make([]*ComponentUpgradePlan, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ComponentUpgradePlan) + **out = **in + } + } } if in.ConfigVersions != nil { in, out := &in.ConfigVersions, &out.ConfigVersions diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 8fc9358e486..7c8d4e5fb3a 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -135,14 +135,14 @@ func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgr cfg, legacyReconfigure, err := loadConfig(flags.cfgPath, client, !upgradeApply, printer) if err != nil { if apierrors.IsNotFound(err) { - fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) - fmt.Println("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.") - fmt.Println("") - fmt.Println("[upgrade/config] Next steps:") - fmt.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your control-plane.\n") - fmt.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your control-plane.\n") - fmt.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n") - fmt.Println("") + printer.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) + printer.Printf("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.\n") + printer.Printf("\n") + printer.Printf("[upgrade/config] Next steps:\n") + printer.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your control-plane.\n") + printer.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your control-plane.\n") + printer.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n") + printer.Printf("\n") err = errors.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) } return nil, nil, nil, errors.Wrap(err, "[upgrade/config] FATAL") @@ -203,14 +203,14 @@ func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgr // Check if feature gate flags used in the cluster are consistent with the set of features currently supported by kubeadm if msg := features.CheckDeprecatedFlags(&features.InitFeatureGates, cfg.FeatureGates); len(msg) > 0 { for _, m := range msg { - fmt.Printf("[upgrade/config] %s\n", m) + printer.Printf("[upgrade/config] %s\n", m) } return nil, nil, nil, errors.New("[upgrade/config] FATAL. Unable to upgrade a cluster using deprecated feature-gate flags. Please see the release notes") } // If the user told us to print this information out; do it! if flags.printConfig { - printConfiguration(&cfg.ClusterConfiguration, os.Stdout) + printConfiguration(&cfg.ClusterConfiguration, os.Stdout, printer) } // Use a real version getter interface that queries the API server, the kubeadm client and the Kubernetes CI system for latest versions @@ -218,7 +218,7 @@ func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgr } // printConfiguration prints the external version of the API to yaml -func printConfiguration(clustercfg *kubeadmapi.ClusterConfiguration, w io.Writer) { +func printConfiguration(clustercfg *kubeadmapi.ClusterConfiguration, w io.Writer, printer output.Printer) { // Short-circuit if cfg is nil, so we can safely get the value of the pointer below if clustercfg == nil { return @@ -226,11 +226,11 @@ func printConfiguration(clustercfg *kubeadmapi.ClusterConfiguration, w io.Writer cfgYaml, err := configutil.MarshalKubeadmConfigObject(clustercfg) if err == nil { - fmt.Fprintln(w, "[upgrade/config] Configuration used:") + printer.Fprintln(w, "[upgrade/config] Configuration used:") scanner := bufio.NewScanner(bytes.NewReader(cfgYaml)) for scanner.Scan() { - fmt.Fprintf(w, "\t%s\n", scanner.Text()) + printer.Fprintf(w, "\t%s\n", scanner.Text()) } } } diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go index 357148e9afc..091b80f2f7b 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/common_test.go @@ -139,7 +139,7 @@ func TestPrintConfiguration(t *testing.T) { for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { rt.buf = bytes.NewBufferString("") - printConfiguration(rt.cfg, rt.buf) + printConfiguration(rt.cfg, rt.buf, &output.TextPrinter{}) actualBytes := rt.buf.Bytes() if !bytes.Equal(actualBytes, rt.expectedBytes) { t.Errorf( diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go index 92ff3d4db4b..b707d1d96d2 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -35,7 +35,6 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - outputapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" outputapischeme "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme" outputapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha2" "k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs" @@ -77,9 +76,9 @@ func newCmdPlan(apf *applyPlanFlags) *cobra.Command { return cmd } -// newComponentUpgradePlan helper creates outputapiv1alpha1.ComponentUpgradePlan object -func newComponentUpgradePlan(name, currentVersion, newVersion string) outputapi.ComponentUpgradePlan { - return outputapi.ComponentUpgradePlan{ +// newComponentUpgradePlan helper creates outputapiv1alpha2.ComponentUpgradePlan object +func newComponentUpgradePlan(name, currentVersion, newVersion string) *outputapiv1alpha2.ComponentUpgradePlan { + return &outputapiv1alpha2.ComponentUpgradePlan{ Name: name, CurrentVersion: currentVersion, NewVersion: newVersion, @@ -151,7 +150,7 @@ func (pf *upgradePlanJSONYamlPrintFlags) AllowedFormats() []string { // upgradePlanJSONYAMLPrinter prints upgrade plan in a JSON or YAML format type upgradePlanJSONYAMLPrinter struct { output.ResourcePrinterWrapper - Buffer []outputapiv1alpha2.ComponentUpgradePlan + Buffer []*outputapiv1alpha2.ComponentUpgradePlan } // newUpgradePlanJSONYAMLPrinter creates new upgradePlanJSONYAMLPrinter object @@ -168,15 +167,16 @@ func (p *upgradePlanJSONYAMLPrinter) PrintObj(obj runtime.Object, writer io.Writ if !ok { return fmt.Errorf("expected ComponentUpgradePlan, but got %+v", obj) } - p.Buffer = append(p.Buffer, *item) + p.Buffer = append(p.Buffer, item) return nil } // Close writes any buffered data and empties list of buffered components func (p *upgradePlanJSONYAMLPrinter) Close(writer io.Writer) { plan := &outputapiv1alpha2.UpgradePlan{Components: p.Buffer} + // p.ResourcePrinterWrapper.Printer.PrintObj(plan, writer) p.Printer.PrintObj(plan, writer) - p.Buffer = []outputapiv1alpha2.ComponentUpgradePlan{} + p.Buffer = []*outputapiv1alpha2.ComponentUpgradePlan{} } // upgradePlanTextPrinter prints upgrade plan in a text form @@ -226,9 +226,9 @@ func (pf *upgradePlanTextPrintFlags) AllowedFormats() []string { // ToPrinter returns kubeadm printer for the text output format func (pf *upgradePlanTextPrintFlags) ToPrinter(outputFormat string) (output.Printer, error) { if outputFormat == output.TextOutput { - return &upgradePlanTextPrinter{columns: []string{"COMPONENT", "CURRENT", "AVAILABLE"}}, nil + return &upgradePlanTextPrinter{columns: []string{"COMPONENT", "CURRENT", "TARGET"}}, nil } - return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: &outputFormat, AllowedFormats: []string{output.TextOutput}} + return nil, genericclioptions.NoCompatiblePrinterError{OutputFormat: &outputFormat, AllowedFormats: []string{output.JSONOutput, output.YAMLOutput, output.TextOutput}} } // runPlan takes care of outputting available versions to upgrade to for the user @@ -247,8 +247,7 @@ func runPlan(flags *planFlags, args []string, printer output.Printer) error { // Compute which upgrade possibilities there are klog.V(1).Infoln("[upgrade/plan] computing upgrade possibilities") - klog.V(1).Infoln("[upgrade/plan] Fetching available versions to upgrade to") - availUpgrades, err := upgrade.GetAvailableUpgrades(versionGetter, flags.allowExperimentalUpgrades, flags.allowRCUpgrades, isExternalEtcd, client, constants.GetStaticPodDirectory()) + availUpgrades, err := upgrade.GetAvailableUpgrades(versionGetter, flags.allowExperimentalUpgrades, flags.allowRCUpgrades, isExternalEtcd, client, constants.GetStaticPodDirectory(), printer) if err != nil { return errors.Wrap(err, "[upgrade/versions] FATAL") } @@ -291,7 +290,7 @@ func runPlan(flags *planFlags, args []string, printer output.Printer) error { // TODO There is currently no way to cleanly output upgrades that involve adding, removing, or changing components // https://github.com/kubernetes/kubeadm/issues/810 was created to track addressing this. -func appendDNSComponent(components []outputapi.ComponentUpgradePlan, up *upgrade.Upgrade, name string) []outputapi.ComponentUpgradePlan { +func appendDNSComponent(components []*outputapiv1alpha2.ComponentUpgradePlan, up *upgrade.Upgrade, name string) []*outputapiv1alpha2.ComponentUpgradePlan { beforeVersion := up.Before.DNSVersion afterVersion := up.After.DNSVersion @@ -302,7 +301,7 @@ func appendDNSComponent(components []outputapi.ComponentUpgradePlan, up *upgrade } // genUpgradePlan generates output-friendly upgrade plan out of upgrade.Upgrade structure -func genUpgradePlan(up *upgrade.Upgrade, isExternalEtcd bool) (*outputapi.UpgradePlan, string, error) { +func genUpgradePlan(up *upgrade.Upgrade, isExternalEtcd bool) (*outputapiv1alpha2.UpgradePlan, string, error) { newK8sVersion, err := version.ParseSemantic(up.After.KubeVersion) if err != nil { return nil, "", errors.Wrapf(err, "Unable to parse normalized version %q as a semantic version", up.After.KubeVersion) @@ -317,7 +316,7 @@ func genUpgradePlan(up *upgrade.Upgrade, isExternalEtcd bool) (*outputapi.Upgrad } } - components := []outputapi.ComponentUpgradePlan{} + components := []*outputapiv1alpha2.ComponentUpgradePlan{} if up.CanUpgradeKubelets() { // The map is of the form :. Here all the keys are put into a slice and sorted @@ -339,10 +338,10 @@ func genUpgradePlan(up *upgrade.Upgrade, isExternalEtcd bool) (*outputapi.Upgrad components = append(components, newComponentUpgradePlan(constants.Etcd, up.Before.EtcdVersion, up.After.EtcdVersion)) } - return &outputapi.UpgradePlan{Components: components}, unstableVersionFlag, nil + return &outputapiv1alpha2.UpgradePlan{Components: components}, unstableVersionFlag, nil } -func getComponentConfigVersionStates(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, cfgPath string) ([]outputapi.ComponentConfigVersionState, error) { +func getComponentConfigVersionStates(cfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, cfgPath string) ([]outputapiv1alpha2.ComponentConfigVersionState, error) { docmap := kubeadmapi.DocumentMap{} if cfgPath != "" { @@ -361,16 +360,7 @@ func getComponentConfigVersionStates(cfg *kubeadmapi.ClusterConfiguration, clien } // printUpgradePlan prints a UX-friendly overview of what versions are available to upgrade to -func printUpgradePlan(up *upgrade.Upgrade, plan *outputapi.UpgradePlan, unstableVersionFlag string, isExternalEtcd bool, w io.Writer, printer output.Printer) { - // The tab writer writes to the "real" writer w - tabw := tabwriter.NewWriter(w, 10, 4, 3, ' ', 0) - - // endOfTable helper function flashes table writer - endOfTable := func() { - tabw.Flush() - printer.Fprintln(w, "") - } - +func printUpgradePlan(up *upgrade.Upgrade, plan *outputapiv1alpha2.UpgradePlan, unstableVersionFlag string, isExternalEtcd bool, writer io.Writer, printer output.Printer) { printHeader := true printManualUpgradeHeader := true for _, component := range plan.Components { @@ -379,41 +369,40 @@ func printUpgradePlan(up *upgrade.Upgrade, plan *outputapi.UpgradePlan, unstable continue } else if component.Name == constants.Kubelet { if printManualUpgradeHeader { - printer.Fprintln(w, "Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':") - printer.Fprintln(tabw, "COMPONENT\tCURRENT\tTARGET") - printer.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) + printer.Fprintln(writer, "Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':") + printer.PrintObj(newComponentUpgradePlan(component.Name, component.CurrentVersion, component.NewVersion), writer) printManualUpgradeHeader = false } else { - printer.Fprintf(tabw, "%s\t%s\t%s\n", "", component.CurrentVersion, component.NewVersion) + printer.PrintObj(newComponentUpgradePlan("", component.CurrentVersion, component.NewVersion), writer) } } else { if printHeader { // End of manual upgrades table - endOfTable() + printer.Flush(writer) - printer.Fprintf(w, "Upgrade to the latest %s:\n", up.Description) - printer.Fprintln(w, "") - printer.Fprintln(tabw, "COMPONENT\tCURRENT\tTARGET") + printer.Fprintln(writer, "") + printer.Fprintf(writer, "Upgrade to the latest %s:\n", up.Description) + printer.Fprintln(writer, "") printHeader = false } - printer.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) + printer.PrintObj(newComponentUpgradePlan(component.Name, component.CurrentVersion, component.NewVersion), writer) } } - // End of control plane table - endOfTable() + printer.Flush(writer) + printer.Fprintln(writer, "") - //fmt.Fprintln(w, "") - printer.Fprintln(w, "You can now apply the upgrade by executing the following command:") - printer.Fprintln(w, "") - printer.Fprintf(w, "\tkubeadm upgrade apply %s%s\n", up.After.KubeVersion, unstableVersionFlag) - printer.Fprintln(w, "") + printer.Fprintln(writer, "You can now apply the upgrade by executing the following command:") + printer.Fprintln(writer, "") + printer.Fprintf(writer, "\tkubeadm upgrade apply %s%s\n", up.After.KubeVersion, unstableVersionFlag) + printer.Fprintln(writer, "") if up.Before.KubeadmVersion != up.After.KubeadmVersion { - printer.Fprintf(w, "Note: Before you can perform this upgrade, you have to update kubeadm to %s.\n", up.After.KubeadmVersion) - printer.Fprintln(w, "") + printer.Fprintf(writer, "Note: Before you can perform this upgrade, you have to update kubeadm to %s.\n", up.After.KubeadmVersion) + printer.Fprintln(writer, "") } - printLineSeparator(w, printer) + printLineSeparator(writer, printer) + printer.Close(writer) } // sortedSliceFromStringIntMap returns a slice of the keys in the map sorted alphabetically @@ -445,7 +434,7 @@ func printLineSeparator(w io.Writer, printer output.Printer) { printer.Fprintln(w, "") } -func printComponentConfigVersionStates(versionStates []outputapi.ComponentConfigVersionState, w io.Writer, printer output.Printer) { +func printComponentConfigVersionStates(versionStates []outputapiv1alpha2.ComponentConfigVersionState, w io.Writer, printer output.Printer) { if len(versionStates) == 0 { printer.Fprintln(w, "No information available on component configs.") return diff --git a/cmd/kubeadm/app/cmd/upgrade/plan_test.go b/cmd/kubeadm/app/cmd/upgrade/plan_test.go index 4ed26780e42..82b489c8e88 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan_test.go @@ -22,6 +22,7 @@ import ( "testing" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) func TestSortedSliceFromStringIntMap(t *testing.T) { @@ -441,13 +442,19 @@ _____________________________________________________________________ for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { rt.buf = bytes.NewBufferString("") + outputFlags := newUpgradePlanPrintFlags(output.TextOutput) + printer, err := outputFlags.ToPrinter() + if err != nil { + t.Errorf("failed ToPrinter, err: %+v", err) + } + // Generate and print upgrade plans for _, up := range rt.upgrades { plan, unstableVersionFlag, err := genUpgradePlan(&up, rt.externalEtcd) if err != nil { t.Errorf("failed genUpgradePlan, err: %+v", err) } - printUpgradePlan(&up, plan, unstableVersionFlag, rt.externalEtcd, rt.buf) + printUpgradePlan(&up, plan, unstableVersionFlag, rt.externalEtcd, rt.buf, printer) } actualBytes := rt.buf.Bytes() if !bytes.Equal(actualBytes, rt.expectedBytes) { @@ -460,3 +467,164 @@ _____________________________________________________________________ }) } } + +func TestPrintAvailableUpgradesStructured(t *testing.T) { + upgrades := []upgrade.Upgrade{ + { + Description: "version in the v1.8 series", + Before: upgrade.ClusterState{ + KubeVersion: "v1.8.1", + KubeletVersions: map[string]uint16{ + "v1.8.1": 1, + }, + KubeadmVersion: "v1.8.2", + DNSVersion: "1.14.5", + EtcdVersion: "3.0.17", + }, + After: upgrade.ClusterState{ + KubeVersion: "v1.8.3", + KubeadmVersion: "v1.8.3", + DNSVersion: "1.14.5", + EtcdVersion: "3.0.17", + }, + }, + } + + var tests = []struct { + name string + outputFormat string + buf *bytes.Buffer + expected string + externalEtcd bool + }{ + { + name: "JSON output", + outputFormat: "json", + expected: `{ + "kind": "UpgradePlan", + "apiVersion": "output.kubeadm.k8s.io/v1alpha2", + "components": [ + { + "name": "kubelet", + "currentVersion": "1 x v1.8.1", + "newVersion": "v1.8.3" + }, + { + "name": "kube-apiserver", + "currentVersion": "v1.8.1", + "newVersion": "v1.8.3" + }, + { + "name": "kube-controller-manager", + "currentVersion": "v1.8.1", + "newVersion": "v1.8.3" + }, + { + "name": "kube-scheduler", + "currentVersion": "v1.8.1", + "newVersion": "v1.8.3" + }, + { + "name": "kube-proxy", + "currentVersion": "v1.8.1", + "newVersion": "v1.8.3" + }, + { + "name": "CoreDNS", + "currentVersion": "1.14.5", + "newVersion": "1.14.5" + }, + { + "name": "etcd", + "currentVersion": "3.0.17", + "newVersion": "3.0.17" + } + ], + "configVersions": null +} +`, + }, + { + name: "YAML output", + outputFormat: "yaml", + expected: `apiVersion: output.kubeadm.k8s.io/v1alpha2 +components: +- currentVersion: 1 x v1.8.1 + name: kubelet + newVersion: v1.8.3 +- currentVersion: v1.8.1 + name: kube-apiserver + newVersion: v1.8.3 +- currentVersion: v1.8.1 + name: kube-controller-manager + newVersion: v1.8.3 +- currentVersion: v1.8.1 + name: kube-scheduler + newVersion: v1.8.3 +- currentVersion: v1.8.1 + name: kube-proxy + newVersion: v1.8.3 +- currentVersion: 1.14.5 + name: CoreDNS + newVersion: 1.14.5 +- currentVersion: 3.0.17 + name: etcd + newVersion: 3.0.17 +configVersions: null +kind: UpgradePlan +`, + }, + { + name: "Text output", + outputFormat: "Text", + expected: `Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': +COMPONENT CURRENT TARGET +kubelet 1 x v1.8.1 v1.8.3 + +Upgrade to the latest version in the v1.8 series: + +COMPONENT CURRENT TARGET +kube-apiserver v1.8.1 v1.8.3 +kube-controller-manager v1.8.1 v1.8.3 +kube-scheduler v1.8.1 v1.8.3 +kube-proxy v1.8.1 v1.8.3 +CoreDNS 1.14.5 1.14.5 +etcd 3.0.17 3.0.17 + +You can now apply the upgrade by executing the following command: + + kubeadm upgrade apply v1.8.3 + +Note: Before you can perform this upgrade, you have to update kubeadm to v1.8.3. + +_____________________________________________________________________ + +`, + }, + } + + for _, rt := range tests { + t.Run(rt.name, func(t *testing.T) { + rt.buf = bytes.NewBufferString("") + outputFlags := newUpgradePlanPrintFlags(rt.outputFormat) + printer, err := outputFlags.ToPrinter() + if err != nil { + t.Errorf("failed ToPrinter, err: %+v", err) + } + + // Generate and print upgrade plans + for _, up := range upgrades { + plan, unstableVersionFlag, err := genUpgradePlan(&up, rt.externalEtcd) + if err != nil { + t.Errorf("failed genUpgradePlan, err: %+v", err) + } + printUpgradePlan(&up, plan, unstableVersionFlag, rt.externalEtcd, rt.buf, printer) + } + + actual := rt.buf.String() + if actual != rt.expected { + t.Errorf("failed PrintAvailableUpgrades:\n\nexpected:\n%s\n\nactual:\n%s", rt.expected, actual) + } + }) + } +} diff --git a/cmd/kubeadm/app/componentconfigs/configset.go b/cmd/kubeadm/app/componentconfigs/configset.go index 55cc2a3aa79..3ad5cfad108 100644 --- a/cmd/kubeadm/app/componentconfigs/configset.go +++ b/cmd/kubeadm/app/componentconfigs/configset.go @@ -26,9 +26,9 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + outputapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha2" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" - "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict" @@ -289,7 +289,7 @@ func FetchFromClusterWithLocalOverwrites(clusterCfg *kubeadmapi.ClusterConfigura // GetVersionStates returns a slice of ComponentConfigVersionState structs // describing all supported component config groups that were identified on the cluster -func GetVersionStates(clusterCfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, docmap kubeadmapi.DocumentMap) ([]output.ComponentConfigVersionState, error) { +func GetVersionStates(clusterCfg *kubeadmapi.ClusterConfiguration, client clientset.Interface, docmap kubeadmapi.DocumentMap) ([]outputapiv1alpha2.ComponentConfigVersionState, error) { // We don't want to modify clusterCfg so we make a working deep copy of it. // Also, we don't want the defaulted component configs so we get rid of them. scratchClusterCfg := clusterCfg.DeepCopy() @@ -308,13 +308,13 @@ func GetVersionStates(clusterCfg *kubeadmapi.ClusterConfiguration, client client } } - results := []output.ComponentConfigVersionState{} + results := []outputapiv1alpha2.ComponentConfigVersionState{} for _, handler := range known { group := handler.GroupVersion.Group if vererr, ok := multipleVerErrs[group]; ok { // If there is an UnsupportedConfigVersionError then we are dealing with a case where the config was user // supplied and requires manual upgrade - results = append(results, output.ComponentConfigVersionState{ + results = append(results, outputapiv1alpha2.ComponentConfigVersionState{ Group: group, CurrentVersion: vererr.OldVersion.Version, PreferredVersion: vererr.CurrentVersion.Version, @@ -322,7 +322,7 @@ func GetVersionStates(clusterCfg *kubeadmapi.ClusterConfiguration, client client }) } else if _, ok := scratchClusterCfg.ComponentConfigs[group]; ok { // Normally loaded component config. No manual upgrade required on behalf of users. - results = append(results, output.ComponentConfigVersionState{ + results = append(results, outputapiv1alpha2.ComponentConfigVersionState{ Group: group, CurrentVersion: handler.GroupVersion.Version, // Currently kubeadm supports only one version per API PreferredVersion: handler.GroupVersion.Version, // group so we can get away with these being the same @@ -331,7 +331,7 @@ func GetVersionStates(clusterCfg *kubeadmapi.ClusterConfiguration, client client // This config was either not present (user did not install an addon) or the config was unsupported kubeadm // generated one and is therefore skipped so we can automatically re-generate it (no action required on // behalf of the user). - results = append(results, output.ComponentConfigVersionState{ + results = append(results, outputapiv1alpha2.ComponentConfigVersionState{ Group: group, PreferredVersion: handler.GroupVersion.Version, }) diff --git a/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go b/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go index 18e78d54f59..df6b3fff383 100644 --- a/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go +++ b/cmd/kubeadm/app/componentconfigs/fakeconfig_test.go @@ -35,7 +35,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" - outputapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/output" + outputapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha2" "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -615,12 +615,12 @@ func TestFetchFromClusterWithLocalOverwrites(t *testing.T) { func TestGetVersionStates(t *testing.T) { fakeKnownContext(func() { - versionStateCurrent := outputapi.ComponentConfigVersionState{ + versionStateCurrent := outputapiv1alpha2.ComponentConfigVersionState{ Group: kubeadmapiv1.GroupName, CurrentVersion: currentClusterConfigVersion, PreferredVersion: currentClusterConfigVersion, } - versionStateOld := outputapi.ComponentConfigVersionState{ + versionStateOld := outputapiv1alpha2.ComponentConfigVersionState{ Group: kubeadmapiv1.GroupName, CurrentVersion: oldClusterConfigVersion, PreferredVersion: currentClusterConfigVersion, @@ -631,7 +631,7 @@ func TestGetVersionStates(t *testing.T) { desc string obj runtime.Object config string - expected outputapi.ComponentConfigVersionState + expected outputapiv1alpha2.ComponentConfigVersionState }{ { desc: "appropriate cluster object without overwrite", @@ -687,7 +687,7 @@ func TestGetVersionStates(t *testing.T) { { desc: "old signed config without an overwrite", obj: testClusterConfigMap(oldFooClusterConfig, true), - expected: outputapi.ComponentConfigVersionState{ + expected: outputapiv1alpha2.ComponentConfigVersionState{ Group: kubeadmapiv1.GroupName, CurrentVersion: "", // The config is treated as if it's missing PreferredVersion: currentClusterConfigVersion, diff --git a/cmd/kubeadm/app/phases/upgrade/compute.go b/cmd/kubeadm/app/phases/upgrade/compute.go index 52008b30c6a..8a54a1081a0 100644 --- a/cmd/kubeadm/app/phases/upgrade/compute.go +++ b/cmd/kubeadm/app/phases/upgrade/compute.go @@ -26,6 +26,7 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/dns" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) // Upgrade defines an upgrade possibility to upgrade from a current version to a new one @@ -72,7 +73,9 @@ type ClusterState struct { // GetAvailableUpgrades fetches all versions from the specified VersionGetter and computes which // kinds of upgrades can be performed -func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed, externalEtcd bool, client clientset.Interface, manifestsDir string) ([]Upgrade, error) { +func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed, externalEtcd bool, client clientset.Interface, manifestsDir string, printer output.Printer) ([]Upgrade, error) { + printer.Printf("[upgrade] Fetching available versions to upgrade to\n") + // Collect the upgrades kubeadm can do in this list upgrades := []Upgrade{} @@ -81,14 +84,14 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA if err != nil { return upgrades, err } - fmt.Printf("[upgrade/versions] Cluster version: %s\n", clusterVersionStr) + printer.Printf("[upgrade/versions] Cluster version: %s\n", clusterVersionStr) // Get current kubeadm CLI version kubeadmVersionStr, kubeadmVersion, err := versionGetterImpl.KubeadmVersion() if err != nil { return upgrades, err } - fmt.Printf("[upgrade/versions] kubeadm version: %s\n", kubeadmVersionStr) + printer.Printf("[upgrade/versions] kubeadm version: %s\n", kubeadmVersionStr) // Get and output the current latest stable version stableVersionStr, stableVersion, err := versionGetterImpl.VersionFromCILabel("stable", "stable version") @@ -97,7 +100,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA klog.Warningf("[upgrade/versions] WARNING: Falling back to current kubeadm version as latest stable version") stableVersionStr, stableVersion = kubeadmVersionStr, kubeadmVersion } else { - fmt.Printf("[upgrade/versions] Target version: %s\n", stableVersionStr) + printer.Printf("[upgrade/versions] Target version: %s\n", stableVersionStr) } // Get the kubelet versions in the cluster @@ -146,7 +149,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA if err != nil { klog.Warningf("[upgrade/versions] WARNING: %v\n", err) } else { - fmt.Printf("[upgrade/versions] Latest %s: %s\n", description, patchVersionStr) + printer.Printf("[upgrade/versions] Latest %s: %s\n", description, patchVersionStr) // Check if a minor version upgrade is possible when a patch release exists // It's only possible if the latest patch version is higher than the current patch version @@ -264,6 +267,9 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA } } + // Add a newline in the end of this output to leave some space to the next output section + printer.Printf("\n") + return upgrades, nil } diff --git a/cmd/kubeadm/app/phases/upgrade/compute_test.go b/cmd/kubeadm/app/phases/upgrade/compute_test.go index 81444448964..aef4af9a361 100644 --- a/cmd/kubeadm/app/phases/upgrade/compute_test.go +++ b/cmd/kubeadm/app/phases/upgrade/compute_test.go @@ -31,6 +31,7 @@ import ( clientsetfake "k8s.io/client-go/kubernetes/fake" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) type fakeVersionGetter struct { @@ -652,7 +653,7 @@ func TestGetAvailableUpgrades(t *testing.T) { t.Fatalf("Unable to create test static pod manifest: %v", err) } - actualUpgrades, actualErr := GetAvailableUpgrades(rt.vg, rt.allowExperimental, rt.allowRCs, rt.externalEtcd, client, manifestsDir) + actualUpgrades, actualErr := GetAvailableUpgrades(rt.vg, rt.allowExperimental, rt.allowRCs, rt.externalEtcd, client, manifestsDir, &output.TextPrinter{}) if !reflect.DeepEqual(actualUpgrades, rt.expectedUpgrades) { t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades: %v\n\tgot: %v", rt.expectedUpgrades, actualUpgrades) } From 22fb3be96d31583cf6faba470a085c01135e733f Mon Sep 17 00:00:00 2001 From: Paco Xu Date: Thu, 17 Mar 2022 11:49:08 +0800 Subject: [PATCH 3/3] cleanup: kubeadm upgrade plan supports json/yaml output Co-authored-by: Lubomir I. Ivanov --- cmd/kubeadm/app/apis/output/types.go | 2 +- cmd/kubeadm/app/apis/output/v1alpha2/types.go | 2 +- .../v1alpha2/zz_generated.conversion.go | 4 +- .../output/v1alpha2/zz_generated.deepcopy.go | 10 +-- .../app/apis/output/zz_generated.deepcopy.go | 10 +-- cmd/kubeadm/app/cmd/certs.go | 3 +- cmd/kubeadm/app/cmd/config.go | 4 +- cmd/kubeadm/app/cmd/join.go | 3 +- cmd/kubeadm/app/cmd/reset.go | 3 +- cmd/kubeadm/app/cmd/token.go | 4 +- cmd/kubeadm/app/cmd/upgrade/common.go | 4 +- cmd/kubeadm/app/cmd/upgrade/diff.go | 3 +- cmd/kubeadm/app/cmd/upgrade/node.go | 3 +- cmd/kubeadm/app/cmd/upgrade/plan.go | 88 +++++++++++-------- cmd/kubeadm/app/componentconfigs/configset.go | 2 +- cmd/kubeadm/app/phases/upgrade/compute.go | 2 +- cmd/kubeadm/app/util/config/cluster.go | 3 + cmd/kubeadm/app/util/output/output.go | 55 +++++++----- 18 files changed, 106 insertions(+), 99 deletions(-) diff --git a/cmd/kubeadm/app/apis/output/types.go b/cmd/kubeadm/app/apis/output/types.go index d0229a28e04..9dda9491425 100644 --- a/cmd/kubeadm/app/apis/output/types.go +++ b/cmd/kubeadm/app/apis/output/types.go @@ -78,7 +78,7 @@ type ComponentConfigVersionState struct { type UpgradePlan struct { metav1.TypeMeta - Components []*ComponentUpgradePlan + Components []ComponentUpgradePlan ConfigVersions []ComponentConfigVersionState } diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/types.go b/cmd/kubeadm/app/apis/output/v1alpha2/types.go index 7938849a356..529c029dd59 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/types.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/types.go @@ -78,7 +78,7 @@ type ComponentConfigVersionState struct { type UpgradePlan struct { metav1.TypeMeta - Components []*ComponentUpgradePlan `json:"components"` + Components []ComponentUpgradePlan `json:"components"` ConfigVersions []ComponentConfigVersionState `json:"configVersions"` } diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go index 005b39773fb..5893ba6c309 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go @@ -180,7 +180,7 @@ func Convert_output_Images_To_v1alpha2_Images(in *output.Images, out *Images, s } func autoConvert_v1alpha2_UpgradePlan_To_output_UpgradePlan(in *UpgradePlan, out *output.UpgradePlan, s conversion.Scope) error { - out.Components = *(*[]*output.ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) + out.Components = *(*[]output.ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) out.ConfigVersions = *(*[]output.ComponentConfigVersionState)(unsafe.Pointer(&in.ConfigVersions)) return nil } @@ -191,7 +191,7 @@ func Convert_v1alpha2_UpgradePlan_To_output_UpgradePlan(in *UpgradePlan, out *ou } func autoConvert_output_UpgradePlan_To_v1alpha2_UpgradePlan(in *output.UpgradePlan, out *UpgradePlan, s conversion.Scope) error { - out.Components = *(*[]*ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) + out.Components = *(*[]ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) out.ConfigVersions = *(*[]ComponentConfigVersionState)(unsafe.Pointer(&in.ConfigVersions)) return nil } diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go index dec35b0d537..73e8c32def2 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.deepcopy.go @@ -128,14 +128,8 @@ func (in *UpgradePlan) DeepCopyInto(out *UpgradePlan) { out.TypeMeta = in.TypeMeta if in.Components != nil { in, out := &in.Components, &out.Components - *out = make([]*ComponentUpgradePlan, len(*in)) - for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = new(ComponentUpgradePlan) - **out = **in - } - } + *out = make([]ComponentUpgradePlan, len(*in)) + copy(*out, *in) } if in.ConfigVersions != nil { in, out := &in.ConfigVersions, &out.ConfigVersions diff --git a/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go b/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go index d09de881e1c..8d553d35c15 100644 --- a/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go +++ b/cmd/kubeadm/app/apis/output/zz_generated.deepcopy.go @@ -128,14 +128,8 @@ func (in *UpgradePlan) DeepCopyInto(out *UpgradePlan) { out.TypeMeta = in.TypeMeta if in.Components != nil { in, out := &in.Components, &out.Components - *out = make([]*ComponentUpgradePlan, len(*in)) - for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = new(ComponentUpgradePlan) - **out = **in - } - } + *out = make([]ComponentUpgradePlan, len(*in)) + copy(*out, *in) } if in.ConfigVersions != nil { in, out := &in.ConfigVersions, &out.ConfigVersions diff --git a/cmd/kubeadm/app/cmd/certs.go b/cmd/kubeadm/app/cmd/certs.go index a5265b4b1c3..62cf2fcdf0c 100644 --- a/cmd/kubeadm/app/cmd/certs.go +++ b/cmd/kubeadm/app/cmd/certs.go @@ -39,7 +39,6 @@ import ( kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) var ( @@ -333,7 +332,7 @@ func getInternalCfg(cfgPath string, kubeconfigPath string, cfg kubeadmapiv1.Clus if cfgPath == "" { client, err := kubeconfigutil.ClientSetFromFile(kubeconfigPath) if err == nil { - internalcfg, err := configutil.FetchInitConfigurationFromCluster(client, &output.TextPrinter{}, logPrefix, false, false) + internalcfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, logPrefix, false, false) if err == nil { fmt.Println() // add empty line to separate the FetchInitConfigurationFromCluster output from the command output return internalcfg, nil diff --git a/cmd/kubeadm/app/cmd/config.go b/cmd/kubeadm/app/cmd/config.go index 9994b80d79d..99e68706b4f 100644 --- a/cmd/kubeadm/app/cmd/config.go +++ b/cmd/kubeadm/app/cmd/config.go @@ -371,7 +371,7 @@ func newCmdConfigImagesList(out io.Writer, mockK8sVersion *string) *cobra.Comman printer, err := outputFlags.ToPrinter() if err != nil { - return err + return errors.Wrap(err, "could not construct output printer") } imagesList, err := NewImagesList(cfgPath, externalcfg) @@ -424,7 +424,7 @@ func (itp *imageTextPrinter) PrintObj(obj runtime.Object, writer io.Writer) erro // imageTextPrintFlags provides flags necessary for printing image in a text form. type imageTextPrintFlags struct{} -// ToPrinter returns kubeadm printer for the text output format +// ToPrinter returns a kubeadm printer for the text output format func (ipf *imageTextPrintFlags) ToPrinter(outputFormat string) (output.Printer, error) { if outputFormat == output.TextOutput { return &imageTextPrinter{}, nil diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 61c6c36c5b1..9a78a94866a 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -47,7 +47,6 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/discovery" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) var ( @@ -619,7 +618,7 @@ func fetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.I } // Fetches the init configuration - initConfiguration, err := configutil.FetchInitConfigurationFromCluster(tlsClient, &output.TextPrinter{}, "preflight", true, false) + initConfiguration, err := configutil.FetchInitConfigurationFromCluster(tlsClient, nil, "preflight", true, false) if err != nil { return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") } diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index ef33b1ad8b8..407603d1def 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -37,7 +37,6 @@ import ( cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - "k8s.io/kubernetes/cmd/kubeadm/app/util/output" utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" ) @@ -99,7 +98,7 @@ func newResetData(cmd *cobra.Command, options *resetOptions, in io.Reader, out i client, err := getClientset(options.kubeconfigPath, false) if err == nil { klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", options.kubeconfigPath) - cfg, err = configutil.FetchInitConfigurationFromCluster(client, &output.TextPrinter{}, "reset", false, false) + cfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", false, false) if err != nil { klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap from cluster: %v", err) } diff --git a/cmd/kubeadm/app/cmd/token.go b/cmd/kubeadm/app/cmd/token.go index 529efac51af..85e81eae6a3 100644 --- a/cmd/kubeadm/app/cmd/token.go +++ b/cmd/kubeadm/app/cmd/token.go @@ -169,7 +169,7 @@ func newCmdToken(out io.Writer, errW io.Writer) *cobra.Command { printer, err := outputFlags.ToPrinter() if err != nil { - return err + return errors.Wrap(err, "could not construct output printer") } return RunListTokens(out, errW, client, printer) @@ -354,7 +354,7 @@ func (ttp *tokenTextPrinter) PrintObj(obj runtime.Object, writer io.Writer) erro // tokenTextPrintFlags provides flags necessary for printing bootstrap token in a text form. type tokenTextPrintFlags struct{} -// ToPrinter returns kubeadm printer for the text output format +// ToPrinter returns a kubeadm printer for the text output format func (tpf *tokenTextPrintFlags) ToPrinter(outputFormat string) (output.Printer, error) { if outputFormat == output.TextOutput { return &tokenTextPrinter{columns: []string{"TOKEN", "TTL", "EXPIRES", "USAGES", "DESCRIPTION", "EXTRA GROUPS"}}, nil diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 7c8d4e5fb3a..2623b823487 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -137,12 +137,12 @@ func enforceRequirements(flags *applyPlanFlags, args []string, dryRun bool, upgr if apierrors.IsNotFound(err) { printer.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) printer.Printf("[upgrade/config] Without this information, 'kubeadm upgrade' won't know how to configure your upgraded cluster.\n") - printer.Printf("\n") + printer.Println() printer.Printf("[upgrade/config] Next steps:\n") printer.Printf("\t- OPTION 1: Run 'kubeadm config upload from-flags' and specify the same CLI arguments you passed to 'kubeadm init' when you created your control-plane.\n") printer.Printf("\t- OPTION 2: Run 'kubeadm config upload from-file' and specify the same config file you passed to 'kubeadm init' when you created your control-plane.\n") printer.Printf("\t- OPTION 3: Pass a config file to 'kubeadm upgrade' using the --config flag.\n") - printer.Printf("\n") + printer.Println() err = errors.Errorf("the ConfigMap %q in the %s namespace used for getting configuration information was not found", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) } return nil, nil, nil, errors.Wrap(err, "[upgrade/config] FATAL") diff --git a/cmd/kubeadm/app/cmd/upgrade/diff.go b/cmd/kubeadm/app/cmd/upgrade/diff.go index fe421e94811..8d0ebbc3703 100644 --- a/cmd/kubeadm/app/cmd/upgrade/diff.go +++ b/cmd/kubeadm/app/cmd/upgrade/diff.go @@ -37,7 +37,6 @@ import ( kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" - "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) type diffFlags struct { @@ -119,7 +118,7 @@ func runDiff(flags *diffFlags, args []string) error { if err != nil { return errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) } - cfg, err = configutil.FetchInitConfigurationFromCluster(client, &output.TextPrinter{}, "upgrade/diff", false, false) + cfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade/diff", false, false) } if err != nil { return err diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index 5224de86afb..25adaa48fce 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -33,7 +33,6 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow" "k8s.io/kubernetes/cmd/kubeadm/app/constants" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) // nodeOptions defines all the options exposed via flags by kubeadm upgrade node. @@ -140,7 +139,7 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*node // Fetches the cluster configuration // NB in case of control-plane node, we are reading all the info for the node; in case of NOT control-plane node // (worker node), we are not reading local API address and the CRI socket from the node object - cfg, err := configutil.FetchInitConfigurationFromCluster(client, &output.TextPrinter{}, "upgrade", !isControlPlaneNode, false) + cfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", !isControlPlaneNode, false) if err != nil { return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") } diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go index b707d1d96d2..ad8812bb918 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -34,6 +34,7 @@ import ( "k8s.io/cli-runtime/pkg/printers" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" outputapischeme "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/scheme" outputapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha2" @@ -62,7 +63,7 @@ func newCmdPlan(apf *applyPlanFlags) *cobra.Command { RunE: func(_ *cobra.Command, args []string) error { printer, err := outputFlags.ToPrinter() if err != nil { - return err + return errors.Wrap(err, "could not construct output printer") } return runPlan(flags, args, printer) @@ -77,21 +78,21 @@ func newCmdPlan(apf *applyPlanFlags) *cobra.Command { } // newComponentUpgradePlan helper creates outputapiv1alpha2.ComponentUpgradePlan object -func newComponentUpgradePlan(name, currentVersion, newVersion string) *outputapiv1alpha2.ComponentUpgradePlan { - return &outputapiv1alpha2.ComponentUpgradePlan{ +func newComponentUpgradePlan(name, currentVersion, newVersion string) outputapiv1alpha2.ComponentUpgradePlan { + return outputapiv1alpha2.ComponentUpgradePlan{ Name: name, CurrentVersion: currentVersion, NewVersion: newVersion, } } -// upgradePlanPrintFlags defines printer flag structure for the +// upgradePlanPrintFlags defines a printer flag structure for the // upgrade plan kubeadm command and provides a method // of retrieving a known printer based on flag values provided. type upgradePlanPrintFlags struct { - // JSONYamlPrintFlags provides default flags necessary for json/yaml printing. + // JSONYamlPrintFlags provides default flags necessary for json/yaml printing JSONYamlPrintFlags *upgradePlanJSONYamlPrintFlags - // TextPrintFlags provides default flags necessary for text printing. + // TextPrintFlags provides default flags necessary for text printing TextPrintFlags *upgradePlanTextPrintFlags // TypeSetterPrinter is an implementation of ResourcePrinter that wraps another printer with types set on the objects TypeSetterPrinter *printers.TypeSetterPrinter @@ -108,7 +109,7 @@ func newUpgradePlanPrintFlags(outputFormat string) *upgradePlanPrintFlags { } } -// AllowedFormats returns list of allowed output formats +// AllowedFormats returns a list of allowed output formats func (pf *upgradePlanPrintFlags) AllowedFormats() []string { ret := pf.TextPrintFlags.AllowedFormats() return append(ret, pf.JSONYamlPrintFlags.AllowedFormats()...) @@ -119,7 +120,9 @@ func (pf *upgradePlanPrintFlags) AllowedFormats() []string { func (pf *upgradePlanPrintFlags) AddFlags(cmd *cobra.Command) { pf.TextPrintFlags.AddFlags(cmd) pf.JSONYamlPrintFlags.AddFlags(cmd) - cmd.Flags().StringVarP(&pf.OutputFormat, "experimental-output", "o", pf.OutputFormat, fmt.Sprintf("Output format. One of: %s.", strings.Join(pf.AllowedFormats(), "|"))) + // TODO: once we are confident the feature is graduated we should remove the EXPERIMENTAL text below: + // https://github.com/kubernetes/kubeadm/issues/494 + cmd.Flags().StringVarP(&pf.OutputFormat, "output", "o", pf.OutputFormat, fmt.Sprintf("EXPERIMENTAL: Output format. One of: %s.", strings.Join(pf.AllowedFormats(), "|"))) } // ToPrinter receives an outputFormat and returns a printer capable of @@ -142,7 +145,7 @@ type upgradePlanJSONYamlPrintFlags struct { genericclioptions.JSONYamlPrintFlags } -// AllowedFormats returns list of allowed output formats +// AllowedFormats returns a list of allowed output formats func (pf *upgradePlanJSONYamlPrintFlags) AllowedFormats() []string { return []string{output.JSONOutput, output.YAMLOutput} } @@ -150,10 +153,10 @@ func (pf *upgradePlanJSONYamlPrintFlags) AllowedFormats() []string { // upgradePlanJSONYAMLPrinter prints upgrade plan in a JSON or YAML format type upgradePlanJSONYAMLPrinter struct { output.ResourcePrinterWrapper - Buffer []*outputapiv1alpha2.ComponentUpgradePlan + Buffer []outputapiv1alpha2.ComponentUpgradePlan } -// newUpgradePlanJSONYAMLPrinter creates new upgradePlanJSONYAMLPrinter object +// newUpgradePlanJSONYAMLPrinter creates a new upgradePlanJSONYAMLPrinter object func newUpgradePlanJSONYAMLPrinter(resourcePrinter printers.ResourcePrinter, err error) (output.Printer, error) { if err != nil { return nil, err @@ -165,18 +168,29 @@ func newUpgradePlanJSONYAMLPrinter(resourcePrinter printers.ResourcePrinter, err func (p *upgradePlanJSONYAMLPrinter) PrintObj(obj runtime.Object, writer io.Writer) error { item, ok := obj.(*outputapiv1alpha2.ComponentUpgradePlan) if !ok { - return fmt.Errorf("expected ComponentUpgradePlan, but got %+v", obj) + return errors.Errorf("expected ComponentUpgradePlan, but got %+v", obj) } - p.Buffer = append(p.Buffer, item) + p.Buffer = append(p.Buffer, *item) return nil } -// Close writes any buffered data and empties list of buffered components -func (p *upgradePlanJSONYAMLPrinter) Close(writer io.Writer) { +// Flush writes any buffered data once last object is added +func (p *upgradePlanJSONYAMLPrinter) Flush(writer io.Writer, last bool) { + if !last { + return + } + if len(p.Buffer) == 0 { + return + } plan := &outputapiv1alpha2.UpgradePlan{Components: p.Buffer} - // p.ResourcePrinterWrapper.Printer.PrintObj(plan, writer) - p.Printer.PrintObj(plan, writer) - p.Buffer = []*outputapiv1alpha2.ComponentUpgradePlan{} + if err := p.Printer.PrintObj(plan, writer); err != nil { + fmt.Fprintf(os.Stderr, "could not flush output buffer: %v\n", err) + } +} + +// Close empties the list of buffered components +func (p *upgradePlanJSONYAMLPrinter) Close(writer io.Writer) { + p.Buffer = p.Buffer[:0] } // upgradePlanTextPrinter prints upgrade plan in a text form @@ -187,10 +201,11 @@ type upgradePlanTextPrinter struct { } // Flush writes any buffered data -func (p *upgradePlanTextPrinter) Flush(writer io.Writer) { +func (p *upgradePlanTextPrinter) Flush(writer io.Writer, last bool) { if p.tabwriter != nil { p.tabwriter.Flush() p.tabwriter = nil + p.Fprintln(writer, "") } } @@ -204,26 +219,25 @@ func (p *upgradePlanTextPrinter) PrintObj(obj runtime.Object, writer io.Writer) item, ok := obj.(*outputapiv1alpha2.ComponentUpgradePlan) if !ok { - return fmt.Errorf("expected ComponentUpgradePlan, but got %+v", obj) + return errors.Errorf("expected ComponentUpgradePlan, but got %+v", obj) } // Print item fmt.Fprintf(p.tabwriter, "%s\t%s\t%s\n", item.Name, item.CurrentVersion, item.NewVersion) - return nil } -// upgradePlanTextPrintFlags provides flags necessary for printing upgrade plan in a text form. +// upgradePlanTextPrintFlags provides flags necessary for printing upgrade plan in a text form type upgradePlanTextPrintFlags struct{} func (pf *upgradePlanTextPrintFlags) AddFlags(cmd *cobra.Command) {} -// AllowedFormats returns list of allowed output formats +// AllowedFormats returns a list of allowed output formats func (pf *upgradePlanTextPrintFlags) AllowedFormats() []string { return []string{output.TextOutput} } -// ToPrinter returns kubeadm printer for the text output format +// ToPrinter returns a kubeadm printer for the text output format func (pf *upgradePlanTextPrintFlags) ToPrinter(outputFormat string) (output.Printer, error) { if outputFormat == output.TextOutput { return &upgradePlanTextPrinter{columns: []string{"COMPONENT", "CURRENT", "TARGET"}}, nil @@ -282,15 +296,12 @@ func runPlan(flags *planFlags, args []string, printer output.Printer) error { // Finally, print the component config state table printComponentConfigVersionStates(configVersionStates, os.Stdout, printer) - - // Add a newline in the end of this output to leave some space to the next output section - klog.V(1).Infoln() return nil } // TODO There is currently no way to cleanly output upgrades that involve adding, removing, or changing components // https://github.com/kubernetes/kubeadm/issues/810 was created to track addressing this. -func appendDNSComponent(components []*outputapiv1alpha2.ComponentUpgradePlan, up *upgrade.Upgrade, name string) []*outputapiv1alpha2.ComponentUpgradePlan { +func appendDNSComponent(components []outputapiv1alpha2.ComponentUpgradePlan, up *upgrade.Upgrade, name string) []outputapiv1alpha2.ComponentUpgradePlan { beforeVersion := up.Before.DNSVersion afterVersion := up.After.DNSVersion @@ -316,7 +327,7 @@ func genUpgradePlan(up *upgrade.Upgrade, isExternalEtcd bool) (*outputapiv1alpha } } - components := []*outputapiv1alpha2.ComponentUpgradePlan{} + components := []outputapiv1alpha2.ComponentUpgradePlan{} if up.CanUpgradeKubelets() { // The map is of the form :. Here all the keys are put into a slice and sorted @@ -370,26 +381,26 @@ func printUpgradePlan(up *upgrade.Upgrade, plan *outputapiv1alpha2.UpgradePlan, } else if component.Name == constants.Kubelet { if printManualUpgradeHeader { printer.Fprintln(writer, "Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':") - printer.PrintObj(newComponentUpgradePlan(component.Name, component.CurrentVersion, component.NewVersion), writer) + plan := newComponentUpgradePlan(component.Name, component.CurrentVersion, component.NewVersion) + printer.PrintObj(&plan, writer) printManualUpgradeHeader = false } else { - printer.PrintObj(newComponentUpgradePlan("", component.CurrentVersion, component.NewVersion), writer) + plan := newComponentUpgradePlan("", component.CurrentVersion, component.NewVersion) + printer.PrintObj(&plan, writer) } } else { if printHeader { // End of manual upgrades table - printer.Flush(writer) - - printer.Fprintln(writer, "") + printer.Flush(writer, false) printer.Fprintf(writer, "Upgrade to the latest %s:\n", up.Description) printer.Fprintln(writer, "") printHeader = false } - printer.PrintObj(newComponentUpgradePlan(component.Name, component.CurrentVersion, component.NewVersion), writer) + plan := newComponentUpgradePlan(component.Name, component.CurrentVersion, component.NewVersion) + printer.PrintObj(&plan, writer) } } - printer.Flush(writer) - printer.Fprintln(writer, "") + printer.Flush(writer, true) printer.Fprintln(writer, "You can now apply the upgrade by executing the following command:") printer.Fprintln(writer, "") @@ -430,8 +441,7 @@ func yesOrNo(b bool) string { } func printLineSeparator(w io.Writer, printer output.Printer) { - printer.Fprintln(w, "_____________________________________________________________________") - printer.Fprintln(w, "") + printer.Fprintf(w, "_____________________________________________________________________\n\n") } func printComponentConfigVersionStates(versionStates []outputapiv1alpha2.ComponentConfigVersionState, w io.Writer, printer output.Printer) { diff --git a/cmd/kubeadm/app/componentconfigs/configset.go b/cmd/kubeadm/app/componentconfigs/configset.go index 3ad5cfad108..6c2f8e74b80 100644 --- a/cmd/kubeadm/app/componentconfigs/configset.go +++ b/cmd/kubeadm/app/componentconfigs/configset.go @@ -26,9 +26,9 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - outputapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha2" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + outputapiv1alpha2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/output/v1alpha2" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" "k8s.io/kubernetes/cmd/kubeadm/app/util/config/strict" diff --git a/cmd/kubeadm/app/phases/upgrade/compute.go b/cmd/kubeadm/app/phases/upgrade/compute.go index 8a54a1081a0..71fde4770af 100644 --- a/cmd/kubeadm/app/phases/upgrade/compute.go +++ b/cmd/kubeadm/app/phases/upgrade/compute.go @@ -268,7 +268,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA } // Add a newline in the end of this output to leave some space to the next output section - printer.Printf("\n") + printer.Println() return upgrades, nil } diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go index ed9648f61e0..937832ea7ac 100644 --- a/cmd/kubeadm/app/util/config/cluster.go +++ b/cmd/kubeadm/app/util/config/cluster.go @@ -46,6 +46,9 @@ import ( // FetchInitConfigurationFromCluster fetches configuration from a ConfigMap in the cluster func FetchInitConfigurationFromCluster(client clientset.Interface, printer output.Printer, logPrefix string, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { + if printer == nil { + printer = &output.TextPrinter{} + } printer.Printf("[%s] Reading configuration from the cluster...\n", logPrefix) printer.Printf("[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -o yaml'\n", logPrefix, metav1.NamespaceSystem, constants.KubeadmConfigConfigMap) diff --git a/cmd/kubeadm/app/util/output/output.go b/cmd/kubeadm/app/util/output/output.go index b99f51eaac7..e662c943abe 100644 --- a/cmd/kubeadm/app/util/output/output.go +++ b/cmd/kubeadm/app/util/output/output.go @@ -28,14 +28,16 @@ import ( "k8s.io/cli-runtime/pkg/printers" ) -// TextOutput describes the plain text output -const TextOutput = "text" +const ( + // TextOutput describes the plain text output + TextOutput = "text" -// JSONOutput describes the JSON output -const JSONOutput = "json" + // JSONOutput describes the JSON output + JSONOutput = "json" -// YAMLOutput describes the YAML output -const YAMLOutput = "yaml" + // YAMLOutput describes the YAML output + YAMLOutput = "yaml" +) // TextPrintFlags is an interface to handle custom text output type TextPrintFlags interface { @@ -58,7 +60,7 @@ type PrintFlags struct { OutputFormat *string } -// AllowedFormats returns list of allowed output formats +// AllowedFormats returns a list of allowed output formats func (pf *PrintFlags) AllowedFormats() []string { ret := []string{TextOutput} ret = append(ret, pf.JSONYamlPrintFlags.AllowedFormats()...) @@ -141,8 +143,9 @@ type Printer interface { Fprintf(writer io.Writer, format string, args ...interface{}) (n int, err error) Fprintln(writer io.Writer, args ...interface{}) (n int, err error) Printf(format string, args ...interface{}) (n int, err error) + Println(args ...interface{}) (n int, err error) - Flush(writer io.Writer) + Flush(writer io.Writer, last bool) Close(writer io.Writer) } @@ -150,16 +153,6 @@ type Printer interface { type TextPrinter struct { } -// Flush writes any buffered data -func (tp *TextPrinter) Flush(writer io.Writer) { - return -} - -// Close flushes any buffered data and closes the printer -func (tp *TextPrinter) Close(writer io.Writer) { - return -} - // PrintObj is an implementation of ResourcePrinter.PrintObj that prints object func (tp *TextPrinter) PrintObj(obj runtime.Object, writer io.Writer) error { _, err := fmt.Fprintf(writer, "%+v\n", obj) @@ -181,6 +174,19 @@ func (tp *TextPrinter) Printf(format string, args ...interface{}) (n int, err er return fmt.Printf(format, args...) } +// Println is a wrapper around fmt.Printf +func (tp *TextPrinter) Println(args ...interface{}) (n int, err error) { + return fmt.Println(args...) +} + +// Flush writes any buffered data +func (tp *TextPrinter) Flush(writer io.Writer, last bool) { +} + +// Close flushes any buffered data and closes the printer +func (tp *TextPrinter) Close(writer io.Writer) { +} + // ResourcePrinterWrapper wraps ResourcePrinter and implements Printer interface type ResourcePrinterWrapper struct { Printer printers.ResourcePrinter @@ -195,13 +201,11 @@ func NewResourcePrinterWrapper(resourcePrinter printers.ResourcePrinter, err err } // Flush writes any buffered data -func (rpw *ResourcePrinterWrapper) Flush(writer io.Writer) { - return +func (rpw *ResourcePrinterWrapper) Flush(writer io.Writer, last bool) { } // Close flushes any buffered data and closes the printer func (rpw *ResourcePrinterWrapper) Close(writer io.Writer) { - return } // PrintObj is an implementation of ResourcePrinter.PrintObj that calls underlying printer API @@ -216,7 +220,7 @@ func (rpw *ResourcePrinterWrapper) Fprintf(writer io.Writer, format string, args return 0, nil } -// Fprintln is an empty method to satisfy Printer interface +// Fprintln is an empty method to satisfy the Printer interface // and silent info printing for structured output // This method is usually redefined for the text output func (rpw *ResourcePrinterWrapper) Fprintln(writer io.Writer, args ...interface{}) (n int, err error) { @@ -229,3 +233,10 @@ func (rpw *ResourcePrinterWrapper) Fprintln(writer io.Writer, args ...interface{ func (rpw *ResourcePrinterWrapper) Printf(format string, args ...interface{}) (n int, err error) { return 0, nil } + +// Println is an empty method to satisfy Printer interface +// and silent info printing for structured output +// This method is usually redefined for the text output +func (rpw *ResourcePrinterWrapper) Println(args ...interface{}) (n int, err error) { + return 0, nil +}