From 22adf221a1405c93f21dcfbfa08722c22d44d8fe Mon Sep 17 00:00:00 2001 From: carlory Date: Thu, 29 Feb 2024 16:49:25 +0800 Subject: [PATCH] print nodename --- cmd/kubeadm/app/apis/output/types.go | 1 + .../app/apis/output/v1alpha2/conversion.go | 5 + .../v1alpha2/zz_generated.conversion.go | 40 +- cmd/kubeadm/app/apis/output/v1alpha3/types.go | 1 + .../v1alpha3/zz_generated.conversion.go | 2 + cmd/kubeadm/app/cmd/upgrade/plan.go | 86 ++-- cmd/kubeadm/app/cmd/upgrade/plan_test.go | 390 ++++++++-------- cmd/kubeadm/app/phases/upgrade/compute.go | 109 +++-- .../app/phases/upgrade/compute_test.go | 421 ++++++++++++------ cmd/kubeadm/app/phases/upgrade/policy.go | 2 +- .../app/phases/upgrade/versiongetter.go | 53 ++- .../app/phases/upgrade/versiongetter_test.go | 14 +- 12 files changed, 709 insertions(+), 415 deletions(-) diff --git a/cmd/kubeadm/app/apis/output/types.go b/cmd/kubeadm/app/apis/output/types.go index 2847694de31..bb36bc3aede 100644 --- a/cmd/kubeadm/app/apis/output/types.go +++ b/cmd/kubeadm/app/apis/output/types.go @@ -49,6 +49,7 @@ type ComponentUpgradePlan struct { Name string CurrentVersion string NewVersion string + NodeName string } // ComponentConfigVersionState describes the current and desired version of a component config diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/conversion.go b/cmd/kubeadm/app/apis/output/v1alpha2/conversion.go index 464fb243535..d533ded71cd 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/conversion.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/conversion.go @@ -26,3 +26,8 @@ import ( func Convert_output_UpgradePlan_To_v1alpha2_UpgradePlan(in *output.UpgradePlan, out *UpgradePlan, s conversion.Scope) error { return autoConvert_output_UpgradePlan_To_v1alpha2_UpgradePlan(in, out, s) } + +// Convert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan converts a private ComponentUpgradePlan to public ComponentUpgradePlan. +func Convert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan(in *output.ComponentUpgradePlan, out *ComponentUpgradePlan, s conversion.Scope) error { + return autoConvert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan(in, out, s) +} diff --git a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go index 201661f39e1..a04031bdb17 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/output/v1alpha2/zz_generated.conversion.go @@ -61,11 +61,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*output.ComponentUpgradePlan)(nil), (*ComponentUpgradePlan)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan(a.(*output.ComponentUpgradePlan), b.(*ComponentUpgradePlan), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*Images)(nil), (*output.Images)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_Images_To_output_Images(a.(*Images), b.(*output.Images), scope) }); err != nil { @@ -81,6 +76,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*output.ComponentUpgradePlan)(nil), (*ComponentUpgradePlan)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan(a.(*output.ComponentUpgradePlan), b.(*ComponentUpgradePlan), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*output.UpgradePlan)(nil), (*UpgradePlan)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_output_UpgradePlan_To_v1alpha2_UpgradePlan(a.(*output.UpgradePlan), b.(*UpgradePlan), scope) }); err != nil { @@ -151,14 +151,10 @@ func autoConvert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan(in out.Name = in.Name out.CurrentVersion = in.CurrentVersion out.NewVersion = in.NewVersion + // WARNING: in.NodeName requires manual conversion: does not exist in peer-type return nil } -// Convert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan is an autogenerated conversion function. -func Convert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan(in *output.ComponentUpgradePlan, out *ComponentUpgradePlan, s conversion.Scope) error { - return autoConvert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan(in, out, s) -} - func autoConvert_v1alpha2_Images_To_output_Images(in *Images, out *output.Images, s conversion.Scope) error { out.Images = *(*[]string)(unsafe.Pointer(&in.Images)) return nil @@ -180,7 +176,17 @@ func Convert_output_Images_To_v1alpha2_Images(in *output.Images, out *Images, s } func autoConvert_v1alpha2_UpgradePlan_To_output_UpgradePlan(in *UpgradePlan, out *output.UpgradePlan, s conversion.Scope) error { - out.Components = *(*[]output.ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) + if in.Components != nil { + in, out := &in.Components, &out.Components + *out = make([]output.ComponentUpgradePlan, len(*in)) + for i := range *in { + if err := Convert_v1alpha2_ComponentUpgradePlan_To_output_ComponentUpgradePlan(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Components = nil + } out.ConfigVersions = *(*[]output.ComponentConfigVersionState)(unsafe.Pointer(&in.ConfigVersions)) return nil } @@ -193,6 +199,16 @@ func Convert_v1alpha2_UpgradePlan_To_output_UpgradePlan(in *UpgradePlan, out *ou func autoConvert_output_UpgradePlan_To_v1alpha2_UpgradePlan(in *output.UpgradePlan, out *UpgradePlan, s conversion.Scope) error { // WARNING: in.AvailableUpgrades requires manual conversion: does not exist in peer-type out.ConfigVersions = *(*[]ComponentConfigVersionState)(unsafe.Pointer(&in.ConfigVersions)) - out.Components = *(*[]ComponentUpgradePlan)(unsafe.Pointer(&in.Components)) + if in.Components != nil { + in, out := &in.Components, &out.Components + *out = make([]ComponentUpgradePlan, len(*in)) + for i := range *in { + if err := Convert_output_ComponentUpgradePlan_To_v1alpha2_ComponentUpgradePlan(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Components = nil + } return nil } diff --git a/cmd/kubeadm/app/apis/output/v1alpha3/types.go b/cmd/kubeadm/app/apis/output/v1alpha3/types.go index 103cc2efd21..96fb3fb70b8 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha3/types.go +++ b/cmd/kubeadm/app/apis/output/v1alpha3/types.go @@ -49,6 +49,7 @@ type ComponentUpgradePlan struct { Name string `json:"name"` CurrentVersion string `json:"currentVersion"` NewVersion string `json:"newVersion"` + NodeName string `json:"nodeName,omitempty"` } // ComponentConfigVersionState describes the current and desired version of a component config diff --git a/cmd/kubeadm/app/apis/output/v1alpha3/zz_generated.conversion.go b/cmd/kubeadm/app/apis/output/v1alpha3/zz_generated.conversion.go index f849ed2b8ad..42cc642a3bf 100644 --- a/cmd/kubeadm/app/apis/output/v1alpha3/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/output/v1alpha3/zz_generated.conversion.go @@ -243,6 +243,7 @@ func autoConvert_v1alpha3_ComponentUpgradePlan_To_output_ComponentUpgradePlan(in out.Name = in.Name out.CurrentVersion = in.CurrentVersion out.NewVersion = in.NewVersion + out.NodeName = in.NodeName return nil } @@ -255,6 +256,7 @@ func autoConvert_output_ComponentUpgradePlan_To_v1alpha3_ComponentUpgradePlan(in out.Name = in.Name out.CurrentVersion = in.CurrentVersion out.NewVersion = in.NewVersion + out.NodeName = in.NodeName return nil } diff --git a/cmd/kubeadm/app/cmd/upgrade/plan.go b/cmd/kubeadm/app/cmd/upgrade/plan.go index a60c98e35b8..68b5d527a2f 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan.go @@ -17,7 +17,6 @@ limitations under the License. package upgrade import ( - "fmt" "io" "os" "sort" @@ -88,11 +87,12 @@ func newCmdPlan(apf *applyPlanFlags) *cobra.Command { } // newComponentUpgradePlan helper creates outputapiv1alpha3.ComponentUpgradePlan object -func newComponentUpgradePlan(name, currentVersion, newVersion string) outputapiv1alpha3.ComponentUpgradePlan { +func newComponentUpgradePlan(name, currentVersion, newVersion, nodeName string) outputapiv1alpha3.ComponentUpgradePlan { return outputapiv1alpha3.ComponentUpgradePlan{ Name: name, CurrentVersion: currentVersion, NewVersion: newVersion, + NodeName: nodeName, } } @@ -106,10 +106,6 @@ func runPlan(flagSet *pflag.FlagSet, flags *planFlags, args []string, printer ou return err } - // Currently this is the only method we have for distinguishing - // external etcd vs static pod etcd - isExternalEtcd := initCfg.Etcd.External != nil - // Compute which upgrade possibilities there are klog.V(1).Infoln("[upgrade/plan] computing upgrade possibilities") @@ -124,7 +120,7 @@ func runPlan(flagSet *pflag.FlagSet, flags *planFlags, args []string, printer ou return cmdutil.TypeMismatchErr("allowExperimentalUpgrades", "bool") } - availUpgrades, err := upgrade.GetAvailableUpgrades(versionGetter, *allowExperimentalUpgrades, *allowRCUpgrades, isExternalEtcd, client, constants.GetStaticPodDirectory(), printer) + availUpgrades, err := upgrade.GetAvailableUpgrades(versionGetter, *allowExperimentalUpgrades, *allowRCUpgrades, client, printer) if err != nil { return errors.Wrap(err, "[upgrade/versions] FATAL") } @@ -143,16 +139,15 @@ func runPlan(flagSet *pflag.FlagSet, flags *planFlags, args []string, printer ou } // Generate and print the upgrade plan - plan := genUpgradePlan(availUpgrades, configVersionStates, isExternalEtcd) + plan := genUpgradePlan(availUpgrades, configVersionStates) return printer.PrintObj(plan, os.Stdout) } // genUpgradePlan generates upgrade plan from available upgrades and component config version states -func genUpgradePlan(availUpgrades []upgrade.Upgrade, configVersions []outputapiv1alpha3.ComponentConfigVersionState, isExternalEtcd bool) *outputapiv1alpha3.UpgradePlan { +func genUpgradePlan(availUpgrades []upgrade.Upgrade, configVersions []outputapiv1alpha3.ComponentConfigVersionState) *outputapiv1alpha3.UpgradePlan { plan := &outputapiv1alpha3.UpgradePlan{ConfigVersions: configVersions} for _, up := range availUpgrades { - au := genAvailableUpgrade(&up, isExternalEtcd) - plan.AvailableUpgrades = append(plan.AvailableUpgrades, au) + plan.AvailableUpgrades = append(plan.AvailableUpgrades, genAvailableUpgrade(&up)) } return plan } @@ -164,7 +159,7 @@ func appendDNSComponent(components []outputapiv1alpha3.ComponentUpgradePlan, up afterVersion := up.After.DNSVersion if beforeVersion != "" || afterVersion != "" { - components = append(components, newComponentUpgradePlan(name, beforeVersion, afterVersion)) + components = append(components, newComponentUpgradePlan(name, beforeVersion, afterVersion, "")) } return components } @@ -175,41 +170,64 @@ func appendKubeadmComponent(components []outputapiv1alpha3.ComponentUpgradePlan, afterVersion := up.After.KubeadmVersion if beforeVersion != "" || afterVersion != "" { - components = append(components, newComponentUpgradePlan(name, beforeVersion, afterVersion)) + components = append(components, newComponentUpgradePlan(name, beforeVersion, afterVersion, "")) } return components } -// genAvailableUpgrade generates available upgrade from upgrade object and external etcd boolean -func genAvailableUpgrade(up *upgrade.Upgrade, isExternalEtcd bool) outputapiv1alpha3.AvailableUpgrade { +// genAvailableUpgrade generates available upgrade from upgrade object. +func genAvailableUpgrade(up *upgrade.Upgrade) outputapiv1alpha3.AvailableUpgrade { components := []outputapiv1alpha3.ComponentUpgradePlan{} if up.CanUpgradeKubelets() { - // The map is of the form :. Here all the keys are put into a slice and sorted + // The map is of the form :. Here all the keys are put into a slice and sorted // in order to always get the right order. Then the map value is extracted separately - for _, oldVersion := range sortedSliceFromStringIntMap(up.Before.KubeletVersions) { - nodeCount := up.Before.KubeletVersions[oldVersion] - components = append(components, newComponentUpgradePlan(constants.Kubelet, fmt.Sprintf("%d x %s", nodeCount, oldVersion), up.After.KubeVersion)) + for _, oldVersion := range sortedSliceFromStringStringArrayMap(up.Before.KubeletVersions) { + nodeNames := up.Before.KubeletVersions[oldVersion] + for _, nodeName := range nodeNames { + components = append(components, newComponentUpgradePlan(constants.Kubelet, oldVersion, up.After.KubeVersion, nodeName)) + } } } - components = append(components, newComponentUpgradePlan(constants.KubeAPIServer, up.Before.KubeVersion, up.After.KubeVersion)) - components = append(components, newComponentUpgradePlan(constants.KubeControllerManager, up.Before.KubeVersion, up.After.KubeVersion)) - components = append(components, newComponentUpgradePlan(constants.KubeScheduler, up.Before.KubeVersion, up.After.KubeVersion)) - components = append(components, newComponentUpgradePlan(constants.KubeProxy, up.Before.KubeVersion, up.After.KubeVersion)) + for _, oldVersion := range sortedSliceFromStringStringArrayMap(up.Before.KubeAPIServerVersions) { + nodeNames := up.Before.KubeAPIServerVersions[oldVersion] + for _, nodeName := range nodeNames { + components = append(components, newComponentUpgradePlan(constants.KubeAPIServer, oldVersion, up.After.KubeVersion, nodeName)) + } + } + for _, oldVersion := range sortedSliceFromStringStringArrayMap(up.Before.KubeControllerManagerVersions) { + nodeNames := up.Before.KubeControllerManagerVersions[oldVersion] + for _, nodeName := range nodeNames { + components = append(components, newComponentUpgradePlan(constants.KubeControllerManager, oldVersion, up.After.KubeVersion, nodeName)) + } + } + + for _, oldVersion := range sortedSliceFromStringStringArrayMap(up.Before.KubeSchedulerVersions) { + nodeNames := up.Before.KubeSchedulerVersions[oldVersion] + for _, nodeName := range nodeNames { + components = append(components, newComponentUpgradePlan(constants.KubeScheduler, oldVersion, up.After.KubeVersion, nodeName)) + } + } + + components = append(components, newComponentUpgradePlan(constants.KubeProxy, up.Before.KubeVersion, up.After.KubeVersion, "")) components = appendDNSComponent(components, up, constants.CoreDNS) components = appendKubeadmComponent(components, up, constants.Kubeadm) - if !isExternalEtcd { - components = append(components, newComponentUpgradePlan(constants.Etcd, up.Before.EtcdVersion, up.After.EtcdVersion)) + // If etcd is not external, we should include it in the upgrade plan + for _, oldVersion := range sortedSliceFromStringStringArrayMap(up.Before.EtcdVersions) { + nodeNames := up.Before.EtcdVersions[oldVersion] + for _, nodeName := range nodeNames { + components = append(components, newComponentUpgradePlan(constants.Etcd, oldVersion, up.After.EtcdVersion, nodeName)) + } } return outputapiv1alpha3.AvailableUpgrade{Description: up.Description, Components: components} } -// sortedSliceFromStringIntMap returns a slice of the keys in the map sorted alphabetically -func sortedSliceFromStringIntMap(strMap map[string]uint16) []string { +// sortedSliceFromStringStringArrayMap returns a slice of the keys in the map sorted alphabetically +func sortedSliceFromStringStringArrayMap(strMap map[string][]string) []string { strSlice := []string{} for k := range strMap { strSlice = append(strSlice, k) @@ -280,28 +298,24 @@ func (printer *upgradePlanTextPrinter) printAvailableUpgrade(writer io.Writer, a _, _ = printer.Fprintln(writer, "Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':") tabw := tabwriter.NewWriter(writer, 10, 4, 3, ' ', 0) - _, _ = printer.Fprintln(tabw, strings.Join([]string{"COMPONENT", "CURRENT", "TARGET"}, "\t")) - for i, component := range au.Components { + _, _ = printer.Fprintln(tabw, strings.Join([]string{"COMPONENT", "NODE", "CURRENT", "TARGET"}, "\t")) + for _, component := range au.Components { if component.Name != constants.Kubelet { continue } - if i == 0 { - _, _ = printer.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) - } else { - _, _ = printer.Fprintf(tabw, "%s\t%s\t%s\n", "", component.CurrentVersion, component.NewVersion) - } + _, _ = printer.Fprintf(tabw, "%s\t%s\t%s\t%s\n", component.Name, component.NodeName, component.CurrentVersion, component.NewVersion) } _ = tabw.Flush() _, _ = printer.Fprintln(writer, "") _, _ = printer.Fprintf(writer, "Upgrade to the latest %s:\n", au.Description) _, _ = printer.Fprintln(writer, "") - _, _ = printer.Fprintln(tabw, strings.Join([]string{"COMPONENT", "CURRENT", "TARGET"}, "\t")) + _, _ = printer.Fprintln(tabw, strings.Join([]string{"COMPONENT", "NODE", "CURRENT", "TARGET"}, "\t")) for _, component := range au.Components { if component.Name == constants.Kubelet || component.Name == constants.Kubeadm { continue } - _, _ = printer.Fprintf(tabw, "%s\t%s\t%s\n", component.Name, component.CurrentVersion, component.NewVersion) + _, _ = printer.Fprintf(tabw, "%s\t%s\t%s\t%s\n", component.Name, component.NodeName, component.CurrentVersion, component.NewVersion) } _ = tabw.Flush() diff --git a/cmd/kubeadm/app/cmd/upgrade/plan_test.go b/cmd/kubeadm/app/cmd/upgrade/plan_test.go index ab30daa4828..b1baa75e961 100644 --- a/cmd/kubeadm/app/cmd/upgrade/plan_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/plan_test.go @@ -28,39 +28,39 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/util/output" ) -func TestSortedSliceFromStringIntMap(t *testing.T) { +func TestSortedSliceFromStringStringArrayMap(t *testing.T) { var tests = []struct { name string - strMap map[string]uint16 + strMap map[string][]string expectedSlice []string }{ { name: "the returned slice should be alphabetically sorted based on the string keys in the map", - strMap: map[string]uint16{"foo": 1, "bar": 2}, + strMap: map[string][]string{"foo": {"1"}, "bar": {"1", "2"}}, expectedSlice: []string{"bar", "foo"}, }, { name: "the int value should not affect this func", - strMap: map[string]uint16{"foo": 2, "bar": 1}, + strMap: map[string][]string{"foo": {"1", "2"}, "bar": {"1"}}, expectedSlice: []string{"bar", "foo"}, }, { name: "slice with 4 keys and different values", - strMap: map[string]uint16{"b": 2, "a": 1, "cb": 0, "ca": 1000}, + strMap: map[string][]string{"b": {"1", "2"}, "a": {"1"}, "cb": {}, "ca": {"1", "2", "3"}}, expectedSlice: []string{"a", "b", "ca", "cb"}, }, { name: "this should work for version numbers as well; and the lowest version should come first", - strMap: map[string]uint16{"v1.7.0": 1, "v1.6.1": 1, "v1.6.2": 1, "v1.8.0": 1, "v1.8.0-alpha.1": 1}, + strMap: map[string][]string{"v1.7.0": {"1"}, "v1.6.1": {"1"}, "v1.6.2": {"1"}, "v1.8.0": {"1"}, "v1.8.0-alpha.1": {"1"}}, expectedSlice: []string{"v1.6.1", "v1.6.2", "v1.7.0", "v1.8.0", "v1.8.0-alpha.1"}, }, } for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { - actualSlice := sortedSliceFromStringIntMap(rt.strMap) + actualSlice := sortedSliceFromStringStringArrayMap(rt.strMap) if !reflect.DeepEqual(actualSlice, rt.expectedSlice) { t.Errorf( - "failed SortedSliceFromStringIntMap:\n\texpected: %v\n\t actual: %v", + "failed sortedSliceFromStringStringArrayMap:\n\texpected: %v\n\t actual: %v", rt.expectedSlice, actualSlice, ) @@ -101,12 +101,23 @@ func TestPrintUpgradePlan(t *testing.T) { Description: "version in the v1.18 series", Before: upgrade.ClusterState{ KubeVersion: "v1.18.1", - KubeletVersions: map[string]uint16{ - "v1.18.1": 1, + KubeAPIServerVersions: map[string][]string{ + "v1.18.1": {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + "v1.18.1": {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + "v1.18.1": {"node1"}, + }, + KubeletVersions: map[string][]string{ + "v1.18.1": {"node1"}, + }, + EtcdVersions: map[string][]string{ + "3.4.3-0": {"node1"}, }, KubeadmVersion: "v1.18.1", DNSVersion: "1.6.7", - EtcdVersion: "3.4.3-0", }, After: upgrade.ClusterState{ KubeVersion: "v1.18.4", @@ -118,18 +129,18 @@ func TestPrintUpgradePlan(t *testing.T) { }, versionStates: versionStates, expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT TARGET -kubelet 1 x v1.18.1 v1.18.4 +COMPONENT NODE CURRENT TARGET +kubelet node1 v1.18.1 v1.18.4 Upgrade to the latest version in the v1.18 series: -COMPONENT CURRENT TARGET -kube-apiserver v1.18.1 v1.18.4 -kube-controller-manager v1.18.1 v1.18.4 -kube-scheduler v1.18.1 v1.18.4 -kube-proxy v1.18.1 v1.18.4 -CoreDNS 1.6.7 1.6.7 -etcd 3.4.3-0 3.4.3-0 +COMPONENT NODE CURRENT TARGET +kube-apiserver node1 v1.18.1 v1.18.4 +kube-controller-manager node1 v1.18.1 v1.18.4 +kube-scheduler node1 v1.18.1 v1.18.4 +kube-proxy v1.18.1 v1.18.4 +CoreDNS 1.6.7 1.6.7 +etcd node1 3.4.3-0 3.4.3-0 You can now apply the upgrade by executing the following command: @@ -159,12 +170,23 @@ _____________________________________________________________________ Description: "stable version", Before: upgrade.ClusterState{ KubeVersion: "v1.18.4", - KubeletVersions: map[string]uint16{ - "v1.18.4": 1, + KubeAPIServerVersions: map[string][]string{ + "v1.18.4": {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + "v1.18.4": {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + "v1.18.4": {"node1"}, + }, + KubeletVersions: map[string][]string{ + "v1.18.4": {"node1"}, + }, + EtcdVersions: map[string][]string{ + "3.4.3-0": {"node1"}, }, KubeadmVersion: "v1.18.4", DNSVersion: "1.6.7", - EtcdVersion: "3.4.3-0", }, After: upgrade.ClusterState{ KubeVersion: "v1.19.0", @@ -176,18 +198,18 @@ _____________________________________________________________________ }, versionStates: versionStates, expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT TARGET -kubelet 1 x v1.18.4 v1.19.0 +COMPONENT NODE CURRENT TARGET +kubelet node1 v1.18.4 v1.19.0 Upgrade to the latest stable version: -COMPONENT CURRENT TARGET -kube-apiserver v1.18.4 v1.19.0 -kube-controller-manager v1.18.4 v1.19.0 -kube-scheduler v1.18.4 v1.19.0 -kube-proxy v1.18.4 v1.19.0 -CoreDNS 1.6.7 1.7.0 -etcd 3.4.3-0 3.4.7-0 +COMPONENT NODE CURRENT TARGET +kube-apiserver node1 v1.18.4 v1.19.0 +kube-controller-manager node1 v1.18.4 v1.19.0 +kube-scheduler node1 v1.18.4 v1.19.0 +kube-proxy v1.18.4 v1.19.0 +CoreDNS 1.6.7 1.7.0 +etcd node1 3.4.3-0 3.4.7-0 You can now apply the upgrade by executing the following command: @@ -217,12 +239,23 @@ _____________________________________________________________________ Description: "version in the v1.18 series", Before: upgrade.ClusterState{ KubeVersion: "v1.18.3", - KubeletVersions: map[string]uint16{ - "v1.18.3": 1, + KubeAPIServerVersions: map[string][]string{ + "v1.18.3": {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + "v1.18.3": {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + "v1.18.3": {"node1"}, + }, + KubeletVersions: map[string][]string{ + "v1.18.3": {"node1"}, + }, + EtcdVersions: map[string][]string{ + "3.4.3-0": {"node1"}, }, KubeadmVersion: "v1.18.3", DNSVersion: "1.6.7", - EtcdVersion: "3.4.3-0", }, After: upgrade.ClusterState{ KubeVersion: "v1.18.5", @@ -235,12 +268,23 @@ _____________________________________________________________________ Description: "stable version", Before: upgrade.ClusterState{ KubeVersion: "v1.18.3", - KubeletVersions: map[string]uint16{ - "v1.18.3": 1, + KubeAPIServerVersions: map[string][]string{ + "v1.18.3": {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + "v1.18.3": {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + "v1.18.3": {"node1"}, + }, + KubeletVersions: map[string][]string{ + "v1.18.3": {"node1"}, + }, + EtcdVersions: map[string][]string{ + "3.4.3-0": {"node1"}, }, KubeadmVersion: "v1.18.3", DNSVersion: "1.6.7", - EtcdVersion: "3.4.3-0", }, After: upgrade.ClusterState{ KubeVersion: "v1.19.0", @@ -252,18 +296,18 @@ _____________________________________________________________________ }, versionStates: versionStates, expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT TARGET -kubelet 1 x v1.18.3 v1.18.5 +COMPONENT NODE CURRENT TARGET +kubelet node1 v1.18.3 v1.18.5 Upgrade to the latest version in the v1.18 series: -COMPONENT CURRENT TARGET -kube-apiserver v1.18.3 v1.18.5 -kube-controller-manager v1.18.3 v1.18.5 -kube-scheduler v1.18.3 v1.18.5 -kube-proxy v1.18.3 v1.18.5 -CoreDNS 1.6.7 1.6.7 -etcd 3.4.3-0 3.4.3-0 +COMPONENT NODE CURRENT TARGET +kube-apiserver node1 v1.18.3 v1.18.5 +kube-controller-manager node1 v1.18.3 v1.18.5 +kube-scheduler node1 v1.18.3 v1.18.5 +kube-proxy v1.18.3 v1.18.5 +CoreDNS 1.6.7 1.6.7 +etcd node1 3.4.3-0 3.4.3-0 You can now apply the upgrade by executing the following command: @@ -272,18 +316,18 @@ You can now apply the upgrade by executing the following command: _____________________________________________________________________ Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT TARGET -kubelet 1 x v1.18.3 v1.19.0 +COMPONENT NODE CURRENT TARGET +kubelet node1 v1.18.3 v1.19.0 Upgrade to the latest stable version: -COMPONENT CURRENT TARGET -kube-apiserver v1.18.3 v1.19.0 -kube-controller-manager v1.18.3 v1.19.0 -kube-scheduler v1.18.3 v1.19.0 -kube-proxy v1.18.3 v1.19.0 -CoreDNS 1.6.7 1.7.0 -etcd 3.4.3-0 3.4.7-0 +COMPONENT NODE CURRENT TARGET +kube-apiserver node1 v1.18.3 v1.19.0 +kube-controller-manager node1 v1.18.3 v1.19.0 +kube-scheduler node1 v1.18.3 v1.19.0 +kube-proxy v1.18.3 v1.19.0 +CoreDNS 1.6.7 1.7.0 +etcd node1 3.4.3-0 3.4.7-0 You can now apply the upgrade by executing the following command: @@ -313,12 +357,23 @@ _____________________________________________________________________ Description: "experimental version", Before: upgrade.ClusterState{ KubeVersion: "v1.18.5", - KubeletVersions: map[string]uint16{ - "v1.18.5": 1, + KubeAPIServerVersions: map[string][]string{ + "v1.18.5": {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + "v1.18.5": {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + "v1.18.5": {"node1"}, + }, + KubeletVersions: map[string][]string{ + "v1.18.5": {"node1"}, + }, + EtcdVersions: map[string][]string{ + "3.4.3-0": {"node1"}, }, KubeadmVersion: "v1.18.5", DNSVersion: "1.6.7", - EtcdVersion: "3.4.3-0", }, After: upgrade.ClusterState{ KubeVersion: "v1.19.0-beta.1", @@ -330,18 +385,18 @@ _____________________________________________________________________ }, versionStates: versionStates, expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT TARGET -kubelet 1 x v1.18.5 v1.19.0-beta.1 +COMPONENT NODE CURRENT TARGET +kubelet node1 v1.18.5 v1.19.0-beta.1 Upgrade to the latest experimental version: -COMPONENT CURRENT TARGET -kube-apiserver v1.18.5 v1.19.0-beta.1 -kube-controller-manager v1.18.5 v1.19.0-beta.1 -kube-scheduler v1.18.5 v1.19.0-beta.1 -kube-proxy v1.18.5 v1.19.0-beta.1 -CoreDNS 1.6.7 1.7.0 -etcd 3.4.3-0 3.4.7-0 +COMPONENT NODE CURRENT TARGET +kube-apiserver node1 v1.18.5 v1.19.0-beta.1 +kube-controller-manager node1 v1.18.5 v1.19.0-beta.1 +kube-scheduler node1 v1.18.5 v1.19.0-beta.1 +kube-proxy v1.18.5 v1.19.0-beta.1 +CoreDNS 1.6.7 1.7.0 +etcd node1 3.4.3-0 3.4.7-0 You can now apply the upgrade by executing the following command: @@ -371,12 +426,23 @@ _____________________________________________________________________ Description: "release candidate version", Before: upgrade.ClusterState{ KubeVersion: "v1.18.5", - KubeletVersions: map[string]uint16{ - "v1.18.5": 1, + KubeAPIServerVersions: map[string][]string{ + "v1.18.5": {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + "v1.18.5": {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + "v1.18.5": {"node1"}, + }, + KubeletVersions: map[string][]string{ + "v1.18.5": {"node1"}, + }, + EtcdVersions: map[string][]string{ + "3.4.3-0": {"node1"}, }, KubeadmVersion: "v1.18.5", DNSVersion: "1.6.7", - EtcdVersion: "3.4.3-0", }, After: upgrade.ClusterState{ KubeVersion: "v1.19.0-rc.1", @@ -388,18 +454,18 @@ _____________________________________________________________________ }, versionStates: versionStates, expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT TARGET -kubelet 1 x v1.18.5 v1.19.0-rc.1 +COMPONENT NODE CURRENT TARGET +kubelet node1 v1.18.5 v1.19.0-rc.1 Upgrade to the latest release candidate version: -COMPONENT CURRENT TARGET -kube-apiserver v1.18.5 v1.19.0-rc.1 -kube-controller-manager v1.18.5 v1.19.0-rc.1 -kube-scheduler v1.18.5 v1.19.0-rc.1 -kube-proxy v1.18.5 v1.19.0-rc.1 -CoreDNS 1.6.7 1.7.0 -etcd 3.4.3-0 3.4.7-0 +COMPONENT NODE CURRENT TARGET +kube-apiserver node1 v1.18.5 v1.19.0-rc.1 +kube-controller-manager node1 v1.18.5 v1.19.0-rc.1 +kube-scheduler node1 v1.18.5 v1.19.0-rc.1 +kube-proxy v1.18.5 v1.19.0-rc.1 +CoreDNS 1.6.7 1.7.0 +etcd node1 3.4.3-0 3.4.7-0 You can now apply the upgrade by executing the following command: @@ -429,13 +495,24 @@ _____________________________________________________________________ Description: "version in the v1.19 series", Before: upgrade.ClusterState{ KubeVersion: "v1.19.2", - KubeletVersions: map[string]uint16{ - "v1.19.2": 1, - "v1.19.3": 2, + KubeAPIServerVersions: map[string][]string{ + "v1.19.2": {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + "v1.19.2": {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + "v1.19.2": {"node1"}, + }, + KubeletVersions: map[string][]string{ + "v1.19.2": {"node1"}, + "v1.19.3": {"node2", "node3"}, + }, + EtcdVersions: map[string][]string{ + "3.4.7-0": {"node1"}, }, KubeadmVersion: "v1.19.2", DNSVersion: "1.7.0", - EtcdVersion: "3.4.7-0", }, After: upgrade.ClusterState{ KubeVersion: "v1.19.3", @@ -447,78 +524,20 @@ _____________________________________________________________________ }, versionStates: versionStates, expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT TARGET -kubelet 1 x v1.19.2 v1.19.3 - 2 x v1.19.3 v1.19.3 +COMPONENT NODE CURRENT TARGET +kubelet node1 v1.19.2 v1.19.3 +kubelet node2 v1.19.3 v1.19.3 +kubelet node3 v1.19.3 v1.19.3 Upgrade to the latest version in the v1.19 series: -COMPONENT CURRENT TARGET -kube-apiserver v1.19.2 v1.19.3 -kube-controller-manager v1.19.2 v1.19.3 -kube-scheduler v1.19.2 v1.19.3 -kube-proxy v1.19.2 v1.19.3 -CoreDNS 1.7.0 1.7.0 -etcd 3.4.7-0 3.4.7-0 - -You can now apply the upgrade by executing the following command: - - kubeadm upgrade apply v1.19.3 - -Note: Before you can perform this upgrade, you have to update kubeadm to v1.19.3. - -_____________________________________________________________________ - - -The table below shows the current state of component configs as understood by this version of kubeadm. -Configs that have a "yes" mark in the "MANUAL UPGRADE REQUIRED" column require manual config upgrade or -resetting to kubeadm defaults before a successful upgrade can be performed. The version to manually -upgrade to is denoted in the "PREFERRED VERSION" column. - -API GROUP CURRENT VERSION PREFERRED VERSION MANUAL UPGRADE REQUIRED -kubeproxy.config.k8s.io v1alpha1 v1alpha1 no -kubelet.config.k8s.io v1beta1 v1beta1 no -_____________________________________________________________________ - -`), - }, - - { - name: "external etcd upgrade available", - upgrades: []upgrade.Upgrade{ - { - Description: "version in the v1.19 series", - Before: upgrade.ClusterState{ - KubeVersion: "v1.19.2", - KubeletVersions: map[string]uint16{ - "v1.19.2": 1, - }, - KubeadmVersion: "v1.19.2", - DNSVersion: "1.7.0", - EtcdVersion: "3.4.7-0", - }, - After: upgrade.ClusterState{ - KubeVersion: "v1.19.3", - KubeadmVersion: "v1.19.3", - DNSVersion: "1.7.0", - EtcdVersion: "3.4.7-0", - }, - }, - }, - versionStates: versionStates, - externalEtcd: true, - expectedBytes: []byte(`Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT TARGET -kubelet 1 x v1.19.2 v1.19.3 - -Upgrade to the latest version in the v1.19 series: - -COMPONENT CURRENT TARGET -kube-apiserver v1.19.2 v1.19.3 -kube-controller-manager v1.19.2 v1.19.3 -kube-scheduler v1.19.2 v1.19.3 -kube-proxy v1.19.2 v1.19.3 -CoreDNS 1.7.0 1.7.0 +COMPONENT NODE CURRENT TARGET +kube-apiserver node1 v1.19.2 v1.19.3 +kube-controller-manager node1 v1.19.2 v1.19.3 +kube-scheduler node1 v1.19.2 v1.19.3 +kube-proxy v1.19.2 v1.19.3 +CoreDNS 1.7.0 1.7.0 +etcd node1 3.4.7-0 3.4.7-0 You can now apply the upgrade by executing the following command: @@ -551,7 +570,7 @@ _____________________________________________________________________ t.Errorf("failed ToPrinter, err: %+v", err) } - plan := genUpgradePlan(rt.upgrades, rt.versionStates, rt.externalEtcd) + plan := genUpgradePlan(rt.upgrades, rt.versionStates) if err := printer.PrintObj(plan, rt.buf); err != nil { t.Errorf("unexpected error when print object: %v", err) } @@ -559,7 +578,7 @@ _____________________________________________________________________ actualBytes := rt.buf.Bytes() if !bytes.Equal(actualBytes, rt.expectedBytes) { t.Errorf( - "failed PrintUpgradePlan:\n\texpected: %q\n\n\tactual : %q", + "failed PrintUpgradePlan:\n\texpected: %s\n\n\tactual: %s", string(rt.expectedBytes), string(actualBytes), ) @@ -574,12 +593,23 @@ func TestPrintUpgradePlanStructured(t *testing.T) { Description: "version in the v1.8 series", Before: upgrade.ClusterState{ KubeVersion: "v1.8.1", - KubeletVersions: map[string]uint16{ - "v1.8.1": 1, + KubeAPIServerVersions: map[string][]string{ + "v1.8.1": {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + "v1.8.1": {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + "v1.8.1": {"node1"}, + }, + KubeletVersions: map[string][]string{ + "v1.8.1": {"node1"}, + }, + EtcdVersions: map[string][]string{ + "3.0.17": {"node1"}, }, KubeadmVersion: "v1.8.2", DNSVersion: "1.14.5", - EtcdVersion: "3.0.17", }, After: upgrade.ClusterState{ KubeVersion: "v1.8.3", @@ -624,23 +654,27 @@ func TestPrintUpgradePlanStructured(t *testing.T) { "components": [ { "name": "kubelet", - "currentVersion": "1 x v1.8.1", - "newVersion": "v1.8.3" + "currentVersion": "v1.8.1", + "newVersion": "v1.8.3", + "nodeName": "node1" }, { "name": "kube-apiserver", "currentVersion": "v1.8.1", - "newVersion": "v1.8.3" + "newVersion": "v1.8.3", + "nodeName": "node1" }, { "name": "kube-controller-manager", "currentVersion": "v1.8.1", - "newVersion": "v1.8.3" + "newVersion": "v1.8.3", + "nodeName": "node1" }, { "name": "kube-scheduler", "currentVersion": "v1.8.1", - "newVersion": "v1.8.3" + "newVersion": "v1.8.3", + "nodeName": "node1" }, { "name": "kube-proxy", @@ -660,7 +694,8 @@ func TestPrintUpgradePlanStructured(t *testing.T) { { "name": "etcd", "currentVersion": "3.0.17", - "newVersion": "3.0.17" + "newVersion": "3.0.17", + "nodeName": "node1" } ] } @@ -688,18 +723,22 @@ func TestPrintUpgradePlanStructured(t *testing.T) { expected: `apiVersion: output.kubeadm.k8s.io/v1alpha3 availableUpgrades: - components: - - currentVersion: 1 x v1.8.1 + - currentVersion: v1.8.1 name: kubelet newVersion: v1.8.3 + nodeName: node1 - currentVersion: v1.8.1 name: kube-apiserver newVersion: v1.8.3 + nodeName: node1 - currentVersion: v1.8.1 name: kube-controller-manager newVersion: v1.8.3 + nodeName: node1 - currentVersion: v1.8.1 name: kube-scheduler newVersion: v1.8.3 + nodeName: node1 - currentVersion: v1.8.1 name: kube-proxy newVersion: v1.8.3 @@ -712,6 +751,7 @@ availableUpgrades: - currentVersion: 3.0.17 name: etcd newVersion: 3.0.17 + nodeName: node1 description: version in the v1.8 series configVersions: - currentVersion: v1alpha1 @@ -729,18 +769,18 @@ kind: UpgradePlan name: "Text output", outputFormat: "text", expected: `Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply': -COMPONENT CURRENT TARGET -kubelet 1 x v1.8.1 v1.8.3 +COMPONENT NODE CURRENT TARGET +kubelet node1 v1.8.1 v1.8.3 Upgrade to the latest version in the v1.8 series: -COMPONENT CURRENT TARGET -kube-apiserver v1.8.1 v1.8.3 -kube-controller-manager v1.8.1 v1.8.3 -kube-scheduler v1.8.1 v1.8.3 -kube-proxy v1.8.1 v1.8.3 -CoreDNS 1.14.5 1.14.5 -etcd 3.0.17 3.0.17 +COMPONENT NODE CURRENT TARGET +kube-apiserver node1 v1.8.1 v1.8.3 +kube-controller-manager node1 v1.8.1 v1.8.3 +kube-scheduler node1 v1.8.1 v1.8.3 +kube-proxy v1.8.1 v1.8.3 +CoreDNS 1.14.5 1.14.5 +etcd node1 3.0.17 3.0.17 You can now apply the upgrade by executing the following command: @@ -774,7 +814,7 @@ _____________________________________________________________________ t.Errorf("failed ToPrinter, err: %+v", err) } - plan := genUpgradePlan(upgrades, versionStates, false) + plan := genUpgradePlan(upgrades, versionStates) if err := printer.PrintObj(plan, rt.buf); err != nil { t.Errorf("unexpected error when print object: %v", err) } diff --git a/cmd/kubeadm/app/phases/upgrade/compute.go b/cmd/kubeadm/app/phases/upgrade/compute.go index b1631503cd4..2c6eabb8d36 100644 --- a/cmd/kubeadm/app/phases/upgrade/compute.go +++ b/cmd/kubeadm/app/phases/upgrade/compute.go @@ -52,38 +52,55 @@ func (u *Upgrade) CanUpgradeKubelets() bool { return !sameVersionFound } -// CanUpgradeEtcd returns whether an upgrade of etcd is possible -func (u *Upgrade) CanUpgradeEtcd() bool { - return u.Before.EtcdVersion != u.After.EtcdVersion -} - -// ClusterState describes the state of certain versions for a cluster +// ClusterState describes the state of certain versions for a cluster during an upgrade type ClusterState struct { - // KubeVersion describes the version of the Kubernetes API Server, Controller Manager, Scheduler and Proxy. + // KubeVersion describes the version of latest Kubernetes API Server in the cluster. KubeVersion string // DNSVersion describes the version of the DNS add-on. DNSVersion string // KubeadmVersion describes the version of the kubeadm CLI KubeadmVersion string - // KubeletVersions is a map with a version number linked to the amount of kubelets running that version in the cluster - KubeletVersions map[string]uint16 // EtcdVersion represents the version of etcd used in the cluster EtcdVersion string + + // The following maps describe the versions of the different components in the cluster. + // The key is the version string and the value is a list of nodes that have that version. + KubeAPIServerVersions map[string][]string + KubeControllerManagerVersions map[string][]string + KubeSchedulerVersions map[string][]string + EtcdVersions map[string][]string + KubeletVersions map[string][]string } // GetAvailableUpgrades fetches all versions from the specified VersionGetter and computes which // kinds of upgrades can be performed -func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed, externalEtcd bool, client clientset.Interface, manifestsDir string, printer output.Printer) ([]Upgrade, error) { +func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesAllowed, rcUpgradesAllowed bool, client clientset.Interface, printer output.Printer) ([]Upgrade, error) { printer.Printf("[upgrade] Fetching available versions to upgrade to\n") // Collect the upgrades kubeadm can do in this list var upgrades []Upgrade - // Get the cluster version - clusterVersionStr, clusterVersion, err := versionGetterImpl.ClusterVersion() + // Get the kube-apiserver versions in the cluster + kubeAPIServerVersions, err := versionGetterImpl.ComponentVersions(kubeadmconstants.KubeAPIServer) if err != nil { return upgrades, err } + if len(kubeAPIServerVersions) > 1 { + verMsg := []string{} + for version, nodes := range kubeAPIServerVersions { + verMsg = append(verMsg, fmt.Sprintf("%s on nodes %v", version, nodes)) + } + klog.Warningf("Different API server versions in the cluster were discovered: %v. Please upgrade your control plane"+ + " nodes to the same version of Kubernetes", strings.Join(verMsg, ", ")) + } + + // Get the lastest cluster version + clusterVersion, err := getLatestClusterVersion(kubeAPIServerVersions) + if err != nil { + return upgrades, err + } + clusterVersionStr := clusterVersion.String() + printer.Printf("[upgrade/versions] Cluster version: %s\n", clusterVersionStr) // Get current kubeadm CLI version @@ -109,15 +126,25 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA return upgrades, err } - // Get current stacked etcd version on the local node - var etcdVersion string - if !externalEtcd { - etcdVersion, err = GetEtcdImageTagFromStaticPod(manifestsDir) - if err != nil { - return upgrades, err - } + // Get the kube-controller-manager versions in the cluster + kubeControllerManagerVersions, err := versionGetterImpl.ComponentVersions(kubeadmconstants.KubeControllerManager) + if err != nil { + return upgrades, err } + // Get the kube-scheduler versions in the cluster + kubeSchedulerVersions, err := versionGetterImpl.ComponentVersions(kubeadmconstants.KubeScheduler) + if err != nil { + return upgrades, err + } + + // Get the etcd versions in the cluster + etcdVersions, err := versionGetterImpl.ComponentVersions(kubeadmconstants.Etcd) + if err != nil { + return upgrades, err + } + isExternalEtcd := len(etcdVersions) == 0 + dnsVersion, err := dns.DeployedDNSAddon(client) if err != nil { return nil, err @@ -125,11 +152,14 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA // Construct a descriptor for the current state of the world beforeState := ClusterState{ - KubeVersion: clusterVersionStr, - DNSVersion: dnsVersion, - KubeadmVersion: kubeadmVersionStr, - KubeletVersions: kubeletVersions, - EtcdVersion: etcdVersion, + KubeVersion: clusterVersionStr, + DNSVersion: dnsVersion, + KubeadmVersion: kubeadmVersionStr, + KubeAPIServerVersions: kubeAPIServerVersions, + KubeControllerManagerVersions: kubeControllerManagerVersions, + KubeSchedulerVersions: kubeSchedulerVersions, + KubeletVersions: kubeletVersions, + EtcdVersions: etcdVersions, } // Do a "dumb guess" that a new minor upgrade is available just because the latest stable version is higher than the cluster version @@ -173,8 +203,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA KubeVersion: patchVersionStr, DNSVersion: kubeadmconstants.CoreDNSVersion, KubeadmVersion: newKubeadmVer, - EtcdVersion: getSuggestedEtcdVersion(externalEtcd, patchVersionStr), - // KubeletVersions is unset here as it is not used anywhere in .After + EtcdVersion: getSuggestedEtcdVersion(isExternalEtcd, patchVersionStr), }, }) } @@ -189,8 +218,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA KubeVersion: stableVersionStr, DNSVersion: kubeadmconstants.CoreDNSVersion, KubeadmVersion: stableVersionStr, - EtcdVersion: getSuggestedEtcdVersion(externalEtcd, stableVersionStr), - // KubeletVersions is unset here as it is not used anywhere in .After + EtcdVersion: getSuggestedEtcdVersion(isExternalEtcd, stableVersionStr), }, }) } @@ -236,8 +264,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA KubeVersion: previousBranchLatestVersionStr, DNSVersion: kubeadmconstants.CoreDNSVersion, KubeadmVersion: previousBranchLatestVersionStr, - EtcdVersion: getSuggestedEtcdVersion(externalEtcd, previousBranchLatestVersionStr), - // KubeletVersions is unset here as it is not used anywhere in .After + EtcdVersion: getSuggestedEtcdVersion(isExternalEtcd, previousBranchLatestVersionStr), }, }) } @@ -260,8 +287,7 @@ func GetAvailableUpgrades(versionGetterImpl VersionGetter, experimentalUpgradesA KubeVersion: unstableKubeVersion, DNSVersion: kubeadmconstants.CoreDNSVersion, KubeadmVersion: unstableKubeVersion, - EtcdVersion: getSuggestedEtcdVersion(externalEtcd, unstableKubeVersion), - // KubeletVersions is unset here as it is not used anywhere in .After + EtcdVersion: getSuggestedEtcdVersion(isExternalEtcd, unstableKubeVersion), }, }) } @@ -294,8 +320,8 @@ func minorUpgradePossibleWithPatchRelease(stableVersion, patchVersion *versionut return patchVersion.LessThan(stableVersion) } -func getSuggestedEtcdVersion(externalEtcd bool, kubernetesVersion string) string { - if externalEtcd { +func getSuggestedEtcdVersion(isExternalEtcd bool, kubernetesVersion string) string { + if isExternalEtcd { return "" } etcdVersion, warning, err := kubeadmconstants.EtcdSupportedVersion(kubeadmconstants.SupportedEtcdVersion, kubernetesVersion) @@ -308,3 +334,18 @@ func getSuggestedEtcdVersion(externalEtcd bool, kubernetesVersion string) string } return etcdVersion.String() } + +func getLatestClusterVersion(kubeAPIServerVersions map[string][]string) (*versionutil.Version, error) { + var latestVersion *versionutil.Version + for versionStr, nodes := range kubeAPIServerVersions { + ver, err := versionutil.ParseSemantic(versionStr) + if err != nil { + return nil, fmt.Errorf("couldn't parse kube-apiserver version %s from nodes %v", versionStr, nodes) + } + if latestVersion == nil || ver.AtLeast(latestVersion) { + latestVersion = ver + } + } + + return latestVersion, nil +} diff --git a/cmd/kubeadm/app/phases/upgrade/compute_test.go b/cmd/kubeadm/app/phases/upgrade/compute_test.go index 801e6eb0dcb..b2d7a93e075 100644 --- a/cmd/kubeadm/app/phases/upgrade/compute_test.go +++ b/cmd/kubeadm/app/phases/upgrade/compute_test.go @@ -18,11 +18,10 @@ package upgrade import ( "fmt" - "os" - "reflect" "strings" "testing" + "github.com/google/go-cmp/cmp" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,7 +33,16 @@ import ( ) type fakeVersionGetter struct { - clusterVersion, kubeadmVersion, stableVersion, latestVersion, latestDevBranchVersion, stablePatchVersion, kubeletVersion string + clusterVersion string + kubeadmVersion string + stableVersion string + latestVersion string + latestDevBranchVersion string + stablePatchVersion string + kubeletVersion string + componentVersion string + etcdVersion string + isExternalEtcd bool } var _ VersionGetter = &fakeVersionGetter{} @@ -63,26 +71,30 @@ func (f *fakeVersionGetter) VersionFromCILabel(ciVersionLabel, _ string) (string return f.stablePatchVersion, versionutil.MustParseSemantic(f.stablePatchVersion), nil } -// KubeletVersions gets the versions of the kubelets in the cluster -func (f *fakeVersionGetter) KubeletVersions() (map[string]uint16, error) { - return map[string]uint16{ - f.kubeletVersion: 1, +// KubeletVersions should return a map with a version and a list of node names that describes how many kubelets there are for that version +func (f *fakeVersionGetter) KubeletVersions() (map[string][]string, error) { + return map[string][]string{ + f.kubeletVersion: {"node1"}, + }, nil +} + +// ComponentVersions should return a map with a version and a list of node names that describes how many a given control-plane components there are for that version +func (f *fakeVersionGetter) ComponentVersions(name string) (map[string][]string, error) { + if name == constants.Etcd { + if f.isExternalEtcd { + return map[string][]string{}, nil + } + return map[string][]string{ + f.etcdVersion: {"node1"}, + }, nil + } + + return map[string][]string{ + f.componentVersion: {"node1"}, }, nil } const fakeCurrentEtcdVersion = "3.1.12" -const etcdStaticPod = `apiVersion: v1 -kind: Pod -metadata: - labels: - component: etcd - tier: control-plane - name: etcd - namespace: kube-system -spec: - containers: - - name: etcd - image: registry.k8s.io/etcd:` + fakeCurrentEtcdVersion func getEtcdVersion(v *versionutil.Version) string { etcdVer, _, _ := constants.EtcdSupportedVersion(constants.SupportedEtcdVersion, v.String()) @@ -123,15 +135,16 @@ func TestGetAvailableUpgrades(t *testing.T) { expectedUpgrades []Upgrade allowExperimental, allowRCs bool errExpected bool - externalEtcd bool beforeDNSVersion string }{ { name: "no action needed, already up-to-date", vg: &fakeVersionGetter{ - clusterVersion: v1Y0.String(), - kubeletVersion: v1Y0.String(), - kubeadmVersion: v1Y0.String(), + clusterVersion: v1Y0.String(), + componentVersion: v1Y0.String(), + kubeletVersion: v1Y0.String(), + kubeadmVersion: v1Y0.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1Y0.String(), stableVersion: v1Y0.String(), @@ -144,9 +157,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "simple patch version upgrade", vg: &fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Y2.String(), + clusterVersion: v1Y1.String(), + componentVersion: v1Y1.String(), + kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane + kubeadmVersion: v1Y2.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1Y3.String(), stableVersion: v1Y3.String(), @@ -157,12 +172,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), Before: ClusterState{ KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y1.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1Y1.String(): {"node1"}, }, KubeadmVersion: v1Y2.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Y3.String(), @@ -178,26 +202,36 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "simple patch version upgrade with external etcd", vg: &fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Y2.String(), + clusterVersion: v1Y1.String(), + componentVersion: v1Y1.String(), + kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane + kubeadmVersion: v1Y2.String(), + isExternalEtcd: true, stablePatchVersion: v1Y3.String(), stableVersion: v1Y3.String(), }, beforeDNSVersion: fakeCurrentCoreDNSVersion, - externalEtcd: true, expectedUpgrades: []Upgrade{ { Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), Before: ClusterState{ KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y1.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, }, + KubeControllerManagerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + EtcdVersions: map[string][]string{}, KubeadmVersion: v1Y2.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: "", }, After: ClusterState{ KubeVersion: v1Y3.String(), @@ -213,9 +247,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "no version provided to offline version getter does not change behavior", vg: NewOfflineVersionGetter(&fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Y2.String(), + clusterVersion: v1Y1.String(), + componentVersion: v1Y1.String(), + kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane + kubeadmVersion: v1Y2.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1Y3.String(), stableVersion: v1Y3.String(), @@ -226,12 +262,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), Before: ClusterState{ KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y1.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1Y1.String(): {"node1"}, }, KubeadmVersion: v1Y2.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Y3.String(), @@ -247,9 +292,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "minor version upgrade only", vg: &fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Z0.String(), + clusterVersion: v1Y1.String(), + componentVersion: v1Y1.String(), + kubeletVersion: v1Y1.String(), // the kubelet are on the same version as the control plane + kubeadmVersion: v1Z0.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1Y1.String(), stableVersion: v1Z0.String(), @@ -260,12 +307,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: "stable version", Before: ClusterState{ KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y1.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1Y1.String(): {"node1"}, }, KubeadmVersion: v1Z0.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Z0.String(), @@ -281,9 +337,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "both minor version upgrade and patch version upgrade available", vg: &fakeVersionGetter{ - clusterVersion: v1Y3.String(), - kubeletVersion: v1Y3.String(), // the kubelet are on the same version as the control plane - kubeadmVersion: v1Y5.String(), + clusterVersion: v1Y3.String(), + componentVersion: v1Y3.String(), + kubeletVersion: v1Y3.String(), // the kubelet are on the same version as the control plane + kubeadmVersion: v1Y5.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1Y5.String(), stableVersion: v1Z1.String(), @@ -294,12 +352,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), Before: ClusterState{ KubeVersion: v1Y3.String(), - KubeletVersions: map[string]uint16{ - v1Y3.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1Y3.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1Y3.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1Y3.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1Y3.String(): {"node1"}, }, KubeadmVersion: v1Y5.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Y5.String(), @@ -312,12 +379,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: "stable version", Before: ClusterState{ KubeVersion: v1Y3.String(), - KubeletVersions: map[string]uint16{ - v1Y3.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1Y3.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1Y3.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1Y3.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1Y3.String(): {"node1"}, }, KubeadmVersion: v1Y5.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Z1.String(), @@ -333,9 +409,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "allow experimental upgrades, but no upgrade available", vg: &fakeVersionGetter{ - clusterVersion: v1Z0alpha2.String(), - kubeletVersion: v1Y5.String(), - kubeadmVersion: v1Y5.String(), + clusterVersion: v1Z0alpha2.String(), + componentVersion: v1Z0alpha2.String(), + kubeletVersion: v1Y5.String(), + kubeadmVersion: v1Y5.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1Y5.String(), stableVersion: v1Y5.String(), @@ -349,9 +427,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "upgrade to an unstable version should be supported", vg: &fakeVersionGetter{ - clusterVersion: v1Y5.String(), - kubeletVersion: v1Y5.String(), - kubeadmVersion: v1Y5.String(), + clusterVersion: v1Y5.String(), + componentVersion: v1Y5.String(), + kubeletVersion: v1Y5.String(), + kubeadmVersion: v1Y5.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1Y5.String(), stableVersion: v1Y5.String(), @@ -363,12 +443,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: "experimental version", Before: ClusterState{ KubeVersion: v1Y5.String(), - KubeletVersions: map[string]uint16{ - v1Y5.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1Y5.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1Y5.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1Y5.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1Y5.String(): {"node1"}, }, KubeadmVersion: v1Y5.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Z0alpha2.String(), @@ -384,9 +473,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "upgrade from an unstable version to an unstable version should be supported", vg: &fakeVersionGetter{ - clusterVersion: v1Z0alpha1.String(), - kubeletVersion: v1Y5.String(), - kubeadmVersion: v1Y5.String(), + clusterVersion: v1Z0alpha1.String(), + componentVersion: v1Z0alpha1.String(), + kubeletVersion: v1Y5.String(), + kubeadmVersion: v1Y5.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1Y5.String(), stableVersion: v1Y5.String(), @@ -398,12 +489,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: "experimental version", Before: ClusterState{ KubeVersion: v1Z0alpha1.String(), - KubeletVersions: map[string]uint16{ - v1Y5.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1Z0alpha1.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1Z0alpha1.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1Z0alpha1.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1Y5.String(): {"node1"}, }, KubeadmVersion: v1Y5.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Z0alpha2.String(), @@ -419,9 +519,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "v1.X.0-alpha.0 should be ignored", vg: &fakeVersionGetter{ - clusterVersion: v1X5.String(), - kubeletVersion: v1X5.String(), - kubeadmVersion: v1X5.String(), + clusterVersion: v1X5.String(), + componentVersion: v1X5.String(), + kubeletVersion: v1X5.String(), + kubeadmVersion: v1X5.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1X5.String(), stableVersion: v1X5.String(), @@ -434,12 +536,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: "experimental version", Before: ClusterState{ KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1X5.String(): {"node1"}, }, KubeadmVersion: v1X5.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Z0beta1.String(), @@ -455,9 +566,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "upgrade to an RC version should be supported", vg: &fakeVersionGetter{ - clusterVersion: v1X5.String(), - kubeletVersion: v1X5.String(), - kubeadmVersion: v1X5.String(), + clusterVersion: v1X5.String(), + componentVersion: v1X5.String(), + kubeletVersion: v1X5.String(), + kubeadmVersion: v1X5.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1X5.String(), stableVersion: v1X5.String(), @@ -470,12 +583,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: "release candidate version", Before: ClusterState{ KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1X5.String(): {"node1"}, }, KubeadmVersion: v1X5.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Z0rc1.String(), @@ -491,9 +613,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "it is possible (but very uncommon) that the latest version from the previous branch is an rc and the current latest version is alpha.0. In that case, show the RC", vg: &fakeVersionGetter{ - clusterVersion: v1X5.String(), - kubeletVersion: v1X5.String(), - kubeadmVersion: v1X5.String(), + clusterVersion: v1X5.String(), + componentVersion: v1X5.String(), + kubeletVersion: v1X5.String(), + kubeadmVersion: v1X5.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1X5.String(), stableVersion: v1X5.String(), @@ -506,12 +630,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: "experimental version", // Note that this is considered an experimental version in this uncommon scenario Before: ClusterState{ KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1X5.String(): {"node1"}, }, KubeadmVersion: v1X5.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Z0rc1.String(), @@ -527,9 +660,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "upgrade to an RC version should be supported. There may also be an even newer unstable version.", vg: &fakeVersionGetter{ - clusterVersion: v1X5.String(), - kubeletVersion: v1X5.String(), - kubeadmVersion: v1X5.String(), + clusterVersion: v1X5.String(), + componentVersion: v1X5.String(), + kubeletVersion: v1X5.String(), + kubeadmVersion: v1X5.String(), + etcdVersion: fakeCurrentEtcdVersion, stablePatchVersion: v1X5.String(), stableVersion: v1X5.String(), @@ -542,12 +677,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: "release candidate version", Before: ClusterState{ KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1X5.String(): {"node1"}, }, KubeadmVersion: v1X5.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Z0rc1.String(), @@ -560,12 +704,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: "experimental version", Before: ClusterState{ KubeVersion: v1X5.String(), - KubeletVersions: map[string]uint16{ - v1X5.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1X5.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1X5.String(): {"node1"}, }, KubeadmVersion: v1X5.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Y0alpha1.String(), @@ -582,9 +735,11 @@ func TestGetAvailableUpgrades(t *testing.T) { { name: "offline version getter", vg: NewOfflineVersionGetter(&fakeVersionGetter{ - clusterVersion: v1Y1.String(), - kubeletVersion: v1Y0.String(), - kubeadmVersion: v1Y1.String(), + clusterVersion: v1Y1.String(), + componentVersion: v1Y1.String(), + kubeletVersion: v1Y0.String(), + kubeadmVersion: v1Y1.String(), + etcdVersion: fakeCurrentEtcdVersion, }, v1Z1.String()), beforeDNSVersion: fakeCurrentCoreDNSVersion, expectedUpgrades: []Upgrade{ @@ -592,12 +747,21 @@ func TestGetAvailableUpgrades(t *testing.T) { Description: fmt.Sprintf("version in the v%d.%d series", v1Y0.Major(), v1Y0.Minor()), Before: ClusterState{ KubeVersion: v1Y1.String(), - KubeletVersions: map[string]uint16{ - v1Y0.String(): 1, + KubeAPIServerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeControllerManagerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeSchedulerVersions: map[string][]string{ + v1Y1.String(): {"node1"}, + }, + KubeletVersions: map[string][]string{ + v1Y0.String(): {"node1"}, }, KubeadmVersion: v1Y1.String(), DNSVersion: fakeCurrentCoreDNSVersion, - EtcdVersion: fakeCurrentEtcdVersion, + EtcdVersions: map[string][]string{fakeCurrentEtcdVersion: {"node1"}}, }, After: ClusterState{ KubeVersion: v1Z1.String(), @@ -642,27 +806,18 @@ func TestGetAvailableUpgrades(t *testing.T) { }, }) - manifestsDir, err := os.MkdirTemp("", "GetAvailableUpgrades-test-manifests") - if err != nil { - t.Fatalf("Unable to create temporary directory: %v", err) - } - defer os.RemoveAll(manifestsDir) - - if err = os.WriteFile(constants.GetStaticPodFilepath(constants.Etcd, manifestsDir), []byte(etcdStaticPod), 0644); err != nil { - t.Fatalf("Unable to create test static pod manifest: %v", err) - } - - actualUpgrades, actualErr := GetAvailableUpgrades(rt.vg, rt.allowExperimental, rt.allowRCs, rt.externalEtcd, client, manifestsDir, &output.TextPrinter{}) - if !reflect.DeepEqual(actualUpgrades, rt.expectedUpgrades) { - t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades: %v\n\tgot: %v", rt.expectedUpgrades, actualUpgrades) + actualUpgrades, actualErr := GetAvailableUpgrades(rt.vg, rt.allowExperimental, rt.allowRCs, client, &output.TextPrinter{}) + if diff := cmp.Diff(rt.expectedUpgrades, actualUpgrades); len(diff) > 0 { + t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades:\n%v\n\tgot:\n%v\n\tdiff:\n%v", rt.expectedUpgrades, actualUpgrades, diff) } if rt.errExpected && actualErr == nil { t.Error("unexpected success") } else if !rt.errExpected && actualErr != nil { t.Errorf("unexpected failure: %v", actualErr) } - if !reflect.DeepEqual(actualUpgrades, rt.expectedUpgrades) { - t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades: %v\n\tgot: %v", rt.expectedUpgrades, actualUpgrades) + if diff := cmp.Diff(rt.expectedUpgrades, actualUpgrades); len(diff) > 0 { + t.Logf("diff: %s", cmp.Diff(rt.expectedUpgrades, actualUpgrades)) + t.Errorf("failed TestGetAvailableUpgrades\n\texpected upgrades:\n%v\n\tgot:\n%v\n\tdiff:\n%v", rt.expectedUpgrades, actualUpgrades, diff) } }) } @@ -671,46 +826,46 @@ func TestGetAvailableUpgrades(t *testing.T) { func TestKubeletUpgrade(t *testing.T) { tests := []struct { name string - before map[string]uint16 + before map[string][]string after string expected bool }{ { name: "upgrade from v1.10.1 to v1.10.3 is available", - before: map[string]uint16{ - "v1.10.1": 1, + before: map[string][]string{ + "v1.10.1": {"node1"}, }, after: "v1.10.3", expected: true, }, { - name: "upgrade from v1.10.1 and v1.10.3/100 to v1.10.3 is available", - before: map[string]uint16{ - "v1.10.1": 1, - "v1.10.3": 100, + name: "upgrade from v1.10.1 and v1.10.3/2 to v1.10.3 is available", + before: map[string][]string{ + "v1.10.1": {"node1"}, + "v1.10.3": {"node2", "node3"}, }, after: "v1.10.3", expected: true, }, { name: "upgrade from v1.10.3 to v1.10.3 is not available", - before: map[string]uint16{ - "v1.10.3": 1, + before: map[string][]string{ + "v1.10.3": {"node1"}, }, after: "v1.10.3", expected: false, }, { - name: "upgrade from v1.10.3/100 to v1.10.3 is not available", - before: map[string]uint16{ - "v1.10.3": 100, + name: "upgrade from v1.10.3/2 to v1.10.3 is not available", + before: map[string][]string{ + "v1.10.3": {"node1", "node2"}, }, after: "v1.10.3", expected: false, }, { name: "upgrade is not available if we don't know anything about the earlier state", - before: map[string]uint16{}, + before: map[string][]string{}, after: "v1.10.3", expected: false, }, diff --git a/cmd/kubeadm/app/phases/upgrade/policy.go b/cmd/kubeadm/app/phases/upgrade/policy.go index 497e03747dd..69abf1387d2 100644 --- a/cmd/kubeadm/app/phases/upgrade/policy.go +++ b/cmd/kubeadm/app/phases/upgrade/policy.go @@ -168,7 +168,7 @@ func detectUnstableVersionError(newK8sVersion *version.Version, newK8sVersionStr } // detectTooOldKubelets errors out if the kubelet versions are so old that an unsupported skew would happen if the cluster was upgraded -func detectTooOldKubelets(newK8sVersion *version.Version, kubeletVersions map[string]uint16) error { +func detectTooOldKubelets(newK8sVersion *version.Version, kubeletVersions map[string][]string) error { var tooOldKubeletVersions []string for versionStr := range kubeletVersions { diff --git a/cmd/kubeadm/app/phases/upgrade/versiongetter.go b/cmd/kubeadm/app/phases/upgrade/versiongetter.go index 12bebe5166b..836eed06139 100644 --- a/cmd/kubeadm/app/phases/upgrade/versiongetter.go +++ b/cmd/kubeadm/app/phases/upgrade/versiongetter.go @@ -18,10 +18,10 @@ package upgrade import ( "context" + "fmt" "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" versionutil "k8s.io/apimachinery/pkg/util/version" pkgversion "k8s.io/apimachinery/pkg/version" @@ -29,7 +29,9 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/component-base/version" + "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + "k8s.io/kubernetes/cmd/kubeadm/app/util/image" ) // VersionGetter defines an interface for fetching different versions. @@ -41,8 +43,10 @@ type VersionGetter interface { KubeadmVersion() (string, *versionutil.Version, error) // VersionFromCILabel should resolve CI labels like `latest`, `stable`, `stable-1.8`, etc. to real versions VersionFromCILabel(string, string) (string, *versionutil.Version, error) - // KubeletVersions should return a map with a version and a number that describes how many kubelets there are for that version - KubeletVersions() (map[string]uint16, error) + // KubeletVersions should return a map with a version and a list of node names that describes how many kubelets there are for that version + KubeletVersions() (map[string][]string, error) + // ComponentVersions should return a map with a version and a list of node names that describes how many a given control-plane components there are for that version + ComponentVersions(string) (map[string][]string, error) } // KubeVersionGetter handles the version-fetching mechanism from external sources @@ -111,27 +115,42 @@ func (g *KubeVersionGetter) VersionFromCILabel(ciVersionLabel, description strin return versionStr, ver, nil } -// KubeletVersions gets the versions of the kubelets in the cluster -func (g *KubeVersionGetter) KubeletVersions() (map[string]uint16, error) { +// KubeletVersions gets the versions of the kubelets in the cluster. +func (g *KubeVersionGetter) KubeletVersions() (map[string][]string, error) { nodes, err := g.client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) if err != nil { return nil, errors.New("couldn't list all nodes in cluster") } - return computeKubeletVersions(nodes.Items), nil + + // map kubelet version to a list of node names + kubeletVersions := make(map[string][]string) + for _, node := range nodes.Items { + kver := node.Status.NodeInfo.KubeletVersion + kubeletVersions[kver] = append(kubeletVersions[kver], node.Name) + } + return kubeletVersions, nil } -// computeKubeletVersions returns a string-int map that describes how many nodes are of a specific version -func computeKubeletVersions(nodes []v1.Node) map[string]uint16 { - kubeletVersions := make(map[string]uint16) - for _, node := range nodes { - kver := node.Status.NodeInfo.KubeletVersion - if _, found := kubeletVersions[kver]; !found { - kubeletVersions[kver] = 1 - continue - } - kubeletVersions[kver]++ +// ComponentVersions gets the versions of the control-plane components in the cluster. +// The name parameter is the name of the component to get the versions for. +// The function returns a map with the version as the key and a list of node names as the value. +func (g *KubeVersionGetter) ComponentVersions(name string) (map[string][]string, error) { + podList, err := g.client.CoreV1().Pods(metav1.NamespaceSystem).List( + context.TODO(), + metav1.ListOptions{ + LabelSelector: fmt.Sprintf("component=%s,tier=%s", name, constants.ControlPlaneTier), + }, + ) + if err != nil { + return nil, errors.Wrap(err, "couldn't list pods in cluster") } - return kubeletVersions + + componentVersions := make(map[string][]string) + for _, pod := range podList.Items { + tag := convertImageTagMetadataToSemver(image.TagFromImage(pod.Spec.Containers[0].Image)) + componentVersions[tag] = append(componentVersions[tag], pod.Spec.NodeName) + } + return componentVersions, nil } // OfflineVersionGetter will use the version provided or diff --git a/cmd/kubeadm/app/phases/upgrade/versiongetter_test.go b/cmd/kubeadm/app/phases/upgrade/versiongetter_test.go index 874c7fe8ee4..e0027ea3917 100644 --- a/cmd/kubeadm/app/phases/upgrade/versiongetter_test.go +++ b/cmd/kubeadm/app/phases/upgrade/versiongetter_test.go @@ -148,7 +148,7 @@ func TestKubeVersionGetterKubeletVersions(t *testing.T) { tests := []struct { name string nodes *v1.NodeList - want map[string]uint16 + want map[string][]string wantErr bool }{ { @@ -181,9 +181,9 @@ func TestKubeVersionGetterKubeletVersions(t *testing.T) { }, }, }, - want: map[string]uint16{ - "v1.28.0": 1, - "v1.28.1": 2, + want: map[string][]string{ + "v1.28.0": {"node1"}, + "v1.28.1": {"node2", "node3"}, }, wantErr: false, }, @@ -201,15 +201,15 @@ func TestKubeVersionGetterKubeletVersions(t *testing.T) { }, }, }, - want: map[string]uint16{ - "": 2, + want: map[string][]string{ + "": {"node2", "node3"}, }, wantErr: false, }, { name: "node list is empty", nodes: &v1.NodeList{}, - want: map[string]uint16{}, + want: map[string][]string{}, wantErr: false, }, }