From 207ffa7bdc63da125838037961f1bc54e827bca3 Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Thu, 29 Jul 2021 00:13:26 +0300 Subject: [PATCH 1/2] kubeadm: dynamically populate the current/minimum k8s versions Kubeadm requires manual version updates of its current supported k8s control plane version and minimally supported k8s control plane and kubelet versions every release cycle. To avoid that, in constants.go: - Add the helper function getSkewedKubernetesVersion() that can be used to retrieve a MAJOR.(MINOR+n).0 version of k8s. It currently uses the kubeadm version populated in "component-base/version" during the kubeadm build process. - Use the function to set existing version constants (variables). Update util/config/common.go#NormalizeKubernetesVersion() to tolerate the case where a k8s version in the ClusterConfiguration is too old for the kubeadm binary to use during code freeze. Include unit tests for the new utilities. --- cmd/kubeadm/app/constants/constants.go | 52 ++++++++++++++++-- cmd/kubeadm/app/constants/constants_test.go | 59 +++++++++++++++++++++ cmd/kubeadm/app/util/config/common.go | 38 ++++++++++++- cmd/kubeadm/app/util/config/common_test.go | 54 +++++++++++++++++++ 4 files changed, 198 insertions(+), 5 deletions(-) diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index b3559734a33..a64d65abdb0 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -29,7 +29,9 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" + apimachineryversion "k8s.io/apimachinery/pkg/version" bootstrapapi "k8s.io/cluster-bootstrap/token/api" + componentversion "k8s.io/component-base/version" utilnet "k8s.io/utils/net" "github.com/pkg/errors" @@ -448,13 +450,13 @@ var ( ControlPlaneComponents = []string{KubeAPIServer, KubeControllerManager, KubeScheduler} // MinimumControlPlaneVersion specifies the minimum control plane version kubeadm can deploy - MinimumControlPlaneVersion = version.MustParseSemantic("v1.21.0") + MinimumControlPlaneVersion = getSkewedKubernetesVersion(-1) // MinimumKubeletVersion specifies the minimum version of kubelet which kubeadm supports - MinimumKubeletVersion = version.MustParseSemantic("v1.21.0") + MinimumKubeletVersion = getSkewedKubernetesVersion(-1) // CurrentKubernetesVersion specifies current Kubernetes version supported by kubeadm - CurrentKubernetesVersion = version.MustParseSemantic("v1.22.0") + CurrentKubernetesVersion = getSkewedKubernetesVersion(0) // SupportedEtcdVersion lists officially supported etcd versions with corresponding Kubernetes releases SupportedEtcdVersion = map[uint8]string{ @@ -483,8 +485,52 @@ var ( Factor: 1.0, Jitter: 0.1, } + + // defaultKubernetesVersionForTests is the default version used for unit tests. + // The MINOR should be at least 3 as some tests subtract 3 from the MINOR version. + defaultKubernetesVersionForTests = version.MustParseSemantic("v1.3.0") ) +// isRunningInTest can be used to determine if the code in this file is being run in a test. +func isRunningInTest() bool { + return strings.HasSuffix(os.Args[0], ".test") +} + +// getSkewedKubernetesVersion returns the current MAJOR.(MINOR+n).0 Kubernetes version with a skew of 'n' +// It uses the kubeadm version provided by the 'component-base/version' package. This version must be populated +// by passing linker flags during the kubeadm build process. If the version is empty, assume that kubeadm +// was either build incorrectly or this code is running in unit tests. +func getSkewedKubernetesVersion(n int) *version.Version { + versionInfo := componentversion.Get() + ver := getSkewedKubernetesVersionImpl(&versionInfo, n, isRunningInTest) + if ver == nil { + panic("kubeadm is not build properly using 'make ...': missing component version information") + } + return ver +} + +func getSkewedKubernetesVersionImpl(versionInfo *apimachineryversion.Info, n int, isRunningInTestFunc func() bool) *version.Version { + // TODO: update if the kubeadm version gets decoupled from the Kubernetes version. + // This would require keeping track of the supported skew in a table. + // More changes would be required if the kubelet version one day decouples from that of Kubernetes. + var ver *version.Version + if len(versionInfo.Major) == 0 { + if isRunningInTestFunc() { + ver = defaultKubernetesVersionForTests // An arbitrary version for testing purposes + } else { + // If this is not running in tests assume that the kubeadm binary is not build properly + return nil + } + } else { + ver = version.MustParseSemantic(versionInfo.GitVersion) + } + // Append the MINOR version skew. + // TODO: handle the case of Kubernetes moving to v2.0 or having MAJOR version updates in the future. + // This would require keeping track (in a table) of the last MINOR for a particular MAJOR. + minor := uint(int(ver.Minor()) + n) + return version.MustParseSemantic(fmt.Sprintf("v%d.%d.0", ver.Major(), minor)) +} + // EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release // If passed version is not in the given list, the function returns the nearest version with a warning func EtcdSupportedVersion(supportedEtcdVersion map[uint8]string, versionString string) (etcdVersion *version.Version, warning, err error) { diff --git a/cmd/kubeadm/app/constants/constants_test.go b/cmd/kubeadm/app/constants/constants_test.go index 30872c4d123..6e05fc4ad83 100644 --- a/cmd/kubeadm/app/constants/constants_test.go +++ b/cmd/kubeadm/app/constants/constants_test.go @@ -21,6 +21,7 @@ import ( "testing" "k8s.io/apimachinery/pkg/util/version" + apimachineryversion "k8s.io/apimachinery/pkg/version" ) func TestGetStaticPodDirectory(t *testing.T) { @@ -237,3 +238,61 @@ func TestGetKubernetesServiceCIDR(t *testing.T) { }) } } + +func TestGetSkewedKubernetesVersionImpl(t *testing.T) { + tests := []struct { + name string + versionInfo *apimachineryversion.Info + n int + isRunningInTestFunc func() bool + expectedResult *version.Version + }{ + { + name: "invalid versionInfo; running in test", + versionInfo: &apimachineryversion.Info{}, + expectedResult: defaultKubernetesVersionForTests, + }, + { + name: "invalid versionInfo; not running in test", + versionInfo: &apimachineryversion.Info{}, + isRunningInTestFunc: func() bool { return false }, + expectedResult: nil, + }, + { + name: "valid skew of -1", + versionInfo: &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"}, + n: -1, + expectedResult: version.MustParseSemantic("v1.22.0"), + }, + { + name: "valid skew of 0", + versionInfo: &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"}, + n: 0, + expectedResult: version.MustParseSemantic("v1.23.0"), + }, + { + name: "valid skew of +1", + versionInfo: &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"}, + n: 1, + expectedResult: version.MustParseSemantic("v1.24.0"), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if tc.isRunningInTestFunc == nil { + tc.isRunningInTestFunc = func() bool { return true } + } + result := getSkewedKubernetesVersionImpl(tc.versionInfo, tc.n, tc.isRunningInTestFunc) + if (tc.expectedResult == nil) != (result == nil) { + t.Errorf("expected result: %v, got: %v", tc.expectedResult, result) + } + if result == nil { + return + } + if cmp, _ := result.Compare(tc.expectedResult.String()); cmp != 0 { + t.Errorf("expected result: %v, got %v", tc.expectedResult, result) + } + }) + } +} diff --git a/cmd/kubeadm/app/util/config/common.go b/cmd/kubeadm/app/util/config/common.go index 184d6d11632..04440549468 100644 --- a/cmd/kubeadm/app/util/config/common.go +++ b/cmd/kubeadm/app/util/config/common.go @@ -32,6 +32,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" netutil "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/version" + apimachineryversion "k8s.io/apimachinery/pkg/version" + componentversion "k8s.io/component-base/version" "k8s.io/klog/v2" "github.com/pkg/errors" @@ -102,8 +104,23 @@ func NormalizeKubernetesVersion(cfg *kubeadmapi.ClusterConfiguration) error { if err != nil { return errors.Wrapf(err, "couldn't parse Kubernetes version %q", cfg.KubernetesVersion) } - if k8sVersion.LessThan(constants.MinimumControlPlaneVersion) { - return errors.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", constants.MinimumControlPlaneVersion.String(), cfg.KubernetesVersion) + + // During the k8s release process, a kubeadm version in the main branch could be 1.23.0-pre, + // while the 1.22.0 version is not released yet. The MinimumControlPlaneVersion validation + // in such a case will not pass, since the value of MinimumControlPlaneVersion would be + // calculated as kubeadm version - 1 (1.22) and k8sVersion would still be at 1.21.x + // (fetched from the 'stable' marker). Handle this case by only showing a warning. + mcpVersion := constants.MinimumControlPlaneVersion + versionInfo := componentversion.Get() + if isKubeadmPrereleaseVersion(&versionInfo, k8sVersion, mcpVersion) { + klog.V(1).Infof("WARNING: tolerating control plane version %s, assuming that k8s version %s is not released yet", + cfg.KubernetesVersion, mcpVersion) + return nil + } + // If not a pre-release version, handle the validation normally. + if k8sVersion.LessThan(mcpVersion) { + return errors.Errorf("this version of kubeadm only supports deploying clusters with the control plane version >= %s. Current version: %s", + mcpVersion, cfg.KubernetesVersion) } return nil } @@ -204,3 +221,20 @@ func MigrateOldConfig(oldConfig []byte) ([]byte, error) { return bytes.Join(newConfig, []byte(constants.YAMLDocumentSeparator)), nil } + +// isKubeadmPrereleaseVersion returns true if the kubeadm version is a pre-release version and +// the minimum control plane version is N+2 MINOR version of the given k8sVersion. +func isKubeadmPrereleaseVersion(versionInfo *apimachineryversion.Info, k8sVersion, mcpVersion *version.Version) bool { + if len(versionInfo.Major) != 0 { // Make sure the component version is populated + kubeadmVersion := version.MustParseSemantic(versionInfo.String()) + if len(kubeadmVersion.PreRelease()) != 0 { // Only handle this if the kubeadm binary is a pre-release + // After incrementing the k8s MINOR version by one, if this version is equal or greater than the + // MCP version, return true. + v := k8sVersion.WithMinor(k8sVersion.Minor() + 1) + if comp, _ := v.Compare(mcpVersion.String()); comp != -1 { + return true + } + } + } + return false +} diff --git a/cmd/kubeadm/app/util/config/common_test.go b/cmd/kubeadm/app/util/config/common_test.go index 9b01e848bff..58eb93d87b7 100644 --- a/cmd/kubeadm/app/util/config/common_test.go +++ b/cmd/kubeadm/app/util/config/common_test.go @@ -26,6 +26,8 @@ import ( kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/version" + apimachineryversion "k8s.io/apimachinery/pkg/version" "github.com/lithammer/dedent" ) @@ -397,3 +399,55 @@ func TestMigrateOldConfigFromFile(t *testing.T) { }) } } + +func TestIsKubeadmPrereleaseVersion(t *testing.T) { + validVersionInfo := &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0-alpha.1"} + tests := []struct { + name string + versionInfo *apimachineryversion.Info + k8sVersion *version.Version + mcpVersion *version.Version + expectedResult bool + }{ + { + name: "invalid versionInfo", + versionInfo: &apimachineryversion.Info{}, + expectedResult: false, + }, + { + name: "kubeadm is not a prerelease version", + versionInfo: &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"}, + expectedResult: false, + }, + { + name: "mcpVersion is equal to k8sVersion", + versionInfo: validVersionInfo, + k8sVersion: version.MustParseSemantic("v1.21.0"), + mcpVersion: version.MustParseSemantic("v1.21.0"), + expectedResult: true, + }, + { + name: "k8sVersion is 1 MINOR version older than mcpVersion", + versionInfo: validVersionInfo, + k8sVersion: version.MustParseSemantic("v1.21.0"), + mcpVersion: version.MustParseSemantic("v1.22.0"), + expectedResult: true, + }, + { + name: "k8sVersion is 2 MINOR versions older than mcpVersion", + versionInfo: validVersionInfo, + k8sVersion: version.MustParseSemantic("v1.21.0"), + mcpVersion: version.MustParseSemantic("v1.23.0"), + expectedResult: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := isKubeadmPrereleaseVersion(tc.versionInfo, tc.k8sVersion, tc.mcpVersion) + if result != tc.expectedResult { + t.Errorf("expected result: %v, got %v", tc.expectedResult, result) + } + }) + } +} From e3538edc227b3495bbedaf1b646317f9e1a022fe Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Thu, 29 Jul 2021 04:30:27 +0300 Subject: [PATCH 2/2] kubeadm: update unit tests to support dynamic version updates Tests under /app and /test would fail if the current/minimum k8s version is dynamically populated from the version in the kubeadm binary. Adapt the tests to support that. --- cmd/kubeadm/app/cmd/config_test.go | 6 +++--- cmd/kubeadm/app/phases/upgrade/compute_test.go | 3 ++- cmd/kubeadm/app/phases/upgrade/policy_test.go | 14 +++++++------- cmd/kubeadm/app/preflight/checks_test.go | 2 +- cmd/kubeadm/test/cmd/init_test.go | 15 +++++++++++++-- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/cmd/kubeadm/app/cmd/config_test.go b/cmd/kubeadm/app/cmd/config_test.go index 46644acd891..8477fcec644 100644 --- a/cmd/kubeadm/app/cmd/config_test.go +++ b/cmd/kubeadm/app/cmd/config_test.go @@ -208,8 +208,8 @@ func TestConfigImagesListRunWithoutPath(t *testing.T) { func TestConfigImagesListOutput(t *testing.T) { - etcdVersion, ok := constants.SupportedEtcdVersion[uint8(dummyKubernetesVersion.Minor())] - if !ok { + etcdVersion, _, err := constants.EtcdSupportedVersion(constants.SupportedEtcdVersion, dummyKubernetesVersionStr) + if err != nil { t.Fatalf("cannot determine etcd version for Kubernetes version %s", dummyKubernetesVersionStr) } versionMapping := struct { @@ -218,7 +218,7 @@ func TestConfigImagesListOutput(t *testing.T) { PauseVersion string CoreDNSVersion string }{ - EtcdVersion: etcdVersion, + EtcdVersion: etcdVersion.String(), KubeVersion: "v" + dummyKubernetesVersionStr, PauseVersion: constants.PauseVersion, CoreDNSVersion: constants.CoreDNSVersion, diff --git a/cmd/kubeadm/app/phases/upgrade/compute_test.go b/cmd/kubeadm/app/phases/upgrade/compute_test.go index 85b3e78cdc0..220232bd7e6 100644 --- a/cmd/kubeadm/app/phases/upgrade/compute_test.go +++ b/cmd/kubeadm/app/phases/upgrade/compute_test.go @@ -85,7 +85,8 @@ spec: image: k8s.gcr.io/etcd:` + fakeCurrentEtcdVersion func getEtcdVersion(v *versionutil.Version) string { - return constants.SupportedEtcdVersion[uint8(v.Minor())] + etcdVer, _, _ := constants.EtcdSupportedVersion(constants.SupportedEtcdVersion, v.String()) + return etcdVer.String() } const fakeCurrentCoreDNSVersion = "1.0.6" diff --git a/cmd/kubeadm/app/phases/upgrade/policy_test.go b/cmd/kubeadm/app/phases/upgrade/policy_test.go index 81e35898395..9cebdf8b045 100644 --- a/cmd/kubeadm/app/phases/upgrade/policy_test.go +++ b/cmd/kubeadm/app/phases/upgrade/policy_test.go @@ -76,7 +76,7 @@ func TestEnforceVersionPolicies(t *testing.T) { kubeletVersion: "v1.12.3", kubeadmVersion: "v1.12.3", }, - newK8sVersion: "v1.11.10", + newK8sVersion: "v1.10.10", expectedMandatoryErrs: 1, // version must be higher than v1.12.0 expectedSkippableErrs: 1, // can't upgrade old k8s with newer kubeadm }, @@ -85,9 +85,9 @@ func TestEnforceVersionPolicies(t *testing.T) { vg: &fakeVersionGetter{ clusterVersion: "v1.11.3", kubeletVersion: "v1.11.3", - kubeadmVersion: constants.CurrentKubernetesVersion.String(), + kubeadmVersion: "v1.13.0", }, - newK8sVersion: constants.CurrentKubernetesVersion.String(), + newK8sVersion: "v1.13.0", expectedMandatoryErrs: 1, // can't upgrade two minor versions expectedSkippableErrs: 1, // kubelet <-> apiserver skew too large }, @@ -124,11 +124,11 @@ func TestEnforceVersionPolicies(t *testing.T) { { name: "the maximum skew between the cluster version and the kubelet versions should be one minor version. This may be forced through though.", vg: &fakeVersionGetter{ - clusterVersion: constants.MinimumControlPlaneVersion.WithPatch(3).String(), - kubeletVersion: "v1.12.8", - kubeadmVersion: constants.CurrentKubernetesVersion.String(), + clusterVersion: "v1.12.0", + kubeletVersion: "v1.10.8", + kubeadmVersion: "v1.12.0", }, - newK8sVersion: constants.CurrentKubernetesVersion.String(), + newK8sVersion: "v1.12.0", expectedSkippableErrs: 1, }, { diff --git a/cmd/kubeadm/app/preflight/checks_test.go b/cmd/kubeadm/app/preflight/checks_test.go index 80b51a8f0cb..760689025ae 100644 --- a/cmd/kubeadm/app/preflight/checks_test.go +++ b/cmd/kubeadm/app/preflight/checks_test.go @@ -793,7 +793,7 @@ func TestKubeletVersionCheck(t *testing.T) { expectWarnings bool }{ {"v" + constants.CurrentKubernetesVersion.WithPatch(2).String(), "", false, false}, // check minimally supported version when there is no information about control plane - {"v1.11.3", "v1.11.8", true, false}, // too old kubelet (older than kubeadmconstants.MinimumKubeletVersion), should fail. + {"v1.1.0", "v1.11.8", true, false}, // too old kubelet, should fail. {"v" + constants.MinimumKubeletVersion.String(), constants.MinimumControlPlaneVersion.WithPatch(5).String(), false, false}, // kubelet within same major.minor as control plane {"v" + constants.MinimumKubeletVersion.WithPatch(5).String(), constants.MinimumControlPlaneVersion.WithPatch(1).String(), false, false}, // kubelet is newer, but still within same major.minor as control plane {"v" + constants.MinimumKubeletVersion.String(), constants.CurrentKubernetesVersion.WithPatch(1).String(), false, false}, // kubelet is lower than control plane, but newer than minimally supported diff --git a/cmd/kubeadm/test/cmd/init_test.go b/cmd/kubeadm/test/cmd/init_test.go index 5c376b7a2d1..1e2d82ae9c9 100644 --- a/cmd/kubeadm/test/cmd/init_test.go +++ b/cmd/kubeadm/test/cmd/init_test.go @@ -19,9 +19,10 @@ package kubeadm import ( "fmt" "os" + "strings" "testing" - "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/apimachinery/pkg/util/version" "github.com/lithammer/dedent" ) @@ -37,6 +38,16 @@ func runKubeadmInit(args ...string) (string, string, int, error) { return RunCmd(kubeadmPath, kubeadmArgs...) } +func getKubeadmVersion() *version.Version { + kubeadmPath := getKubeadmPath() + kubeadmArgs := []string{"version", "-o=short"} + out, _, _, err := RunCmd(kubeadmPath, kubeadmArgs...) + if err != nil { + panic(fmt.Sprintf("could not run 'kubeadm version -o=short': %v", err)) + } + return version.MustParseSemantic(strings.TrimSpace(out)) +} + func TestCmdInitToken(t *testing.T) { initTest := []struct { name string @@ -94,7 +105,7 @@ func TestCmdInitKubernetesVersion(t *testing.T) { }, { name: "valid version is accepted", - args: "--kubernetes-version=" + constants.CurrentKubernetesVersion.String(), + args: "--kubernetes-version=" + getKubeadmVersion().String(), expected: true, }, }