From b551ad596a6e5972ecf7cc3ad7ed29150d9d8b25 Mon Sep 17 00:00:00 2001 From: "Rostislav M. Georgiev" Date: Wed, 7 Nov 2018 14:26:08 +0200 Subject: [PATCH] kubeadm: UnifiedControlPlaneImage -> UseHyperKubeImage Up until now UnifiedControlPlaneImage existed as a string value as part of the ClusterConfiguration. This provided an override for the Kubernetes core component images with a single custom image. It is mostly used to override the control plane images with the hyperkube image. This saves both bandwith and disk space on the control plane nodes. Unfortunately, this specified an entire image string (complete with its prefix, image name and tag). This disables upgrades of setups that use hyperkube. Therefore, to enable upgrades on hyperkube setups and to make configuration more convenient, the UnifiedControlPlaneImage option is replaced with a boolean option, called UseHyperKubeImage. If set to true, this option replaces the image name of any Kubernetes core components with hyperkube, thus allowing for upgrades and respecting the image repository and version, specified in the ClusterConfiguration. Signed-off-by: Rostislav M. Georgiev --- cmd/kubeadm/app/apis/kubeadm/types.go | 5 +- cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD | 2 + .../app/apis/kubeadm/v1alpha3/conversion.go | 29 ++++++++ .../apis/kubeadm/v1alpha3/conversion_test.go | 72 +++++++++++++++++++ .../v1alpha3/zz_generated.conversion.go | 4 +- cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go | 2 +- cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go | 6 +- .../v1beta1/zz_generated.conversion.go | 4 +- cmd/kubeadm/app/cmd/upgrade/common_test.go | 2 - cmd/kubeadm/app/constants/constants.go | 2 + cmd/kubeadm/app/images/images.go | 18 +++-- cmd/kubeadm/app/images/images_test.go | 6 +- .../app/phases/upgrade/staticpods_test.go | 2 +- .../testdata/conversion/master/internal.yaml | 2 +- .../testdata/conversion/master/v1alpha3.yaml | 2 +- .../testdata/conversion/master/v1beta1.yaml | 2 +- .../testdata/defaulting/master/defaulted.yaml | 1 - .../validation/invalid_mastercfg.yaml | 2 +- 18 files changed, 136 insertions(+), 27 deletions(-) diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 5d15fa2731a..253c355b358 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -101,9 +101,8 @@ type ClusterConfiguration struct { // +k8s:conversion-gen=false CIImageRepository string - // UnifiedControlPlaneImage specifies if a specific container image should be - // used for all control plane components. - UnifiedControlPlaneImage string + // UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images + UseHyperKubeImage bool // AuditPolicyConfiguration defines the options for the api server audit system. AuditPolicyConfiguration AuditPolicyConfiguration diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD index a5600c77782..720238d8455 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD @@ -20,6 +20,8 @@ go_library( deps = [ "//cmd/kubeadm/app/apis/kubeadm:go_default_library", "//cmd/kubeadm/app/constants:go_default_library", + "//cmd/kubeadm/app/images:go_default_library", + "//cmd/kubeadm/app/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library", diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go index 4c5bdd73fcd..6245106ed06 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go @@ -18,10 +18,13 @@ package v1alpha3 import ( "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/conversion" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/images" + kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) func Convert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error { @@ -108,9 +111,29 @@ func Convert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in *C return err } + if err := Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(in, out); err != nil { + return err + } + return nil } +func Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(in *ClusterConfiguration, out *kubeadm.ClusterConfiguration) error { + if len(in.UnifiedControlPlaneImage) == 0 { + out.UseHyperKubeImage = false + return nil + } + + k8sImageTag := kubeadmutil.KubernetesVersionToImageTag(in.KubernetesVersion) + expectedImage := images.GetGenericImage(in.ImageRepository, constants.HyperKube, k8sImageTag) + if expectedImage == in.UnifiedControlPlaneImage { + out.UseHyperKubeImage = true + return nil + } + + return errors.Errorf("cannot convert unifiedControlPlaneImage=%q to useHyperKubeImage", in.UnifiedControlPlaneImage) +} + func Convert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in *kubeadm.ClusterConfiguration, out *ClusterConfiguration, s conversion.Scope) error { if err := autoConvert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in, out, s); err != nil { return err @@ -132,6 +155,12 @@ func Convert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in *k return err } + if in.UseHyperKubeImage { + out.UnifiedControlPlaneImage = images.GetKubeControlPlaneImage("", in) + } else { + out.UnifiedControlPlaneImage = "" + } + return nil } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion_test.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion_test.go index 55df7850024..7b27aa7c14a 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion_test.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion_test.go @@ -53,3 +53,75 @@ func TestJoinConfigurationConversion(t *testing.T) { } } } + +func TestConvertToUseHyperKubeImage(t *testing.T) { + tests := []struct { + desc string + in *v1alpha3.ClusterConfiguration + useHyperKubeImage bool + expectedErr bool + }{ + { + desc: "unset UnifiedControlPlaneImage sets UseHyperKubeImage to false", + in: &v1alpha3.ClusterConfiguration{}, + useHyperKubeImage: false, + expectedErr: false, + }, + { + desc: "matching UnifiedControlPlaneImage sets UseHyperKubeImage to true", + in: &v1alpha3.ClusterConfiguration{ + ImageRepository: "k8s.gcr.io", + KubernetesVersion: "v1.12.2", + UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2", + }, + useHyperKubeImage: true, + expectedErr: false, + }, + { + desc: "mismatching UnifiedControlPlaneImage tag causes an error", + in: &v1alpha3.ClusterConfiguration{ + ImageRepository: "k8s.gcr.io", + KubernetesVersion: "v1.12.0", + UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2", + }, + expectedErr: true, + }, + { + desc: "mismatching UnifiedControlPlaneImage repo causes an error", + in: &v1alpha3.ClusterConfiguration{ + ImageRepository: "my.repo", + KubernetesVersion: "v1.12.2", + UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2", + }, + expectedErr: true, + }, + { + desc: "mismatching UnifiedControlPlaneImage image name causes an error", + in: &v1alpha3.ClusterConfiguration{ + ImageRepository: "k8s.gcr.io", + KubernetesVersion: "v1.12.2", + UnifiedControlPlaneImage: "k8s.gcr.io/otherimage:v1.12.2", + }, + expectedErr: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + out := &kubeadm.ClusterConfiguration{} + err := v1alpha3.Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(test.in, out) + if test.expectedErr { + if err == nil { + t.Fatalf("unexpected success, UseHyperKubeImage: %t", out.UseHyperKubeImage) + } + } else { + if err != nil { + t.Fatalf("unexpected failure: %v", err) + } + if out.UseHyperKubeImage != test.useHyperKubeImage { + t.Fatalf("mismatching result from conversion:\n\tExpected: %t\n\tReceived: %t", test.useHyperKubeImage, out.UseHyperKubeImage) + } + } + }) + } +} diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go index a278d956346..20db83eb8d4 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go @@ -326,7 +326,7 @@ func autoConvert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(i // WARNING: in.APIServerCertSANs requires manual conversion: does not exist in peer-type out.CertificatesDir = in.CertificatesDir out.ImageRepository = in.ImageRepository - out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage + // WARNING: in.UnifiedControlPlaneImage requires manual conversion: does not exist in peer-type if err := Convert_v1alpha3_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil { return err } @@ -351,7 +351,7 @@ func autoConvert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(i out.CertificatesDir = in.CertificatesDir out.ImageRepository = in.ImageRepository // INFO: in.CIImageRepository opted out of conversion generation - out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage + // WARNING: in.UseHyperKubeImage requires manual conversion: does not exist in peer-type if err := Convert_kubeadm_AuditPolicyConfiguration_To_v1alpha3_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil { return err } diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go index e9c3843d93c..3ea272e2173 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go @@ -233,7 +233,7 @@ limitations under the License. // pathType: File // certificatesDir: "/etc/kubernetes/pki" // imageRepository: "k8s.gcr.io" -// unifiedControlPlaneImage: "k8s.gcr.io/controlplane:v1.12.0" +// useHyperKubeImage: false // auditPolicy: // # https://kubernetes.io/docs/tasks/debug-application-cluster/audit/#audit-policy // path: "/var/log/audit/audit.json" diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go index 9a112686a27..66eb9b6d33b 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go @@ -91,9 +91,9 @@ type ClusterConfiguration struct { // ImageRepository what container registry to pull control plane images from ImageRepository string `json:"imageRepository"` - // UnifiedControlPlaneImage specifies if a specific container image should - // be used for all control plane components. - UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"` + + // UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images + UseHyperKubeImage bool `json:"useHyperKubeImage,omitempty"` // AuditPolicyConfiguration defines the options for the api server audit system AuditPolicyConfiguration AuditPolicyConfiguration `json:"auditPolicy"` diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go index 579cba96325..075f3c31822 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go @@ -402,7 +402,7 @@ func autoConvert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in } out.CertificatesDir = in.CertificatesDir out.ImageRepository = in.ImageRepository - out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage + out.UseHyperKubeImage = in.UseHyperKubeImage if err := Convert_v1beta1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil { return err } @@ -438,7 +438,7 @@ func autoConvert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in out.CertificatesDir = in.CertificatesDir out.ImageRepository = in.ImageRepository // INFO: in.CIImageRepository opted out of conversion generation - out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage + out.UseHyperKubeImage = in.UseHyperKubeImage if err := Convert_kubeadm_AuditPolicyConfiguration_To_v1beta1_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil { return err } diff --git a/cmd/kubeadm/app/cmd/upgrade/common_test.go b/cmd/kubeadm/app/cmd/upgrade/common_test.go index 5547b71cd2a..ae884c65fcf 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/common_test.go @@ -63,7 +63,6 @@ func TestPrintConfiguration(t *testing.T) { podSubnet: "" serviceSubnet: "" scheduler: {} - unifiedControlPlaneImage: "" `), }, { @@ -102,7 +101,6 @@ func TestPrintConfiguration(t *testing.T) { podSubnet: "" serviceSubnet: 10.96.0.1/12 scheduler: {} - unifiedControlPlaneImage: "" `), }, } diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index a2dc78056e1..f1a239fff15 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -265,6 +265,8 @@ const ( KubeScheduler = "kube-scheduler" // KubeProxy defines variable used internally when referring to kube-proxy component KubeProxy = "kube-proxy" + // HyperKube defines variable used internally when referring to the hyperkube image + HyperKube = "hyperkube" // SelfHostingPrefix describes the prefix workloads that are self-hosted by kubeadm has SelfHostingPrefix = "self-hosted-" diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index 3d520cb3cee..38be281a362 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -32,8 +32,8 @@ func GetGenericImage(prefix, image, tag string) string { // GetKubeControlPlaneImage generates and returns the image for the core Kubernetes components or returns the unified control plane image if specified func GetKubeControlPlaneImage(image string, cfg *kubeadmapi.ClusterConfiguration) string { - if cfg.UnifiedControlPlaneImage != "" { - return cfg.UnifiedControlPlaneImage + if cfg.UseHyperKubeImage { + image = constants.HyperKube } repoPrefix := cfg.GetControlPlaneImageRepository() kubernetesImageTag := kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion) @@ -56,10 +56,16 @@ func GetEtcdImage(cfg *kubeadmapi.ClusterConfiguration) string { // GetAllImages returns a list of container images kubeadm expects to use on a control plane node func GetAllImages(cfg *kubeadmapi.ClusterConfiguration) []string { imgs := []string{} - imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeAPIServer, cfg)) - imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeControllerManager, cfg)) - imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeScheduler, cfg)) - imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeProxy, cfg)) + + // start with core kubernetes images + if cfg.UseHyperKubeImage { + imgs = append(imgs, GetKubeControlPlaneImage(constants.HyperKube, cfg)) + } else { + imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeAPIServer, cfg)) + imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeControllerManager, cfg)) + imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeScheduler, cfg)) + imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeProxy, cfg)) + } // pause, etcd and kube-dns are not available on the ci image repository so use the default image repository. imgs = append(imgs, GetGenericImage(cfg.ImageRepository, "pause", constants.PauseVersion)) diff --git a/cmd/kubeadm/app/images/images_test.go b/cmd/kubeadm/app/images/images_test.go index a7bba7477f5..2b7a19e8418 100644 --- a/cmd/kubeadm/app/images/images_test.go +++ b/cmd/kubeadm/app/images/images_test.go @@ -51,9 +51,11 @@ func TestGetKubeControlPlaneImage(t *testing.T) { cfg *kubeadmapi.ClusterConfiguration }{ { - expected: "override", + expected: GetGenericImage(gcrPrefix, constants.HyperKube, expected), cfg: &kubeadmapi.ClusterConfiguration{ - UnifiedControlPlaneImage: "override", + ImageRepository: gcrPrefix, + KubernetesVersion: testversion, + UseHyperKubeImage: true, }, }, { diff --git a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go index 96ed5ff2621..05aa483ea5c 100644 --- a/cmd/kubeadm/app/phases/upgrade/staticpods_test.go +++ b/cmd/kubeadm/app/phases/upgrade/staticpods_test.go @@ -82,7 +82,7 @@ networking: schedulerExtraArgs: null token: ce3aa5.5ec8455bb76b379f tokenTTL: 24h -unifiedControlPlaneImage: "" +useHyperKubeImage: false ` ) diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml index af08a33e5e5..175e6b2ab68 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml @@ -200,4 +200,4 @@ NodeRegistration: Scheduler: ExtraArgs: null ExtraVolumes: null -UnifiedControlPlaneImage: "" +UseHyperKubeImage: true diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml index f6ad739d51e..339e4f07957 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml @@ -47,7 +47,7 @@ networking: dnsDomain: cluster.local podSubnet: "" serviceSubnet: 10.96.0.0/12 -unifiedControlPlaneImage: "" +unifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.11.2" --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 diff --git a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml index 92758f57c30..c8c62ca208c 100644 --- a/cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml +++ b/cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml @@ -51,7 +51,7 @@ networking: podSubnet: "" serviceSubnet: 10.96.0.0/12 scheduler: {} -unifiedControlPlaneImage: "" +useHyperKubeImage: true --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 diff --git a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml index 1df4353c21f..10566d3209d 100644 --- a/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml +++ b/cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml @@ -41,7 +41,6 @@ networking: podSubnet: 10.148.0.0/16 serviceSubnet: 10.196.0.0/12 scheduler: {} -unifiedControlPlaneImage: "" --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 bindAddress: 0.0.0.0 diff --git a/cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml b/cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml index 34924a75acc..55c7fb1b5ad 100644 --- a/cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml +++ b/cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml @@ -4,4 +4,4 @@ networking: dnsDomain: INVALID-DOMAIN-!!!! podSubnet: "" serviceSubnet: 10.96.0.0/12 -unifiedControlPlaneImage: "" \ No newline at end of file +useHyperKubeImage: false \ No newline at end of file