kubeadm: UnifiedControlPlaneImage -> UseHyperKubeImage

Up until now UnifiedControlPlaneImage existed as a string value as part of the
ClusterConfiguration. This provided an override for the Kubernetes core
component images with a single custom image. It is mostly used to override the
control plane images with the hyperkube image. This saves both bandwith and
disk space on the control plane nodes.
Unfortunately, this specified an entire image string (complete with its prefix,
image name and tag). This disables upgrades of setups that use hyperkube.
Therefore, to enable upgrades on hyperkube setups and to make configuration
more convenient, the UnifiedControlPlaneImage option is replaced with a boolean
option, called UseHyperKubeImage. If set to true, this option replaces the
image name of any Kubernetes core components with hyperkube, thus allowing for
upgrades and respecting the image repository and version, specified in the
ClusterConfiguration.

Signed-off-by: Rostislav M. Georgiev <rostislavg@vmware.com>

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -101,9 +101,8 @@ type ClusterConfiguration struct {
 	// +k8s:conversion-gen=false
 	CIImageRepository string
 
-	// UnifiedControlPlaneImage specifies if a specific container image should be
+	// UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images
-	// used for all control plane components.
+	UseHyperKubeImage bool
-	UnifiedControlPlaneImage string
 
 	// AuditPolicyConfiguration defines the options for the api server audit system.
 	AuditPolicyConfiguration AuditPolicyConfiguration

cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD

@@ -20,6 +20,8 @@ go_library(
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/images:go_default_library",
+        "//cmd/kubeadm/app/util:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library",

cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go

@@ -18,10 +18,13 @@ package v1alpha3
 
 import (
 	"github.com/pkg/errors"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/images"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 )
 
 func Convert_v1alpha3_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error {
@@ -108,9 +111,29 @@ func Convert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in *C
 		return err
 	}
 
+	if err := Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(in, out); err != nil {
+		return err
+	}
+
 	return nil
 }
 
+func Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(in *ClusterConfiguration, out *kubeadm.ClusterConfiguration) error {
+	if len(in.UnifiedControlPlaneImage) == 0 {
+		out.UseHyperKubeImage = false
+		return nil
+	}
+
+	k8sImageTag := kubeadmutil.KubernetesVersionToImageTag(in.KubernetesVersion)
+	expectedImage := images.GetGenericImage(in.ImageRepository, constants.HyperKube, k8sImageTag)
+	if expectedImage == in.UnifiedControlPlaneImage {
+		out.UseHyperKubeImage = true
+		return nil
+	}
+
+	return errors.Errorf("cannot convert unifiedControlPlaneImage=%q to useHyperKubeImage", in.UnifiedControlPlaneImage)
+}
+
 func Convert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in *kubeadm.ClusterConfiguration, out *ClusterConfiguration, s conversion.Scope) error {
 	if err := autoConvert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in, out, s); err != nil {
 		return err
@@ -132,6 +155,12 @@ func Convert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in *k
 		return err
 	}
 
+	if in.UseHyperKubeImage {
+		out.UnifiedControlPlaneImage = images.GetKubeControlPlaneImage("", in)
+	} else {
+		out.UnifiedControlPlaneImage = ""
+	}
+
 	return nil
 }
 

cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion_test.go

@@ -53,3 +53,75 @@ func TestJoinConfigurationConversion(t *testing.T) {
 		}
 	}
 }
+
+func TestConvertToUseHyperKubeImage(t *testing.T) {
+	tests := []struct {
+		desc              string
+		in                *v1alpha3.ClusterConfiguration
+		useHyperKubeImage bool
+		expectedErr       bool
+	}{
+		{
+			desc:              "unset UnifiedControlPlaneImage sets UseHyperKubeImage to false",
+			in:                &v1alpha3.ClusterConfiguration{},
+			useHyperKubeImage: false,
+			expectedErr:       false,
+		},
+		{
+			desc: "matching UnifiedControlPlaneImage sets UseHyperKubeImage to true",
+			in: &v1alpha3.ClusterConfiguration{
+				ImageRepository:          "k8s.gcr.io",
+				KubernetesVersion:        "v1.12.2",
+				UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2",
+			},
+			useHyperKubeImage: true,
+			expectedErr:       false,
+		},
+		{
+			desc: "mismatching UnifiedControlPlaneImage tag causes an error",
+			in: &v1alpha3.ClusterConfiguration{
+				ImageRepository:          "k8s.gcr.io",
+				KubernetesVersion:        "v1.12.0",
+				UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2",
+			},
+			expectedErr: true,
+		},
+		{
+			desc: "mismatching UnifiedControlPlaneImage repo causes an error",
+			in: &v1alpha3.ClusterConfiguration{
+				ImageRepository:          "my.repo",
+				KubernetesVersion:        "v1.12.2",
+				UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2",
+			},
+			expectedErr: true,
+		},
+		{
+			desc: "mismatching UnifiedControlPlaneImage image name causes an error",
+			in: &v1alpha3.ClusterConfiguration{
+				ImageRepository:          "k8s.gcr.io",
+				KubernetesVersion:        "v1.12.2",
+				UnifiedControlPlaneImage: "k8s.gcr.io/otherimage:v1.12.2",
+			},
+			expectedErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			out := &kubeadm.ClusterConfiguration{}
+			err := v1alpha3.Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(test.in, out)
+			if test.expectedErr {
+				if err == nil {
+					t.Fatalf("unexpected success, UseHyperKubeImage: %t", out.UseHyperKubeImage)
+				}
+			} else {
+				if err != nil {
+					t.Fatalf("unexpected failure: %v", err)
+				}
+				if out.UseHyperKubeImage != test.useHyperKubeImage {
+					t.Fatalf("mismatching result from conversion:\n\tExpected: %t\n\tReceived: %t", test.useHyperKubeImage, out.UseHyperKubeImage)
+				}
+			}
+		})
+	}
+}

cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go

@@ -326,7 +326,7 @@ func autoConvert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(i
 	// WARNING: in.APIServerCertSANs requires manual conversion: does not exist in peer-type
 	out.CertificatesDir = in.CertificatesDir
 	out.ImageRepository = in.ImageRepository
-	out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage
+	// WARNING: in.UnifiedControlPlaneImage requires manual conversion: does not exist in peer-type
 	if err := Convert_v1alpha3_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil {
 		return err
 	}
@@ -351,7 +351,7 @@ func autoConvert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(i
 	out.CertificatesDir = in.CertificatesDir
 	out.ImageRepository = in.ImageRepository
 	// INFO: in.CIImageRepository opted out of conversion generation
-	out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage
+	// WARNING: in.UseHyperKubeImage requires manual conversion: does not exist in peer-type
 	if err := Convert_kubeadm_AuditPolicyConfiguration_To_v1alpha3_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil {
 		return err
 	}

cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go

@@ -233,7 +233,7 @@ limitations under the License.
 //	    pathType: File
 //	certificatesDir: "/etc/kubernetes/pki"
 //	imageRepository: "k8s.gcr.io"
-//	unifiedControlPlaneImage: "k8s.gcr.io/controlplane:v1.12.0"
+//	useHyperKubeImage: false
 //	auditPolicy:
 //	  # https://kubernetes.io/docs/tasks/debug-application-cluster/audit/#audit-policy
 //	  path: "/var/log/audit/audit.json"

cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go

@@ -91,9 +91,9 @@ type ClusterConfiguration struct {
 
 	// ImageRepository what container registry to pull control plane images from
 	ImageRepository string `json:"imageRepository"`
-	// UnifiedControlPlaneImage specifies if a specific container image should
+
-	// be used for all control plane components.
+	// UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images
-	UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"`
+	UseHyperKubeImage bool `json:"useHyperKubeImage,omitempty"`
 
 	// AuditPolicyConfiguration defines the options for the api server audit system
 	AuditPolicyConfiguration AuditPolicyConfiguration `json:"auditPolicy"`

cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go

@@ -402,7 +402,7 @@ func autoConvert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in
 	}
 	out.CertificatesDir = in.CertificatesDir
 	out.ImageRepository = in.ImageRepository
-	out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage
+	out.UseHyperKubeImage = in.UseHyperKubeImage
 	if err := Convert_v1beta1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil {
 		return err
 	}
@@ -438,7 +438,7 @@ func autoConvert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in
 	out.CertificatesDir = in.CertificatesDir
 	out.ImageRepository = in.ImageRepository
 	// INFO: in.CIImageRepository opted out of conversion generation
-	out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage
+	out.UseHyperKubeImage = in.UseHyperKubeImage
 	if err := Convert_kubeadm_AuditPolicyConfiguration_To_v1beta1_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil {
 		return err
 	}

cmd/kubeadm/app/cmd/upgrade/common_test.go

@@ -63,7 +63,6 @@ func TestPrintConfiguration(t *testing.T) {
 	  podSubnet: ""
 	  serviceSubnet: ""
 	scheduler: {}
-	unifiedControlPlaneImage: ""
 `),
 		},
 		{
@@ -102,7 +101,6 @@ func TestPrintConfiguration(t *testing.T) {
 	  podSubnet: ""
 	  serviceSubnet: 10.96.0.1/12
 	scheduler: {}
-	unifiedControlPlaneImage: ""
 `),
 		},
 	}

cmd/kubeadm/app/constants/constants.go

@@ -265,6 +265,8 @@ const (
 	KubeScheduler = "kube-scheduler"
 	// KubeProxy defines variable used internally when referring to kube-proxy component
 	KubeProxy = "kube-proxy"
+	// HyperKube defines variable used internally when referring to the hyperkube image
+	HyperKube = "hyperkube"
 
 	// SelfHostingPrefix describes the prefix workloads that are self-hosted by kubeadm has
 	SelfHostingPrefix = "self-hosted-"

cmd/kubeadm/app/images/images.go

@@ -32,8 +32,8 @@ func GetGenericImage(prefix, image, tag string) string {
 
 // GetKubeControlPlaneImage generates and returns the image for the core Kubernetes components or returns the unified control plane image if specified
 func GetKubeControlPlaneImage(image string, cfg *kubeadmapi.ClusterConfiguration) string {
-	if cfg.UnifiedControlPlaneImage != "" {
+	if cfg.UseHyperKubeImage {
-		return cfg.UnifiedControlPlaneImage
+		image = constants.HyperKube
 	}
 	repoPrefix := cfg.GetControlPlaneImageRepository()
 	kubernetesImageTag := kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion)
@@ -56,10 +56,16 @@ func GetEtcdImage(cfg *kubeadmapi.ClusterConfiguration) string {
 // GetAllImages returns a list of container images kubeadm expects to use on a control plane node
 func GetAllImages(cfg *kubeadmapi.ClusterConfiguration) []string {
 	imgs := []string{}
-	imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeAPIServer, cfg))
+
-	imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeControllerManager, cfg))
+	// start with core kubernetes images
-	imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeScheduler, cfg))
+	if cfg.UseHyperKubeImage {
-	imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeProxy, cfg))
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.HyperKube, cfg))
+	} else {
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeAPIServer, cfg))
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeControllerManager, cfg))
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeScheduler, cfg))
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeProxy, cfg))
+	}
 
 	// pause, etcd and kube-dns are not available on the ci image repository so use the default image repository.
 	imgs = append(imgs, GetGenericImage(cfg.ImageRepository, "pause", constants.PauseVersion))

cmd/kubeadm/app/images/images_test.go

@@ -51,9 +51,11 @@ func TestGetKubeControlPlaneImage(t *testing.T) {
 		cfg      *kubeadmapi.ClusterConfiguration
 	}{
 		{
-			expected: "override",
+			expected: GetGenericImage(gcrPrefix, constants.HyperKube, expected),
 			cfg: &kubeadmapi.ClusterConfiguration{
-				UnifiedControlPlaneImage: "override",
+				ImageRepository:   gcrPrefix,
+				KubernetesVersion: testversion,
+				UseHyperKubeImage: true,
 			},
 		},
 		{

cmd/kubeadm/app/phases/upgrade/staticpods_test.go

@@ -82,7 +82,7 @@ networking:
 schedulerExtraArgs: null
 token: ce3aa5.5ec8455bb76b379f
 tokenTTL: 24h
-unifiedControlPlaneImage: ""
+useHyperKubeImage: false
 `
 )
 

cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml

@@ -200,4 +200,4 @@ NodeRegistration:
 Scheduler:
   ExtraArgs: null
   ExtraVolumes: null
-UnifiedControlPlaneImage: ""
+UseHyperKubeImage: true

cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml

@@ -47,7 +47,7 @@ networking:
   dnsDomain: cluster.local
   podSubnet: ""
   serviceSubnet: 10.96.0.0/12
-unifiedControlPlaneImage: ""
+unifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.11.2"
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 bindAddress: 0.0.0.0

cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml

@@ -51,7 +51,7 @@ networking:
   podSubnet: ""
   serviceSubnet: 10.96.0.0/12
 scheduler: {}
-unifiedControlPlaneImage: ""
+useHyperKubeImage: true
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 bindAddress: 0.0.0.0

cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml

@@ -41,7 +41,6 @@ networking:
   podSubnet: 10.148.0.0/16
   serviceSubnet: 10.196.0.0/12
 scheduler: {}
-unifiedControlPlaneImage: ""
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 bindAddress: 0.0.0.0

cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml

@@ -4,4 +4,4 @@ networking:
   dnsDomain: INVALID-DOMAIN-!!!!
   podSubnet: ""
   serviceSubnet: 10.96.0.0/12
-unifiedControlPlaneImage: ""
+useHyperKubeImage: false