Merge pull request #70793 from rosti/use-hyperkube

kubeadm: UnifiedControlPlaneImage string -> UseHyperKubeImage bool

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -106,9 +106,8 @@ type ClusterConfiguration struct {
 	// +k8s:conversion-gen=false
 	CIImageRepository string
 
-	// UnifiedControlPlaneImage specifies if a specific container image should be
+	// UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images
-	// used for all control plane components.
+	UseHyperKubeImage bool
-	UnifiedControlPlaneImage string
 
 	// AuditPolicyConfiguration defines the options for the api server audit system.
 	AuditPolicyConfiguration AuditPolicyConfiguration

cmd/kubeadm/app/apis/kubeadm/v1alpha3/BUILD

@@ -20,6 +20,8 @@ go_library(
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/images:go_default_library",
+        "//cmd/kubeadm/app/util:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/conversion:go_default_library",

cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion.go

@@ -18,10 +18,13 @@ package v1alpha3
 
 import (
 	"github.com/pkg/errors"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/images"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 )
 
 func Convert_v1alpha3_InitConfiguration_To_kubeadm_InitConfiguration(in *InitConfiguration, out *kubeadm.InitConfiguration, s conversion.Scope) error {
@@ -126,9 +129,29 @@ func Convert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in *C
 		return err
 	}
 
+	if err := Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(in, out); err != nil {
+		return err
+	}
+
 	return nil
 }
 
+func Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(in *ClusterConfiguration, out *kubeadm.ClusterConfiguration) error {
+	if len(in.UnifiedControlPlaneImage) == 0 {
+		out.UseHyperKubeImage = false
+		return nil
+	}
+
+	k8sImageTag := kubeadmutil.KubernetesVersionToImageTag(in.KubernetesVersion)
+	expectedImage := images.GetGenericImage(in.ImageRepository, constants.HyperKube, k8sImageTag)
+	if expectedImage == in.UnifiedControlPlaneImage {
+		out.UseHyperKubeImage = true
+		return nil
+	}
+
+	return errors.Errorf("cannot convert unifiedControlPlaneImage=%q to useHyperKubeImage", in.UnifiedControlPlaneImage)
+}
+
 func Convert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in *kubeadm.ClusterConfiguration, out *ClusterConfiguration, s conversion.Scope) error {
 	if err := autoConvert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in, out, s); err != nil {
 		return err
@@ -150,6 +173,12 @@ func Convert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(in *k
 		return err
 	}
 
+	if in.UseHyperKubeImage {
+		out.UnifiedControlPlaneImage = images.GetKubeControlPlaneImage("", in)
+	} else {
+		out.UnifiedControlPlaneImage = ""
+	}
+
 	return nil
 }
 

cmd/kubeadm/app/apis/kubeadm/v1alpha3/conversion_test.go

@@ -59,3 +59,75 @@ func TestJoinConfigurationConversion(t *testing.T) {
 		}
 	}
 }
+
+func TestConvertToUseHyperKubeImage(t *testing.T) {
+	tests := []struct {
+		desc              string
+		in                *v1alpha3.ClusterConfiguration
+		useHyperKubeImage bool
+		expectedErr       bool
+	}{
+		{
+			desc:              "unset UnifiedControlPlaneImage sets UseHyperKubeImage to false",
+			in:                &v1alpha3.ClusterConfiguration{},
+			useHyperKubeImage: false,
+			expectedErr:       false,
+		},
+		{
+			desc: "matching UnifiedControlPlaneImage sets UseHyperKubeImage to true",
+			in: &v1alpha3.ClusterConfiguration{
+				ImageRepository:          "k8s.gcr.io",
+				KubernetesVersion:        "v1.12.2",
+				UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2",
+			},
+			useHyperKubeImage: true,
+			expectedErr:       false,
+		},
+		{
+			desc: "mismatching UnifiedControlPlaneImage tag causes an error",
+			in: &v1alpha3.ClusterConfiguration{
+				ImageRepository:          "k8s.gcr.io",
+				KubernetesVersion:        "v1.12.0",
+				UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2",
+			},
+			expectedErr: true,
+		},
+		{
+			desc: "mismatching UnifiedControlPlaneImage repo causes an error",
+			in: &v1alpha3.ClusterConfiguration{
+				ImageRepository:          "my.repo",
+				KubernetesVersion:        "v1.12.2",
+				UnifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.12.2",
+			},
+			expectedErr: true,
+		},
+		{
+			desc: "mismatching UnifiedControlPlaneImage image name causes an error",
+			in: &v1alpha3.ClusterConfiguration{
+				ImageRepository:          "k8s.gcr.io",
+				KubernetesVersion:        "v1.12.2",
+				UnifiedControlPlaneImage: "k8s.gcr.io/otherimage:v1.12.2",
+			},
+			expectedErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			out := &kubeadm.ClusterConfiguration{}
+			err := v1alpha3.Convert_v1alpha3_UnifiedControlPlaneImage_To_kubeadm_UseHyperKubeImage(test.in, out)
+			if test.expectedErr {
+				if err == nil {
+					t.Fatalf("unexpected success, UseHyperKubeImage: %t", out.UseHyperKubeImage)
+				}
+			} else {
+				if err != nil {
+					t.Fatalf("unexpected failure: %v", err)
+				}
+				if out.UseHyperKubeImage != test.useHyperKubeImage {
+					t.Fatalf("mismatching result from conversion:\n\tExpected: %t\n\tReceived: %t", test.useHyperKubeImage, out.UseHyperKubeImage)
+				}
+			}
+		})
+	}
+}

cmd/kubeadm/app/apis/kubeadm/v1alpha3/zz_generated.conversion.go

@@ -336,7 +336,7 @@ func autoConvert_v1alpha3_ClusterConfiguration_To_kubeadm_ClusterConfiguration(i
 	// WARNING: in.APIServerCertSANs requires manual conversion: does not exist in peer-type
 	out.CertificatesDir = in.CertificatesDir
 	out.ImageRepository = in.ImageRepository
-	out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage
+	// WARNING: in.UnifiedControlPlaneImage requires manual conversion: does not exist in peer-type
 	if err := Convert_v1alpha3_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil {
 		return err
 	}
@@ -361,7 +361,7 @@ func autoConvert_kubeadm_ClusterConfiguration_To_v1alpha3_ClusterConfiguration(i
 	out.CertificatesDir = in.CertificatesDir
 	out.ImageRepository = in.ImageRepository
 	// INFO: in.CIImageRepository opted out of conversion generation
-	out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage
+	// WARNING: in.UseHyperKubeImage requires manual conversion: does not exist in peer-type
 	if err := Convert_kubeadm_AuditPolicyConfiguration_To_v1alpha3_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil {
 		return err
 	}

cmd/kubeadm/app/apis/kubeadm/v1beta1/doc.go

@@ -233,7 +233,7 @@ limitations under the License.
 //	    pathType: File
 //	certificatesDir: "/etc/kubernetes/pki"
 //	imageRepository: "k8s.gcr.io"
-//	unifiedControlPlaneImage: "k8s.gcr.io/controlplane:v1.12.0"
+//	useHyperKubeImage: false
 //	auditPolicy:
 //	  # https://kubernetes.io/docs/tasks/debug-application-cluster/audit/#audit-policy
 //	  path: "/var/log/audit/audit.json"

cmd/kubeadm/app/apis/kubeadm/v1beta1/types.go

@@ -96,9 +96,9 @@ type ClusterConfiguration struct {
 
 	// ImageRepository what container registry to pull control plane images from
 	ImageRepository string `json:"imageRepository"`
-	// UnifiedControlPlaneImage specifies if a specific container image should
+
-	// be used for all control plane components.
+	// UseHyperKubeImage controls if hyperkube should be used for Kubernetes components instead of their respective separate images
-	UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"`
+	UseHyperKubeImage bool `json:"useHyperKubeImage,omitempty"`
 
 	// AuditPolicyConfiguration defines the options for the api server audit system
 	AuditPolicyConfiguration AuditPolicyConfiguration `json:"auditPolicy"`

cmd/kubeadm/app/apis/kubeadm/v1beta1/zz_generated.conversion.go

@@ -402,7 +402,7 @@ func autoConvert_v1beta1_ClusterConfiguration_To_kubeadm_ClusterConfiguration(in
 	}
 	out.CertificatesDir = in.CertificatesDir
 	out.ImageRepository = in.ImageRepository
-	out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage
+	out.UseHyperKubeImage = in.UseHyperKubeImage
 	if err := Convert_v1beta1_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil {
 		return err
 	}
@@ -438,7 +438,7 @@ func autoConvert_kubeadm_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in
 	out.CertificatesDir = in.CertificatesDir
 	out.ImageRepository = in.ImageRepository
 	// INFO: in.CIImageRepository opted out of conversion generation
-	out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage
+	out.UseHyperKubeImage = in.UseHyperKubeImage
 	if err := Convert_kubeadm_AuditPolicyConfiguration_To_v1beta1_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil {
 		return err
 	}

cmd/kubeadm/app/cmd/upgrade/common_test.go

@@ -63,7 +63,6 @@ func TestPrintConfiguration(t *testing.T) {
 	  podSubnet: ""
 	  serviceSubnet: ""
 	scheduler: {}
-	unifiedControlPlaneImage: ""
 `),
 		},
 		{
@@ -102,7 +101,6 @@ func TestPrintConfiguration(t *testing.T) {
 	  podSubnet: ""
 	  serviceSubnet: 10.96.0.1/12
 	scheduler: {}
-	unifiedControlPlaneImage: ""
 `),
 		},
 	}

cmd/kubeadm/app/constants/constants.go

@@ -265,6 +265,8 @@ const (
 	KubeScheduler = "kube-scheduler"
 	// KubeProxy defines variable used internally when referring to kube-proxy component
 	KubeProxy = "kube-proxy"
+	// HyperKube defines variable used internally when referring to the hyperkube image
+	HyperKube = "hyperkube"
 
 	// SelfHostingPrefix describes the prefix workloads that are self-hosted by kubeadm has
 	SelfHostingPrefix = "self-hosted-"

cmd/kubeadm/app/images/images.go

@@ -32,8 +32,8 @@ func GetGenericImage(prefix, image, tag string) string {
 
 // GetKubeControlPlaneImage generates and returns the image for the core Kubernetes components or returns the unified control plane image if specified
 func GetKubeControlPlaneImage(image string, cfg *kubeadmapi.ClusterConfiguration) string {
-	if cfg.UnifiedControlPlaneImage != "" {
+	if cfg.UseHyperKubeImage {
-		return cfg.UnifiedControlPlaneImage
+		image = constants.HyperKube
 	}
 	repoPrefix := cfg.GetControlPlaneImageRepository()
 	kubernetesImageTag := kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion)
@@ -56,10 +56,16 @@ func GetEtcdImage(cfg *kubeadmapi.ClusterConfiguration) string {
 // GetAllImages returns a list of container images kubeadm expects to use on a control plane node
 func GetAllImages(cfg *kubeadmapi.ClusterConfiguration) []string {
 	imgs := []string{}
-	imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeAPIServer, cfg))
+
-	imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeControllerManager, cfg))
+	// start with core kubernetes images
-	imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeScheduler, cfg))
+	if cfg.UseHyperKubeImage {
-	imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeProxy, cfg))
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.HyperKube, cfg))
+	} else {
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeAPIServer, cfg))
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeControllerManager, cfg))
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeScheduler, cfg))
+		imgs = append(imgs, GetKubeControlPlaneImage(constants.KubeProxy, cfg))
+	}
 
 	// pause, etcd and kube-dns are not available on the ci image repository so use the default image repository.
 	imgs = append(imgs, GetGenericImage(cfg.ImageRepository, "pause", constants.PauseVersion))

cmd/kubeadm/app/images/images_test.go

@@ -51,9 +51,11 @@ func TestGetKubeControlPlaneImage(t *testing.T) {
 		cfg      *kubeadmapi.ClusterConfiguration
 	}{
 		{
-			expected: "override",
+			expected: GetGenericImage(gcrPrefix, constants.HyperKube, expected),
 			cfg: &kubeadmapi.ClusterConfiguration{
-				UnifiedControlPlaneImage: "override",
+				ImageRepository:   gcrPrefix,
+				KubernetesVersion: testversion,
+				UseHyperKubeImage: true,
 			},
 		},
 		{

cmd/kubeadm/app/phases/upgrade/staticpods_test.go

@@ -82,7 +82,7 @@ networking:
 schedulerExtraArgs: null
 token: ce3aa5.5ec8455bb76b379f
 tokenTTL: 24h
-unifiedControlPlaneImage: ""
+useHyperKubeImage: false
 `
 )
 

cmd/kubeadm/app/util/config/testdata/conversion/master/internal.yaml

@@ -200,4 +200,4 @@ NodeRegistration:
 Scheduler:
   ExtraArgs: null
   ExtraVolumes: null
-UnifiedControlPlaneImage: ""
+UseHyperKubeImage: true

cmd/kubeadm/app/util/config/testdata/conversion/master/v1alpha3.yaml

@@ -47,7 +47,7 @@ networking:
   dnsDomain: cluster.local
   podSubnet: ""
   serviceSubnet: 10.96.0.0/12
-unifiedControlPlaneImage: ""
+unifiedControlPlaneImage: "k8s.gcr.io/hyperkube:v1.11.2"
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 bindAddress: 0.0.0.0

cmd/kubeadm/app/util/config/testdata/conversion/master/v1beta1.yaml

@@ -51,7 +51,7 @@ networking:
   podSubnet: ""
   serviceSubnet: 10.96.0.0/12
 scheduler: {}
-unifiedControlPlaneImage: ""
+useHyperKubeImage: true
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 bindAddress: 0.0.0.0

cmd/kubeadm/app/util/config/testdata/defaulting/master/defaulted.yaml

@@ -41,7 +41,6 @@ networking:
   podSubnet: 10.148.0.0/16
   serviceSubnet: 10.196.0.0/12
 scheduler: {}
-unifiedControlPlaneImage: ""
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 bindAddress: 0.0.0.0

cmd/kubeadm/app/util/config/testdata/validation/invalid_mastercfg.yaml

@@ -4,4 +4,4 @@ networking:
   dnsDomain: INVALID-DOMAIN-!!!!
   podSubnet: ""
   serviceSubnet: 10.96.0.0/12
-unifiedControlPlaneImage: ""
+useHyperKubeImage: false