Merge pull request #48053 from luxas/kubeadm_move_envparam_to_api

Automatic merge from submit-queue (batch tested with PRs 48781, 48817, 48830, 48829, 48053)

kubeadm: Start to remove old envparams

**What this PR does / why we need it**:

Historically, the envparams feature was used as a way to easily debug `kubeadm` while developing it in the v1.3-v1.4 timeframe. Since then some parameters have been hanging around and not being moved into the API as they should have.

Note: This is a temporary step; moving things into the API. Still, the API is gonna change, this is not the end state. But this is better than keeping the envparams.

I'm gonna deal with `KubernetesDir` in the next PR.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #

Fixes: kubernetes/kubeadm#326

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```
@timothysc @pipejakob

cmd/kubeadm/app/apis/kubeadm/env.go

@@ -25,15 +25,10 @@ import (
 
 var GlobalEnvParams = SetEnvParams()
 
-// TODO(phase1+) Move these paramaters to the API group
-// we need some params for testing etc, let's keep these hidden for now
 func SetEnvParams() *EnvParams {
 
 	envParams := map[string]string{
-		"kubernetes_dir":  "/etc/kubernetes",
+		"kubernetes_dir": "/etc/kubernetes",
-		"hyperkube_image": "",
-		"repo_prefix":     "gcr.io/google_containers",
-		"etcd_image":      "",
 	}
 
 	for k := range envParams {
@@ -43,9 +38,6 @@ func SetEnvParams() *EnvParams {
 	}
 
 	return &EnvParams{
-		KubernetesDir:    path.Clean(envParams["kubernetes_dir"]),
+		KubernetesDir: path.Clean(envParams["kubernetes_dir"]),
-		HyperkubeImage:   envParams["hyperkube_image"],
-		RepositoryPrefix: envParams["repo_prefix"],
-		EtcdImage:        envParams["etcd_image"],
 	}
 }

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -36,7 +36,10 @@ func KubeadmFuzzerFuncs(t apitesting.TestingCommon) []interface{} {
 			obj.CertificatesDir = "foo"
 			obj.APIServerCertSANs = []string{}
 			obj.Token = "foo"
+			obj.Etcd.Image = "foo"
 			obj.Etcd.DataDir = "foo"
+			obj.ImageRepository = "foo"
+			obj.UnifiedControlPlaneImage = "foo"
 		},
 		func(obj *kubeadm.NodeConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -23,10 +23,7 @@ import (
 )
 
 type EnvParams struct {
-	KubernetesDir    string
+	KubernetesDir string
-	HyperkubeImage   string
-	RepositoryPrefix string
-	EtcdImage        string
 }
 
 type MasterConfiguration struct {
@@ -56,6 +53,11 @@ type MasterConfiguration struct {
 	APIServerCertSANs []string
 	// CertificatesDir specifies where to store or look for all required certificates
 	CertificatesDir string
+
+	// ImageRepository what container registry to pull control plane images from
+	ImageRepository string
+	// UnifiedControlPlaneImage specifies if a specific container image should be used for all control plane components
+	UnifiedControlPlaneImage string
 }
 
 type API struct {
@@ -84,6 +86,8 @@ type Etcd struct {
 	KeyFile   string
 	DataDir   string
 	ExtraArgs map[string]string
+	// Image specifies which container image to use for running etcd. If empty, automatically populated by kubeadm using the image repository and default etcd version
+	Image string
 }
 
 type NodeConfiguration struct {

cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go

@@ -33,6 +33,7 @@ const (
 	DefaultCACertPath         = "/etc/kubernetes/pki/ca.crt"
 	DefaultCertificatesDir    = "/etc/kubernetes/pki"
 	DefaultEtcdDataDir        = "/var/lib/etcd"
+	DefaultImageRepository    = "gcr.io/google_containers"
 )
 
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
@@ -68,6 +69,10 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 		obj.TokenTTL = constants.DefaultTokenDuration
 	}
 
+	if obj.ImageRepository == "" {
+		obj.ImageRepository = DefaultImageRepository
+	}
+
 	if obj.Etcd.DataDir == "" {
 		obj.Etcd.DataDir = DefaultEtcdDataDir
 	}

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -49,6 +49,11 @@ type MasterConfiguration struct {
 	APIServerCertSANs []string `json:"apiServerCertSANs"`
 	// CertificatesDir specifies where to store or look for all required certificates
 	CertificatesDir string `json:"certificatesDir"`
+
+	// ImageRepository what container registry to pull control plane images from
+	ImageRepository string `json:"imageRepository"`
+	// UnifiedControlPlaneImage specifies if a specific container image should be used for all control plane components
+	UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"`
 }
 
 type API struct {
@@ -77,6 +82,8 @@ type Etcd struct {
 	KeyFile   string            `json:"keyFile"`
 	DataDir   string            `json:"dataDir"`
 	ExtraArgs map[string]string `json:"extraArgs"`
+	// Image specifies which container image to use for running etcd. If empty, automatically populated by kubeadm using the image repository and default etcd version
+	Image string `json:"image"`
 }
 
 type NodeConfiguration struct {

cmd/kubeadm/app/images/images.go

@@ -37,7 +37,7 @@ func GetCoreImage(image string, cfg *kubeadmapi.MasterConfiguration, overrideIma
 	if overrideImage != "" {
 		return overrideImage
 	}
-	repoPrefix := kubeadmapi.GlobalEnvParams.RepositoryPrefix
+	repoPrefix := cfg.ImageRepository
 	return map[string]string{
 		KubeEtcdImage:              fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "etcd", runtime.GOARCH, etcdVersion),
 		KubeAPIServerImage:         fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "kube-apiserver", runtime.GOARCH, cfg.KubernetesVersion),

cmd/kubeadm/app/images/images_test.go

@@ -43,27 +43,27 @@ func TestGetCoreImage(t *testing.T) {
 		{getCoreImageTest{o: "override"}, "override"},
 		{getCoreImageTest{
 			i: KubeEtcdImage,
-			c: &kubeadmapi.MasterConfiguration{}},
+			c: &kubeadmapi.MasterConfiguration{ImageRepository: gcrPrefix}},
 			fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion),
 		},
 		{getCoreImageTest{
 			i: KubeAPIServerImage,
-			c: &kubeadmapi.MasterConfiguration{KubernetesVersion: testversion}},
+			c: &kubeadmapi.MasterConfiguration{ImageRepository: gcrPrefix, KubernetesVersion: testversion}},
 			fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, testversion),
 		},
 		{getCoreImageTest{
 			i: KubeControllerManagerImage,
-			c: &kubeadmapi.MasterConfiguration{KubernetesVersion: testversion}},
+			c: &kubeadmapi.MasterConfiguration{ImageRepository: gcrPrefix, KubernetesVersion: testversion}},
 			fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, testversion),
 		},
 		{getCoreImageTest{
 			i: KubeSchedulerImage,
-			c: &kubeadmapi.MasterConfiguration{KubernetesVersion: testversion}},
+			c: &kubeadmapi.MasterConfiguration{ImageRepository: gcrPrefix, KubernetesVersion: testversion}},
 			fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, testversion),
 		},
 		{getCoreImageTest{
 			i: KubeProxyImage,
-			c: &kubeadmapi.MasterConfiguration{KubernetesVersion: testversion}},
+			c: &kubeadmapi.MasterConfiguration{ImageRepository: gcrPrefix, KubernetesVersion: testversion}},
 			fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, testversion),
 		},
 	}

cmd/kubeadm/app/phases/addons/addons.go

@@ -45,7 +45,7 @@ func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientse
 	}
 
 	proxyDaemonSetBytes, err := kubeadmutil.ParseTemplate(KubeProxyDaemonSet, struct{ Image, ClusterCIDR, MasterTaintKey string }{
-		Image:          images.GetCoreImage("proxy", cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
+		Image:          images.GetCoreImage("proxy", cfg, cfg.UnifiedControlPlaneImage),
 		ClusterCIDR:    getClusterCIDR(cfg.Networking.PodSubnet),
 		MasterTaintKey: kubeadmconstants.LabelNodeRoleMaster,
 	})
@@ -54,7 +54,7 @@ func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientse
 	}
 
 	dnsDeploymentBytes, err := kubeadmutil.ParseTemplate(KubeDNSDeployment, struct{ ImageRepository, Arch, Version, DNSDomain, MasterTaintKey string }{
-		ImageRepository: kubeadmapi.GlobalEnvParams.RepositoryPrefix,
+		ImageRepository: cfg.ImageRepository,
 		Arch:            runtime.GOARCH,
 		Version:         KubeDNSVersion,
 		DNSDomain:       cfg.Networking.DNSDomain,

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -82,7 +82,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 	staticPodSpecs := map[string]v1.Pod{
 		kubeAPIServer: componentPod(v1.Container{
 			Name:          kubeAPIServer,
-			Image:         images.GetCoreImage(images.KubeAPIServerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
+			Image:         images.GetCoreImage(images.KubeAPIServerImage, cfg, cfg.UnifiedControlPlaneImage),
 			Command:       getAPIServerCommand(cfg, false, k8sVersion),
 			VolumeMounts:  volumeMounts,
 			LivenessProbe: componentProbe(int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS),
@@ -91,7 +91,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 		}, volumes...),
 		kubeControllerManager: componentPod(v1.Container{
 			Name:          kubeControllerManager,
-			Image:         images.GetCoreImage(images.KubeControllerManagerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
+			Image:         images.GetCoreImage(images.KubeControllerManagerImage, cfg, cfg.UnifiedControlPlaneImage),
 			Command:       getControllerManagerCommand(cfg, false, k8sVersion),
 			VolumeMounts:  volumeMounts,
 			LivenessProbe: componentProbe(10252, "/healthz", v1.URISchemeHTTP),
@@ -100,7 +100,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 		}, volumes...),
 		kubeScheduler: componentPod(v1.Container{
 			Name:          kubeScheduler,
-			Image:         images.GetCoreImage(images.KubeSchedulerImage, cfg, kubeadmapi.GlobalEnvParams.HyperkubeImage),
+			Image:         images.GetCoreImage(images.KubeSchedulerImage, cfg, cfg.UnifiedControlPlaneImage),
 			Command:       getSchedulerCommand(cfg, false),
 			VolumeMounts:  []v1.VolumeMount{k8sVolumeMount()},
 			LivenessProbe: componentProbe(10251, "/healthz", v1.URISchemeHTTP),
@@ -115,7 +115,7 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 			Name:          etcd,
 			Command:       getEtcdCommand(cfg),
 			VolumeMounts:  []v1.VolumeMount{certsVolumeMount(), etcdVolumeMount(cfg.Etcd.DataDir), k8sVolumeMount()},
-			Image:         images.GetCoreImage(images.KubeEtcdImage, cfg, kubeadmapi.GlobalEnvParams.EtcdImage),
+			Image:         images.GetCoreImage(images.KubeEtcdImage, cfg, cfg.Etcd.Image),
 			LivenessProbe: componentProbe(2379, "/health", v1.URISchemeHTTP),
 		}, certsVolume(cfg), etcdVolume(cfg), k8sVolume())