diff --git a/cmd/kubeadm/app/apis/kubeadm/BUILD b/cmd/kubeadm/app/apis/kubeadm/BUILD index 9d6eb70ee96..bf5435c0779 100644 --- a/cmd/kubeadm/app/apis/kubeadm/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/BUILD @@ -17,6 +17,7 @@ go_library( deps = [ "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go index 7373a2ec0c5..b0c26279348 100644 --- a/cmd/kubeadm/app/apis/kubeadm/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/types.go @@ -17,6 +17,7 @@ limitations under the License. package kubeadm import ( + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1" @@ -92,6 +93,9 @@ type MasterConfiguration struct { // CertificatesDir specifies where to store or look for all required certificates. CertificatesDir string + // ImagePullPolicy for control plane images. Can be Always, IfNotPresent or Never. + ImagePullPolicy v1.PullPolicy + // ImageRepository is the container registry to pull control plane images from. ImageRepository string diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD index 389e161b4b4..03e3826aa09 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/BUILD @@ -56,6 +56,7 @@ go_library( "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library", "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/conversion:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go index d1edd0e1e97..22a86c14dda 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1" kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1" @@ -93,6 +94,8 @@ type MasterConfiguration struct { // ImageRepository what container registry to pull control plane images from ImageRepository string `json:"imageRepository"` + // ImagePullPolicy that control plane images. Can be Always, IfNotPresent or Never. + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` // UnifiedControlPlaneImage specifies if a specific container image should // be used for all control plane components. UnifiedControlPlaneImage string `json:"unifiedControlPlaneImage"` diff --git a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go index 8afb09456b2..c76bcd5143c 100644 --- a/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go +++ b/cmd/kubeadm/app/apis/kubeadm/v1alpha1/zz_generated.conversion.go @@ -23,6 +23,7 @@ package v1alpha1 import ( unsafe "unsafe" + core_v1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" @@ -214,6 +215,7 @@ func autoConvert_v1alpha1_MasterConfiguration_To_kubeadm_MasterConfiguration(in out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs)) out.CertificatesDir = in.CertificatesDir out.ImageRepository = in.ImageRepository + out.ImagePullPolicy = core_v1.PullPolicy(in.ImagePullPolicy) out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) return nil @@ -255,6 +257,7 @@ func autoConvert_kubeadm_MasterConfiguration_To_v1alpha1_MasterConfiguration(in out.SchedulerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.SchedulerExtraVolumes)) out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs)) out.CertificatesDir = in.CertificatesDir + out.ImagePullPolicy = core_v1.PullPolicy(in.ImagePullPolicy) out.ImageRepository = in.ImageRepository // INFO: in.CIImageRepository opted out of conversion generation out.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index 658d3237e57..70ff745c5ea 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -30,6 +30,7 @@ import ( "github.com/spf13/cobra" flag "github.com/spf13/pflag" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" utilflag "k8s.io/apiserver/pkg/util/flag" @@ -92,10 +93,12 @@ var ( This error is likely caused by: - The kubelet is not running - The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled) - - There is no internet connection, so the kubelet cannot pull the following control plane images: + - Either there is no internet connection, or imagePullPolicy is set to "Never", + so the kubelet cannot pull or find the following control plane images: - {{ .APIServerImage }} - {{ .ControllerManagerImage }} - {{ .SchedulerImage }} + - {{ .EtcdImage }} (only if no external etcd endpoints are configured) If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands: - 'systemctl status kubelet' @@ -357,7 +360,7 @@ func (i *Init) Run(out io.Writer) error { } // waiter holds the apiclient.Waiter implementation of choice, responsible for querying the API server in various ways and waiting for conditions to be fulfilled - waiter := getWaiter(i.dryRun, client) + waiter := getWaiter(i, client) if err := waitForAPIAndKubelet(waiter); err != nil { ctx := map[string]string{ @@ -365,6 +368,7 @@ func (i *Init) Run(out io.Writer) error { "APIServerImage": images.GetCoreImage(kubeadmconstants.KubeAPIServer, i.cfg.GetControlPlaneImageRepository(), i.cfg.KubernetesVersion, i.cfg.UnifiedControlPlaneImage), "ControllerManagerImage": images.GetCoreImage(kubeadmconstants.KubeControllerManager, i.cfg.GetControlPlaneImageRepository(), i.cfg.KubernetesVersion, i.cfg.UnifiedControlPlaneImage), "SchedulerImage": images.GetCoreImage(kubeadmconstants.KubeScheduler, i.cfg.GetControlPlaneImageRepository(), i.cfg.KubernetesVersion, i.cfg.UnifiedControlPlaneImage), + "EtcdImage": images.GetCoreImage(kubeadmconstants.Etcd, i.cfg.ImageRepository, i.cfg.KubernetesVersion, i.cfg.Etcd.Image), } kubeletFailTempl.Execute(out, ctx) @@ -522,11 +526,18 @@ func printFilesIfDryRunning(dryRun bool, manifestDir string) error { } // getWaiter gets the right waiter implementation for the right occasion -func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter { - if dryRun { +func getWaiter(i *Init, client clientset.Interface) apiclient.Waiter { + if i.dryRun { return dryrunutil.NewWaiter() } - return apiclient.NewKubeWaiter(client, 30*time.Minute, os.Stdout) + + timeout := 30 * time.Minute + + // No need for a large timeout if we don't expect downloads + if i.cfg.ImagePullPolicy == v1.PullNever { + timeout = 60 * time.Second + } + return apiclient.NewKubeWaiter(client, timeout, os.Stdout) } // waitForAPIAndKubelet waits primarily for the API server to come up. If that takes a long time, and the kubelet diff --git a/cmd/kubeadm/app/phases/controlplane/manifests.go b/cmd/kubeadm/app/phases/controlplane/manifests.go index 41b5b0c7db4..dea08e79d55 100644 --- a/cmd/kubeadm/app/phases/controlplane/manifests.go +++ b/cmd/kubeadm/app/phases/controlplane/manifests.go @@ -77,31 +77,34 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version. // Prepare static pod specs staticPodSpecs := map[string]v1.Pod{ kubeadmconstants.KubeAPIServer: staticpodutil.ComponentPod(v1.Container{ - Name: kubeadmconstants.KubeAPIServer, - Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), - Command: getAPIServerCommand(cfg, k8sVersion), - VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)), - LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS), - Resources: staticpodutil.ComponentResources("250m"), - Env: getProxyEnvVars(), + Name: kubeadmconstants.KubeAPIServer, + Image: images.GetCoreImage(kubeadmconstants.KubeAPIServer, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), + ImagePullPolicy: cfg.ImagePullPolicy, + Command: getAPIServerCommand(cfg, k8sVersion), + VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeAPIServer)), + LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeAPIServer, int(cfg.API.BindPort), "/healthz", v1.URISchemeHTTPS), + Resources: staticpodutil.ComponentResources("250m"), + Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)), kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{ - Name: kubeadmconstants.KubeControllerManager, - Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), - Command: getControllerManagerCommand(cfg, k8sVersion), - VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), - LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeControllerManager, 10252, "/healthz", v1.URISchemeHTTP), - Resources: staticpodutil.ComponentResources("200m"), - Env: getProxyEnvVars(), + Name: kubeadmconstants.KubeControllerManager, + Image: images.GetCoreImage(kubeadmconstants.KubeControllerManager, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), + ImagePullPolicy: cfg.ImagePullPolicy, + Command: getControllerManagerCommand(cfg, k8sVersion), + VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeControllerManager)), + LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeControllerManager, 10252, "/healthz", v1.URISchemeHTTP), + Resources: staticpodutil.ComponentResources("200m"), + Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)), kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{ - Name: kubeadmconstants.KubeScheduler, - Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), - Command: getSchedulerCommand(cfg), - VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), - LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeScheduler, 10251, "/healthz", v1.URISchemeHTTP), - Resources: staticpodutil.ComponentResources("100m"), - Env: getProxyEnvVars(), + Name: kubeadmconstants.KubeScheduler, + Image: images.GetCoreImage(kubeadmconstants.KubeScheduler, cfg.GetControlPlaneImageRepository(), cfg.KubernetesVersion, cfg.UnifiedControlPlaneImage), + ImagePullPolicy: cfg.ImagePullPolicy, + Command: getSchedulerCommand(cfg), + VolumeMounts: staticpodutil.VolumeMountMapToSlice(mounts.GetVolumeMounts(kubeadmconstants.KubeScheduler)), + LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.KubeScheduler, 10251, "/healthz", v1.URISchemeHTTP), + Resources: staticpodutil.ComponentResources("100m"), + Env: getProxyEnvVars(), }, mounts.GetVolumes(kubeadmconstants.KubeScheduler)), } diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index a56ce49e13e..1de1e38adbe 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -53,9 +53,10 @@ func GetEtcdPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.Pod { etcdVolumeName: staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.DataDir, &pathType), } return staticpodutil.ComponentPod(v1.Container{ - Name: kubeadmconstants.Etcd, - Command: getEtcdCommand(cfg), - Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Image), + Name: kubeadmconstants.Etcd, + Command: getEtcdCommand(cfg), + Image: images.GetCoreImage(kubeadmconstants.Etcd, cfg.ImageRepository, cfg.KubernetesVersion, cfg.Etcd.Image), + ImagePullPolicy: cfg.ImagePullPolicy, // Mount the etcd datadir path read-write so etcd can store data in a more persistent manner VolumeMounts: []v1.VolumeMount{staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.DataDir, false)}, LivenessProbe: staticpodutil.ComponentProbe(cfg, kubeadmconstants.Etcd, 2379, "/health", v1.URISchemeHTTP),