Merge pull request #124442 from neolit123/1.31-add-image-pull-policy-to-upgrade-config

kubeadm: support image pull mode and policy in UpgradeConfiguration

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -162,8 +162,14 @@ func fuzzUpgradeConfiguration(obj *kubeadm.UpgradeConfiguration, c fuzz.Continue
 
 	// Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail)
 	obj.Node.EtcdUpgrade = ptr.To(true)
+	obj.Node.CertificateRenewal = ptr.To(false)
+	obj.Node.ImagePullPolicy = corev1.PullIfNotPresent
+	obj.Node.ImagePullSerial = ptr.To(true)
+
 	obj.Apply.EtcdUpgrade = ptr.To(true)
 	obj.Apply.CertificateRenewal = ptr.To(false)
-	obj.Node.CertificateRenewal = ptr.To(false)
+	obj.Apply.ImagePullPolicy = corev1.PullIfNotPresent
+	obj.Apply.ImagePullSerial = ptr.To(true)
+
 	kubeadm.SetDefaultTimeouts(&obj.Timeouts)
 }

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -588,6 +588,14 @@ type UpgradeApplyConfiguration struct {
 	// SkipPhases is a list of phases to skip during command execution.
 	// NOTE: This field is currently ignored for "kubeadm upgrade apply", but in the future it will be supported.
 	SkipPhases []string
+
+	// ImagePullPolicy specifies the policy for image pulling during kubeadm "upgrade apply" operations.
+	// The value of this field must be one of "Always", "IfNotPresent" or "Never".
+	// If this field is unset kubeadm will default it to "IfNotPresent", or pull the required images if not present on the host.
+	ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"`
+
+	// ImagePullSerial specifies if image pulling performed by kubeadm must be done serially or in parallel.
+	ImagePullSerial *bool
 }
 
 // UpgradeDiffConfiguration contains a list of configurable options which are specific to the "kubeadm upgrade diff" command.
@@ -620,6 +628,14 @@ type UpgradeNodeConfiguration struct {
 
 	// Patches contains options related to applying patches to components deployed by kubeadm during "kubeadm upgrade".
 	Patches *Patches
+
+	// ImagePullPolicy specifies the policy for image pulling during kubeadm "upgrade node" operations.
+	// The value of this field must be one of "Always", "IfNotPresent" or "Never".
+	// If this field is unset kubeadm will default it to "IfNotPresent", or pull the required images if not present on the host.
+	ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"`
+
+	// ImagePullSerial specifies if image pulling performed by kubeadm must be done serially or in parallel.
+	ImagePullSerial *bool
 }
 
 // UpgradePlanConfiguration contains a list of configurable options which are specific to the "kubeadm upgrade plan" command.

cmd/kubeadm/app/apis/kubeadm/v1beta4/defaults.go

@@ -278,18 +278,29 @@ func SetDefaults_UpgradeConfiguration(obj *UpgradeConfiguration) {
 	if obj.Node.EtcdUpgrade == nil {
 		obj.Node.EtcdUpgrade = ptr.To(true)
 	}
-
 	if obj.Node.CertificateRenewal == nil {
 		obj.Node.CertificateRenewal = ptr.To(true)
 	}
+	if len(obj.Node.ImagePullPolicy) == 0 {
+		obj.Node.ImagePullPolicy = DefaultImagePullPolicy
+	}
+	if obj.Node.ImagePullSerial == nil {
+		obj.Node.ImagePullSerial = ptr.To(true)
+	}
 
 	if obj.Apply.EtcdUpgrade == nil {
 		obj.Apply.EtcdUpgrade = ptr.To(true)
 	}
-
 	if obj.Apply.CertificateRenewal == nil {
 		obj.Apply.CertificateRenewal = ptr.To(true)
 	}
+	if len(obj.Apply.ImagePullPolicy) == 0 {
+		obj.Apply.ImagePullPolicy = DefaultImagePullPolicy
+	}
+	if obj.Apply.ImagePullSerial == nil {
+		obj.Apply.ImagePullSerial = ptr.To(true)
+	}
+
 	if obj.Timeouts == nil {
 		obj.Timeouts = &Timeouts{}
 	}

cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go

@@ -663,6 +663,17 @@ type UpgradeApplyConfiguration struct {
 	// SkipPhases is a list of phases to skip during command execution.
 	// NOTE: This field is currently ignored for "kubeadm upgrade apply", but in the future it will be supported.
 	SkipPhases []string
+
+	// ImagePullPolicy specifies the policy for image pulling during kubeadm "upgrade apply" operations.
+	// The value of this field must be one of "Always", "IfNotPresent" or "Never".
+	// If this field is unset kubeadm will default it to "IfNotPresent", or pull the required images if not present on the host.
+	// +optional
+	ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
+
+	// ImagePullSerial specifies if image pulling performed by kubeadm must be done serially or in parallel.
+	// Default: true
+	// +optional
+	ImagePullSerial *bool `json:"imagePullSerial,omitempty"`
 }
 
 // UpgradeDiffConfiguration contains a list of configurable options which are specific to the "kubeadm upgrade diff" command.
@@ -705,6 +716,17 @@ type UpgradeNodeConfiguration struct {
 	// Patches contains options related to applying patches to components deployed by kubeadm during "kubeadm upgrade".
 	// +optional
 	Patches *Patches `json:"patches,omitempty"`
+
+	// ImagePullPolicy specifies the policy for image pulling during kubeadm "upgrade node" operations.
+	// The value of this field must be one of "Always", "IfNotPresent" or "Never".
+	// If this field is unset kubeadm will default it to "IfNotPresent", or pull the required images if not present on the host.
+	// +optional
+	ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
+
+	// ImagePullSerial specifies if image pulling performed by kubeadm must be done serially or in parallel.
+	// Default: true
+	// +optional
+	ImagePullSerial *bool `json:"imagePullSerial,omitempty"`
 }
 
 // UpgradePlanConfiguration contains a list of configurable options which are specific to the "kubeadm upgrade plan" command.

cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.conversion.go

@@ -1009,6 +1009,8 @@ func autoConvert_v1beta4_UpgradeApplyConfiguration_To_kubeadm_UpgradeApplyConfig
 	out.Patches = (*kubeadm.Patches)(unsafe.Pointer(in.Patches))
 	out.PrintConfig = (*bool)(unsafe.Pointer(in.PrintConfig))
 	out.SkipPhases = *(*[]string)(unsafe.Pointer(&in.SkipPhases))
+	out.ImagePullPolicy = corev1.PullPolicy(in.ImagePullPolicy)
+	out.ImagePullSerial = (*bool)(unsafe.Pointer(in.ImagePullSerial))
 	return nil
 }
 
@@ -1029,6 +1031,8 @@ func autoConvert_kubeadm_UpgradeApplyConfiguration_To_v1beta4_UpgradeApplyConfig
 	out.Patches = (*Patches)(unsafe.Pointer(in.Patches))
 	out.PrintConfig = (*bool)(unsafe.Pointer(in.PrintConfig))
 	out.SkipPhases = *(*[]string)(unsafe.Pointer(&in.SkipPhases))
+	out.ImagePullPolicy = corev1.PullPolicy(in.ImagePullPolicy)
+	out.ImagePullSerial = (*bool)(unsafe.Pointer(in.ImagePullSerial))
 	return nil
 }
 
@@ -1110,6 +1114,8 @@ func autoConvert_v1beta4_UpgradeNodeConfiguration_To_kubeadm_UpgradeNodeConfigur
 	out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors))
 	out.SkipPhases = *(*[]string)(unsafe.Pointer(&in.SkipPhases))
 	out.Patches = (*kubeadm.Patches)(unsafe.Pointer(in.Patches))
+	out.ImagePullPolicy = corev1.PullPolicy(in.ImagePullPolicy)
+	out.ImagePullSerial = (*bool)(unsafe.Pointer(in.ImagePullSerial))
 	return nil
 }
 
@@ -1125,6 +1131,8 @@ func autoConvert_kubeadm_UpgradeNodeConfiguration_To_v1beta4_UpgradeNodeConfigur
 	out.IgnorePreflightErrors = *(*[]string)(unsafe.Pointer(&in.IgnorePreflightErrors))
 	out.SkipPhases = *(*[]string)(unsafe.Pointer(&in.SkipPhases))
 	out.Patches = (*Patches)(unsafe.Pointer(in.Patches))
+	out.ImagePullPolicy = corev1.PullPolicy(in.ImagePullPolicy)
+	out.ImagePullSerial = (*bool)(unsafe.Pointer(in.ImagePullSerial))
 	return nil
 }
 

cmd/kubeadm/app/apis/kubeadm/v1beta4/zz_generated.deepcopy.go

@@ -727,6 +727,11 @@ func (in *UpgradeApplyConfiguration) DeepCopyInto(out *UpgradeApplyConfiguration
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.ImagePullSerial != nil {
+		in, out := &in.ImagePullSerial, &out.ImagePullSerial
+		*out = new(bool)
+		**out = **in
+	}
 	return
 }
 
@@ -823,6 +828,11 @@ func (in *UpgradeNodeConfiguration) DeepCopyInto(out *UpgradeNodeConfiguration)
 		*out = new(Patches)
 		**out = **in
 	}
+	if in.ImagePullSerial != nil {
+		in, out := &in.ImagePullSerial, &out.ImagePullSerial
+		*out = new(bool)
+		**out = **in
+	}
 	return
 }
 

cmd/kubeadm/app/apis/kubeadm/zz_generated.deepcopy.go

@@ -767,6 +767,11 @@ func (in *UpgradeApplyConfiguration) DeepCopyInto(out *UpgradeApplyConfiguration
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.ImagePullSerial != nil {
+		in, out := &in.ImagePullSerial, &out.ImagePullSerial
+		*out = new(bool)
+		**out = **in
+	}
 	return
 }
 
@@ -863,6 +868,11 @@ func (in *UpgradeNodeConfiguration) DeepCopyInto(out *UpgradeNodeConfiguration)
 		*out = new(Patches)
 		**out = **in
 	}
+	if in.ImagePullSerial != nil {
+		in, out := &in.ImagePullSerial, &out.ImagePullSerial
+		*out = new(bool)
+		**out = **in
+	}
 	return
 }
 

cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go

@@ -54,13 +54,19 @@ func runPreflight(c workflow.RunData) error {
 		return err
 	}
 
-	// if this is a control-plane node, pull the basic images
+	// If this is a control-plane node, pull the basic images
 	if data.IsControlPlaneNode() {
+		// Update the InitConfiguration used for RunPullImagesCheck with ImagePullPolicy and ImagePullSerial
+		// that come from UpgradeNodeConfiguration.
+		initConfig := data.InitCfg()
+		initConfig.NodeRegistration.ImagePullPolicy = data.Cfg().Node.ImagePullPolicy
+		initConfig.NodeRegistration.ImagePullSerial = data.Cfg().Node.ImagePullSerial
+
 		if !data.DryRun() {
 			fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
 			fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
 			fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'")
-			if err := preflight.RunPullImagesCheck(utilsexec.New(), data.InitCfg(), data.IgnorePreflightErrors()); err != nil {
+			if err := preflight.RunPullImagesCheck(utilsexec.New(), initConfig, data.IgnorePreflightErrors()); err != nil {
 				return err
 			}
 		} else {

cmd/kubeadm/app/cmd/upgrade/common.go

@@ -106,6 +106,13 @@ func enforceRequirements(flagSet *pflag.FlagSet, flags *applyPlanFlags, args []s
 		return nil, nil, nil, nil, errors.Wrap(err, "[upgrade/init config] FATAL")
 	}
 
+	// Set the ImagePullPolicy and ImagePullSerial from the UpgradeApplyConfiguration to the InitConfiguration.
+	// These are used by preflight.RunPullImagesCheck() when running 'apply'.
+	if upgradeApply {
+		initCfg.NodeRegistration.ImagePullPolicy = upgradeCfg.Apply.ImagePullPolicy
+		initCfg.NodeRegistration.ImagePullSerial = upgradeCfg.Apply.ImagePullSerial
+	}
+
 	newK8sVersion := upgradeCfg.Plan.KubernetesVersion
 	if upgradeApply {
 		newK8sVersion = upgradeCfg.Apply.KubernetesVersion

cmd/kubeadm/app/cmd/upgrade/node.go

@@ -180,11 +180,11 @@ func newNodeData(cmd *cobra.Command, args []string, nodeOptions *nodeOptions, ou
 		return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap")
 	}
 
-	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(nodeOptions.ignorePreflightErrors, initCfg.NodeRegistration.IgnorePreflightErrors)
+	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(nodeOptions.ignorePreflightErrors, upgradeCfg.Node.IgnorePreflightErrors)
 	if err != nil {
 		return nil, err
 	}
-	// Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration:
+	// Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration:
 	initCfg.NodeRegistration.IgnorePreflightErrors = sets.List(ignorePreflightErrorsSet)
 
 	var patchesDir string

cmd/kubeadm/app/util/config/upgradeconfiguration_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/lithammer/dedent"
 
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/yaml"
@@ -53,10 +54,14 @@ func TestDocMapToUpgradeConfiguration(t *testing.T) {
 				Apply: kubeadmapi.UpgradeApplyConfiguration{
 					CertificateRenewal: ptr.To(true),
 					EtcdUpgrade:        ptr.To(true),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 				Node: kubeadmapi.UpgradeNodeConfiguration{
 					CertificateRenewal: ptr.To(true),
 					EtcdUpgrade:        ptr.To(true),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 			},
 		},
@@ -78,10 +83,14 @@ func TestDocMapToUpgradeConfiguration(t *testing.T) {
 				Apply: kubeadmapi.UpgradeApplyConfiguration{
 					CertificateRenewal: ptr.To(false),
 					EtcdUpgrade:        ptr.To(true),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 				Node: kubeadmapi.UpgradeNodeConfiguration{
 					CertificateRenewal: ptr.To(true),
 					EtcdUpgrade:        ptr.To(false),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 			},
 		},
@@ -162,10 +171,14 @@ func TestLoadUpgradeConfigurationFromFile(t *testing.T) {
 				Apply: kubeadmapi.UpgradeApplyConfiguration{
 					CertificateRenewal: ptr.To(true),
 					EtcdUpgrade:        ptr.To(true),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 				Node: kubeadmapi.UpgradeNodeConfiguration{
 					CertificateRenewal: ptr.To(true),
 					EtcdUpgrade:        ptr.To(true),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 			},
 			wantErr: false,
@@ -214,10 +227,14 @@ func TestDefaultedUpgradeConfiguration(t *testing.T) {
 				Apply: kubeadmapi.UpgradeApplyConfiguration{
 					CertificateRenewal: ptr.To(true),
 					EtcdUpgrade:        ptr.To(true),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 				Node: kubeadmapi.UpgradeNodeConfiguration{
 					CertificateRenewal: ptr.To(true),
 					EtcdUpgrade:        ptr.To(true),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 			},
 		},
@@ -226,9 +243,13 @@ func TestDefaultedUpgradeConfiguration(t *testing.T) {
 			cfg: &kubeadmapiv1.UpgradeConfiguration{
 				Apply: kubeadmapiv1.UpgradeApplyConfiguration{
 					CertificateRenewal: ptr.To(false),
+					ImagePullPolicy:    v1.PullAlways,
+					ImagePullSerial:    ptr.To(false),
 				},
 				Node: kubeadmapiv1.UpgradeNodeConfiguration{
 					EtcdUpgrade:     ptr.To(false),
+					ImagePullPolicy: v1.PullAlways,
+					ImagePullSerial: ptr.To(false),
 				},
 				TypeMeta: metav1.TypeMeta{
 					APIVersion: kubeadmapiv1.SchemeGroupVersion.String(),
@@ -239,10 +260,14 @@ func TestDefaultedUpgradeConfiguration(t *testing.T) {
 				Apply: kubeadmapi.UpgradeApplyConfiguration{
 					CertificateRenewal: ptr.To(false),
 					EtcdUpgrade:        ptr.To(true),
+					ImagePullPolicy:    v1.PullAlways,
+					ImagePullSerial:    ptr.To(false),
 				},
 				Node: kubeadmapi.UpgradeNodeConfiguration{
 					CertificateRenewal: ptr.To(true),
 					EtcdUpgrade:        ptr.To(false),
+					ImagePullPolicy:    v1.PullAlways,
+					ImagePullSerial:    ptr.To(false),
 				},
 			},
 		},
@@ -269,15 +294,6 @@ func TestLoadOrDefaultUpgradeConfiguration(t *testing.T) {
 	}()
 	filename := "kubeadmConfig"
 	filePath := filepath.Join(tmpdir, filename)
-	fileContents := dedent.Dedent(`
-				apiVersion: kubeadm.k8s.io/v1beta4
-				kind: UpgradeConfiguration
-			`)
-	err = os.WriteFile(filePath, []byte(fileContents), 0644)
-	if err != nil {
-		t.Fatalf("Couldn't write content to file: %v", err)
-	}
-
 	options := LoadOrDefaultConfigurationOptions{}
 
 	tests := []struct {
@@ -305,10 +321,14 @@ func TestLoadOrDefaultUpgradeConfiguration(t *testing.T) {
 				Apply: kubeadmapi.UpgradeApplyConfiguration{
 					CertificateRenewal: ptr.To(false),
 					EtcdUpgrade:        ptr.To(true),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 				Node: kubeadmapi.UpgradeNodeConfiguration{
 					CertificateRenewal: ptr.To(true),
 					EtcdUpgrade:        ptr.To(false),
+					ImagePullPolicy:    v1.PullIfNotPresent,
+					ImagePullSerial:    ptr.To(true),
 				},
 			},
 		},
@@ -318,9 +338,15 @@ func TestLoadOrDefaultUpgradeConfiguration(t *testing.T) {
 			cfg: &kubeadmapiv1.UpgradeConfiguration{
 				Apply: kubeadmapiv1.UpgradeApplyConfiguration{
 					CertificateRenewal: ptr.To(false),
+					EtcdUpgrade:        ptr.To(false),
+					ImagePullPolicy:    v1.PullNever,
+					ImagePullSerial:    ptr.To(false),
 				},
 				Node: kubeadmapiv1.UpgradeNodeConfiguration{
+					CertificateRenewal: ptr.To(false),
 					EtcdUpgrade:        ptr.To(false),
+					ImagePullPolicy:    v1.PullNever,
+					ImagePullSerial:    ptr.To(false),
 				},
 				TypeMeta: metav1.TypeMeta{
 					APIVersion: kubeadmapiv1.SchemeGroupVersion.String(),
@@ -329,18 +355,31 @@ func TestLoadOrDefaultUpgradeConfiguration(t *testing.T) {
 			},
 			want: &kubeadmapi.UpgradeConfiguration{
 				Apply: kubeadmapi.UpgradeApplyConfiguration{
-					CertificateRenewal: ptr.To(true),
+					CertificateRenewal: ptr.To(false),
-					EtcdUpgrade:        ptr.To(true),
+					EtcdUpgrade:        ptr.To(false),
+					ImagePullPolicy:    v1.PullNever,
+					ImagePullSerial:    ptr.To(false),
 				},
 				Node: kubeadmapi.UpgradeNodeConfiguration{
-					CertificateRenewal: ptr.To(true),
+					CertificateRenewal: ptr.To(false),
-					EtcdUpgrade:        ptr.To(true),
+					EtcdUpgrade:        ptr.To(false),
+					ImagePullPolicy:    v1.PullNever,
+					ImagePullSerial:    ptr.To(false),
 				},
 			},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			bytes, err := yaml.Marshal(tt.cfg)
+			if err != nil {
+				t.Fatalf("Could not marshal test config: %v", err)
+			}
+			err = os.WriteFile(filePath, bytes, 0644)
+			if err != nil {
+				t.Fatalf("Couldn't write content to file: %v", err)
+			}
+
 			got, _ := LoadOrDefaultUpgradeConfiguration(tt.cfgPath, tt.cfg, options)
 			if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreFields(kubeadmapi.UpgradeConfiguration{}, "Timeouts")); diff != "" {
 				t.Errorf("LoadOrDefaultUpgradeConfiguration returned unexpected diff (-want,+got):\n%s", diff)