kubelet: modify KubeletConfiguration API with image pull policies

Also adds PreloadedImagesVerificationAllowlist to API exceptions list
for missing list type as this is not a part of the REST API.

pkg/kubelet/apis/config/helpers_test.go

@@ -243,6 +243,7 @@ var (
 		"ImageGCLowThresholdPercent",
 		"ImageMinimumGCAge.Duration",
 		"ImageMaximumGCAge.Duration",
+		"ImagePullCredentialsVerificationPolicy",
 		"KernelMemcgNotification",
 		"KubeAPIBurst",
 		"KubeAPIQPS",
@@ -268,6 +269,7 @@ var (
 		"PodPidsLimit",
 		"PodsPerCore",
 		"Port",
+		"PreloadedImagesVerificationAllowlist[*]",
 		"ProtectKernelDefaults",
 		"ProviderID",
 		"ReadOnlyPort",

pkg/kubelet/apis/config/types.go

@@ -155,6 +155,25 @@ type KubeletConfiguration struct {
 	// pulls to burst to this number, while still not exceeding registryPullQPS.
 	// Only used if registryPullQPS > 0.
 	RegistryBurst int32
+	// imagePullCredentialsVerificationPolicy determines how credentials should be
+	// verified when pod requests an image that is already present on the node:
+	//   - NeverVerify
+	//       - anyone on a node can use any image present on the node
+	//   - NeverVerifyPreloadedImages
+	//       - images that were pulled to the node by something else than the kubelet
+	//         can be used without reverifying pull credentials
+	//   - NeverVerifyAllowlistedImages
+	//       - like "NeverVerifyPreloadedImages" but only node images from
+	//         `preloadedImagesVerificationAllowlist` don't require reverification
+	//   - AlwaysVerify
+	//       - all images require credential reverification
+	ImagePullCredentialsVerificationPolicy string
+	// preloadedImagesVerificationAllowlist specifies a list of images that are
+	// exempted from credential reverification for the "NeverVerifyAllowlistedImages"
+	// `imagePullCredentialsVerificationPolicy`.
+	// The list accepts a full path segment wildcard suffix "/*".
+	// Only use image specs without an image tag or digest.
+	PreloadedImagesVerificationAllowlist []string
 	// eventRecordQPS is the maximum event creations per second. If 0, there
 	// is no limit enforced.
 	EventRecordQPS int32
@@ -770,6 +789,25 @@ type CrashLoopBackOffConfig struct {
 	MaxContainerRestartPeriod *metav1.Duration
 }
 
+// ImagePullCredentialsVerificationPolicy is an enum for the policy that is enforced
+// when pod is requesting an image that appears on the system
+type ImagePullCredentialsVerificationPolicy string
+
+const (
+	// NeverVerify will never require credential verification for images that
+	// already exist on the node
+	NeverVerify ImagePullCredentialsVerificationPolicy = "NeverVerify"
+	// NeverVerifyPreloadedImages does not require credential verification for images
+	// pulled outside the kubelet process
+	NeverVerifyPreloadedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyPreloadedImages"
+	// NeverVerifyAllowlistedImages does not require credential verification for
+	// a list of images that were pulled outside the kubelet process
+	NeverVerifyAllowlistedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyAllowlistedImages"
+	// AlwaysVerify requires credential verification for accessing any image on the
+	// node irregardless how it was pulled
+	AlwaysVerify ImagePullCredentialsVerificationPolicy = "AlwaysVerify"
+)
+
 // ImagePullIntent is a record of the kubelet attempting to pull an image.
 //
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/kubelet/apis/config/v1beta1/defaults.go

@@ -313,4 +313,10 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura
 			obj.CrashLoopBackOff.MaxContainerRestartPeriod = &metav1.Duration{Duration: MaxContainerBackOff}
 		}
 	}
+
+	if localFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) {
+		if obj.ImagePullCredentialsVerificationPolicy == "" {
+			obj.ImagePullCredentialsVerificationPolicy = kubeletconfigv1beta1.NeverVerifyPreloadedImages
+		}
+	}
 }

pkg/kubelet/apis/config/v1beta1/defaults_test.go

@@ -77,6 +77,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				NodeLeaseDurationSeconds:                  40,
 				ImageMinimumGCAge:                         metav1.Duration{Duration: 2 * time.Minute},
 				ImageMaximumGCAge:                         metav1.Duration{},
+				ImagePullCredentialsVerificationPolicy:    "",
 				ImageGCHighThresholdPercent:               ptr.To[int32](85),
 				ImageGCLowThresholdPercent:                ptr.To[int32](80),
 				ContainerRuntimeEndpoint:                  "unix:///run/containerd/containerd.sock",
@@ -185,6 +186,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				NodeLeaseDurationSeconds:               0,
 				ContainerRuntimeEndpoint:               "unix:///run/containerd/containerd.sock",
 				ImageMinimumGCAge:                      zeroDuration,
+				ImagePullCredentialsVerificationPolicy: "",
 				ImageGCHighThresholdPercent:            ptr.To[int32](0),
 				ImageGCLowThresholdPercent:             ptr.To[int32](0),
 				VolumeStatsAggPeriod:                   zeroDuration,
@@ -307,6 +309,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				ImageMinimumGCAge:                         metav1.Duration{Duration: 2 * time.Minute},
 				ImageGCHighThresholdPercent:               ptr.To[int32](0),
 				ImageGCLowThresholdPercent:                ptr.To[int32](0),
+				ImagePullCredentialsVerificationPolicy:    "",
 				VolumeStatsAggPeriod:                      metav1.Duration{Duration: time.Minute},
 				CgroupsPerQOS:                             ptr.To(false),
 				CgroupDriver:                              "cgroupfs",
@@ -426,6 +429,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				ImageMinimumGCAge:                    metav1.Duration{Duration: 60 * time.Second},
 				ImageGCHighThresholdPercent:          ptr.To[int32](1),
 				ImageGCLowThresholdPercent:           ptr.To[int32](1),
+				PreloadedImagesVerificationAllowlist: []string{"test.test/repo/image"},
 				VolumeStatsAggPeriod:                 metav1.Duration{Duration: 60 * time.Second},
 				KubeletCgroups:                       "kubelet-cgroup",
 				SystemCgroups:                        "system-cgroup",
@@ -582,6 +586,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
 				ImageMinimumGCAge:                    metav1.Duration{Duration: 60 * time.Second},
 				ImageGCHighThresholdPercent:          ptr.To[int32](1),
 				ImageGCLowThresholdPercent:           ptr.To[int32](1),
+				PreloadedImagesVerificationAllowlist: []string{"test.test/repo/image"},
 				VolumeStatsAggPeriod:                 metav1.Duration{Duration: 60 * time.Second},
 				KubeletCgroups:                       "kubelet-cgroup",
 				SystemCgroups:                        "system-cgroup",

pkg/kubelet/apis/config/validation/validation.go

@@ -32,6 +32,7 @@ import (
 	tracingapi "k8s.io/component-base/tracing/api/v1"
 	"k8s.io/kubernetes/pkg/features"
 	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
+	"k8s.io/kubernetes/pkg/kubelet/images"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	utilfs "k8s.io/kubernetes/pkg/util/filesystem"
 	utiltaints "k8s.io/kubernetes/pkg/util/taints"
@@ -286,6 +287,32 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur
 		allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for hairpinMode (--hairpin-mode). Valid options are %q, %q or %q",
 			kc.HairpinMode, kubeletconfig.HairpinNone, kubeletconfig.HairpinVeth, kubeletconfig.PromiscuousBridge))
 	}
+
+	if localFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) {
+		switch kc.ImagePullCredentialsVerificationPolicy {
+		case string(kubeletconfig.NeverVerify),
+			string(kubeletconfig.NeverVerifyPreloadedImages),
+			string(kubeletconfig.NeverVerifyAllowlistedImages),
+			string(kubeletconfig.AlwaysVerify):
+		default:
+			allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for imagePullCredentialsVerificationPolicy. Valid options are %q, %q, %q or %q",
+				kc.ImagePullCredentialsVerificationPolicy, kubeletconfig.NeverVerify, kubeletconfig.NeverVerifyPreloadedImages, kubeletconfig.NeverVerifyAllowlistedImages, kubeletconfig.AlwaysVerify))
+		}
+
+		if len(kc.PreloadedImagesVerificationAllowlist) > 0 && kc.ImagePullCredentialsVerificationPolicy != string(kubeletconfig.NeverVerifyAllowlistedImages) {
+			allErrors = append(allErrors, fmt.Errorf("invalid configuration: can't set `preloadedImagesVerificationAllowlist` if `imagePullCredentialsVertificationPolicy` is not \"NeverVerifyAllowlistedImages\""))
+		} else if err := images.ValidateAllowlistImagesPatterns(kc.PreloadedImagesVerificationAllowlist); err != nil {
+			allErrors = append(allErrors, fmt.Errorf("invalid configuration: invalid image pattern in `preloadedImagesVerificationAllowlist`: %w", err))
+		}
+	} else {
+		if len(kc.ImagePullCredentialsVerificationPolicy) > 0 {
+			allErrors = append(allErrors, fmt.Errorf("invalid configuration: `imagePullCredentialsVerificationPolicy` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled"))
+		}
+		if len(kc.PreloadedImagesVerificationAllowlist) > 0 {
+			allErrors = append(allErrors, fmt.Errorf("invalid configuration: `preloadedImagesVerificationAllowlist` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled"))
+		}
+	}
+
 	if kc.ReservedSystemCPUs != "" {
 		// --reserved-cpus does not support --system-reserved-cgroup or --kube-reserved-cgroup
 		if kc.SystemReservedCgroup != "" || kc.KubeReservedCgroup != "" {

pkg/kubelet/apis/config/validation/validation_test.go

@@ -728,6 +728,39 @@ func TestValidateKubeletConfiguration(t *testing.T) {
 				return conf
 			},
 			errMsg: "logging.format: Invalid value: \"invalid\": Unsupported log format",
+		}, {
+			name: "invalid imagePullCredentialsVerificationPolicy configuration",
+			configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
+				conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": true}
+				conf.ImagePullCredentialsVerificationPolicy = "invalid"
+				return conf
+			},
+			errMsg: `option "invalid" specified for imagePullCredentialsVerificationPolicy. Valid options are "NeverVerify", "NeverVerifyPreloadedImages", "NeverVerifyAllowlistedImages" or "AlwaysVerify"]`,
+		}, {
+			name: "invalid PreloadedImagesVerificationAllowlist configuration - featuregate enabled",
+			configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
+				conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": true}
+				conf.ImagePullCredentialsVerificationPolicy = string(kubeletconfig.NeverVerify)
+				conf.PreloadedImagesVerificationAllowlist = []string{"test.test/repo"}
+				return conf
+			},
+			errMsg: "can't set `preloadedImagesVerificationAllowlist` if `imagePullCredentialsVertificationPolicy` is not \"NeverVerifyAllowlistedImages\"]",
+		}, {
+			name: "invalid PreloadedImagesVerificationAllowlist configuration - featuregate disabled",
+			configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
+				conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": false}
+				conf.ImagePullCredentialsVerificationPolicy = string(kubeletconfig.NeverVerify)
+				return conf
+			},
+			errMsg: "invalid configuration: `imagePullCredentialsVerificationPolicy` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled",
+		}, {
+			name: "invalid PreloadedImagesVerificationAllowlist configuration",
+			configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
+				conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": false}
+				conf.PreloadedImagesVerificationAllowlist = []string{"test.test/repo"}
+				return conf
+			},
+			errMsg: "invalid configuration: `preloadedImagesVerificationAllowlist` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled",
 		}, {
 			name: "invalid FeatureGate",
 			configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {

staging/src/k8s.io/kubelet/config/v1beta1/types.go

@@ -83,6 +83,25 @@ const (
 	StaticMemoryManagerPolicy = "Static"
 )
 
+// ImagePullCredentialsVerificationPolicy is an enum for the policy that is enforced
+// when pod is requesting an image that appears on the system
+type ImagePullCredentialsVerificationPolicy string
+
+const (
+	// NeverVerify will never require credential verification for images that
+	// already exist on the node
+	NeverVerify ImagePullCredentialsVerificationPolicy = "NeverVerify"
+	// NeverVerifyPreloadedImages does not require credential verification for images
+	// pulled outside the kubelet process
+	NeverVerifyPreloadedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyPreloadedImages"
+	// NeverVerifyAllowlistedImages does not require credential verification for
+	// a list of images that were pulled outside the kubelet process
+	NeverVerifyAllowlistedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyAllowlistedImages"
+	// AlwaysVerify requires credential verification for accessing any image on the
+	// node irregardless how it was pulled
+	AlwaysVerify ImagePullCredentialsVerificationPolicy = "AlwaysVerify"
+)
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // KubeletConfiguration contains the configuration for the Kubelet
@@ -210,6 +229,28 @@ type KubeletConfiguration struct {
 	// Default: 10
 	// +optional
 	RegistryBurst int32 `json:"registryBurst,omitempty"`
+	// imagePullCredentialsVerificationPolicy determines how credentials should be
+	// verified when pod requests an image that is already present on the node:
+	//   - NeverVerify
+	//       - anyone on a node can use any image present on the node
+	//   - NeverVerifyPreloadedImages
+	//       - images that were pulled to the node by something else than the kubelet
+	//         can be used without reverifying pull credentials
+	//   - NeverVerifyAllowlistedImages
+	//       - like "NeverVerifyPreloadedImages" but only node images from
+	//         `preloadedImagesVerificationAllowlist` don't require reverification
+	//   - AlwaysVerify
+	//       - all images require credential reverification
+	// +optional
+	ImagePullCredentialsVerificationPolicy ImagePullCredentialsVerificationPolicy `json:"imagePullCredentialsVerificationPolicy,omitempty"`
+	// preloadedImagesVerificationAllowlist specifies a list of images that are
+	// exempted from credential reverification for the "NeverVerifyAllowlistedImages"
+	// `imagePullCredentialsVerificationPolicy`.
+	// The list accepts a full path segment wildcard suffix "/*".
+	// Only use image specs without an image tag or digest.
+	// +optional
+	// +listType=set
+	PreloadedImagesVerificationAllowlist []string `json:"preloadedImagesVerificationAllowlist,omitempty"`
 	// eventRecordQPS is the maximum event creations per second. If 0, there
 	// is no limit enforced. The value cannot be a negative number.
 	// Default: 50