From 47827f4d9a51d3bcc0ed0a6893969d00d45cce22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanislav=20L=C3=A1zni=C4=8Dka?= Date: Tue, 15 Oct 2024 17:04:36 +0200 Subject: [PATCH] kubelet: modify KubeletConfiguration API with image pull policies Also adds PreloadedImagesVerificationAllowlist to API exceptions list for missing list type as this is not a part of the REST API. --- pkg/kubelet/apis/config/helpers_test.go | 2 + pkg/kubelet/apis/config/types.go | 38 ++ pkg/kubelet/apis/config/v1beta1/defaults.go | 6 + .../apis/config/v1beta1/defaults_test.go | 333 +++++++++--------- .../apis/config/validation/validation.go | 27 ++ .../apis/config/validation/validation_test.go | 33 ++ .../k8s.io/kubelet/config/v1beta1/types.go | 41 +++ 7 files changed, 316 insertions(+), 164 deletions(-) diff --git a/pkg/kubelet/apis/config/helpers_test.go b/pkg/kubelet/apis/config/helpers_test.go index 568b0be20df..b170def02dc 100644 --- a/pkg/kubelet/apis/config/helpers_test.go +++ b/pkg/kubelet/apis/config/helpers_test.go @@ -243,6 +243,7 @@ var ( "ImageGCLowThresholdPercent", "ImageMinimumGCAge.Duration", "ImageMaximumGCAge.Duration", + "ImagePullCredentialsVerificationPolicy", "KernelMemcgNotification", "KubeAPIBurst", "KubeAPIQPS", @@ -268,6 +269,7 @@ var ( "PodPidsLimit", "PodsPerCore", "Port", + "PreloadedImagesVerificationAllowlist[*]", "ProtectKernelDefaults", "ProviderID", "ReadOnlyPort", diff --git a/pkg/kubelet/apis/config/types.go b/pkg/kubelet/apis/config/types.go index 5cb669834d0..d295f16122c 100644 --- a/pkg/kubelet/apis/config/types.go +++ b/pkg/kubelet/apis/config/types.go @@ -155,6 +155,25 @@ type KubeletConfiguration struct { // pulls to burst to this number, while still not exceeding registryPullQPS. // Only used if registryPullQPS > 0. RegistryBurst int32 + // imagePullCredentialsVerificationPolicy determines how credentials should be + // verified when pod requests an image that is already present on the node: + // - NeverVerify + // - anyone on a node can use any image present on the node + // - NeverVerifyPreloadedImages + // - images that were pulled to the node by something else than the kubelet + // can be used without reverifying pull credentials + // - NeverVerifyAllowlistedImages + // - like "NeverVerifyPreloadedImages" but only node images from + // `preloadedImagesVerificationAllowlist` don't require reverification + // - AlwaysVerify + // - all images require credential reverification + ImagePullCredentialsVerificationPolicy string + // preloadedImagesVerificationAllowlist specifies a list of images that are + // exempted from credential reverification for the "NeverVerifyAllowlistedImages" + // `imagePullCredentialsVerificationPolicy`. + // The list accepts a full path segment wildcard suffix "/*". + // Only use image specs without an image tag or digest. + PreloadedImagesVerificationAllowlist []string // eventRecordQPS is the maximum event creations per second. If 0, there // is no limit enforced. EventRecordQPS int32 @@ -770,6 +789,25 @@ type CrashLoopBackOffConfig struct { MaxContainerRestartPeriod *metav1.Duration } +// ImagePullCredentialsVerificationPolicy is an enum for the policy that is enforced +// when pod is requesting an image that appears on the system +type ImagePullCredentialsVerificationPolicy string + +const ( + // NeverVerify will never require credential verification for images that + // already exist on the node + NeverVerify ImagePullCredentialsVerificationPolicy = "NeverVerify" + // NeverVerifyPreloadedImages does not require credential verification for images + // pulled outside the kubelet process + NeverVerifyPreloadedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyPreloadedImages" + // NeverVerifyAllowlistedImages does not require credential verification for + // a list of images that were pulled outside the kubelet process + NeverVerifyAllowlistedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyAllowlistedImages" + // AlwaysVerify requires credential verification for accessing any image on the + // node irregardless how it was pulled + AlwaysVerify ImagePullCredentialsVerificationPolicy = "AlwaysVerify" +) + // ImagePullIntent is a record of the kubelet attempting to pull an image. // // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index 6a7ea7fd468..04d71d00bd7 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -313,4 +313,10 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura obj.CrashLoopBackOff.MaxContainerRestartPeriod = &metav1.Duration{Duration: MaxContainerBackOff} } } + + if localFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) { + if obj.ImagePullCredentialsVerificationPolicy == "" { + obj.ImagePullCredentialsVerificationPolicy = kubeletconfigv1beta1.NeverVerifyPreloadedImages + } + } } diff --git a/pkg/kubelet/apis/config/v1beta1/defaults_test.go b/pkg/kubelet/apis/config/v1beta1/defaults_test.go index 391bdbe0b9d..9c79a088cc0 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults_test.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults_test.go @@ -77,6 +77,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { NodeLeaseDurationSeconds: 40, ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute}, ImageMaximumGCAge: metav1.Duration{}, + ImagePullCredentialsVerificationPolicy: "", ImageGCHighThresholdPercent: ptr.To[int32](85), ImageGCLowThresholdPercent: ptr.To[int32](80), ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", @@ -168,74 +169,75 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { CacheUnauthorizedTTL: zeroDuration, }, }, - RegistryPullQPS: ptr.To[int32](0), - RegistryBurst: 0, - EventRecordQPS: ptr.To[int32](0), - EventBurst: 0, - EnableDebuggingHandlers: ptr.To(false), - EnableContentionProfiling: false, - HealthzPort: ptr.To[int32](0), - HealthzBindAddress: "", - OOMScoreAdj: ptr.To[int32](0), - ClusterDomain: "", - ClusterDNS: []string{}, - StreamingConnectionIdleTimeout: zeroDuration, - NodeStatusUpdateFrequency: zeroDuration, - NodeStatusReportFrequency: zeroDuration, - NodeLeaseDurationSeconds: 0, - ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", - ImageMinimumGCAge: zeroDuration, - ImageGCHighThresholdPercent: ptr.To[int32](0), - ImageGCLowThresholdPercent: ptr.To[int32](0), - VolumeStatsAggPeriod: zeroDuration, - KubeletCgroups: "", - SystemCgroups: "", - CgroupRoot: "", - CgroupsPerQOS: ptr.To(false), - CgroupDriver: "", - CPUManagerPolicy: "", - CPUManagerPolicyOptions: map[string]string{}, - CPUManagerReconcilePeriod: zeroDuration, - MemoryManagerPolicy: "", - TopologyManagerPolicy: "", - TopologyManagerScope: "", - QOSReserved: map[string]string{}, - RuntimeRequestTimeout: zeroDuration, - HairpinMode: "", - MaxPods: 0, - PodCIDR: "", - PodPidsLimit: ptr.To[int64](0), - ResolverConfig: ptr.To(""), - RunOnce: false, - CPUCFSQuota: ptr.To(false), - CPUCFSQuotaPeriod: &zeroDuration, - NodeStatusMaxImages: ptr.To[int32](0), - MaxOpenFiles: 0, - ContentType: "", - KubeAPIQPS: ptr.To[int32](0), - KubeAPIBurst: 0, - SerializeImagePulls: ptr.To(false), - MaxParallelImagePulls: nil, - EvictionHard: map[string]string{}, - EvictionSoft: map[string]string{}, - EvictionSoftGracePeriod: map[string]string{}, - EvictionPressureTransitionPeriod: zeroDuration, - EvictionMaxPodGracePeriod: 0, - EvictionMinimumReclaim: map[string]string{}, - MergeDefaultEvictionSettings: ptr.To(false), - PodsPerCore: 0, - EnableControllerAttachDetach: ptr.To(false), - ProtectKernelDefaults: false, - MakeIPTablesUtilChains: ptr.To(false), - IPTablesMasqueradeBit: ptr.To[int32](0), - IPTablesDropBit: ptr.To[int32](0), - FeatureGates: map[string]bool{}, - FailSwapOn: ptr.To(false), - MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: ""}, - ContainerLogMaxSize: "", - ContainerLogMaxFiles: ptr.To[int32](0), - ContainerLogMaxWorkers: ptr.To[int32](1), - ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second}, + RegistryPullQPS: ptr.To[int32](0), + RegistryBurst: 0, + EventRecordQPS: ptr.To[int32](0), + EventBurst: 0, + EnableDebuggingHandlers: ptr.To(false), + EnableContentionProfiling: false, + HealthzPort: ptr.To[int32](0), + HealthzBindAddress: "", + OOMScoreAdj: ptr.To[int32](0), + ClusterDomain: "", + ClusterDNS: []string{}, + StreamingConnectionIdleTimeout: zeroDuration, + NodeStatusUpdateFrequency: zeroDuration, + NodeStatusReportFrequency: zeroDuration, + NodeLeaseDurationSeconds: 0, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", + ImageMinimumGCAge: zeroDuration, + ImagePullCredentialsVerificationPolicy: "", + ImageGCHighThresholdPercent: ptr.To[int32](0), + ImageGCLowThresholdPercent: ptr.To[int32](0), + VolumeStatsAggPeriod: zeroDuration, + KubeletCgroups: "", + SystemCgroups: "", + CgroupRoot: "", + CgroupsPerQOS: ptr.To(false), + CgroupDriver: "", + CPUManagerPolicy: "", + CPUManagerPolicyOptions: map[string]string{}, + CPUManagerReconcilePeriod: zeroDuration, + MemoryManagerPolicy: "", + TopologyManagerPolicy: "", + TopologyManagerScope: "", + QOSReserved: map[string]string{}, + RuntimeRequestTimeout: zeroDuration, + HairpinMode: "", + MaxPods: 0, + PodCIDR: "", + PodPidsLimit: ptr.To[int64](0), + ResolverConfig: ptr.To(""), + RunOnce: false, + CPUCFSQuota: ptr.To(false), + CPUCFSQuotaPeriod: &zeroDuration, + NodeStatusMaxImages: ptr.To[int32](0), + MaxOpenFiles: 0, + ContentType: "", + KubeAPIQPS: ptr.To[int32](0), + KubeAPIBurst: 0, + SerializeImagePulls: ptr.To(false), + MaxParallelImagePulls: nil, + EvictionHard: map[string]string{}, + EvictionSoft: map[string]string{}, + EvictionSoftGracePeriod: map[string]string{}, + EvictionPressureTransitionPeriod: zeroDuration, + EvictionMaxPodGracePeriod: 0, + EvictionMinimumReclaim: map[string]string{}, + MergeDefaultEvictionSettings: ptr.To(false), + PodsPerCore: 0, + EnableControllerAttachDetach: ptr.To(false), + ProtectKernelDefaults: false, + MakeIPTablesUtilChains: ptr.To(false), + IPTablesMasqueradeBit: ptr.To[int32](0), + IPTablesDropBit: ptr.To[int32](0), + FeatureGates: map[string]bool{}, + FailSwapOn: ptr.To(false), + MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: ""}, + ContainerLogMaxSize: "", + ContainerLogMaxFiles: ptr.To[int32](0), + ContainerLogMaxWorkers: ptr.To[int32](1), + ContainerLogMonitorInterval: &metav1.Duration{Duration: 10 * time.Second}, ConfigMapAndSecretChangeDetectionStrategy: v1beta1.WatchChangeDetectionStrategy, SystemReserved: map[string]string{}, KubeReserved: map[string]string{}, @@ -307,6 +309,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { ImageMinimumGCAge: metav1.Duration{Duration: 2 * time.Minute}, ImageGCHighThresholdPercent: ptr.To[int32](0), ImageGCLowThresholdPercent: ptr.To[int32](0), + ImagePullCredentialsVerificationPolicy: "", VolumeStatsAggPeriod: metav1.Duration{Duration: time.Minute}, CgroupsPerQOS: ptr.To(false), CgroupDriver: "cgroupfs", @@ -407,54 +410,55 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { CacheUnauthorizedTTL: metav1.Duration{Duration: 60 * time.Second}, }, }, - RegistryPullQPS: ptr.To[int32](1), - RegistryBurst: 1, - EventRecordQPS: ptr.To[int32](1), - EventBurst: 1, - EnableDebuggingHandlers: ptr.To(true), - EnableContentionProfiling: true, - HealthzPort: ptr.To[int32](1), - HealthzBindAddress: "127.0.0.2", - OOMScoreAdj: ptr.To[int32](1), - ClusterDomain: "cluster-domain", - ClusterDNS: []string{"192.168.1.3"}, - StreamingConnectionIdleTimeout: metav1.Duration{Duration: 60 * time.Second}, - NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, - NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, - NodeLeaseDurationSeconds: 1, - ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", - ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, - ImageGCHighThresholdPercent: ptr.To[int32](1), - ImageGCLowThresholdPercent: ptr.To[int32](1), - VolumeStatsAggPeriod: metav1.Duration{Duration: 60 * time.Second}, - KubeletCgroups: "kubelet-cgroup", - SystemCgroups: "system-cgroup", - CgroupRoot: "root-cgroup", - CgroupsPerQOS: ptr.To(true), - CgroupDriver: "systemd", - CPUManagerPolicy: "cpu-manager-policy", - CPUManagerPolicyOptions: map[string]string{"key": "value"}, - CPUManagerReconcilePeriod: metav1.Duration{Duration: 60 * time.Second}, - MemoryManagerPolicy: v1beta1.StaticMemoryManagerPolicy, - TopologyManagerPolicy: v1beta1.RestrictedTopologyManagerPolicy, - TopologyManagerScope: v1beta1.PodTopologyManagerScope, - QOSReserved: map[string]string{"memory": "10%"}, - RuntimeRequestTimeout: metav1.Duration{Duration: 60 * time.Second}, - HairpinMode: v1beta1.HairpinVeth, - MaxPods: 1, - PodCIDR: "192.168.1.0/24", - PodPidsLimit: ptr.To[int64](1), - ResolverConfig: ptr.To("resolver-config"), - RunOnce: true, - CPUCFSQuota: ptr.To(true), - CPUCFSQuotaPeriod: &metav1.Duration{Duration: 60 * time.Second}, - NodeStatusMaxImages: ptr.To[int32](1), - MaxOpenFiles: 1, - ContentType: "application/protobuf", - KubeAPIQPS: ptr.To[int32](1), - KubeAPIBurst: 1, - SerializeImagePulls: ptr.To(true), - MaxParallelImagePulls: ptr.To[int32](5), + RegistryPullQPS: ptr.To[int32](1), + RegistryBurst: 1, + EventRecordQPS: ptr.To[int32](1), + EventBurst: 1, + EnableDebuggingHandlers: ptr.To(true), + EnableContentionProfiling: true, + HealthzPort: ptr.To[int32](1), + HealthzBindAddress: "127.0.0.2", + OOMScoreAdj: ptr.To[int32](1), + ClusterDomain: "cluster-domain", + ClusterDNS: []string{"192.168.1.3"}, + StreamingConnectionIdleTimeout: metav1.Duration{Duration: 60 * time.Second}, + NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, + NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, + NodeLeaseDurationSeconds: 1, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", + ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, + ImageGCHighThresholdPercent: ptr.To[int32](1), + ImageGCLowThresholdPercent: ptr.To[int32](1), + PreloadedImagesVerificationAllowlist: []string{"test.test/repo/image"}, + VolumeStatsAggPeriod: metav1.Duration{Duration: 60 * time.Second}, + KubeletCgroups: "kubelet-cgroup", + SystemCgroups: "system-cgroup", + CgroupRoot: "root-cgroup", + CgroupsPerQOS: ptr.To(true), + CgroupDriver: "systemd", + CPUManagerPolicy: "cpu-manager-policy", + CPUManagerPolicyOptions: map[string]string{"key": "value"}, + CPUManagerReconcilePeriod: metav1.Duration{Duration: 60 * time.Second}, + MemoryManagerPolicy: v1beta1.StaticMemoryManagerPolicy, + TopologyManagerPolicy: v1beta1.RestrictedTopologyManagerPolicy, + TopologyManagerScope: v1beta1.PodTopologyManagerScope, + QOSReserved: map[string]string{"memory": "10%"}, + RuntimeRequestTimeout: metav1.Duration{Duration: 60 * time.Second}, + HairpinMode: v1beta1.HairpinVeth, + MaxPods: 1, + PodCIDR: "192.168.1.0/24", + PodPidsLimit: ptr.To[int64](1), + ResolverConfig: ptr.To("resolver-config"), + RunOnce: true, + CPUCFSQuota: ptr.To(true), + CPUCFSQuotaPeriod: &metav1.Duration{Duration: 60 * time.Second}, + NodeStatusMaxImages: ptr.To[int32](1), + MaxOpenFiles: 1, + ContentType: "application/protobuf", + KubeAPIQPS: ptr.To[int32](1), + KubeAPIBurst: 1, + SerializeImagePulls: ptr.To(true), + MaxParallelImagePulls: ptr.To[int32](5), EvictionHard: map[string]string{ "memory.available": "1Mi", "nodefs.available": "1%", @@ -563,54 +567,55 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { CacheUnauthorizedTTL: metav1.Duration{Duration: 60 * time.Second}, }, }, - RegistryPullQPS: ptr.To[int32](1), - RegistryBurst: 1, - EventRecordQPS: ptr.To[int32](1), - EventBurst: 1, - EnableDebuggingHandlers: ptr.To(true), - EnableContentionProfiling: true, - HealthzPort: ptr.To[int32](1), - HealthzBindAddress: "127.0.0.2", - OOMScoreAdj: ptr.To[int32](1), - ClusterDomain: "cluster-domain", - ClusterDNS: []string{"192.168.1.3"}, - StreamingConnectionIdleTimeout: metav1.Duration{Duration: 60 * time.Second}, - NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, - NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, - NodeLeaseDurationSeconds: 1, - ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", - ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, - ImageGCHighThresholdPercent: ptr.To[int32](1), - ImageGCLowThresholdPercent: ptr.To[int32](1), - VolumeStatsAggPeriod: metav1.Duration{Duration: 60 * time.Second}, - KubeletCgroups: "kubelet-cgroup", - SystemCgroups: "system-cgroup", - CgroupRoot: "root-cgroup", - CgroupsPerQOS: ptr.To(true), - CgroupDriver: "systemd", - CPUManagerPolicy: "cpu-manager-policy", - CPUManagerPolicyOptions: map[string]string{"key": "value"}, - CPUManagerReconcilePeriod: metav1.Duration{Duration: 60 * time.Second}, - MemoryManagerPolicy: v1beta1.StaticMemoryManagerPolicy, - TopologyManagerPolicy: v1beta1.RestrictedTopologyManagerPolicy, - TopologyManagerScope: v1beta1.PodTopologyManagerScope, - QOSReserved: map[string]string{"memory": "10%"}, - RuntimeRequestTimeout: metav1.Duration{Duration: 60 * time.Second}, - HairpinMode: v1beta1.HairpinVeth, - MaxPods: 1, - PodCIDR: "192.168.1.0/24", - PodPidsLimit: ptr.To[int64](1), - ResolverConfig: ptr.To("resolver-config"), - RunOnce: true, - CPUCFSQuota: ptr.To(true), - CPUCFSQuotaPeriod: &metav1.Duration{Duration: 60 * time.Second}, - NodeStatusMaxImages: ptr.To[int32](1), - MaxOpenFiles: 1, - ContentType: "application/protobuf", - KubeAPIQPS: ptr.To[int32](1), - KubeAPIBurst: 1, - SerializeImagePulls: ptr.To(true), - MaxParallelImagePulls: ptr.To[int32](5), + RegistryPullQPS: ptr.To[int32](1), + RegistryBurst: 1, + EventRecordQPS: ptr.To[int32](1), + EventBurst: 1, + EnableDebuggingHandlers: ptr.To(true), + EnableContentionProfiling: true, + HealthzPort: ptr.To[int32](1), + HealthzBindAddress: "127.0.0.2", + OOMScoreAdj: ptr.To[int32](1), + ClusterDomain: "cluster-domain", + ClusterDNS: []string{"192.168.1.3"}, + StreamingConnectionIdleTimeout: metav1.Duration{Duration: 60 * time.Second}, + NodeStatusUpdateFrequency: metav1.Duration{Duration: 60 * time.Second}, + NodeStatusReportFrequency: metav1.Duration{Duration: 60 * time.Second}, + NodeLeaseDurationSeconds: 1, + ContainerRuntimeEndpoint: "unix:///run/containerd/containerd.sock", + ImageMinimumGCAge: metav1.Duration{Duration: 60 * time.Second}, + ImageGCHighThresholdPercent: ptr.To[int32](1), + ImageGCLowThresholdPercent: ptr.To[int32](1), + PreloadedImagesVerificationAllowlist: []string{"test.test/repo/image"}, + VolumeStatsAggPeriod: metav1.Duration{Duration: 60 * time.Second}, + KubeletCgroups: "kubelet-cgroup", + SystemCgroups: "system-cgroup", + CgroupRoot: "root-cgroup", + CgroupsPerQOS: ptr.To(true), + CgroupDriver: "systemd", + CPUManagerPolicy: "cpu-manager-policy", + CPUManagerPolicyOptions: map[string]string{"key": "value"}, + CPUManagerReconcilePeriod: metav1.Duration{Duration: 60 * time.Second}, + MemoryManagerPolicy: v1beta1.StaticMemoryManagerPolicy, + TopologyManagerPolicy: v1beta1.RestrictedTopologyManagerPolicy, + TopologyManagerScope: v1beta1.PodTopologyManagerScope, + QOSReserved: map[string]string{"memory": "10%"}, + RuntimeRequestTimeout: metav1.Duration{Duration: 60 * time.Second}, + HairpinMode: v1beta1.HairpinVeth, + MaxPods: 1, + PodCIDR: "192.168.1.0/24", + PodPidsLimit: ptr.To[int64](1), + ResolverConfig: ptr.To("resolver-config"), + RunOnce: true, + CPUCFSQuota: ptr.To(true), + CPUCFSQuotaPeriod: &metav1.Duration{Duration: 60 * time.Second}, + NodeStatusMaxImages: ptr.To[int32](1), + MaxOpenFiles: 1, + ContentType: "application/protobuf", + KubeAPIQPS: ptr.To[int32](1), + KubeAPIBurst: 1, + SerializeImagePulls: ptr.To(true), + MaxParallelImagePulls: ptr.To[int32](5), EvictionHard: map[string]string{ "memory.available": "1Mi", "nodefs.available": "1%", diff --git a/pkg/kubelet/apis/config/validation/validation.go b/pkg/kubelet/apis/config/validation/validation.go index a4195a56a68..29d74a70e71 100644 --- a/pkg/kubelet/apis/config/validation/validation.go +++ b/pkg/kubelet/apis/config/validation/validation.go @@ -32,6 +32,7 @@ import ( tracingapi "k8s.io/component-base/tracing/api/v1" "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + "k8s.io/kubernetes/pkg/kubelet/images" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" utilfs "k8s.io/kubernetes/pkg/util/filesystem" utiltaints "k8s.io/kubernetes/pkg/util/taints" @@ -286,6 +287,32 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for hairpinMode (--hairpin-mode). Valid options are %q, %q or %q", kc.HairpinMode, kubeletconfig.HairpinNone, kubeletconfig.HairpinVeth, kubeletconfig.PromiscuousBridge)) } + + if localFeatureGate.Enabled(features.KubeletEnsureSecretPulledImages) { + switch kc.ImagePullCredentialsVerificationPolicy { + case string(kubeletconfig.NeverVerify), + string(kubeletconfig.NeverVerifyPreloadedImages), + string(kubeletconfig.NeverVerifyAllowlistedImages), + string(kubeletconfig.AlwaysVerify): + default: + allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for imagePullCredentialsVerificationPolicy. Valid options are %q, %q, %q or %q", + kc.ImagePullCredentialsVerificationPolicy, kubeletconfig.NeverVerify, kubeletconfig.NeverVerifyPreloadedImages, kubeletconfig.NeverVerifyAllowlistedImages, kubeletconfig.AlwaysVerify)) + } + + if len(kc.PreloadedImagesVerificationAllowlist) > 0 && kc.ImagePullCredentialsVerificationPolicy != string(kubeletconfig.NeverVerifyAllowlistedImages) { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: can't set `preloadedImagesVerificationAllowlist` if `imagePullCredentialsVertificationPolicy` is not \"NeverVerifyAllowlistedImages\"")) + } else if err := images.ValidateAllowlistImagesPatterns(kc.PreloadedImagesVerificationAllowlist); err != nil { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: invalid image pattern in `preloadedImagesVerificationAllowlist`: %w", err)) + } + } else { + if len(kc.ImagePullCredentialsVerificationPolicy) > 0 { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: `imagePullCredentialsVerificationPolicy` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled")) + } + if len(kc.PreloadedImagesVerificationAllowlist) > 0 { + allErrors = append(allErrors, fmt.Errorf("invalid configuration: `preloadedImagesVerificationAllowlist` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled")) + } + } + if kc.ReservedSystemCPUs != "" { // --reserved-cpus does not support --system-reserved-cgroup or --kube-reserved-cgroup if kc.SystemReservedCgroup != "" || kc.KubeReservedCgroup != "" { diff --git a/pkg/kubelet/apis/config/validation/validation_test.go b/pkg/kubelet/apis/config/validation/validation_test.go index b09677228b5..08da05c2efc 100644 --- a/pkg/kubelet/apis/config/validation/validation_test.go +++ b/pkg/kubelet/apis/config/validation/validation_test.go @@ -728,6 +728,39 @@ func TestValidateKubeletConfiguration(t *testing.T) { return conf }, errMsg: "logging.format: Invalid value: \"invalid\": Unsupported log format", + }, { + name: "invalid imagePullCredentialsVerificationPolicy configuration", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": true} + conf.ImagePullCredentialsVerificationPolicy = "invalid" + return conf + }, + errMsg: `option "invalid" specified for imagePullCredentialsVerificationPolicy. Valid options are "NeverVerify", "NeverVerifyPreloadedImages", "NeverVerifyAllowlistedImages" or "AlwaysVerify"]`, + }, { + name: "invalid PreloadedImagesVerificationAllowlist configuration - featuregate enabled", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": true} + conf.ImagePullCredentialsVerificationPolicy = string(kubeletconfig.NeverVerify) + conf.PreloadedImagesVerificationAllowlist = []string{"test.test/repo"} + return conf + }, + errMsg: "can't set `preloadedImagesVerificationAllowlist` if `imagePullCredentialsVertificationPolicy` is not \"NeverVerifyAllowlistedImages\"]", + }, { + name: "invalid PreloadedImagesVerificationAllowlist configuration - featuregate disabled", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": false} + conf.ImagePullCredentialsVerificationPolicy = string(kubeletconfig.NeverVerify) + return conf + }, + errMsg: "invalid configuration: `imagePullCredentialsVerificationPolicy` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled", + }, { + name: "invalid PreloadedImagesVerificationAllowlist configuration", + configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { + conf.FeatureGates = map[string]bool{"KubeletEnsureSecretPulledImages": false} + conf.PreloadedImagesVerificationAllowlist = []string{"test.test/repo"} + return conf + }, + errMsg: "invalid configuration: `preloadedImagesVerificationAllowlist` must not be set if KubeletEnsureSecretPulledImages feature gate is not enabled", }, { name: "invalid FeatureGate", configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index edbee00b45f..b08cd91ad51 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -83,6 +83,25 @@ const ( StaticMemoryManagerPolicy = "Static" ) +// ImagePullCredentialsVerificationPolicy is an enum for the policy that is enforced +// when pod is requesting an image that appears on the system +type ImagePullCredentialsVerificationPolicy string + +const ( + // NeverVerify will never require credential verification for images that + // already exist on the node + NeverVerify ImagePullCredentialsVerificationPolicy = "NeverVerify" + // NeverVerifyPreloadedImages does not require credential verification for images + // pulled outside the kubelet process + NeverVerifyPreloadedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyPreloadedImages" + // NeverVerifyAllowlistedImages does not require credential verification for + // a list of images that were pulled outside the kubelet process + NeverVerifyAllowlistedImages ImagePullCredentialsVerificationPolicy = "NeverVerifyAllowlistedImages" + // AlwaysVerify requires credential verification for accessing any image on the + // node irregardless how it was pulled + AlwaysVerify ImagePullCredentialsVerificationPolicy = "AlwaysVerify" +) + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // KubeletConfiguration contains the configuration for the Kubelet @@ -210,6 +229,28 @@ type KubeletConfiguration struct { // Default: 10 // +optional RegistryBurst int32 `json:"registryBurst,omitempty"` + // imagePullCredentialsVerificationPolicy determines how credentials should be + // verified when pod requests an image that is already present on the node: + // - NeverVerify + // - anyone on a node can use any image present on the node + // - NeverVerifyPreloadedImages + // - images that were pulled to the node by something else than the kubelet + // can be used without reverifying pull credentials + // - NeverVerifyAllowlistedImages + // - like "NeverVerifyPreloadedImages" but only node images from + // `preloadedImagesVerificationAllowlist` don't require reverification + // - AlwaysVerify + // - all images require credential reverification + // +optional + ImagePullCredentialsVerificationPolicy ImagePullCredentialsVerificationPolicy `json:"imagePullCredentialsVerificationPolicy,omitempty"` + // preloadedImagesVerificationAllowlist specifies a list of images that are + // exempted from credential reverification for the "NeverVerifyAllowlistedImages" + // `imagePullCredentialsVerificationPolicy`. + // The list accepts a full path segment wildcard suffix "/*". + // Only use image specs without an image tag or digest. + // +optional + // +listType=set + PreloadedImagesVerificationAllowlist []string `json:"preloadedImagesVerificationAllowlist,omitempty"` // eventRecordQPS is the maximum event creations per second. If 0, there // is no limit enforced. The value cannot be a negative number. // Default: 50