Merge pull request #130384 from zhifei92/migrate-kubelet-images-to-contextual-logging

chore(kubelet): migrate images to contextual logging
This commit is contained in:
Kubernetes Prow Robot
2025-08-27 23:17:09 -07:00
committed by GitHub
10 changed files with 129 additions and 103 deletions

View File

@@ -60,7 +60,7 @@ const (
// PostImageGCHook allows external sources to react to GC collect events.
// `remainingImages` is a list of images that were left on the system after garbage
// collection finished.
type PostImageGCHook func(remainingImages []string, gcStart time.Time)
type PostImageGCHook func(ctx context.Context, remainingImages []string, gcStart time.Time)
// StatsProvider is an interface for fetching stats used during image garbage
// collection.
@@ -77,7 +77,7 @@ type ImageGCManager interface {
GarbageCollect(ctx context.Context, beganGC time.Time) error
// Start async garbage collection of images.
Start()
Start(ctx context.Context)
GetImageList() ([]container.Image, error)
@@ -214,12 +214,12 @@ func NewImageGCManager(runtime container.Runtime, statsProvider StatsProvider, p
return im, nil
}
func (im *realImageGCManager) Start() {
ctx := context.Background()
func (im *realImageGCManager) Start(ctx context.Context) {
logger := klog.FromContext(ctx)
go wait.Until(func() {
_, err := im.detectImages(ctx, time.Now())
if err != nil {
klog.InfoS("Failed to monitor images", "err", err)
logger.Info("Failed to monitor images", "err", err)
}
}, 5*time.Minute, wait.NeverStop)
@@ -227,7 +227,7 @@ func (im *realImageGCManager) Start() {
go wait.Until(func() {
images, err := im.runtime.ListImages(ctx)
if err != nil {
klog.InfoS("Failed to update image list", "err", err)
logger.Info("Failed to update image list", "err", err)
} else {
im.imageCache.set(images)
}
@@ -241,6 +241,7 @@ func (im *realImageGCManager) GetImageList() ([]container.Image, error) {
}
func (im *realImageGCManager) detectImages(ctx context.Context, detectTime time.Time) (sets.Set[string], error) {
logger := klog.FromContext(ctx)
isRuntimeClassInImageCriAPIEnabled := utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClassInImageCriAPI)
imagesInUse := sets.New[string]()
@@ -261,11 +262,11 @@ func (im *realImageGCManager) detectImages(ctx context.Context, detectTime time.
}
if !isRuntimeClassInImageCriAPIEnabled {
klog.V(5).InfoS("Container uses image", "pod", klog.KRef(pod.Namespace, pod.Name), "containerName", container.Name, "containerImage", container.Image, "imageID", container.ImageID, "imageRef", container.ImageRef)
logger.V(5).Info("Container uses image", "pod", klog.KRef(pod.Namespace, pod.Name), "containerName", container.Name, "containerImage", container.Image, "imageID", container.ImageID, "imageRef", container.ImageRef)
imagesInUse.Insert(container.ImageID)
} else {
imageKey := getImageTuple(container.ImageID, container.ImageRuntimeHandler)
klog.V(5).InfoS("Container uses image", "pod", klog.KRef(pod.Namespace, pod.Name), "containerName", container.Name, "containerImage", container.Image, "imageID", container.ImageID, "imageRef", container.ImageRef, "imageKey", imageKey)
logger.V(5).Info("Container uses image", "pod", klog.KRef(pod.Namespace, pod.Name), "containerName", container.Name, "containerImage", container.Image, "imageID", container.ImageID, "imageRef", container.ImageRef, "imageKey", imageKey)
imagesInUse.Insert(imageKey)
}
}
@@ -279,17 +280,17 @@ func (im *realImageGCManager) detectImages(ctx context.Context, detectTime time.
for _, image := range images {
imageKey := image.ID
if !isRuntimeClassInImageCriAPIEnabled {
klog.V(5).InfoS("Adding image ID to currentImages", "imageID", imageKey)
logger.V(5).Info("Adding image ID to currentImages", "imageID", imageKey)
} else {
imageKey = getImageTuple(image.ID, image.Spec.RuntimeHandler)
klog.V(5).InfoS("Adding image ID with runtime class to currentImages", "imageKey", imageKey, "runtimeHandler", image.Spec.RuntimeHandler)
logger.V(5).Info("Adding image ID with runtime class to currentImages", "imageKey", imageKey, "runtimeHandler", image.Spec.RuntimeHandler)
}
currentImages.Insert(imageKey)
// New image, set it as detected now.
if _, ok := im.imageRecords[imageKey]; !ok {
klog.V(5).InfoS("Image ID is new", "imageID", imageKey, "runtimeHandler", image.Spec.RuntimeHandler)
logger.V(5).Info("Image ID is new", "imageID", imageKey, "runtimeHandler", image.Spec.RuntimeHandler)
im.imageRecords[imageKey] = &imageRecord{
firstDetected: detectTime,
runtimeHandlerUsedToPullImage: image.Spec.RuntimeHandler,
@@ -298,21 +299,21 @@ func (im *realImageGCManager) detectImages(ctx context.Context, detectTime time.
// Set last used time to now if the image is being used.
if isImageUsed(imageKey, imagesInUse) {
klog.V(5).InfoS("Setting Image ID lastUsed", "imageID", imageKey, "lastUsed", now)
logger.V(5).Info("Setting Image ID lastUsed", "imageID", imageKey, "lastUsed", now)
im.imageRecords[imageKey].lastUsed = now
}
klog.V(5).InfoS("Image ID has size", "imageID", imageKey, "size", image.Size)
logger.V(5).Info("Image ID has size", "imageID", imageKey, "size", image.Size)
im.imageRecords[imageKey].size = image.Size
klog.V(5).InfoS("Image ID is pinned", "imageID", imageKey, "pinned", image.Pinned)
logger.V(5).Info("Image ID is pinned", "imageID", imageKey, "pinned", image.Pinned)
im.imageRecords[imageKey].pinned = image.Pinned
}
// Remove old images from our records.
for image := range im.imageRecords {
if !currentImages.Has(image) {
klog.V(5).InfoS("Image ID is no longer present; removing from imageRecords", "imageID", image)
logger.V(5).Info("Image ID is no longer present; removing from imageRecords", "imageID", image)
delete(im.imageRecords, image)
}
}
@@ -322,6 +323,7 @@ func (im *realImageGCManager) detectImages(ctx context.Context, detectTime time.
// handleImageVolumes ensures that image volumes are considered as images in use.
func (im *realImageGCManager) handleImageVolumes(ctx context.Context, imagesInUse sets.Set[string], container *container.Container, pod *container.Pod, images []container.Image) error {
logger := klog.FromContext(ctx)
if !utilfeature.DefaultFeatureGate.Enabled(features.ImageVolume) {
return nil
}
@@ -334,7 +336,7 @@ func (im *realImageGCManager) handleImageVolumes(ctx context.Context, imagesInUs
for _, mount := range status.Mounts {
for _, image := range images {
if mount.Image != nil && mount.Image.Image == image.ID {
klog.V(5).InfoS("Container uses image as mount", "pod", klog.KRef(pod.Namespace, pod.Name), "containerName", container.Name, "imageID", image.ID)
logger.V(5).Info("Container uses image as mount", "pod", klog.KRef(pod.Namespace, pod.Name), "containerName", container.Name, "imageID", image.ID)
imagesInUse.Insert(image.ID)
}
}
@@ -345,6 +347,7 @@ func (im *realImageGCManager) handleImageVolumes(ctx context.Context, imagesInUs
func (im *realImageGCManager) GarbageCollect(ctx context.Context, beganGC time.Time) error {
ctx, otelSpan := im.tracer.Start(ctx, "Images/GarbageCollect")
logger := klog.FromContext(ctx)
defer otelSpan.End()
freeTime := time.Now()
@@ -373,7 +376,7 @@ func (im *realImageGCManager) GarbageCollect(ctx context.Context, beganGC time.T
}
if available > capacity {
klog.InfoS("Availability is larger than capacity", "available", available, "capacity", capacity)
logger.Info("Availability is larger than capacity", "available", available, "capacity", capacity)
available = capacity
}
@@ -388,14 +391,15 @@ func (im *realImageGCManager) GarbageCollect(ctx context.Context, beganGC time.T
usagePercent := 100 - int(available*100/capacity)
if usagePercent >= im.policy.HighThresholdPercent {
amountToFree := capacity*int64(100-im.policy.LowThresholdPercent)/100 - available
klog.InfoS("Disk usage on image filesystem is over the high threshold, trying to free bytes down to the low threshold", "usage", usagePercent, "highThreshold", im.policy.HighThresholdPercent, "amountToFree", amountToFree, "lowThreshold", im.policy.LowThresholdPercent)
logger.Info("Disk usage on image filesystem is over the high threshold, trying to free bytes down to the low threshold", "usage", usagePercent, "highThreshold", im.policy.HighThresholdPercent, "amountToFree", amountToFree, "lowThreshold", im.policy.LowThresholdPercent)
remainingImages, freed, err := im.freeSpace(ctx, amountToFree, freeTime, images)
logger.Info("Disk usage on image filesystem is over the high threshold, trying to free bytes down to the low threshold", "usage", usagePercent, "highThreshold", im.policy.HighThresholdPercent, "amountToFree", amountToFree, "lowThreshold", im.policy.LowThresholdPercent)
if err != nil {
// Failed to delete images, eg due to a read-only filesystem.
return err
}
im.runPostGCHooks(remainingImages, freeTime)
im.runPostGCHooks(ctx, remainingImages, freeTime)
if freed < amountToFree {
// This usually means the disk is full for reasons other than container
@@ -413,9 +417,9 @@ func (im *realImageGCManager) GarbageCollect(ctx context.Context, beganGC time.T
return nil
}
func (im *realImageGCManager) runPostGCHooks(remainingImages []string, gcStartTime time.Time) {
func (im *realImageGCManager) runPostGCHooks(ctx context.Context, remainingImages []string, gcStartTime time.Time) {
for _, h := range im.postGCHooks {
h(remainingImages, gcStartTime)
h(ctx, remainingImages, gcStartTime)
}
}
@@ -430,9 +434,10 @@ func (im *realImageGCManager) freeOldImages(ctx context.Context, images []evicti
return images, nil
}
var deletionErrors []error
logger := klog.FromContext(ctx)
remainingImages := make([]evictionInfo, 0)
for _, image := range images {
klog.V(5).InfoS("Evaluating image ID for possible garbage collection based on image age", "imageID", image.id)
logger.V(5).Info("Evaluating image ID for possible garbage collection based on image age", "imageID", image.id)
// Evaluate whether image is older than MaxAge.
if freeTime.Sub(image.lastUsed) > im.policy.MaxAge {
if err := im.freeImage(ctx, image, ImageGarbageCollectedTotalReasonAge); err != nil {
@@ -451,7 +456,8 @@ func (im *realImageGCManager) freeOldImages(ctx context.Context, images []evicti
}
func (im *realImageGCManager) DeleteUnusedImages(ctx context.Context) error {
klog.InfoS("Attempting to delete unused images")
logger := klog.FromContext(ctx)
logger.Info("Attempting to delete unused images")
freeTime := time.Now()
images, err := im.imagesInEvictionOrder(ctx, freeTime)
@@ -464,7 +470,7 @@ func (im *realImageGCManager) DeleteUnusedImages(ctx context.Context) error {
return err
}
im.runPostGCHooks(remainingImages, freeTime)
im.runPostGCHooks(ctx, remainingImages, freeTime)
return nil
}
@@ -477,22 +483,23 @@ func (im *realImageGCManager) DeleteUnusedImages(ctx context.Context) error {
func (im *realImageGCManager) freeSpace(ctx context.Context, bytesToFree int64, freeTime time.Time, images []evictionInfo) ([]string, int64, error) {
// Delete unused images until we've freed up enough space.
var deletionErrors []error
logger := klog.FromContext(ctx)
spaceFreed := int64(0)
var imagesLeft []string
for _, image := range images {
klog.V(5).InfoS("Evaluating image ID for possible garbage collection based on disk usage", "imageID", image.id, "runtimeHandler", image.imageRecord.runtimeHandlerUsedToPullImage)
logger.V(5).Info("Evaluating image ID for possible garbage collection based on disk usage", "imageID", image.id, "runtimeHandler", image.runtimeHandlerUsedToPullImage)
// Images that are currently in used were given a newer lastUsed.
if image.lastUsed.Equal(freeTime) || image.lastUsed.After(freeTime) {
klog.V(5).InfoS("Image ID was used too recently, not eligible for garbage collection", "imageID", image.id, "lastUsed", image.lastUsed, "freeTime", freeTime)
imagesLeft = append(imagesLeft, image.id)
logger.V(5).Info("Image ID was used too recently, not eligible for garbage collection", "imageID", image.id, "lastUsed", image.lastUsed, "freeTime", freeTime)
continue
}
// Avoid garbage collect the image if the image is not old enough.
// In such a case, the image may have just been pulled down, and will be used by a container right away.
if freeTime.Sub(image.firstDetected) < im.policy.MinAge {
klog.V(5).InfoS("Image ID's age is less than the policy's minAge, not eligible for garbage collection", "imageID", image.id, "age", freeTime.Sub(image.firstDetected), "minAge", im.policy.MinAge)
imagesLeft = append(imagesLeft, image.id)
logger.V(5).Info("Image ID's age is less than the policy's minAge, not eligible for garbage collection", "imageID", image.id, "age", freeTime.Sub(image.firstDetected), "minAge", im.policy.MinAge)
continue
}
@@ -518,7 +525,8 @@ func (im *realImageGCManager) freeImage(ctx context.Context, image evictionInfo,
isRuntimeClassInImageCriAPIEnabled := utilfeature.DefaultFeatureGate.Enabled(features.RuntimeClassInImageCriAPI)
// Remove image. Continue despite errors.
var err error
klog.InfoS("Removing image to free bytes", "imageID", image.id, "size", image.size, "runtimeHandler", image.runtimeHandlerUsedToPullImage)
logger := klog.FromContext(ctx)
logger.Info("Removing image to free bytes", "imageID", image.id, "size", image.size, "runtimeHandler", image.runtimeHandlerUsedToPullImage)
err = im.runtime.RemoveImage(ctx, container.ImageSpec{Image: image.id, RuntimeHandler: image.runtimeHandlerUsedToPullImage})
if err != nil {
return err
@@ -547,17 +555,18 @@ func (im *realImageGCManager) imagesInEvictionOrder(ctx context.Context, freeTim
im.imageRecordsLock.Lock()
defer im.imageRecordsLock.Unlock()
logger := klog.FromContext(ctx)
// Get all images in eviction order.
images := make([]evictionInfo, 0, len(im.imageRecords))
for image, record := range im.imageRecords {
if isImageUsed(image, imagesInUse) {
klog.V(5).InfoS("Image ID is being used", "imageID", image)
logger.V(5).Info("Image ID is being used", "imageID", image)
continue
}
// Check if image is pinned, prevent garbage collection
if record.pinned {
klog.V(5).InfoS("Image is pinned, skipping garbage collection", "imageID", image)
logger.V(5).Info("Image is pinned, skipping garbage collection", "imageID", image)
continue
}

View File

@@ -37,6 +37,7 @@ import (
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
stats "k8s.io/kubernetes/pkg/kubelet/server/stats"
statstest "k8s.io/kubernetes/pkg/kubelet/server/stats/testing"
"k8s.io/kubernetes/test/utils/ktesting"
testingclock "k8s.io/utils/clock/testing"
"k8s.io/utils/ptr"
)
@@ -125,7 +126,7 @@ func makeContainer(id int) *container.Container {
}
func TestDetectImagesInitialDetect(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -173,7 +174,7 @@ func TestDetectImagesInitialDetect(t *testing.T) {
func TestDetectImagesInitialDetectWithRuntimeHandlerInImageCriAPIFeatureGate(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RuntimeClassInImageCriAPI, true)
testRuntimeHandler := "test-runtimeHandler"
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -227,7 +228,7 @@ func TestDetectImagesInitialDetectWithRuntimeHandlerInImageCriAPIFeatureGate(t *
}
func TestDetectImagesWithNewImage(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
// Just one image initially.
@@ -279,7 +280,7 @@ func TestDetectImagesWithNewImage(t *testing.T) {
}
func TestDeleteUnusedImagesExemptSandboxImage(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -298,7 +299,7 @@ func TestDeleteUnusedImagesExemptSandboxImage(t *testing.T) {
}
func TestDeletePinnedImage(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -321,7 +322,7 @@ func TestDeletePinnedImage(t *testing.T) {
}
func TestDoNotDeletePinnedImage(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -342,7 +343,7 @@ func TestDoNotDeletePinnedImage(t *testing.T) {
}
func TestDeleteUnPinnedImage(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -363,7 +364,7 @@ func TestDeleteUnPinnedImage(t *testing.T) {
}
func TestAllPinnedImages(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -385,7 +386,7 @@ func TestAllPinnedImages(t *testing.T) {
}
func TestDetectImagesContainerStopped(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -424,7 +425,7 @@ func TestDetectImagesContainerStopped(t *testing.T) {
}
func TestDetectImagesWithRemovedImages(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -453,7 +454,7 @@ func TestDetectImagesWithRemovedImages(t *testing.T) {
}
func TestFreeSpaceImagesInUseContainersAreIgnored(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -474,7 +475,7 @@ func TestFreeSpaceImagesInUseContainersAreIgnored(t *testing.T) {
}
func TestDeleteUnusedImagesRemoveAllUnusedImages(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -498,7 +499,7 @@ func TestDeleteUnusedImagesRemoveAllUnusedImages(t *testing.T) {
}
func TestDeleteUnusedImagesLimitByImageLiveTime(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{
@@ -517,7 +518,7 @@ func TestDeleteUnusedImagesLimitByImageLiveTime(t *testing.T) {
}},
}
// start to detect images
manager.Start()
manager.Start(ctx)
// try to delete images, but images are not old enough,so no image will be deleted
err := manager.DeleteUnusedImages(ctx)
assert := assert.New(t)
@@ -531,7 +532,7 @@ func TestDeleteUnusedImagesLimitByImageLiveTime(t *testing.T) {
}
func TestFreeSpaceRemoveByLeastRecentlyUsed(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -583,7 +584,7 @@ func TestFreeSpaceRemoveByLeastRecentlyUsed(t *testing.T) {
}
func TestFreeSpaceTiesBrokenByDetectedTime(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
mockStatsProvider := statstest.NewMockProvider(t)
manager, fakeRuntime := newRealImageGCManager(ImageGCPolicy{}, mockStatsProvider)
@@ -617,7 +618,7 @@ func TestFreeSpaceTiesBrokenByDetectedTime(t *testing.T) {
}
func TestGarbageCollectBelowLowThreshold(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
policy := ImageGCPolicy{
HighThresholdPercent: 90,
LowThresholdPercent: 80,
@@ -636,7 +637,7 @@ func TestGarbageCollectBelowLowThreshold(t *testing.T) {
}
func TestGarbageCollectCadvisorFailure(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
policy := ImageGCPolicy{
HighThresholdPercent: 90,
LowThresholdPercent: 80,
@@ -649,7 +650,7 @@ func TestGarbageCollectCadvisorFailure(t *testing.T) {
}
func TestGarbageCollectBelowSuccess(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
policy := ImageGCPolicy{
HighThresholdPercent: 90,
LowThresholdPercent: 80,
@@ -672,7 +673,7 @@ func TestGarbageCollectBelowSuccess(t *testing.T) {
}
func TestGarbageCollectNotEnoughFreed(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
policy := ImageGCPolicy{
HighThresholdPercent: 90,
LowThresholdPercent: 80,
@@ -720,7 +721,7 @@ func TestGarbageCollectNotEnoughFreed(t *testing.T) {
}
func TestGarbageCollectImageNotOldEnough(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
policy := ImageGCPolicy{
HighThresholdPercent: 90,
LowThresholdPercent: 80,
@@ -773,7 +774,7 @@ func getImagesAndFreeSpace(ctx context.Context, t *testing.T, assert *assert.Ass
}
func TestGarbageCollectImageTooOld(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
policy := ImageGCPolicy{
HighThresholdPercent: 90,
LowThresholdPercent: 80,
@@ -828,7 +829,7 @@ func TestGarbageCollectImageTooOld(t *testing.T) {
}
func TestGarbageCollectImageMaxAgeDisabled(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
policy := ImageGCPolicy{
HighThresholdPercent: 90,
LowThresholdPercent: 80,

View File

@@ -239,7 +239,7 @@ func (m *imageManager) EnsureImageExists(ctx context.Context, objRef *v1.ObjectR
}
}
pullRequired := m.imagePullManager.MustAttemptImagePull(requestedImage, imageRef, imagePullSecrets, imagePullServiceAccount)
pullRequired := m.imagePullManager.MustAttemptImagePull(ctx, requestedImage, imageRef, imagePullSecrets, imagePullServiceAccount)
if !pullRequired {
msg := fmt.Sprintf("Container image %q already present on machine and can be accessed by the pod", requestedImage)
m.logIt(objRef, v1.EventTypeNormal, events.PulledImage, logPrefix, msg, klog.Info)
@@ -268,9 +268,9 @@ func (m *imageManager) pullImage(ctx context.Context, logPrefix string, objRef *
defer func() {
if pullSucceeded {
m.imagePullManager.RecordImagePulled(image, imageRef, trackedToImagePullCreds(finalPullCredentials))
m.imagePullManager.RecordImagePulled(ctx, image, imageRef, trackedToImagePullCreds(finalPullCredentials))
} else {
m.imagePullManager.RecordImagePullFailed(image)
m.imagePullManager.RecordImagePullFailed(ctx, image)
}
}()
}

View File

@@ -694,7 +694,7 @@ type mockImagePullManager struct {
config *mockImagePullManagerConfig
}
func (m *mockImagePullManager) MustAttemptImagePull(image, _ string, podSecrets []kubeletconfiginternal.ImagePullSecret, podServiceAccount *kubeletconfiginternal.ImagePullServiceAccount) bool {
func (m *mockImagePullManager) MustAttemptImagePull(ctx context.Context, image, _ string, podSecrets []kubeletconfiginternal.ImagePullSecret, podServiceAccount *kubeletconfiginternal.ImagePullServiceAccount) bool {
if m.config == nil || m.config.allowAll {
return false
}
@@ -736,7 +736,7 @@ type mockImagePullManagerWithTracking struct {
recordedCredentials *kubeletconfiginternal.ImagePullCredentials
}
func (m *mockImagePullManagerWithTracking) MustAttemptImagePull(image, imageRef string, podSecrets []kubeletconfiginternal.ImagePullSecret, podServiceAccount *kubeletconfiginternal.ImagePullServiceAccount) bool {
func (m *mockImagePullManagerWithTracking) MustAttemptImagePull(ctx context.Context, image, imageRef string, podSecrets []kubeletconfiginternal.ImagePullSecret, podServiceAccount *kubeletconfiginternal.ImagePullServiceAccount) bool {
m.mustAttemptCalled = true
m.lastImage = image
m.lastImageRef = imageRef
@@ -751,7 +751,7 @@ func (m *mockImagePullManagerWithTracking) MustAttemptImagePull(image, imageRef
return m.mustAttemptReturn
}
func (m *mockImagePullManagerWithTracking) RecordImagePulled(image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials) {
func (m *mockImagePullManagerWithTracking) RecordImagePulled(ctx context.Context, image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials) {
m.recordedCredentials = credentials
}
@@ -834,7 +834,7 @@ func TestParallelPuller(t *testing.T) {
useSerializedEnv := false
for _, c := range cases {
t.Run(c.testName, func(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
puller, fakeClock, fakeRuntime, container, fakePodPullingTimeRecorder, _ := pullerTestEnv(t, c, useSerializedEnv, nil)
pod := &v1.Pod{
@@ -880,7 +880,7 @@ func TestSerializedPuller(t *testing.T) {
useSerializedEnv := true
for _, c := range cases {
t.Run(c.testName, func(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
puller, fakeClock, fakeRuntime, container, fakePodPullingTimeRecorder, _ := pullerTestEnv(t, c, useSerializedEnv, nil)
pod := &v1.Pod{
@@ -975,7 +975,7 @@ func TestPullAndListImageWithPodAnnotations(t *testing.T) {
useSerializedEnv := true
t.Run(c.testName, func(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
puller, fakeClock, fakeRuntime, container, fakePodPullingTimeRecorder, _ := pullerTestEnv(t, c, useSerializedEnv, nil)
fakeRuntime.CalledFunctions = nil
fakeRuntime.ImageList = []Image{}
@@ -1039,7 +1039,7 @@ func TestPullAndListImageWithRuntimeHandlerInImageCriAPIFeatureGate(t *testing.T
useSerializedEnv := true
t.Run(c.testName, func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RuntimeClassInImageCriAPI, true)
ctx := context.Background()
ctx := ktesting.Init(t)
puller, fakeClock, fakeRuntime, container, fakePodPullingTimeRecorder, _ := pullerTestEnv(t, c, useSerializedEnv, nil)
fakeRuntime.CalledFunctions = nil
fakeRuntime.ImageList = []Image{}
@@ -1071,7 +1071,7 @@ func TestPullAndListImageWithRuntimeHandlerInImageCriAPIFeatureGate(t *testing.T
}
func TestMaxParallelImagePullsLimit(t *testing.T) {
ctx := context.Background()
ctx := ktesting.Init(t)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test_pod",

View File

@@ -25,6 +25,7 @@ import (
"github.com/google/uuid"
"k8s.io/apimachinery/pkg/util/rand"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/test/utils/ktesting"
)
type namedAccessor struct {
@@ -72,7 +73,8 @@ func directRecordReadFunc(expectHit bool) benchmarkedCheckFunc {
func mustAttemptPullReadFunc(expectHit bool) benchmarkedCheckFunc {
return func(b *testing.B, pullManager PullManager, imgRef string) {
mustPull := pullManager.MustAttemptImagePull("test.repo/org/"+imgRef, imgRef, nil, nil)
tCtx := ktesting.Init(b)
mustPull := pullManager.MustAttemptImagePull(tCtx, "test.repo/org/"+imgRef, imgRef, nil, nil)
if mustPull != !expectHit {
b.Fatalf("MustAttemptImagePull() expected %t, got %t", !expectHit, mustPull)
}
@@ -93,7 +95,9 @@ func BenchmarkPullManagerWriteImagePullIntent(b *testing.B) {
func BenchmarkPullManagerWriteIfNotChanged(b *testing.B) {
benchmarkAllPullAccessorsWrite(b, func(b *testing.B, pullManager PullManager, imgRef string) {
tCtx := ktesting.Init(b)
if err := pullManager.writePulledRecordIfChanged(
tCtx,
"test.repo/org/"+imgRef,
imgRef,
&kubeletconfig.ImagePullCredentials{NodePodsAccessible: true},

View File

@@ -89,9 +89,10 @@ func (f *PullManager) RecordPullIntent(image string) error {
return nil
}
func (f *PullManager) RecordImagePulled(image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials) {
if err := f.writePulledRecordIfChanged(image, imageRef, credentials); err != nil {
klog.ErrorS(err, "failed to write image pulled record", "imageRef", imageRef)
func (f *PullManager) RecordImagePulled(ctx context.Context, image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials) {
logger := klog.FromContext(ctx)
if err := f.writePulledRecordIfChanged(ctx, image, imageRef, credentials); err != nil {
logger.Error(err, "failed to write image pulled record", "imageRef", imageRef)
return
}
@@ -100,7 +101,7 @@ func (f *PullManager) RecordImagePulled(image, imageRef string, credentials *kub
// This is done so that the successfully pulled image is still considered as pulled by the kubelet.
// The kubelet will attempt to turn the imagePullIntent into a pulled record again when
// it's restarted.
f.decrementImagePullIntent(image)
f.decrementImagePullIntent(ctx, image)
}
// writePulledRecordIfChanged writes an ImagePulledRecord into the f.pulledDir directory.
@@ -113,7 +114,8 @@ func (f *PullManager) RecordImagePulled(image, imageRef string, credentials *kub
// unknown circumstances. We should record the image as tracked but no credentials
// should be written in order to force credential verification when the image is
// accessed the next time.
func (f *PullManager) writePulledRecordIfChanged(image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials) error {
func (f *PullManager) writePulledRecordIfChanged(ctx context.Context, image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials) error {
logger := klog.FromContext(ctx)
f.pulledAccessors.Lock(imageRef)
defer f.pulledAccessors.Unlock(imageRef)
@@ -124,7 +126,7 @@ func (f *PullManager) writePulledRecordIfChanged(image, imageRef string, credent
pulledRecord, _, err := f.recordsAccessor.GetImagePulledRecord(imageRef)
if err != nil {
klog.InfoS("failed to retrieve an ImagePulledRecord", "image", image, "err", err)
logger.Info("failed to retrieve an ImagePulledRecord", "image", image, "err", err)
pulledRecord = nil
}
@@ -154,20 +156,21 @@ func (f *PullManager) writePulledRecordIfChanged(image, imageRef string, credent
return f.recordsAccessor.WriteImagePulledRecord(pulledRecord)
}
func (f *PullManager) RecordImagePullFailed(image string) {
f.decrementImagePullIntent(image)
func (f *PullManager) RecordImagePullFailed(ctx context.Context, image string) {
f.decrementImagePullIntent(ctx, image)
}
// decrementImagePullIntent decreses the number of how many times image pull
// intent for a given `image` was requested, and removes the ImagePullIntent file
// if the reference counter for the image reaches zero.
func (f *PullManager) decrementImagePullIntent(image string) {
func (f *PullManager) decrementImagePullIntent(ctx context.Context, image string) {
logger := klog.FromContext(ctx)
f.intentAccessors.Lock(image)
defer f.intentAccessors.Unlock(image)
if f.getIntentCounterForImage(image) <= 1 {
if err := f.recordsAccessor.DeleteImagePullIntent(image); err != nil {
klog.ErrorS(err, "failed to remove image pull intent", "image", image)
logger.Error(err, "failed to remove image pull intent", "image", image)
return
}
// only delete the intent counter once the file was deleted to be consistent
@@ -179,11 +182,12 @@ func (f *PullManager) decrementImagePullIntent(image string) {
f.decrementIntentCounterForImage(image)
}
func (f *PullManager) MustAttemptImagePull(image, imageRef string, podSecrets []kubeletconfiginternal.ImagePullSecret, podServiceAccount *kubeletconfiginternal.ImagePullServiceAccount) bool {
func (f *PullManager) MustAttemptImagePull(ctx context.Context, image, imageRef string, podSecrets []kubeletconfiginternal.ImagePullSecret, podServiceAccount *kubeletconfiginternal.ImagePullServiceAccount) bool {
if len(imageRef) == 0 {
return true
}
logger := klog.FromContext(ctx)
var imagePulledByKubelet bool
var pulledRecord *kubeletconfiginternal.ImagePulledRecord
@@ -224,7 +228,7 @@ func (f *PullManager) MustAttemptImagePull(image, imageRef string, podSecrets []
}()
if err != nil {
klog.ErrorS(err, "Unable to access cache records about image pulls")
logger.Error(err, "Unable to access cache records about image pulls")
return true
}
@@ -239,7 +243,7 @@ func (f *PullManager) MustAttemptImagePull(image, imageRef string, podSecrets []
sanitizedImage, err := trimImageTagDigest(image)
if err != nil {
klog.ErrorS(err, "failed to parse image name, forcing image credentials reverification", "image", sanitizedImage)
logger.Error(err, "failed to parse image name, forcing image credentials reverification", "image", sanitizedImage)
return true
}
@@ -271,8 +275,8 @@ func (f *PullManager) MustAttemptImagePull(image, imageRef string, podSecrets []
// While we're only matching at this point, we want to ensure this secret is considered valid in the future
// and so we make an additional write to the cache.
// writePulledRecord() is a noop in case the secret with the updated hash already appears in the cache.
if err := f.writePulledRecordIfChanged(image, imageRef, &kubeletconfiginternal.ImagePullCredentials{KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{podSecret}}); err != nil {
klog.ErrorS(err, "failed to write an image pulled record", "image", image, "imageRef", imageRef)
if err := f.writePulledRecordIfChanged(ctx, image, imageRef, &kubeletconfiginternal.ImagePullCredentials{KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{podSecret}}); err != nil {
logger.Error(err, "failed to write an image pulled record", "image", image, "imageRef", imageRef)
}
}
return false
@@ -283,8 +287,8 @@ func (f *PullManager) MustAttemptImagePull(image, imageRef string, podSecrets []
// While we're only matching at this point, we want to ensure the updated credentials are considered valid in the future
// and so we make an additional write to the cache.
// writePulledRecord() is a noop in case the hash got updated in the meantime.
if err := f.writePulledRecordIfChanged(image, imageRef, &kubeletconfiginternal.ImagePullCredentials{KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{podSecret}}); err != nil {
klog.ErrorS(err, "failed to write an image pulled record", "image", image, "imageRef", imageRef)
if err := f.writePulledRecordIfChanged(ctx, image, imageRef, &kubeletconfiginternal.ImagePullCredentials{KubernetesSecrets: []kubeletconfiginternal.ImagePullSecret{podSecret}}); err != nil {
logger.Error(err, "failed to write an image pulled record", "image", image, "imageRef", imageRef)
}
return false
}
@@ -300,13 +304,14 @@ func (f *PullManager) MustAttemptImagePull(image, imageRef string, podSecrets []
return true
}
func (f *PullManager) PruneUnknownRecords(imageList []string, until time.Time) {
func (f *PullManager) PruneUnknownRecords(ctx context.Context, imageList []string, until time.Time) {
f.pulledAccessors.GlobalLock()
defer f.pulledAccessors.GlobalUnlock()
logger := klog.FromContext(ctx)
pulledRecords, err := f.recordsAccessor.ListImagePulledRecords()
if err != nil {
klog.ErrorS(err, "there were errors listing ImagePulledRecords, garbage collection will proceed with incomplete records list")
logger.Error(err, "there were errors listing ImagePulledRecords, garbage collection will proceed with incomplete records list")
}
imagesInUse := sets.New(imageList...)
@@ -321,7 +326,7 @@ func (f *PullManager) PruneUnknownRecords(imageList []string, until time.Time) {
}
if err := f.recordsAccessor.DeleteImagePulledRecord(imageRecord.ImageRef); err != nil {
klog.ErrorS(err, "failed to remove an ImagePulledRecord", "imageRef", imageRecord.ImageRef)
logger.Error(err, "failed to remove an ImagePulledRecord", "imageRef", imageRecord.ImageRef)
}
}
@@ -335,9 +340,10 @@ func (f *PullManager) PruneUnknownRecords(imageList []string, until time.Time) {
// This method is not thread-safe and it should only be called upon the creation
// of the PullManager.
func (f *PullManager) initialize(ctx context.Context) {
logger := klog.FromContext(ctx)
pullIntents, err := f.recordsAccessor.ListImagePullIntents()
if err != nil {
klog.ErrorS(err, "there were errors listing ImagePullIntents, continuing with an incomplete records list")
logger.Error(err, "there were errors listing ImagePullIntents, continuing with an incomplete records list")
}
if len(pullIntents) == 0 {
@@ -346,7 +352,7 @@ func (f *PullManager) initialize(ctx context.Context) {
imageObjs, err := f.imageService.ListImages(ctx)
if err != nil {
klog.ErrorS(err, "failed to list images")
logger.Error(err, "failed to list images")
}
inFlightPulls := sets.New[string]()
@@ -362,13 +368,13 @@ func (f *PullManager) initialize(ctx context.Context) {
for _, image := range existingRecordedImages.UnsortedList() {
if err := f.writePulledRecordIfChanged(image, imageObj.ID, nil); err != nil {
klog.ErrorS(err, "failed to write an image pull record", "imageRef", imageObj.ID)
if err := f.writePulledRecordIfChanged(ctx, image, imageObj.ID, nil); err != nil {
logger.Error(err, "failed to write an image pull record", "imageRef", imageObj.ID)
continue
}
if err := f.recordsAccessor.DeleteImagePullIntent(image); err != nil {
klog.V(2).InfoS("failed to remove image pull intent file", "imageName", image, "error", err)
logger.V(2).Info("failed to remove image pull intent file", "imageName", image, "error", err)
}
}
}

View File

@@ -726,6 +726,7 @@ func TestFileBasedImagePullManager_MustAttemptImagePull(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
encoder, decoder, err := createKubeletConfigSchemeEncoderDecoder()
require.NoError(t, err)
_, tCtx := ktesting.NewTestContext(t)
testDir := t.TempDir()
pullingDir := filepath.Join(testDir, "pulling")
@@ -750,7 +751,8 @@ func TestFileBasedImagePullManager_MustAttemptImagePull(t *testing.T) {
intentCounters: &sync.Map{},
pulledAccessors: NewStripedLockSet(10),
}
if got := f.MustAttemptImagePull(tt.image, tt.imageRef, tt.podSecrets, tt.podServiceAccount); got != tt.want {
if got := f.MustAttemptImagePull(tCtx, tt.image, tt.imageRef, tt.podSecrets, tt.podServiceAccount); got != tt.want {
t.Errorf("FileBasedImagePullManager.MustAttemptImagePull() = %v, want %v", got, tt.want)
}
@@ -1110,6 +1112,7 @@ func TestFileBasedImagePullManager_RecordImagePulled(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, tCtx := ktesting.NewTestContext(t)
encoder, decoder, err := createKubeletConfigSchemeEncoderDecoder()
require.NoError(t, err)
@@ -1135,7 +1138,7 @@ func TestFileBasedImagePullManager_RecordImagePulled(t *testing.T) {
}
f.intentCounters.Store(tt.image, tt.pullsInFlight)
origIntentCounter := f.getIntentCounterForImage(tt.image)
f.RecordImagePulled(tt.image, tt.imageRef, tt.creds)
f.RecordImagePulled(tCtx, tt.image, tt.imageRef, tt.creds)
require.Equal(t, f.getIntentCounterForImage(tt.image), origIntentCounter-1, "intent counter for %s was not decremented", tt.image)
for _, fname := range tt.expectPulled {
@@ -1399,6 +1402,7 @@ func TestFileBasedImagePullManager_PruneUnknownRecords(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, tCtx := ktesting.NewTestContext(t)
encoder, decoder, err := createKubeletConfigSchemeEncoderDecoder()
require.NoError(t, err)
@@ -1420,7 +1424,7 @@ func TestFileBasedImagePullManager_PruneUnknownRecords(t *testing.T) {
recordsAccessor: fsRecordAccessor,
pulledAccessors: NewStripedLockSet(10),
}
f.PruneUnknownRecords(tt.imageList, tt.gcStartTime)
f.PruneUnknownRecords(tCtx, tt.imageList, tt.gcStartTime)
filesLeft := sets.New[string]()
err = filepath.Walk(pulledDir, func(path string, info fs.FileInfo, err error) error {

View File

@@ -17,6 +17,7 @@ limitations under the License.
package pullmanager
import (
"context"
"time"
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
@@ -44,14 +45,14 @@ type ImagePullManager interface {
// to `true`.
//
// `image` is the content of the pod's container `image` field.
RecordImagePulled(image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials)
RecordImagePulled(ctx context.Context, image, imageRef string, credentials *kubeletconfiginternal.ImagePullCredentials)
// RecordImagePullFailed should be called if an image failed to pull.
//
// Internally, it lowers its reference counter for the given image. If the
// counter reaches zero, the pull intent record for the image is removed.
//
// `image` is the content of the pod's container `image` field.
RecordImagePullFailed(image string)
RecordImagePullFailed(ctx context.Context, image string)
// MustAttemptImagePull evaluates the policy for the image specified in
// `image` and if the policy demands verification, it checks the internal
// cache to see if there's a record of pulling the image with the presented
@@ -62,14 +63,14 @@ type ImagePullManager interface {
// was found in the cache.
//
// `image` is the content of the pod's container `image` field.
MustAttemptImagePull(image, imageRef string, credentials []kubeletconfiginternal.ImagePullSecret, serviceAccount *kubeletconfiginternal.ImagePullServiceAccount) bool
MustAttemptImagePull(ctx context.Context, image, imageRef string, credentials []kubeletconfiginternal.ImagePullSecret, serviceAccount *kubeletconfiginternal.ImagePullServiceAccount) bool
// PruneUnknownRecords deletes all of the cache ImagePulledRecords for each of the images
// whose imageRef does not appear in the `imageList` iff such an record was last updated
// _before_ the `until` timestamp.
//
// This method is only expected to be called by the kubelet's image garbage collector.
// `until` is a timestamp created _before_ the `imageList` was requested from the CRI.
PruneUnknownRecords(imageList []string, until time.Time)
PruneUnknownRecords(ctx context.Context, imageList []string, until time.Time)
}
// PullRecordsAccessor allows unified access to ImagePullIntents/ImagePulledRecords

View File

@@ -17,6 +17,7 @@ limitations under the License.
package pullmanager
import (
"context"
"time"
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config"
@@ -26,11 +27,11 @@ var _ ImagePullManager = &NoopImagePullManager{}
type NoopImagePullManager struct{}
func (m *NoopImagePullManager) RecordPullIntent(_ string) error { return nil }
func (m *NoopImagePullManager) RecordImagePulled(_, _ string, _ *kubeletconfiginternal.ImagePullCredentials) {
func (m *NoopImagePullManager) RecordPullIntent(string) error { return nil }
func (m *NoopImagePullManager) RecordImagePulled(context.Context, string, string, *kubeletconfiginternal.ImagePullCredentials) {
}
func (m *NoopImagePullManager) RecordImagePullFailed(image string) {}
func (m *NoopImagePullManager) MustAttemptImagePull(_, _ string, _ []kubeletconfiginternal.ImagePullSecret, _ *kubeletconfiginternal.ImagePullServiceAccount) bool {
func (m *NoopImagePullManager) RecordImagePullFailed(context.Context, string) {}
func (m *NoopImagePullManager) MustAttemptImagePull(context.Context, string, string, []kubeletconfiginternal.ImagePullSecret, *kubeletconfiginternal.ImagePullServiceAccount) bool {
return false
}
func (m *NoopImagePullManager) PruneUnknownRecords(_ []string, _ time.Time) {}
func (m *NoopImagePullManager) PruneUnknownRecords(context.Context, []string, time.Time) {}

View File

@@ -1659,7 +1659,7 @@ func (kl *Kubelet) initializeModules(ctx context.Context) error {
}
// Start the image manager.
kl.imageManager.Start()
kl.imageManager.Start(ctx)
// Start the certificate manager if it was enabled.
if kl.serverCertificateManager != nil {