mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-24 12:15:52 +00:00
Implementing ImageManager to take over image lifecycle.
All images are tracked, when they were created and when they were last used. FreeSpace() evicts these images in least recently used order, breaking ties with oldest first.
This commit is contained in:
Victor Marmol
parent
f3315b8350
commit
4f3f073f3c
222
pkg/kubelet/image_manager.go
Normal file
222
pkg/kubelet/image_manager.go
Normal file
@ -0,0 +1,222 @@
|
||||
/*
|
||||
Copyright 2015 Google Inc. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kubelet
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
|
||||
docker "github.com/fsouza/go-dockerclient"
|
||||
"github.com/golang/glog"
|
||||
)
|
||||
|
||||
// Manages lifecycle of all images.
|
||||
//
|
||||
// Class is thread-safe.
|
||||
type imageManager interface {
|
||||
// Starts the image manager.
|
||||
Start() error
|
||||
|
||||
// Tries to free bytesToFree worth of images on the disk.
|
||||
//
|
||||
// Returns the number of bytes free and an error if any occured. The number of
|
||||
// bytes freed is always returned.
|
||||
// Note that error may be nil and the number of bytes free may be less
|
||||
// than bytesToFree.
|
||||
FreeSpace(bytesToFree int64) (int64, error)
|
||||
|
||||
// TODO(vmarmol): Have this subsume pulls as well.
|
||||
}
|
||||
|
||||
type realImageManager struct {
|
||||
// Connection to the Docker daemon.
|
||||
dockerClient dockertools.DockerInterface
|
||||
|
||||
// Records of images and their use.
|
||||
imageRecords map[string]*imageRecord
|
||||
|
||||
// Lock for imageRecords.
|
||||
imageRecordsLock sync.Mutex
|
||||
}
|
||||
|
||||
// Information about the images we track.
|
||||
type imageRecord struct {
|
||||
// Time when this image was first detected.
|
||||
detected time.Time
|
||||
|
||||
// Time when we last saw this image being used.
|
||||
lastUsed time.Time
|
||||
|
||||
// Size of the image in bytes.
|
||||
size int64
|
||||
}
|
||||
|
||||
func newImageManager(dockerClient dockertools.DockerInterface) imageManager {
|
||||
return &realImageManager{
|
||||
dockerClient: dockerClient,
|
||||
imageRecords: make(map[string]*imageRecord),
|
||||
}
|
||||
}
|
||||
|
||||
func (self *realImageManager) Start() error {
|
||||
// Initial detection make detected time "unknown" in the past.
|
||||
var zero time.Time
|
||||
err := self.detectImages(zero)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
util.Forever(func() {
|
||||
err := self.detectImages(time.Now())
|
||||
if err != nil {
|
||||
glog.Warningf("[ImageManager] Failed to monitor images: %v", err)
|
||||
}
|
||||
}, 5*time.Minute)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (self *realImageManager) detectImages(detected time.Time) error {
|
||||
images, err := self.dockerClient.ListImages(docker.ListImagesOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
containers, err := self.dockerClient.ListContainers(docker.ListContainersOptions{
|
||||
All: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Make a set of images in use by containers.
|
||||
imagesInUse := util.NewStringSet()
|
||||
for _, container := range containers {
|
||||
imagesInUse.Insert(container.Image)
|
||||
}
|
||||
|
||||
// Add new images and record those being used.
|
||||
now := time.Now()
|
||||
currentImages := util.NewStringSet()
|
||||
self.imageRecordsLock.Lock()
|
||||
defer self.imageRecordsLock.Unlock()
|
||||
for _, image := range images {
|
||||
currentImages.Insert(image.ID)
|
||||
|
||||
// New image, set it as detected now.
|
||||
if _, ok := self.imageRecords[image.ID]; !ok {
|
||||
self.imageRecords[image.ID] = &imageRecord{
|
||||
detected: detected,
|
||||
}
|
||||
}
|
||||
|
||||
// Set last used time to now if the image is being used.
|
||||
if isImageUsed(&image, imagesInUse) {
|
||||
self.imageRecords[image.ID].lastUsed = now
|
||||
}
|
||||
|
||||
self.imageRecords[image.ID].size = image.VirtualSize
|
||||
}
|
||||
|
||||
// Remove old images from our records.
|
||||
for image := range self.imageRecords {
|
||||
if !currentImages.Has(image) {
|
||||
delete(self.imageRecords, image)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (self *realImageManager) FreeSpace(bytesToFree int64) (int64, error) {
|
||||
startTime := time.Now()
|
||||
err := self.detectImages(startTime)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
self.imageRecordsLock.Lock()
|
||||
defer self.imageRecordsLock.Unlock()
|
||||
|
||||
// Get all images in eviction order.
|
||||
images := make([]evictionInfo, 0, len(self.imageRecords))
|
||||
for image, record := range self.imageRecords {
|
||||
images = append(images, evictionInfo{
|
||||
id: image,
|
||||
imageRecord: *record,
|
||||
})
|
||||
}
|
||||
sort.Sort(byLastUsedAndDetected(images))
|
||||
|
||||
// Delete unused images until we've freed up enough space.
|
||||
var lastErr error
|
||||
spaceFreed := int64(0)
|
||||
for _, image := range images {
|
||||
// Images that are currently in used were given a newer lastUsed.
|
||||
if image.lastUsed.After(startTime) {
|
||||
break
|
||||
}
|
||||
|
||||
// Remove image. Continue despite errors.
|
||||
err := self.dockerClient.RemoveImage(image.id)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
delete(self.imageRecords, image.id)
|
||||
spaceFreed += image.size
|
||||
|
||||
if spaceFreed >= bytesToFree {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return spaceFreed, lastErr
|
||||
}
|
||||
|
||||
type evictionInfo struct {
|
||||
id string
|
||||
imageRecord
|
||||
}
|
||||
|
||||
type byLastUsedAndDetected []evictionInfo
|
||||
|
||||
func (self byLastUsedAndDetected) Len() int { return len(self) }
|
||||
func (self byLastUsedAndDetected) Swap(i, j int) { self[i], self[j] = self[j], self[i] }
|
||||
func (self byLastUsedAndDetected) Less(i, j int) bool {
|
||||
// Sort by last used, break ties by detected.
|
||||
if self[i].lastUsed.Equal(self[j].lastUsed) {
|
||||
return self[i].detected.Before(self[j].detected)
|
||||
} else {
|
||||
return self[i].lastUsed.Before(self[j].lastUsed)
|
||||
}
|
||||
}
|
||||
|
||||
func isImageUsed(image *docker.APIImages, imagesInUse util.StringSet) bool {
|
||||
// Check the image ID and all the RepoTags.
|
||||
if _, ok := imagesInUse[image.ID]; ok {
|
||||
return true
|
||||
}
|
||||
for _, tag := range image.RepoTags {
|
||||
if _, ok := imagesInUse[tag]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
301
pkg/kubelet/image_manager_test.go
Normal file
301
pkg/kubelet/image_manager_test.go
Normal file
@ -0,0 +1,301 @@
|
||||
/*
|
||||
Copyright 2015 Google Inc. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kubelet
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
|
||||
docker "github.com/fsouza/go-dockerclient"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var zero time.Time
|
||||
|
||||
func newRealImageManager(dockerClient dockertools.DockerInterface) *realImageManager {
|
||||
return newImageManager(dockerClient).(*realImageManager)
|
||||
}
|
||||
|
||||
// Returns the name of the image with the given ID.
|
||||
func imageName(id int) string {
|
||||
return fmt.Sprintf("image-%d", id)
|
||||
}
|
||||
|
||||
// Make an image with the specified ID.
|
||||
func makeImage(id int, size int64) docker.APIImages {
|
||||
return docker.APIImages{
|
||||
ID: imageName(id),
|
||||
VirtualSize: size,
|
||||
}
|
||||
}
|
||||
|
||||
// Make a container with the specified ID. It will use the image with the same ID.
|
||||
func makeContainer(id int) docker.APIContainers {
|
||||
return docker.APIContainers{
|
||||
ID: fmt.Sprintf("container-%d", id),
|
||||
Image: imageName(id),
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectImagesInitialDetect(t *testing.T) {
|
||||
fakeDocker := &dockertools.FakeDockerClient{
|
||||
Images: []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
makeImage(1, 2048),
|
||||
},
|
||||
ContainerList: []docker.APIContainers{
|
||||
makeContainer(1),
|
||||
},
|
||||
}
|
||||
manager := newRealImageManager(fakeDocker)
|
||||
|
||||
startTime := time.Now().Add(-time.Millisecond)
|
||||
err := manager.detectImages(zero)
|
||||
assert := assert.New(t)
|
||||
require.Nil(t, err)
|
||||
assert.Len(manager.imageRecords, 2)
|
||||
noContainer, ok := manager.imageRecords[imageName(0)]
|
||||
require.True(t, ok)
|
||||
assert.Equal(zero, noContainer.detected)
|
||||
assert.Equal(zero, noContainer.lastUsed)
|
||||
withContainer, ok := manager.imageRecords[imageName(1)]
|
||||
require.True(t, ok)
|
||||
assert.Equal(zero, withContainer.detected)
|
||||
assert.True(withContainer.lastUsed.After(startTime))
|
||||
}
|
||||
|
||||
func TestDetectImagesWithNewImage(t *testing.T) {
|
||||
// Just one image initially.
|
||||
fakeDocker := &dockertools.FakeDockerClient{
|
||||
Images: []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
makeImage(1, 2048),
|
||||
},
|
||||
ContainerList: []docker.APIContainers{
|
||||
makeContainer(1),
|
||||
},
|
||||
}
|
||||
manager := newRealImageManager(fakeDocker)
|
||||
|
||||
err := manager.detectImages(zero)
|
||||
assert := assert.New(t)
|
||||
require.Nil(t, err)
|
||||
assert.Len(manager.imageRecords, 2)
|
||||
|
||||
// Add a new image.
|
||||
fakeDocker.Images = []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
makeImage(1, 1024),
|
||||
makeImage(2, 1024),
|
||||
}
|
||||
|
||||
detectedTime := zero.Add(time.Second)
|
||||
startTime := time.Now().Add(-time.Millisecond)
|
||||
err = manager.detectImages(detectedTime)
|
||||
require.Nil(t, err)
|
||||
assert.Len(manager.imageRecords, 3)
|
||||
noContainer, ok := manager.imageRecords[imageName(0)]
|
||||
require.True(t, ok)
|
||||
assert.Equal(zero, noContainer.detected)
|
||||
assert.Equal(zero, noContainer.lastUsed)
|
||||
withContainer, ok := manager.imageRecords[imageName(1)]
|
||||
require.True(t, ok)
|
||||
assert.Equal(zero, withContainer.detected)
|
||||
assert.True(withContainer.lastUsed.After(startTime))
|
||||
newContainer, ok := manager.imageRecords[imageName(2)]
|
||||
require.True(t, ok)
|
||||
assert.Equal(detectedTime, newContainer.detected)
|
||||
assert.Equal(zero, noContainer.lastUsed)
|
||||
}
|
||||
|
||||
func TestDetectImagesContainerStopped(t *testing.T) {
|
||||
fakeDocker := &dockertools.FakeDockerClient{
|
||||
Images: []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
makeImage(1, 2048),
|
||||
},
|
||||
ContainerList: []docker.APIContainers{
|
||||
makeContainer(1),
|
||||
},
|
||||
}
|
||||
manager := newRealImageManager(fakeDocker)
|
||||
|
||||
err := manager.detectImages(zero)
|
||||
assert := assert.New(t)
|
||||
require.Nil(t, err)
|
||||
assert.Len(manager.imageRecords, 2)
|
||||
withContainer, ok := manager.imageRecords[imageName(1)]
|
||||
require.True(t, ok)
|
||||
|
||||
// Simulate container being stopped.
|
||||
fakeDocker.ContainerList = []docker.APIContainers{}
|
||||
err = manager.detectImages(time.Now())
|
||||
require.Nil(t, err)
|
||||
assert.Len(manager.imageRecords, 2)
|
||||
container1, ok := manager.imageRecords[imageName(0)]
|
||||
require.True(t, ok)
|
||||
assert.Equal(zero, container1.detected)
|
||||
assert.Equal(zero, container1.lastUsed)
|
||||
container2, ok := manager.imageRecords[imageName(1)]
|
||||
require.True(t, ok)
|
||||
assert.Equal(zero, container2.detected)
|
||||
assert.True(container2.lastUsed.Equal(withContainer.lastUsed))
|
||||
}
|
||||
|
||||
func TestDetectImagesWithRemovedImages(t *testing.T) {
|
||||
fakeDocker := &dockertools.FakeDockerClient{
|
||||
Images: []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
makeImage(1, 2048),
|
||||
},
|
||||
ContainerList: []docker.APIContainers{
|
||||
makeContainer(1),
|
||||
},
|
||||
}
|
||||
manager := newRealImageManager(fakeDocker)
|
||||
|
||||
err := manager.detectImages(zero)
|
||||
assert := assert.New(t)
|
||||
require.Nil(t, err)
|
||||
assert.Len(manager.imageRecords, 2)
|
||||
|
||||
// Simulate both images being removed.
|
||||
fakeDocker.Images = []docker.APIImages{}
|
||||
err = manager.detectImages(time.Now())
|
||||
require.Nil(t, err)
|
||||
assert.Len(manager.imageRecords, 0)
|
||||
}
|
||||
|
||||
func TestFreeSpaceImagesInUseContainersAreIgnored(t *testing.T) {
|
||||
fakeDocker := &dockertools.FakeDockerClient{
|
||||
Images: []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
makeImage(1, 2048),
|
||||
},
|
||||
ContainerList: []docker.APIContainers{
|
||||
makeContainer(1),
|
||||
},
|
||||
RemovedImages: util.NewStringSet(),
|
||||
}
|
||||
manager := newRealImageManager(fakeDocker)
|
||||
|
||||
spaceFreed, err := manager.FreeSpace(2048)
|
||||
assert := assert.New(t)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(1024, spaceFreed)
|
||||
assert.Len(fakeDocker.RemovedImages, 1)
|
||||
assert.True(fakeDocker.RemovedImages.Has(imageName(0)))
|
||||
}
|
||||
|
||||
func TestFreeSpaceRemoveByLeastRecentlyUsed(t *testing.T) {
|
||||
fakeDocker := &dockertools.FakeDockerClient{
|
||||
Images: []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
makeImage(1, 2048),
|
||||
},
|
||||
ContainerList: []docker.APIContainers{
|
||||
makeContainer(0),
|
||||
makeContainer(1),
|
||||
},
|
||||
RemovedImages: util.NewStringSet(),
|
||||
}
|
||||
manager := newRealImageManager(fakeDocker)
|
||||
|
||||
// Make 1 be more recently used than 0.
|
||||
require.Nil(t, manager.detectImages(zero))
|
||||
fakeDocker.ContainerList = []docker.APIContainers{
|
||||
makeContainer(1),
|
||||
}
|
||||
require.Nil(t, manager.detectImages(time.Now()))
|
||||
fakeDocker.ContainerList = []docker.APIContainers{}
|
||||
require.Nil(t, manager.detectImages(time.Now()))
|
||||
require.Len(t, manager.imageRecords, 2)
|
||||
|
||||
spaceFreed, err := manager.FreeSpace(1024)
|
||||
assert := assert.New(t)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(1024, spaceFreed)
|
||||
assert.Len(fakeDocker.RemovedImages, 1)
|
||||
assert.True(fakeDocker.RemovedImages.Has(imageName(0)))
|
||||
}
|
||||
|
||||
func TestFreeSpaceTiesBrokenByDetectedTime(t *testing.T) {
|
||||
fakeDocker := &dockertools.FakeDockerClient{
|
||||
Images: []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
},
|
||||
ContainerList: []docker.APIContainers{
|
||||
makeContainer(0),
|
||||
},
|
||||
RemovedImages: util.NewStringSet(),
|
||||
}
|
||||
manager := newRealImageManager(fakeDocker)
|
||||
|
||||
// Make 1 more recently detected but used at the same time as 0.
|
||||
require.Nil(t, manager.detectImages(zero))
|
||||
fakeDocker.Images = []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
makeImage(1, 2048),
|
||||
}
|
||||
fakeDocker.ContainerList = []docker.APIContainers{
|
||||
makeContainer(0),
|
||||
makeContainer(1),
|
||||
}
|
||||
require.Nil(t, manager.detectImages(time.Now()))
|
||||
fakeDocker.ContainerList = []docker.APIContainers{}
|
||||
require.Nil(t, manager.detectImages(time.Now()))
|
||||
require.Len(t, manager.imageRecords, 2)
|
||||
|
||||
spaceFreed, err := manager.FreeSpace(1024)
|
||||
assert := assert.New(t)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(1024, spaceFreed)
|
||||
assert.Len(fakeDocker.RemovedImages, 1)
|
||||
assert.True(fakeDocker.RemovedImages.Has(imageName(0)))
|
||||
}
|
||||
|
||||
func TestFreeSpaceImagesAlsoDoesLookupByRepoTags(t *testing.T) {
|
||||
fakeDocker := &dockertools.FakeDockerClient{
|
||||
Images: []docker.APIImages{
|
||||
makeImage(0, 1024),
|
||||
{
|
||||
ID: "5678",
|
||||
RepoTags: []string{"potato", "salad"},
|
||||
VirtualSize: 2048,
|
||||
},
|
||||
},
|
||||
ContainerList: []docker.APIContainers{
|
||||
{
|
||||
ID: "c5678",
|
||||
Image: "salad",
|
||||
},
|
||||
},
|
||||
RemovedImages: util.NewStringSet(),
|
||||
}
|
||||
manager := newRealImageManager(fakeDocker)
|
||||
|
||||
spaceFreed, err := manager.FreeSpace(1024)
|
||||
assert := assert.New(t)
|
||||
require.Nil(t, err)
|
||||
assert.Equal(1024, spaceFreed)
|
||||
assert.Len(fakeDocker.RemovedImages, 1)
|
||||
assert.True(fakeDocker.RemovedImages.Has(imageName(0)))
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user