diff --git a/pkg/kubelet/image_manager.go b/pkg/kubelet/image_manager.go new file mode 100644 index 00000000000..d01e1edaca6 --- /dev/null +++ b/pkg/kubelet/image_manager.go @@ -0,0 +1,222 @@ +/* +Copyright 2015 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "sort" + "sync" + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + docker "github.com/fsouza/go-dockerclient" + "github.com/golang/glog" +) + +// Manages lifecycle of all images. +// +// Class is thread-safe. +type imageManager interface { + // Starts the image manager. + Start() error + + // Tries to free bytesToFree worth of images on the disk. + // + // Returns the number of bytes free and an error if any occured. The number of + // bytes freed is always returned. + // Note that error may be nil and the number of bytes free may be less + // than bytesToFree. + FreeSpace(bytesToFree int64) (int64, error) + + // TODO(vmarmol): Have this subsume pulls as well. +} + +type realImageManager struct { + // Connection to the Docker daemon. + dockerClient dockertools.DockerInterface + + // Records of images and their use. + imageRecords map[string]*imageRecord + + // Lock for imageRecords. + imageRecordsLock sync.Mutex +} + +// Information about the images we track. +type imageRecord struct { + // Time when this image was first detected. + detected time.Time + + // Time when we last saw this image being used. + lastUsed time.Time + + // Size of the image in bytes. + size int64 +} + +func newImageManager(dockerClient dockertools.DockerInterface) imageManager { + return &realImageManager{ + dockerClient: dockerClient, + imageRecords: make(map[string]*imageRecord), + } +} + +func (self *realImageManager) Start() error { + // Initial detection make detected time "unknown" in the past. + var zero time.Time + err := self.detectImages(zero) + if err != nil { + return err + } + + util.Forever(func() { + err := self.detectImages(time.Now()) + if err != nil { + glog.Warningf("[ImageManager] Failed to monitor images: %v", err) + } + }, 5*time.Minute) + + return nil +} + +func (self *realImageManager) detectImages(detected time.Time) error { + images, err := self.dockerClient.ListImages(docker.ListImagesOptions{}) + if err != nil { + return err + } + containers, err := self.dockerClient.ListContainers(docker.ListContainersOptions{ + All: true, + }) + if err != nil { + return err + } + + // Make a set of images in use by containers. + imagesInUse := util.NewStringSet() + for _, container := range containers { + imagesInUse.Insert(container.Image) + } + + // Add new images and record those being used. + now := time.Now() + currentImages := util.NewStringSet() + self.imageRecordsLock.Lock() + defer self.imageRecordsLock.Unlock() + for _, image := range images { + currentImages.Insert(image.ID) + + // New image, set it as detected now. + if _, ok := self.imageRecords[image.ID]; !ok { + self.imageRecords[image.ID] = &imageRecord{ + detected: detected, + } + } + + // Set last used time to now if the image is being used. + if isImageUsed(&image, imagesInUse) { + self.imageRecords[image.ID].lastUsed = now + } + + self.imageRecords[image.ID].size = image.VirtualSize + } + + // Remove old images from our records. + for image := range self.imageRecords { + if !currentImages.Has(image) { + delete(self.imageRecords, image) + } + } + + return nil +} + +func (self *realImageManager) FreeSpace(bytesToFree int64) (int64, error) { + startTime := time.Now() + err := self.detectImages(startTime) + if err != nil { + return 0, err + } + + self.imageRecordsLock.Lock() + defer self.imageRecordsLock.Unlock() + + // Get all images in eviction order. + images := make([]evictionInfo, 0, len(self.imageRecords)) + for image, record := range self.imageRecords { + images = append(images, evictionInfo{ + id: image, + imageRecord: *record, + }) + } + sort.Sort(byLastUsedAndDetected(images)) + + // Delete unused images until we've freed up enough space. + var lastErr error + spaceFreed := int64(0) + for _, image := range images { + // Images that are currently in used were given a newer lastUsed. + if image.lastUsed.After(startTime) { + break + } + + // Remove image. Continue despite errors. + err := self.dockerClient.RemoveImage(image.id) + if err != nil { + lastErr = err + continue + } + delete(self.imageRecords, image.id) + spaceFreed += image.size + + if spaceFreed >= bytesToFree { + break + } + } + + return spaceFreed, lastErr +} + +type evictionInfo struct { + id string + imageRecord +} + +type byLastUsedAndDetected []evictionInfo + +func (self byLastUsedAndDetected) Len() int { return len(self) } +func (self byLastUsedAndDetected) Swap(i, j int) { self[i], self[j] = self[j], self[i] } +func (self byLastUsedAndDetected) Less(i, j int) bool { + // Sort by last used, break ties by detected. + if self[i].lastUsed.Equal(self[j].lastUsed) { + return self[i].detected.Before(self[j].detected) + } else { + return self[i].lastUsed.Before(self[j].lastUsed) + } +} + +func isImageUsed(image *docker.APIImages, imagesInUse util.StringSet) bool { + // Check the image ID and all the RepoTags. + if _, ok := imagesInUse[image.ID]; ok { + return true + } + for _, tag := range image.RepoTags { + if _, ok := imagesInUse[tag]; ok { + return true + } + } + return false +} diff --git a/pkg/kubelet/image_manager_test.go b/pkg/kubelet/image_manager_test.go new file mode 100644 index 00000000000..d71d3d91d4e --- /dev/null +++ b/pkg/kubelet/image_manager_test.go @@ -0,0 +1,301 @@ +/* +Copyright 2015 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" + "testing" + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + docker "github.com/fsouza/go-dockerclient" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var zero time.Time + +func newRealImageManager(dockerClient dockertools.DockerInterface) *realImageManager { + return newImageManager(dockerClient).(*realImageManager) +} + +// Returns the name of the image with the given ID. +func imageName(id int) string { + return fmt.Sprintf("image-%d", id) +} + +// Make an image with the specified ID. +func makeImage(id int, size int64) docker.APIImages { + return docker.APIImages{ + ID: imageName(id), + VirtualSize: size, + } +} + +// Make a container with the specified ID. It will use the image with the same ID. +func makeContainer(id int) docker.APIContainers { + return docker.APIContainers{ + ID: fmt.Sprintf("container-%d", id), + Image: imageName(id), + } +} + +func TestDetectImagesInitialDetect(t *testing.T) { + fakeDocker := &dockertools.FakeDockerClient{ + Images: []docker.APIImages{ + makeImage(0, 1024), + makeImage(1, 2048), + }, + ContainerList: []docker.APIContainers{ + makeContainer(1), + }, + } + manager := newRealImageManager(fakeDocker) + + startTime := time.Now().Add(-time.Millisecond) + err := manager.detectImages(zero) + assert := assert.New(t) + require.Nil(t, err) + assert.Len(manager.imageRecords, 2) + noContainer, ok := manager.imageRecords[imageName(0)] + require.True(t, ok) + assert.Equal(zero, noContainer.detected) + assert.Equal(zero, noContainer.lastUsed) + withContainer, ok := manager.imageRecords[imageName(1)] + require.True(t, ok) + assert.Equal(zero, withContainer.detected) + assert.True(withContainer.lastUsed.After(startTime)) +} + +func TestDetectImagesWithNewImage(t *testing.T) { + // Just one image initially. + fakeDocker := &dockertools.FakeDockerClient{ + Images: []docker.APIImages{ + makeImage(0, 1024), + makeImage(1, 2048), + }, + ContainerList: []docker.APIContainers{ + makeContainer(1), + }, + } + manager := newRealImageManager(fakeDocker) + + err := manager.detectImages(zero) + assert := assert.New(t) + require.Nil(t, err) + assert.Len(manager.imageRecords, 2) + + // Add a new image. + fakeDocker.Images = []docker.APIImages{ + makeImage(0, 1024), + makeImage(1, 1024), + makeImage(2, 1024), + } + + detectedTime := zero.Add(time.Second) + startTime := time.Now().Add(-time.Millisecond) + err = manager.detectImages(detectedTime) + require.Nil(t, err) + assert.Len(manager.imageRecords, 3) + noContainer, ok := manager.imageRecords[imageName(0)] + require.True(t, ok) + assert.Equal(zero, noContainer.detected) + assert.Equal(zero, noContainer.lastUsed) + withContainer, ok := manager.imageRecords[imageName(1)] + require.True(t, ok) + assert.Equal(zero, withContainer.detected) + assert.True(withContainer.lastUsed.After(startTime)) + newContainer, ok := manager.imageRecords[imageName(2)] + require.True(t, ok) + assert.Equal(detectedTime, newContainer.detected) + assert.Equal(zero, noContainer.lastUsed) +} + +func TestDetectImagesContainerStopped(t *testing.T) { + fakeDocker := &dockertools.FakeDockerClient{ + Images: []docker.APIImages{ + makeImage(0, 1024), + makeImage(1, 2048), + }, + ContainerList: []docker.APIContainers{ + makeContainer(1), + }, + } + manager := newRealImageManager(fakeDocker) + + err := manager.detectImages(zero) + assert := assert.New(t) + require.Nil(t, err) + assert.Len(manager.imageRecords, 2) + withContainer, ok := manager.imageRecords[imageName(1)] + require.True(t, ok) + + // Simulate container being stopped. + fakeDocker.ContainerList = []docker.APIContainers{} + err = manager.detectImages(time.Now()) + require.Nil(t, err) + assert.Len(manager.imageRecords, 2) + container1, ok := manager.imageRecords[imageName(0)] + require.True(t, ok) + assert.Equal(zero, container1.detected) + assert.Equal(zero, container1.lastUsed) + container2, ok := manager.imageRecords[imageName(1)] + require.True(t, ok) + assert.Equal(zero, container2.detected) + assert.True(container2.lastUsed.Equal(withContainer.lastUsed)) +} + +func TestDetectImagesWithRemovedImages(t *testing.T) { + fakeDocker := &dockertools.FakeDockerClient{ + Images: []docker.APIImages{ + makeImage(0, 1024), + makeImage(1, 2048), + }, + ContainerList: []docker.APIContainers{ + makeContainer(1), + }, + } + manager := newRealImageManager(fakeDocker) + + err := manager.detectImages(zero) + assert := assert.New(t) + require.Nil(t, err) + assert.Len(manager.imageRecords, 2) + + // Simulate both images being removed. + fakeDocker.Images = []docker.APIImages{} + err = manager.detectImages(time.Now()) + require.Nil(t, err) + assert.Len(manager.imageRecords, 0) +} + +func TestFreeSpaceImagesInUseContainersAreIgnored(t *testing.T) { + fakeDocker := &dockertools.FakeDockerClient{ + Images: []docker.APIImages{ + makeImage(0, 1024), + makeImage(1, 2048), + }, + ContainerList: []docker.APIContainers{ + makeContainer(1), + }, + RemovedImages: util.NewStringSet(), + } + manager := newRealImageManager(fakeDocker) + + spaceFreed, err := manager.FreeSpace(2048) + assert := assert.New(t) + require.Nil(t, err) + assert.Equal(1024, spaceFreed) + assert.Len(fakeDocker.RemovedImages, 1) + assert.True(fakeDocker.RemovedImages.Has(imageName(0))) +} + +func TestFreeSpaceRemoveByLeastRecentlyUsed(t *testing.T) { + fakeDocker := &dockertools.FakeDockerClient{ + Images: []docker.APIImages{ + makeImage(0, 1024), + makeImage(1, 2048), + }, + ContainerList: []docker.APIContainers{ + makeContainer(0), + makeContainer(1), + }, + RemovedImages: util.NewStringSet(), + } + manager := newRealImageManager(fakeDocker) + + // Make 1 be more recently used than 0. + require.Nil(t, manager.detectImages(zero)) + fakeDocker.ContainerList = []docker.APIContainers{ + makeContainer(1), + } + require.Nil(t, manager.detectImages(time.Now())) + fakeDocker.ContainerList = []docker.APIContainers{} + require.Nil(t, manager.detectImages(time.Now())) + require.Len(t, manager.imageRecords, 2) + + spaceFreed, err := manager.FreeSpace(1024) + assert := assert.New(t) + require.Nil(t, err) + assert.Equal(1024, spaceFreed) + assert.Len(fakeDocker.RemovedImages, 1) + assert.True(fakeDocker.RemovedImages.Has(imageName(0))) +} + +func TestFreeSpaceTiesBrokenByDetectedTime(t *testing.T) { + fakeDocker := &dockertools.FakeDockerClient{ + Images: []docker.APIImages{ + makeImage(0, 1024), + }, + ContainerList: []docker.APIContainers{ + makeContainer(0), + }, + RemovedImages: util.NewStringSet(), + } + manager := newRealImageManager(fakeDocker) + + // Make 1 more recently detected but used at the same time as 0. + require.Nil(t, manager.detectImages(zero)) + fakeDocker.Images = []docker.APIImages{ + makeImage(0, 1024), + makeImage(1, 2048), + } + fakeDocker.ContainerList = []docker.APIContainers{ + makeContainer(0), + makeContainer(1), + } + require.Nil(t, manager.detectImages(time.Now())) + fakeDocker.ContainerList = []docker.APIContainers{} + require.Nil(t, manager.detectImages(time.Now())) + require.Len(t, manager.imageRecords, 2) + + spaceFreed, err := manager.FreeSpace(1024) + assert := assert.New(t) + require.Nil(t, err) + assert.Equal(1024, spaceFreed) + assert.Len(fakeDocker.RemovedImages, 1) + assert.True(fakeDocker.RemovedImages.Has(imageName(0))) +} + +func TestFreeSpaceImagesAlsoDoesLookupByRepoTags(t *testing.T) { + fakeDocker := &dockertools.FakeDockerClient{ + Images: []docker.APIImages{ + makeImage(0, 1024), + { + ID: "5678", + RepoTags: []string{"potato", "salad"}, + VirtualSize: 2048, + }, + }, + ContainerList: []docker.APIContainers{ + { + ID: "c5678", + Image: "salad", + }, + }, + RemovedImages: util.NewStringSet(), + } + manager := newRealImageManager(fakeDocker) + + spaceFreed, err := manager.FreeSpace(1024) + assert := assert.New(t) + require.Nil(t, err) + assert.Equal(1024, spaceFreed) + assert.Len(fakeDocker.RemovedImages, 1) + assert.True(fakeDocker.RemovedImages.Has(imageName(0))) +}