From d25ae52d9b4c9cb38745d153f4e9b74b25f659e0 Mon Sep 17 00:00:00 2001 From: feisky Date: Sat, 26 Sep 2015 08:29:08 +0800 Subject: [PATCH] Refactor image manager for client/server implementation of the container runtime --- pkg/kubelet/image_manager.go | 33 +++--- pkg/kubelet/image_manager_test.go | 185 +++++++++++++++++------------- pkg/kubelet/kubelet.go | 13 ++- 3 files changed, 131 insertions(+), 100 deletions(-) diff --git a/pkg/kubelet/image_manager.go b/pkg/kubelet/image_manager.go index a198644f347..1f40e286881 100644 --- a/pkg/kubelet/image_manager.go +++ b/pkg/kubelet/image_manager.go @@ -22,12 +22,11 @@ import ( "sync" "time" - docker "github.com/fsouza/go-dockerclient" "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/client/record" "k8s.io/kubernetes/pkg/kubelet/cadvisor" - "k8s.io/kubernetes/pkg/kubelet/dockertools" + "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/util" "k8s.io/kubernetes/pkg/util/sets" ) @@ -59,8 +58,8 @@ type ImageGCPolicy struct { } type realImageManager struct { - // Connection to the Docker daemon. - dockerClient dockertools.DockerInterface + // Container runtime + runtime container.Runtime // Records of images and their use. imageRecords map[string]*imageRecord @@ -91,7 +90,7 @@ type imageRecord struct { size int64 } -func newImageManager(dockerClient dockertools.DockerInterface, cadvisorInterface cadvisor.Interface, recorder record.EventRecorder, nodeRef *api.ObjectReference, policy ImageGCPolicy) (imageManager, error) { +func newImageManager(runtime container.Runtime, cadvisorInterface cadvisor.Interface, recorder record.EventRecorder, nodeRef *api.ObjectReference, policy ImageGCPolicy) (imageManager, error) { // Validate policy. if policy.HighThresholdPercent < 0 || policy.HighThresholdPercent > 100 { return nil, fmt.Errorf("invalid HighThresholdPercent %d, must be in range [0-100]", policy.HighThresholdPercent) @@ -100,7 +99,7 @@ func newImageManager(dockerClient dockertools.DockerInterface, cadvisorInterface return nil, fmt.Errorf("invalid LowThresholdPercent %d, must be in range [0-100]", policy.LowThresholdPercent) } im := &realImageManager{ - dockerClient: dockerClient, + runtime: runtime, policy: policy, imageRecords: make(map[string]*imageRecord), cadvisor: cadvisorInterface, @@ -130,21 +129,21 @@ func (im *realImageManager) Start() error { } func (im *realImageManager) detectImages(detected time.Time) error { - images, err := im.dockerClient.ListImages(docker.ListImagesOptions{}) + images, err := im.runtime.ListImages() if err != nil { return err } - containers, err := im.dockerClient.ListContainers(docker.ListContainersOptions{ - All: true, - }) + pods, err := im.runtime.GetPods(true) if err != nil { return err } // Make a set of images in use by containers. imagesInUse := sets.NewString() - for _, container := range containers { - imagesInUse.Insert(container.Image) + for _, pod := range pods { + for _, container := range pod.Containers { + imagesInUse.Insert(container.Image) + } } // Add new images and record those being used. @@ -163,11 +162,11 @@ func (im *realImageManager) detectImages(detected time.Time) error { } // Set last used time to now if the image is being used. - if isImageUsed(&image, imagesInUse) { + if isImageUsed(image, imagesInUse) { im.imageRecords[image.ID].lastUsed = now } - im.imageRecords[image.ID].size = image.VirtualSize + im.imageRecords[image.ID].size = image.Size } // Remove old images from our records. @@ -253,7 +252,7 @@ func (im *realImageManager) freeSpace(bytesToFree int64) (int64, error) { // Remove image. Continue despite errors. glog.Infof("[ImageManager]: Removing image %q to free %d bytes", image.id, image.size) - err := im.dockerClient.RemoveImage(image.id) + err := im.runtime.RemoveImage(container.ImageSpec{Image: image.id}) if err != nil { lastErr = err continue @@ -287,12 +286,12 @@ func (ev byLastUsedAndDetected) Less(i, j int) bool { } } -func isImageUsed(image *docker.APIImages, imagesInUse sets.String) bool { +func isImageUsed(image container.Image, imagesInUse sets.String) bool { // Check the image ID and all the RepoTags. if _, ok := imagesInUse[image.ID]; ok { return true } - for _, tag := range image.RepoTags { + for _, tag := range image.Tags { if _, ok := imagesInUse[tag]; ok { return true } diff --git a/pkg/kubelet/image_manager_test.go b/pkg/kubelet/image_manager_test.go index 2cff1944c70..f32a10505e5 100644 --- a/pkg/kubelet/image_manager_test.go +++ b/pkg/kubelet/image_manager_test.go @@ -21,30 +21,27 @@ import ( "testing" "time" - docker "github.com/fsouza/go-dockerclient" cadvisorApiV2 "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/kubernetes/pkg/client/record" "k8s.io/kubernetes/pkg/kubelet/cadvisor" - "k8s.io/kubernetes/pkg/kubelet/dockertools" - "k8s.io/kubernetes/pkg/util/sets" + "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/types" ) var zero time.Time -func newRealImageManager(policy ImageGCPolicy) (*realImageManager, *dockertools.FakeDockerClient, *cadvisor.Mock) { - fakeDocker := &dockertools.FakeDockerClient{ - RemovedImages: sets.NewString(), - } +func newRealImageManager(policy ImageGCPolicy) (*realImageManager, *container.FakeRuntime, *cadvisor.Mock) { + fakeRuntime := &container.FakeRuntime{} mockCadvisor := new(cadvisor.Mock) return &realImageManager{ - dockerClient: fakeDocker, + runtime: fakeRuntime, policy: policy, imageRecords: make(map[string]*imageRecord), cadvisor: mockCadvisor, recorder: &record.FakeRecorder{}, - }, fakeDocker, mockCadvisor + }, fakeRuntime, mockCadvisor } // Accessors used for thread-safe testing. @@ -67,29 +64,33 @@ func imageName(id int) string { } // Make an image with the specified ID. -func makeImage(id int, size int64) docker.APIImages { - return docker.APIImages{ - ID: imageName(id), - VirtualSize: size, +func makeImage(id int, size int64) container.Image { + return container.Image{ + ID: imageName(id), + Size: size, } } // Make a container with the specified ID. It will use the image with the same ID. -func makeContainer(id int) docker.APIContainers { - return docker.APIContainers{ - ID: fmt.Sprintf("container-%d", id), +func makeContainer(id int) *container.Container { + return &container.Container{ + ID: types.UID(fmt.Sprintf("container-%d", id)), Image: imageName(id), } } func TestDetectImagesInitialDetect(t *testing.T) { - manager, fakeDocker, _ := newRealImageManager(ImageGCPolicy{}) - fakeDocker.Images = []docker.APIImages{ + manager, fakeRuntime, _ := newRealImageManager(ImageGCPolicy{}) + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), makeImage(1, 2048), } - fakeDocker.ContainerList = []docker.APIContainers{ - makeContainer(1), + fakeRuntime.PodList = []*container.Pod{ + { + Containers: []*container.Container{ + makeContainer(1), + }, + }, } startTime := time.Now().Add(-time.Millisecond) @@ -109,13 +110,17 @@ func TestDetectImagesInitialDetect(t *testing.T) { func TestDetectImagesWithNewImage(t *testing.T) { // Just one image initially. - manager, fakeDocker, _ := newRealImageManager(ImageGCPolicy{}) - fakeDocker.Images = []docker.APIImages{ + manager, fakeRuntime, _ := newRealImageManager(ImageGCPolicy{}) + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), makeImage(1, 2048), } - fakeDocker.ContainerList = []docker.APIContainers{ - makeContainer(1), + fakeRuntime.PodList = []*container.Pod{ + { + Containers: []*container.Container{ + makeContainer(1), + }, + }, } err := manager.detectImages(zero) @@ -124,7 +129,7 @@ func TestDetectImagesWithNewImage(t *testing.T) { assert.Equal(manager.imageRecordsLen(), 2) // Add a new image. - fakeDocker.Images = []docker.APIImages{ + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), makeImage(1, 1024), makeImage(2, 1024), @@ -150,13 +155,17 @@ func TestDetectImagesWithNewImage(t *testing.T) { } func TestDetectImagesContainerStopped(t *testing.T) { - manager, fakeDocker, _ := newRealImageManager(ImageGCPolicy{}) - fakeDocker.Images = []docker.APIImages{ + manager, fakeRuntime, _ := newRealImageManager(ImageGCPolicy{}) + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), makeImage(1, 2048), } - fakeDocker.ContainerList = []docker.APIContainers{ - makeContainer(1), + fakeRuntime.PodList = []*container.Pod{ + { + Containers: []*container.Container{ + makeContainer(1), + }, + }, } err := manager.detectImages(zero) @@ -167,7 +176,7 @@ func TestDetectImagesContainerStopped(t *testing.T) { require.True(t, ok) // Simulate container being stopped. - fakeDocker.ContainerList = []docker.APIContainers{} + fakeRuntime.PodList = []*container.Pod{} err = manager.detectImages(time.Now()) require.NoError(t, err) assert.Equal(manager.imageRecordsLen(), 2) @@ -182,13 +191,17 @@ func TestDetectImagesContainerStopped(t *testing.T) { } func TestDetectImagesWithRemovedImages(t *testing.T) { - manager, fakeDocker, _ := newRealImageManager(ImageGCPolicy{}) - fakeDocker.Images = []docker.APIImages{ + manager, fakeRuntime, _ := newRealImageManager(ImageGCPolicy{}) + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), makeImage(1, 2048), } - fakeDocker.ContainerList = []docker.APIContainers{ - makeContainer(1), + fakeRuntime.PodList = []*container.Pod{ + { + Containers: []*container.Container{ + makeContainer(1), + }, + }, } err := manager.detectImages(zero) @@ -197,48 +210,63 @@ func TestDetectImagesWithRemovedImages(t *testing.T) { assert.Equal(manager.imageRecordsLen(), 2) // Simulate both images being removed. - fakeDocker.Images = []docker.APIImages{} + fakeRuntime.ImageList = []container.Image{} err = manager.detectImages(time.Now()) require.NoError(t, err) assert.Equal(manager.imageRecordsLen(), 0) } func TestFreeSpaceImagesInUseContainersAreIgnored(t *testing.T) { - manager, fakeDocker, _ := newRealImageManager(ImageGCPolicy{}) - fakeDocker.Images = []docker.APIImages{ + manager, fakeRuntime, _ := newRealImageManager(ImageGCPolicy{}) + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), makeImage(1, 2048), } - fakeDocker.ContainerList = []docker.APIContainers{ - makeContainer(1), + fakeRuntime.PodList = []*container.Pod{ + { + Containers: []*container.Container{ + makeContainer(1), + }, + }, } spaceFreed, err := manager.freeSpace(2048) assert := assert.New(t) require.NoError(t, err) assert.EqualValues(1024, spaceFreed) - assert.Len(fakeDocker.RemovedImages, 1) - assert.True(fakeDocker.RemovedImages.Has(imageName(0))) + assert.Len(fakeRuntime.ImageList, 1) } func TestFreeSpaceRemoveByLeastRecentlyUsed(t *testing.T) { - manager, fakeDocker, _ := newRealImageManager(ImageGCPolicy{}) - fakeDocker.Images = []docker.APIImages{ + manager, fakeRuntime, _ := newRealImageManager(ImageGCPolicy{}) + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), makeImage(1, 2048), } - fakeDocker.ContainerList = []docker.APIContainers{ - makeContainer(0), - makeContainer(1), + fakeRuntime.PodList = []*container.Pod{ + { + Containers: []*container.Container{ + makeContainer(0), + makeContainer(1), + }, + }, } // Make 1 be more recently used than 0. require.NoError(t, manager.detectImages(zero)) - fakeDocker.ContainerList = []docker.APIContainers{ - makeContainer(1), + fakeRuntime.PodList = []*container.Pod{ + { + Containers: []*container.Container{ + makeContainer(1), + }, + }, } require.NoError(t, manager.detectImages(time.Now())) - fakeDocker.ContainerList = []docker.APIContainers{} + fakeRuntime.PodList = []*container.Pod{ + { + Containers: []*container.Container{}, + }, + } require.NoError(t, manager.detectImages(time.Now())) require.Equal(t, manager.imageRecordsLen(), 2) @@ -246,56 +274,58 @@ func TestFreeSpaceRemoveByLeastRecentlyUsed(t *testing.T) { assert := assert.New(t) require.NoError(t, err) assert.EqualValues(1024, spaceFreed) - assert.Len(fakeDocker.RemovedImages, 1) - assert.True(fakeDocker.RemovedImages.Has(imageName(0))) + assert.Len(fakeRuntime.ImageList, 1) } func TestFreeSpaceTiesBrokenByDetectedTime(t *testing.T) { - manager, fakeDocker, _ := newRealImageManager(ImageGCPolicy{}) - fakeDocker.Images = []docker.APIImages{ + manager, fakeRuntime, _ := newRealImageManager(ImageGCPolicy{}) + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), } - fakeDocker.ContainerList = []docker.APIContainers{ - makeContainer(0), + fakeRuntime.PodList = []*container.Pod{ + { + Containers: []*container.Container{ + makeContainer(0), + }, + }, } // Make 1 more recently detected but used at the same time as 0. require.NoError(t, manager.detectImages(zero)) - fakeDocker.Images = []docker.APIImages{ + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), makeImage(1, 2048), } - fakeDocker.ContainerList = []docker.APIContainers{ - makeContainer(0), - makeContainer(1), - } require.NoError(t, manager.detectImages(time.Now())) - fakeDocker.ContainerList = []docker.APIContainers{} + fakeRuntime.PodList = []*container.Pod{} require.NoError(t, manager.detectImages(time.Now())) require.Equal(t, manager.imageRecordsLen(), 2) spaceFreed, err := manager.freeSpace(1024) assert := assert.New(t) require.NoError(t, err) - assert.EqualValues(1024, spaceFreed) - assert.Len(fakeDocker.RemovedImages, 1) - assert.True(fakeDocker.RemovedImages.Has(imageName(0))) + assert.EqualValues(2048, spaceFreed) + assert.Len(fakeRuntime.ImageList, 1) } func TestFreeSpaceImagesAlsoDoesLookupByRepoTags(t *testing.T) { - manager, fakeDocker, _ := newRealImageManager(ImageGCPolicy{}) - fakeDocker.Images = []docker.APIImages{ + manager, fakeRuntime, _ := newRealImageManager(ImageGCPolicy{}) + fakeRuntime.ImageList = []container.Image{ makeImage(0, 1024), { - ID: "5678", - RepoTags: []string{"potato", "salad"}, - VirtualSize: 2048, + ID: "5678", + Tags: []string{"potato", "salad"}, + Size: 2048, }, } - fakeDocker.ContainerList = []docker.APIContainers{ + fakeRuntime.PodList = []*container.Pod{ { - ID: "c5678", - Image: "salad", + Containers: []*container.Container{ + { + ID: "c5678", + Image: "salad", + }, + }, }, } @@ -303,8 +333,7 @@ func TestFreeSpaceImagesAlsoDoesLookupByRepoTags(t *testing.T) { assert := assert.New(t) require.NoError(t, err) assert.EqualValues(1024, spaceFreed) - assert.Len(fakeDocker.RemovedImages, 1) - assert.True(fakeDocker.RemovedImages.Has(imageName(0))) + assert.Len(fakeRuntime.ImageList, 1) } func TestGarbageCollectBelowLowThreshold(t *testing.T) { @@ -339,14 +368,14 @@ func TestGarbageCollectBelowSuccess(t *testing.T) { HighThresholdPercent: 90, LowThresholdPercent: 80, } - manager, fakeDocker, mockCadvisor := newRealImageManager(policy) + manager, fakeRuntime, mockCadvisor := newRealImageManager(policy) // Expect 95% usage and most of it gets freed. mockCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiV2.FsInfo{ Usage: 950, Capacity: 1000, }, nil) - fakeDocker.Images = []docker.APIImages{ + fakeRuntime.ImageList = []container.Image{ makeImage(0, 450), } @@ -358,14 +387,14 @@ func TestGarbageCollectNotEnoughFreed(t *testing.T) { HighThresholdPercent: 90, LowThresholdPercent: 80, } - manager, fakeDocker, mockCadvisor := newRealImageManager(policy) + manager, fakeRuntime, mockCadvisor := newRealImageManager(policy) // Expect 95% usage and little of it gets freed. mockCadvisor.On("DockerImagesFsInfo").Return(cadvisorApiV2.FsInfo{ Usage: 950, Capacity: 1000, }, nil) - fakeDocker.Images = []docker.APIImages{ + fakeRuntime.ImageList = []container.Image{ makeImage(0, 50), } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 851eb1a52e9..99016bcd30d 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -237,10 +237,7 @@ func NewMainKubelet( if err != nil { return nil, err } - imageManager, err := newImageManager(dockerClient, cadvisorInterface, recorder, nodeRef, imageGCPolicy) - if err != nil { - return nil, fmt.Errorf("failed to initialize image manager: %v", err) - } + diskSpaceManager, err := newDiskSpaceManager(cadvisorInterface, diskSpacePolicy) if err != nil { return nil, fmt.Errorf("failed to initialize disk manager: %v", err) @@ -278,7 +275,6 @@ func NewMainKubelet( recorder: recorder, cadvisor: cadvisorInterface, containerGC: containerGC, - imageManager: imageManager, diskSpaceManager: diskSpaceManager, statusManager: statusManager, volumeManager: volumeManager, @@ -360,6 +356,13 @@ func NewMainKubelet( return nil, fmt.Errorf("unsupported container runtime %q specified", containerRuntime) } + // setup imageManager + imageManager, err := newImageManager(klet.containerRuntime, cadvisorInterface, recorder, nodeRef, imageGCPolicy) + if err != nil { + return nil, fmt.Errorf("failed to initialize image manager: %v", err) + } + klet.imageManager = imageManager + // Setup container manager, can fail if the devices hierarchy is not mounted // (it is required by Docker however). containerManager, err := newContainerManager(mounter, cadvisorInterface, dockerDaemonContainer, systemContainer, resourceContainer)