From 68dadcfd15137529e833ffa81fca288e95544d19 Mon Sep 17 00:00:00 2001 From: Lantao Liu Date: Tue, 30 Jan 2018 00:24:03 +0000 Subject: [PATCH] Make eviction manager work with CRI container runtime. Signed-off-by: Lantao Liu --- pkg/kubelet/cadvisor/cadvisor_linux.go | 13 ------ pkg/kubelet/cadvisor/cadvisor_unsupported.go | 4 -- pkg/kubelet/cadvisor/cadvisor_windows.go | 4 -- pkg/kubelet/cadvisor/testing/cadvisor_fake.go | 4 -- pkg/kubelet/cadvisor/testing/cadvisor_mock.go | 5 --- pkg/kubelet/cadvisor/types.go | 3 -- pkg/kubelet/kubelet.go | 2 +- pkg/kubelet/stats/cadvisor_stats_provider.go | 10 +++++ pkg/kubelet/stats/cri_stats_provider.go | 17 ++++++++ pkg/kubelet/stats/stats_provider.go | 14 +++++++ pkg/kubelet/stats/stats_provider_test.go | 40 +++++++++++++++++++ 11 files changed, 82 insertions(+), 34 deletions(-) diff --git a/pkg/kubelet/cadvisor/cadvisor_linux.go b/pkg/kubelet/cadvisor/cadvisor_linux.go index 948cb651b30..6a6ef44c5d5 100644 --- a/pkg/kubelet/cadvisor/cadvisor_linux.go +++ b/pkg/kubelet/cadvisor/cadvisor_linux.go @@ -242,16 +242,3 @@ func (cc *cadvisorClient) getFsInfo(label string) (cadvisorapiv2.FsInfo, error) func (cc *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) { return cc.WatchForEvents(request) } - -// HasDedicatedImageFs returns true if the imagefs has a dedicated device. -func (cc *cadvisorClient) HasDedicatedImageFs() (bool, error) { - imageFsInfo, err := cc.ImagesFsInfo() - if err != nil { - return false, err - } - rootFsInfo, err := cc.RootFsInfo() - if err != nil { - return false, err - } - return imageFsInfo.Device != rootFsInfo.Device, nil -} diff --git a/pkg/kubelet/cadvisor/cadvisor_unsupported.go b/pkg/kubelet/cadvisor/cadvisor_unsupported.go index f1ae9486d87..954dca2260b 100644 --- a/pkg/kubelet/cadvisor/cadvisor_unsupported.go +++ b/pkg/kubelet/cadvisor/cadvisor_unsupported.go @@ -77,10 +77,6 @@ func (cu *cadvisorUnsupported) WatchEvents(request *events.Request) (*events.Eve return nil, unsupportedErr } -func (cu *cadvisorUnsupported) HasDedicatedImageFs() (bool, error) { - return false, unsupportedErr -} - func (c *cadvisorUnsupported) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) { return cadvisorapiv2.FsInfo{}, nil } diff --git a/pkg/kubelet/cadvisor/cadvisor_windows.go b/pkg/kubelet/cadvisor/cadvisor_windows.go index 6ce1f8d1b38..6120bb762c4 100644 --- a/pkg/kubelet/cadvisor/cadvisor_windows.go +++ b/pkg/kubelet/cadvisor/cadvisor_windows.go @@ -77,10 +77,6 @@ func (cu *cadvisorClient) WatchEvents(request *events.Request) (*events.EventCha return &events.EventChannel{}, nil } -func (cu *cadvisorClient) HasDedicatedImageFs() (bool, error) { - return false, nil -} - func (c *cadvisorClient) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) { return cadvisorapiv2.FsInfo{}, nil } diff --git a/pkg/kubelet/cadvisor/testing/cadvisor_fake.go b/pkg/kubelet/cadvisor/testing/cadvisor_fake.go index 9ab24f07acb..2432f9ad2a2 100644 --- a/pkg/kubelet/cadvisor/testing/cadvisor_fake.go +++ b/pkg/kubelet/cadvisor/testing/cadvisor_fake.go @@ -76,10 +76,6 @@ func (c *Fake) WatchEvents(request *events.Request) (*events.EventChannel, error return new(events.EventChannel), nil } -func (c *Fake) HasDedicatedImageFs() (bool, error) { - return false, nil -} - func (c *Fake) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) { return cadvisorapiv2.FsInfo{}, nil } diff --git a/pkg/kubelet/cadvisor/testing/cadvisor_mock.go b/pkg/kubelet/cadvisor/testing/cadvisor_mock.go index d192c16594d..e503a8eabc5 100644 --- a/pkg/kubelet/cadvisor/testing/cadvisor_mock.go +++ b/pkg/kubelet/cadvisor/testing/cadvisor_mock.go @@ -84,11 +84,6 @@ func (c *Mock) WatchEvents(request *events.Request) (*events.EventChannel, error return args.Get(0).(*events.EventChannel), args.Error(1) } -func (c *Mock) HasDedicatedImageFs() (bool, error) { - args := c.Called() - return args.Get(0).(bool), args.Error(1) -} - func (c *Mock) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) { args := c.Called(uuid) return args.Get(0).(cadvisorapiv2.FsInfo), args.Error(1) diff --git a/pkg/kubelet/cadvisor/types.go b/pkg/kubelet/cadvisor/types.go index 4777798f2ff..1aa37506e6f 100644 --- a/pkg/kubelet/cadvisor/types.go +++ b/pkg/kubelet/cadvisor/types.go @@ -42,9 +42,6 @@ type Interface interface { // Get events streamed through passedChannel that fit the request. WatchEvents(request *events.Request) (*events.EventChannel, error) - // HasDedicatedImageFs returns true iff a dedicated image filesystem exists for storing images. - HasDedicatedImageFs() (bool, error) - // GetFsInfoByFsUUID returns the stats of the filesystem with the specified // uuid. GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index f6ae638db2f..657c5c65c29 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1321,7 +1321,7 @@ func (kl *Kubelet) initializeRuntimeDependentModules() { glog.Fatalf("Failed to start cAdvisor %v", err) } // eviction manager must start after cadvisor because it needs to know if the container runtime has a dedicated imagefs - kl.evictionManager.Start(kl.cadvisor, kl.GetActivePods, kl.podResourcesAreReclaimed, kl.containerManager, evictionMonitoringPeriod) + kl.evictionManager.Start(kl.StatsProvider, kl.GetActivePods, kl.podResourcesAreReclaimed, kl.containerManager, evictionMonitoringPeriod) } // Run starts the kubelet reacting to config updates diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index 17a10df2f6a..891a38f5915 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -220,6 +220,16 @@ func (p *cadvisorStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { }, nil } +// ImageFsDevice returns name of the device where the image filesystem locates, +// e.g. /dev/sda1. +func (p *cadvisorStatsProvider) ImageFsDevice() (string, error) { + imageFsInfo, err := p.cadvisor.ImagesFsInfo() + if err != nil { + return "", err + } + return imageFsInfo.Device, nil +} + // buildPodRef returns a PodReference that identifies the Pod managing cinfo func buildPodRef(containerLabels map[string]string) statsapi.PodReference { podName := kubetypes.GetPodName(containerLabels) diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index 152344c3c31..950368b6ea5 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -17,6 +17,7 @@ limitations under the License. package stats import ( + "errors" "fmt" "path" "sort" @@ -202,6 +203,22 @@ func (p *criStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { return nil, fmt.Errorf("imageFs information is unavailable") } +// ImageFsDevice returns name of the device where the image filesystem locates, +// e.g. /dev/sda1. +func (p *criStatsProvider) ImageFsDevice() (string, error) { + resp, err := p.imageService.ImageFsInfo() + if err != nil { + return "", err + } + for _, fs := range resp { + fsInfo := p.getFsInfo(fs.GetStorageId()) + if fsInfo != nil { + return fsInfo.Device, nil + } + } + return "", errors.New("imagefs device is not found") +} + // getFsInfo returns the information of the filesystem with the specified // storageID. If any error occurs, this function logs the error and returns // nil. diff --git a/pkg/kubelet/stats/stats_provider.go b/pkg/kubelet/stats/stats_provider.go index 235f1a46d2d..bf2302920f6 100644 --- a/pkg/kubelet/stats/stats_provider.go +++ b/pkg/kubelet/stats/stats_provider.go @@ -84,6 +84,7 @@ type StatsProvider struct { type containerStatsProvider interface { ListPodStats() ([]statsapi.PodStats, error) ImageFsStats() (*statsapi.FsStats, error) + ImageFsDevice() (string, error) } // GetCgroupStats returns the stats of the cgroup with the cgroupName. Note that @@ -167,3 +168,16 @@ func (p *StatsProvider) GetRawContainerInfo(containerName string, req *cadvisora containerInfo.Name: containerInfo, }, nil } + +// HasDedicatedImageFs returns true if a dedicated image filesystem exists for storing images. +func (p *StatsProvider) HasDedicatedImageFs() (bool, error) { + device, err := p.containerStatsProvider.ImageFsDevice() + if err != nil { + return false, err + } + rootFsInfo, err := p.cadvisor.RootFsInfo() + if err != nil { + return false, err + } + return device != rootFsInfo.Device, nil +} diff --git a/pkg/kubelet/stats/stats_provider_test.go b/pkg/kubelet/stats/stats_provider_test.go index 1ff7f6524b4..e2abed50991 100644 --- a/pkg/kubelet/stats/stats_provider_test.go +++ b/pkg/kubelet/stats/stats_provider_test.go @@ -362,6 +362,41 @@ func TestGetRawContainerInfoSubcontainers(t *testing.T) { mockCadvisor.AssertExpectations(t) } +func TestHasDedicatedImageFs(t *testing.T) { + for desc, test := range map[string]struct { + rootfsDevice string + imagefsDevice string + dedicated bool + }{ + "dedicated device for image filesystem": { + rootfsDevice: "root/device", + imagefsDevice: "image/device", + dedicated: true, + }, + "shared device for image filesystem": { + rootfsDevice: "share/device", + imagefsDevice: "share/device", + dedicated: false, + }, + } { + t.Logf("TestCase %q", desc) + var ( + mockCadvisor = new(cadvisortest.Mock) + mockPodManager = new(kubepodtest.MockManager) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + ) + mockCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{Device: test.rootfsDevice}, nil) + + provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{ + device: test.imagefsDevice, + }) + dedicated, err := provider.HasDedicatedImageFs() + assert.NoError(t, err) + assert.Equal(t, test.dedicated, dedicated) + mockCadvisor.AssertExpectations(t) + } +} + func getTerminatedContainerInfo(seed int, podName string, podNamespace string, containerName string) cadvisorapiv2.ContainerInfo { cinfo := getTestContainerInfo(seed, podName, podNamespace, containerName) cinfo.Stats[0].Memory.RSS = 0 @@ -606,6 +641,7 @@ func (o *fakeResourceAnalyzer) GetPodVolumeStats(uid types.UID) (serverstats.Pod } type fakeContainerStatsProvider struct { + device string } func (p fakeContainerStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { @@ -614,3 +650,7 @@ func (p fakeContainerStatsProvider) ListPodStats() ([]statsapi.PodStats, error) func (p fakeContainerStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { return nil, fmt.Errorf("not implemented") } + +func (p fakeContainerStatsProvider) ImageFsDevice() (string, error) { + return p.device, nil +}