diff --git a/pkg/kubelet/disk_manager.go b/pkg/kubelet/disk_manager.go index b5b0d499dd9..bbb2ea0bb8f 100644 --- a/pkg/kubelet/disk_manager.go +++ b/pkg/kubelet/disk_manager.go @@ -46,10 +46,11 @@ type DiskSpacePolicy struct { } type fsInfo struct { - Usage int64 - Capacity int64 - Available int64 - Timestamp time.Time + Usage int64 + Capacity int64 + Available int64 + Timestamp time.Time + InodesFree int64 } type realDiskSpaceManager struct { @@ -78,6 +79,7 @@ func (dm *realDiskSpaceManager) getFsInfo(fsType string, f func() (cadvisorapi.F fsi.Usage = int64(fs.Usage) fsi.Capacity = int64(fs.Capacity) fsi.Available = int64(fs.Available) + fsi.InodesFree = int64(fs.InodesFree) dm.cachedInfo[fsType] = fsi } return fsi, nil @@ -102,11 +104,14 @@ func (dm *realDiskSpaceManager) isSpaceAvailable(fsType string, threshold int, f if fsInfo.Available < 0 { return true, fmt.Errorf("wrong available space for %q: %+v", fsType, fsInfo) } - if fsInfo.Available < int64(threshold)*mb { glog.Infof("Running out of space on disk for %q: available %d MB, threshold %d MB", fsType, fsInfo.Available/mb, threshold) return false, nil } + if fsInfo.InodesFree == 0 { + glog.Infof("Running out of inodes for %q", fsType) + return false, nil + } return true, nil } diff --git a/pkg/kubelet/disk_manager_test.go b/pkg/kubelet/disk_manager_test.go index e9ff9cf7733..3e44d25b9c1 100644 --- a/pkg/kubelet/disk_manager_test.go +++ b/pkg/kubelet/disk_manager_test.go @@ -62,13 +62,15 @@ func TestSpaceAvailable(t *testing.T) { assert.NoError(err) mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 400 * mb, - Capacity: 1000 * mb, - Available: 600 * mb, + Usage: 400 * mb, + Capacity: 1000 * mb, + Available: 600 * mb, + InodesFree: 5, }, nil) mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 9 * mb, - Capacity: 10 * mb, + Usage: 9 * mb, + Capacity: 10 * mb, + InodesFree: 5, }, nil) ok, err := dm.IsRuntimeDiskSpaceAvailable() @@ -81,7 +83,7 @@ func TestSpaceAvailable(t *testing.T) { } // TestIsRuntimeDiskSpaceAvailableWithSpace verifies IsRuntimeDiskSpaceAvailable results when -// space is available. +// space and inodes are available. func TestIsRuntimeDiskSpaceAvailableWithSpace(t *testing.T) { assert, policy, mockCadvisor := setUp(t) dm, err := newDiskSpaceManager(mockCadvisor, policy) @@ -89,9 +91,10 @@ func TestIsRuntimeDiskSpaceAvailableWithSpace(t *testing.T) { // 500MB available mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 9500 * mb, - Capacity: 10000 * mb, - Available: 500 * mb, + Usage: 9500 * mb, + Capacity: 10000 * mb, + Available: 500 * mb, + InodesFree: 10, }, nil) ok, err := dm.IsRuntimeDiskSpaceAvailable() @@ -105,9 +108,30 @@ func TestIsRuntimeDiskSpaceAvailableWithoutSpace(t *testing.T) { // 1MB available assert, policy, mockCadvisor := setUp(t) mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 999 * mb, - Capacity: 1000 * mb, - Available: 1 * mb, + Usage: 999 * mb, + Capacity: 1000 * mb, + Available: 1 * mb, + InodesFree: 10, + }, nil) + + dm, err := newDiskSpaceManager(mockCadvisor, policy) + require.NoError(t, err) + + ok, err := dm.IsRuntimeDiskSpaceAvailable() + assert.NoError(err) + assert.False(ok) +} + +// TestIsRuntimeDiskSpaceAvailableWithoutInode verifies IsRuntimeDiskSpaceAvailable results when +// inode is not available. +func TestIsRuntimeDiskSpaceAvailableWithoutInode(t *testing.T) { + // 1MB available + assert, policy, mockCadvisor := setUp(t) + mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{ + Usage: 999 * mb, + Capacity: 1000 * mb, + Available: 500 * mb, + InodesFree: 0, }, nil) dm, err := newDiskSpaceManager(mockCadvisor, policy) @@ -119,7 +143,7 @@ func TestIsRuntimeDiskSpaceAvailableWithoutSpace(t *testing.T) { } // TestIsRootDiskSpaceAvailableWithSpace verifies IsRootDiskSpaceAvailable results when -// space is available. +// space and inodes are available. func TestIsRootDiskSpaceAvailableWithSpace(t *testing.T) { assert, policy, mockCadvisor := setUp(t) policy.RootFreeDiskMB = 10 @@ -128,9 +152,10 @@ func TestIsRootDiskSpaceAvailableWithSpace(t *testing.T) { // 999MB available mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 1 * mb, - Capacity: 1000 * mb, - Available: 999 * mb, + Usage: 1 * mb, + Capacity: 1000 * mb, + Available: 999 * mb, + InodesFree: 10, }, nil) ok, err := dm.IsRootDiskSpaceAvailable() @@ -148,9 +173,31 @@ func TestIsRootDiskSpaceAvailableWithoutSpace(t *testing.T) { // 9MB available mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 990 * mb, - Capacity: 1000 * mb, - Available: 9 * mb, + Usage: 990 * mb, + Capacity: 1000 * mb, + Available: 9 * mb, + InodesFree: 10, + }, nil) + + ok, err := dm.IsRootDiskSpaceAvailable() + assert.NoError(err) + assert.False(ok) +} + +// TestIsRootDiskSpaceAvailableWithoutInode verifies IsRootDiskSpaceAvailable results when +// inode is not available. +func TestIsRootDiskSpaceAvailableWithoutInode(t *testing.T) { + assert, policy, mockCadvisor := setUp(t) + policy.RootFreeDiskMB = 10 + dm, err := newDiskSpaceManager(mockCadvisor, policy) + assert.NoError(err) + + // 9MB available + mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{ + Usage: 990 * mb, + Capacity: 1000 * mb, + Available: 500 * mb, + InodesFree: 0, }, nil) ok, err := dm.IsRootDiskSpaceAvailable() @@ -165,14 +212,16 @@ func TestCache(t *testing.T) { assert.NoError(err) mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 400 * mb, - Capacity: 1000 * mb, - Available: 300 * mb, + Usage: 400 * mb, + Capacity: 1000 * mb, + Available: 300 * mb, + InodesFree: 10, }, nil).Once() mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 500 * mb, - Capacity: 1000 * mb, - Available: 500 * mb, + Usage: 500 * mb, + Capacity: 1000 * mb, + Available: 500 * mb, + InodesFree: 5, }, nil).Once() // Initial calls which should be recorded in mockCadvisor @@ -224,9 +273,10 @@ func Test_getFsInfo(t *testing.T) { // Sunny day case mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 10 * mb, - Capacity: 100 * mb, - Available: 90 * mb, + Usage: 10 * mb, + Capacity: 100 * mb, + Available: 90 * mb, + InodesFree: 5, }, nil).Once() dm := &realDiskSpaceManager{ @@ -242,9 +292,10 @@ func Test_getFsInfo(t *testing.T) { // Threshold case mockCadvisor = new(cadvisortest.Mock) mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 9 * mb, - Capacity: 100 * mb, - Available: 9 * mb, + Usage: 9 * mb, + Capacity: 100 * mb, + Available: 9 * mb, + InodesFree: 5, }, nil).Once() dm = &realDiskSpaceManager{ @@ -258,9 +309,10 @@ func Test_getFsInfo(t *testing.T) { // Frozen case mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 9 * mb, - Capacity: 10 * mb, - Available: 500 * mb, + Usage: 9 * mb, + Capacity: 10 * mb, + Available: 500 * mb, + InodesFree: 5, }, nil).Once() dm = &realDiskSpaceManager{ @@ -275,9 +327,10 @@ func Test_getFsInfo(t *testing.T) { // Capacity error case mockCadvisor = new(cadvisortest.Mock) mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{ - Usage: 9 * mb, - Capacity: 0, - Available: 500 * mb, + Usage: 9 * mb, + Capacity: 0, + Available: 500 * mb, + InodesFree: 5, }, nil).Once() dm = &realDiskSpaceManager{ diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index 8a6f9e8ecc8..bf7bad4464e 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -2695,8 +2695,8 @@ func TestValidateContainerLogStatus(t *testing.T) { // sufficient disk space or it is out of disk, depending on the capacity, availability and // threshold values. func updateDiskSpacePolicy(kubelet *Kubelet, mockCadvisor *cadvisortest.Mock, rootCap, dockerCap, rootAvail, dockerAvail uint64, rootThreshold, dockerThreshold int) error { - dockerimagesFsInfo := cadvisorapiv2.FsInfo{Capacity: rootCap * mb, Available: rootAvail * mb} - rootFsInfo := cadvisorapiv2.FsInfo{Capacity: dockerCap * mb, Available: dockerAvail * mb} + dockerimagesFsInfo := cadvisorapiv2.FsInfo{Capacity: rootCap * mb, Available: rootAvail * mb, InodesFree: 5} + rootFsInfo := cadvisorapiv2.FsInfo{Capacity: dockerCap * mb, Available: dockerAvail * mb, InodesFree: 5} mockCadvisor.On("ImagesFsInfo").Return(dockerimagesFsInfo, nil) mockCadvisor.On("RootFsInfo").Return(rootFsInfo, nil)