Merge pull request #28176 from ronnielai/inode-check-dep

Automatic merge from submit-queue

Declare out of disk when there is no free inodes

#21546

pkg/kubelet/disk_manager.go

@@ -46,10 +46,11 @@ type DiskSpacePolicy struct {
 }
 
 type fsInfo struct {
-	Usage     int64
+	Usage      int64
-	Capacity  int64
+	Capacity   int64
-	Available int64
+	Available  int64
-	Timestamp time.Time
+	Timestamp  time.Time
+	InodesFree int64
 }
 
 type realDiskSpaceManager struct {
@@ -78,6 +79,7 @@ func (dm *realDiskSpaceManager) getFsInfo(fsType string, f func() (cadvisorapi.F
 		fsi.Usage = int64(fs.Usage)
 		fsi.Capacity = int64(fs.Capacity)
 		fsi.Available = int64(fs.Available)
+		fsi.InodesFree = int64(fs.InodesFree)
 		dm.cachedInfo[fsType] = fsi
 	}
 	return fsi, nil
@@ -102,11 +104,14 @@ func (dm *realDiskSpaceManager) isSpaceAvailable(fsType string, threshold int, f
 	if fsInfo.Available < 0 {
 		return true, fmt.Errorf("wrong available space for %q: %+v", fsType, fsInfo)
 	}
-
 	if fsInfo.Available < int64(threshold)*mb {
 		glog.Infof("Running out of space on disk for %q: available %d MB, threshold %d MB", fsType, fsInfo.Available/mb, threshold)
 		return false, nil
 	}
+	if fsInfo.InodesFree == 0 {
+		glog.Infof("Running out of inodes for %q", fsType)
+		return false, nil
+	}
 	return true, nil
 }
 

pkg/kubelet/disk_manager_test.go

@@ -62,13 +62,15 @@ func TestSpaceAvailable(t *testing.T) {
 	assert.NoError(err)
 
 	mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     400 * mb,
+		Usage:      400 * mb,
-		Capacity:  1000 * mb,
+		Capacity:   1000 * mb,
-		Available: 600 * mb,
+		Available:  600 * mb,
+		InodesFree: 5,
 	}, nil)
 	mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:    9 * mb,
+		Usage:      9 * mb,
-		Capacity: 10 * mb,
+		Capacity:   10 * mb,
+		InodesFree: 5,
 	}, nil)
 
 	ok, err := dm.IsRuntimeDiskSpaceAvailable()
@@ -81,7 +83,7 @@ func TestSpaceAvailable(t *testing.T) {
 }
 
 // TestIsRuntimeDiskSpaceAvailableWithSpace verifies IsRuntimeDiskSpaceAvailable results when
-// space is available.
+// space and inodes are available.
 func TestIsRuntimeDiskSpaceAvailableWithSpace(t *testing.T) {
 	assert, policy, mockCadvisor := setUp(t)
 	dm, err := newDiskSpaceManager(mockCadvisor, policy)
@@ -89,9 +91,10 @@ func TestIsRuntimeDiskSpaceAvailableWithSpace(t *testing.T) {
 
 	// 500MB available
 	mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     9500 * mb,
+		Usage:      9500 * mb,
-		Capacity:  10000 * mb,
+		Capacity:   10000 * mb,
-		Available: 500 * mb,
+		Available:  500 * mb,
+		InodesFree: 10,
 	}, nil)
 
 	ok, err := dm.IsRuntimeDiskSpaceAvailable()
@@ -105,9 +108,30 @@ func TestIsRuntimeDiskSpaceAvailableWithoutSpace(t *testing.T) {
 	// 1MB available
 	assert, policy, mockCadvisor := setUp(t)
 	mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     999 * mb,
+		Usage:      999 * mb,
-		Capacity:  1000 * mb,
+		Capacity:   1000 * mb,
-		Available: 1 * mb,
+		Available:  1 * mb,
+		InodesFree: 10,
+	}, nil)
+
+	dm, err := newDiskSpaceManager(mockCadvisor, policy)
+	require.NoError(t, err)
+
+	ok, err := dm.IsRuntimeDiskSpaceAvailable()
+	assert.NoError(err)
+	assert.False(ok)
+}
+
+// TestIsRuntimeDiskSpaceAvailableWithoutInode verifies IsRuntimeDiskSpaceAvailable results when
+// inode is not available.
+func TestIsRuntimeDiskSpaceAvailableWithoutInode(t *testing.T) {
+	// 1MB available
+	assert, policy, mockCadvisor := setUp(t)
+	mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
+		Usage:      999 * mb,
+		Capacity:   1000 * mb,
+		Available:  500 * mb,
+		InodesFree: 0,
 	}, nil)
 
 	dm, err := newDiskSpaceManager(mockCadvisor, policy)
@@ -119,7 +143,7 @@ func TestIsRuntimeDiskSpaceAvailableWithoutSpace(t *testing.T) {
 }
 
 // TestIsRootDiskSpaceAvailableWithSpace verifies IsRootDiskSpaceAvailable results when
-// space is available.
+// space and inodes are available.
 func TestIsRootDiskSpaceAvailableWithSpace(t *testing.T) {
 	assert, policy, mockCadvisor := setUp(t)
 	policy.RootFreeDiskMB = 10
@@ -128,9 +152,10 @@ func TestIsRootDiskSpaceAvailableWithSpace(t *testing.T) {
 
 	// 999MB available
 	mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     1 * mb,
+		Usage:      1 * mb,
-		Capacity:  1000 * mb,
+		Capacity:   1000 * mb,
-		Available: 999 * mb,
+		Available:  999 * mb,
+		InodesFree: 10,
 	}, nil)
 
 	ok, err := dm.IsRootDiskSpaceAvailable()
@@ -148,9 +173,31 @@ func TestIsRootDiskSpaceAvailableWithoutSpace(t *testing.T) {
 
 	// 9MB available
 	mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     990 * mb,
+		Usage:      990 * mb,
-		Capacity:  1000 * mb,
+		Capacity:   1000 * mb,
-		Available: 9 * mb,
+		Available:  9 * mb,
+		InodesFree: 10,
+	}, nil)
+
+	ok, err := dm.IsRootDiskSpaceAvailable()
+	assert.NoError(err)
+	assert.False(ok)
+}
+
+// TestIsRootDiskSpaceAvailableWithoutInode verifies IsRootDiskSpaceAvailable results when
+// inode is not available.
+func TestIsRootDiskSpaceAvailableWithoutInode(t *testing.T) {
+	assert, policy, mockCadvisor := setUp(t)
+	policy.RootFreeDiskMB = 10
+	dm, err := newDiskSpaceManager(mockCadvisor, policy)
+	assert.NoError(err)
+
+	// 9MB available
+	mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
+		Usage:      990 * mb,
+		Capacity:   1000 * mb,
+		Available:  500 * mb,
+		InodesFree: 0,
 	}, nil)
 
 	ok, err := dm.IsRootDiskSpaceAvailable()
@@ -165,14 +212,16 @@ func TestCache(t *testing.T) {
 	assert.NoError(err)
 
 	mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     400 * mb,
+		Usage:      400 * mb,
-		Capacity:  1000 * mb,
+		Capacity:   1000 * mb,
-		Available: 300 * mb,
+		Available:  300 * mb,
+		InodesFree: 10,
 	}, nil).Once()
 	mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     500 * mb,
+		Usage:      500 * mb,
-		Capacity:  1000 * mb,
+		Capacity:   1000 * mb,
-		Available: 500 * mb,
+		Available:  500 * mb,
+		InodesFree: 5,
 	}, nil).Once()
 
 	// Initial calls which should be recorded in mockCadvisor
@@ -224,9 +273,10 @@ func Test_getFsInfo(t *testing.T) {
 
 	// Sunny day case
 	mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     10 * mb,
+		Usage:      10 * mb,
-		Capacity:  100 * mb,
+		Capacity:   100 * mb,
-		Available: 90 * mb,
+		Available:  90 * mb,
+		InodesFree: 5,
 	}, nil).Once()
 
 	dm := &realDiskSpaceManager{
@@ -242,9 +292,10 @@ func Test_getFsInfo(t *testing.T) {
 	// Threshold case
 	mockCadvisor = new(cadvisortest.Mock)
 	mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     9 * mb,
+		Usage:      9 * mb,
-		Capacity:  100 * mb,
+		Capacity:   100 * mb,
-		Available: 9 * mb,
+		Available:  9 * mb,
+		InodesFree: 5,
 	}, nil).Once()
 
 	dm = &realDiskSpaceManager{
@@ -258,9 +309,10 @@ func Test_getFsInfo(t *testing.T) {
 
 	// Frozen case
 	mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     9 * mb,
+		Usage:      9 * mb,
-		Capacity:  10 * mb,
+		Capacity:   10 * mb,
-		Available: 500 * mb,
+		Available:  500 * mb,
+		InodesFree: 5,
 	}, nil).Once()
 
 	dm = &realDiskSpaceManager{
@@ -275,9 +327,10 @@ func Test_getFsInfo(t *testing.T) {
 	// Capacity error case
 	mockCadvisor = new(cadvisortest.Mock)
 	mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
-		Usage:     9 * mb,
+		Usage:      9 * mb,
-		Capacity:  0,
+		Capacity:   0,
-		Available: 500 * mb,
+		Available:  500 * mb,
+		InodesFree: 5,
 	}, nil).Once()
 
 	dm = &realDiskSpaceManager{

pkg/kubelet/kubelet_test.go

@@ -2695,8 +2695,8 @@ func TestValidateContainerLogStatus(t *testing.T) {
 // sufficient disk space or it is out of disk, depending on the capacity, availability and
 // threshold values.
 func updateDiskSpacePolicy(kubelet *Kubelet, mockCadvisor *cadvisortest.Mock, rootCap, dockerCap, rootAvail, dockerAvail uint64, rootThreshold, dockerThreshold int) error {
-	dockerimagesFsInfo := cadvisorapiv2.FsInfo{Capacity: rootCap * mb, Available: rootAvail * mb}
+	dockerimagesFsInfo := cadvisorapiv2.FsInfo{Capacity: rootCap * mb, Available: rootAvail * mb, InodesFree: 5}
-	rootFsInfo := cadvisorapiv2.FsInfo{Capacity: dockerCap * mb, Available: dockerAvail * mb}
+	rootFsInfo := cadvisorapiv2.FsInfo{Capacity: dockerCap * mb, Available: dockerAvail * mb, InodesFree: 5}
 	mockCadvisor.On("ImagesFsInfo").Return(dockerimagesFsInfo, nil)
 	mockCadvisor.On("RootFsInfo").Return(rootFsInfo, nil)