Merge pull request #35132 from dashpole/per_volume_inode

Automatic merge from submit-queue

Per Volume Inode Accounting

Collects volume inode stats using the same find command as cadvisor.  The command is "find _path_ -xdev -printf '.' | wc -c".  The output is passed to the summary api, and will be consumed by the eviction manager.

This cannot be merged yet, as it depends on changes adding the InodesUsed field to the summary api, and the eviction manager consuming this.  Expect tests to fail until this happens.
DEPENDS ON #35137

pkg/kubelet/server/stats/volume_stat_calculator.go

@@ -114,9 +114,13 @@ func (s *volumeStatCalculator) calcAndStoreStats() {
 func (s *volumeStatCalculator) parsePodVolumeStats(podName string, metric *volume.Metrics) stats.VolumeStats {
 	available := uint64(metric.Available.Value())
 	capacity := uint64(metric.Capacity.Value())
-	used := uint64((metric.Used.Value()))
+	used := uint64(metric.Used.Value())
+	inodes := uint64(metric.Inodes.Value())
+	inodesFree := uint64(metric.InodesFree.Value())
+	inodesUsed := uint64(metric.InodesUsed.Value())
 	return stats.VolumeStats{
-		Name:    podName,
-		FsStats: stats.FsStats{AvailableBytes: &available, CapacityBytes: &capacity, UsedBytes: &used},
+		Name: podName,
+		FsStats: stats.FsStats{AvailableBytes: &available, CapacityBytes: &capacity, UsedBytes: &used,
+			Inodes: &inodes, InodesFree: &inodesFree, InodesUsed: &inodesUsed},
 	}
 }

pkg/volume/metrics_du.go

@@ -50,6 +50,11 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) {
 		return metrics, err
 	}
 
+	err = md.runFind(metrics)
+	if err != nil {
+		return metrics, err
+	}
+
 	err = md.getFsInfo(metrics)
 	if err != nil {
 		return metrics, err
@@ -68,14 +73,26 @@ func (md *metricsDu) runDu(metrics *Metrics) error {
 	return nil
 }
 
+// runFind executes the "find" command and writes the results to metrics.InodesUsed
+func (md *metricsDu) runFind(metrics *Metrics) error {
+	inodesUsed, err := util.Find(md.path)
+	if err != nil {
+		return err
+	}
+	metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
+	return nil
+}
+
 // getFsInfo writes metrics.Capacity and metrics.Available from the filesystem
 // info
 func (md *metricsDu) getFsInfo(metrics *Metrics) error {
-	available, capacity, _, err := util.FsInfo(md.path)
+	available, capacity, _, inodes, inodesFree, _, err := util.FsInfo(md.path)
 	if err != nil {
 		return NewFsInfoFailedError(err)
 	}
 	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
 	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
+	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
+	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
 	return nil
 }

pkg/volume/metrics_statfs.go

@@ -54,12 +54,15 @@ func (md *metricsStatFS) GetMetrics() (*Metrics, error) {
 
 // getFsInfo writes metrics.Capacity, metrics.Used and metrics.Available from the filesystem info
 func (md *metricsStatFS) getFsInfo(metrics *Metrics) error {
-	available, capacity, usage, err := util.FsInfo(md.path)
+	available, capacity, usage, inodes, inodesFree, inodesUsed, err := util.FsInfo(md.path)
 	if err != nil {
 		return NewFsInfoFailedError(err)
 	}
 	metrics.Available = resource.NewQuantity(available, resource.BinarySI)
 	metrics.Capacity = resource.NewQuantity(capacity, resource.BinarySI)
 	metrics.Used = resource.NewQuantity(usage, resource.BinarySI)
+	metrics.Inodes = resource.NewQuantity(inodes, resource.BinarySI)
+	metrics.InodesFree = resource.NewQuantity(inodesFree, resource.BinarySI)
+	metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
 	return nil
 }

pkg/volume/util/fs.go

@@ -19,23 +19,25 @@ limitations under the License.
 package util
 
 import (
+	"bytes"
 	"fmt"
 	"os/exec"
+	"strconv"
 	"strings"
 	"syscall"
 
 	"k8s.io/kubernetes/pkg/api/resource"
 )
 
-// FSInfo linux returns (available bytes, byte capacity, byte usage, error) for the filesystem that
-// path resides upon.
-func FsInfo(path string) (int64, int64, int64, error) {
+// FSInfo linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
+// for the filesystem that path resides upon.
+func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
 	statfs := &syscall.Statfs_t{}
 	err := syscall.Statfs(path, statfs)
 	if err != nil {
-		return 0, 0, 0, err
+		return 0, 0, 0, 0, 0, 0, err
 	}
-	// TODO(vishh): Include inodes space
+
 	// Available is blocks available * fragment size
 	available := int64(statfs.Bavail) * int64(statfs.Bsize)
 
@@ -45,7 +47,11 @@ func FsInfo(path string) (int64, int64, int64, error) {
 	// Usage is block being used * fragment size (aka block size).
 	usage := (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize)
 
-	return available, capacity, usage, nil
+	inodes := int64(statfs.Files)
+	inodesFree := int64(statfs.Ffree)
+	inodesUsed := inodes - inodesFree
+
+	return available, capacity, usage, inodes, inodesFree, inodesUsed, nil
 }
 
 func Du(path string) (*resource.Quantity, error) {
@@ -62,3 +68,36 @@ func Du(path string) (*resource.Quantity, error) {
 	used.Format = resource.BinarySI
 	return &used, nil
 }
+
+// Find uses the command `find <path> -dev -printf '.' | wc -c` to count files and directories.
+// While this is not an exact measure of inodes used, it is a very good approximation.
+func Find(path string) (int64, error) {
+	var stdout, stdwcerr, stdfinderr bytes.Buffer
+	var err error
+	findCmd := exec.Command("find", path, "-xdev", "-printf", ".")
+	wcCmd := exec.Command("wc", "-c")
+	if wcCmd.Stdin, err = findCmd.StdoutPipe(); err != nil {
+		return 0, fmt.Errorf("failed to setup stdout for cmd %v - %v", findCmd.Args, err)
+	}
+	wcCmd.Stdout, wcCmd.Stderr, findCmd.Stderr = &stdout, &stdwcerr, &stdfinderr
+	if err = findCmd.Start(); err != nil {
+		return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stdfinderr.String())
+	}
+
+	if err = wcCmd.Start(); err != nil {
+		return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr %v", wcCmd.Args, err, stdwcerr.String())
+	}
+	err = findCmd.Wait()
+	if err != nil {
+		return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stdfinderr.String(), err)
+	}
+	err = wcCmd.Wait()
+	if err != nil {
+		return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", wcCmd.Args, stdwcerr.String(), err)
+	}
+	inodeUsage, err := strconv.ParseInt(strings.TrimSpace(stdout.String()), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse cmds: %v, %v output %s - %s", findCmd.Args, wcCmd.Args, stdout.String(), err)
+	}
+	return inodeUsage, nil
+}

pkg/volume/volume.go

@@ -64,6 +64,20 @@ type Metrics struct {
 	// emptydir, hostpath), this is the available space on the underlying
 	// storage, and is shared with host processes and other Volumes.
 	Available *resource.Quantity
+
+	// InodesUsed represents the total inodes used by the Volume.
+	InodesUsed *resource.Quantity
+
+	// Inodes represents the total number of inodes availible in the volume.
+	// For volumes that share a filesystem with the host (e.g. emptydir, hostpath),
+	// this is the inodes available in the underlying storage,
+	// and will not equal InodesUsed + InodesFree as the fs is shared.
+	Inodes *resource.Quantity
+
+	// InodesFree represent the inodes available for the volume.  For Volues that share
+	// a filesystem with the host (e.g. emptydir, hostpath), this is the free inodes
+	// on the underlying sporage, and is shared with host processes and other volumes
+	InodesFree *resource.Quantity
 }
 
 // Attributes represents the attributes of this mounter.

test/e2e_node/summary_test.go

@@ -138,10 +138,9 @@ var _ = framework.KubeDescribe("Summary API", func() {
 							"AvailableBytes": fsCapacityBounds,
 							"CapacityBytes":  fsCapacityBounds,
 							"UsedBytes":      bounded(kb, 1*mb),
-							// Inodes are not reported for Volumes.
-							"InodesFree": BeNil(),
-							"Inodes":     BeNil(),
-							"InodesUsed": BeNil(),
+							"InodesFree":     bounded(1E4, 1E8),
+							"Inodes":         bounded(1E4, 1E8),
+							"InodesUsed":     bounded(0, 1E8),
 						}),
 					}),
 				}),