Merge pull request #96115 from ncopa/disk-usage

Get inodes and disk usage via pure go

pkg/volume/metrics_du.go

@@ -46,12 +46,7 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) {
 		return metrics, NewNoPathDefinedError()
 	}
 
-	err := md.runDiskUsage(metrics)
+	err := md.getDiskUsage(metrics)
-	if err != nil {
-		return metrics, err
-	}
-
-	err = md.runFind(metrics)
 	if err != nil {
 		return metrics, err
 	}
@@ -64,23 +59,14 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) {
 	return metrics, nil
 }
 
-// runDiskUsage gets disk usage of md.path and writes the results to metrics.Used
+// getDiskUsage writes metrics.Used and metric.InodesUsed from fs.DiskUsage
-func (md *metricsDu) runDiskUsage(metrics *Metrics) error {
+func (md *metricsDu) getDiskUsage(metrics *Metrics) error {
-	used, err := fs.DiskUsage(md.path)
+	usage, err := fs.DiskUsage(md.path)
 	if err != nil {
 		return err
 	}
-	metrics.Used = used
+	metrics.Used = resource.NewQuantity(usage.Bytes, resource.BinarySI)
-	return nil
+	metrics.InodesUsed = resource.NewQuantity(usage.Inodes, resource.BinarySI)
-}
-
-// runFind executes the "find" command and writes the results to metrics.InodesUsed
-func (md *metricsDu) runFind(metrics *Metrics) error {
-	inodesUsed, err := fs.Find(md.path)
-	if err != nil {
-		return err
-	}
-	metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
 	return nil
 }
 

pkg/volume/metrics_du_test.go

@@ -81,6 +81,21 @@ func TestMetricsDuGetCapacity(t *testing.T) {
 	if e, a := (expectedEmptyDirUsage.Value() + getExpectedBlockSize(filepath.Join(tmpDir, "f1"))), actual.Used.Value(); e != a {
 		t.Errorf("Unexpected Used for directory with file.  Expected %v, got %d.", e, a)
 	}
+
+	// create a hardlink and expect inodes count to stay the same
+	previousInodes := actual.InodesUsed.Value()
+	err = os.Link(filepath.Join(tmpDir, "f1"), filepath.Join(tmpDir, "f2"))
+	if err != nil {
+		t.Errorf("Unexpected error when creating hard link %v", err)
+	}
+	actual, err = metrics.GetMetrics()
+	if err != nil {
+		t.Errorf("Unexpected error when calling GetMetrics %v", err)
+	}
+	if e, a := previousInodes, actual.InodesUsed.Value(); e != a {
+		t.Errorf("Unexpected Used for directory with file.  Expected %v, got %d.", e, a)
+	}
+
 }
 
 // TestMetricsDuRequireInit tests that if MetricsDu is not initialized with a path, GetMetrics

pkg/volume/util/fs/fs.go

@@ -19,17 +19,21 @@ limitations under the License.
 package fs
 
 import (
-	"bytes"
 	"fmt"
-	"os/exec"
+	"os"
-	"strings"
+	"path/filepath"
+	"syscall"
 
 	"golang.org/x/sys/unix"
 
-	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/volume/util/fsquota"
 )
 
+type UsageInfo struct {
+	Bytes  int64
+	Inodes int64
+}
+
 // Info linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
 // for the filesystem that path resides upon.
 func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
@@ -55,63 +59,83 @@ func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
 	return available, capacity, usage, inodes, inodesFree, inodesUsed, nil
 }
 
-// DiskUsage gets disk usage of specified path.
+// DiskUsage calculates the number of inodes and disk usage for a given directory
-func DiskUsage(path string) (*resource.Quantity, error) {
+func DiskUsage(path string) (UsageInfo, error) {
-	// First check whether the quota system knows about this directory
+	var usage UsageInfo
-	// A nil quantity with no error means that the path does not support quotas
-	// and we should use other mechanisms.
-	data, err := fsquota.GetConsumption(path)
-	if data != nil {
-		return data, nil
-	} else if err != nil {
-		return nil, fmt.Errorf("unable to retrieve disk consumption via quota for %s: %v", path, err)
-	}
-	// Uses the same niceness level as cadvisor.fs does when running du
-	// Uses -B 1 to always scale to a blocksize of 1 byte
-	out, err := exec.Command("nice", "-n", "19", "du", "-x", "-s", "-B", "1", path).CombinedOutput()
-	if err != nil {
-		return nil, fmt.Errorf("failed command 'du' ($ nice -n 19 du -x -s -B 1) on path %s with error %v", path, err)
-	}
-	used, err := resource.ParseQuantity(strings.Fields(string(out))[0])
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse 'du' output %s due to error %v", out, err)
-	}
-	used.Format = resource.BinarySI
-	return &used, nil
-}
 
-// Find uses the equivalent of the command `find <path> -dev -printf '.' | wc -c` to count files and directories.
-// While this is not an exact measure of inodes used, it is a very good approximation.
-func Find(path string) (int64, error) {
 	if path == "" {
-		return 0, fmt.Errorf("invalid directory")
+		return usage, fmt.Errorf("invalid directory")
 	}
+
 	// First check whether the quota system knows about this directory
-	// A nil quantity with no error means that the path does not support quotas
+	// A nil quantity or error means that the path does not support quotas
-	// and we should use other mechanisms.
+	// or xfs_quota tool is missing and we should use other mechanisms.
-	inodes, err := fsquota.GetInodes(path)
+	consumption, _ := fsquota.GetConsumption(path)
+	if consumption != nil {
+		usage.Bytes = consumption.Value()
+	}
+
+	inodes, _ := fsquota.GetInodes(path)
 	if inodes != nil {
-		return inodes.Value(), nil
+		usage.Inodes = inodes.Value()
-	} else if err != nil {
-		return 0, fmt.Errorf("unable to retrieve inode consumption via quota for %s: %v", path, err)
 	}
-	var counter byteCounter
-	var stderr bytes.Buffer
-	findCmd := exec.Command("find", path, "-xdev", "-printf", ".")
-	findCmd.Stdout, findCmd.Stderr = &counter, &stderr
-	if err := findCmd.Start(); err != nil {
-		return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stderr.String())
-	}
-	if err := findCmd.Wait(); err != nil {
-		return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stderr.String(), err)
-	}
-	return counter.bytesWritten, nil
-}
 
-// Simple io.Writer implementation that counts how many bytes were written.
+	if inodes != nil && consumption != nil {
-type byteCounter struct{ bytesWritten int64 }
+		return usage, nil
+	}
 
-func (b *byteCounter) Write(p []byte) (int, error) {
+	topLevelStat := &unix.Stat_t{}
-	b.bytesWritten += int64(len(p))
+	err := unix.Stat(path, topLevelStat)
-	return len(p), nil
+	if err != nil {
+		return usage, err
+	}
+
+	// dedupedInode stores inodes that could be duplicates (nlink > 1)
+	dedupedInodes := make(map[uint64]struct{})
+
+	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		// ignore files that have been deleted after directory was read
+		if os.IsNotExist(err) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("unable to count inodes for %s: %s", path, err)
+		}
+
+		// according to the docs, Sys can be nil
+		if info.Sys() == nil {
+			return fmt.Errorf("fileinfo Sys is nil")
+		}
+
+		s, ok := info.Sys().(*syscall.Stat_t)
+		if !ok {
+			return fmt.Errorf("unsupported fileinfo; could not convert to stat_t")
+		}
+
+		if s.Dev != topLevelStat.Dev {
+			// don't descend into directories on other devices
+			return filepath.SkipDir
+		}
+
+		// Dedupe hardlinks
+		if s.Nlink > 1 {
+			if _, ok := dedupedInodes[s.Ino]; !ok {
+				dedupedInodes[s.Ino] = struct{}{}
+			} else {
+				return nil
+			}
+		}
+
+		if consumption == nil {
+			usage.Bytes += int64(s.Blocks) * int64(512) // blocksize in bytes
+		}
+
+		if inodes == nil {
+			usage.Inodes++
+		}
+
+		return nil
+	})
+
+	return usage, err
 }

pkg/volume/util/fs/fs_unsupported.go

@@ -20,21 +20,20 @@ package fs
 
 import (
 	"fmt"
-
-	"k8s.io/apimachinery/pkg/api/resource"
 )
 
+type UsageInfo struct {
+	Bytes  int64
+	Inodes int64
+}
+
 // Info unsupported returns 0 values for available and capacity and an error.
 func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
 	return 0, 0, 0, 0, 0, 0, fmt.Errorf("fsinfo not supported for this build")
 }
 
 // DiskUsage gets disk usage of specified path.
-func DiskUsage(path string) (*resource.Quantity, error) {
+func DiskUsage(path string) (UsageInfo, error) {
-	return nil, fmt.Errorf("du not supported for this build")
+	var usage UsageInfo
-}
+	return usage, fmt.Errorf("directory disk usage not supported for this build.")
-
-// Find will always return zero since is on unsupported platform.
-func Find(path string) (int64, error) {
-	return 0, fmt.Errorf("find not supported for this build")
 }

pkg/volume/util/fs/fs_windows.go

@@ -26,8 +26,6 @@ import (
 	"unsafe"
 
 	"golang.org/x/sys/windows"
-
-	"k8s.io/apimachinery/pkg/api/resource"
 )
 
 var (
@@ -35,6 +33,11 @@ var (
 	procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW")
 )
 
+type UsageInfo struct {
+	Bytes  int64
+	Inodes int64
+}
+
 // Info returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
 // for the filesystem that path resides upon.
 func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
@@ -64,28 +67,15 @@ func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
 }
 
 // DiskUsage gets disk usage of specified path.
-func DiskUsage(path string) (*resource.Quantity, error) {
+func DiskUsage(path string) (UsageInfo, error) {
+	var usage UsageInfo
 	info, err := os.Lstat(path)
 	if err != nil {
-		return nil, err
+		return usage, err
 	}
 
-	usage, err := diskUsage(path, info)
+	usage.Bytes, err = diskUsage(path, info)
-	if err != nil {
+	return usage, err
-		return nil, err
-	}
-
-	used, err := resource.ParseQuantity(fmt.Sprintf("%d", usage))
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse fs usage %d due to %v", usage, err)
-	}
-	used.Format = resource.BinarySI
-	return &used, nil
-}
-
-// Find will always return zero since inodes is not supported on Windows.
-func Find(path string) (int64, error) {
-	return 0, nil
 }
 
 func diskUsage(currPath string, info os.FileInfo) (int64, error) {

pkg/volume/util/fs/fs_windows_test.go

@@ -67,10 +67,15 @@ func TestDiskUsage(t *testing.T) {
 	}
 	total := dirInfo1.Size() + dirInfo2.Size() + fileInfo1.Size() + fileInfo2.Size()
 
-	size, err := DiskUsage(dir1)
+	usage, err := DiskUsage(dir1)
 	if err != nil {
 		t.Fatalf("TestDiskUsage failed: %s", err.Error())
 	}
+	size, err := resource.ParseQuantity(fmt.Sprintf("%d", usage.Bytes))
+	if err != nil {
+		t.Fatalf("TestDiskUsage failed: %s", err.Error())
+	}
+
 	used, err := resource.ParseQuantity(fmt.Sprintf("%d", total))
 	if err != nil {
 		t.Fatalf("TestDiskUsage failed: %s", err.Error())