vendor: cadvisor v0.39.0

Main upgrades:
- github.com/opencontainers/runc v1.0.0-rc93
- github.com/containerd/containerd v1.4.4
- github.com/docker/docker v20.10.2
- github.com/mrunalp/fileutils v0.5.0
- github.com/opencontainers/selinux v1.8.0
- github.com/cilium/ebpf v0.2.0

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go

@@ -6,7 +6,6 @@ import (
 	"bufio"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -105,9 +104,9 @@ func splitBlkioStatLine(r rune) bool {
 	return r == ' ' || r == ':'
 }
 
-func getBlkioStat(path string) ([]cgroups.BlkioStatEntry, error) {
+func getBlkioStat(dir, file string) ([]cgroups.BlkioStatEntry, error) {
 	var blkioStats []cgroups.BlkioStatEntry
-	f, err := os.Open(path)
+	f, err := fscommon.OpenFile(dir, file, os.O_RDONLY)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return blkioStats, nil
@@ -125,7 +124,7 @@ func getBlkioStat(path string) ([]cgroups.BlkioStatEntry, error) {
 				// skip total line
 				continue
 			} else {
-				return nil, fmt.Errorf("Invalid line found while parsing %s: %s", path, sc.Text())
+				return nil, fmt.Errorf("Invalid line found while parsing %s/%s: %s", dir, file, sc.Text())
 			}
 		}
 
@@ -158,73 +157,134 @@ func getBlkioStat(path string) ([]cgroups.BlkioStatEntry, error) {
 }
 
 func (s *BlkioGroup) GetStats(path string, stats *cgroups.Stats) error {
-	// Try to read CFQ stats available on all CFQ enabled kernels first
-	if blkioStats, err := getBlkioStat(filepath.Join(path, "blkio.io_serviced_recursive")); err == nil && blkioStats != nil {
-		return getCFQStats(path, stats)
+	type blkioStatInfo struct {
+		filename            string
+		blkioStatEntriesPtr *[]cgroups.BlkioStatEntry
+	}
+	var bfqDebugStats = []blkioStatInfo{
+		{
+			filename:            "blkio.bfq.sectors_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.SectorsRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_service_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceTimeRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_wait_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoWaitTimeRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_merged_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoMergedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_queued_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoQueuedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoTimeRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var bfqStats = []blkioStatInfo{
+		{
+			filename:            "blkio.bfq.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.bfq.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var cfqStats = []blkioStatInfo{
+		{
+			filename:            "blkio.sectors_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.SectorsRecursive,
+		},
+		{
+			filename:            "blkio.io_service_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceTimeRecursive,
+		},
+		{
+			filename:            "blkio.io_wait_time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoWaitTimeRecursive,
+		},
+		{
+			filename:            "blkio.io_merged_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoMergedRecursive,
+		},
+		{
+			filename:            "blkio.io_queued_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoQueuedRecursive,
+		},
+		{
+			filename:            "blkio.time_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoTimeRecursive,
+		},
+		{
+			filename:            "blkio.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var throttleRecursiveStats = []blkioStatInfo{
+		{
+			filename:            "blkio.throttle.io_serviced_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.throttle.io_service_bytes_recursive",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var baseStats = []blkioStatInfo{
+		{
+			filename:            "blkio.throttle.io_serviced",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServicedRecursive,
+		},
+		{
+			filename:            "blkio.throttle.io_service_bytes",
+			blkioStatEntriesPtr: &stats.BlkioStats.IoServiceBytesRecursive,
+		},
+	}
+	var orderedStats = [][]blkioStatInfo{
+		bfqDebugStats,
+		bfqStats,
+		cfqStats,
+		throttleRecursiveStats,
+		baseStats,
 	}
-	return getStats(path, stats) // Use generic stats as fallback
-}
 
-func getCFQStats(path string, stats *cgroups.Stats) error {
 	var blkioStats []cgroups.BlkioStatEntry
 	var err error
 
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.sectors_recursive")); err != nil {
-		return err
+	for _, statGroup := range orderedStats {
+		for i, statInfo := range statGroup {
+			if blkioStats, err = getBlkioStat(path, statInfo.filename); err != nil || blkioStats == nil {
+				// if error occurs on first file, move to next group
+				if i == 0 {
+					break
+				}
+				return err
+			}
+			*statInfo.blkioStatEntriesPtr = blkioStats
+			//finish if all stats are gathered
+			if i == len(statGroup)-1 {
+				return nil
+			}
+		}
 	}
-	stats.BlkioStats.SectorsRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_service_bytes_recursive")); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServiceBytesRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_serviced_recursive")); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServicedRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_queued_recursive")); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoQueuedRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_service_time_recursive")); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServiceTimeRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_wait_time_recursive")); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoWaitTimeRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_merged_recursive")); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoMergedRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.time_recursive")); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoTimeRecursive = blkioStats
-
-	return nil
-}
-
-func getStats(path string, stats *cgroups.Stats) error {
-	var blkioStats []cgroups.BlkioStatEntry
-	var err error
-
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.throttle.io_service_bytes")); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServiceBytesRecursive = blkioStats
-
-	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.throttle.io_serviced")); err != nil {
-		return err
-	}
-	stats.BlkioStats.IoServicedRecursive = blkioStats
-
 	return nil
 }

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpu.go

@@ -6,7 +6,6 @@ import (
 	"bufio"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strconv"
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
@@ -87,7 +86,7 @@ func (s *CpuGroup) Set(path string, cgroup *configs.Cgroup) error {
 }
 
 func (s *CpuGroup) GetStats(path string, stats *cgroups.Stats) error {
-	f, err := os.Open(filepath.Join(path, "cpu.stat"))
+	f, err := fscommon.OpenFile(path, "cpu.stat", os.O_RDONLY)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuacct.go

@@ -5,7 +5,6 @@ package fs
 import (
 	"bufio"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -83,8 +82,7 @@ func (s *CpuacctGroup) GetStats(path string, stats *cgroups.Stats) error {
 
 // Returns user and kernel usage breakdown in nanoseconds.
 func getCpuUsageBreakdown(path string) (uint64, uint64, error) {
-	userModeUsage := uint64(0)
-	kernelModeUsage := uint64(0)
+	var userModeUsage, kernelModeUsage uint64
 	const (
 		userField   = "user"
 		systemField = "system"
@@ -93,11 +91,11 @@ func getCpuUsageBreakdown(path string) (uint64, uint64, error) {
 	// Expected format:
 	// user <usage in ticks>
 	// system <usage in ticks>
-	data, err := ioutil.ReadFile(filepath.Join(path, cgroupCpuacctStat))
+	data, err := fscommon.ReadFile(path, cgroupCpuacctStat)
 	if err != nil {
 		return 0, 0, err
 	}
-	fields := strings.Fields(string(data))
+	fields := strings.Fields(data)
 	if len(fields) < 4 {
 		return 0, 0, fmt.Errorf("failure - %s is expected to have at least 4 fields", filepath.Join(path, cgroupCpuacctStat))
 	}
@@ -119,11 +117,11 @@ func getCpuUsageBreakdown(path string) (uint64, uint64, error) {
 
 func getPercpuUsage(path string) ([]uint64, error) {
 	percpuUsage := []uint64{}
-	data, err := ioutil.ReadFile(filepath.Join(path, "cpuacct.usage_percpu"))
+	data, err := fscommon.ReadFile(path, "cpuacct.usage_percpu")
 	if err != nil {
 		return percpuUsage, err
 	}
-	for _, value := range strings.Fields(string(data)) {
+	for _, value := range strings.Fields(data) {
 		value, err := strconv.ParseUint(value, 10, 64)
 		if err != nil {
 			return percpuUsage, fmt.Errorf("Unable to convert param value to uint64: %s", err)
@@ -137,7 +135,7 @@ func getPercpuUsageInModes(path string) ([]uint64, []uint64, error) {
 	usageKernelMode := []uint64{}
 	usageUserMode := []uint64{}
 
-	file, err := os.Open(filepath.Join(path, cgroupCpuacctUsageAll))
+	file, err := fscommon.OpenFile(path, cgroupCpuacctUsageAll, os.O_RDONLY)
 	if os.IsNotExist(err) {
 		return usageKernelMode, usageUserMode, nil
 	} else if err != nil {

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuset.go

@@ -3,17 +3,17 @@
 package fs
 
 import (
-	"bytes"
-	"io/ioutil"
+	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
 
-	"github.com/moby/sys/mountinfo"
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
 	"github.com/opencontainers/runc/libcontainer/configs"
-	libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
 	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 )
 
 type CpusetGroup struct {
@@ -41,30 +41,107 @@ func (s *CpusetGroup) Set(path string, cgroup *configs.Cgroup) error {
 	return nil
 }
 
-func (s *CpusetGroup) GetStats(path string, stats *cgroups.Stats) error {
-	return nil
-}
-
-// Get the source mount point of directory passed in as argument.
-func getMount(dir string) (string, error) {
-	mi, err := mountinfo.GetMounts(mountinfo.ParentsFilter(dir))
+func getCpusetStat(path string, filename string) ([]uint16, error) {
+	var extracted []uint16
+	fileContent, err := fscommon.GetCgroupParamString(path, filename)
 	if err != nil {
-		return "", err
+		return extracted, err
 	}
-	if len(mi) < 1 {
-		return "", errors.Errorf("Can't find mount point of %s", dir)
+	if len(fileContent) == 0 {
+		return extracted, fmt.Errorf("%s found to be empty", filepath.Join(path, filename))
 	}
 
-	// find the longest mount point
-	var idx, maxlen int
-	for i := range mi {
-		if len(mi[i].Mountpoint) > maxlen {
-			maxlen = len(mi[i].Mountpoint)
-			idx = i
+	for _, s := range strings.Split(fileContent, ",") {
+		splitted := strings.SplitN(s, "-", 3)
+		switch len(splitted) {
+		case 3:
+			return extracted, fmt.Errorf("invalid values in %s", filepath.Join(path, filename))
+		case 2:
+			min, err := strconv.ParseUint(splitted[0], 10, 16)
+			if err != nil {
+				return extracted, err
+			}
+			max, err := strconv.ParseUint(splitted[1], 10, 16)
+			if err != nil {
+				return extracted, err
+			}
+			if min > max {
+				return extracted, fmt.Errorf("invalid values in %s", filepath.Join(path, filename))
+			}
+			for i := min; i <= max; i++ {
+				extracted = append(extracted, uint16(i))
+			}
+		case 1:
+			value, err := strconv.ParseUint(s, 10, 16)
+			if err != nil {
+				return extracted, err
+			}
+			extracted = append(extracted, uint16(value))
 		}
 	}
 
-	return mi[idx].Mountpoint, nil
+	return extracted, nil
+}
+
+func (s *CpusetGroup) GetStats(path string, stats *cgroups.Stats) error {
+	var err error
+
+	stats.CPUSetStats.CPUs, err = getCpusetStat(path, "cpuset.cpus")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.CPUExclusive, err = fscommon.GetCgroupParamUint(path, "cpuset.cpu_exclusive")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.Mems, err = getCpusetStat(path, "cpuset.mems")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.MemHardwall, err = fscommon.GetCgroupParamUint(path, "cpuset.mem_hardwall")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.MemExclusive, err = fscommon.GetCgroupParamUint(path, "cpuset.mem_exclusive")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.MemoryMigrate, err = fscommon.GetCgroupParamUint(path, "cpuset.memory_migrate")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.MemorySpreadPage, err = fscommon.GetCgroupParamUint(path, "cpuset.memory_spread_page")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.MemorySpreadSlab, err = fscommon.GetCgroupParamUint(path, "cpuset.memory_spread_slab")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.MemoryPressure, err = fscommon.GetCgroupParamUint(path, "cpuset.memory_pressure")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.SchedLoadBalance, err = fscommon.GetCgroupParamUint(path, "cpuset.sched_load_balance")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	stats.CPUSetStats.SchedRelaxDomainLevel, err = fscommon.GetCgroupParamInt(path, "cpuset.sched_relax_domain_level")
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+
+	return nil
 }
 
 func (s *CpusetGroup) ApplyDir(dir string, cgroup *configs.Cgroup, pid int) error {
@@ -73,18 +150,13 @@ func (s *CpusetGroup) ApplyDir(dir string, cgroup *configs.Cgroup, pid int) erro
 	if dir == "" {
 		return nil
 	}
-	root, err := getMount(dir)
-	if err != nil {
-		return err
-	}
-	root = filepath.Dir(root)
 	// 'ensureParent' start with parent because we don't want to
 	// explicitly inherit from parent, it could conflict with
 	// 'cpuset.cpu_exclusive'.
-	if err := s.ensureParent(filepath.Dir(dir), root); err != nil {
+	if err := cpusetEnsureParent(filepath.Dir(dir)); err != nil {
 		return err
 	}
-	if err := os.MkdirAll(dir, 0755); err != nil {
+	if err := os.Mkdir(dir, 0755); err != nil && !os.IsExist(err) {
 		return err
 	}
 	// We didn't inherit cpuset configs from parent, but we have
@@ -103,59 +175,61 @@ func (s *CpusetGroup) ApplyDir(dir string, cgroup *configs.Cgroup, pid int) erro
 	return cgroups.WriteCgroupProc(dir, pid)
 }
 
-func (s *CpusetGroup) getSubsystemSettings(parent string) (cpus []byte, mems []byte, err error) {
-	if cpus, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.cpus")); err != nil {
+func getCpusetSubsystemSettings(parent string) (cpus, mems string, err error) {
+	if cpus, err = fscommon.ReadFile(parent, "cpuset.cpus"); err != nil {
 		return
 	}
-	if mems, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.mems")); err != nil {
+	if mems, err = fscommon.ReadFile(parent, "cpuset.mems"); err != nil {
 		return
 	}
 	return cpus, mems, nil
 }
 
-// ensureParent makes sure that the parent directory of current is created
-// and populated with the proper cpus and mems files copied from
-// it's parent.
-func (s *CpusetGroup) ensureParent(current, root string) error {
+// cpusetEnsureParent makes sure that the parent directories of current
+// are created and populated with the proper cpus and mems files copied
+// from their respective parent. It does that recursively, starting from
+// the top of the cpuset hierarchy (i.e. cpuset cgroup mount point).
+func cpusetEnsureParent(current string) error {
+	var st unix.Statfs_t
+
 	parent := filepath.Dir(current)
-	if libcontainerUtils.CleanPath(parent) == root {
+	err := unix.Statfs(parent, &st)
+	if err == nil && st.Type != unix.CGROUP_SUPER_MAGIC {
 		return nil
 	}
-	// Avoid infinite recursion.
-	if parent == current {
-		return errors.New("cpuset: cgroup parent path outside cgroup root")
+	// Treat non-existing directory as cgroupfs as it will be created,
+	// and the root cpuset directory obviously exists.
+	if err != nil && err != unix.ENOENT {
+		return &os.PathError{Op: "statfs", Path: parent, Err: err}
 	}
-	if err := s.ensureParent(parent, root); err != nil {
+
+	if err := cpusetEnsureParent(parent); err != nil {
 		return err
 	}
-	if err := os.MkdirAll(current, 0755); err != nil {
+	if err := os.Mkdir(current, 0755); err != nil && !os.IsExist(err) {
 		return err
 	}
-	return s.copyIfNeeded(current, parent)
+	return cpusetCopyIfNeeded(current, parent)
 }
 
-// copyIfNeeded copies the cpuset.cpus and cpuset.mems from the parent
+// cpusetCopyIfNeeded copies the cpuset.cpus and cpuset.mems from the parent
 // directory to the current directory if the file's contents are 0
-func (s *CpusetGroup) copyIfNeeded(current, parent string) error {
-	var (
-		err                      error
-		currentCpus, currentMems []byte
-		parentCpus, parentMems   []byte
-	)
-
-	if currentCpus, currentMems, err = s.getSubsystemSettings(current); err != nil {
+func cpusetCopyIfNeeded(current, parent string) error {
+	currentCpus, currentMems, err := getCpusetSubsystemSettings(current)
+	if err != nil {
 		return err
 	}
-	if parentCpus, parentMems, err = s.getSubsystemSettings(parent); err != nil {
+	parentCpus, parentMems, err := getCpusetSubsystemSettings(parent)
+	if err != nil {
 		return err
 	}
 
-	if s.isEmpty(currentCpus) {
+	if isEmptyCpuset(currentCpus) {
 		if err := fscommon.WriteFile(current, "cpuset.cpus", string(parentCpus)); err != nil {
 			return err
 		}
 	}
-	if s.isEmpty(currentMems) {
+	if isEmptyCpuset(currentMems) {
 		if err := fscommon.WriteFile(current, "cpuset.mems", string(parentMems)); err != nil {
 			return err
 		}
@@ -163,13 +237,13 @@ func (s *CpusetGroup) copyIfNeeded(current, parent string) error {
 	return nil
 }
 
-func (s *CpusetGroup) isEmpty(b []byte) bool {
-	return len(bytes.Trim(b, "\n")) == 0
+func isEmptyCpuset(str string) bool {
+	return str == "" || str == "\n"
 }
 
 func (s *CpusetGroup) ensureCpusAndMems(path string, cgroup *configs.Cgroup) error {
 	if err := s.Set(path, cgroup); err != nil {
 		return err
 	}
-	return s.copyIfNeeded(path, filepath.Dir(path))
+	return cpusetCopyIfNeeded(path, filepath.Dir(path))
 }

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/devices.go

@@ -8,9 +8,10 @@ import (
 	"reflect"
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
-	"github.com/opencontainers/runc/libcontainer/cgroups/devices"
+	cgroupdevices "github.com/opencontainers/runc/libcontainer/cgroups/devices"
 	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
 	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/devices"
 	"github.com/opencontainers/runc/libcontainer/system"
 )
 
@@ -34,17 +35,17 @@ func (s *DevicesGroup) Apply(path string, d *cgroupData) error {
 	return join(path, d.pid)
 }
 
-func loadEmulator(path string) (*devices.Emulator, error) {
+func loadEmulator(path string) (*cgroupdevices.Emulator, error) {
 	list, err := fscommon.ReadFile(path, "devices.list")
 	if err != nil {
 		return nil, err
 	}
-	return devices.EmulatorFromList(bytes.NewBufferString(list))
+	return cgroupdevices.EmulatorFromList(bytes.NewBufferString(list))
 }
 
-func buildEmulator(rules []*configs.DeviceRule) (*devices.Emulator, error) {
+func buildEmulator(rules []*devices.Rule) (*cgroupdevices.Emulator, error) {
 	// This defaults to a white-list -- which is what we want!
-	emu := &devices.Emulator{}
+	emu := &cgroupdevices.Emulator{}
 	for _, rule := range rules {
 		if err := emu.Apply(*rule); err != nil {
 			return nil, err

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/freezer.go

@@ -28,33 +28,54 @@ func (s *FreezerGroup) Apply(path string, d *cgroupData) error {
 
 func (s *FreezerGroup) Set(path string, cgroup *configs.Cgroup) error {
 	switch cgroup.Resources.Freezer {
-	case configs.Frozen, configs.Thawed:
-		for {
-			// In case this loop does not exit because it doesn't get the expected
-			// state, let's write again this state, hoping it's going to be properly
-			// set this time. Otherwise, this loop could run infinitely, waiting for
-			// a state change that would never happen.
-			if err := fscommon.WriteFile(path, "freezer.state", string(cgroup.Resources.Freezer)); err != nil {
+	case configs.Frozen:
+		// As per older kernel docs (freezer-subsystem.txt before
+		// kernel commit ef9fe980c6fcc1821), if FREEZING is seen,
+		// userspace should either retry or thaw. While current
+		// kernel cgroup v1 docs no longer mention a need to retry,
+		// the kernel (tested on v5.4, Ubuntu 20.04) can't reliably
+		// freeze a cgroup while new processes keep appearing in it
+		// (either via fork/clone or by writing new PIDs to
+		// cgroup.procs).
+		//
+		// The number of retries below is chosen to have a decent
+		// chance to succeed even in the worst case scenario (runc
+		// pause/unpause with parallel runc exec).
+		//
+		// Adding any amount of sleep in between retries did not
+		// increase the chances of successful freeze.
+		for i := 0; i < 1000; i++ {
+			if err := fscommon.WriteFile(path, "freezer.state", string(configs.Frozen)); err != nil {
 				return err
 			}
 
-			state, err := s.GetState(path)
+			state, err := fscommon.ReadFile(path, "freezer.state")
 			if err != nil {
 				return err
 			}
-			if state == cgroup.Resources.Freezer {
-				break
+			state = strings.TrimSpace(state)
+			switch state {
+			case "FREEZING":
+				continue
+			case string(configs.Frozen):
+				return nil
+			default:
+				// should never happen
+				return fmt.Errorf("unexpected state %s while freezing", strings.TrimSpace(state))
 			}
-
-			time.Sleep(1 * time.Millisecond)
 		}
+		// Despite our best efforts, it got stuck in FREEZING.
+		// Leaving it in this state is bad and dangerous, so
+		// let's (try to) thaw it back and error out.
+		_ = fscommon.WriteFile(path, "freezer.state", string(configs.Thawed))
+		return errors.New("unable to freeze")
+	case configs.Thawed:
+		return fscommon.WriteFile(path, "freezer.state", string(configs.Thawed))
 	case configs.Undefined:
 		return nil
 	default:
 		return fmt.Errorf("Invalid argument '%s' to freezer.state", string(cgroup.Resources.Freezer))
 	}
-
-	return nil
 }
 
 func (s *FreezerGroup) GetStats(path string, stats *cgroups.Stats) error {

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/fs.go

@@ -3,11 +3,9 @@
 package fs
 
 import (
-	"bufio"
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"sync"
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
@@ -133,46 +131,19 @@ func getCgroupRoot() (string, error) {
 		return cgroupRoot, nil
 	}
 
-	// slow path: parse mountinfo, find the first mount where fs=cgroup
-	// (e.g. "/sys/fs/cgroup/memory"), use its parent.
-	f, err := os.Open("/proc/self/mountinfo")
+	// slow path: parse mountinfo
+	mi, err := cgroups.GetCgroupMounts(false)
 	if err != nil {
 		return "", err
 	}
-	defer f.Close()
-
-	var root string
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		text := scanner.Text()
-		fields := strings.Split(text, " ")
-		// Safe as mountinfo encodes mountpoints with spaces as \040.
-		index := strings.Index(text, " - ")
-		postSeparatorFields := strings.Fields(text[index+3:])
-		numPostFields := len(postSeparatorFields)
-
-		// This is an error as we can't detect if the mount is for "cgroup"
-		if numPostFields == 0 {
-			return "", fmt.Errorf("mountinfo: found no fields post '-' in %q", text)
-		}
-
-		if postSeparatorFields[0] == "cgroup" {
-			// Check that the mount is properly formatted.
-			if numPostFields < 3 {
-				return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text)
-			}
-
-			root = filepath.Dir(fields[4])
-			break
-		}
-	}
-	if err := scanner.Err(); err != nil {
-		return "", err
-	}
-	if root == "" {
+	if len(mi) < 1 {
 		return "", errors.New("no cgroup mount found in mountinfo")
 	}
 
+	// Get the first cgroup mount (e.g. "/sys/fs/cgroup/memory"),
+	// use its parent directory.
+	root := filepath.Dir(mi[0].Mountpoint)
+
 	if _, err := os.Stat(root); err != nil {
 		return "", err
 	}
@@ -218,28 +189,31 @@ func (m *manager) Apply(pid int) (err error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
-	var c = m.cgroups
-
-	d, err := getCgroupData(m.cgroups, pid)
-	if err != nil {
-		return err
+	c := m.cgroups
+	if c.Resources.Unified != nil {
+		return cgroups.ErrV1NoUnified
 	}
 
 	m.paths = make(map[string]string)
 	if c.Paths != nil {
+		cgMap, err := cgroups.ParseCgroupFile("/proc/self/cgroup")
+		if err != nil {
+			return err
+		}
 		for name, path := range c.Paths {
-			_, err := d.path(name)
-			if err != nil {
-				if cgroups.IsNotFound(err) {
-					continue
-				}
-				return err
+			// XXX(kolyshkin@): why this check is needed?
+			if _, ok := cgMap[name]; ok {
+				m.paths[name] = path
 			}
-			m.paths[name] = path
 		}
 		return cgroups.EnterPid(m.paths, pid)
 	}
 
+	d, err := getCgroupData(m.cgroups, pid)
+	if err != nil {
+		return err
+	}
+
 	for _, sys := range subsystems {
 		p, err := d.path(sys.Name())
 		if err != nil {
@@ -274,11 +248,7 @@ func (m *manager) Destroy() error {
 	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	if err := cgroups.RemovePaths(m.paths); err != nil {
-		return err
-	}
-	m.paths = make(map[string]string)
-	return nil
+	return cgroups.RemovePaths(m.paths)
 }
 
 func (m *manager) Path(subsys string) string {
@@ -313,6 +283,9 @@ func (m *manager) Set(container *configs.Config) error {
 	if m.cgroups != nil && m.cgroups.Paths != nil {
 		return nil
 	}
+	if container.Cgroups.Resources.Unified != nil {
+		return cgroups.ErrV1NoUnified
+	}
 
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -425,16 +398,6 @@ func join(path string, pid int) error {
 	return cgroups.WriteCgroupProc(path, pid)
 }
 
-func removePath(p string, err error) error {
-	if err != nil {
-		return err
-	}
-	if p != "" {
-		return os.RemoveAll(p)
-	}
-	return nil
-}
-
 func (m *manager) GetPaths() map[string]string {
 	m.mu.Lock()
 	defer m.mu.Unlock()

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/hugetlb.go

@@ -5,7 +5,6 @@ package fs
 import (
 	"fmt"
 	"strconv"
-	"strings"
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
@@ -25,7 +24,7 @@ func (s *HugetlbGroup) Apply(path string, d *cgroupData) error {
 
 func (s *HugetlbGroup) Set(path string, cgroup *configs.Cgroup) error {
 	for _, hugetlb := range cgroup.Resources.HugetlbLimit {
-		if err := fscommon.WriteFile(path, strings.Join([]string{"hugetlb", hugetlb.Pagesize, "limit_in_bytes"}, "."), strconv.FormatUint(hugetlb.Limit, 10)); err != nil {
+		if err := fscommon.WriteFile(path, "hugetlb."+hugetlb.Pagesize+".limit_in_bytes", strconv.FormatUint(hugetlb.Limit, 10)); err != nil {
 			return err
 		}
 	}
@@ -39,21 +38,21 @@ func (s *HugetlbGroup) GetStats(path string, stats *cgroups.Stats) error {
 		return nil
 	}
 	for _, pageSize := range HugePageSizes {
-		usage := strings.Join([]string{"hugetlb", pageSize, "usage_in_bytes"}, ".")
+		usage := "hugetlb." + pageSize + ".usage_in_bytes"
 		value, err := fscommon.GetCgroupParamUint(path, usage)
 		if err != nil {
 			return fmt.Errorf("failed to parse %s - %v", usage, err)
 		}
 		hugetlbStats.Usage = value
 
-		maxUsage := strings.Join([]string{"hugetlb", pageSize, "max_usage_in_bytes"}, ".")
+		maxUsage := "hugetlb." + pageSize + ".max_usage_in_bytes"
 		value, err = fscommon.GetCgroupParamUint(path, maxUsage)
 		if err != nil {
 			return fmt.Errorf("failed to parse %s - %v", maxUsage, err)
 		}
 		hugetlbStats.MaxUsage = value
 
-		failcnt := strings.Join([]string{"hugetlb", pageSize, "failcnt"}, ".")
+		failcnt := "hugetlb." + pageSize + ".failcnt"
 		value, err = fscommon.GetCgroupParamUint(path, failcnt)
 		if err != nil {
 			return fmt.Errorf("failed to parse %s - %v", failcnt, err)

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/kmem.go

@@ -5,11 +5,11 @@ package fs
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"path/filepath"
 	"strconv"
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
 	"golang.org/x/sys/unix"
 )
 
@@ -42,7 +42,7 @@ func setKernelMemory(path string, kernelMemoryLimit int64) error {
 		// doesn't support it we *must* error out.
 		return errors.New("kernel memory accounting not supported by this kernel")
 	}
-	if err := ioutil.WriteFile(filepath.Join(path, cgroupKernelMemoryLimit), []byte(strconv.FormatInt(kernelMemoryLimit, 10)), 0700); err != nil {
+	if err := fscommon.WriteFile(path, cgroupKernelMemoryLimit, strconv.FormatInt(kernelMemoryLimit, 10)); err != nil {
 		// Check if the error number returned by the syscall is "EBUSY"
 		// The EBUSY signal is returned on attempts to write to the
 		// memory.kmem.limit_in_bytes file if the cgroup has children or
@@ -50,7 +50,7 @@ func setKernelMemory(path string, kernelMemoryLimit int64) error {
 		if errors.Is(err, unix.EBUSY) {
 			return fmt.Errorf("failed to set %s, because either tasks have already joined this cgroup or it has children", cgroupKernelMemoryLimit)
 		}
-		return fmt.Errorf("failed to write %v to %v: %v", kernelMemoryLimit, cgroupKernelMemoryLimit, err)
+		return err
 	}
 	return nil
 }

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/memory.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -18,16 +17,8 @@ import (
 )
 
 const (
-	numaNodeSymbol            = "N"
-	numaStatColumnSeparator   = " "
-	numaStatKeyValueSeparator = "="
-	numaStatMaxColumns        = math.MaxUint8 + 1
-	numaStatValueIndex        = 1
-	numaStatTypeIndex         = 0
-	numaStatColumnSliceLength = 2
-	cgroupMemorySwapLimit     = "memory.memsw.limit_in_bytes"
-	cgroupMemoryLimit         = "memory.limit_in_bytes"
-	cgroupMemoryPagesByNuma   = "memory.numa_stat"
+	cgroupMemorySwapLimit = "memory.memsw.limit_in_bytes"
+	cgroupMemoryLimit     = "memory.limit_in_bytes"
 )
 
 type MemoryGroup struct {
@@ -160,7 +151,7 @@ func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error {
 
 func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
 	// Set stats from memory.stat.
-	statsFile, err := os.Open(filepath.Join(path, "memory.stat"))
+	statsFile, err := fscommon.OpenFile(path, "memory.stat", os.O_RDONLY)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil
@@ -200,8 +191,7 @@ func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
 	}
 	stats.MemoryStats.KernelTCPUsage = kernelTCPUsage
 
-	useHierarchy := strings.Join([]string{"memory", "use_hierarchy"}, ".")
-	value, err := fscommon.GetCgroupParamUint(path, useHierarchy)
+	value, err := fscommon.GetCgroupParamUint(path, "memory.use_hierarchy")
 	if err != nil {
 		return err
 	}
@@ -233,12 +223,14 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) {
 
 	moduleName := "memory"
 	if name != "" {
-		moduleName = strings.Join([]string{"memory", name}, ".")
+		moduleName = "memory." + name
 	}
-	usage := strings.Join([]string{moduleName, "usage_in_bytes"}, ".")
-	maxUsage := strings.Join([]string{moduleName, "max_usage_in_bytes"}, ".")
-	failcnt := strings.Join([]string{moduleName, "failcnt"}, ".")
-	limit := strings.Join([]string{moduleName, "limit_in_bytes"}, ".")
+	var (
+		usage    = moduleName + ".usage_in_bytes"
+		maxUsage = moduleName + ".max_usage_in_bytes"
+		failcnt  = moduleName + ".failcnt"
+		limit    = moduleName + ".limit_in_bytes"
+	)
 
 	value, err := fscommon.GetCgroupParamUint(path, usage)
 	if err != nil {
@@ -277,47 +269,81 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) {
 }
 
 func getPageUsageByNUMA(cgroupPath string) (cgroups.PageUsageByNUMA, error) {
+	const (
+		maxColumns = math.MaxUint8 + 1
+		filename   = "memory.numa_stat"
+	)
 	stats := cgroups.PageUsageByNUMA{}
 
-	file, err := os.Open(path.Join(cgroupPath, cgroupMemoryPagesByNuma))
+	file, err := fscommon.OpenFile(cgroupPath, filename, os.O_RDONLY)
 	if os.IsNotExist(err) {
 		return stats, nil
 	} else if err != nil {
 		return stats, err
 	}
 
+	// File format is documented in linux/Documentation/cgroup-v1/memory.txt
+	// and it looks like this:
+	//
+	// total=<total pages> N0=<node 0 pages> N1=<node 1 pages> ...
+	// file=<total file pages> N0=<node 0 pages> N1=<node 1 pages> ...
+	// anon=<total anon pages> N0=<node 0 pages> N1=<node 1 pages> ...
+	// unevictable=<total anon pages> N0=<node 0 pages> N1=<node 1 pages> ...
+	// hierarchical_<counter>=<counter pages> N0=<node 0 pages> N1=<node 1 pages> ...
+
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
-		var statsType string
-		statsByType := cgroups.PageStats{Nodes: map[uint8]uint64{}}
-		columns := strings.SplitN(scanner.Text(), numaStatColumnSeparator, numaStatMaxColumns)
+		var field *cgroups.PageStats
 
-		for _, column := range columns {
-			pagesByNode := strings.SplitN(column, numaStatKeyValueSeparator, numaStatColumnSliceLength)
+		line := scanner.Text()
+		columns := strings.SplitN(line, " ", maxColumns)
+		for i, column := range columns {
+			byNode := strings.SplitN(column, "=", 2)
+			// Some custom kernels have non-standard fields, like
+			//   numa_locality 0 0 0 0 0 0 0 0 0 0
+			//   numa_exectime 0
+			if len(byNode) < 2 {
+				if i == 0 {
+					// Ignore/skip those.
+					break
+				} else {
+					// The first column was already validated,
+					// so be strict to the rest.
+					return stats, fmt.Errorf("malformed line %q in %s",
+						line, filename)
+				}
+			}
+			key, val := byNode[0], byNode[1]
+			if i == 0 { // First column: key is name, val is total.
+				field = getNUMAField(&stats, key)
+				if field == nil { // unknown field (new kernel?)
+					break
+				}
+				field.Total, err = strconv.ParseUint(val, 0, 64)
+				if err != nil {
+					return stats, err
+				}
+				field.Nodes = map[uint8]uint64{}
+			} else { // Subsequent columns: key is N<id>, val is usage.
+				if len(key) < 2 || key[0] != 'N' {
+					// This is definitely an error.
+					return stats, fmt.Errorf("malformed line %q in %s",
+						line, filename)
+				}
 
-			if strings.HasPrefix(pagesByNode[numaStatTypeIndex], numaNodeSymbol) {
-				nodeID, err := strconv.ParseUint(pagesByNode[numaStatTypeIndex][1:], 10, 8)
+				n, err := strconv.ParseUint(key[1:], 10, 8)
 				if err != nil {
 					return cgroups.PageUsageByNUMA{}, err
 				}
 
-				statsByType.Nodes[uint8(nodeID)], err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
-				if err != nil {
-					return cgroups.PageUsageByNUMA{}, err
-				}
-			} else {
-				statsByType.Total, err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
+				usage, err := strconv.ParseUint(val, 10, 64)
 				if err != nil {
 					return cgroups.PageUsageByNUMA{}, err
 				}
 
-				statsType = pagesByNode[numaStatTypeIndex]
+				field.Nodes[uint8(n)] = usage
 			}
 
-			err := addNUMAStatsByType(&stats, statsByType, statsType)
-			if err != nil {
-				return cgroups.PageUsageByNUMA{}, err
-			}
 		}
 	}
 	err = scanner.Err()
@@ -328,26 +354,24 @@ func getPageUsageByNUMA(cgroupPath string) (cgroups.PageUsageByNUMA, error) {
 	return stats, nil
 }
 
-func addNUMAStatsByType(stats *cgroups.PageUsageByNUMA, byTypeStats cgroups.PageStats, statsType string) error {
-	switch statsType {
+func getNUMAField(stats *cgroups.PageUsageByNUMA, name string) *cgroups.PageStats {
+	switch name {
 	case "total":
-		stats.Total = byTypeStats
+		return &stats.Total
 	case "file":
-		stats.File = byTypeStats
+		return &stats.File
 	case "anon":
-		stats.Anon = byTypeStats
+		return &stats.Anon
 	case "unevictable":
-		stats.Unevictable = byTypeStats
+		return &stats.Unevictable
 	case "hierarchical_total":
-		stats.Hierarchical.Total = byTypeStats
+		return &stats.Hierarchical.Total
 	case "hierarchical_file":
-		stats.Hierarchical.File = byTypeStats
+		return &stats.Hierarchical.File
 	case "hierarchical_anon":
-		stats.Hierarchical.Anon = byTypeStats
+		return &stats.Hierarchical.Anon
 	case "hierarchical_unevictable":
-		stats.Hierarchical.Unevictable = byTypeStats
-	default:
-		return fmt.Errorf("unsupported NUMA page type found: %s", statsType)
+		return &stats.Hierarchical.Unevictable
 	}
 	return nil
 }

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/name.go

@@ -19,7 +19,7 @@ func (s *NameGroup) Name() string {
 func (s *NameGroup) Apply(path string, d *cgroupData) error {
 	if s.Join {
 		// ignore errors if the named cgroup does not exist
-		join(path, d.pid)
+		_ = join(path, d.pid)
 	}
 	return nil
 }