github/kata-containers
1
0
Fork 2
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-10-21 20:08:54 +00:00
Code Issues Projects Releases Wiki Activity

cgroups: add host cgroup support

Browse Source 
Fixes #344

Add host cgroup support for kata.

This commits only adds cpu.cfs_period and cpu.cfs_quota support.

It will create 3-level hierarchy, take "cpu" cgroup as an example:

```
/sys/fs/cgroup
|---cpu
   |---kata
      |---<sandbox-id>
         |--vcpu
      |---<sandbox-id>
```

* `vc` cgroup is common parent for all kata-container sandbox, it won't be removed
after sandbox removed. This cgroup has no limitation.
* `<sandbox-id>` cgroup is the layer for each sandbox, it contains all other qemu
threads except for vcpu threads. In future, we can consider putting all shim
processes and proxy process here. This cgroup has no limitation yet.
* `vcpu` cgroup contains vcpu threads from qemu. Currently cpu quota and period
constraint applies to this cgroup.

Signed-off-by: Wei Zhang <zhangwei555@huawei.com>
Signed-off-by: Jingxiao Lu <lujingxiao@huawei.com>
This commit is contained in:
Wei Zhang
2018-09-13 20:10:29 +08:00
parent 523d49c076
commit 34fe3b9d6d
13 changed files with 468 additions and 586 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
cli/create.go
View File

@@ -10,9 +10,7 @@ import (
"context"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
vc "github.com/kata-containers/runtime/virtcontainers"
@@ -177,26 +175,6 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s
}
}
// config.json provides a cgroups path that has to be used to create "tasks"
// and "cgroups.procs" files. Those files have to be filled with a PID, which
// is shim's in our case. This is mandatory to make sure there is no one
// else (like Docker) trying to create those files on our behalf. We want to
// know those files location so that we can remove them when delete is called.
cgroupsPathList, err := processCgroupsPath(ctx, ociSpec, containerType.IsSandbox())
if err != nil {
return err
}
// cgroupsDirPath is CgroupsPath fetch from OCI spec
var cgroupsDirPath string
if ociSpec.Linux != nil {
cgroupsDirPath = ociSpec.Linux.CgroupsPath
}
if err := createCgroupsFiles(ctx, containerID, cgroupsDirPath, cgroupsPathList, process.Pid); err != nil {
return err
}
// Creation of PID file has to be the last thing done in the create
// because containerd considers the create complete after this file
// is created.
@@ -379,52 +357,6 @@ func createContainer(ctx context.Context, ociSpec oci.CompatOCISpec, containerID
return c.Process(), nil
}
func createCgroupsFiles(ctx context.Context, containerID string, cgroupsDirPath string, cgroupsPathList []string, pid int) error {
span, _ := trace(ctx, "createCgroupsFiles")
defer span.Finish()
if len(cgroupsPathList) == 0 {
kataLog.WithField("pid", pid).Info("Cgroups files not created because cgroupsPath was empty")
return nil
}
for _, cgroupsPath := range cgroupsPathList {
if err := os.MkdirAll(cgroupsPath, cgroupsDirMode); err != nil {
return err
}
if strings.Contains(cgroupsPath, "cpu") && cgroupsDirPath != "" {
parent := strings.TrimSuffix(cgroupsPath, cgroupsDirPath)
copyParentCPUSet(cgroupsPath, parent)
}
tasksFilePath := filepath.Join(cgroupsPath, cgroupsTasksFile)
procsFilePath := filepath.Join(cgroupsPath, cgroupsProcsFile)
pidStr := fmt.Sprintf("%d", pid)
for _, path := range []string{tasksFilePath, procsFilePath} {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, cgroupsFileMode)
if err != nil {
return err
}
defer f.Close()
n, err := f.WriteString(pidStr)
if err != nil {
return err
}
if n < len(pidStr) {
return fmt.Errorf("Could not write pid to %q: only %d bytes written out of %d",
path, n, len(pidStr))
}
}
}
return nil
}
func createPIDFile(ctx context.Context, pidFilePath string, pid int) error {
span, _ := trace(ctx, "createPIDFile")
defer span.Finish()
@@ -457,48 +389,3 @@ func createPIDFile(ctx context.Context, pidFilePath string, pid int) error {
return nil
}
// copyParentCPUSet copies the cpuset.cpus and cpuset.mems from the parent
// directory to the current directory if the file's contents are 0
func copyParentCPUSet(current, parent string) error {
currentCpus, currentMems, err := getCPUSet(current)
if err != nil {
return err
}
parentCpus, parentMems, err := getCPUSet(parent)
if err != nil {
return err
}
if len(parentCpus) < 1 || len(parentMems) < 1 {
return nil
}
var cgroupsFileMode = os.FileMode(0600)
if isEmptyString(currentCpus) {
if err := writeFile(filepath.Join(current, "cpuset.cpus"), string(parentCpus), cgroupsFileMode); err != nil {
return err
}
}
if isEmptyString(currentMems) {
if err := writeFile(filepath.Join(current, "cpuset.mems"), string(parentMems), cgroupsFileMode); err != nil {
return err
}
}
return nil
}
func getCPUSet(parent string) (cpus []byte, mems []byte, err error) {
if cpus, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.cpus")); err != nil {
return
}
if mems, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.mems")); err != nil {
return
}
return cpus, mems, nil
}

122
cli/create_test.go
View File

@@ -36,22 +36,6 @@ const (
var testStrPID = fmt.Sprintf("%d", testPID)
func mockCPUSetContent(contents map[string]string) error {
for filePath, data := range contents {
if err := writeFile(filePath, data, testFileMode); err != nil {
return err
}
}
return nil
}
func testCreateCgroupsFilesSuccessful(t *testing.T, cgroupsDirPath string, cgroupsPathList []string, pid int) {
if err := createCgroupsFiles(context.Background(), "foo", cgroupsDirPath, cgroupsPathList, pid); err != nil {
t.Fatalf("This test should succeed (cgroupsPath %q, pid %d): %s", cgroupsPathList, pid, err)
}
}
// return the value of the *last* param with the specified key
func findLastParam(key string, params []vc.Param) (string, error) {
if key == "" {
@@ -74,62 +58,6 @@ func findLastParam(key string, params []vc.Param) (string, error) {
return "", fmt.Errorf("no param called %q found", name)
}
func TestCgroupsFilesEmptyCgroupsPathSuccessful(t *testing.T) {
testCreateCgroupsFilesSuccessful(t, "", []string{}, testPID)
}
func TestCreateCgroupsFilesFailToWriteFile(t *testing.T) {
if os.Geteuid() == 0 {
// The os.FileMode(0000) trick doesn't work for root.
t.Skip(testDisabledNeedNonRoot)
}
assert := assert.New(t)
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
// create the file as a directory to force an error
file := filepath.Join(tmpdir, "cgroups-file")
err = os.MkdirAll(file, os.FileMode(0000))
assert.NoError(err)
files := []string{file}
err = createCgroupsFiles(context.Background(), "foo", "cgroups-file", files, testPID)
assert.Error(err)
}
func TestCgroupsFilesNonEmptyCgroupsPathSuccessful(t *testing.T) {
cgroupsPath, err := ioutil.TempDir(testDir, "cgroups-path-")
if err != nil {
t.Fatalf("Could not create temporary cgroups directory: %s", err)
}
testCreateCgroupsFilesSuccessful(t, "cgroups-path-", []string{cgroupsPath}, testPID)
defer os.RemoveAll(cgroupsPath)
tasksPath := filepath.Join(cgroupsPath, cgroupsTasksFile)
procsPath := filepath.Join(cgroupsPath, cgroupsProcsFile)
for _, path := range []string{tasksPath, procsPath} {
if _, err := os.Stat(path); err != nil {
t.Fatalf("Path %q should have been created: %s", path, err)
}
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("Could not read %q previously created: %s", path, err)
}
if string(fileBytes) != testStrPID {
t.Fatalf("PID %s read from %q different from expected PID %s", string(fileBytes), path, testStrPID)
}
}
}
func TestCreatePIDFileSuccessful(t *testing.T) {
pidDirPath, err := ioutil.TempDir(testDir, "pid-path-")
if err != nil {
@@ -1087,56 +1015,6 @@ func TestCreateCreateContainer(t *testing.T) {
}
}
func TestCopyParentCPUSetFail(t *testing.T) {
assert := assert.New(t)
cgroupsPath, err := ioutil.TempDir(testDir, "cgroups-path-")
assert.NoError(err)
defer os.RemoveAll(cgroupsPath)
err = copyParentCPUSet(cgroupsPath, testDir)
assert.Error(err)
}
func TestCopyParentCPUSetSuccessful(t *testing.T) {
assert := assert.New(t)
cgroupsPath, err := ioutil.TempDir(testDir, "cgroups-path-")
assert.NoError(err)
defer os.RemoveAll(cgroupsPath)
cgroupsSrcPath := filepath.Join(cgroupsPath, "src")
err = os.Mkdir(cgroupsSrcPath, testDirMode)
assert.NoError(err)
err = mockCPUSetContent(map[string]string{
filepath.Join(cgroupsSrcPath, "cpuset.cpus"): "0-1",
filepath.Join(cgroupsSrcPath, "cpuset.mems"): "0-1",
})
assert.NoError(err)
cgroupsDstPath := filepath.Join(cgroupsPath, "dst")
err = os.Mkdir(cgroupsDstPath, testDirMode)
assert.NoError(err)
fd, err := os.Create(filepath.Join(cgroupsDstPath, "cpuset.cpus"))
assert.NoError(err)
fd.Close()
fd, err = os.Create(filepath.Join(cgroupsDstPath, "cpuset.mems"))
assert.NoError(err)
fd.Close()
err = copyParentCPUSet(cgroupsDstPath, cgroupsSrcPath)
assert.NoError(err)
currentCpus, currentMems, err := getCPUSet(cgroupsDstPath)
assert.NoError(err)
assert.False(isEmptyString(currentCpus))
assert.False(isEmptyString(currentMems))
}
func TestSetKernelParams(t *testing.T) {
assert := assert.New(t)

14
cli/delete.go
View File

@@ -123,19 +123,7 @@ func delete(ctx context.Context, containerID string, force bool) error {
return err
}
// In order to prevent any file descriptor leak related to cgroups files
// that have been previously created, we have to remove them before this
// function returns.
cgroupsPathList, err := processCgroupsPath(ctx, ociSpec, containerType.IsSandbox())
if err != nil {
return err
}
if err := delContainerIDMapping(ctx, containerID); err != nil {
return err
}
return removeCgroupsPath(ctx, containerID, cgroupsPathList)
return delContainerIDMapping(ctx, containerID)
}
func deleteSandbox(ctx context.Context, sandboxID string) error {

132
cli/oci.go
View File

@@ -8,7 +8,6 @@ package main
import (
"bufio"
"context"
"errors"
"fmt"
"io/ioutil"
"net"
@@ -20,18 +19,11 @@ import (
"github.com/containernetworking/plugins/pkg/ns"
vc "github.com/kata-containers/runtime/virtcontainers"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
"github.com/opencontainers/runc/libcontainer/utils"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/sirupsen/logrus"
)
// Contants related to cgroup memory directory
const (
cgroupsTasksFile = "tasks"
cgroupsProcsFile = "cgroup.procs"
cgroupsDirMode = os.FileMode(0750)
cgroupsFileMode = os.FileMode(0640)
ctrsMappingDirMode = os.FileMode(0750)
// Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed
@@ -39,8 +31,6 @@ const (
cgroupFsType = 0x27e0eb
)
var errNeedLinuxResource = errors.New("Linux resource cannot be empty")
var cgroupsDirPath string
var procMountInfo = "/proc/self/mountinfo"
@@ -125,128 +115,6 @@ func validCreateParams(ctx context.Context, containerID, bundlePath string) (str
return resolved, nil
}
// processCgroupsPath process the cgroups path as expected from the
// OCI runtime specification. It returns a list of complete paths
// that should be created and used for every specified resource.
func processCgroupsPath(ctx context.Context, ociSpec oci.CompatOCISpec, isSandbox bool) ([]string, error) {
span, _ := trace(ctx, "processCgroupsPath")
defer span.Finish()
var cgroupsPathList []string
if ociSpec.Linux.CgroupsPath == "" {
return []string{}, nil
}
if ociSpec.Linux.Resources == nil {
return []string{}, nil
}
if ociSpec.Linux.Resources.Memory != nil {
memCgroupsPath, err := processCgroupsPathForResource(ociSpec, "memory", isSandbox)
if err != nil {
return []string{}, err
}
if memCgroupsPath != "" {
cgroupsPathList = append(cgroupsPathList, memCgroupsPath)
}
}
if ociSpec.Linux.Resources.CPU != nil {
cpuCgroupsPath, err := processCgroupsPathForResource(ociSpec, "cpu", isSandbox)
if err != nil {
return []string{}, err
}
if cpuCgroupsPath != "" {
cgroupsPathList = append(cgroupsPathList, cpuCgroupsPath)
}
}
if ociSpec.Linux.Resources.Pids != nil {
pidsCgroupsPath, err := processCgroupsPathForResource(ociSpec, "pids", isSandbox)
if err != nil {
return []string{}, err
}
if pidsCgroupsPath != "" {
cgroupsPathList = append(cgroupsPathList, pidsCgroupsPath)
}
}
if ociSpec.Linux.Resources.BlockIO != nil {
blkIOCgroupsPath, err := processCgroupsPathForResource(ociSpec, "blkio", isSandbox)
if err != nil {
return []string{}, err
}
if blkIOCgroupsPath != "" {
cgroupsPathList = append(cgroupsPathList, blkIOCgroupsPath)
}
}
return cgroupsPathList, nil
}
func processCgroupsPathForResource(ociSpec oci.CompatOCISpec, resource string, isSandbox bool) (string, error) {
if resource == "" {
return "", errNeedLinuxResource
}
var err error
cgroupsDirPath, err = getCgroupsDirPath(procMountInfo)
if err != nil {
return "", fmt.Errorf("get CgroupsDirPath error: %s", err)
}
// Relative cgroups path provided.
if filepath.IsAbs(ociSpec.Linux.CgroupsPath) == false {
return filepath.Join(cgroupsDirPath, resource, ociSpec.Linux.CgroupsPath), nil
}
// Absolute cgroups path provided.
var cgroupMount specs.Mount
cgroupMountFound := false
for _, mount := range ociSpec.Mounts {
if mount.Type == "cgroup" {
cgroupMount = mount
cgroupMountFound = true
break
}
}
if !cgroupMountFound {
// According to the OCI spec, an absolute path should be
// interpreted as relative to the system cgroup mount point
// when there is no cgroup mount point.
return filepath.Join(cgroupsDirPath, resource, ociSpec.Linux.CgroupsPath), nil
}
if cgroupMount.Destination == "" {
return "", fmt.Errorf("cgroupsPath is absolute, cgroup mount destination cannot be empty")
}
cgroupPath := filepath.Join(cgroupMount.Destination, resource)
// It is not an error to have this cgroup not mounted. It is usually
// due to an old kernel version with missing support for specific
// cgroups.
fields := logrus.Fields{
"path": cgroupPath,
"type": "cgroup",
}
if !isCgroupMounted(cgroupPath) {
kataLog.WithFields(fields).Info("path not mounted")
return "", nil
}
kataLog.WithFields(fields).Info("path mounted")
return filepath.Join(cgroupPath, ociSpec.Linux.CgroupsPath), nil
}
func isCgroupMounted(cgroupPath string) bool {
var statFs syscall.Statfs_t

196
cli/oci_test.go
View File

@@ -13,17 +13,13 @@ import (
"net"
"os"
"path/filepath"
"reflect"
"syscall"
"testing"
"time"
vc "github.com/kata-containers/runtime/virtcontainers"
vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations"
"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
"github.com/kata-containers/runtime/virtcontainers/pkg/vcmock"
"github.com/opencontainers/runc/libcontainer/utils"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
)
@@ -32,38 +28,6 @@ var (
consoleSocketPathTest = "console-socket-test"
)
type cgroupTestDataType struct {
resource string
linuxSpec *specs.LinuxResources
}
var cgroupTestData = []cgroupTestDataType{
{
"memory",
&specs.LinuxResources{
Memory: &specs.LinuxMemory{},
},
},
{
"cpu",
&specs.LinuxResources{
CPU: &specs.LinuxCPU{},
},
},
{
"pids",
&specs.LinuxResources{
Pids: &specs.LinuxPids{},
},
},
{
"blkio",
&specs.LinuxResources{
BlockIO: &specs.LinuxBlockIO{},
},
},
}
func TestGetContainerInfoContainerIDEmptyFailure(t *testing.T) {
assert := assert.New(t)
status, _, err := getContainerInfo(context.Background(), "")
@@ -181,141 +145,6 @@ func TestValidCreateParamsBundleIsAFile(t *testing.T) {
assert.False(vcmock.IsMockError(err))
}
func testProcessCgroupsPath(t *testing.T, ociSpec oci.CompatOCISpec, expected []string) {
assert := assert.New(t)
result, err := processCgroupsPath(context.Background(), ociSpec, true)
assert.NoError(err)
if reflect.DeepEqual(result, expected) == false {
assert.FailNow("DeepEqual failed", "Result path %q should match the expected one %q", result, expected)
}
}
func TestProcessCgroupsPathEmptyPathSuccessful(t *testing.T) {
ociSpec := oci.CompatOCISpec{}
ociSpec.Linux = &specs.Linux{
CgroupsPath: "",
}
testProcessCgroupsPath(t, ociSpec, []string{})
}
func TestProcessCgroupsPathEmptyResources(t *testing.T) {
ociSpec := oci.CompatOCISpec{}
ociSpec.Linux = &specs.Linux{
CgroupsPath: "foo",
}
testProcessCgroupsPath(t, ociSpec, []string{})
}
func TestProcessCgroupsPathRelativePathSuccessful(t *testing.T) {
relativeCgroupsPath := "relative/cgroups/path"
cgroupsDirPath = "/foo/runtime/base"
ociSpec := oci.CompatOCISpec{}
ociSpec.Linux = &specs.Linux{
CgroupsPath: relativeCgroupsPath,
}
for _, d := range cgroupTestData {
ociSpec.Linux.Resources = d.linuxSpec
p := filepath.Join(cgroupsDirPath, d.resource, relativeCgroupsPath)
testProcessCgroupsPath(t, ociSpec, []string{p})
}
}
func TestProcessCgroupsPathAbsoluteNoCgroupMountSuccessful(t *testing.T) {
absoluteCgroupsPath := "/absolute/cgroups/path"
cgroupsDirPath = "/foo/runtime/base"
ociSpec := oci.CompatOCISpec{}
ociSpec.Linux = &specs.Linux{
CgroupsPath: absoluteCgroupsPath,
}
for _, d := range cgroupTestData {
ociSpec.Linux.Resources = d.linuxSpec
p := filepath.Join(cgroupsDirPath, d.resource, absoluteCgroupsPath)
testProcessCgroupsPath(t, ociSpec, []string{p})
}
}
func TestProcessCgroupsPathAbsoluteNoCgroupMountDestinationFailure(t *testing.T) {
assert := assert.New(t)
absoluteCgroupsPath := "/absolute/cgroups/path"
ociSpec := oci.CompatOCISpec{}
ociSpec.Mounts = []specs.Mount{
{
Type: "cgroup",
},
}
ociSpec.Linux = &specs.Linux{
CgroupsPath: absoluteCgroupsPath,
}
for _, d := range cgroupTestData {
ociSpec.Linux.Resources = d.linuxSpec
for _, isSandbox := range []bool{true, false} {
_, err := processCgroupsPath(context.Background(), ociSpec, isSandbox)
assert.Error(err, "This test should fail because no cgroup mount destination provided")
}
}
}
func TestProcessCgroupsPathAbsoluteSuccessful(t *testing.T) {
assert := assert.New(t)
if os.Geteuid() != 0 {
t.Skip(testDisabledNeedRoot)
}
memoryResource := "memory"
absoluteCgroupsPath := "/cgroup/mount/destination"
cgroupMountDest, err := ioutil.TempDir("", "cgroup-memory-")
assert.NoError(err)
defer os.RemoveAll(cgroupMountDest)
resourceMountPath := filepath.Join(cgroupMountDest, memoryResource)
err = os.MkdirAll(resourceMountPath, cgroupsDirMode)
assert.NoError(err)
err = syscall.Mount("go-test", resourceMountPath, "cgroup", 0, memoryResource)
assert.NoError(err)
defer syscall.Unmount(resourceMountPath, 0)
ociSpec := oci.CompatOCISpec{}
ociSpec.Linux = &specs.Linux{
Resources: &specs.LinuxResources{
Memory: &specs.LinuxMemory{},
},
CgroupsPath: absoluteCgroupsPath,
}
ociSpec.Mounts = []specs.Mount{
{
Type: "cgroup",
Destination: cgroupMountDest,
},
}
testProcessCgroupsPath(t, ociSpec, []string{filepath.Join(resourceMountPath, absoluteCgroupsPath)})
}
func TestSetupConsoleExistingConsolePathSuccessful(t *testing.T) {
assert := assert.New(t)
console, err := setupConsole(consolePathTest, "")
@@ -436,31 +265,6 @@ func TestIsCgroupMounted(t *testing.T) {
assert.True(isCgroupMounted(memoryCgroupPath), "%s is a cgroup", memoryCgroupPath)
}
func TestProcessCgroupsPathForResource(t *testing.T) {
assert := assert.New(t)
tmpdir, err := ioutil.TempDir("", "")
assert.NoError(err)
defer os.RemoveAll(tmpdir)
bundlePath := filepath.Join(tmpdir, "bundle")
err = makeOCIBundle(bundlePath)
assert.NoError(err)
ociConfigFile := filepath.Join(bundlePath, specConfig)
assert.True(fileExists(ociConfigFile))
spec, err := readOCIConfigFile(ociConfigFile)
assert.NoError(err)
for _, isSandbox := range []bool{true, false} {
_, err := processCgroupsPathForResource(spec, "", isSandbox)
assert.Error(err)
assert.False(vcmock.IsMockError(err))
}
}
func TestGetCgroupsDirPath(t *testing.T) {
assert := assert.New(t)

6
cli/utils.go
View File

@@ -6,7 +6,6 @@
package main
import (
"bytes"
"fmt"
"io/ioutil"
"os"
@@ -227,11 +226,6 @@ func writeFile(filePath string, data string, fileMode os.FileMode) error {
return nil
}
// isEmptyString return if string is empty
func isEmptyString(b []byte) bool {
return len(bytes.Trim(b, "\n")) == 0
}
// fileSize returns the number of bytes in the specified file
func fileSize(file string) (int64, error) {
st := syscall.Stat_t{}