Merge pull request #933 from egernst/cgroup-updates

cgroup and cpuset fixes from 1.x

src/runtime/go.sum

@@ -198,6 +198,7 @@ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0b
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kata-containers/kata-containers v0.0.0-20201013034856-c88820454d08 h1:yk9fzLKb9RmV9xuT5mkJw4owk/K0rX5cusm2ukEEDro=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=

src/runtime/virtcontainers/container.go

@@ -1139,6 +1139,12 @@ func (c *Container) update(resources specs.LinuxResources) error {
 		if q := cpu.Quota; q != nil && *q != 0 {
 			c.config.Resources.CPU.Quota = q
 		}
+		if cpu.Cpus != "" {
+			c.config.Resources.CPU.Cpus = cpu.Cpus
+		}
+		if cpu.Mems != "" {
+			c.config.Resources.CPU.Mems = cpu.Mems
+		}
 	}
 
 	if c.config.Resources.Memory == nil {
@@ -1159,6 +1165,14 @@ func (c *Container) update(resources specs.LinuxResources) error {
 		}
 	}
 
+	// There currently isn't a notion of cpusets.cpus or mems being tracked
+	// inside of the guest. Make sure we clear these before asking agent to update
+	// the container's cgroups.
+	if resources.CPU != nil {
+		resources.CPU.Mems = ""
+		resources.CPU.Cpus = ""
+	}
+
 	return c.sandbox.agent.updateContainer(c.sandbox, *c, resources)
 }
 

src/runtime/virtcontainers/pkg/cgroups/manager.go

@@ -331,3 +331,17 @@ func (m *Manager) RemoveDevice(device string) error {
 	m.Unlock()
 	return fmt.Errorf("device %v not found in the cgroup", device)
 }
+
+func (m *Manager) SetCPUSet(cpuset, memset string) error {
+	cgroups, err := m.GetCgroups()
+	if err != nil {
+		return err
+	}
+
+	m.Lock()
+	cgroups.CpusetCpus = cpuset
+	cgroups.CpusetMems = memset
+	m.Unlock()
+
+	return m.Apply()
+}

src/runtime/virtcontainers/pkg/cpuset/cpuset.go

@@ -0,0 +1,296 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+// Copyright (c) 2017 The Kubernetes Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package cpuset
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// Builder is a mutable builder for CPUSet. Functions that mutate instances
+// of this type are not thread-safe.
+type Builder struct {
+	result CPUSet
+	done   bool
+}
+
+// NewBuilder returns a mutable CPUSet builder.
+func NewBuilder() Builder {
+	return Builder{
+		result: CPUSet{
+			elems: map[int]struct{}{},
+		},
+	}
+}
+
+// Add adds the supplied elements to the result. Calling Add after calling
+// Result has no effect.
+func (b Builder) Add(elems ...int) {
+	if b.done {
+		return
+	}
+	for _, elem := range elems {
+		b.result.elems[elem] = struct{}{}
+	}
+}
+
+// Result returns the result CPUSet containing all elements that were
+// previously added to this builder. Subsequent calls to Add have no effect.
+func (b Builder) Result() CPUSet {
+	b.done = true
+	return b.result
+}
+
+// CPUSet is a thread-safe, immutable set-like data structure for CPU IDs.
+type CPUSet struct {
+	elems map[int]struct{}
+}
+
+// NewCPUSet returns a new CPUSet containing the supplied elements.
+func NewCPUSet(cpus ...int) CPUSet {
+	b := NewBuilder()
+	for _, c := range cpus {
+		b.Add(c)
+	}
+	return b.Result()
+}
+
+// Size returns the number of elements in this set.
+func (s CPUSet) Size() int {
+	return len(s.elems)
+}
+
+// IsEmpty returns true if there are zero elements in this set.
+func (s CPUSet) IsEmpty() bool {
+	return s.Size() == 0
+}
+
+// Contains returns true if the supplied element is present in this set.
+func (s CPUSet) Contains(cpu int) bool {
+	_, found := s.elems[cpu]
+	return found
+}
+
+// Equals returns true if the supplied set contains exactly the same elements
+// as this set (s IsSubsetOf s2 and s2 IsSubsetOf s).
+func (s CPUSet) Equals(s2 CPUSet) bool {
+	return reflect.DeepEqual(s.elems, s2.elems)
+}
+
+// Filter returns a new CPU set that contains all of the elements from this
+// set that match the supplied predicate, without mutating the source set.
+func (s CPUSet) Filter(predicate func(int) bool) CPUSet {
+	b := NewBuilder()
+	for cpu := range s.elems {
+		if predicate(cpu) {
+			b.Add(cpu)
+		}
+	}
+	return b.Result()
+}
+
+// FilterNot returns a new CPU set that contains all of the elements from this
+// set that do not match the supplied predicate, without mutating the source
+// set.
+func (s CPUSet) FilterNot(predicate func(int) bool) CPUSet {
+	b := NewBuilder()
+	for cpu := range s.elems {
+		if !predicate(cpu) {
+			b.Add(cpu)
+		}
+	}
+	return b.Result()
+}
+
+// IsSubsetOf returns true if the supplied set contains all the elements
+func (s CPUSet) IsSubsetOf(s2 CPUSet) bool {
+	result := true
+	for cpu := range s.elems {
+		if !s2.Contains(cpu) {
+			result = false
+			break
+		}
+	}
+	return result
+}
+
+// Union returns a new CPU set that contains all of the elements from this
+// set and all of the elements from the supplied set, without mutating
+// either source set.
+func (s CPUSet) Union(s2 CPUSet) CPUSet {
+	b := NewBuilder()
+	for cpu := range s.elems {
+		b.Add(cpu)
+	}
+	for cpu := range s2.elems {
+		b.Add(cpu)
+	}
+	return b.Result()
+}
+
+// UnionAll returns a new CPU set that contains all of the elements from this
+// set and all of the elements from the supplied sets, without mutating
+// either source set.
+func (s CPUSet) UnionAll(s2 []CPUSet) CPUSet {
+	b := NewBuilder()
+	for cpu := range s.elems {
+		b.Add(cpu)
+	}
+	for _, cs := range s2 {
+		for cpu := range cs.elems {
+			b.Add(cpu)
+		}
+	}
+	return b.Result()
+}
+
+// Intersection returns a new CPU set that contains all of the elements
+// that are present in both this set and the supplied set, without mutating
+// either source set.
+func (s CPUSet) Intersection(s2 CPUSet) CPUSet {
+	return s.Filter(func(cpu int) bool { return s2.Contains(cpu) })
+}
+
+// Difference returns a new CPU set that contains all of the elements that
+// are present in this set and not the supplied set, without mutating either
+// source set.
+func (s CPUSet) Difference(s2 CPUSet) CPUSet {
+	return s.FilterNot(func(cpu int) bool { return s2.Contains(cpu) })
+}
+
+// ToSlice returns a slice of integers that contains all elements from
+// this set.
+func (s CPUSet) ToSlice() []int {
+	result := []int{}
+	for cpu := range s.elems {
+		result = append(result, cpu)
+	}
+	sort.Ints(result)
+	return result
+}
+
+// ToSliceNoSort returns a slice of integers that contains all elements from
+// this set.
+func (s CPUSet) ToSliceNoSort() []int {
+	result := []int{}
+	for cpu := range s.elems {
+		result = append(result, cpu)
+	}
+	return result
+}
+
+// String returns a new string representation of the elements in this CPU set
+// in canonical linux CPU list format.
+//
+// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS
+func (s CPUSet) String() string {
+	if s.IsEmpty() {
+		return ""
+	}
+
+	elems := s.ToSlice()
+
+	type rng struct {
+		start int
+		end   int
+	}
+
+	ranges := []rng{{elems[0], elems[0]}}
+
+	for i := 1; i < len(elems); i++ {
+		lastRange := &ranges[len(ranges)-1]
+		// if this element is adjacent to the high end of the last range
+		if elems[i] == lastRange.end+1 {
+			// then extend the last range to include this element
+			lastRange.end = elems[i]
+			continue
+		}
+		// otherwise, start a new range beginning with this element
+		ranges = append(ranges, rng{elems[i], elems[i]})
+	}
+
+	// construct string from ranges
+	var result bytes.Buffer
+	for _, r := range ranges {
+		if r.start == r.end {
+			result.WriteString(strconv.Itoa(r.start))
+		} else {
+			result.WriteString(fmt.Sprintf("%d-%d", r.start, r.end))
+		}
+		result.WriteString(",")
+	}
+	return strings.TrimRight(result.String(), ",")
+}
+
+// Parse CPUSet constructs a new CPU set from a Linux CPU list formatted string.
+//
+// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS
+func Parse(s string) (CPUSet, error) {
+	b := NewBuilder()
+
+	// Handle empty string.
+	if s == "" {
+		return b.Result(), nil
+	}
+
+	// Split CPU list string:
+	// "0-5,34,46-48 => ["0-5", "34", "46-48"]
+	ranges := strings.Split(s, ",")
+
+	for _, r := range ranges {
+		boundaries := strings.Split(r, "-")
+		if len(boundaries) == 1 {
+			// Handle ranges that consist of only one element like "34".
+			elem, err := strconv.Atoi(boundaries[0])
+			if err != nil {
+				return NewCPUSet(), err
+			}
+			b.Add(elem)
+		} else if len(boundaries) == 2 {
+			// Handle multi-element ranges like "0-5".
+			start, err := strconv.Atoi(boundaries[0])
+			if err != nil {
+				return NewCPUSet(), err
+			}
+			end, err := strconv.Atoi(boundaries[1])
+			if err != nil {
+				return NewCPUSet(), err
+			}
+			// Add all elements to the result.
+			// e.g. "0-5", "46-48" => [0, 1, 2, 3, 4, 5, 46, 47, 48].
+			for e := start; e <= end; e++ {
+				b.Add(e)
+			}
+		}
+	}
+	return b.Result(), nil
+}
+
+// Clone returns a copy of this CPU set.
+func (s CPUSet) Clone() CPUSet {
+	b := NewBuilder()
+	for elem := range s.elems {
+		b.Add(elem)
+	}
+	return b.Result()
+}

src/runtime/virtcontainers/pkg/cpuset/cpuset_test.go

@@ -0,0 +1,348 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+// Copyright (c) 2017 The Kubernetes Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package cpuset
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestCPUSetBuilder(t *testing.T) {
+	b := NewBuilder()
+	elems := []int{1, 2, 3, 4, 5}
+	for _, elem := range elems {
+		b.Add(elem)
+	}
+	result := b.Result()
+	for _, elem := range elems {
+		if !result.Contains(elem) {
+			t.Fatalf("expected cpuset to contain element %d: [%v]", elem, result)
+		}
+	}
+	if len(elems) != result.Size() {
+		t.Fatalf("expected cpuset %s to have the same size as %v", result, elems)
+	}
+}
+
+func TestCPUSetSize(t *testing.T) {
+	testCases := []struct {
+		cpuset   CPUSet
+		expected int
+	}{
+		{NewCPUSet(), 0},
+		{NewCPUSet(5), 1},
+		{NewCPUSet(1, 2, 3, 4, 5), 5},
+	}
+
+	for _, c := range testCases {
+		actual := c.cpuset.Size()
+		if actual != c.expected {
+			t.Fatalf("expected: %d, actual: %d, cpuset: [%v]", c.expected, actual, c.cpuset)
+		}
+	}
+}
+
+func TestCPUSetIsEmpty(t *testing.T) {
+	testCases := []struct {
+		cpuset   CPUSet
+		expected bool
+	}{
+		{NewCPUSet(), true},
+		{NewCPUSet(5), false},
+		{NewCPUSet(1, 2, 3, 4, 5), false},
+	}
+
+	for _, c := range testCases {
+		actual := c.cpuset.IsEmpty()
+		if actual != c.expected {
+			t.Fatalf("expected: %t, IsEmpty() returned: %t, cpuset: [%v]", c.expected, actual, c.cpuset)
+		}
+	}
+}
+
+func TestCPUSetContains(t *testing.T) {
+	testCases := []struct {
+		cpuset         CPUSet
+		mustContain    []int
+		mustNotContain []int
+	}{
+		{NewCPUSet(), []int{}, []int{1, 2, 3, 4, 5}},
+		{NewCPUSet(5), []int{5}, []int{1, 2, 3, 4}},
+		{NewCPUSet(1, 2, 4, 5), []int{1, 2, 4, 5}, []int{0, 3, 6}},
+	}
+
+	for _, c := range testCases {
+		for _, elem := range c.mustContain {
+			if !c.cpuset.Contains(elem) {
+				t.Fatalf("expected cpuset to contain element %d: [%v]", elem, c.cpuset)
+			}
+		}
+		for _, elem := range c.mustNotContain {
+			if c.cpuset.Contains(elem) {
+				t.Fatalf("expected cpuset not to contain element %d: [%v]", elem, c.cpuset)
+			}
+		}
+	}
+}
+
+func TestCPUSetEqual(t *testing.T) {
+	shouldEqual := []struct {
+		s1 CPUSet
+		s2 CPUSet
+	}{
+		{NewCPUSet(), NewCPUSet()},
+		{NewCPUSet(5), NewCPUSet(5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+	}
+
+	shouldNotEqual := []struct {
+		s1 CPUSet
+		s2 CPUSet
+	}{
+		{NewCPUSet(), NewCPUSet(5)},
+		{NewCPUSet(5), NewCPUSet()},
+		{NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()},
+		{NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5)},
+	}
+
+	for _, c := range shouldEqual {
+		if !c.s1.Equals(c.s2) {
+			t.Fatalf("expected cpusets to be equal: s1: [%v], s2: [%v]", c.s1, c.s2)
+		}
+	}
+	for _, c := range shouldNotEqual {
+		if c.s1.Equals(c.s2) {
+			t.Fatalf("expected cpusets to not be equal: s1: [%v], s2: [%v]", c.s1, c.s2)
+		}
+	}
+}
+
+func TestCPUSetIsSubsetOf(t *testing.T) {
+	shouldBeSubset := []struct {
+		s1 CPUSet
+		s2 CPUSet
+	}{
+		// A set is a subset of itself
+		{NewCPUSet(), NewCPUSet()},
+		{NewCPUSet(5), NewCPUSet(5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+
+		// Empty set is a subset of every set
+		{NewCPUSet(), NewCPUSet(5)},
+		{NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)},
+
+		{NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(2, 3), NewCPUSet(1, 2, 3, 4, 5)},
+	}
+
+	shouldNotBeSubset := []struct {
+		s1 CPUSet
+		s2 CPUSet
+	}{}
+
+	for _, c := range shouldBeSubset {
+		if !c.s1.IsSubsetOf(c.s2) {
+			t.Fatalf("expected s1 to be a subset of s2: s1: [%v], s2: [%v]", c.s1, c.s2)
+		}
+	}
+	for _, c := range shouldNotBeSubset {
+		if c.s1.IsSubsetOf(c.s2) {
+			t.Fatalf("expected s1 to not be a subset of s2: s1: [%v], s2: [%v]", c.s1, c.s2)
+		}
+	}
+}
+
+func TestCPUSetUnionAll(t *testing.T) {
+	testCases := []struct {
+		s1       CPUSet
+		s2       CPUSet
+		s3       CPUSet
+		expected CPUSet
+	}{
+		{NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet(4), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 5), NewCPUSet(1, 2, 3, 4, 5)},
+	}
+	for _, c := range testCases {
+		s := []CPUSet{}
+		s = append(s, c.s2)
+		s = append(s, c.s3)
+		result := c.s1.UnionAll(s)
+		if !result.Equals(c.expected) {
+			t.Fatalf("expected the union of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2)
+		}
+	}
+}
+
+func TestCPUSetUnion(t *testing.T) {
+	testCases := []struct {
+		s1       CPUSet
+		s2       CPUSet
+		expected CPUSet
+	}{
+		{NewCPUSet(), NewCPUSet(), NewCPUSet()},
+
+		{NewCPUSet(), NewCPUSet(5), NewCPUSet(5)},
+		{NewCPUSet(5), NewCPUSet(), NewCPUSet(5)},
+		{NewCPUSet(5), NewCPUSet(5), NewCPUSet(5)},
+
+		{NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+
+		{NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5)},
+
+		{NewCPUSet(1, 2), NewCPUSet(3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3), NewCPUSet(3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+	}
+
+	for _, c := range testCases {
+		result := c.s1.Union(c.s2)
+		if !result.Equals(c.expected) {
+			t.Fatalf("expected the union of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2)
+		}
+	}
+}
+
+func TestCPUSetIntersection(t *testing.T) {
+	testCases := []struct {
+		s1       CPUSet
+		s2       CPUSet
+		expected CPUSet
+	}{
+		{NewCPUSet(), NewCPUSet(), NewCPUSet()},
+
+		{NewCPUSet(), NewCPUSet(5), NewCPUSet()},
+		{NewCPUSet(5), NewCPUSet(), NewCPUSet()},
+		{NewCPUSet(5), NewCPUSet(5), NewCPUSet(5)},
+
+		{NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet()},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5)},
+
+		{NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5), NewCPUSet(5)},
+
+		{NewCPUSet(1, 2), NewCPUSet(3, 4, 5), NewCPUSet()},
+		{NewCPUSet(1, 2, 3), NewCPUSet(3, 4, 5), NewCPUSet(3)},
+	}
+
+	for _, c := range testCases {
+		result := c.s1.Intersection(c.s2)
+		if !result.Equals(c.expected) {
+			t.Fatalf("expected the intersection of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2)
+		}
+	}
+}
+
+func TestCPUSetDifference(t *testing.T) {
+	testCases := []struct {
+		s1       CPUSet
+		s2       CPUSet
+		expected CPUSet
+	}{
+		{NewCPUSet(), NewCPUSet(), NewCPUSet()},
+
+		{NewCPUSet(), NewCPUSet(5), NewCPUSet()},
+		{NewCPUSet(5), NewCPUSet(), NewCPUSet(5)},
+		{NewCPUSet(5), NewCPUSet(5), NewCPUSet()},
+
+		{NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(), NewCPUSet(1, 2, 3, 4, 5)},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()},
+
+		{NewCPUSet(5), NewCPUSet(1, 2, 3, 4, 5), NewCPUSet()},
+		{NewCPUSet(1, 2, 3, 4, 5), NewCPUSet(5), NewCPUSet(1, 2, 3, 4)},
+
+		{NewCPUSet(1, 2), NewCPUSet(3, 4, 5), NewCPUSet(1, 2)},
+		{NewCPUSet(1, 2, 3), NewCPUSet(3, 4, 5), NewCPUSet(1, 2)},
+	}
+
+	for _, c := range testCases {
+		result := c.s1.Difference(c.s2)
+		if !result.Equals(c.expected) {
+			t.Fatalf("expected the difference of s1 and s2 to be [%v] (got [%v]), s1: [%v], s2: [%v]", c.expected, result, c.s1, c.s2)
+		}
+	}
+}
+
+func TestCPUSetToSlice(t *testing.T) {
+	testCases := []struct {
+		set      CPUSet
+		expected []int
+	}{
+		{NewCPUSet(), []int{}},
+		{NewCPUSet(5), []int{5}},
+		{NewCPUSet(1, 2, 3, 4, 5), []int{1, 2, 3, 4, 5}},
+	}
+
+	for _, c := range testCases {
+		result := c.set.ToSlice()
+		if !reflect.DeepEqual(result, c.expected) {
+			t.Fatalf("expected set as slice to be [%v] (got [%v]), s: [%v]", c.expected, result, c.set)
+		}
+	}
+}
+
+func TestCPUSetString(t *testing.T) {
+	testCases := []struct {
+		set      CPUSet
+		expected string
+	}{
+		{NewCPUSet(), ""},
+		{NewCPUSet(5), "5"},
+		{NewCPUSet(1, 2, 3, 4, 5), "1-5"},
+		{NewCPUSet(1, 2, 3, 5, 6, 8), "1-3,5-6,8"},
+	}
+
+	for _, c := range testCases {
+		result := c.set.String()
+		if result != c.expected {
+			t.Fatalf("expected set as string to be %s (got \"%s\"), s: [%v]", c.expected, result, c.set)
+		}
+	}
+}
+
+func TestParse(t *testing.T) {
+	testCases := []struct {
+		cpusetString string
+		expected     CPUSet
+	}{
+		{"", NewCPUSet()},
+		{"5", NewCPUSet(5)},
+		{"1,2,3,4,5", NewCPUSet(1, 2, 3, 4, 5)},
+		{"1-5", NewCPUSet(1, 2, 3, 4, 5)},
+		{"1-2,3-5", NewCPUSet(1, 2, 3, 4, 5)},
+	}
+
+	for _, c := range testCases {
+		result, err := Parse(c.cpusetString)
+		if err != nil {
+			t.Fatalf("expected error not to have occurred: %v", err)
+		}
+		if !result.Equals(c.expected) {
+			t.Fatalf("expected string \"%s\" to parse as [%v] (got [%v])", c.cpusetString, c.expected, result)
+		}
+	}
+}

src/runtime/virtcontainers/sandbox.go

@@ -39,6 +39,7 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	vccgroups "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cgroups"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cpuset"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
@@ -564,15 +565,25 @@ func (s *Sandbox) createCgroupManager() error {
 	}
 
 	spec := s.GetPatchedOCISpec()
-	if spec != nil {
+	if spec != nil && spec.Linux != nil {
 		cgroupPath = spec.Linux.CgroupsPath
 
 		// Kata relies on the cgroup parent created and configured by the container
-		// engine, but sometimes the sandbox cgroup is not configured and the container
-		// may have access to all the resources, hence the runtime must constrain the
-		// sandbox and update the list of devices with the devices hotplugged in the
-		// hypervisor.
-		resources = *spec.Linux.Resources
+		// engine by default. The exception is for devices whitelist as well as sandbox-level
+		// CPUSet.
+		if spec.Linux.Resources != nil {
+			resources.Devices = spec.Linux.Resources.Devices
+
+			if spec.Linux.Resources.CPU != nil {
+				resources.CPU = &specs.LinuxCPU{
+					Cpus: spec.Linux.Resources.CPU.Cpus,
+				}
+			}
+		}
+
+		//TODO: in Docker or Podman use case, it is reasonable to set a constraint. Need to add a flag
+		// to allow users to configure Kata to constrain CPUs and Memory in this alternative
+		// scenario. See https://github.com/kata-containers/runtime/issues/2811
 	}
 
 	if s.devManager != nil {
@@ -1215,7 +1226,7 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
 		}
 	}()
 
-	// Sandbox is reponsable to update VM resources needed by Containers
+	// Sandbox is responsible to update VM resources needed by Containers
 	// Update resources after having added containers to the sandbox, since
 	// container status is requiered to know if more resources should be added.
 	err = s.updateResources()
@@ -1329,6 +1340,11 @@ func (s *Sandbox) DeleteContainer(containerID string) (VCContainer, error) {
 		}
 	}
 
+	// update the sandbox cgroup
+	if err = s.cgroupsUpdate(); err != nil {
+		return nil, err
+	}
+
 	if err = s.storeSandbox(); err != nil {
 		return nil, err
 	}
@@ -1866,11 +1882,12 @@ func (s *Sandbox) AddDevice(info config.DeviceInfo) (api.Device, error) {
 	return b, nil
 }
 
-// updateResources will calculate the resources required for the virtual machine, and
-// adjust the virtual machine sizing accordingly. For a given sandbox, it will calculate the
-// number of vCPUs required based on the sum of container requests, plus default CPUs for the VM.
-// Similar is done for memory. If changes in memory or CPU are made, the VM will be updated and
-// the agent will online the applicable CPU and memory.
+// updateResources will:
+// - calculate the resources required for the virtual machine, and adjust the virtual machine
+// sizing accordingly. For a given sandbox, it will calculate the number of vCPUs required based
+// on the sum of container requests, plus default CPUs for the VM. Similar is done for memory.
+// If changes in memory or CPU are made, the VM will be updated and the agent will online the
+// applicable CPU and memory.
 func (s *Sandbox) updateResources() error {
 	if s == nil {
 		return errors.New("sandbox is nil")
@@ -1880,7 +1897,10 @@ func (s *Sandbox) updateResources() error {
 		return fmt.Errorf("sandbox config is nil")
 	}
 
-	sandboxVCPUs := s.calculateSandboxCPUs()
+	sandboxVCPUs, err := s.calculateSandboxCPUs()
+	if err != nil {
+		return err
+	}
 	// Add default vcpus for sandbox
 	sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs
 
@@ -1942,8 +1962,9 @@ func (s *Sandbox) calculateSandboxMemory() int64 {
 	return memorySandbox
 }
 
-func (s *Sandbox) calculateSandboxCPUs() uint32 {
+func (s *Sandbox) calculateSandboxCPUs() (uint32, error) {
 	mCPU := uint32(0)
+	cpusetCount := int(0)
 
 	for _, c := range s.config.Containers {
 		// Do not hot add again non-running containers resources
@@ -1957,9 +1978,22 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 {
 				mCPU += utils.CalculateMilliCPUs(*cpu.Quota, *cpu.Period)
 			}
 
+			set, err := cpuset.Parse(cpu.Cpus)
+			if err != nil {
+				return 0, nil
+			}
+			cpusetCount += set.Size()
 		}
 	}
-	return utils.CalculateVCpusFromMilliCpus(mCPU)
+
+	// If we aren't being constrained, then we could have two scenarios:
+	//  1. BestEffort QoS: no proper support today in Kata.
+	//  2. We could be constrained only by CPUSets. Check for this:
+	if mCPU == 0 && cpusetCount > 0 {
+		return uint32(cpusetCount), nil
+	}
+
+	return utils.CalculateVCpusFromMilliCpus(mCPU), nil
 }
 
 // GetHypervisorType is used for getting Hypervisor name currently used.
@@ -1975,9 +2009,18 @@ func (s *Sandbox) GetHypervisorType() string {
 func (s *Sandbox) cgroupsUpdate() error {
 
 	// If Kata is configured for SandboxCgroupOnly, the VMM and its processes are already
-	// in the Kata sandbox cgroup (inherited). No need to move threads/processes, and we should
-	// rely on parent's cgroup CPU/memory values
+	// in the Kata sandbox cgroup (inherited). Check to see if sandbox cpuset needs to be
+	// updated.
 	if s.config.SandboxCgroupOnly {
+		cpuset, memset, err := s.getSandboxCPUSet()
+		if err != nil {
+			return err
+		}
+
+		if err := s.cgroupMgr.SetCPUSet(cpuset, memset); err != nil {
+			return err
+		}
+
 		return nil
 	}
 
@@ -2275,3 +2318,31 @@ func (s *Sandbox) GetOOMEvent() (string, error) {
 func (s *Sandbox) GetAgentURL() (string, error) {
 	return s.agent.getAgentURL()
 }
+
+// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets'
+// cpus and mems as a string in canonical linux CPU/mems list format
+func (s *Sandbox) getSandboxCPUSet() (string, string, error) {
+	if s.config == nil {
+		return "", "", nil
+	}
+
+	cpuResult := cpuset.NewCPUSet()
+	memResult := cpuset.NewCPUSet()
+	for _, ctr := range s.config.Containers {
+		if ctr.Resources.CPU != nil {
+			currCPUSet, err := cpuset.Parse(ctr.Resources.CPU.Cpus)
+			if err != nil {
+				return "", "", fmt.Errorf("unable to parse CPUset.cpus for container %s: %v", ctr.ID, err)
+			}
+			cpuResult = cpuResult.Union(currCPUSet)
+
+			currMemSet, err := cpuset.Parse(ctr.Resources.CPU.Mems)
+			if err != nil {
+				return "", "", fmt.Errorf("unable to parse CPUset.mems for container %s: %v", ctr.ID, err)
+			}
+			memResult = memResult.Union(currMemSet)
+		}
+	}
+
+	return cpuResult.String(), memResult.String(), nil
+}

src/runtime/virtcontainers/sandbox_test.go

@@ -106,12 +106,18 @@ func TestCreateMockSandbox(t *testing.T) {
 func TestCalculateSandboxCPUs(t *testing.T) {
 	sandbox := &Sandbox{}
 	sandbox.config = &SandboxConfig{}
+
 	unconstrained := newTestContainerConfigNoop("cont-00001")
-	constrained := newTestContainerConfigNoop("cont-00001")
+	constrained := newTestContainerConfigNoop("cont-00002")
+	unconstrainedCpusets0_1 := newTestContainerConfigNoop("cont-00003")
+	unconstrainedCpusets2 := newTestContainerConfigNoop("cont-00004")
+	constrainedCpusets0_7 := newTestContainerConfigNoop("cont-00005")
 	quota := int64(4000)
 	period := uint64(1000)
 	constrained.Resources.CPU = &specs.LinuxCPU{Period: &period, Quota: &quota}
-
+	unconstrainedCpusets0_1.Resources.CPU = &specs.LinuxCPU{Cpus: "0-1"}
+	unconstrainedCpusets2.Resources.CPU = &specs.LinuxCPU{Cpus: "2"}
+	constrainedCpusets0_7.Resources.CPU = &specs.LinuxCPU{Period: &period, Quota: &quota, Cpus: "0-7"}
 	tests := []struct {
 		name       string
 		containers []ContainerConfig
@@ -123,11 +129,14 @@ func TestCalculateSandboxCPUs(t *testing.T) {
 		{"2-constrained", []ContainerConfig{constrained, constrained}, 8},
 		{"3-mix-constraints", []ContainerConfig{unconstrained, constrained, constrained}, 8},
 		{"3-constrained", []ContainerConfig{constrained, constrained, constrained}, 12},
+		{"unconstrained-1-cpuset", []ContainerConfig{unconstrained, unconstrained, unconstrainedCpusets0_1}, 2},
+		{"unconstrained-2-cpuset", []ContainerConfig{unconstrainedCpusets0_1, unconstrainedCpusets2}, 3},
+		{"constrained-cpuset", []ContainerConfig{constrainedCpusets0_7}, 4},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			sandbox.config.Containers = tt.containers
-			got := sandbox.calculateSandboxCPUs()
+			got, _ := sandbox.calculateSandboxCPUs()
 			assert.Equal(t, got, tt.want)
 		})
 	}
@@ -1419,3 +1428,127 @@ func TestSandbox_SetupSandboxCgroup(t *testing.T) {
 		})
 	}
 }
+
+func getContainerConfigWithCPUSet(cpuset, memset string) ContainerConfig {
+	return ContainerConfig{
+		Resources: specs.LinuxResources{
+			CPU: &specs.LinuxCPU{
+				Cpus: cpuset,
+				Mems: memset,
+			},
+		},
+	}
+}
+
+func getSimpleSandbox(cpusets, memsets [3]string) *Sandbox {
+	sandbox := Sandbox{}
+
+	sandbox.config = &SandboxConfig{
+		Containers: []ContainerConfig{
+			getContainerConfigWithCPUSet(cpusets[0], memsets[0]),
+			getContainerConfigWithCPUSet(cpusets[1], memsets[1]),
+			getContainerConfigWithCPUSet(cpusets[2], memsets[2]),
+		},
+	}
+
+	return &sandbox
+}
+
+func TestGetSandboxCpuSet(t *testing.T) {
+
+	tests := []struct {
+		name      string
+		cpusets   [3]string
+		memsets   [3]string
+		cpuResult string
+		memResult string
+		wantErr   bool
+	}{
+		{
+			"single, no cpuset",
+			[3]string{"", "", ""},
+			[3]string{"", "", ""},
+			"",
+			"",
+			false,
+		},
+		{
+			"single cpuset",
+			[3]string{"0", "", ""},
+			[3]string{"", "", ""},
+			"0",
+			"",
+			false,
+		},
+		{
+			"two duplicate cpuset",
+			[3]string{"0", "0", ""},
+			[3]string{"", "", ""},
+			"0",
+			"",
+			false,
+		},
+		{
+			"3 cpusets",
+			[3]string{"0-3", "5-7", "1"},
+			[3]string{"", "", ""},
+			"0-3,5-7",
+			"",
+			false,
+		},
+
+		{
+			"weird, but should be okay",
+			[3]string{"0-3", "99999", ""},
+			[3]string{"", "", ""},
+			"0-3,99999",
+			"",
+			false,
+		},
+		{
+			"two, overlapping cpuset",
+			[3]string{"0-3", "1-2", ""},
+			[3]string{"", "", ""},
+			"0-3",
+			"",
+			false,
+		},
+		{
+			"garbage, should fail",
+			[3]string{"7 beard-seconds", "Audrey + 7", "Elliott - 17"},
+			[3]string{"", "", ""},
+			"",
+			"",
+			true,
+		},
+		{
+			"cpuset and memset",
+			[3]string{"0-3", "1-2", ""},
+			[3]string{"0", "1", "0-1"},
+			"0-3",
+			"0-1",
+			false,
+		},
+		{
+			"memset",
+			[3]string{"0-3", "1-2", ""},
+			[3]string{"0", "3", ""},
+			"0-3",
+			"0,3",
+			false,
+		},
+	}
+	for _, tt := range tests {
+
+		t.Run(tt.name, func(t *testing.T) {
+			s := getSimpleSandbox(tt.cpusets, tt.memsets)
+			res, _, err := s.getSandboxCPUSet()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("getSandboxCPUSet() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if res != tt.cpuResult {
+				t.Errorf("getSandboxCPUSet() result = %s, wanted result %s", res, tt.cpuResult)
+			}
+		})
+	}
+}