mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-13 13:55:41 +00:00
Unit test: Swap - Limited/Unlimited Swap, cgroups v1/v2, etc
Signed-off-by: Itamar Holder <iholder@redhat.com>
This commit is contained in:
Itamar Holder
parent
a30410d9ce
commit
4b6314f815
@ -21,6 +21,9 @@ package kuberuntime
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/types"
|
||||||
"math"
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"reflect"
|
"reflect"
|
||||||
@ -38,7 +41,6 @@ import (
|
|||||||
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
|
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
|
||||||
"k8s.io/kubernetes/pkg/features"
|
"k8s.io/kubernetes/pkg/features"
|
||||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||||
kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerIndex int, enforceMemoryQoS bool) *runtimeapi.ContainerConfig {
|
func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerIndex int, enforceMemoryQoS bool) *runtimeapi.ContainerConfig {
|
||||||
@ -695,96 +697,6 @@ func TestGenerateLinuxContainerConfigNamespaces(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGenerateLinuxContainerConfigSwap(t *testing.T) {
|
|
||||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeSwap, true)()
|
|
||||||
_, _, m, err := createTestRuntimeManager()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("error creating test RuntimeManager: %v", err)
|
|
||||||
}
|
|
||||||
m.machineInfo.MemoryCapacity = 1000000
|
|
||||||
containerName := "test"
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
swapSetting string
|
|
||||||
pod *v1.Pod
|
|
||||||
expected int64
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "config unset, memory limit set",
|
|
||||||
// no swap setting
|
|
||||||
pod: &v1.Pod{
|
|
||||||
Spec: v1.PodSpec{
|
|
||||||
Containers: []v1.Container{{
|
|
||||||
Name: containerName,
|
|
||||||
Resources: v1.ResourceRequirements{
|
|
||||||
Limits: v1.ResourceList{
|
|
||||||
"memory": resource.MustParse("1000"),
|
|
||||||
},
|
|
||||||
Requests: v1.ResourceList{
|
|
||||||
"memory": resource.MustParse("1000"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expected: 1000,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "config unset, no memory limit",
|
|
||||||
// no swap setting
|
|
||||||
pod: &v1.Pod{
|
|
||||||
Spec: v1.PodSpec{
|
|
||||||
Containers: []v1.Container{
|
|
||||||
{Name: containerName},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expected: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Note: behaviour will be the same as previous two cases
|
|
||||||
name: "config set to LimitedSwap, memory limit set",
|
|
||||||
swapSetting: kubelettypes.LimitedSwap,
|
|
||||||
pod: &v1.Pod{
|
|
||||||
Spec: v1.PodSpec{
|
|
||||||
Containers: []v1.Container{{
|
|
||||||
Name: containerName,
|
|
||||||
Resources: v1.ResourceRequirements{
|
|
||||||
Limits: v1.ResourceList{
|
|
||||||
"memory": resource.MustParse("1000"),
|
|
||||||
},
|
|
||||||
Requests: v1.ResourceList{
|
|
||||||
"memory": resource.MustParse("1000"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expected: 1000,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "UnlimitedSwap enabled",
|
|
||||||
swapSetting: kubelettypes.UnlimitedSwap,
|
|
||||||
pod: &v1.Pod{
|
|
||||||
Spec: v1.PodSpec{
|
|
||||||
Containers: []v1.Container{
|
|
||||||
{Name: containerName},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
expected: -1,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
m.memorySwapBehavior = tc.swapSetting
|
|
||||||
actual, err := m.generateLinuxContainerConfig(&tc.pod.Spec.Containers[0], tc.pod, nil, "", nil, false)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
assert.Equal(t, tc.expected, actual.Resources.MemorySwapLimitInBytes, "memory swap config for %s", tc.name)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateLinuxContainerResources(t *testing.T) {
|
func TestGenerateLinuxContainerResources(t *testing.T) {
|
||||||
_, _, m, err := createTestRuntimeManager()
|
_, _, m, err := createTestRuntimeManager()
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
@ -936,6 +848,10 @@ func TestGenerateLinuxContainerResources(t *testing.T) {
|
|||||||
if tc.scalingFg {
|
if tc.scalingFg {
|
||||||
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InPlacePodVerticalScaling, true)()
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InPlacePodVerticalScaling, true)()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setCgroupVersionDuringTest(cgroupV1)
|
||||||
|
tc.expected.MemorySwapLimitInBytes = tc.expected.MemoryLimitInBytes
|
||||||
|
|
||||||
pod.Spec.Containers[0].Resources = v1.ResourceRequirements{Limits: tc.limits, Requests: tc.requests}
|
pod.Spec.Containers[0].Resources = v1.ResourceRequirements{Limits: tc.limits, Requests: tc.requests}
|
||||||
if len(tc.cStatus) > 0 {
|
if len(tc.cStatus) > 0 {
|
||||||
pod.Status.ContainerStatuses = tc.cStatus
|
pod.Status.ContainerStatuses = tc.cStatus
|
||||||
@ -950,6 +866,279 @@ func TestGenerateLinuxContainerResources(t *testing.T) {
|
|||||||
//TODO(vinaykul,InPlacePodVerticalScaling): Add unit tests for cgroup v1 & v2
|
//TODO(vinaykul,InPlacePodVerticalScaling): Add unit tests for cgroup v1 & v2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
||||||
|
_, _, m, err := createTestRuntimeManager()
|
||||||
|
assert.NoError(t, err)
|
||||||
|
m.machineInfo.MemoryCapacity = 42949672960 // 40Gb == 40 * 1024^3
|
||||||
|
m.machineInfo.SwapCapacity = 5368709120 // 5Gb == 5 * 1024^3
|
||||||
|
|
||||||
|
pod := &v1.Pod{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
UID: "12345678",
|
||||||
|
Name: "foo",
|
||||||
|
Namespace: "bar",
|
||||||
|
},
|
||||||
|
Spec: v1.PodSpec{
|
||||||
|
Containers: []v1.Container{
|
||||||
|
{
|
||||||
|
Name: "c1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "c2",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Status: v1.PodStatus{},
|
||||||
|
}
|
||||||
|
|
||||||
|
expectNoSwap := func(cgroupVersion CgroupVersion, resources ...*runtimeapi.LinuxContainerResources) {
|
||||||
|
const msg = "container is expected to not have swap access"
|
||||||
|
|
||||||
|
for _, r := range resources {
|
||||||
|
switch cgroupVersion {
|
||||||
|
case cgroupV1:
|
||||||
|
assert.Equal(t, r.MemoryLimitInBytes, r.MemorySwapLimitInBytes, msg)
|
||||||
|
case cgroupV2:
|
||||||
|
assert.Equal(t, "0", r.Unified[cm.Cgroup2MaxSwapFilename], msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
expectUnlimitedSwap := func(cgroupVersion CgroupVersion, resources ...*runtimeapi.LinuxContainerResources) {
|
||||||
|
const msg = "container is expected to have unlimited swap access"
|
||||||
|
|
||||||
|
for _, r := range resources {
|
||||||
|
switch cgroupVersion {
|
||||||
|
case cgroupV1:
|
||||||
|
assert.Equal(t, int64(-1), r.MemorySwapLimitInBytes, msg)
|
||||||
|
case cgroupV2:
|
||||||
|
assert.Equal(t, "max", r.Unified[cm.Cgroup2MaxSwapFilename], msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
expectSwap := func(cgroupVersion CgroupVersion, swapBytesExpected int64, resources *runtimeapi.LinuxContainerResources) {
|
||||||
|
msg := fmt.Sprintf("container swap is expected to be limited by %d bytes", swapBytesExpected)
|
||||||
|
|
||||||
|
switch cgroupVersion {
|
||||||
|
case cgroupV1:
|
||||||
|
assert.Equal(t, resources.MemoryLimitInBytes+swapBytesExpected, resources.MemorySwapLimitInBytes, msg)
|
||||||
|
case cgroupV2:
|
||||||
|
assert.Equal(t, fmt.Sprintf("%d", swapBytesExpected), resources.Unified[cm.Cgroup2MaxSwapFilename], msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
calcSwapForBurstablePods := func(containerMemoryRequest int64) int64 {
|
||||||
|
swapSize, err := calcSwapForBurstablePods(containerMemoryRequest, int64(m.machineInfo.MemoryCapacity), int64(m.machineInfo.SwapCapacity))
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
return swapSize
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
cgroupVersion CgroupVersion
|
||||||
|
qosClass v1.PodQOSClass
|
||||||
|
nodeSwapFeatureGateEnabled bool
|
||||||
|
swapBehavior string
|
||||||
|
addContainerWithoutRequests bool
|
||||||
|
addGuaranteedContainer bool
|
||||||
|
}{
|
||||||
|
// With cgroup v1
|
||||||
|
{
|
||||||
|
name: "cgroups v1, LimitedSwap, Burstable QoS",
|
||||||
|
cgroupVersion: cgroupV1,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.LimitedSwap,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cgroups v1, UnlimitedSwap, Burstable QoS",
|
||||||
|
cgroupVersion: cgroupV1,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.UnlimitedSwap,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cgroups v1, LimitedSwap, Best-effort QoS",
|
||||||
|
cgroupVersion: cgroupV1,
|
||||||
|
qosClass: v1.PodQOSBestEffort,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.LimitedSwap,
|
||||||
|
},
|
||||||
|
|
||||||
|
// With feature gate turned off
|
||||||
|
{
|
||||||
|
name: "NodeSwap feature gate turned off, cgroups v2, LimitedSwap",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: false,
|
||||||
|
swapBehavior: types.LimitedSwap,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "NodeSwap feature gate turned off, cgroups v2, UnlimitedSwap",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: false,
|
||||||
|
swapBehavior: types.UnlimitedSwap,
|
||||||
|
},
|
||||||
|
|
||||||
|
// With no swapBehavior, UnlimitedSwap should be the default
|
||||||
|
{
|
||||||
|
name: "With no swapBehavior - UnlimitedSwap should be the default",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBestEffort,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: "",
|
||||||
|
},
|
||||||
|
|
||||||
|
// With Guaranteed and Best-effort QoS
|
||||||
|
{
|
||||||
|
name: "Best-effort Qos, cgroups v2, LimitedSwap",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.LimitedSwap,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Best-effort Qos, cgroups v2, UnlimitedSwap",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.UnlimitedSwap,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Guaranteed Qos, cgroups v2, LimitedSwap",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSGuaranteed,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.LimitedSwap,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Guaranteed Qos, cgroups v2, UnlimitedSwap",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSGuaranteed,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.UnlimitedSwap,
|
||||||
|
},
|
||||||
|
|
||||||
|
// With a "guaranteed" container (when memory requests equal to limits)
|
||||||
|
{
|
||||||
|
name: "Burstable Qos, cgroups v2, LimitedSwap, with a guaranteed container",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.LimitedSwap,
|
||||||
|
addContainerWithoutRequests: false,
|
||||||
|
addGuaranteedContainer: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Burstable Qos, cgroups v2, UnlimitedSwap, with a guaranteed container",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.UnlimitedSwap,
|
||||||
|
addContainerWithoutRequests: false,
|
||||||
|
addGuaranteedContainer: true,
|
||||||
|
},
|
||||||
|
|
||||||
|
// Swap is expected to be allocated
|
||||||
|
{
|
||||||
|
name: "Burstable Qos, cgroups v2, LimitedSwap",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.LimitedSwap,
|
||||||
|
addContainerWithoutRequests: false,
|
||||||
|
addGuaranteedContainer: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Burstable Qos, cgroups v2, UnlimitedSwap",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.UnlimitedSwap,
|
||||||
|
addContainerWithoutRequests: false,
|
||||||
|
addGuaranteedContainer: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Burstable Qos, cgroups v2, LimitedSwap, with a container with no requests",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.LimitedSwap,
|
||||||
|
addContainerWithoutRequests: true,
|
||||||
|
addGuaranteedContainer: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Burstable Qos, cgroups v2, UnlimitedSwap, with a container with no requests",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBurstable,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: types.UnlimitedSwap,
|
||||||
|
addContainerWithoutRequests: true,
|
||||||
|
addGuaranteedContainer: false,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
setCgroupVersionDuringTest(tc.cgroupVersion)
|
||||||
|
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeSwap, tc.nodeSwapFeatureGateEnabled)()
|
||||||
|
m.memorySwapBehavior = tc.swapBehavior
|
||||||
|
|
||||||
|
var resourceReqsC1, resourceReqsC2 v1.ResourceRequirements
|
||||||
|
switch tc.qosClass {
|
||||||
|
case v1.PodQOSBurstable:
|
||||||
|
resourceReqsC1 = v1.ResourceRequirements{
|
||||||
|
Requests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("1Gi")},
|
||||||
|
}
|
||||||
|
|
||||||
|
if !tc.addContainerWithoutRequests {
|
||||||
|
resourceReqsC2 = v1.ResourceRequirements{
|
||||||
|
Requests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("2Gi")},
|
||||||
|
}
|
||||||
|
|
||||||
|
if tc.addGuaranteedContainer {
|
||||||
|
resourceReqsC2.Limits = v1.ResourceList{v1.ResourceMemory: resource.MustParse("2Gi")}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case v1.PodQOSGuaranteed:
|
||||||
|
resourceReqsC1 = v1.ResourceRequirements{
|
||||||
|
Requests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("1Gi"), v1.ResourceCPU: resource.MustParse("1")},
|
||||||
|
Limits: v1.ResourceList{v1.ResourceMemory: resource.MustParse("1Gi"), v1.ResourceCPU: resource.MustParse("1")},
|
||||||
|
}
|
||||||
|
resourceReqsC2 = v1.ResourceRequirements{
|
||||||
|
Requests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("2Gi"), v1.ResourceCPU: resource.MustParse("1")},
|
||||||
|
Limits: v1.ResourceList{v1.ResourceMemory: resource.MustParse("2Gi"), v1.ResourceCPU: resource.MustParse("1")},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pod.Spec.Containers[0].Resources = resourceReqsC1
|
||||||
|
pod.Spec.Containers[1].Resources = resourceReqsC2
|
||||||
|
|
||||||
|
resourcesC1 := m.generateLinuxContainerResources(pod, &pod.Spec.Containers[0], false)
|
||||||
|
resourcesC2 := m.generateLinuxContainerResources(pod, &pod.Spec.Containers[1], false)
|
||||||
|
|
||||||
|
if !tc.nodeSwapFeatureGateEnabled || tc.cgroupVersion == cgroupV1 || (tc.swapBehavior == types.LimitedSwap && tc.qosClass != v1.PodQOSBurstable) {
|
||||||
|
expectNoSwap(tc.cgroupVersion, resourcesC1, resourcesC2)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if tc.swapBehavior == types.UnlimitedSwap || tc.swapBehavior == "" {
|
||||||
|
expectUnlimitedSwap(tc.cgroupVersion, resourcesC1, resourcesC2)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c1ExpectedSwap := calcSwapForBurstablePods(resourceReqsC1.Requests.Memory().Value())
|
||||||
|
c2ExpectedSwap := int64(0)
|
||||||
|
if !tc.addContainerWithoutRequests && !tc.addGuaranteedContainer {
|
||||||
|
c2ExpectedSwap = calcSwapForBurstablePods(resourceReqsC2.Requests.Memory().Value())
|
||||||
|
}
|
||||||
|
|
||||||
|
expectSwap(tc.cgroupVersion, c1ExpectedSwap, resourcesC1)
|
||||||
|
expectSwap(tc.cgroupVersion, c2ExpectedSwap, resourcesC2)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type CgroupVersion string
|
type CgroupVersion string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user