add no swap as the default option for swap

This commit is contained in:
Kevin Hannon 2024-01-12 14:42:51 -05:00
parent b0ee334374
commit 6a4e19a4ec
13 changed files with 62 additions and 195 deletions

View File

@ -105,6 +105,7 @@ import (
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/pkg/kubelet/server" "k8s.io/kubernetes/pkg/kubelet/server"
"k8s.io/kubernetes/pkg/kubelet/stats/pidlimit" "k8s.io/kubernetes/pkg/kubelet/stats/pidlimit"
kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
kubeletutil "k8s.io/kubernetes/pkg/kubelet/util" kubeletutil "k8s.io/kubernetes/pkg/kubelet/util"
utilfs "k8s.io/kubernetes/pkg/util/filesystem" utilfs "k8s.io/kubernetes/pkg/util/filesystem"
"k8s.io/kubernetes/pkg/util/flock" "k8s.io/kubernetes/pkg/util/flock"
@ -803,6 +804,14 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
return fmt.Errorf("topology manager policy options %v require feature gates %q enabled", return fmt.Errorf("topology manager policy options %v require feature gates %q enabled",
s.TopologyManagerPolicyOptions, features.TopologyManagerPolicyOptions) s.TopologyManagerPolicyOptions, features.TopologyManagerPolicyOptions)
} }
if utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) {
if !isCgroup2UnifiedMode() && s.MemorySwap.SwapBehavior == kubelettypes.LimitedSwap {
klog.InfoS("Swap feature is enabled and LimitedSwap but it is only supported with cgroupv2", "CGroupV2", isCgroup2UnifiedMode(), "SwapBehavior", s.MemorySwap.SwapBehavior)
}
if !s.FailSwapOn && s.MemorySwap.SwapBehavior == "" {
klog.InfoS("NoSwap is set due to memorySwapBehavior not specified", "memorySwapBehavior", s.MemorySwap.SwapBehavior, "FailSwapOn", s.FailSwapOn)
}
}
kubeDeps.ContainerManager, err = cm.NewContainerManager( kubeDeps.ContainerManager, err = cm.NewContainerManager(
kubeDeps.Mounter, kubeDeps.Mounter,

View File

@ -539,11 +539,13 @@ const (
// Allow pods to failover to a different node in case of non graceful node shutdown // Allow pods to failover to a different node in case of non graceful node shutdown
NodeOutOfServiceVolumeDetach featuregate.Feature = "NodeOutOfServiceVolumeDetach" NodeOutOfServiceVolumeDetach featuregate.Feature = "NodeOutOfServiceVolumeDetach"
// owner: @iholder101 // owner: @iholder101 @kannon92
// kep: https://kep.k8s.io/2400
// alpha: v1.22 // alpha: v1.22
// beta1: v1.28. For more info, please look at the KEP: https://kep.k8s.io/2400. // beta1: v1.28 (default=false)
// // beta2: v.1.30 (default=true)
// Permits kubelet to run with swap enabled
// Permits kubelet to run with swap enabled.
NodeSwap featuregate.Feature = "NodeSwap" NodeSwap featuregate.Feature = "NodeSwap"
// owner: @mortent, @atiratree, @ravig // owner: @mortent, @atiratree, @ravig
@ -1105,7 +1107,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
NodeOutOfServiceVolumeDetach: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31 NodeOutOfServiceVolumeDetach: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31
NodeSwap: {Default: false, PreRelease: featuregate.Beta}, NodeSwap: {Default: true, PreRelease: featuregate.Beta},
PDBUnhealthyPodEvictionPolicy: {Default: true, PreRelease: featuregate.Beta}, PDBUnhealthyPodEvictionPolicy: {Default: true, PreRelease: featuregate.Beta},

View File

@ -59686,7 +59686,7 @@ func schema_k8sio_kubelet_config_v1beta1_MemorySwapConfiguration(ref common.Refe
Properties: map[string]spec.Schema{ Properties: map[string]spec.Schema{
"swapBehavior": { "swapBehavior": {
SchemaProps: spec.SchemaProps{ SchemaProps: spec.SchemaProps{
Description: "swapBehavior configures swap memory available to container workloads. May be one of \"\", \"LimitedSwap\": workload combined memory and swap usage cannot exceed pod memory limit \"UnlimitedSwap\": workloads can use unlimited swap, up to the allocatable limit.", Description: "swapBehavior configures swap memory available to container workloads. May be one of \"\", \"NoSwap\": workloads can not use swap, default option. \"LimitedSwap\": workload swap usage is limited. The swap limit is proportionate to the container's memory request.",
Type: []string{"string"}, Type: []string{"string"},
Format: "", Format: "",
}, },

View File

@ -666,8 +666,8 @@ type ShutdownGracePeriodByPodPriority struct {
type MemorySwapConfiguration struct { type MemorySwapConfiguration struct {
// swapBehavior configures swap memory available to container workloads. May be one of // swapBehavior configures swap memory available to container workloads. May be one of
// "", "LimitedSwap": workload combined memory and swap usage cannot exceed pod memory limit // "", "NoSwap": workloads can not use swap, default option.
// "UnlimitedSwap": workloads can use unlimited swap, up to the allocatable limit. // "LimitedSwap": workload swap usage is limited. The swap limit is proportionate to the container's memory request.
// +featureGate=NodeSwap // +featureGate=NodeSwap
// +optional // +optional
SwapBehavior string SwapBehavior string

View File

@ -471,7 +471,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
IPTablesMasqueradeBit: utilpointer.Int32(1), IPTablesMasqueradeBit: utilpointer.Int32(1),
IPTablesDropBit: utilpointer.Int32(1), IPTablesDropBit: utilpointer.Int32(1),
FailSwapOn: utilpointer.Bool(true), FailSwapOn: utilpointer.Bool(true),
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "UnlimitedSwap"}, MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "NoSwap"},
ContainerLogMaxSize: "1Mi", ContainerLogMaxSize: "1Mi",
ContainerLogMaxFiles: utilpointer.Int32(1), ContainerLogMaxFiles: utilpointer.Int32(1),
ContainerLogMaxWorkers: utilpointer.Int32(1), ContainerLogMaxWorkers: utilpointer.Int32(1),
@ -620,7 +620,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
IPTablesMasqueradeBit: utilpointer.Int32(1), IPTablesMasqueradeBit: utilpointer.Int32(1),
IPTablesDropBit: utilpointer.Int32(1), IPTablesDropBit: utilpointer.Int32(1),
FailSwapOn: utilpointer.Bool(true), FailSwapOn: utilpointer.Bool(true),
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "UnlimitedSwap"}, MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "NoSwap"},
ContainerLogMaxSize: "1Mi", ContainerLogMaxSize: "1Mi",
ContainerLogMaxFiles: utilpointer.Int32(1), ContainerLogMaxFiles: utilpointer.Int32(1),
ContainerLogMaxWorkers: utilpointer.Int32(1), ContainerLogMaxWorkers: utilpointer.Int32(1),

View File

@ -195,10 +195,10 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur
if localFeatureGate.Enabled(features.NodeSwap) { if localFeatureGate.Enabled(features.NodeSwap) {
switch kc.MemorySwap.SwapBehavior { switch kc.MemorySwap.SwapBehavior {
case "": case "":
case kubetypes.NoSwap:
case kubetypes.LimitedSwap: case kubetypes.LimitedSwap:
case kubetypes.UnlimitedSwap:
default: default:
allErrors = append(allErrors, fmt.Errorf("invalid configuration: memorySwap.swapBehavior %q must be one of: \"\", %q, or %q", kc.MemorySwap.SwapBehavior, kubetypes.LimitedSwap, kubetypes.UnlimitedSwap)) allErrors = append(allErrors, fmt.Errorf("invalid configuration: memorySwap.swapBehavior %q must be one of: \"\", %q or %q", kc.MemorySwap.SwapBehavior, kubetypes.LimitedSwap, kubetypes.NoSwap))
} }
} }
if !localFeatureGate.Enabled(features.NodeSwap) && kc.MemorySwap != (kubeletconfig.MemorySwapConfiguration{}) { if !localFeatureGate.Enabled(features.NodeSwap) && kc.MemorySwap != (kubeletconfig.MemorySwapConfiguration{}) {

View File

@ -367,7 +367,7 @@ func TestValidateKubeletConfiguration(t *testing.T) {
conf.MemorySwap.SwapBehavior = "invalid-behavior" conf.MemorySwap.SwapBehavior = "invalid-behavior"
return conf return conf
}, },
errMsg: "invalid configuration: memorySwap.swapBehavior \"invalid-behavior\" must be one of: \"\", \"LimitedSwap\", or \"UnlimitedSwap\"", errMsg: "invalid configuration: memorySwap.swapBehavior \"invalid-behavior\" must be one of: \"\", \"LimitedSwap\" or \"NoSwap\"",
}, { }, {
name: "specify MemorySwap.SwapBehavior without enabling NodeSwap", name: "specify MemorySwap.SwapBehavior without enabling NodeSwap",
configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration { configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {

View File

@ -168,6 +168,12 @@ func (m *kubeGenericRuntimeManager) configureContainerSwapResources(lcr *runtime
return return
} }
swapConfigurationHelper := newSwapConfigurationHelper(*m.machineInfo) swapConfigurationHelper := newSwapConfigurationHelper(*m.machineInfo)
if m.memorySwapBehavior == kubelettypes.LimitedSwap {
if !isCgroup2UnifiedMode() {
swapConfigurationHelper.ConfigureNoSwap(lcr)
return
}
}
if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.NodeSwap) { if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.NodeSwap) {
swapConfigurationHelper.ConfigureNoSwap(lcr) swapConfigurationHelper.ConfigureNoSwap(lcr)
@ -177,10 +183,12 @@ func (m *kubeGenericRuntimeManager) configureContainerSwapResources(lcr *runtime
// NOTE(ehashman): Behavior is defined in the opencontainers runtime spec: // NOTE(ehashman): Behavior is defined in the opencontainers runtime spec:
// https://github.com/opencontainers/runtime-spec/blob/1c3f411f041711bbeecf35ff7e93461ea6789220/config-linux.md#memory // https://github.com/opencontainers/runtime-spec/blob/1c3f411f041711bbeecf35ff7e93461ea6789220/config-linux.md#memory
switch m.memorySwapBehavior { switch m.memorySwapBehavior {
case kubelettypes.NoSwap:
swapConfigurationHelper.ConfigureNoSwap(lcr)
case kubelettypes.LimitedSwap: case kubelettypes.LimitedSwap:
swapConfigurationHelper.ConfigureLimitedSwap(lcr, pod, container) swapConfigurationHelper.ConfigureLimitedSwap(lcr, pod, container)
default: default:
swapConfigurationHelper.ConfigureUnlimitedSwap(lcr) swapConfigurationHelper.ConfigureNoSwap(lcr)
} }
} }
@ -401,19 +409,6 @@ func (m swapConfigurationHelper) ConfigureNoSwap(lcr *runtimeapi.LinuxContainerR
m.configureSwap(lcr, 0) m.configureSwap(lcr, 0)
} }
func (m swapConfigurationHelper) ConfigureUnlimitedSwap(lcr *runtimeapi.LinuxContainerResources) {
if !isCgroup2UnifiedMode() {
m.ConfigureNoSwap(lcr)
return
}
if lcr.Unified == nil {
lcr.Unified = map[string]string{}
}
lcr.Unified[cm.Cgroup2MaxSwapFilename] = "max"
}
func (m swapConfigurationHelper) configureSwap(lcr *runtimeapi.LinuxContainerResources, swapMemory int64) { func (m swapConfigurationHelper) configureSwap(lcr *runtimeapi.LinuxContainerResources, swapMemory int64) {
if !isCgroup2UnifiedMode() { if !isCgroup2UnifiedMode() {
klog.ErrorS(fmt.Errorf("swap configuration is not supported with cgroup v1"), "swap configuration under cgroup v1 is unexpected") klog.ErrorS(fmt.Errorf("swap configuration is not supported with cgroup v1"), "swap configuration under cgroup v1 is unexpected")

View File

@ -918,19 +918,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
} }
} }
expectUnlimitedSwap := func(cgroupVersion CgroupVersion, resources ...*runtimeapi.LinuxContainerResources) {
const msg = "container is expected to have unlimited swap access"
for _, r := range resources {
switch cgroupVersion {
case cgroupV1:
assert.Equal(t, int64(-1), r.MemorySwapLimitInBytes, msg)
case cgroupV2:
assert.Equal(t, "max", r.Unified[cm.Cgroup2MaxSwapFilename], msg)
}
}
}
expectSwap := func(cgroupVersion CgroupVersion, swapBytesExpected int64, resources *runtimeapi.LinuxContainerResources) { expectSwap := func(cgroupVersion CgroupVersion, swapBytesExpected int64, resources *runtimeapi.LinuxContainerResources) {
msg := fmt.Sprintf("container swap is expected to be limited by %d bytes", swapBytesExpected) msg := fmt.Sprintf("container swap is expected to be limited by %d bytes", swapBytesExpected)
@ -967,13 +954,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
nodeSwapFeatureGateEnabled: true, nodeSwapFeatureGateEnabled: true,
swapBehavior: types.LimitedSwap, swapBehavior: types.LimitedSwap,
}, },
{
name: "cgroups v1, UnlimitedSwap, Burstable QoS",
cgroupVersion: cgroupV1,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
},
{ {
name: "cgroups v1, LimitedSwap, Best-effort QoS", name: "cgroups v1, LimitedSwap, Best-effort QoS",
cgroupVersion: cgroupV1, cgroupVersion: cgroupV1,
@ -990,17 +970,10 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
nodeSwapFeatureGateEnabled: false, nodeSwapFeatureGateEnabled: false,
swapBehavior: types.LimitedSwap, swapBehavior: types.LimitedSwap,
}, },
{
name: "NodeSwap feature gate turned off, cgroups v2, UnlimitedSwap",
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: false,
swapBehavior: types.UnlimitedSwap,
},
// With no swapBehavior, UnlimitedSwap should be the default // With no swapBehavior, NoSwap should be the default
{ {
name: "With no swapBehavior - UnlimitedSwap should be the default", name: "With no swapBehavior - NoSwap should be the default",
cgroupVersion: cgroupV2, cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBestEffort, qosClass: v1.PodQOSBestEffort,
nodeSwapFeatureGateEnabled: true, nodeSwapFeatureGateEnabled: true,
@ -1008,6 +981,13 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
}, },
// With Guaranteed and Best-effort QoS // With Guaranteed and Best-effort QoS
{
name: "Best-effort QoS, cgroups v2, NoSwap",
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBestEffort,
nodeSwapFeatureGateEnabled: true,
swapBehavior: "NoSwap",
},
{ {
name: "Best-effort QoS, cgroups v2, LimitedSwap", name: "Best-effort QoS, cgroups v2, LimitedSwap",
cgroupVersion: cgroupV2, cgroupVersion: cgroupV2,
@ -1015,13 +995,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
nodeSwapFeatureGateEnabled: true, nodeSwapFeatureGateEnabled: true,
swapBehavior: types.LimitedSwap, swapBehavior: types.LimitedSwap,
}, },
{
name: "Best-effort QoS, cgroups v2, UnlimitedSwap",
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
},
{ {
name: "Guaranteed QoS, cgroups v2, LimitedSwap", name: "Guaranteed QoS, cgroups v2, LimitedSwap",
cgroupVersion: cgroupV2, cgroupVersion: cgroupV2,
@ -1029,13 +1002,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
nodeSwapFeatureGateEnabled: true, nodeSwapFeatureGateEnabled: true,
swapBehavior: types.LimitedSwap, swapBehavior: types.LimitedSwap,
}, },
{
name: "Guaranteed QoS, cgroups v2, UnlimitedSwap",
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSGuaranteed,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
},
// With a "guaranteed" container (when memory requests equal to limits) // With a "guaranteed" container (when memory requests equal to limits)
{ {
@ -1047,15 +1013,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
addContainerWithoutRequests: false, addContainerWithoutRequests: false,
addGuaranteedContainer: true, addGuaranteedContainer: true,
}, },
{
name: "Burstable QoS, cgroups v2, UnlimitedSwap, with a guaranteed container",
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
addContainerWithoutRequests: false,
addGuaranteedContainer: true,
},
// Swap is expected to be allocated // Swap is expected to be allocated
{ {
@ -1067,15 +1024,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
addContainerWithoutRequests: false, addContainerWithoutRequests: false,
addGuaranteedContainer: false, addGuaranteedContainer: false,
}, },
{
name: "Burstable QoS, cgroups v2, UnlimitedSwap",
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
addContainerWithoutRequests: false,
addGuaranteedContainer: false,
},
{ {
name: "Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests", name: "Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests",
cgroupVersion: cgroupV2, cgroupVersion: cgroupV2,
@ -1085,15 +1033,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
addContainerWithoutRequests: true, addContainerWithoutRequests: true,
addGuaranteedContainer: false, addGuaranteedContainer: false,
}, },
{
name: "Burstable QoS, cgroups v2, UnlimitedSwap, with a container with no requests",
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
addContainerWithoutRequests: true,
addGuaranteedContainer: false,
},
// All the above examples with Swap disabled on node // All the above examples with Swap disabled on node
{ {
name: "Swap disabled on node, cgroups v1, LimitedSwap, Burstable QoS", name: "Swap disabled on node, cgroups v1, LimitedSwap, Burstable QoS",
@ -1103,14 +1042,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
nodeSwapFeatureGateEnabled: true, nodeSwapFeatureGateEnabled: true,
swapBehavior: types.LimitedSwap, swapBehavior: types.LimitedSwap,
}, },
{
name: "Swap disabled on node, cgroups v1, UnlimitedSwap, Burstable QoS",
swapDisabledOnNode: true,
cgroupVersion: cgroupV1,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
},
{ {
name: "Swap disabled on node, cgroups v1, LimitedSwap, Best-effort QoS", name: "Swap disabled on node, cgroups v1, LimitedSwap, Best-effort QoS",
swapDisabledOnNode: true, swapDisabledOnNode: true,
@ -1129,18 +1060,10 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
nodeSwapFeatureGateEnabled: false, nodeSwapFeatureGateEnabled: false,
swapBehavior: types.LimitedSwap, swapBehavior: types.LimitedSwap,
}, },
{
name: "Swap disabled on node, NodeSwap feature gate turned off, cgroups v2, UnlimitedSwap",
swapDisabledOnNode: true,
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: false,
swapBehavior: types.UnlimitedSwap,
},
// With no swapBehavior, UnlimitedSwap should be the default // With no swapBehavior, NoSwap should be the default
{ {
name: "Swap disabled on node, With no swapBehavior - UnlimitedSwap should be the default", name: "Swap disabled on node, With no swapBehavior - NoSwap should be the default",
swapDisabledOnNode: true, swapDisabledOnNode: true,
cgroupVersion: cgroupV2, cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBestEffort, qosClass: v1.PodQOSBestEffort,
@ -1157,14 +1080,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
nodeSwapFeatureGateEnabled: true, nodeSwapFeatureGateEnabled: true,
swapBehavior: types.LimitedSwap, swapBehavior: types.LimitedSwap,
}, },
{
name: "Swap disabled on node, Best-effort QoS, cgroups v2, UnlimitedSwap",
swapDisabledOnNode: true,
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
},
{ {
name: "Swap disabled on node, Guaranteed QoS, cgroups v2, LimitedSwap", name: "Swap disabled on node, Guaranteed QoS, cgroups v2, LimitedSwap",
swapDisabledOnNode: true, swapDisabledOnNode: true,
@ -1173,14 +1088,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
nodeSwapFeatureGateEnabled: true, nodeSwapFeatureGateEnabled: true,
swapBehavior: types.LimitedSwap, swapBehavior: types.LimitedSwap,
}, },
{
name: "Swap disabled on node, Guaranteed QoS, cgroups v2, UnlimitedSwap",
swapDisabledOnNode: true,
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSGuaranteed,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
},
// With a "guaranteed" container (when memory requests equal to limits) // With a "guaranteed" container (when memory requests equal to limits)
{ {
@ -1193,16 +1100,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
addContainerWithoutRequests: false, addContainerWithoutRequests: false,
addGuaranteedContainer: true, addGuaranteedContainer: true,
}, },
{
name: "Swap disabled on node, Burstable QoS, cgroups v2, UnlimitedSwap, with a guaranteed container",
swapDisabledOnNode: true,
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
addContainerWithoutRequests: false,
addGuaranteedContainer: true,
},
// Swap is expected to be allocated // Swap is expected to be allocated
{ {
@ -1215,16 +1112,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
addContainerWithoutRequests: false, addContainerWithoutRequests: false,
addGuaranteedContainer: false, addGuaranteedContainer: false,
}, },
{
name: "Swap disabled on node, Burstable QoS, cgroups v2, UnlimitedSwap",
swapDisabledOnNode: true,
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
addContainerWithoutRequests: false,
addGuaranteedContainer: false,
},
{ {
name: "Swap disabled on node, Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests", name: "Swap disabled on node, Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests",
swapDisabledOnNode: true, swapDisabledOnNode: true,
@ -1235,16 +1122,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
addContainerWithoutRequests: true, addContainerWithoutRequests: true,
addGuaranteedContainer: false, addGuaranteedContainer: false,
}, },
{
name: "Swap disabled on node, Burstable QoS, cgroups v2, UnlimitedSwap, with a container with no requests",
swapDisabledOnNode: true,
cgroupVersion: cgroupV2,
qosClass: v1.PodQOSBurstable,
nodeSwapFeatureGateEnabled: true,
swapBehavior: types.UnlimitedSwap,
addContainerWithoutRequests: true,
addGuaranteedContainer: false,
},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
setCgroupVersionDuringTest(tc.cgroupVersion) setCgroupVersionDuringTest(tc.cgroupVersion)
@ -1294,8 +1171,8 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
return return
} }
if tc.swapBehavior == types.UnlimitedSwap || tc.swapBehavior == "" { if tc.swapBehavior == types.NoSwap || tc.swapBehavior == "" {
expectUnlimitedSwap(tc.cgroupVersion, resourcesC1, resourcesC2) expectNoSwap(tc.cgroupVersion, resourcesC1, resourcesC2)
return return
} }

View File

@ -36,5 +36,5 @@ const (
// SwapBehavior types // SwapBehavior types
const ( const (
LimitedSwap = "LimitedSwap" LimitedSwap = "LimitedSwap"
UnlimitedSwap = "UnlimitedSwap" NoSwap = "NoSwap"
) )

View File

@ -954,8 +954,8 @@ type ShutdownGracePeriodByPodPriority struct {
type MemorySwapConfiguration struct { type MemorySwapConfiguration struct {
// swapBehavior configures swap memory available to container workloads. May be one of // swapBehavior configures swap memory available to container workloads. May be one of
// "", "LimitedSwap": workload combined memory and swap usage cannot exceed pod memory limit // "", "NoSwap": workloads can not use swap, default option.
// "UnlimitedSwap": workloads can use unlimited swap, up to the allocatable limit. // "LimitedSwap": workload swap usage is limited. The swap limit is proportionate to the container's memory request.
// +featureGate=NodeSwap // +featureGate=NodeSwap
// +optional // +optional
SwapBehavior string `json:"swapBehavior,omitempty"` SwapBehavior string `json:"swapBehavior,omitempty"`

View File

@ -24,8 +24,6 @@ import (
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
"k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@ -119,18 +117,6 @@ func runOomKillerTest(f *framework.Framework, testCase testCase, kubeReservedMem
}) })
ginkgo.It("The containers terminated by OOM killer should have the reason set to OOMKilled", func() { ginkgo.It("The containers terminated by OOM killer should have the reason set to OOMKilled", func() {
cfg, configErr := getCurrentKubeletConfig(context.TODO())
framework.ExpectNoError(configErr)
if utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) {
// If Swap is enabled, we should test OOM with LimitedSwap.
// UnlimitedSwap allows for workloads to use unbounded swap which
// makes testing OOM challenging.
// We are not able to change the default for these conformance tests,
// so we will skip these tests if swap is enabled.
if cfg.MemorySwap.SwapBehavior == "" || cfg.MemorySwap.SwapBehavior == "UnlimitedSwap" {
ginkgo.Skip("OOMKiller should not run with UnlimitedSwap")
}
}
ginkgo.By("Waiting for the pod to be failed") ginkgo.By("Waiting for the pod to be failed")
err := e2epod.WaitForPodTerminatedInNamespace(context.TODO(), f.ClientSet, testCase.podSpec.Name, "", f.Namespace.Name) err := e2epod.WaitForPodTerminatedInNamespace(context.TODO(), f.ClientSet, testCase.podSpec.Name, "", f.Namespace.Name)
framework.ExpectNoError(err, "Failed waiting for pod to terminate, %s/%s", f.Namespace.Name, testCase.podSpec.Name) framework.ExpectNoError(err, "Failed waiting for pod to terminate, %s/%s", f.Namespace.Name, testCase.podSpec.Name)

View File

@ -55,16 +55,17 @@ var _ = SIGDescribe("Swap", framework.WithNodeConformance(), "[LinuxOnly]", func
isCgroupV2 := isPodCgroupV2(f, pod) isCgroupV2 := isPodCgroupV2(f, pod)
isLimitedSwap := isLimitedSwap(f, pod) isLimitedSwap := isLimitedSwap(f, pod)
isNoSwap := isNoSwap(f, pod)
if !isSwapFeatureGateEnabled() || !isCgroupV2 || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) { if !isSwapFeatureGateEnabled() || !isCgroupV2 || isNoSwap || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) {
ginkgo.By(fmt.Sprintf("Expecting no swap. feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable)) ginkgo.By(fmt.Sprintf("Expecting no swap. isNoSwap? %t, feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isNoSwap, isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable))
expectNoSwap(f, pod, isCgroupV2) expectNoSwap(f, pod, isCgroupV2)
return return
} }
if !isLimitedSwap { if !isLimitedSwap {
ginkgo.By("expecting unlimited swap") ginkgo.By("expecting no swap")
expectUnlimitedSwap(f, pod, isCgroupV2) expectNoSwap(f, pod, isCgroupV2)
return return
} }
@ -176,16 +177,6 @@ func expectNoSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) {
} }
} }
func expectUnlimitedSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) {
if isCgroupV2 {
swapLimit := readCgroupFile(f, pod, cgroupV2SwapLimitFile)
gomega.ExpectWithOffset(1, swapLimit).To(gomega.Equal("max"), "max swap allowed should be \"max\"")
} else {
swapPlusMemLimit := readCgroupFile(f, pod, cgroupV1SwapLimitFile)
gomega.ExpectWithOffset(1, swapPlusMemLimit).To(gomega.Equal("-1"))
}
}
// supports v2 only as v1 shouldn't support LimitedSwap // supports v2 only as v1 shouldn't support LimitedSwap
func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) { func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) {
swapLimitStr := readCgroupFile(f, pod, cgroupV2SwapLimitFile) swapLimitStr := readCgroupFile(f, pod, cgroupV2SwapLimitFile)
@ -253,3 +244,10 @@ func isLimitedSwap(f *framework.Framework, pod *v1.Pod) bool {
return kubeletCfg.MemorySwap.SwapBehavior == types.LimitedSwap return kubeletCfg.MemorySwap.SwapBehavior == types.LimitedSwap
} }
func isNoSwap(f *framework.Framework, pod *v1.Pod) bool {
kubeletCfg, err := getCurrentKubeletConfig(context.Background())
framework.ExpectNoError(err, "cannot get kubelet config")
return kubeletCfg.MemorySwap.SwapBehavior == types.NoSwap || kubeletCfg.MemorySwap.SwapBehavior == ""
}