mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-11 04:52:08 +00:00
add no swap as the default option for swap
This commit is contained in:
parent
b0ee334374
commit
6a4e19a4ec
@ -105,6 +105,7 @@ import (
|
|||||||
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
|
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/server"
|
"k8s.io/kubernetes/pkg/kubelet/server"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/stats/pidlimit"
|
"k8s.io/kubernetes/pkg/kubelet/stats/pidlimit"
|
||||||
|
kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||||
kubeletutil "k8s.io/kubernetes/pkg/kubelet/util"
|
kubeletutil "k8s.io/kubernetes/pkg/kubelet/util"
|
||||||
utilfs "k8s.io/kubernetes/pkg/util/filesystem"
|
utilfs "k8s.io/kubernetes/pkg/util/filesystem"
|
||||||
"k8s.io/kubernetes/pkg/util/flock"
|
"k8s.io/kubernetes/pkg/util/flock"
|
||||||
@ -803,6 +804,14 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
|
|||||||
return fmt.Errorf("topology manager policy options %v require feature gates %q enabled",
|
return fmt.Errorf("topology manager policy options %v require feature gates %q enabled",
|
||||||
s.TopologyManagerPolicyOptions, features.TopologyManagerPolicyOptions)
|
s.TopologyManagerPolicyOptions, features.TopologyManagerPolicyOptions)
|
||||||
}
|
}
|
||||||
|
if utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) {
|
||||||
|
if !isCgroup2UnifiedMode() && s.MemorySwap.SwapBehavior == kubelettypes.LimitedSwap {
|
||||||
|
klog.InfoS("Swap feature is enabled and LimitedSwap but it is only supported with cgroupv2", "CGroupV2", isCgroup2UnifiedMode(), "SwapBehavior", s.MemorySwap.SwapBehavior)
|
||||||
|
}
|
||||||
|
if !s.FailSwapOn && s.MemorySwap.SwapBehavior == "" {
|
||||||
|
klog.InfoS("NoSwap is set due to memorySwapBehavior not specified", "memorySwapBehavior", s.MemorySwap.SwapBehavior, "FailSwapOn", s.FailSwapOn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
kubeDeps.ContainerManager, err = cm.NewContainerManager(
|
kubeDeps.ContainerManager, err = cm.NewContainerManager(
|
||||||
kubeDeps.Mounter,
|
kubeDeps.Mounter,
|
||||||
|
@ -539,11 +539,13 @@ const (
|
|||||||
// Allow pods to failover to a different node in case of non graceful node shutdown
|
// Allow pods to failover to a different node in case of non graceful node shutdown
|
||||||
NodeOutOfServiceVolumeDetach featuregate.Feature = "NodeOutOfServiceVolumeDetach"
|
NodeOutOfServiceVolumeDetach featuregate.Feature = "NodeOutOfServiceVolumeDetach"
|
||||||
|
|
||||||
// owner: @iholder101
|
// owner: @iholder101 @kannon92
|
||||||
|
// kep: https://kep.k8s.io/2400
|
||||||
// alpha: v1.22
|
// alpha: v1.22
|
||||||
// beta1: v1.28. For more info, please look at the KEP: https://kep.k8s.io/2400.
|
// beta1: v1.28 (default=false)
|
||||||
//
|
// beta2: v.1.30 (default=true)
|
||||||
// Permits kubelet to run with swap enabled
|
|
||||||
|
// Permits kubelet to run with swap enabled.
|
||||||
NodeSwap featuregate.Feature = "NodeSwap"
|
NodeSwap featuregate.Feature = "NodeSwap"
|
||||||
|
|
||||||
// owner: @mortent, @atiratree, @ravig
|
// owner: @mortent, @atiratree, @ravig
|
||||||
@ -1105,7 +1107,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
|
|||||||
|
|
||||||
NodeOutOfServiceVolumeDetach: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31
|
NodeOutOfServiceVolumeDetach: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31
|
||||||
|
|
||||||
NodeSwap: {Default: false, PreRelease: featuregate.Beta},
|
NodeSwap: {Default: true, PreRelease: featuregate.Beta},
|
||||||
|
|
||||||
PDBUnhealthyPodEvictionPolicy: {Default: true, PreRelease: featuregate.Beta},
|
PDBUnhealthyPodEvictionPolicy: {Default: true, PreRelease: featuregate.Beta},
|
||||||
|
|
||||||
|
2
pkg/generated/openapi/zz_generated.openapi.go
generated
2
pkg/generated/openapi/zz_generated.openapi.go
generated
@ -59686,7 +59686,7 @@ func schema_k8sio_kubelet_config_v1beta1_MemorySwapConfiguration(ref common.Refe
|
|||||||
Properties: map[string]spec.Schema{
|
Properties: map[string]spec.Schema{
|
||||||
"swapBehavior": {
|
"swapBehavior": {
|
||||||
SchemaProps: spec.SchemaProps{
|
SchemaProps: spec.SchemaProps{
|
||||||
Description: "swapBehavior configures swap memory available to container workloads. May be one of \"\", \"LimitedSwap\": workload combined memory and swap usage cannot exceed pod memory limit \"UnlimitedSwap\": workloads can use unlimited swap, up to the allocatable limit.",
|
Description: "swapBehavior configures swap memory available to container workloads. May be one of \"\", \"NoSwap\": workloads can not use swap, default option. \"LimitedSwap\": workload swap usage is limited. The swap limit is proportionate to the container's memory request.",
|
||||||
Type: []string{"string"},
|
Type: []string{"string"},
|
||||||
Format: "",
|
Format: "",
|
||||||
},
|
},
|
||||||
|
@ -666,8 +666,8 @@ type ShutdownGracePeriodByPodPriority struct {
|
|||||||
|
|
||||||
type MemorySwapConfiguration struct {
|
type MemorySwapConfiguration struct {
|
||||||
// swapBehavior configures swap memory available to container workloads. May be one of
|
// swapBehavior configures swap memory available to container workloads. May be one of
|
||||||
// "", "LimitedSwap": workload combined memory and swap usage cannot exceed pod memory limit
|
// "", "NoSwap": workloads can not use swap, default option.
|
||||||
// "UnlimitedSwap": workloads can use unlimited swap, up to the allocatable limit.
|
// "LimitedSwap": workload swap usage is limited. The swap limit is proportionate to the container's memory request.
|
||||||
// +featureGate=NodeSwap
|
// +featureGate=NodeSwap
|
||||||
// +optional
|
// +optional
|
||||||
SwapBehavior string
|
SwapBehavior string
|
||||||
|
@ -471,7 +471,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
|
|||||||
IPTablesMasqueradeBit: utilpointer.Int32(1),
|
IPTablesMasqueradeBit: utilpointer.Int32(1),
|
||||||
IPTablesDropBit: utilpointer.Int32(1),
|
IPTablesDropBit: utilpointer.Int32(1),
|
||||||
FailSwapOn: utilpointer.Bool(true),
|
FailSwapOn: utilpointer.Bool(true),
|
||||||
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "UnlimitedSwap"},
|
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "NoSwap"},
|
||||||
ContainerLogMaxSize: "1Mi",
|
ContainerLogMaxSize: "1Mi",
|
||||||
ContainerLogMaxFiles: utilpointer.Int32(1),
|
ContainerLogMaxFiles: utilpointer.Int32(1),
|
||||||
ContainerLogMaxWorkers: utilpointer.Int32(1),
|
ContainerLogMaxWorkers: utilpointer.Int32(1),
|
||||||
@ -620,7 +620,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) {
|
|||||||
IPTablesMasqueradeBit: utilpointer.Int32(1),
|
IPTablesMasqueradeBit: utilpointer.Int32(1),
|
||||||
IPTablesDropBit: utilpointer.Int32(1),
|
IPTablesDropBit: utilpointer.Int32(1),
|
||||||
FailSwapOn: utilpointer.Bool(true),
|
FailSwapOn: utilpointer.Bool(true),
|
||||||
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "UnlimitedSwap"},
|
MemorySwap: v1beta1.MemorySwapConfiguration{SwapBehavior: "NoSwap"},
|
||||||
ContainerLogMaxSize: "1Mi",
|
ContainerLogMaxSize: "1Mi",
|
||||||
ContainerLogMaxFiles: utilpointer.Int32(1),
|
ContainerLogMaxFiles: utilpointer.Int32(1),
|
||||||
ContainerLogMaxWorkers: utilpointer.Int32(1),
|
ContainerLogMaxWorkers: utilpointer.Int32(1),
|
||||||
|
@ -195,10 +195,10 @@ func ValidateKubeletConfiguration(kc *kubeletconfig.KubeletConfiguration, featur
|
|||||||
if localFeatureGate.Enabled(features.NodeSwap) {
|
if localFeatureGate.Enabled(features.NodeSwap) {
|
||||||
switch kc.MemorySwap.SwapBehavior {
|
switch kc.MemorySwap.SwapBehavior {
|
||||||
case "":
|
case "":
|
||||||
|
case kubetypes.NoSwap:
|
||||||
case kubetypes.LimitedSwap:
|
case kubetypes.LimitedSwap:
|
||||||
case kubetypes.UnlimitedSwap:
|
|
||||||
default:
|
default:
|
||||||
allErrors = append(allErrors, fmt.Errorf("invalid configuration: memorySwap.swapBehavior %q must be one of: \"\", %q, or %q", kc.MemorySwap.SwapBehavior, kubetypes.LimitedSwap, kubetypes.UnlimitedSwap))
|
allErrors = append(allErrors, fmt.Errorf("invalid configuration: memorySwap.swapBehavior %q must be one of: \"\", %q or %q", kc.MemorySwap.SwapBehavior, kubetypes.LimitedSwap, kubetypes.NoSwap))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !localFeatureGate.Enabled(features.NodeSwap) && kc.MemorySwap != (kubeletconfig.MemorySwapConfiguration{}) {
|
if !localFeatureGate.Enabled(features.NodeSwap) && kc.MemorySwap != (kubeletconfig.MemorySwapConfiguration{}) {
|
||||||
|
@ -367,7 +367,7 @@ func TestValidateKubeletConfiguration(t *testing.T) {
|
|||||||
conf.MemorySwap.SwapBehavior = "invalid-behavior"
|
conf.MemorySwap.SwapBehavior = "invalid-behavior"
|
||||||
return conf
|
return conf
|
||||||
},
|
},
|
||||||
errMsg: "invalid configuration: memorySwap.swapBehavior \"invalid-behavior\" must be one of: \"\", \"LimitedSwap\", or \"UnlimitedSwap\"",
|
errMsg: "invalid configuration: memorySwap.swapBehavior \"invalid-behavior\" must be one of: \"\", \"LimitedSwap\" or \"NoSwap\"",
|
||||||
}, {
|
}, {
|
||||||
name: "specify MemorySwap.SwapBehavior without enabling NodeSwap",
|
name: "specify MemorySwap.SwapBehavior without enabling NodeSwap",
|
||||||
configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
|
configure: func(conf *kubeletconfig.KubeletConfiguration) *kubeletconfig.KubeletConfiguration {
|
||||||
|
@ -168,6 +168,12 @@ func (m *kubeGenericRuntimeManager) configureContainerSwapResources(lcr *runtime
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
swapConfigurationHelper := newSwapConfigurationHelper(*m.machineInfo)
|
swapConfigurationHelper := newSwapConfigurationHelper(*m.machineInfo)
|
||||||
|
if m.memorySwapBehavior == kubelettypes.LimitedSwap {
|
||||||
|
if !isCgroup2UnifiedMode() {
|
||||||
|
swapConfigurationHelper.ConfigureNoSwap(lcr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.NodeSwap) {
|
if !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.NodeSwap) {
|
||||||
swapConfigurationHelper.ConfigureNoSwap(lcr)
|
swapConfigurationHelper.ConfigureNoSwap(lcr)
|
||||||
@ -177,10 +183,12 @@ func (m *kubeGenericRuntimeManager) configureContainerSwapResources(lcr *runtime
|
|||||||
// NOTE(ehashman): Behavior is defined in the opencontainers runtime spec:
|
// NOTE(ehashman): Behavior is defined in the opencontainers runtime spec:
|
||||||
// https://github.com/opencontainers/runtime-spec/blob/1c3f411f041711bbeecf35ff7e93461ea6789220/config-linux.md#memory
|
// https://github.com/opencontainers/runtime-spec/blob/1c3f411f041711bbeecf35ff7e93461ea6789220/config-linux.md#memory
|
||||||
switch m.memorySwapBehavior {
|
switch m.memorySwapBehavior {
|
||||||
|
case kubelettypes.NoSwap:
|
||||||
|
swapConfigurationHelper.ConfigureNoSwap(lcr)
|
||||||
case kubelettypes.LimitedSwap:
|
case kubelettypes.LimitedSwap:
|
||||||
swapConfigurationHelper.ConfigureLimitedSwap(lcr, pod, container)
|
swapConfigurationHelper.ConfigureLimitedSwap(lcr, pod, container)
|
||||||
default:
|
default:
|
||||||
swapConfigurationHelper.ConfigureUnlimitedSwap(lcr)
|
swapConfigurationHelper.ConfigureNoSwap(lcr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -401,19 +409,6 @@ func (m swapConfigurationHelper) ConfigureNoSwap(lcr *runtimeapi.LinuxContainerR
|
|||||||
m.configureSwap(lcr, 0)
|
m.configureSwap(lcr, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m swapConfigurationHelper) ConfigureUnlimitedSwap(lcr *runtimeapi.LinuxContainerResources) {
|
|
||||||
if !isCgroup2UnifiedMode() {
|
|
||||||
m.ConfigureNoSwap(lcr)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if lcr.Unified == nil {
|
|
||||||
lcr.Unified = map[string]string{}
|
|
||||||
}
|
|
||||||
|
|
||||||
lcr.Unified[cm.Cgroup2MaxSwapFilename] = "max"
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m swapConfigurationHelper) configureSwap(lcr *runtimeapi.LinuxContainerResources, swapMemory int64) {
|
func (m swapConfigurationHelper) configureSwap(lcr *runtimeapi.LinuxContainerResources, swapMemory int64) {
|
||||||
if !isCgroup2UnifiedMode() {
|
if !isCgroup2UnifiedMode() {
|
||||||
klog.ErrorS(fmt.Errorf("swap configuration is not supported with cgroup v1"), "swap configuration under cgroup v1 is unexpected")
|
klog.ErrorS(fmt.Errorf("swap configuration is not supported with cgroup v1"), "swap configuration under cgroup v1 is unexpected")
|
||||||
|
@ -918,19 +918,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
expectUnlimitedSwap := func(cgroupVersion CgroupVersion, resources ...*runtimeapi.LinuxContainerResources) {
|
|
||||||
const msg = "container is expected to have unlimited swap access"
|
|
||||||
|
|
||||||
for _, r := range resources {
|
|
||||||
switch cgroupVersion {
|
|
||||||
case cgroupV1:
|
|
||||||
assert.Equal(t, int64(-1), r.MemorySwapLimitInBytes, msg)
|
|
||||||
case cgroupV2:
|
|
||||||
assert.Equal(t, "max", r.Unified[cm.Cgroup2MaxSwapFilename], msg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
expectSwap := func(cgroupVersion CgroupVersion, swapBytesExpected int64, resources *runtimeapi.LinuxContainerResources) {
|
expectSwap := func(cgroupVersion CgroupVersion, swapBytesExpected int64, resources *runtimeapi.LinuxContainerResources) {
|
||||||
msg := fmt.Sprintf("container swap is expected to be limited by %d bytes", swapBytesExpected)
|
msg := fmt.Sprintf("container swap is expected to be limited by %d bytes", swapBytesExpected)
|
||||||
|
|
||||||
@ -967,13 +954,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
nodeSwapFeatureGateEnabled: true,
|
nodeSwapFeatureGateEnabled: true,
|
||||||
swapBehavior: types.LimitedSwap,
|
swapBehavior: types.LimitedSwap,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "cgroups v1, UnlimitedSwap, Burstable QoS",
|
|
||||||
cgroupVersion: cgroupV1,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "cgroups v1, LimitedSwap, Best-effort QoS",
|
name: "cgroups v1, LimitedSwap, Best-effort QoS",
|
||||||
cgroupVersion: cgroupV1,
|
cgroupVersion: cgroupV1,
|
||||||
@ -990,17 +970,10 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
nodeSwapFeatureGateEnabled: false,
|
nodeSwapFeatureGateEnabled: false,
|
||||||
swapBehavior: types.LimitedSwap,
|
swapBehavior: types.LimitedSwap,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "NodeSwap feature gate turned off, cgroups v2, UnlimitedSwap",
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: false,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
},
|
|
||||||
|
|
||||||
// With no swapBehavior, UnlimitedSwap should be the default
|
// With no swapBehavior, NoSwap should be the default
|
||||||
{
|
{
|
||||||
name: "With no swapBehavior - UnlimitedSwap should be the default",
|
name: "With no swapBehavior - NoSwap should be the default",
|
||||||
cgroupVersion: cgroupV2,
|
cgroupVersion: cgroupV2,
|
||||||
qosClass: v1.PodQOSBestEffort,
|
qosClass: v1.PodQOSBestEffort,
|
||||||
nodeSwapFeatureGateEnabled: true,
|
nodeSwapFeatureGateEnabled: true,
|
||||||
@ -1008,6 +981,13 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
},
|
},
|
||||||
|
|
||||||
// With Guaranteed and Best-effort QoS
|
// With Guaranteed and Best-effort QoS
|
||||||
|
{
|
||||||
|
name: "Best-effort QoS, cgroups v2, NoSwap",
|
||||||
|
cgroupVersion: cgroupV2,
|
||||||
|
qosClass: v1.PodQOSBestEffort,
|
||||||
|
nodeSwapFeatureGateEnabled: true,
|
||||||
|
swapBehavior: "NoSwap",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "Best-effort QoS, cgroups v2, LimitedSwap",
|
name: "Best-effort QoS, cgroups v2, LimitedSwap",
|
||||||
cgroupVersion: cgroupV2,
|
cgroupVersion: cgroupV2,
|
||||||
@ -1015,13 +995,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
nodeSwapFeatureGateEnabled: true,
|
nodeSwapFeatureGateEnabled: true,
|
||||||
swapBehavior: types.LimitedSwap,
|
swapBehavior: types.LimitedSwap,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Best-effort QoS, cgroups v2, UnlimitedSwap",
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "Guaranteed QoS, cgroups v2, LimitedSwap",
|
name: "Guaranteed QoS, cgroups v2, LimitedSwap",
|
||||||
cgroupVersion: cgroupV2,
|
cgroupVersion: cgroupV2,
|
||||||
@ -1029,13 +1002,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
nodeSwapFeatureGateEnabled: true,
|
nodeSwapFeatureGateEnabled: true,
|
||||||
swapBehavior: types.LimitedSwap,
|
swapBehavior: types.LimitedSwap,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Guaranteed QoS, cgroups v2, UnlimitedSwap",
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSGuaranteed,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
},
|
|
||||||
|
|
||||||
// With a "guaranteed" container (when memory requests equal to limits)
|
// With a "guaranteed" container (when memory requests equal to limits)
|
||||||
{
|
{
|
||||||
@ -1047,15 +1013,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
addContainerWithoutRequests: false,
|
addContainerWithoutRequests: false,
|
||||||
addGuaranteedContainer: true,
|
addGuaranteedContainer: true,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Burstable QoS, cgroups v2, UnlimitedSwap, with a guaranteed container",
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
addContainerWithoutRequests: false,
|
|
||||||
addGuaranteedContainer: true,
|
|
||||||
},
|
|
||||||
|
|
||||||
// Swap is expected to be allocated
|
// Swap is expected to be allocated
|
||||||
{
|
{
|
||||||
@ -1067,15 +1024,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
addContainerWithoutRequests: false,
|
addContainerWithoutRequests: false,
|
||||||
addGuaranteedContainer: false,
|
addGuaranteedContainer: false,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Burstable QoS, cgroups v2, UnlimitedSwap",
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
addContainerWithoutRequests: false,
|
|
||||||
addGuaranteedContainer: false,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests",
|
name: "Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests",
|
||||||
cgroupVersion: cgroupV2,
|
cgroupVersion: cgroupV2,
|
||||||
@ -1085,15 +1033,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
addContainerWithoutRequests: true,
|
addContainerWithoutRequests: true,
|
||||||
addGuaranteedContainer: false,
|
addGuaranteedContainer: false,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Burstable QoS, cgroups v2, UnlimitedSwap, with a container with no requests",
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
addContainerWithoutRequests: true,
|
|
||||||
addGuaranteedContainer: false,
|
|
||||||
},
|
|
||||||
// All the above examples with Swap disabled on node
|
// All the above examples with Swap disabled on node
|
||||||
{
|
{
|
||||||
name: "Swap disabled on node, cgroups v1, LimitedSwap, Burstable QoS",
|
name: "Swap disabled on node, cgroups v1, LimitedSwap, Burstable QoS",
|
||||||
@ -1103,14 +1042,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
nodeSwapFeatureGateEnabled: true,
|
nodeSwapFeatureGateEnabled: true,
|
||||||
swapBehavior: types.LimitedSwap,
|
swapBehavior: types.LimitedSwap,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Swap disabled on node, cgroups v1, UnlimitedSwap, Burstable QoS",
|
|
||||||
swapDisabledOnNode: true,
|
|
||||||
cgroupVersion: cgroupV1,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "Swap disabled on node, cgroups v1, LimitedSwap, Best-effort QoS",
|
name: "Swap disabled on node, cgroups v1, LimitedSwap, Best-effort QoS",
|
||||||
swapDisabledOnNode: true,
|
swapDisabledOnNode: true,
|
||||||
@ -1129,18 +1060,10 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
nodeSwapFeatureGateEnabled: false,
|
nodeSwapFeatureGateEnabled: false,
|
||||||
swapBehavior: types.LimitedSwap,
|
swapBehavior: types.LimitedSwap,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Swap disabled on node, NodeSwap feature gate turned off, cgroups v2, UnlimitedSwap",
|
|
||||||
swapDisabledOnNode: true,
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: false,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
},
|
|
||||||
|
|
||||||
// With no swapBehavior, UnlimitedSwap should be the default
|
// With no swapBehavior, NoSwap should be the default
|
||||||
{
|
{
|
||||||
name: "Swap disabled on node, With no swapBehavior - UnlimitedSwap should be the default",
|
name: "Swap disabled on node, With no swapBehavior - NoSwap should be the default",
|
||||||
swapDisabledOnNode: true,
|
swapDisabledOnNode: true,
|
||||||
cgroupVersion: cgroupV2,
|
cgroupVersion: cgroupV2,
|
||||||
qosClass: v1.PodQOSBestEffort,
|
qosClass: v1.PodQOSBestEffort,
|
||||||
@ -1157,14 +1080,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
nodeSwapFeatureGateEnabled: true,
|
nodeSwapFeatureGateEnabled: true,
|
||||||
swapBehavior: types.LimitedSwap,
|
swapBehavior: types.LimitedSwap,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Swap disabled on node, Best-effort QoS, cgroups v2, UnlimitedSwap",
|
|
||||||
swapDisabledOnNode: true,
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "Swap disabled on node, Guaranteed QoS, cgroups v2, LimitedSwap",
|
name: "Swap disabled on node, Guaranteed QoS, cgroups v2, LimitedSwap",
|
||||||
swapDisabledOnNode: true,
|
swapDisabledOnNode: true,
|
||||||
@ -1173,14 +1088,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
nodeSwapFeatureGateEnabled: true,
|
nodeSwapFeatureGateEnabled: true,
|
||||||
swapBehavior: types.LimitedSwap,
|
swapBehavior: types.LimitedSwap,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Swap disabled on node, Guaranteed QoS, cgroups v2, UnlimitedSwap",
|
|
||||||
swapDisabledOnNode: true,
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSGuaranteed,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
},
|
|
||||||
|
|
||||||
// With a "guaranteed" container (when memory requests equal to limits)
|
// With a "guaranteed" container (when memory requests equal to limits)
|
||||||
{
|
{
|
||||||
@ -1193,16 +1100,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
addContainerWithoutRequests: false,
|
addContainerWithoutRequests: false,
|
||||||
addGuaranteedContainer: true,
|
addGuaranteedContainer: true,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Swap disabled on node, Burstable QoS, cgroups v2, UnlimitedSwap, with a guaranteed container",
|
|
||||||
swapDisabledOnNode: true,
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
addContainerWithoutRequests: false,
|
|
||||||
addGuaranteedContainer: true,
|
|
||||||
},
|
|
||||||
|
|
||||||
// Swap is expected to be allocated
|
// Swap is expected to be allocated
|
||||||
{
|
{
|
||||||
@ -1215,16 +1112,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
addContainerWithoutRequests: false,
|
addContainerWithoutRequests: false,
|
||||||
addGuaranteedContainer: false,
|
addGuaranteedContainer: false,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Swap disabled on node, Burstable QoS, cgroups v2, UnlimitedSwap",
|
|
||||||
swapDisabledOnNode: true,
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
addContainerWithoutRequests: false,
|
|
||||||
addGuaranteedContainer: false,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "Swap disabled on node, Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests",
|
name: "Swap disabled on node, Burstable QoS, cgroups v2, LimitedSwap, with a container with no requests",
|
||||||
swapDisabledOnNode: true,
|
swapDisabledOnNode: true,
|
||||||
@ -1235,16 +1122,6 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
addContainerWithoutRequests: true,
|
addContainerWithoutRequests: true,
|
||||||
addGuaranteedContainer: false,
|
addGuaranteedContainer: false,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "Swap disabled on node, Burstable QoS, cgroups v2, UnlimitedSwap, with a container with no requests",
|
|
||||||
swapDisabledOnNode: true,
|
|
||||||
cgroupVersion: cgroupV2,
|
|
||||||
qosClass: v1.PodQOSBurstable,
|
|
||||||
nodeSwapFeatureGateEnabled: true,
|
|
||||||
swapBehavior: types.UnlimitedSwap,
|
|
||||||
addContainerWithoutRequests: true,
|
|
||||||
addGuaranteedContainer: false,
|
|
||||||
},
|
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
setCgroupVersionDuringTest(tc.cgroupVersion)
|
setCgroupVersionDuringTest(tc.cgroupVersion)
|
||||||
@ -1294,8 +1171,8 @@ func TestGenerateLinuxContainerResourcesWithSwap(t *testing.T) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if tc.swapBehavior == types.UnlimitedSwap || tc.swapBehavior == "" {
|
if tc.swapBehavior == types.NoSwap || tc.swapBehavior == "" {
|
||||||
expectUnlimitedSwap(tc.cgroupVersion, resourcesC1, resourcesC2)
|
expectNoSwap(tc.cgroupVersion, resourcesC1, resourcesC2)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -36,5 +36,5 @@ const (
|
|||||||
// SwapBehavior types
|
// SwapBehavior types
|
||||||
const (
|
const (
|
||||||
LimitedSwap = "LimitedSwap"
|
LimitedSwap = "LimitedSwap"
|
||||||
UnlimitedSwap = "UnlimitedSwap"
|
NoSwap = "NoSwap"
|
||||||
)
|
)
|
||||||
|
@ -954,8 +954,8 @@ type ShutdownGracePeriodByPodPriority struct {
|
|||||||
|
|
||||||
type MemorySwapConfiguration struct {
|
type MemorySwapConfiguration struct {
|
||||||
// swapBehavior configures swap memory available to container workloads. May be one of
|
// swapBehavior configures swap memory available to container workloads. May be one of
|
||||||
// "", "LimitedSwap": workload combined memory and swap usage cannot exceed pod memory limit
|
// "", "NoSwap": workloads can not use swap, default option.
|
||||||
// "UnlimitedSwap": workloads can use unlimited swap, up to the allocatable limit.
|
// "LimitedSwap": workload swap usage is limited. The swap limit is proportionate to the container's memory request.
|
||||||
// +featureGate=NodeSwap
|
// +featureGate=NodeSwap
|
||||||
// +optional
|
// +optional
|
||||||
SwapBehavior string `json:"swapBehavior,omitempty"`
|
SwapBehavior string `json:"swapBehavior,omitempty"`
|
||||||
|
@ -24,8 +24,6 @@ import (
|
|||||||
v1 "k8s.io/api/core/v1"
|
v1 "k8s.io/api/core/v1"
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
|
||||||
"k8s.io/kubernetes/pkg/features"
|
|
||||||
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
|
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
"k8s.io/kubernetes/test/e2e/framework"
|
||||||
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
||||||
@ -119,18 +117,6 @@ func runOomKillerTest(f *framework.Framework, testCase testCase, kubeReservedMem
|
|||||||
})
|
})
|
||||||
|
|
||||||
ginkgo.It("The containers terminated by OOM killer should have the reason set to OOMKilled", func() {
|
ginkgo.It("The containers terminated by OOM killer should have the reason set to OOMKilled", func() {
|
||||||
cfg, configErr := getCurrentKubeletConfig(context.TODO())
|
|
||||||
framework.ExpectNoError(configErr)
|
|
||||||
if utilfeature.DefaultFeatureGate.Enabled(features.NodeSwap) {
|
|
||||||
// If Swap is enabled, we should test OOM with LimitedSwap.
|
|
||||||
// UnlimitedSwap allows for workloads to use unbounded swap which
|
|
||||||
// makes testing OOM challenging.
|
|
||||||
// We are not able to change the default for these conformance tests,
|
|
||||||
// so we will skip these tests if swap is enabled.
|
|
||||||
if cfg.MemorySwap.SwapBehavior == "" || cfg.MemorySwap.SwapBehavior == "UnlimitedSwap" {
|
|
||||||
ginkgo.Skip("OOMKiller should not run with UnlimitedSwap")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ginkgo.By("Waiting for the pod to be failed")
|
ginkgo.By("Waiting for the pod to be failed")
|
||||||
err := e2epod.WaitForPodTerminatedInNamespace(context.TODO(), f.ClientSet, testCase.podSpec.Name, "", f.Namespace.Name)
|
err := e2epod.WaitForPodTerminatedInNamespace(context.TODO(), f.ClientSet, testCase.podSpec.Name, "", f.Namespace.Name)
|
||||||
framework.ExpectNoError(err, "Failed waiting for pod to terminate, %s/%s", f.Namespace.Name, testCase.podSpec.Name)
|
framework.ExpectNoError(err, "Failed waiting for pod to terminate, %s/%s", f.Namespace.Name, testCase.podSpec.Name)
|
||||||
|
@ -55,16 +55,17 @@ var _ = SIGDescribe("Swap", framework.WithNodeConformance(), "[LinuxOnly]", func
|
|||||||
|
|
||||||
isCgroupV2 := isPodCgroupV2(f, pod)
|
isCgroupV2 := isPodCgroupV2(f, pod)
|
||||||
isLimitedSwap := isLimitedSwap(f, pod)
|
isLimitedSwap := isLimitedSwap(f, pod)
|
||||||
|
isNoSwap := isNoSwap(f, pod)
|
||||||
|
|
||||||
if !isSwapFeatureGateEnabled() || !isCgroupV2 || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) {
|
if !isSwapFeatureGateEnabled() || !isCgroupV2 || isNoSwap || (isLimitedSwap && (qosClass != v1.PodQOSBurstable || memoryRequestEqualLimit)) {
|
||||||
ginkgo.By(fmt.Sprintf("Expecting no swap. feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable))
|
ginkgo.By(fmt.Sprintf("Expecting no swap. isNoSwap? %t, feature gate on? %t isCgroupV2? %t is QoS burstable? %t", isNoSwap, isSwapFeatureGateEnabled(), isCgroupV2, qosClass == v1.PodQOSBurstable))
|
||||||
expectNoSwap(f, pod, isCgroupV2)
|
expectNoSwap(f, pod, isCgroupV2)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !isLimitedSwap {
|
if !isLimitedSwap {
|
||||||
ginkgo.By("expecting unlimited swap")
|
ginkgo.By("expecting no swap")
|
||||||
expectUnlimitedSwap(f, pod, isCgroupV2)
|
expectNoSwap(f, pod, isCgroupV2)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -176,16 +177,6 @@ func expectNoSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func expectUnlimitedSwap(f *framework.Framework, pod *v1.Pod, isCgroupV2 bool) {
|
|
||||||
if isCgroupV2 {
|
|
||||||
swapLimit := readCgroupFile(f, pod, cgroupV2SwapLimitFile)
|
|
||||||
gomega.ExpectWithOffset(1, swapLimit).To(gomega.Equal("max"), "max swap allowed should be \"max\"")
|
|
||||||
} else {
|
|
||||||
swapPlusMemLimit := readCgroupFile(f, pod, cgroupV1SwapLimitFile)
|
|
||||||
gomega.ExpectWithOffset(1, swapPlusMemLimit).To(gomega.Equal("-1"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// supports v2 only as v1 shouldn't support LimitedSwap
|
// supports v2 only as v1 shouldn't support LimitedSwap
|
||||||
func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) {
|
func expectLimitedSwap(f *framework.Framework, pod *v1.Pod, expectedSwapLimit int64) {
|
||||||
swapLimitStr := readCgroupFile(f, pod, cgroupV2SwapLimitFile)
|
swapLimitStr := readCgroupFile(f, pod, cgroupV2SwapLimitFile)
|
||||||
@ -253,3 +244,10 @@ func isLimitedSwap(f *framework.Framework, pod *v1.Pod) bool {
|
|||||||
|
|
||||||
return kubeletCfg.MemorySwap.SwapBehavior == types.LimitedSwap
|
return kubeletCfg.MemorySwap.SwapBehavior == types.LimitedSwap
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isNoSwap(f *framework.Framework, pod *v1.Pod) bool {
|
||||||
|
kubeletCfg, err := getCurrentKubeletConfig(context.Background())
|
||||||
|
framework.ExpectNoError(err, "cannot get kubelet config")
|
||||||
|
|
||||||
|
return kubeletCfg.MemorySwap.SwapBehavior == types.NoSwap || kubeletCfg.MemorySwap.SwapBehavior == ""
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user