Merge pull request #42204 from dashpole/allocatable_eviction

Automatic merge from submit-queue

Eviction Manager Enforces Allocatable Thresholds

This PR modifies the eviction manager to enforce node allocatable thresholds for memory as described in kubernetes/community#348.
This PR should be merged after #41234. 

cc @kubernetes/sig-node-pr-reviews @kubernetes/sig-node-feature-requests @vishh 

** Why is this a bug/regression**

Kubelet uses `oom_score_adj` to enforce QoS policies. But the `oom_score_adj` is based on overall memory requested, which means that a Burstable pod that requested a lot of memory can lead to OOM kills for Guaranteed pods, which violates QoS. Even worse, we have observed system daemons like kubelet or kube-proxy being killed by the OOM killer.
Without this PR, v1.6 will have node stability issues and regressions in an existing GA feature `out of Resource` handling.
This commit is contained in:
Kubernetes Submit Queue
2017-03-03 20:20:12 -08:00
committed by GitHub
14 changed files with 474 additions and 83 deletions

View File

@@ -33,6 +33,7 @@ go_test(
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/kubelet/api/v1alpha1/stats:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/eviction/api:go_default_library",
"//pkg/kubelet/lifecycle:go_default_library",
"//pkg/kubelet/types:go_default_library",

View File

@@ -36,6 +36,8 @@ const (
SignalImageFsAvailable Signal = "imagefs.available"
// SignalImageFsInodesFree is amount of inodes available on filesystem that container runtime uses for storing images and container writeable layers.
SignalImageFsInodesFree Signal = "imagefs.inodesFree"
// SignalAllocatableMemoryAvailable is amount of memory available for pod allocation (i.e. allocatable - workingSet (of pods), in bytes.
SignalAllocatableMemoryAvailable Signal = "allocatableMemory.available"
)
// ThresholdOperator is the operator used to express a Threshold.

View File

@@ -134,9 +134,9 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
}
// Start starts the control loop to observe and response to low compute resources.
func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, monitoringInterval time.Duration) {
func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, nodeProvider NodeProvider, monitoringInterval time.Duration) {
// start the eviction manager monitoring
go wait.Until(func() { m.synchronize(diskInfoProvider, podFunc) }, monitoringInterval, wait.NeverStop)
go wait.Until(func() { m.synchronize(diskInfoProvider, podFunc, nodeProvider) }, monitoringInterval, wait.NeverStop)
}
// IsUnderMemoryPressure returns true if the node is under memory pressure.
@@ -187,7 +187,7 @@ func startMemoryThresholdNotifier(thresholds []evictionapi.Threshold, observatio
}
// synchronize is the main control loop that enforces eviction thresholds.
func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc) {
func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, nodeProvider NodeProvider) {
// if we have nothing to do, just return
thresholds := m.config.Thresholds
if len(thresholds) == 0 {
@@ -209,7 +209,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
}
// make observations and get a function to derive pod usage stats relative to those observations.
observations, statsFunc, err := makeSignalObservations(m.summaryProvider)
observations, statsFunc, err := makeSignalObservations(m.summaryProvider, nodeProvider)
if err != nil {
glog.Errorf("eviction manager: unexpected err: %v", err)
return
@@ -224,7 +224,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
err = startMemoryThresholdNotifier(m.config.Thresholds, observations, false, func(desc string) {
glog.Infof("soft memory eviction threshold crossed at %s", desc)
// TODO wait grace period for soft memory limit
m.synchronize(diskInfoProvider, podFunc)
m.synchronize(diskInfoProvider, podFunc, nodeProvider)
})
if err != nil {
glog.Warningf("eviction manager: failed to create hard memory threshold notifier: %v", err)
@@ -232,7 +232,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
// start hard memory notification
err = startMemoryThresholdNotifier(m.config.Thresholds, observations, true, func(desc string) {
glog.Infof("hard memory eviction threshold crossed at %s", desc)
m.synchronize(diskInfoProvider, podFunc)
m.synchronize(diskInfoProvider, podFunc, nodeProvider)
})
if err != nil {
glog.Warningf("eviction manager: failed to create soft memory threshold notifier: %v", err)

View File

@@ -59,6 +59,24 @@ func (m *mockDiskInfoProvider) HasDedicatedImageFs() (bool, error) {
return m.dedicatedImageFs, nil
}
func newMockNodeProvider(allocatableCapacity v1.ResourceList) *mockNodeProvider {
return &mockNodeProvider{
node: v1.Node{
Status: v1.NodeStatus{
Allocatable: allocatableCapacity,
},
},
}
}
type mockNodeProvider struct {
node v1.Node
}
func (m *mockNodeProvider) GetNode() (*v1.Node, error) {
return &m.node, nil
}
// mockImageGC is used to simulate invoking image garbage collection.
type mockImageGC struct {
err error
@@ -175,6 +193,7 @@ func TestMemoryPressure(t *testing.T) {
fakeClock := clock.NewFakeClock(time.Now())
podKiller := &mockPodKiller{}
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
nodeProvider := newMockNodeProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("2Gi")})
imageGC := &mockImageGC{freed: int64(0), err: nil}
nodeRef := &clientv1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
@@ -217,7 +236,7 @@ func TestMemoryPressure(t *testing.T) {
burstablePodToAdmit, _ := podMaker("burst-admit", newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi"), "0Gi")
// synchronize
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have memory pressure
if manager.IsUnderMemoryPressure() {
@@ -235,7 +254,7 @@ func TestMemoryPressure(t *testing.T) {
// induce soft threshold
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
@@ -250,7 +269,7 @@ func TestMemoryPressure(t *testing.T) {
// step forward in time pass the grace period
fakeClock.Step(3 * time.Minute)
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
@@ -275,7 +294,7 @@ func TestMemoryPressure(t *testing.T) {
// remove memory pressure
fakeClock.Step(20 * time.Minute)
summaryProvider.result = summaryStatsMaker("3Gi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have memory pressure
if manager.IsUnderMemoryPressure() {
@@ -285,7 +304,7 @@ func TestMemoryPressure(t *testing.T) {
// induce memory pressure!
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("500Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
@@ -313,7 +332,7 @@ func TestMemoryPressure(t *testing.T) {
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure (because transition period not yet met)
if !manager.IsUnderMemoryPressure() {
@@ -337,7 +356,7 @@ func TestMemoryPressure(t *testing.T) {
fakeClock.Step(5 * time.Minute)
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have memory pressure (because transition period met)
if manager.IsUnderMemoryPressure() {
@@ -392,6 +411,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
fakeClock := clock.NewFakeClock(time.Now())
podKiller := &mockPodKiller{}
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
nodeProvider := newMockNodeProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("2Gi")})
imageGC := &mockImageGC{freed: int64(0), err: nil}
nodeRef := &clientv1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
@@ -433,7 +453,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
podToAdmit, _ := podMaker("pod-to-admit", newResourceList("", ""), newResourceList("", ""), "0Gi", "0Gi", "0Gi")
// synchronize
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have disk pressure
if manager.IsUnderDiskPressure() {
@@ -448,7 +468,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
// induce soft threshold
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("1.5Gi", "200Gi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure
if !manager.IsUnderDiskPressure() {
@@ -463,7 +483,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
// step forward in time pass the grace period
fakeClock.Step(3 * time.Minute)
summaryProvider.result = summaryStatsMaker("1.5Gi", "200Gi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure
if !manager.IsUnderDiskPressure() {
@@ -488,7 +508,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
// remove disk pressure
fakeClock.Step(20 * time.Minute)
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have disk pressure
if manager.IsUnderDiskPressure() {
@@ -498,7 +518,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
// induce disk pressure!
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("500Mi", "200Gi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure
if !manager.IsUnderDiskPressure() {
@@ -523,7 +543,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure (because transition period not yet met)
if !manager.IsUnderDiskPressure() {
@@ -544,7 +564,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
fakeClock.Step(5 * time.Minute)
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have disk pressure (because transition period met)
if manager.IsUnderDiskPressure() {
@@ -589,6 +609,7 @@ func TestMinReclaim(t *testing.T) {
fakeClock := clock.NewFakeClock(time.Now())
podKiller := &mockPodKiller{}
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
nodeProvider := newMockNodeProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("2Gi")})
imageGC := &mockImageGC{freed: int64(0), err: nil}
nodeRef := &clientv1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
@@ -622,7 +643,7 @@ func TestMinReclaim(t *testing.T) {
}
// synchronize
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have memory pressure
if manager.IsUnderMemoryPressure() {
@@ -632,7 +653,7 @@ func TestMinReclaim(t *testing.T) {
// induce memory pressure!
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("500Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
@@ -652,7 +673,7 @@ func TestMinReclaim(t *testing.T) {
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("1.2Gi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure (because transition period not yet met)
if !manager.IsUnderMemoryPressure() {
@@ -672,7 +693,7 @@ func TestMinReclaim(t *testing.T) {
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure (because transition period not yet met)
if !manager.IsUnderMemoryPressure() {
@@ -688,7 +709,7 @@ func TestMinReclaim(t *testing.T) {
fakeClock.Step(5 * time.Minute)
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have memory pressure (because transition period met)
if manager.IsUnderMemoryPressure() {
@@ -727,6 +748,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
fakeClock := clock.NewFakeClock(time.Now())
podKiller := &mockPodKiller{}
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
nodeProvider := newMockNodeProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("2Gi")})
imageGcFree := resource.MustParse("700Mi")
imageGC := &mockImageGC{freed: imageGcFree.Value(), err: nil}
nodeRef := &clientv1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
@@ -761,7 +783,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
}
// synchronize
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have disk pressure
if manager.IsUnderDiskPressure() {
@@ -771,7 +793,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
// induce hard threshold
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker(".9Gi", "200Gi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure
if !manager.IsUnderDiskPressure() {
@@ -794,7 +816,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
// remove disk pressure
fakeClock.Step(20 * time.Minute)
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have disk pressure
if manager.IsUnderDiskPressure() {
@@ -804,7 +826,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
// induce disk pressure!
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("400Mi", "200Gi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure
if !manager.IsUnderDiskPressure() {
@@ -830,7 +852,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
imageGC.invoked = false // reset state
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure (because transition period not yet met)
if !manager.IsUnderDiskPressure() {
@@ -852,7 +874,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
imageGC.invoked = false // reset state
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have disk pressure (because transition period met)
if manager.IsUnderDiskPressure() {
@@ -920,6 +942,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
fakeClock := clock.NewFakeClock(time.Now())
podKiller := &mockPodKiller{}
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
nodeProvider := newMockNodeProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("2Gi")})
imageGC := &mockImageGC{freed: int64(0), err: nil}
nodeRef := &clientv1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
@@ -961,7 +984,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
podToAdmit, _ := podMaker("pod-to-admit", newResourceList("", ""), newResourceList("", ""), "0", "0", "0")
// synchronize
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have disk pressure
if manager.IsUnderDiskPressure() {
@@ -976,7 +999,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
// induce soft threshold
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("1.5Mi", "4Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure
if !manager.IsUnderDiskPressure() {
@@ -991,7 +1014,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
// step forward in time pass the grace period
fakeClock.Step(3 * time.Minute)
summaryProvider.result = summaryStatsMaker("1.5Mi", "4Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure
if !manager.IsUnderDiskPressure() {
@@ -1016,7 +1039,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
// remove inode pressure
fakeClock.Step(20 * time.Minute)
summaryProvider.result = summaryStatsMaker("3Mi", "4Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have disk pressure
if manager.IsUnderDiskPressure() {
@@ -1026,7 +1049,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
// induce inode pressure!
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("0.5Mi", "4Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure
if !manager.IsUnderDiskPressure() {
@@ -1051,7 +1074,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("3Mi", "4Mi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have disk pressure (because transition period not yet met)
if !manager.IsUnderDiskPressure() {
@@ -1072,7 +1095,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
fakeClock.Step(5 * time.Minute)
summaryProvider.result = summaryStatsMaker("3Mi", "4Mi", podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have disk pressure (because transition period met)
if manager.IsUnderDiskPressure() {
@@ -1120,6 +1143,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
fakeClock := clock.NewFakeClock(time.Now())
podKiller := &mockPodKiller{}
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
nodeProvider := newMockNodeProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("2Gi")})
imageGC := &mockImageGC{freed: int64(0), err: nil}
nodeRef := &clientv1.ObjectReference{
Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: "",
@@ -1164,7 +1188,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
// induce soft threshold
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
@@ -1179,7 +1203,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
// step forward in time pass the grace period
fakeClock.Step(3 * time.Minute)
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
@@ -1197,7 +1221,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
// remove memory pressure
fakeClock.Step(20 * time.Minute)
summaryProvider.result = summaryStatsMaker("3Gi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have memory pressure
if manager.IsUnderMemoryPressure() {
@@ -1210,7 +1234,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
// induce memory pressure!
fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("500Mi", podStats)
manager.synchronize(diskInfoProvider, activePodsFunc)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
@@ -1222,3 +1246,167 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
t.Errorf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod.Name, podToEvict.Name)
}
}
// TestAllocatableMemoryPressure
func TestAllocatableMemoryPressure(t *testing.T) {
podMaker := makePodWithMemoryStats
summaryStatsMaker := makeMemoryStats
constantCapacity := "4Gi"
podsToMake := []podToMake{
{name: "guaranteed-low", requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "200Mi"},
{name: "guaranteed-high", requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "400Mi"},
{name: "burstable-low", requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "300Mi"},
{name: "burstable-high", requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "500Mi"},
{name: "best-effort-low", requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "100Mi"},
{name: "best-effort-high", requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "200Mi"},
}
pods := []*v1.Pod{}
podStats := map[*v1.Pod]statsapi.PodStats{}
for _, podToMake := range podsToMake {
pod, podStat := podMaker(podToMake.name, podToMake.requests, podToMake.limits, podToMake.memoryWorkingSet)
pods = append(pods, pod)
podStats[pod] = podStat
}
podToEvict := pods[5]
activePodsFunc := func() []*v1.Pod {
return pods
}
fakeClock := clock.NewFakeClock(time.Now())
podKiller := &mockPodKiller{}
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
nodeProvider := newMockNodeProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("2Gi")})
imageGC := &mockImageGC{freed: int64(0), err: nil}
nodeRef := &clientv1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
config := Config{
MaxPodGracePeriodSeconds: 5,
PressureTransitionPeriod: time.Minute * 5,
Thresholds: []evictionapi.Threshold{
{
Signal: evictionapi.SignalAllocatableMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: quantityMustParse("1Ki"),
},
},
},
}
summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker(constantCapacity, podStats)}
manager := &managerImpl{
clock: fakeClock,
killPodFunc: podKiller.killPodNow,
imageGC: imageGC,
config: config,
recorder: &record.FakeRecorder{},
summaryProvider: summaryProvider,
nodeRef: nodeRef,
nodeConditionsLastObservedAt: nodeConditionsObservedAt{},
thresholdsFirstObservedAt: thresholdsObservedAt{},
}
// create a best effort pod to test admission
bestEffortPodToAdmit, _ := podMaker("best-admit", newResourceList("", ""), newResourceList("", ""), "0Gi")
burstablePodToAdmit, _ := podMaker("burst-admit", newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi"), "0Gi")
// synchronize
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have memory pressure
if manager.IsUnderMemoryPressure() {
t.Errorf("Manager should not report memory pressure")
}
// try to admit our pods (they should succeed)
expected := []bool{true, true}
for i, pod := range []*v1.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
}
}
// induce memory pressure!
fakeClock.Step(1 * time.Minute)
pod, podStat := podMaker("guaranteed-high-2", newResourceList("100m", "1Gi"), newResourceList("100m", "1Gi"), "1Gi")
podStats[pod] = podStat
summaryProvider.result = summaryStatsMaker(constantCapacity, podStats)
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure
if !manager.IsUnderMemoryPressure() {
t.Errorf("Manager should report memory pressure")
}
// check the right pod was killed
if podKiller.pod != podToEvict {
t.Errorf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod.Name, podToEvict.Name)
}
observedGracePeriod := *podKiller.gracePeriodOverride
if observedGracePeriod != int64(0) {
t.Errorf("Manager chose to kill pod with incorrect grace period. Expected: %d, actual: %d", 0, observedGracePeriod)
}
// reset state
podKiller.pod = nil
podKiller.gracePeriodOverride = nil
// the best-effort pod should not admit, burstable should
expected = []bool{false, true}
for i, pod := range []*v1.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
}
}
// reduce memory pressure
fakeClock.Step(1 * time.Minute)
for pod := range podStats {
if pod.Name == "guaranteed-high-2" {
delete(podStats, pod)
}
}
summaryProvider.result = summaryStatsMaker(constantCapacity, podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should have memory pressure (because transition period not yet met)
if !manager.IsUnderMemoryPressure() {
t.Errorf("Manager should report memory pressure")
}
// no pod should have been killed
if podKiller.pod != nil {
t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod.Name)
}
// the best-effort pod should not admit, burstable should
expected = []bool{false, true}
for i, pod := range []*v1.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
}
}
// move the clock past transition period to ensure that we stop reporting pressure
fakeClock.Step(5 * time.Minute)
summaryProvider.result = summaryStatsMaker(constantCapacity, podStats)
podKiller.pod = nil // reset state
manager.synchronize(diskInfoProvider, activePodsFunc, nodeProvider)
// we should not have memory pressure (because transition period met)
if manager.IsUnderMemoryPressure() {
t.Errorf("Manager should not report memory pressure")
}
// no pod should have been killed
if podKiller.pod != nil {
t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod.Name)
}
// all pods should admit now
expected = []bool{true, true}
for i, pod := range []*v1.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
}
}
}

View File

@@ -29,6 +29,7 @@ import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
statsapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
"k8s.io/kubernetes/pkg/kubelet/cm"
evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
"k8s.io/kubernetes/pkg/kubelet/qos"
"k8s.io/kubernetes/pkg/kubelet/server/stats"
@@ -68,6 +69,7 @@ func init() {
// map eviction signals to node conditions
signalToNodeCondition = map[evictionapi.Signal]v1.NodeConditionType{}
signalToNodeCondition[evictionapi.SignalMemoryAvailable] = v1.NodeMemoryPressure
signalToNodeCondition[evictionapi.SignalAllocatableMemoryAvailable] = v1.NodeMemoryPressure
signalToNodeCondition[evictionapi.SignalImageFsAvailable] = v1.NodeDiskPressure
signalToNodeCondition[evictionapi.SignalNodeFsAvailable] = v1.NodeDiskPressure
signalToNodeCondition[evictionapi.SignalImageFsInodesFree] = v1.NodeDiskPressure
@@ -76,6 +78,7 @@ func init() {
// map signals to resources (and vice-versa)
signalToResource = map[evictionapi.Signal]v1.ResourceName{}
signalToResource[evictionapi.SignalMemoryAvailable] = v1.ResourceMemory
signalToResource[evictionapi.SignalAllocatableMemoryAvailable] = v1.ResourceMemory
signalToResource[evictionapi.SignalImageFsAvailable] = resourceImageFs
signalToResource[evictionapi.SignalImageFsInodesFree] = resourceImageFsInodes
signalToResource[evictionapi.SignalNodeFsAvailable] = resourceNodeFs
@@ -93,8 +96,10 @@ func validSignal(signal evictionapi.Signal) bool {
}
// ParseThresholdConfig parses the flags for thresholds.
func ParseThresholdConfig(evictionHard, evictionSoft, evictionSoftGracePeriod, evictionMinimumReclaim string) ([]evictionapi.Threshold, error) {
func ParseThresholdConfig(allocatableConfig []string, evictionHard, evictionSoft, evictionSoftGracePeriod, evictionMinimumReclaim string) ([]evictionapi.Threshold, error) {
results := []evictionapi.Threshold{}
allocatableThresholds := getAllocatableThreshold(allocatableConfig)
results = append(results, allocatableThresholds...)
hardThresholds, err := parseThresholdStatements(evictionHard)
if err != nil {
@@ -214,6 +219,27 @@ func parseThresholdStatement(statement string) (evictionapi.Threshold, error) {
}, nil
}
// getAllocatableThreshold returns the thresholds applicable for the allocatable configuration
func getAllocatableThreshold(allocatableConfig []string) []evictionapi.Threshold {
for _, key := range allocatableConfig {
if key == cm.NodeAllocatableEnforcementKey {
return []evictionapi.Threshold{
{
Signal: evictionapi.SignalAllocatableMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: resource.NewQuantity(int64(0), resource.BinarySI),
},
MinReclaim: &evictionapi.ThresholdValue{
Quantity: resource.NewQuantity(int64(0), resource.BinarySI),
},
},
}
}
}
return []evictionapi.Threshold{}
}
// parsePercentage parses a string representing a percentage value
func parsePercentage(input string) (float32, error) {
value, err := strconv.ParseFloat(strings.TrimRight(input, "%"), 32)
@@ -611,11 +637,15 @@ func (a byEvictionPriority) Less(i, j int) bool {
}
// makeSignalObservations derives observations using the specified summary provider.
func makeSignalObservations(summaryProvider stats.SummaryProvider) (signalObservations, statsFunc, error) {
func makeSignalObservations(summaryProvider stats.SummaryProvider, nodeProvider NodeProvider) (signalObservations, statsFunc, error) {
summary, err := summaryProvider.Get()
if err != nil {
return nil, nil, err
}
node, err := nodeProvider.GetNode()
if err != nil {
return nil, nil, err
}
// build the function to work against for pod stats
statsFunc := cachedStatsFunc(summary.Pods)
@@ -663,6 +693,19 @@ func makeSignalObservations(summaryProvider stats.SummaryProvider) (signalObserv
}
}
}
if memoryAllocatableCapacity, ok := node.Status.Allocatable[v1.ResourceMemory]; ok {
memoryAllocatableAvailable := memoryAllocatableCapacity.Copy()
for _, pod := range summary.Pods {
mu, err := podMemoryUsage(pod)
if err == nil {
memoryAllocatableAvailable.Sub(mu[v1.ResourceMemory])
}
}
result[evictionapi.SignalAllocatableMemoryAvailable] = signalObservation{
available: memoryAllocatableAvailable,
capacity: memoryAllocatableCapacity.Copy(),
}
}
return result, statsFunc, nil
}

View File

@@ -28,6 +28,7 @@ import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
statsapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
"k8s.io/kubernetes/pkg/kubelet/cm"
evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
"k8s.io/kubernetes/pkg/quota"
)
@@ -40,6 +41,7 @@ func quantityMustParse(value string) *resource.Quantity {
func TestParseThresholdConfig(t *testing.T) {
gracePeriod, _ := time.ParseDuration("30s")
testCases := map[string]struct {
allocatableConfig []string
evictionHard string
evictionSoft string
evictionSoftGracePeriod string
@@ -48,6 +50,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds []evictionapi.Threshold
}{
"no values": {
allocatableConfig: []string{},
evictionHard: "",
evictionSoft: "",
evictionSoftGracePeriod: "",
@@ -56,12 +59,23 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"all flag values": {
allocatableConfig: []string{cm.NodeAllocatableEnforcementKey},
evictionHard: "memory.available<150Mi",
evictionSoft: "memory.available<300Mi",
evictionSoftGracePeriod: "memory.available=30s",
evictionMinReclaim: "memory.available=0",
expectErr: false,
expectThresholds: []evictionapi.Threshold{
{
Signal: evictionapi.SignalAllocatableMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: quantityMustParse("0"),
},
MinReclaim: &evictionapi.ThresholdValue{
Quantity: quantityMustParse("0"),
},
},
{
Signal: evictionapi.SignalMemoryAvailable,
Operator: evictionapi.OpLessThan,
@@ -86,6 +100,7 @@ func TestParseThresholdConfig(t *testing.T) {
},
},
"all flag values in percentages": {
allocatableConfig: []string{},
evictionHard: "memory.available<10%",
evictionSoft: "memory.available<30%",
evictionSoftGracePeriod: "memory.available=30s",
@@ -116,6 +131,7 @@ func TestParseThresholdConfig(t *testing.T) {
},
},
"disk flag values": {
allocatableConfig: []string{},
evictionHard: "imagefs.available<150Mi,nodefs.available<100Mi",
evictionSoft: "imagefs.available<300Mi,nodefs.available<200Mi",
evictionSoftGracePeriod: "imagefs.available=30s,nodefs.available=30s",
@@ -167,6 +183,7 @@ func TestParseThresholdConfig(t *testing.T) {
},
},
"disk flag values in percentages": {
allocatableConfig: []string{},
evictionHard: "imagefs.available<15%,nodefs.available<10.5%",
evictionSoft: "imagefs.available<30%,nodefs.available<20.5%",
evictionSoftGracePeriod: "imagefs.available=30s,nodefs.available=30s",
@@ -218,6 +235,7 @@ func TestParseThresholdConfig(t *testing.T) {
},
},
"inode flag values": {
allocatableConfig: []string{},
evictionHard: "imagefs.inodesFree<150Mi,nodefs.inodesFree<100Mi",
evictionSoft: "imagefs.inodesFree<300Mi,nodefs.inodesFree<200Mi",
evictionSoftGracePeriod: "imagefs.inodesFree=30s,nodefs.inodesFree=30s",
@@ -269,6 +287,7 @@ func TestParseThresholdConfig(t *testing.T) {
},
},
"invalid-signal": {
allocatableConfig: []string{},
evictionHard: "mem.available<150Mi",
evictionSoft: "",
evictionSoftGracePeriod: "",
@@ -277,6 +296,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"hard-signal-negative": {
allocatableConfig: []string{},
evictionHard: "memory.available<-150Mi",
evictionSoft: "",
evictionSoftGracePeriod: "",
@@ -285,6 +305,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"hard-signal-negative-percentage": {
allocatableConfig: []string{},
evictionHard: "memory.available<-15%",
evictionSoft: "",
evictionSoftGracePeriod: "",
@@ -293,6 +314,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"soft-signal-negative": {
allocatableConfig: []string{},
evictionHard: "",
evictionSoft: "memory.available<-150Mi",
evictionSoftGracePeriod: "",
@@ -301,6 +323,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"duplicate-signal": {
allocatableConfig: []string{},
evictionHard: "memory.available<150Mi,memory.available<100Mi",
evictionSoft: "",
evictionSoftGracePeriod: "",
@@ -309,6 +332,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"valid-and-invalid-signal": {
allocatableConfig: []string{},
evictionHard: "memory.available<150Mi,invalid.foo<150Mi",
evictionSoft: "",
evictionSoftGracePeriod: "",
@@ -317,6 +341,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"soft-no-grace-period": {
allocatableConfig: []string{},
evictionHard: "",
evictionSoft: "memory.available<150Mi",
evictionSoftGracePeriod: "",
@@ -325,6 +350,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"soft-neg-grace-period": {
allocatableConfig: []string{},
evictionHard: "",
evictionSoft: "memory.available<150Mi",
evictionSoftGracePeriod: "memory.available=-30s",
@@ -333,6 +359,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"neg-reclaim": {
allocatableConfig: []string{},
evictionHard: "",
evictionSoft: "",
evictionSoftGracePeriod: "",
@@ -341,6 +368,7 @@ func TestParseThresholdConfig(t *testing.T) {
expectThresholds: []evictionapi.Threshold{},
},
"duplicate-reclaim": {
allocatableConfig: []string{},
evictionHard: "",
evictionSoft: "",
evictionSoftGracePeriod: "",
@@ -350,7 +378,7 @@ func TestParseThresholdConfig(t *testing.T) {
},
}
for testName, testCase := range testCases {
thresholds, err := ParseThresholdConfig(testCase.evictionHard, testCase.evictionSoft, testCase.evictionSoftGracePeriod, testCase.evictionMinReclaim)
thresholds, err := ParseThresholdConfig(testCase.allocatableConfig, testCase.evictionHard, testCase.evictionSoft, testCase.evictionSoftGracePeriod, testCase.evictionMinReclaim)
if testCase.expectErr != (err != nil) {
t.Errorf("Err not as expected, test: %v, error expected: %v, actual: %v", testName, testCase.expectErr, err)
}
@@ -699,6 +727,7 @@ func TestMakeSignalObservations(t *testing.T) {
}
nodeAvailableBytes := uint64(1024 * 1024 * 1024)
nodeWorkingSetBytes := uint64(1024 * 1024 * 1024)
allocatableMemoryCapacity := uint64(5 * 1024 * 1024 * 1024)
imageFsAvailableBytes := uint64(1024 * 1024)
imageFsCapacityBytes := uint64(1024 * 1024 * 2)
nodeFsAvailableBytes := uint64(1024)
@@ -738,15 +767,31 @@ func TestMakeSignalObservations(t *testing.T) {
podMaker("pod1", "ns2", "uuid2", 1),
podMaker("pod3", "ns3", "uuid3", 1),
}
containerWorkingSetBytes := int64(1024 * 1024)
containerWorkingSetBytes := int64(1024 * 1024 * 1024)
for _, pod := range pods {
fakeStats.Pods = append(fakeStats.Pods, newPodStats(pod, containerWorkingSetBytes))
}
actualObservations, statsFunc, err := makeSignalObservations(provider)
res := quantityMustParse("5Gi")
nodeProvider := newMockNodeProvider(v1.ResourceList{v1.ResourceMemory: *res})
// Allocatable thresholds are always 100%. Verify that Threshold == Capacity.
if res.CmpInt64(int64(allocatableMemoryCapacity)) != 0 {
t.Errorf("Expected Threshold %v to be equal to value %v", res.Value(), allocatableMemoryCapacity)
}
actualObservations, statsFunc, err := makeSignalObservations(provider, nodeProvider)
if err != nil {
t.Errorf("Unexpected err: %v", err)
}
allocatableMemQuantity, found := actualObservations[evictionapi.SignalAllocatableMemoryAvailable]
if !found {
t.Errorf("Expected allocatable memory observation, but didnt find one")
}
if allocatableMemQuantity.available.Value() != 2*containerWorkingSetBytes {
t.Errorf("Expected %v, actual: %v", containerWorkingSetBytes, allocatableMemQuantity.available.Value())
}
if expectedBytes := int64(allocatableMemoryCapacity); allocatableMemQuantity.capacity.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, allocatableMemQuantity.capacity.Value())
}
memQuantity, found := actualObservations[evictionapi.SignalMemoryAvailable]
if !found {
t.Errorf("Expected available memory observation: %v", err)

View File

@@ -53,7 +53,7 @@ type Config struct {
// Manager evaluates when an eviction threshold for node stability has been met on the node.
type Manager interface {
// Start starts the control loop to monitor eviction thresholds at specified interval.
Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, monitoringInterval time.Duration)
Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, nodeProvider NodeProvider, monitoringInterval time.Duration)
// IsUnderMemoryPressure returns true if the node is under memory pressure.
IsUnderMemoryPressure() bool
@@ -68,6 +68,12 @@ type DiskInfoProvider interface {
HasDedicatedImageFs() (bool, error)
}
// NodeProvider is responsible for providing the node api object describing this node
type NodeProvider interface {
// GetNode returns the node info for this node
GetNode() (*v1.Node, error)
}
// ImageGC is responsible for performing garbage collection of unused images.
type ImageGC interface {
// DeleteUnusedImages deletes unused images and returns the number of bytes freed, or an error.

View File

@@ -349,7 +349,12 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
RootFreeDiskMB: int(kubeCfg.LowDiskSpaceThresholdMB),
}
thresholds, err := eviction.ParseThresholdConfig(kubeCfg.EvictionHard, kubeCfg.EvictionSoft, kubeCfg.EvictionSoftGracePeriod, kubeCfg.EvictionMinimumReclaim)
enforceNodeAllocatable := kubeCfg.EnforceNodeAllocatable
if kubeCfg.ExperimentalNodeAllocatableIgnoreEvictionThreshold {
// Do not provide kubeCfg.EnforceNodeAllocatable to eviction threshold parsing if we are not enforcing Evictions
enforceNodeAllocatable = []string{}
}
thresholds, err := eviction.ParseThresholdConfig(enforceNodeAllocatable, kubeCfg.EvictionHard, kubeCfg.EvictionSoft, kubeCfg.EvictionSoftGracePeriod, kubeCfg.EvictionMinimumReclaim)
if err != nil {
return nil, err
}
@@ -1222,7 +1227,7 @@ func (kl *Kubelet) initializeRuntimeDependentModules() {
glog.Fatalf("Failed to start cAdvisor %v", err)
}
// eviction manager must start after cadvisor because it needs to know if the container runtime has a dedicated imagefs
kl.evictionManager.Start(kl, kl.getActivePods, evictionMonitoringPeriod)
kl.evictionManager.Start(kl, kl.getActivePods, kl, evictionMonitoringPeriod)
}
// Run starts the kubelet reacting to config updates