diff --git a/pkg/kubelet/eviction/eviction_manager_test.go b/pkg/kubelet/eviction/eviction_manager_test.go index 85f2f75725a..cf0123a8b57 100644 --- a/pkg/kubelet/eviction/eviction_manager_test.go +++ b/pkg/kubelet/eviction/eviction_manager_test.go @@ -264,6 +264,7 @@ type podToMake struct { requests v1.ResourceList limits v1.ResourceList memoryWorkingSet string + pidUsage uint64 rootFsUsed string logsFsUsed string logsFsInodesUsed string @@ -417,7 +418,7 @@ func TestPIDPressure_VerifyPodStatus(t *testing.T) { fakeClock := testingclock.NewFakeClock(time.Now()) podKiller := &mockPodKiller{} - diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false} + diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: ptr.To(false)} diskGC := &mockDiskGC{err: nil} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} @@ -448,7 +449,11 @@ func TestPIDPressure_VerifyPodStatus(t *testing.T) { } // synchronize to detect the memory pressure - manager.synchronize(diskInfoProvider, activePodsFunc) + _, err := manager.synchronize(diskInfoProvider, activePodsFunc) + + if err != nil { + t.Fatalf("Manager expects no error but got %v", err) + } // verify pid pressure is detected if !manager.IsUnderPIDPressure() { @@ -918,6 +923,210 @@ func makeContainersByQOS(class v1.PodQOSClass) []v1.Container { } } +func TestPIDPressure(t *testing.T) { + podMaker := makePodWithPIDStats + summaryStatsMaker := makePIDStats + podsToMake := []podToMake{ + {name: "high-priority-high-usage", priority: highPriority, pidUsage: 900}, + {name: "default-priority-low-usage", priority: defaultPriority, pidUsage: 100}, + {name: "default-priority-medium-usage", priority: defaultPriority, pidUsage: 400}, + {name: "low-priority-high-usage", priority: lowPriority, pidUsage: 600}, + {name: "low-priority-low-usage", priority: lowPriority, pidUsage: 50}, + } + pods := []*v1.Pod{} + podStats := map[*v1.Pod]statsapi.PodStats{} + for _, podToMake := range podsToMake { + pod, podStat := podMaker(podToMake.name, podToMake.priority, podToMake.pidUsage) + pods = append(pods, pod) + podStats[pod] = podStat + } + podToEvict := pods[3] + activePodsFunc := func() []*v1.Pod { + return pods + } + + fakeClock := testingclock.NewFakeClock(time.Now()) + podKiller := &mockPodKiller{} + diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: ptr.To(false)} + diskGC := &mockDiskGC{err: nil} + nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} + + config := Config{ + MaxPodGracePeriodSeconds: 5, + PressureTransitionPeriod: time.Minute * 5, + Thresholds: []evictionapi.Threshold{ + { + Signal: evictionapi.SignalPIDAvailable, + Operator: evictionapi.OpLessThan, + Value: evictionapi.ThresholdValue{ + Quantity: quantityMustParse("1200"), + }, + }, + { + Signal: evictionapi.SignalPIDAvailable, + Operator: evictionapi.OpLessThan, + Value: evictionapi.ThresholdValue{ + Quantity: quantityMustParse("1500"), + }, + GracePeriod: time.Minute * 2, + }, + }, + } + summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker("2000", "300", podStats)} + manager := &managerImpl{ + clock: fakeClock, + killPodFunc: podKiller.killPodNow, + imageGC: diskGC, + containerGC: diskGC, + config: config, + recorder: &record.FakeRecorder{}, + summaryProvider: summaryProvider, + nodeRef: nodeRef, + nodeConditionsLastObservedAt: nodeConditionsObservedAt{}, + thresholdsFirstObservedAt: thresholdsObservedAt{}, + } + + // synchronize + _, err := manager.synchronize(diskInfoProvider, activePodsFunc) + + if err != nil { + t.Fatalf("Manager expects no error but got %v", err) + } + + // we should not have disk pressure + if manager.IsUnderDiskPressure() { + t.Fatalf("Manager should not report disk pressure") + } + + // induce soft threshold for PID pressure + fakeClock.Step(1 * time.Minute) + summaryProvider.result = summaryStatsMaker("2000", "700", podStats) + _, err = manager.synchronize(diskInfoProvider, activePodsFunc) + + if err != nil { + t.Fatalf("Manager expects no error but got %v", err) + } + + // now, we should have PID pressure + if !manager.IsUnderPIDPressure() { + t.Errorf("Manager should report PID pressure since soft threshold was met") + } + + // verify no pod was yet killed because there has not yet been enough time passed + if podKiller.pod != nil { + t.Errorf("Manager should not have killed a pod yet, but killed: %v", podKiller.pod.Name) + } + + // step forward in time past the grace period + fakeClock.Step(3 * time.Minute) + // no change in PID stats to simulate continued pressure + _, err = manager.synchronize(diskInfoProvider, activePodsFunc) + + if err != nil { + t.Fatalf("Manager expects no error but got %v", err) + } + + // verify PID pressure is still reported + if !manager.IsUnderPIDPressure() { + t.Errorf("Manager should still report PID pressure") + } + + // verify the right pod was killed with the right grace period. + if podKiller.pod != podToEvict { + t.Errorf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod.Name, podToEvict.Name) + } + if podKiller.gracePeriodOverride == nil { + t.Errorf("Manager chose to kill pod but should have had a grace period override.") + } + observedGracePeriod := *podKiller.gracePeriodOverride + if observedGracePeriod != manager.config.MaxPodGracePeriodSeconds { + t.Errorf("Manager chose to kill pod with incorrect grace period. Expected: %d, actual: %d", manager.config.MaxPodGracePeriodSeconds, observedGracePeriod) + } + + // reset state + podKiller.pod = nil + podKiller.gracePeriodOverride = nil + + // remove PID pressure by simulating increased PID availability + fakeClock.Step(20 * time.Minute) + summaryProvider.result = summaryStatsMaker("2000", "300", podStats) // Simulate increased PID availability + _, err = manager.synchronize(diskInfoProvider, activePodsFunc) + + if err != nil { + t.Fatalf("Manager expects no error but got %v", err) + } + + // verify PID pressure is resolved + if manager.IsUnderPIDPressure() { + t.Errorf("Manager should not report PID pressure") + } + + // re-induce PID pressure + fakeClock.Step(1 * time.Minute) + summaryProvider.result = summaryStatsMaker("2000", "1200", podStats) + _, err = manager.synchronize(diskInfoProvider, activePodsFunc) + + if err != nil { + t.Fatalf("Manager expects no error but got %v", err) + } + + // verify PID pressure is reported again + if !manager.IsUnderPIDPressure() { + t.Errorf("Manager should report PID pressure") + } + + // verify the right pod was killed with the right grace period. + if podKiller.pod != podToEvict { + t.Errorf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod.Name, podToEvict.Name) + } + if podKiller.gracePeriodOverride == nil { + t.Errorf("Manager chose to kill pod but should have had a grace period override.") + } + observedGracePeriod = *podKiller.gracePeriodOverride + if observedGracePeriod != int64(0) { + t.Errorf("Manager chose to kill pod with incorrect grace period. Expected: %d, actual: %d", 0, observedGracePeriod) + } + + // reduce PID pressure + fakeClock.Step(1 * time.Minute) + summaryProvider.result = summaryStatsMaker("2000", "300", podStats) + podKiller.pod = nil // reset state + _, err = manager.synchronize(diskInfoProvider, activePodsFunc) + + if err != nil { + t.Fatalf("Manager expects no error but got %v", err) + } + + // we should have memory pressure (because transition period not yet met) + if !manager.IsUnderPIDPressure() { + t.Errorf("Manager should report PID pressure") + } + + // no pod should have been killed + if podKiller.pod != nil { + t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod.Name) + } + + // move the clock past the transition period + fakeClock.Step(5 * time.Minute) + summaryProvider.result = summaryStatsMaker("2000", "300", podStats) + _, err = manager.synchronize(diskInfoProvider, activePodsFunc) + + if err != nil { + t.Fatalf("Manager expects no error but got %v", err) + } + + // we should not have PID pressure (because transition period met) + if manager.IsUnderPIDPressure() { + t.Errorf("Manager should not report memory pressure") + } + + // no pod should have been killed + if podKiller.pod != nil { + t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod.Name) + } +} + func TestAdmitUnderNodeConditions(t *testing.T) { manager := &managerImpl{} pods := []*v1.Pod{