mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-29 22:46:12 +00:00
Merge pull request #57802 from dashpole/allocatable_monitoring
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Monitor the /kubepods cgroup for allocatable metrics **What this PR does / why we need it**: The current implementation of allocatable memory evictions sums the usage of pods in order to compute the total usage by user processes. This PR changes this to instead monitor the `/kubepods` cgroup, which contains all pods, and use this value directly. This is more accurate than summing pod usage, as it is measured at a single point in time. This also collects metrics from this cgroup on-demand. This PR is a precursor to memcg notifications on the `/kubepods` cgroup. This removes the dependency the eviction manager has on the container manager, and adds a dependency for the summary collector on the container manager (to get Cgroup Root) This also changes the way that the allocatable memory eviction signal and threshold are added to make them in-line with the memory eviction signal to address #53902 **Which issue(s) this PR fixes**: Fixes #55638 Fixes #53902 **Special notes for your reviewer**: I have tested this, and can confirm that it works when CgroupsPerQos is set to false. In this case, it returns node metrics, as it is monitoring the `/` cgroup, rather than the `/kubepods` cgroup (which doesn't exist). **Release note**: ```release-note Expose total usage of pods through the "pods" SystemContainer in the Kubelet Summary API ``` cc @sjenning @derekwaynecarr @vishh @kubernetes/sig-node-pr-reviews
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
236fa894df
@ -87,6 +87,8 @@ const (
|
||||
SystemContainerRuntime = "runtime"
|
||||
// SystemContainerMisc is the container name for the system container tracking non-kubernetes processes.
|
||||
SystemContainerMisc = "misc"
|
||||
// SystemContainerPods is the container name for the system container tracking user pods.
|
||||
SystemContainerPods = "pods"
|
||||
)
|
||||
|
||||
// PodStats holds pod-level unprocessed sample stats.
|
||||
|
||||
@ -91,6 +91,9 @@ type ContainerManager interface {
|
||||
UpdatePluginResources(*schedulercache.NodeInfo, *lifecycle.PodAdmitAttributes) error
|
||||
|
||||
InternalContainerLifecycle() InternalContainerLifecycle
|
||||
|
||||
// GetPodCgroupRoot returns the cgroup which contains all pods.
|
||||
GetPodCgroupRoot() string
|
||||
}
|
||||
|
||||
type NodeConfig struct {
|
||||
|
||||
@ -500,6 +500,11 @@ func (cm *containerManagerImpl) GetNodeConfig() NodeConfig {
|
||||
return cm.NodeConfig
|
||||
}
|
||||
|
||||
// GetPodCgroupRoot returns the literal cgroupfs value for the cgroup containing all pods.
|
||||
func (cm *containerManagerImpl) GetPodCgroupRoot() string {
|
||||
return cm.cgroupManager.Name(CgroupName(cm.cgroupRoot))
|
||||
}
|
||||
|
||||
func (cm *containerManagerImpl) GetMountedSubsystems() *CgroupSubsystems {
|
||||
return cm.subsystems
|
||||
}
|
||||
|
||||
@ -90,6 +90,10 @@ func (cm *containerManagerStub) InternalContainerLifecycle() InternalContainerLi
|
||||
return &internalContainerLifecycleImpl{cpumanager.NewFakeManager()}
|
||||
}
|
||||
|
||||
func (cm *containerManagerStub) GetPodCgroupRoot() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func NewStubContainerManager() ContainerManager {
|
||||
return &containerManagerStub{}
|
||||
}
|
||||
|
||||
@ -149,11 +149,11 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
|
||||
}
|
||||
|
||||
// Start starts the control loop to observe and response to low compute resources.
|
||||
func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, capacityProvider CapacityProvider, monitoringInterval time.Duration) {
|
||||
func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration) {
|
||||
// start the eviction manager monitoring
|
||||
go func() {
|
||||
for {
|
||||
if evictedPods := m.synchronize(diskInfoProvider, podFunc, capacityProvider); evictedPods != nil {
|
||||
if evictedPods := m.synchronize(diskInfoProvider, podFunc); evictedPods != nil {
|
||||
glog.Infof("eviction manager: pods %s evicted, waiting for pod to be cleaned up", format.Pods(evictedPods))
|
||||
m.waitForPodsCleanup(podCleanedUpFunc, evictedPods)
|
||||
} else {
|
||||
@ -219,7 +219,7 @@ func startMemoryThresholdNotifier(thresholds []evictionapi.Threshold, observatio
|
||||
|
||||
// synchronize is the main control loop that enforces eviction thresholds.
|
||||
// Returns the pod that was killed, or nil if no pod was killed.
|
||||
func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, capacityProvider CapacityProvider) []*v1.Pod {
|
||||
func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc) []*v1.Pod {
|
||||
// if we have nothing to do, just return
|
||||
thresholds := m.config.Thresholds
|
||||
if len(thresholds) == 0 {
|
||||
@ -248,7 +248,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
|
||||
}
|
||||
|
||||
// make observations and get a function to derive pod usage stats relative to those observations.
|
||||
observations, statsFunc := makeSignalObservations(summary, capacityProvider, activePods)
|
||||
observations, statsFunc := makeSignalObservations(summary)
|
||||
debugLogObservations("observations", observations)
|
||||
|
||||
// attempt to create a threshold notifier to improve eviction response time
|
||||
@ -259,7 +259,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
|
||||
err = startMemoryThresholdNotifier(m.config.Thresholds, observations, false, func(desc string) {
|
||||
glog.Infof("soft memory eviction threshold crossed at %s", desc)
|
||||
// TODO wait grace period for soft memory limit
|
||||
m.synchronize(diskInfoProvider, podFunc, capacityProvider)
|
||||
m.synchronize(diskInfoProvider, podFunc)
|
||||
})
|
||||
if err != nil {
|
||||
glog.Warningf("eviction manager: failed to create soft memory threshold notifier: %v", err)
|
||||
@ -267,7 +267,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
|
||||
// start hard memory notification
|
||||
err = startMemoryThresholdNotifier(m.config.Thresholds, observations, true, func(desc string) {
|
||||
glog.Infof("hard memory eviction threshold crossed at %s", desc)
|
||||
m.synchronize(diskInfoProvider, podFunc, capacityProvider)
|
||||
m.synchronize(diskInfoProvider, podFunc)
|
||||
})
|
||||
if err != nil {
|
||||
glog.Warningf("eviction manager: failed to create hard memory threshold notifier: %v", err)
|
||||
@ -349,7 +349,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
|
||||
m.recorder.Eventf(m.nodeRef, v1.EventTypeWarning, "EvictionThresholdMet", "Attempting to reclaim %s", resourceToReclaim)
|
||||
|
||||
// check if there are node-level resources we can reclaim to reduce pressure before evicting end-user pods.
|
||||
if m.reclaimNodeLevelResources(resourceToReclaim, capacityProvider, activePods) {
|
||||
if m.reclaimNodeLevelResources(resourceToReclaim) {
|
||||
glog.Infof("eviction manager: able to reduce %v pressure without evicting pods.", resourceToReclaim)
|
||||
return nil
|
||||
}
|
||||
@ -437,7 +437,7 @@ func (m *managerImpl) waitForPodsCleanup(podCleanedUpFunc PodCleanedUpFunc, pods
|
||||
}
|
||||
|
||||
// reclaimNodeLevelResources attempts to reclaim node level resources. returns true if thresholds were satisfied and no pod eviction is required.
|
||||
func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceName, capacityProvider CapacityProvider, pods []*v1.Pod) bool {
|
||||
func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceName) bool {
|
||||
nodeReclaimFuncs := m.resourceToNodeReclaimFuncs[resourceToReclaim]
|
||||
for _, nodeReclaimFunc := range nodeReclaimFuncs {
|
||||
// attempt to reclaim the pressured resource.
|
||||
@ -454,7 +454,7 @@ func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceNam
|
||||
}
|
||||
|
||||
// make observations and get a function to derive pod usage stats relative to those observations.
|
||||
observations, _ := makeSignalObservations(summary, capacityProvider, pods)
|
||||
observations, _ := makeSignalObservations(summary)
|
||||
debugLogObservations("observations after resource reclaim", observations)
|
||||
|
||||
// determine the set of thresholds met independent of grace period
|
||||
|
||||
@ -65,27 +65,6 @@ func (m *mockDiskInfoProvider) HasDedicatedImageFs() (bool, error) {
|
||||
return m.dedicatedImageFs, nil
|
||||
}
|
||||
|
||||
func newMockCapacityProvider(capacity, reservation v1.ResourceList) *mockCapacityProvider {
|
||||
return &mockCapacityProvider{
|
||||
capacity: capacity,
|
||||
reservation: reservation,
|
||||
}
|
||||
}
|
||||
|
||||
type mockCapacityProvider struct {
|
||||
capacity v1.ResourceList
|
||||
reservation v1.ResourceList
|
||||
}
|
||||
|
||||
func (m *mockCapacityProvider) GetCapacity() v1.ResourceList {
|
||||
return m.capacity
|
||||
|
||||
}
|
||||
|
||||
func (m *mockCapacityProvider) GetNodeAllocatableReservation() v1.ResourceList {
|
||||
return m.reservation
|
||||
}
|
||||
|
||||
// mockDiskGC is used to simulate invoking image and container garbage collection.
|
||||
type mockDiskGC struct {
|
||||
err error
|
||||
@ -139,6 +118,15 @@ func makeMemoryStats(nodeAvailableBytes string, podStats map[*v1.Pod]statsapi.Po
|
||||
AvailableBytes: &availableBytes,
|
||||
WorkingSetBytes: &WorkingSetBytes,
|
||||
},
|
||||
SystemContainers: []statsapi.ContainerStats{
|
||||
{
|
||||
Name: statsapi.SystemContainerPods,
|
||||
Memory: &statsapi.MemoryStats{
|
||||
AvailableBytes: &availableBytes,
|
||||
WorkingSetBytes: &WorkingSetBytes,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Pods: []statsapi.PodStats{},
|
||||
}
|
||||
@ -217,7 +205,6 @@ func TestMemoryPressure(t *testing.T) {
|
||||
fakeClock := clock.NewFakeClock(time.Now())
|
||||
podKiller := &mockPodKiller{}
|
||||
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
|
||||
capacityProvider := newMockCapacityProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("3Gi")}, v1.ResourceList{v1.ResourceMemory: *quantityMustParse("1Gi")})
|
||||
diskGC := &mockDiskGC{err: nil}
|
||||
nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
|
||||
|
||||
@ -261,7 +248,7 @@ func TestMemoryPressure(t *testing.T) {
|
||||
burstablePodToAdmit, _ := podMaker("burst-admit", defaultPriority, newResourceList("100m", "100Mi", ""), newResourceList("200m", "200Mi", ""), "0Gi")
|
||||
|
||||
// synchronize
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have memory pressure
|
||||
if manager.IsUnderMemoryPressure() {
|
||||
@ -279,7 +266,7 @@ func TestMemoryPressure(t *testing.T) {
|
||||
// induce soft threshold
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -294,7 +281,7 @@ func TestMemoryPressure(t *testing.T) {
|
||||
// step forward in time pass the grace period
|
||||
fakeClock.Step(3 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -319,7 +306,7 @@ func TestMemoryPressure(t *testing.T) {
|
||||
// remove memory pressure
|
||||
fakeClock.Step(20 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("3Gi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have memory pressure
|
||||
if manager.IsUnderMemoryPressure() {
|
||||
@ -329,7 +316,7 @@ func TestMemoryPressure(t *testing.T) {
|
||||
// induce memory pressure!
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("500Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -357,7 +344,7 @@ func TestMemoryPressure(t *testing.T) {
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure (because transition period not yet met)
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -381,7 +368,7 @@ func TestMemoryPressure(t *testing.T) {
|
||||
fakeClock.Step(5 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have memory pressure (because transition period met)
|
||||
if manager.IsUnderMemoryPressure() {
|
||||
@ -439,7 +426,6 @@ func TestDiskPressureNodeFs(t *testing.T) {
|
||||
fakeClock := clock.NewFakeClock(time.Now())
|
||||
podKiller := &mockPodKiller{}
|
||||
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
|
||||
capacityProvider := newMockCapacityProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("3Gi")}, v1.ResourceList{v1.ResourceMemory: *quantityMustParse("1Gi")})
|
||||
diskGC := &mockDiskGC{err: nil}
|
||||
nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
|
||||
|
||||
@ -482,7 +468,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
|
||||
podToAdmit, _ := podMaker("pod-to-admit", defaultPriority, newResourceList("", "", ""), newResourceList("", "", ""), "0Gi", "0Gi", "0Gi")
|
||||
|
||||
// synchronize
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have disk pressure
|
||||
if manager.IsUnderDiskPressure() {
|
||||
@ -497,7 +483,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
|
||||
// induce soft threshold
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("1.5Gi", "200Gi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -512,7 +498,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
|
||||
// step forward in time pass the grace period
|
||||
fakeClock.Step(3 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("1.5Gi", "200Gi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -537,7 +523,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
|
||||
// remove disk pressure
|
||||
fakeClock.Step(20 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have disk pressure
|
||||
if manager.IsUnderDiskPressure() {
|
||||
@ -547,7 +533,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
|
||||
// induce disk pressure!
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("500Mi", "200Gi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -572,7 +558,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure (because transition period not yet met)
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -593,7 +579,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
|
||||
fakeClock.Step(5 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have disk pressure (because transition period met)
|
||||
if manager.IsUnderDiskPressure() {
|
||||
@ -638,7 +624,6 @@ func TestMinReclaim(t *testing.T) {
|
||||
fakeClock := clock.NewFakeClock(time.Now())
|
||||
podKiller := &mockPodKiller{}
|
||||
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
|
||||
capacityProvider := newMockCapacityProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("3Gi")}, v1.ResourceList{v1.ResourceMemory: *quantityMustParse("1Gi")})
|
||||
diskGC := &mockDiskGC{err: nil}
|
||||
nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
|
||||
|
||||
@ -673,7 +658,7 @@ func TestMinReclaim(t *testing.T) {
|
||||
}
|
||||
|
||||
// synchronize
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have memory pressure
|
||||
if manager.IsUnderMemoryPressure() {
|
||||
@ -683,7 +668,7 @@ func TestMinReclaim(t *testing.T) {
|
||||
// induce memory pressure!
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("500Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -703,7 +688,7 @@ func TestMinReclaim(t *testing.T) {
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("1.2Gi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure (because transition period not yet met)
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -723,7 +708,7 @@ func TestMinReclaim(t *testing.T) {
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure (because transition period not yet met)
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -739,7 +724,7 @@ func TestMinReclaim(t *testing.T) {
|
||||
fakeClock.Step(5 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have memory pressure (because transition period met)
|
||||
if manager.IsUnderMemoryPressure() {
|
||||
@ -781,7 +766,6 @@ func TestNodeReclaimFuncs(t *testing.T) {
|
||||
fakeClock := clock.NewFakeClock(time.Now())
|
||||
podKiller := &mockPodKiller{}
|
||||
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
|
||||
capacityProvider := newMockCapacityProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("3Gi")}, v1.ResourceList{v1.ResourceMemory: *quantityMustParse("1Gi")})
|
||||
nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
|
||||
|
||||
config := Config{
|
||||
@ -816,7 +800,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
|
||||
}
|
||||
|
||||
// synchronize
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have disk pressure
|
||||
if manager.IsUnderDiskPressure() {
|
||||
@ -828,7 +812,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
|
||||
summaryProvider.result = summaryStatsMaker(".9Gi", "200Gi", podStats)
|
||||
// make GC successfully return disk usage to previous levels
|
||||
diskGC.summaryAfterGC = summaryStatsMaker("16Gi", "200Gi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -851,7 +835,8 @@ func TestNodeReclaimFuncs(t *testing.T) {
|
||||
|
||||
// remove disk pressure
|
||||
fakeClock.Step(20 * time.Minute)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have disk pressure
|
||||
if manager.IsUnderDiskPressure() {
|
||||
@ -863,7 +848,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
|
||||
summaryProvider.result = summaryStatsMaker("400Mi", "200Gi", podStats)
|
||||
// Dont reclaim any disk
|
||||
diskGC.summaryAfterGC = summaryStatsMaker("400Mi", "200Gi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -890,7 +875,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
|
||||
diskGC.imageGCInvoked = false // reset state
|
||||
diskGC.containerGCInvoked = false // reset state
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure (because transition period not yet met)
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -913,7 +898,7 @@ func TestNodeReclaimFuncs(t *testing.T) {
|
||||
diskGC.imageGCInvoked = false // reset state
|
||||
diskGC.containerGCInvoked = false // reset state
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have disk pressure (because transition period met)
|
||||
if manager.IsUnderDiskPressure() {
|
||||
@ -981,7 +966,6 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
|
||||
fakeClock := clock.NewFakeClock(time.Now())
|
||||
podKiller := &mockPodKiller{}
|
||||
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
|
||||
capacityProvider := newMockCapacityProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("3Gi")}, v1.ResourceList{v1.ResourceMemory: *quantityMustParse("1Gi")})
|
||||
diskGC := &mockDiskGC{err: nil}
|
||||
nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
|
||||
|
||||
@ -1024,7 +1008,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
|
||||
podToAdmit, _ := podMaker("pod-to-admit", defaultPriority, newResourceList("", "", ""), newResourceList("", "", ""), "0", "0", "0")
|
||||
|
||||
// synchronize
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have disk pressure
|
||||
if manager.IsUnderDiskPressure() {
|
||||
@ -1039,7 +1023,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
|
||||
// induce soft threshold
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("1.5Mi", "4Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -1054,7 +1038,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
|
||||
// step forward in time pass the grace period
|
||||
fakeClock.Step(3 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("1.5Mi", "4Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -1079,7 +1063,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
|
||||
// remove inode pressure
|
||||
fakeClock.Step(20 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("3Mi", "4Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have disk pressure
|
||||
if manager.IsUnderDiskPressure() {
|
||||
@ -1089,7 +1073,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
|
||||
// induce inode pressure!
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("0.5Mi", "4Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -1114,7 +1098,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("3Mi", "4Mi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have disk pressure (because transition period not yet met)
|
||||
if !manager.IsUnderDiskPressure() {
|
||||
@ -1135,7 +1119,7 @@ func TestInodePressureNodeFsInodes(t *testing.T) {
|
||||
fakeClock.Step(5 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("3Mi", "4Mi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have disk pressure (because transition period met)
|
||||
if manager.IsUnderDiskPressure() {
|
||||
@ -1184,7 +1168,6 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
|
||||
fakeClock := clock.NewFakeClock(time.Now())
|
||||
podKiller := &mockPodKiller{}
|
||||
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
|
||||
capacityProvider := newMockCapacityProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("3Gi")}, v1.ResourceList{v1.ResourceMemory: *quantityMustParse("1Gi")})
|
||||
diskGC := &mockDiskGC{err: nil}
|
||||
nodeRef := &v1.ObjectReference{
|
||||
Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: "",
|
||||
@ -1230,7 +1213,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
|
||||
// induce soft threshold
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -1245,7 +1228,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
|
||||
// step forward in time pass the grace period
|
||||
fakeClock.Step(3 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -1263,7 +1246,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
|
||||
// remove memory pressure
|
||||
fakeClock.Step(20 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("3Gi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have memory pressure
|
||||
if manager.IsUnderMemoryPressure() {
|
||||
@ -1276,7 +1259,7 @@ func TestCriticalPodsAreNotEvicted(t *testing.T) {
|
||||
// induce memory pressure!
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker("500Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -1294,7 +1277,6 @@ func TestAllocatableMemoryPressure(t *testing.T) {
|
||||
utilfeature.DefaultFeatureGate.SetFromMap(map[string]bool{string(features.PodPriority): true})
|
||||
podMaker := makePodWithMemoryStats
|
||||
summaryStatsMaker := makeMemoryStats
|
||||
constantCapacity := "4Gi"
|
||||
podsToMake := []podToMake{
|
||||
{name: "guaranteed-low-priority-high-usage", priority: lowPriority, requests: newResourceList("100m", "1Gi", ""), limits: newResourceList("100m", "1Gi", ""), memoryWorkingSet: "900Mi"},
|
||||
{name: "burstable-below-requests", priority: defaultPriority, requests: newResourceList("100m", "100Mi", ""), limits: newResourceList("200m", "1Gi", ""), memoryWorkingSet: "50Mi"},
|
||||
@ -1317,7 +1299,6 @@ func TestAllocatableMemoryPressure(t *testing.T) {
|
||||
fakeClock := clock.NewFakeClock(time.Now())
|
||||
podKiller := &mockPodKiller{}
|
||||
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
|
||||
capacityProvider := newMockCapacityProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("3Gi")}, v1.ResourceList{v1.ResourceMemory: *quantityMustParse("1Gi")})
|
||||
diskGC := &mockDiskGC{err: nil}
|
||||
nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
|
||||
|
||||
@ -1329,12 +1310,12 @@ func TestAllocatableMemoryPressure(t *testing.T) {
|
||||
Signal: evictionapi.SignalAllocatableMemoryAvailable,
|
||||
Operator: evictionapi.OpLessThan,
|
||||
Value: evictionapi.ThresholdValue{
|
||||
Quantity: quantityMustParse("1Ki"),
|
||||
Quantity: quantityMustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker(constantCapacity, podStats)}
|
||||
summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker("4Gi", podStats)}
|
||||
manager := &managerImpl{
|
||||
clock: fakeClock,
|
||||
killPodFunc: podKiller.killPodNow,
|
||||
@ -1353,7 +1334,7 @@ func TestAllocatableMemoryPressure(t *testing.T) {
|
||||
burstablePodToAdmit, _ := podMaker("burst-admit", defaultPriority, newResourceList("100m", "100Mi", ""), newResourceList("200m", "200Mi", ""), "0Gi")
|
||||
|
||||
// synchronize
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have memory pressure
|
||||
if manager.IsUnderMemoryPressure() {
|
||||
@ -1372,8 +1353,8 @@ func TestAllocatableMemoryPressure(t *testing.T) {
|
||||
fakeClock.Step(1 * time.Minute)
|
||||
pod, podStat := podMaker("guaranteed-high-2", defaultPriority, newResourceList("100m", "1Gi", ""), newResourceList("100m", "1Gi", ""), "1Gi")
|
||||
podStats[pod] = podStat
|
||||
summaryProvider.result = summaryStatsMaker(constantCapacity, podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
summaryProvider.result = summaryStatsMaker("500Mi", podStats)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -1407,9 +1388,9 @@ func TestAllocatableMemoryPressure(t *testing.T) {
|
||||
delete(podStats, pod)
|
||||
}
|
||||
}
|
||||
summaryProvider.result = summaryStatsMaker(constantCapacity, podStats)
|
||||
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should have memory pressure (because transition period not yet met)
|
||||
if !manager.IsUnderMemoryPressure() {
|
||||
@ -1431,9 +1412,9 @@ func TestAllocatableMemoryPressure(t *testing.T) {
|
||||
|
||||
// move the clock past transition period to ensure that we stop reporting pressure
|
||||
fakeClock.Step(5 * time.Minute)
|
||||
summaryProvider.result = summaryStatsMaker(constantCapacity, podStats)
|
||||
summaryProvider.result = summaryStatsMaker("2Gi", podStats)
|
||||
podKiller.pod = nil // reset state
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider)
|
||||
manager.synchronize(diskInfoProvider, activePodsFunc)
|
||||
|
||||
// we should not have memory pressure (because transition period met)
|
||||
if manager.IsUnderMemoryPressure() {
|
||||
|
||||
@ -100,9 +100,6 @@ func validSignal(signal evictionapi.Signal) bool {
|
||||
// ParseThresholdConfig parses the flags for thresholds.
|
||||
func ParseThresholdConfig(allocatableConfig []string, evictionHard, evictionSoft, evictionSoftGracePeriod, evictionMinimumReclaim map[string]string) ([]evictionapi.Threshold, error) {
|
||||
results := []evictionapi.Threshold{}
|
||||
allocatableThresholds := getAllocatableThreshold(allocatableConfig)
|
||||
results = append(results, allocatableThresholds...)
|
||||
|
||||
hardThresholds, err := parseThresholdStatements(evictionHard)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -137,9 +134,31 @@ func ParseThresholdConfig(allocatableConfig []string, evictionHard, evictionSoft
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, key := range allocatableConfig {
|
||||
if key == kubetypes.NodeAllocatableEnforcementKey {
|
||||
results = addAllocatableThresholds(results)
|
||||
break
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func addAllocatableThresholds(thresholds []evictionapi.Threshold) []evictionapi.Threshold {
|
||||
additionalThresholds := []evictionapi.Threshold{}
|
||||
for _, threshold := range thresholds {
|
||||
if threshold.Signal == evictionapi.SignalMemoryAvailable && isHardEvictionThreshold(threshold) {
|
||||
// Copy the SignalMemoryAvailable to SignalAllocatableMemoryAvailable
|
||||
additionalThresholds = append(additionalThresholds, evictionapi.Threshold{
|
||||
Signal: evictionapi.SignalAllocatableMemoryAvailable,
|
||||
Operator: threshold.Operator,
|
||||
Value: threshold.Value,
|
||||
MinReclaim: threshold.MinReclaim,
|
||||
})
|
||||
}
|
||||
}
|
||||
return append(thresholds, additionalThresholds...)
|
||||
}
|
||||
|
||||
// parseThresholdStatements parses the input statements into a list of Threshold objects.
|
||||
func parseThresholdStatements(statements map[string]string) ([]evictionapi.Threshold, error) {
|
||||
if len(statements) == 0 {
|
||||
@ -204,27 +223,6 @@ func parseThresholdStatement(signal evictionapi.Signal, val string) (*evictionap
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getAllocatableThreshold returns the thresholds applicable for the allocatable configuration
|
||||
func getAllocatableThreshold(allocatableConfig []string) []evictionapi.Threshold {
|
||||
for _, key := range allocatableConfig {
|
||||
if key == kubetypes.NodeAllocatableEnforcementKey {
|
||||
return []evictionapi.Threshold{
|
||||
{
|
||||
Signal: evictionapi.SignalAllocatableMemoryAvailable,
|
||||
Operator: evictionapi.OpLessThan,
|
||||
Value: evictionapi.ThresholdValue{
|
||||
Quantity: resource.NewQuantity(int64(0), resource.BinarySI),
|
||||
},
|
||||
MinReclaim: &evictionapi.ThresholdValue{
|
||||
Quantity: resource.NewQuantity(int64(0), resource.BinarySI),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
return []evictionapi.Threshold{}
|
||||
}
|
||||
|
||||
// parsePercentage parses a string representing a percentage value
|
||||
func parsePercentage(input string) (float32, error) {
|
||||
value, err := strconv.ParseFloat(strings.TrimRight(input, "%"), 32)
|
||||
@ -724,7 +722,7 @@ func (a byEvictionPriority) Less(i, j int) bool {
|
||||
}
|
||||
|
||||
// makeSignalObservations derives observations using the specified summary provider.
|
||||
func makeSignalObservations(summary *statsapi.Summary, capacityProvider CapacityProvider, pods []*v1.Pod) (signalObservations, statsFunc) {
|
||||
func makeSignalObservations(summary *statsapi.Summary) (signalObservations, statsFunc) {
|
||||
// build the function to work against for pod stats
|
||||
statsFunc := cachedStatsFunc(summary.Pods)
|
||||
// build an evaluation context for current eviction signals
|
||||
@ -737,6 +735,17 @@ func makeSignalObservations(summary *statsapi.Summary, capacityProvider Capacity
|
||||
time: memory.Time,
|
||||
}
|
||||
}
|
||||
if allocatableContainer, err := getSysContainer(summary.Node.SystemContainers, statsapi.SystemContainerPods); err != nil {
|
||||
glog.Errorf("eviction manager: failed to construct signal: %q error: %v", evictionapi.SignalAllocatableMemoryAvailable, err)
|
||||
} else {
|
||||
if memory := allocatableContainer.Memory; memory != nil && memory.AvailableBytes != nil && memory.WorkingSetBytes != nil {
|
||||
result[evictionapi.SignalAllocatableMemoryAvailable] = signalObservation{
|
||||
available: resource.NewQuantity(int64(*memory.AvailableBytes), resource.BinarySI),
|
||||
capacity: resource.NewQuantity(int64(*memory.AvailableBytes+*memory.WorkingSetBytes), resource.BinarySI),
|
||||
time: memory.Time,
|
||||
}
|
||||
}
|
||||
}
|
||||
if nodeFs := summary.Node.Fs; nodeFs != nil {
|
||||
if nodeFs.AvailableBytes != nil && nodeFs.CapacityBytes != nil {
|
||||
result[evictionapi.SignalNodeFsAvailable] = signalObservation{
|
||||
@ -771,7 +780,6 @@ func makeSignalObservations(summary *statsapi.Summary, capacityProvider Capacity
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if rlimit := summary.Node.Rlimit; rlimit != nil {
|
||||
if rlimit.NumOfRunningProcesses != nil && rlimit.MaxPID != nil {
|
||||
available := int64(*rlimit.MaxPID) - int64(*rlimit.NumOfRunningProcesses)
|
||||
@ -782,29 +790,18 @@ func makeSignalObservations(summary *statsapi.Summary, capacityProvider Capacity
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if memoryAllocatableCapacity, ok := capacityProvider.GetCapacity()[v1.ResourceMemory]; ok {
|
||||
memoryAllocatableAvailable := memoryAllocatableCapacity.Copy()
|
||||
if reserved, exists := capacityProvider.GetNodeAllocatableReservation()[v1.ResourceMemory]; exists {
|
||||
memoryAllocatableAvailable.Sub(reserved)
|
||||
}
|
||||
for _, pod := range summary.Pods {
|
||||
mu, err := podMemoryUsage(pod)
|
||||
if err == nil {
|
||||
memoryAllocatableAvailable.Sub(mu[v1.ResourceMemory])
|
||||
}
|
||||
}
|
||||
result[evictionapi.SignalAllocatableMemoryAvailable] = signalObservation{
|
||||
available: memoryAllocatableAvailable,
|
||||
capacity: &memoryAllocatableCapacity,
|
||||
}
|
||||
} else {
|
||||
glog.Errorf("Could not find capacity information for resource %v", v1.ResourceMemory)
|
||||
}
|
||||
|
||||
return result, statsFunc
|
||||
}
|
||||
|
||||
func getSysContainer(sysContainers []statsapi.ContainerStats, name string) (*statsapi.ContainerStats, error) {
|
||||
for _, cont := range sysContainers {
|
||||
if cont.Name == name {
|
||||
return &cont, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("system container %q not found in metrics", name)
|
||||
}
|
||||
|
||||
// thresholdsMet returns the set of thresholds that were met independent of grace period
|
||||
func thresholdsMet(thresholds []evictionapi.Threshold, observations signalObservations, enforceMinReclaim bool) []evictionapi.Threshold {
|
||||
results := []evictionapi.Threshold{}
|
||||
|
||||
@ -72,7 +72,7 @@ func TestParseThresholdConfig(t *testing.T) {
|
||||
Signal: evictionapi.SignalAllocatableMemoryAvailable,
|
||||
Operator: evictionapi.OpLessThan,
|
||||
Value: evictionapi.ThresholdValue{
|
||||
Quantity: quantityMustParse("0"),
|
||||
Quantity: quantityMustParse("150Mi"),
|
||||
},
|
||||
MinReclaim: &evictionapi.ThresholdValue{
|
||||
Quantity: quantityMustParse("0"),
|
||||
@ -1004,6 +1004,15 @@ func TestMakeSignalObservations(t *testing.T) {
|
||||
InodesFree: &nodeFsInodesFree,
|
||||
Inodes: &nodeFsInodes,
|
||||
},
|
||||
SystemContainers: []statsapi.ContainerStats{
|
||||
{
|
||||
Name: statsapi.SystemContainerPods,
|
||||
Memory: &statsapi.MemoryStats{
|
||||
AvailableBytes: &nodeAvailableBytes,
|
||||
WorkingSetBytes: &nodeWorkingSetBytes,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Pods: []statsapi.PodStats{},
|
||||
}
|
||||
@ -1017,20 +1026,19 @@ func TestMakeSignalObservations(t *testing.T) {
|
||||
fakeStats.Pods = append(fakeStats.Pods, newPodStats(pod, containerWorkingSetBytes))
|
||||
}
|
||||
res := quantityMustParse("5Gi")
|
||||
capacityProvider := newMockCapacityProvider(v1.ResourceList{v1.ResourceMemory: *quantityMustParse("5Gi")}, v1.ResourceList{v1.ResourceMemory: *quantityMustParse("0Gi")})
|
||||
// Allocatable thresholds are always 100%. Verify that Threshold == Capacity.
|
||||
if res.CmpInt64(int64(allocatableMemoryCapacity)) != 0 {
|
||||
t.Errorf("Expected Threshold %v to be equal to value %v", res.Value(), allocatableMemoryCapacity)
|
||||
}
|
||||
actualObservations, statsFunc := makeSignalObservations(fakeStats, capacityProvider, pods)
|
||||
actualObservations, statsFunc := makeSignalObservations(fakeStats)
|
||||
allocatableMemQuantity, found := actualObservations[evictionapi.SignalAllocatableMemoryAvailable]
|
||||
if !found {
|
||||
t.Errorf("Expected allocatable memory observation, but didnt find one")
|
||||
}
|
||||
if allocatableMemQuantity.available.Value() != 2*containerWorkingSetBytes {
|
||||
t.Errorf("Expected %v, actual: %v", containerWorkingSetBytes, allocatableMemQuantity.available.Value())
|
||||
if expectedBytes := int64(nodeAvailableBytes); allocatableMemQuantity.available.Value() != expectedBytes {
|
||||
t.Errorf("Expected %v, actual: %v", expectedBytes, allocatableMemQuantity.available.Value())
|
||||
}
|
||||
if expectedBytes := int64(allocatableMemoryCapacity); allocatableMemQuantity.capacity.Value() != expectedBytes {
|
||||
if expectedBytes := int64(nodeWorkingSetBytes + nodeAvailableBytes); allocatableMemQuantity.capacity.Value() != expectedBytes {
|
||||
t.Errorf("Expected %v, actual: %v", expectedBytes, allocatableMemQuantity.capacity.Value())
|
||||
}
|
||||
memQuantity, found := actualObservations[evictionapi.SignalMemoryAvailable]
|
||||
|
||||
@ -53,7 +53,7 @@ type Config struct {
|
||||
// Manager evaluates when an eviction threshold for node stability has been met on the node.
|
||||
type Manager interface {
|
||||
// Start starts the control loop to monitor eviction thresholds at specified interval.
|
||||
Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, capacityProvider CapacityProvider, monitoringInterval time.Duration)
|
||||
Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, podCleanedUpFunc PodCleanedUpFunc, monitoringInterval time.Duration)
|
||||
|
||||
// IsUnderMemoryPressure returns true if the node is under memory pressure.
|
||||
IsUnderMemoryPressure() bool
|
||||
@ -71,14 +71,6 @@ type DiskInfoProvider interface {
|
||||
HasDedicatedImageFs() (bool, error)
|
||||
}
|
||||
|
||||
// CapacityProvider is responsible for providing the resource capacity and reservation information
|
||||
type CapacityProvider interface {
|
||||
// GetCapacity returns the amount of compute resources tracked by container manager available on the node.
|
||||
GetCapacity() v1.ResourceList
|
||||
// GetNodeAllocatableReservation returns the amount of compute resources that have to be reserved from scheduling.
|
||||
GetNodeAllocatableReservation() v1.ResourceList
|
||||
}
|
||||
|
||||
// ImageGC is responsible for performing garbage collection of unused images.
|
||||
type ImageGC interface {
|
||||
// DeleteUnusedImages deletes unused images.
|
||||
|
||||
@ -1318,7 +1318,7 @@ func (kl *Kubelet) initializeRuntimeDependentModules() {
|
||||
glog.Fatalf("Failed to start cAdvisor %v", err)
|
||||
}
|
||||
// eviction manager must start after cadvisor because it needs to know if the container runtime has a dedicated imagefs
|
||||
kl.evictionManager.Start(kl.StatsProvider, kl.GetActivePods, kl.podResourcesAreReclaimed, kl.containerManager, evictionMonitoringPeriod)
|
||||
kl.evictionManager.Start(kl.StatsProvider, kl.GetActivePods, kl.podResourcesAreReclaimed, evictionMonitoringPeriod)
|
||||
|
||||
// trigger on-demand stats collection once so that we have capacity information for ephemeral storage.
|
||||
// ignore any errors, since if stats collection is not successful, the container manager will fail to start below.
|
||||
|
||||
@ -211,6 +211,11 @@ func (kl *Kubelet) GetNodeConfig() cm.NodeConfig {
|
||||
return kl.containerManager.GetNodeConfig()
|
||||
}
|
||||
|
||||
// GetPodCgroupRoot returns the listeral cgroupfs value for the cgroup containing all pods
|
||||
func (kl *Kubelet) GetPodCgroupRoot() string {
|
||||
return kl.containerManager.GetPodCgroupRoot()
|
||||
}
|
||||
|
||||
// GetHostIP returns host IP or nil in case of error.
|
||||
func (kl *Kubelet) GetHostIP() (net.IP, error) {
|
||||
node, err := kl.GetNode()
|
||||
|
||||
@ -168,6 +168,7 @@ func (fk *fakeKubelet) StreamingConnectionIdleTimeout() time.Duration {
|
||||
// Unused functions
|
||||
func (_ *fakeKubelet) GetNode() (*v1.Node, error) { return nil, nil }
|
||||
func (_ *fakeKubelet) GetNodeConfig() cm.NodeConfig { return cm.NodeConfig{} }
|
||||
func (_ *fakeKubelet) GetPodCgroupRoot() string { return "" }
|
||||
|
||||
func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Volume, bool) {
|
||||
return map[string]volume.Volume{}, true
|
||||
|
||||
@ -81,6 +81,9 @@ type StatsProvider interface {
|
||||
|
||||
// RlimitStats returns the rlimit stats of system.
|
||||
RlimitStats() (*statsapi.RlimitStats, error)
|
||||
|
||||
// GetPodCgroupRoot returns the literal cgroupfs value for the cgroup containing all pods
|
||||
GetPodCgroupRoot() string
|
||||
}
|
||||
|
||||
type handler struct {
|
||||
|
||||
@ -83,19 +83,23 @@ func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error)
|
||||
Rlimit: rlimit,
|
||||
}
|
||||
|
||||
systemContainers := map[string]string{
|
||||
statsapi.SystemContainerKubelet: nodeConfig.KubeletCgroupsName,
|
||||
statsapi.SystemContainerRuntime: nodeConfig.RuntimeCgroupsName,
|
||||
statsapi.SystemContainerMisc: nodeConfig.SystemCgroupsName,
|
||||
systemContainers := map[string]struct {
|
||||
name string
|
||||
forceStatsUpdate bool
|
||||
}{
|
||||
statsapi.SystemContainerKubelet: {nodeConfig.KubeletCgroupsName, false},
|
||||
statsapi.SystemContainerRuntime: {nodeConfig.RuntimeCgroupsName, false},
|
||||
statsapi.SystemContainerMisc: {nodeConfig.SystemCgroupsName, false},
|
||||
statsapi.SystemContainerPods: {sp.provider.GetPodCgroupRoot(), updateStats},
|
||||
}
|
||||
for sys, name := range systemContainers {
|
||||
for sys, cont := range systemContainers {
|
||||
// skip if cgroup name is undefined (not all system containers are required)
|
||||
if name == "" {
|
||||
if cont.name == "" {
|
||||
continue
|
||||
}
|
||||
s, _, err := sp.provider.GetCgroupStats(name, false)
|
||||
s, _, err := sp.provider.GetCgroupStats(cont.name, cont.forceStatsUpdate)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get system container stats for %q: %v", name, err)
|
||||
glog.Errorf("Failed to get system container stats for %q: %v", cont.name, err)
|
||||
continue
|
||||
}
|
||||
// System containers don't have a filesystem associated with them.
|
||||
|
||||
@ -49,6 +49,7 @@ func TestSummaryProvider(t *testing.T) {
|
||||
SystemCgroupsName: "/misc",
|
||||
KubeletCgroupsName: "/kubelet",
|
||||
}
|
||||
cgroupRoot = "/kubepods"
|
||||
cgroupStatsMap = map[string]struct {
|
||||
cs *statsapi.ContainerStats
|
||||
ns *statsapi.NetworkStats
|
||||
@ -57,6 +58,7 @@ func TestSummaryProvider(t *testing.T) {
|
||||
"/runtime": {cs: getContainerStats(), ns: getNetworkStats()},
|
||||
"/misc": {cs: getContainerStats(), ns: getNetworkStats()},
|
||||
"/kubelet": {cs: getContainerStats(), ns: getNetworkStats()},
|
||||
"/pods": {cs: getContainerStats(), ns: getNetworkStats()},
|
||||
}
|
||||
rlimitStats = getRlimitStats()
|
||||
)
|
||||
@ -67,6 +69,7 @@ func TestSummaryProvider(t *testing.T) {
|
||||
mockStatsProvider.
|
||||
On("GetNode").Return(node, nil).
|
||||
On("GetNodeConfig").Return(nodeConfig).
|
||||
On("GetPodCgroupRoot").Return(cgroupRoot).
|
||||
On("ListPodStats").Return(podStats, nil).
|
||||
On("ImageFsStats").Return(imageFsStats, nil).
|
||||
On("RootFsStats").Return(rootFsStats, nil).
|
||||
@ -74,7 +77,8 @@ func TestSummaryProvider(t *testing.T) {
|
||||
On("GetCgroupStats", "/", true).Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil).
|
||||
On("GetCgroupStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil).
|
||||
On("GetCgroupStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil).
|
||||
On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil)
|
||||
On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil).
|
||||
On("GetCgroupStats", "/kubepods", true).Return(cgroupStatsMap["/pods"].cs, cgroupStatsMap["/pods"].ns, nil)
|
||||
|
||||
provider := NewSummaryProvider(mockStatsProvider)
|
||||
summary, err := provider.Get(true)
|
||||
@ -88,7 +92,7 @@ func TestSummaryProvider(t *testing.T) {
|
||||
assert.Equal(summary.Node.Fs, rootFsStats)
|
||||
assert.Equal(summary.Node.Runtime, &statsapi.RuntimeStats{ImageFs: imageFsStats})
|
||||
|
||||
assert.Equal(len(summary.Node.SystemContainers), 3)
|
||||
assert.Equal(len(summary.Node.SystemContainers), 4)
|
||||
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
|
||||
Name: "kubelet",
|
||||
StartTime: cgroupStatsMap["/kubelet"].cs.StartTime,
|
||||
@ -113,6 +117,14 @@ func TestSummaryProvider(t *testing.T) {
|
||||
Accelerators: cgroupStatsMap["/runtime"].cs.Accelerators,
|
||||
UserDefinedMetrics: cgroupStatsMap["/runtime"].cs.UserDefinedMetrics,
|
||||
})
|
||||
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
|
||||
Name: "pods",
|
||||
StartTime: cgroupStatsMap["/pods"].cs.StartTime,
|
||||
CPU: cgroupStatsMap["/pods"].cs.CPU,
|
||||
Memory: cgroupStatsMap["/pods"].cs.Memory,
|
||||
Accelerators: cgroupStatsMap["/pods"].cs.Accelerators,
|
||||
UserDefinedMetrics: cgroupStatsMap["/pods"].cs.UserDefinedMetrics,
|
||||
})
|
||||
assert.Equal(summary.Pods, podStats)
|
||||
}
|
||||
|
||||
|
||||
@ -125,6 +125,20 @@ func (_m *StatsProvider) GetNodeConfig() cm.NodeConfig {
|
||||
return r0
|
||||
}
|
||||
|
||||
// GetPodCgroupRoot provides a mock function with given fields:
|
||||
func (_m *StatsProvider) GetPodCgroupRoot() string {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 string
|
||||
if rf, ok := ret.Get(0).(func() string); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Get(0).(string)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// GetPodByName provides a mock function with given fields: namespace, name
|
||||
func (_m *StatsProvider) GetPodByName(namespace string, name string) (*corev1.Pod, bool) {
|
||||
ret := _m.Called(namespace, name)
|
||||
|
||||
@ -107,9 +107,21 @@ var _ = framework.KubeDescribe("Summary API", func() {
|
||||
"UserDefinedMetrics": BeEmpty(),
|
||||
})
|
||||
}
|
||||
podsContExpectations := sysContExpectations().(*gstruct.FieldsMatcher)
|
||||
podsContExpectations.Fields["Memory"] = ptrMatchAllFields(gstruct.Fields{
|
||||
"Time": recent(maxStatsAge),
|
||||
// Pods are limited by Node Allocatable
|
||||
"AvailableBytes": bounded(1*framework.Kb, memoryLimit),
|
||||
"UsageBytes": bounded(10*framework.Kb, 20*framework.Mb),
|
||||
"WorkingSetBytes": bounded(10*framework.Kb, 20*framework.Mb),
|
||||
"RSSBytes": bounded(1*framework.Kb, 20*framework.Mb),
|
||||
"PageFaults": bounded(0, 1000000),
|
||||
"MajorPageFaults": bounded(0, 10),
|
||||
})
|
||||
systemContainers := gstruct.Elements{
|
||||
"kubelet": sysContExpectations(),
|
||||
"runtime": sysContExpectations(),
|
||||
"pods": podsContExpectations,
|
||||
}
|
||||
// The Kubelet only manages the 'misc' system container if the host is not running systemd.
|
||||
if !systemdutil.IsRunningSystemd() {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user