eviction by process number

This commit is contained in:
louisgong 2020-03-23 17:15:59 +08:00
parent 1a334335bc
commit 0efb70c0a2
8 changed files with 109 additions and 3 deletions

View File

@ -91,6 +91,13 @@ const (
SystemContainerPods = "pods"
)
// ProcessStats are stats pertaining to processes.
type ProcessStats struct {
// Number of processes
// +optional
ProcessCount *uint64 `json:"process_count,omitempty"`
}
// PodStats holds pod-level unprocessed sample stats.
type PodStats struct {
// Reference to the measured Pod.
@ -119,6 +126,9 @@ type PodStats struct {
// EphemeralStorage reports the total filesystem usage for the containers and emptyDir-backed volumes in the measured Pod.
// +optional
EphemeralStorage *FsStats `json:"ephemeral-storage,omitempty"`
// ProcessStats pertaining to processes.
// +optional
ProcessStats *ProcessStats `json:"process_stats,omitempty"`
}
// ContainerStats holds container-level unprocessed sample stats.

View File

@ -328,6 +328,15 @@ func memoryUsage(memStats *statsapi.MemoryStats) *resource.Quantity {
return resource.NewQuantity(usage, resource.BinarySI)
}
// processUsage converts working set into a process count.
func processUsage(processStats *statsapi.ProcessStats) uint64 {
if processStats == nil || processStats.ProcessCount == nil {
return 0
}
usage := uint64(*processStats.ProcessCount)
return usage
}
// localVolumeNames returns the set of volumes for the pod that are local
// TODO: summary API should report what volumes consume local storage rather than hard-code here.
func localVolumeNames(pod *v1.Pod) []string {
@ -566,6 +575,23 @@ func memory(stats statsFunc) cmpFunc {
}
}
// process compares pods by largest consumer of process number relative to request.
func process(stats statsFunc) cmpFunc {
return func(p1, p2 *v1.Pod) int {
p1Stats, p1Found := stats(p1)
p2Stats, p2Found := stats(p2)
if !p1Found || !p2Found {
// prioritize evicting the pod for which no stats were found
return cmpBool(!p1Found, !p2Found)
}
p1Process := processUsage(p1Stats.ProcessStats)
p2Process := processUsage(p2Stats.ProcessStats)
// prioritize evicting the pod which has the larger consumption of process
return int(p2Process - p1Process)
}
}
// exceedDiskRequests compares whether or not pods' disk usage exceeds their requests
func exceedDiskRequests(stats statsFunc, fsStatsToMeasure []fsStatsType, diskResource v1.ResourceName) cmpFunc {
return func(p1, p2 *v1.Pod) int {
@ -640,7 +666,7 @@ func rankMemoryPressure(pods []*v1.Pod, stats statsFunc) {
// rankPIDPressure orders the input pods by priority in response to PID pressure.
func rankPIDPressure(pods []*v1.Pod, stats statsFunc) {
orderedBy(priority).Sort(pods)
orderedBy(priority, process(stats)).Sort(pods)
}
// rankDiskPressureFunc returns a rankFunc that measures the specified fs stats.

View File

@ -958,6 +958,32 @@ func TestOrderedByPriorityMemory(t *testing.T) {
}
}
// TestOrderedByPriorityProcess ensures we order by priority and then process consumption relative to request.
func TestOrderedByPriorityProcess(t *testing.T) {
pod1 := newPod("low-priority-high-usage", lowPriority, nil, nil)
pod2 := newPod("low-priority-low-usage", lowPriority, nil, nil)
pod3 := newPod("high-priority-high-usage", highPriority, nil, nil)
pod4 := newPod("high-priority-low-usage", highPriority, nil, nil)
stats := map[*v1.Pod]statsapi.PodStats{
pod1: newPodProcessStats(pod1, 20),
pod2: newPodProcessStats(pod2, 6),
pod3: newPodProcessStats(pod3, 20),
pod4: newPodProcessStats(pod4, 5),
}
statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) {
result, found := stats[pod]
return result, found
}
pods := []*v1.Pod{pod4, pod3, pod2, pod1}
expected := []*v1.Pod{pod1, pod2, pod3, pod4}
orderedBy(priority, process(statsFn)).Sort(pods)
for i := range expected {
if pods[i] != expected[i] {
t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name)
}
}
}
func TestSortByEvictionPriority(t *testing.T) {
for _, tc := range []struct {
name string
@ -1884,6 +1910,17 @@ func newPodMemoryStats(pod *v1.Pod, workingSet resource.Quantity) statsapi.PodSt
}
}
func newPodProcessStats(pod *v1.Pod, num uint64) statsapi.PodStats {
return statsapi.PodStats{
PodRef: statsapi.PodReference{
Name: pod.Name, Namespace: pod.Namespace, UID: string(pod.UID),
},
ProcessStats: &statsapi.ProcessStats{
ProcessCount: &num,
},
}
}
func newResourceList(cpu, memory, disk string) v1.ResourceList {
res := v1.ResourceList{}
if cpu != "" {

View File

@ -144,6 +144,7 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
cpu, memory := cadvisorInfoToCPUandMemoryStats(podInfo)
podStats.CPU = cpu
podStats.Memory = memory
podStats.ProcessStats = cadvisorInfoToProcessStats(podInfo)
}
status, found := p.statusProvider.GetPodStatus(podUID)

View File

@ -199,6 +199,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi
cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata(), updateCPUNanoCoreUsage)
p.addPodNetworkStats(ps, podSandboxID, caInfos, cs, containerNetworkStats[podSandboxID])
p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)
p.addProcessStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)
// If cadvisor stats is available for the container, use it to populate
// container stats
@ -491,6 +492,20 @@ func (p *criStatsProvider) addPodCPUMemoryStats(
}
}
func (p *criStatsProvider) addProcessStats(
ps *statsapi.PodStats,
podUID types.UID,
allInfos map[string]cadvisorapiv2.ContainerInfo,
cs *statsapi.ContainerStats,
) {
// try get process stats from cadvisor only.
info := getCadvisorPodInfoFromPodUID(podUID, allInfos)
if info != nil {
ps.ProcessStats = cadvisorInfoToProcessStats(info)
return
}
}
func (p *criStatsProvider) makeContainerStats(
stats *runtimeapi.ContainerStats,
container *runtimeapi.Container,

View File

@ -153,6 +153,15 @@ func cadvisorInfoToContainerCPUAndMemoryStats(name string, info *cadvisorapiv2.C
return result
}
func cadvisorInfoToProcessStats(info *cadvisorapiv2.ContainerInfo) *statsapi.ProcessStats {
cstat, found := latestContainerStats(info)
if !found || cstat.Processes == nil {
return nil
}
num := cstat.Processes.ProcessCount
return &statsapi.ProcessStats{ProcessCount: uint64Ptr(num)}
}
// cadvisorInfoToNetworkStats returns the statsapi.NetworkStats converted from
// the container info from cadvisor.
func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.NetworkStats {

View File

@ -422,15 +422,20 @@ var _ = framework.KubeDescribe("PriorityPidEvictionOrdering [Slow] [Serial] [Dis
})
specs := []podEvictSpec{
{
evictionPriority: 1,
pod: pidConsumingPod("fork-bomb-container", 12000),
evictionPriority: 2,
pod: pidConsumingPod("fork-bomb-container-with-low-priority", 12000),
},
{
evictionPriority: 0,
pod: innocentPod(),
},
{
evictionPriority: 1,
pod: pidConsumingPod("fork-bomb-container-with-high-priority", 12000),
},
}
specs[1].pod.Spec.PriorityClassName = highPriorityClassName
specs[2].pod.Spec.PriorityClassName = highPriorityClassName
runEvictionTest(f, pressureTimeout, expectedNodeCondition, expectedStarvedResource, logPidMetrics, specs)
})
})

View File

@ -260,6 +260,9 @@ var _ = framework.KubeDescribe("Summary API [NodeConformance]", func() {
"Inodes": bounded(1e4, 1e8),
"InodesUsed": bounded(0, 1e8),
}),
"ProcessStats": ptrMatchAllFields(gstruct.Fields{
"ProcessCount": bounded(0, 1e8),
}),
})
matchExpectations := ptrMatchAllFields(gstruct.Fields{