From 0efb70c0a2e51f7211a76f8af470b7abd5786a98 Mon Sep 17 00:00:00 2001 From: louisgong Date: Mon, 23 Mar 2020 17:15:59 +0800 Subject: [PATCH] eviction by process number --- pkg/kubelet/apis/stats/v1alpha1/types.go | 10 ++++++ pkg/kubelet/eviction/helpers.go | 28 ++++++++++++++- pkg/kubelet/eviction/helpers_test.go | 37 ++++++++++++++++++++ pkg/kubelet/stats/cadvisor_stats_provider.go | 1 + pkg/kubelet/stats/cri_stats_provider.go | 15 ++++++++ pkg/kubelet/stats/helper.go | 9 +++++ test/e2e_node/eviction_test.go | 9 +++-- test/e2e_node/summary_test.go | 3 ++ 8 files changed, 109 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/apis/stats/v1alpha1/types.go b/pkg/kubelet/apis/stats/v1alpha1/types.go index afec6d0cf9f..810232aab64 100644 --- a/pkg/kubelet/apis/stats/v1alpha1/types.go +++ b/pkg/kubelet/apis/stats/v1alpha1/types.go @@ -91,6 +91,13 @@ const ( SystemContainerPods = "pods" ) +// ProcessStats are stats pertaining to processes. +type ProcessStats struct { + // Number of processes + // +optional + ProcessCount *uint64 `json:"process_count,omitempty"` +} + // PodStats holds pod-level unprocessed sample stats. type PodStats struct { // Reference to the measured Pod. @@ -119,6 +126,9 @@ type PodStats struct { // EphemeralStorage reports the total filesystem usage for the containers and emptyDir-backed volumes in the measured Pod. // +optional EphemeralStorage *FsStats `json:"ephemeral-storage,omitempty"` + // ProcessStats pertaining to processes. + // +optional + ProcessStats *ProcessStats `json:"process_stats,omitempty"` } // ContainerStats holds container-level unprocessed sample stats. diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index 65a00b32951..77de81d8463 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -328,6 +328,15 @@ func memoryUsage(memStats *statsapi.MemoryStats) *resource.Quantity { return resource.NewQuantity(usage, resource.BinarySI) } +// processUsage converts working set into a process count. +func processUsage(processStats *statsapi.ProcessStats) uint64 { + if processStats == nil || processStats.ProcessCount == nil { + return 0 + } + usage := uint64(*processStats.ProcessCount) + return usage +} + // localVolumeNames returns the set of volumes for the pod that are local // TODO: summary API should report what volumes consume local storage rather than hard-code here. func localVolumeNames(pod *v1.Pod) []string { @@ -566,6 +575,23 @@ func memory(stats statsFunc) cmpFunc { } } +// process compares pods by largest consumer of process number relative to request. +func process(stats statsFunc) cmpFunc { + return func(p1, p2 *v1.Pod) int { + p1Stats, p1Found := stats(p1) + p2Stats, p2Found := stats(p2) + if !p1Found || !p2Found { + // prioritize evicting the pod for which no stats were found + return cmpBool(!p1Found, !p2Found) + } + + p1Process := processUsage(p1Stats.ProcessStats) + p2Process := processUsage(p2Stats.ProcessStats) + // prioritize evicting the pod which has the larger consumption of process + return int(p2Process - p1Process) + } +} + // exceedDiskRequests compares whether or not pods' disk usage exceeds their requests func exceedDiskRequests(stats statsFunc, fsStatsToMeasure []fsStatsType, diskResource v1.ResourceName) cmpFunc { return func(p1, p2 *v1.Pod) int { @@ -640,7 +666,7 @@ func rankMemoryPressure(pods []*v1.Pod, stats statsFunc) { // rankPIDPressure orders the input pods by priority in response to PID pressure. func rankPIDPressure(pods []*v1.Pod, stats statsFunc) { - orderedBy(priority).Sort(pods) + orderedBy(priority, process(stats)).Sort(pods) } // rankDiskPressureFunc returns a rankFunc that measures the specified fs stats. diff --git a/pkg/kubelet/eviction/helpers_test.go b/pkg/kubelet/eviction/helpers_test.go index 4bb84d70f25..65b5d064170 100644 --- a/pkg/kubelet/eviction/helpers_test.go +++ b/pkg/kubelet/eviction/helpers_test.go @@ -958,6 +958,32 @@ func TestOrderedByPriorityMemory(t *testing.T) { } } +// TestOrderedByPriorityProcess ensures we order by priority and then process consumption relative to request. +func TestOrderedByPriorityProcess(t *testing.T) { + pod1 := newPod("low-priority-high-usage", lowPriority, nil, nil) + pod2 := newPod("low-priority-low-usage", lowPriority, nil, nil) + pod3 := newPod("high-priority-high-usage", highPriority, nil, nil) + pod4 := newPod("high-priority-low-usage", highPriority, nil, nil) + stats := map[*v1.Pod]statsapi.PodStats{ + pod1: newPodProcessStats(pod1, 20), + pod2: newPodProcessStats(pod2, 6), + pod3: newPodProcessStats(pod3, 20), + pod4: newPodProcessStats(pod4, 5), + } + statsFn := func(pod *v1.Pod) (statsapi.PodStats, bool) { + result, found := stats[pod] + return result, found + } + pods := []*v1.Pod{pod4, pod3, pod2, pod1} + expected := []*v1.Pod{pod1, pod2, pod3, pod4} + orderedBy(priority, process(statsFn)).Sort(pods) + for i := range expected { + if pods[i] != expected[i] { + t.Errorf("Expected pod[%d]: %s, but got: %s", i, expected[i].Name, pods[i].Name) + } + } +} + func TestSortByEvictionPriority(t *testing.T) { for _, tc := range []struct { name string @@ -1884,6 +1910,17 @@ func newPodMemoryStats(pod *v1.Pod, workingSet resource.Quantity) statsapi.PodSt } } +func newPodProcessStats(pod *v1.Pod, num uint64) statsapi.PodStats { + return statsapi.PodStats{ + PodRef: statsapi.PodReference{ + Name: pod.Name, Namespace: pod.Namespace, UID: string(pod.UID), + }, + ProcessStats: &statsapi.ProcessStats{ + ProcessCount: &num, + }, + } +} + func newResourceList(cpu, memory, disk string) v1.ResourceList { res := v1.ResourceList{} if cpu != "" { diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index 18b59dbc7e4..96be34ef991 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -144,6 +144,7 @@ func (p *cadvisorStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { cpu, memory := cadvisorInfoToCPUandMemoryStats(podInfo) podStats.CPU = cpu podStats.Memory = memory + podStats.ProcessStats = cadvisorInfoToProcessStats(podInfo) } status, found := p.statusProvider.GetPodStatus(podUID) diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index 737ae59844e..2f16f9881ba 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -199,6 +199,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata(), updateCPUNanoCoreUsage) p.addPodNetworkStats(ps, podSandboxID, caInfos, cs, containerNetworkStats[podSandboxID]) p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs) + p.addProcessStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs) // If cadvisor stats is available for the container, use it to populate // container stats @@ -491,6 +492,20 @@ func (p *criStatsProvider) addPodCPUMemoryStats( } } +func (p *criStatsProvider) addProcessStats( + ps *statsapi.PodStats, + podUID types.UID, + allInfos map[string]cadvisorapiv2.ContainerInfo, + cs *statsapi.ContainerStats, +) { + // try get process stats from cadvisor only. + info := getCadvisorPodInfoFromPodUID(podUID, allInfos) + if info != nil { + ps.ProcessStats = cadvisorInfoToProcessStats(info) + return + } +} + func (p *criStatsProvider) makeContainerStats( stats *runtimeapi.ContainerStats, container *runtimeapi.Container, diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index 2c8a78c9362..882c70dd65e 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -153,6 +153,15 @@ func cadvisorInfoToContainerCPUAndMemoryStats(name string, info *cadvisorapiv2.C return result } +func cadvisorInfoToProcessStats(info *cadvisorapiv2.ContainerInfo) *statsapi.ProcessStats { + cstat, found := latestContainerStats(info) + if !found || cstat.Processes == nil { + return nil + } + num := cstat.Processes.ProcessCount + return &statsapi.ProcessStats{ProcessCount: uint64Ptr(num)} +} + // cadvisorInfoToNetworkStats returns the statsapi.NetworkStats converted from // the container info from cadvisor. func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.NetworkStats { diff --git a/test/e2e_node/eviction_test.go b/test/e2e_node/eviction_test.go index 9b54e50c4a2..282ad12124b 100644 --- a/test/e2e_node/eviction_test.go +++ b/test/e2e_node/eviction_test.go @@ -422,15 +422,20 @@ var _ = framework.KubeDescribe("PriorityPidEvictionOrdering [Slow] [Serial] [Dis }) specs := []podEvictSpec{ { - evictionPriority: 1, - pod: pidConsumingPod("fork-bomb-container", 12000), + evictionPriority: 2, + pod: pidConsumingPod("fork-bomb-container-with-low-priority", 12000), }, { evictionPriority: 0, pod: innocentPod(), }, + { + evictionPriority: 1, + pod: pidConsumingPod("fork-bomb-container-with-high-priority", 12000), + }, } specs[1].pod.Spec.PriorityClassName = highPriorityClassName + specs[2].pod.Spec.PriorityClassName = highPriorityClassName runEvictionTest(f, pressureTimeout, expectedNodeCondition, expectedStarvedResource, logPidMetrics, specs) }) }) diff --git a/test/e2e_node/summary_test.go b/test/e2e_node/summary_test.go index 9f58ca69a79..5de99e1b803 100644 --- a/test/e2e_node/summary_test.go +++ b/test/e2e_node/summary_test.go @@ -260,6 +260,9 @@ var _ = framework.KubeDescribe("Summary API [NodeConformance]", func() { "Inodes": bounded(1e4, 1e8), "InodesUsed": bounded(0, 1e8), }), + "ProcessStats": ptrMatchAllFields(gstruct.Fields{ + "ProcessCount": bounded(0, 1e8), + }), }) matchExpectations := ptrMatchAllFields(gstruct.Fields{