github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-26 13:07:07 +00:00
Code Issues Projects Releases Wiki Activity

Allow cAdvisor to collect fs stats before building resource ranking funcs

Browse Source
This commit is contained in:
derekwaynecarr 2016-07-28 13:02:01 -04:00
parent 0de1e62b30
commit d37710f87b
2 changed files with 40 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/kubelet/eviction/eviction_manager.go
View File

@ -109,15 +109,8 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
// Start starts the control loop to observe and response to low compute resources. // Start starts the control loop to observe and response to low compute resources.
func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, monitoringInterval time.Duration) error { func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, monitoringInterval time.Duration) error {
// build the ranking functions now that we can know if the imagefs is dedicated or not.
hasDedicatedImageFs, err := diskInfoProvider.HasDedicatedImageFs()
if err != nil {
return err
}
m.resourceToRankFunc = buildResourceToRankFunc(hasDedicatedImageFs)
// start the eviction manager monitoring // start the eviction manager monitoring
go wait.Until(func() { m.synchronize(podFunc) }, monitoringInterval, wait.NeverStop) go wait.Until(func() { m.synchronize(diskInfoProvider, podFunc) }, monitoringInterval, wait.NeverStop)
return nil return nil
} }
@ -136,13 +129,24 @@ func (m *managerImpl) IsUnderDiskPressure() bool {
} }
// synchronize is the main control loop that enforces eviction thresholds. // synchronize is the main control loop that enforces eviction thresholds.
func (m *managerImpl) synchronize(podFunc ActivePodsFunc) { func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc) {
// if we have nothing to do, just return // if we have nothing to do, just return
thresholds := m.config.Thresholds thresholds := m.config.Thresholds
if len(thresholds) == 0 { if len(thresholds) == 0 {
return return
} }
// build the ranking functions (if not yet known)
// TODO: have a function in cadvisor that lets us know if global housekeeping has completed
if len(m.resourceToRankFunc) == 0 {
// this may error if cadvisor has yet to complete housekeeping, so we will just try again in next pass.
hasDedicatedImageFs, err := diskInfoProvider.HasDedicatedImageFs()
if err != nil {
return
}
m.resourceToRankFunc = buildResourceToRankFunc(hasDedicatedImageFs)
}
// make observations and get a function to derive pod usage stats relative to those observations. // make observations and get a function to derive pod usage stats relative to those observations.
observations, statsFunc, err := makeSignalObservations(m.summaryProvider) observations, statsFunc, err := makeSignalObservations(m.summaryProvider)
if err != nil { if err != nil {

44
pkg/kubelet/eviction/eviction_manager_test.go
View File

@ -44,6 +44,16 @@ func (m *mockPodKiller) killPodNow(pod *api.Pod, status api.PodStatus, gracePeri
return nil return nil
} }
// mockDiskInfoProvider is used to simulate testing.
type mockDiskInfoProvider struct {
dedicatedImageFs bool
}
// HasDedicatedImageFs returns the mocked value
func (m *mockDiskInfoProvider) HasDedicatedImageFs() (bool, error) {
return m.dedicatedImageFs, nil
}
// TestMemoryPressure // TestMemoryPressure
func TestMemoryPressure(t *testing.T) { func TestMemoryPressure(t *testing.T) {
podMaker := func(name string, requests api.ResourceList, limits api.ResourceList, memoryWorkingSet string) (*api.Pod, statsapi.PodStats) { podMaker := func(name string, requests api.ResourceList, limits api.ResourceList, memoryWorkingSet string) (*api.Pod, statsapi.PodStats) {
@ -95,6 +105,7 @@ func TestMemoryPressure(t *testing.T) {
fakeClock := clock.NewFakeClock(time.Now()) fakeClock := clock.NewFakeClock(time.Now())
podKiller := &mockPodKiller{} podKiller := &mockPodKiller{}
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
nodeRef := &api.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} nodeRef := &api.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
config := Config{ config := Config{
@ -124,7 +135,6 @@ func TestMemoryPressure(t *testing.T) {
nodeRef: nodeRef, nodeRef: nodeRef,
nodeConditionsLastObservedAt: nodeConditionsObservedAt{}, nodeConditionsLastObservedAt: nodeConditionsObservedAt{},
thresholdsFirstObservedAt: thresholdsObservedAt{}, thresholdsFirstObservedAt: thresholdsObservedAt{},
resourceToRankFunc: buildResourceToRankFunc(false),
} }
// create a best effort pod to test admission // create a best effort pod to test admission
@ -132,7 +142,7 @@ func TestMemoryPressure(t *testing.T) {
burstablePodToAdmit, _ := podMaker("burst-admit", newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi"), "0Gi") burstablePodToAdmit, _ := podMaker("burst-admit", newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi"), "0Gi")
// synchronize // synchronize
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should not have memory pressure // we should not have memory pressure
if manager.IsUnderMemoryPressure() { if manager.IsUnderMemoryPressure() {
@ -150,7 +160,7 @@ func TestMemoryPressure(t *testing.T) {
// induce soft threshold // induce soft threshold
fakeClock.Step(1 * time.Minute) fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("1500Mi", podStats) summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should have memory pressure // we should have memory pressure
if !manager.IsUnderMemoryPressure() { if !manager.IsUnderMemoryPressure() {
@ -165,7 +175,7 @@ func TestMemoryPressure(t *testing.T) {
// step forward in time pass the grace period // step forward in time pass the grace period
fakeClock.Step(3 * time.Minute) fakeClock.Step(3 * time.Minute)
summaryProvider.result = summaryStatsMaker("1500Mi", podStats) summaryProvider.result = summaryStatsMaker("1500Mi", podStats)
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should have memory pressure // we should have memory pressure
if !manager.IsUnderMemoryPressure() { if !manager.IsUnderMemoryPressure() {
@ -190,7 +200,7 @@ func TestMemoryPressure(t *testing.T) {
// remove memory pressure // remove memory pressure
fakeClock.Step(20 * time.Minute) fakeClock.Step(20 * time.Minute)
summaryProvider.result = summaryStatsMaker("3Gi", podStats) summaryProvider.result = summaryStatsMaker("3Gi", podStats)
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should not have memory pressure // we should not have memory pressure
if manager.IsUnderMemoryPressure() { if manager.IsUnderMemoryPressure() {
@ -200,7 +210,7 @@ func TestMemoryPressure(t *testing.T) {
// induce memory pressure! // induce memory pressure!
fakeClock.Step(1 * time.Minute) fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("500Mi", podStats) summaryProvider.result = summaryStatsMaker("500Mi", podStats)
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should have memory pressure // we should have memory pressure
if !manager.IsUnderMemoryPressure() { if !manager.IsUnderMemoryPressure() {
@ -228,7 +238,7 @@ func TestMemoryPressure(t *testing.T) {
fakeClock.Step(1 * time.Minute) fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("2Gi", podStats) summaryProvider.result = summaryStatsMaker("2Gi", podStats)
podKiller.pod = nil // reset state podKiller.pod = nil // reset state
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should have memory pressure (because transition period not yet met) // we should have memory pressure (because transition period not yet met)
if !manager.IsUnderMemoryPressure() { if !manager.IsUnderMemoryPressure() {
@ -252,7 +262,7 @@ func TestMemoryPressure(t *testing.T) {
fakeClock.Step(5 * time.Minute) fakeClock.Step(5 * time.Minute)
summaryProvider.result = summaryStatsMaker("2Gi", podStats) summaryProvider.result = summaryStatsMaker("2Gi", podStats)
podKiller.pod = nil // reset state podKiller.pod = nil // reset state
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should not have memory pressure (because transition period met) // we should not have memory pressure (because transition period met)
if manager.IsUnderMemoryPressure() { if manager.IsUnderMemoryPressure() {
@ -338,8 +348,9 @@ func TestDiskPressureNodeFs(t *testing.T) {
return pods return pods
} }
fakeClock := util.NewFakeClock(time.Now()) fakeClock := clock.NewFakeClock(time.Now())
podKiller := &mockPodKiller{} podKiller := &mockPodKiller{}
diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false}
nodeRef := &api.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} nodeRef := &api.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
config := Config{ config := Config{
@ -369,14 +380,13 @@ func TestDiskPressureNodeFs(t *testing.T) {
nodeRef: nodeRef, nodeRef: nodeRef,
nodeConditionsLastObservedAt: nodeConditionsObservedAt{}, nodeConditionsLastObservedAt: nodeConditionsObservedAt{},
thresholdsFirstObservedAt: thresholdsObservedAt{}, thresholdsFirstObservedAt: thresholdsObservedAt{},
resourceToRankFunc: buildResourceToRankFunc(false),
} }
// create a best effort pod to test admission // create a best effort pod to test admission
podToAdmit, _ := podMaker("pod-to-admit", newResourceList("", ""), newResourceList("", ""), "0Gi", "0Gi", "0Gi") podToAdmit, _ := podMaker("pod-to-admit", newResourceList("", ""), newResourceList("", ""), "0Gi", "0Gi", "0Gi")
// synchronize // synchronize
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should not have disk pressure // we should not have disk pressure
if manager.IsUnderDiskPressure() { if manager.IsUnderDiskPressure() {
@ -391,7 +401,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
// induce soft threshold // induce soft threshold
fakeClock.Step(1 * time.Minute) fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("1.5Gi", "200Gi", podStats) summaryProvider.result = summaryStatsMaker("1.5Gi", "200Gi", podStats)
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should have disk pressure // we should have disk pressure
if !manager.IsUnderDiskPressure() { if !manager.IsUnderDiskPressure() {
@ -406,7 +416,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
// step forward in time pass the grace period // step forward in time pass the grace period
fakeClock.Step(3 * time.Minute) fakeClock.Step(3 * time.Minute)
summaryProvider.result = summaryStatsMaker("1.5Gi", "200Gi", podStats) summaryProvider.result = summaryStatsMaker("1.5Gi", "200Gi", podStats)
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should have disk pressure // we should have disk pressure
if !manager.IsUnderDiskPressure() { if !manager.IsUnderDiskPressure() {
@ -431,7 +441,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
// remove disk pressure // remove disk pressure
fakeClock.Step(20 * time.Minute) fakeClock.Step(20 * time.Minute)
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats) summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should not have disk pressure // we should not have disk pressure
if manager.IsUnderDiskPressure() { if manager.IsUnderDiskPressure() {
@ -441,7 +451,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
// induce disk pressure! // induce disk pressure!
fakeClock.Step(1 * time.Minute) fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("500Mi", "200Gi", podStats) summaryProvider.result = summaryStatsMaker("500Mi", "200Gi", podStats)
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should have disk pressure // we should have disk pressure
if !manager.IsUnderDiskPressure() { if !manager.IsUnderDiskPressure() {
@ -466,7 +476,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
fakeClock.Step(1 * time.Minute) fakeClock.Step(1 * time.Minute)
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats) summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
podKiller.pod = nil // reset state podKiller.pod = nil // reset state
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should have disk pressure (because transition period not yet met) // we should have disk pressure (because transition period not yet met)
if !manager.IsUnderDiskPressure() { if !manager.IsUnderDiskPressure() {
@ -487,7 +497,7 @@ func TestDiskPressureNodeFs(t *testing.T) {
fakeClock.Step(5 * time.Minute) fakeClock.Step(5 * time.Minute)
summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats) summaryProvider.result = summaryStatsMaker("16Gi", "200Gi", podStats)
podKiller.pod = nil // reset state podKiller.pod = nil // reset state
manager.synchronize(activePodsFunc) manager.synchronize(diskInfoProvider, activePodsFunc)
// we should not have disk pressure (because transition period met) // we should not have disk pressure (because transition period met)
if manager.IsUnderDiskPressure() { if manager.IsUnderDiskPressure() {