kubernetes/pkg/kubelet/eviction/memory_threshold_notifier_test.go
Danielle Lancashire 3630328fd9 eviction: Deflake TestStart
TestStart was previously flaky. In approx 100_000 local runs, it failed
about 70% of the time, and has been mentioned as a flaky unit test in
the past.

This flake was due to a race condition with the logic as written and the
go scheduler. UpdateThreshold calls `notifier.Start(events)` in a new Go
Routine, which is not guarunteed to be called immediately.

This meant that if `m.Start()` was called before `notifier.Start()`, the
test would fail, as the notifier would not have been started before the
4 events were processed and lock released.

Here, we update the test to more closely match the intended application
behaviour, and have events passed to the channel when `Start` is called
on the notifier.

This ensures that -Start gets called and additionally validates
that the correct channel is provided to the notifier.

Stop was never called previously, as it only gets called on a subsequent
call to UpdateThreshold. `AnyTimes()` hid that this did not occur.
2022-02-08 17:03:44 +01:00

273 lines
8.2 KiB
Go

/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package eviction
import (
"fmt"
"strings"
"sync"
"testing"
"time"
gomock "github.com/golang/mock/gomock"
"k8s.io/apimachinery/pkg/api/resource"
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
)
const testCgroupPath = "/sys/fs/cgroups/memory"
func nodeSummary(available, workingSet, usage resource.Quantity, allocatable bool) *statsapi.Summary {
availableBytes := uint64(available.Value())
workingSetBytes := uint64(workingSet.Value())
usageBytes := uint64(usage.Value())
memoryStats := statsapi.MemoryStats{
AvailableBytes: &availableBytes,
WorkingSetBytes: &workingSetBytes,
UsageBytes: &usageBytes,
}
if allocatable {
return &statsapi.Summary{
Node: statsapi.NodeStats{
SystemContainers: []statsapi.ContainerStats{
{
Name: statsapi.SystemContainerPods,
Memory: &memoryStats,
},
},
},
}
}
return &statsapi.Summary{
Node: statsapi.NodeStats{
Memory: &memoryStats,
},
}
}
func newTestMemoryThresholdNotifier(threshold evictionapi.Threshold, factory NotifierFactory, handler func(string)) *memoryThresholdNotifier {
return &memoryThresholdNotifier{
threshold: threshold,
cgroupPath: testCgroupPath,
events: make(chan struct{}),
factory: factory,
handler: handler,
}
}
func TestUpdateThreshold(t *testing.T) {
testCases := []struct {
description string
available resource.Quantity
workingSet resource.Quantity
usage resource.Quantity
evictionThreshold evictionapi.Threshold
expectedThreshold resource.Quantity
updateThresholdErr error
expectErr bool
}{
{
description: "node level threshold",
available: resource.MustParse("3Gi"),
usage: resource.MustParse("2Gi"),
workingSet: resource.MustParse("1Gi"),
evictionThreshold: evictionapi.Threshold{
Signal: evictionapi.SignalMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
},
expectedThreshold: resource.MustParse("4Gi"),
updateThresholdErr: nil,
expectErr: false,
},
{
description: "allocatable threshold",
available: resource.MustParse("4Gi"),
usage: resource.MustParse("3Gi"),
workingSet: resource.MustParse("1Gi"),
evictionThreshold: evictionapi.Threshold{
Signal: evictionapi.SignalAllocatableMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
},
expectedThreshold: resource.MustParse("6Gi"),
updateThresholdErr: nil,
expectErr: false,
},
{
description: "error updating node level threshold",
available: resource.MustParse("3Gi"),
usage: resource.MustParse("2Gi"),
workingSet: resource.MustParse("1Gi"),
evictionThreshold: evictionapi.Threshold{
Signal: evictionapi.SignalMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
},
expectedThreshold: resource.MustParse("4Gi"),
updateThresholdErr: fmt.Errorf("unexpected error"),
expectErr: true,
},
}
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
notifierFactory := NewMockNotifierFactory(mockCtrl)
notifier := NewMockCgroupNotifier(mockCtrl)
m := newTestMemoryThresholdNotifier(tc.evictionThreshold, notifierFactory, nil)
notifierFactory.EXPECT().NewCgroupNotifier(testCgroupPath, memoryUsageAttribute, tc.expectedThreshold.Value()).Return(notifier, tc.updateThresholdErr).Times(1)
var events chan<- struct{} = m.events
notifier.EXPECT().Start(events).Return().AnyTimes()
err := m.UpdateThreshold(nodeSummary(tc.available, tc.workingSet, tc.usage, isAllocatableEvictionThreshold(tc.evictionThreshold)))
if err != nil && !tc.expectErr {
t.Errorf("Unexpected error updating threshold: %v", err)
} else if err == nil && tc.expectErr {
t.Errorf("Expected error updating threshold, but got nil")
}
})
}
}
func TestStart(t *testing.T) {
noResources := resource.MustParse("0")
threshold := evictionapi.Threshold{
Signal: evictionapi.SignalMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: &noResources,
},
}
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
notifierFactory := NewMockNotifierFactory(mockCtrl)
notifier := NewMockCgroupNotifier(mockCtrl)
var wg sync.WaitGroup
wg.Add(4)
m := newTestMemoryThresholdNotifier(threshold, notifierFactory, func(string) {
wg.Done()
})
notifierFactory.EXPECT().NewCgroupNotifier(testCgroupPath, memoryUsageAttribute, int64(0)).Return(notifier, nil).Times(1)
var events chan<- struct{} = m.events
notifier.EXPECT().Start(events).DoAndReturn(func(events chan<- struct{}) {
for i := 0; i < 4; i++ {
events <- struct{}{}
}
})
err := m.UpdateThreshold(nodeSummary(noResources, noResources, noResources, isAllocatableEvictionThreshold(threshold)))
if err != nil {
t.Errorf("Unexpected error updating threshold: %v", err)
}
go m.Start()
wg.Wait()
}
func TestThresholdDescription(t *testing.T) {
testCases := []struct {
description string
evictionThreshold evictionapi.Threshold
expectedSubstrings []string
omittedSubstrings []string
}{
{
description: "hard node level threshold",
evictionThreshold: evictionapi.Threshold{
Signal: evictionapi.SignalMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: quantityMustParse("2Gi"),
},
},
expectedSubstrings: []string{"hard"},
omittedSubstrings: []string{"allocatable", "soft"},
},
{
description: "soft node level threshold",
evictionThreshold: evictionapi.Threshold{
Signal: evictionapi.SignalMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: quantityMustParse("2Gi"),
},
GracePeriod: time.Minute * 2,
},
expectedSubstrings: []string{"soft"},
omittedSubstrings: []string{"allocatable", "hard"},
},
{
description: "hard allocatable threshold",
evictionThreshold: evictionapi.Threshold{
Signal: evictionapi.SignalAllocatableMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: quantityMustParse("2Gi"),
},
GracePeriod: time.Minute * 2,
},
expectedSubstrings: []string{"soft", "allocatable"},
omittedSubstrings: []string{"hard"},
},
{
description: "soft allocatable threshold",
evictionThreshold: evictionapi.Threshold{
Signal: evictionapi.SignalAllocatableMemoryAvailable,
Operator: evictionapi.OpLessThan,
Value: evictionapi.ThresholdValue{
Quantity: quantityMustParse("2Gi"),
},
},
expectedSubstrings: []string{"hard", "allocatable"},
omittedSubstrings: []string{"soft"},
},
}
for _, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
m := &memoryThresholdNotifier{
notifier: &MockCgroupNotifier{},
threshold: tc.evictionThreshold,
cgroupPath: testCgroupPath,
}
desc := m.Description()
for _, expected := range tc.expectedSubstrings {
if !strings.Contains(desc, expected) {
t.Errorf("expected description for notifier with threshold %+v to contain %s, but it did not", tc.evictionThreshold, expected)
}
}
for _, omitted := range tc.omittedSubstrings {
if strings.Contains(desc, omitted) {
t.Errorf("expected description for notifier with threshold %+v NOT to contain %s, but it did", tc.evictionThreshold, omitted)
}
}
})
}
}