github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 11:21:47 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #30400 from ronnielai/percent1

Browse Source 
Automatic merge from submit-queue

Support percentage threshold for eviction
This commit is contained in:
Kubernetes Submit Queue 2016-08-16 23:03:21 -07:00 committed by GitHub
commit 1c9332ab51
5 changed files with 489 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/kubelet/eviction/eviction_manager.go
View File

@ -285,7 +285,7 @@ func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim api.ResourceNa
glog.Errorf("eviction manager: unable to find value associated with signal %v", signal)
continue
}
value.Add(*reclaimed)
value.available.Add(*reclaimed)
// evaluate all current thresholds to see if with adjusted observations, we think we have met min reclaim goals
if len(thresholdsMet(m.thresholdsMet, observations, true)) == 0 {

56
pkg/kubelet/eviction/eviction_manager_test.go
View File

@ -79,10 +79,12 @@ func TestMemoryPressure(t *testing.T) {
summaryStatsMaker := func(nodeAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary {
val := resource.MustParse(nodeAvailableBytes)
availableBytes := uint64(val.Value())
WorkingSetBytes := uint64(val.Value())
result := &statsapi.Summary{
Node: statsapi.NodeStats{
Memory: &statsapi.MemoryStats{
AvailableBytes: &availableBytes,
AvailableBytes: &availableBytes,
WorkingSetBytes: &WorkingSetBytes,
},
},
Pods: []statsapi.PodStats{},
@ -129,12 +131,16 @@ func TestMemoryPressure(t *testing.T) {
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("1Gi"),
Value: ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
},
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("2Gi"),
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("2Gi"),
},
GracePeriod: time.Minute * 2,
},
},
@ -317,16 +323,20 @@ func TestDiskPressureNodeFs(t *testing.T) {
summaryStatsMaker := func(rootFsAvailableBytes, imageFsAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary {
rootFsVal := resource.MustParse(rootFsAvailableBytes)
rootFsBytes := uint64(rootFsVal.Value())
rootFsCapacityBytes := uint64(rootFsVal.Value() * 2)
imageFsVal := resource.MustParse(imageFsAvailableBytes)
imageFsBytes := uint64(imageFsVal.Value())
imageFsCapacityBytes := uint64(imageFsVal.Value() * 2)
result := &statsapi.Summary{
Node: statsapi.NodeStats{
Fs: &statsapi.FsStats{
AvailableBytes: &rootFsBytes,
CapacityBytes: &rootFsCapacityBytes,
},
Runtime: &statsapi.RuntimeStats{
ImageFs: &statsapi.FsStats{
AvailableBytes: &imageFsBytes,
CapacityBytes: &imageFsCapacityBytes,
},
},
},
@ -376,12 +386,16 @@ func TestDiskPressureNodeFs(t *testing.T) {
{
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: quantityMustParse("1Gi"),
Value: ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
},
{
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: quantityMustParse("2Gi"),
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("2Gi"),
},
GracePeriod: time.Minute * 2,
},
},
@ -544,10 +558,12 @@ func TestMinReclaim(t *testing.T) {
summaryStatsMaker := func(nodeAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary {
val := resource.MustParse(nodeAvailableBytes)
availableBytes := uint64(val.Value())
WorkingSetBytes := uint64(val.Value())
result := &statsapi.Summary{
Node: statsapi.NodeStats{
Memory: &statsapi.MemoryStats{
AvailableBytes: &availableBytes,
AvailableBytes: &availableBytes,
WorkingSetBytes: &WorkingSetBytes,
},
},
Pods: []statsapi.PodStats{},
@ -592,9 +608,11 @@ func TestMinReclaim(t *testing.T) {
PressureTransitionPeriod: time.Minute * 5,
Thresholds: []Threshold{
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("1Gi"),
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
MinReclaim: quantityMustParse("500Mi"),
},
},
@ -703,16 +721,20 @@ func TestNodeReclaimFuncs(t *testing.T) {
summaryStatsMaker := func(rootFsAvailableBytes, imageFsAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary {
rootFsVal := resource.MustParse(rootFsAvailableBytes)
rootFsBytes := uint64(rootFsVal.Value())
rootFsCapacityBytes := uint64(rootFsVal.Value() * 2)
imageFsVal := resource.MustParse(imageFsAvailableBytes)
imageFsBytes := uint64(imageFsVal.Value())
imageFsCapacityBytes := uint64(imageFsVal.Value() * 2)
result := &statsapi.Summary{
Node: statsapi.NodeStats{
Fs: &statsapi.FsStats{
AvailableBytes: &rootFsBytes,
CapacityBytes: &rootFsCapacityBytes,
},
Runtime: &statsapi.RuntimeStats{
ImageFs: &statsapi.FsStats{
AvailableBytes: &imageFsBytes,
CapacityBytes: &imageFsCapacityBytes,
},
},
},
@ -761,9 +783,11 @@ func TestNodeReclaimFuncs(t *testing.T) {
PressureTransitionPeriod: time.Minute * 5,
Thresholds: []Threshold{
{
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: quantityMustParse("1Gi"),
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
MinReclaim: quantityMustParse("500Mi"),
},
},

110
pkg/kubelet/eviction/helpers.go
View File

@ -19,11 +19,11 @@ package eviction
import (
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/resource"
statsapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
@ -169,18 +169,47 @@ func parseThresholdStatement(statement string) (Threshold, error) {
return Threshold{}, fmt.Errorf(unsupportedEvictionSignal, signal)
}
quantity, err := resource.ParseQuantity(parts[1])
quantityValue := parts[1]
if strings.HasSuffix(quantityValue, "%") {
percentage, err := parsePercentage(quantityValue)
if err != nil {
return Threshold{}, err
}
if percentage <= 0 {
return Threshold{}, fmt.Errorf("eviction percentage threshold %v must be positive: %s", signal, quantityValue)
}
return Threshold{
Signal: signal,
Operator: operator,
Value: ThresholdValue{
Percentage: percentage,
},
}, nil
} else {
quantity, err := resource.ParseQuantity(quantityValue)
if err != nil {
return Threshold{}, err
}
if quantity.Sign() < 0 || quantity.IsZero() {
return Threshold{}, fmt.Errorf("eviction threshold %v must be positive: %s", signal, &quantity)
}
return Threshold{
Signal: signal,
Operator: operator,
Value: ThresholdValue{
Quantity: &quantity,
},
}, nil
}
}
// parsePercentage parses a string representing a percentage value
func parsePercentage(input string) (float32, error) {
value, err := strconv.ParseFloat(strings.TrimRight(input, "%"), 32)
if err != nil {
return Threshold{}, err
return 0, err
}
if quantity.Sign() < 0 {
return Threshold{}, fmt.Errorf("eviction threshold %v cannot be negative: %s", signal, &quantity)
}
return Threshold{
Signal: signal,
Operator: operator,
Value: &quantity,
}, nil
return float32(value) / 100, nil
}
// parseGracePeriods parses the grace period statements
@ -329,7 +358,15 @@ func podMemoryUsage(podStats statsapi.PodStats) (api.ResourceList, error) {
// formatThreshold formats a threshold for logging.
func formatThreshold(threshold Threshold) string {
return fmt.Sprintf("threshold(signal=%v, operator=%v, value=%v, gracePeriod=%v)", threshold.Signal, threshold.Value.String(), threshold.Operator, threshold.GracePeriod)
return fmt.Sprintf("threshold(signal=%v, operator=%v, value=%v, gracePeriod=%v)", threshold.Signal, formatThresholdValue(threshold.Value), threshold.Operator, threshold.GracePeriod)
}
// formatThresholdValue formats a thresholdValue for logging.
func formatThresholdValue(value ThresholdValue) string {
if value.Quantity != nil {
return value.Quantity.String()
}
return fmt.Sprintf("%f%%", value.Percentage*float32(100))
}
// cachedStatsFunc returns a statsFunc based on the provided pod stats.
@ -532,15 +569,24 @@ func makeSignalObservations(summaryProvider stats.SummaryProvider) (signalObserv
// build an evaluation context for current eviction signals
result := signalObservations{}
if memory := summary.Node.Memory; memory != nil && memory.AvailableBytes != nil {
result[SignalMemoryAvailable] = resource.NewQuantity(int64(*memory.AvailableBytes), resource.BinarySI)
if memory := summary.Node.Memory; memory != nil && memory.AvailableBytes != nil && memory.WorkingSetBytes != nil {
result[SignalMemoryAvailable] = signalObservation{
available: resource.NewQuantity(int64(*memory.AvailableBytes), resource.BinarySI),
capacity: resource.NewQuantity(int64(*memory.AvailableBytes+*memory.WorkingSetBytes), resource.BinarySI),
}
}
if nodeFs := summary.Node.Fs; nodeFs != nil && nodeFs.AvailableBytes != nil {
result[SignalNodeFsAvailable] = resource.NewQuantity(int64(*nodeFs.AvailableBytes), resource.BinarySI)
if nodeFs := summary.Node.Fs; nodeFs != nil && nodeFs.AvailableBytes != nil && nodeFs.CapacityBytes != nil {
result[SignalNodeFsAvailable] = signalObservation{
available: resource.NewQuantity(int64(*nodeFs.AvailableBytes), resource.BinarySI),
capacity: resource.NewQuantity(int64(*nodeFs.CapacityBytes), resource.BinarySI),
}
}
if summary.Node.Runtime != nil {
if imageFs := summary.Node.Runtime.ImageFs; imageFs != nil && imageFs.AvailableBytes != nil {
result[SignalImageFsAvailable] = resource.NewQuantity(int64(*imageFs.AvailableBytes), resource.BinarySI)
if imageFs := summary.Node.Runtime.ImageFs; imageFs != nil && imageFs.AvailableBytes != nil && imageFs.CapacityBytes != nil {
result[SignalImageFsAvailable] = signalObservation{
available: resource.NewQuantity(int64(*imageFs.AvailableBytes), resource.BinarySI),
capacity: resource.NewQuantity(int64(*imageFs.CapacityBytes), resource.BinarySI),
}
}
}
return result, statsFunc, nil
@ -558,12 +604,12 @@ func thresholdsMet(thresholds []Threshold, observations signalObservations, enfo
}
// determine if we have met the specified threshold
thresholdMet := false
quantity := threshold.Value.Copy()
quantity := getThresholdQuantity(threshold.Value, observed.capacity)
// if enforceMinReclaim is specified, we compare relative to value - minreclaim
if enforceMinReclaim && threshold.MinReclaim != nil {
quantity.Add(*threshold.MinReclaim)
}
thresholdResult := quantity.Cmp(*observed)
thresholdResult := quantity.Cmp(*observed.available)
switch threshold.Operator {
case OpLessThan:
thresholdMet = thresholdResult > 0
@ -575,6 +621,14 @@ func thresholdsMet(thresholds []Threshold, observations signalObservations, enfo
return results
}
// getThresholdQuantity returns the expected quantity value for a thresholdValue
func getThresholdQuantity(value ThresholdValue, capacity *resource.Quantity) *resource.Quantity {
if value.Quantity != nil {
return value.Quantity.Copy()
}
return resource.NewQuantity(int64(float64(capacity.Value())*float64(value.Percentage)), resource.BinarySI)
}
// thresholdsFirstObservedAt merges the input set of thresholds with the previous observation to determine when active set of thresholds were initially met.
func thresholdsFirstObservedAt(thresholds []Threshold, lastObservedAt thresholdsObservedAt, now time.Time) thresholdsObservedAt {
results := thresholdsObservedAt{}
@ -678,13 +732,27 @@ func mergeThresholds(inputsA []Threshold, inputsB []Threshold) []Threshold {
// hasThreshold returns true if the threshold is in the input list
func hasThreshold(inputs []Threshold, item Threshold) bool {
for _, input := range inputs {
if input.GracePeriod == item.GracePeriod && input.Operator == item.Operator && input.Signal == item.Signal && input.Value.Cmp(*item.Value) == 0 {
if input.GracePeriod == item.GracePeriod && input.Operator == item.Operator && input.Signal == item.Signal && compareThresholdValue(input.Value, item.Value) {
return true
}
}
return false
}
// compareThresholdValue returns true if the two thresholdValue objects are logically the same
func compareThresholdValue(a ThresholdValue, b ThresholdValue) bool {
if a.Quantity != nil {
if b.Quantity == nil {
return false
}
return a.Quantity.Cmp(*b.Quantity) == 0
}
if b.Quantity != nil {
return false
}
return a.Percentage == b.Percentage
}
// getStarvedResources returns the set of resources that are starved based on thresholds met.
func getStarvedResources(thresholds []Threshold) []api.ResourceName {
results := []api.ResourceName{}

378
pkg/kubelet/eviction/helpers_test.go
View File

@ -61,15 +61,45 @@ func TestParseThresholdConfig(t *testing.T) {
expectErr: false,
expectThresholds: []Threshold{
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("150Mi"),
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("150Mi"),
},
MinReclaim: quantityMustParse("0"),
},
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("300Mi"),
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("300Mi"),
},
GracePeriod: gracePeriod,
MinReclaim: quantityMustParse("0"),
},
},
},
"all flag values in percentages": {
evictionHard: "memory.available<10%",
evictionSoft: "memory.available<30%",
evictionSoftGracePeriod: "memory.available=30s",
evictionMinReclaim: "memory.available=0",
expectErr: false,
expectThresholds: []Threshold{
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Percentage: 0.1,
},
MinReclaim: quantityMustParse("0"),
},
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Percentage: 0.3,
},
GracePeriod: gracePeriod,
MinReclaim: quantityMustParse("0"),
},
@ -83,28 +113,79 @@ func TestParseThresholdConfig(t *testing.T) {
expectErr: false,
expectThresholds: []Threshold{
{
Signal: SignalImageFsAvailable,
Operator: OpLessThan,
Value: quantityMustParse("150Mi"),
Signal: SignalImageFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("150Mi"),
},
MinReclaim: quantityMustParse("2Gi"),
},
{
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: quantityMustParse("100Mi"),
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("100Mi"),
},
MinReclaim: quantityMustParse("1Gi"),
},
{
Signal: SignalImageFsAvailable,
Operator: OpLessThan,
Value: quantityMustParse("300Mi"),
Signal: SignalImageFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("300Mi"),
},
GracePeriod: gracePeriod,
MinReclaim: quantityMustParse("2Gi"),
},
{
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: quantityMustParse("200Mi"),
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("200Mi"),
},
GracePeriod: gracePeriod,
MinReclaim: quantityMustParse("1Gi"),
},
},
},
"disk flag values in percentages": {
evictionHard: "imagefs.available<15%,nodefs.available<10.5%",
evictionSoft: "imagefs.available<30%,nodefs.available<20.5%",
evictionSoftGracePeriod: "imagefs.available=30s,nodefs.available=30s",
evictionMinReclaim: "imagefs.available=2Gi,nodefs.available=1Gi",
expectErr: false,
expectThresholds: []Threshold{
{
Signal: SignalImageFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Percentage: 0.15,
},
MinReclaim: quantityMustParse("2Gi"),
},
{
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Percentage: 0.105,
},
MinReclaim: quantityMustParse("1Gi"),
},
{
Signal: SignalImageFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Percentage: 0.3,
},
GracePeriod: gracePeriod,
MinReclaim: quantityMustParse("2Gi"),
},
{
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Percentage: 0.205,
},
GracePeriod: gracePeriod,
MinReclaim: quantityMustParse("1Gi"),
},
@ -126,6 +207,14 @@ func TestParseThresholdConfig(t *testing.T) {
expectErr: true,
expectThresholds: []Threshold{},
},
"hard-signal-negative-percentage": {
evictionHard: "memory.available<-15%",
evictionSoft: "",
evictionSoftGracePeriod: "",
evictionMinReclaim: "",
expectErr: true,
expectThresholds: []Threshold{},
},
"soft-signal-negative": {
evictionHard: "",
evictionSoft: "memory.available<-150Mi",
@ -227,8 +316,8 @@ func thresholdEqual(a Threshold, b Threshold) bool {
return a.GracePeriod == b.GracePeriod &&
a.Operator == b.Operator &&
a.Signal == b.Signal &&
a.Value.Cmp(*b.Value) == 0 &&
a.MinReclaim.Cmp(*b.MinReclaim) == 0
a.MinReclaim.Cmp(*b.MinReclaim) == 0 &&
compareThresholdValue(a.Value, b.Value)
}
// TestOrderedByQoS ensures we order BestEffort < Burstable < Guaranteed
@ -514,20 +603,26 @@ func TestMakeSignalObservations(t *testing.T) {
return pod
}
nodeAvailableBytes := uint64(1024 * 1024 * 1024)
nodeWorkingSetBytes := uint64(1024 * 1024 * 1024)
imageFsAvailableBytes := uint64(1024 * 1024)
imageFsCapacityBytes := uint64(1024 * 1024 * 2)
nodeFsAvailableBytes := uint64(1024)
nodeFsCapacityBytes := uint64(1024 * 2)
fakeStats := &statsapi.Summary{
Node: statsapi.NodeStats{
Memory: &statsapi.MemoryStats{
AvailableBytes: &nodeAvailableBytes,
AvailableBytes: &nodeAvailableBytes,
WorkingSetBytes: &nodeWorkingSetBytes,
},
Runtime: &statsapi.RuntimeStats{
ImageFs: &statsapi.FsStats{
AvailableBytes: &imageFsAvailableBytes,
CapacityBytes: &imageFsCapacityBytes,
},
},
Fs: &statsapi.FsStats{
AvailableBytes: &nodeFsAvailableBytes,
CapacityBytes: &nodeFsCapacityBytes,
},
},
Pods: []statsapi.PodStats{},
@ -545,6 +640,7 @@ func TestMakeSignalObservations(t *testing.T) {
fakeStats.Pods = append(fakeStats.Pods, newPodStats(pod, containerWorkingSetBytes))
}
actualObservations, statsFunc, err := makeSignalObservations(provider)
if err != nil {
t.Errorf("Unexpected err: %v", err)
}
@ -552,22 +648,31 @@ func TestMakeSignalObservations(t *testing.T) {
if !found {
t.Errorf("Expected available memory observation: %v", err)
}
if expectedBytes := int64(nodeAvailableBytes); memQuantity.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, memQuantity.Value())
if expectedBytes := int64(nodeAvailableBytes); memQuantity.available.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, memQuantity.available.Value())
}
if expectedBytes := int64(nodeWorkingSetBytes + nodeAvailableBytes); memQuantity.capacity.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, memQuantity.capacity.Value())
}
nodeFsQuantity, found := actualObservations[SignalNodeFsAvailable]
if !found {
t.Errorf("Expected available nodefs observation: %v", err)
}
if expectedBytes := int64(nodeFsAvailableBytes); nodeFsQuantity.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, nodeFsQuantity.Value())
if expectedBytes := int64(nodeFsAvailableBytes); nodeFsQuantity.available.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, nodeFsQuantity.available.Value())
}
if expectedBytes := int64(nodeFsCapacityBytes); nodeFsQuantity.capacity.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, nodeFsQuantity.capacity.Value())
}
imageFsQuantity, found := actualObservations[SignalImageFsAvailable]
if !found {
t.Errorf("Expected available imagefs observation: %v", err)
}
if expectedBytes := int64(imageFsAvailableBytes); imageFsQuantity.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, imageFsQuantity.Value())
if expectedBytes := int64(imageFsAvailableBytes); imageFsQuantity.available.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, imageFsQuantity.available.Value())
}
if expectedBytes := int64(imageFsCapacityBytes); imageFsQuantity.capacity.Value() != expectedBytes {
t.Errorf("Expected %v, actual: %v", expectedBytes, imageFsQuantity.capacity.Value())
}
for _, pod := range pods {
podStats, found := statsFunc(pod)
@ -585,9 +690,11 @@ func TestMakeSignalObservations(t *testing.T) {
func TestThresholdsMet(t *testing.T) {
hardThreshold := Threshold{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("1Gi"),
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
MinReclaim: quantityMustParse("500Mi"),
}
testCases := map[string]struct {
@ -602,11 +709,13 @@ func TestThresholdsMet(t *testing.T) {
observations: signalObservations{},
result: []Threshold{},
},
"threshold-met": {
"threshold-met-memory": {
enforceMinReclaim: false,
thresholds: []Threshold{hardThreshold},
observations: signalObservations{
SignalMemoryAvailable: quantityMustParse("500Mi"),
SignalMemoryAvailable: signalObservation{
available: quantityMustParse("500Mi"),
},
},
result: []Threshold{hardThreshold},
},
@ -614,7 +723,9 @@ func TestThresholdsMet(t *testing.T) {
enforceMinReclaim: false,
thresholds: []Threshold{hardThreshold},
observations: signalObservations{
SignalMemoryAvailable: quantityMustParse("2Gi"),
SignalMemoryAvailable: signalObservation{
available: quantityMustParse("2Gi"),
},
},
result: []Threshold{},
},
@ -622,7 +733,9 @@ func TestThresholdsMet(t *testing.T) {
enforceMinReclaim: true,
thresholds: []Threshold{hardThreshold},
observations: signalObservations{
SignalMemoryAvailable: quantityMustParse("1.05Gi"),
SignalMemoryAvailable: signalObservation{
available: quantityMustParse("1.05Gi"),
},
},
result: []Threshold{hardThreshold},
},
@ -630,7 +743,9 @@ func TestThresholdsMet(t *testing.T) {
enforceMinReclaim: true,
thresholds: []Threshold{hardThreshold},
observations: signalObservations{
SignalMemoryAvailable: quantityMustParse("2Gi"),
SignalMemoryAvailable: signalObservation{
available: quantityMustParse("2Gi"),
},
},
result: []Threshold{},
},
@ -643,11 +758,101 @@ func TestThresholdsMet(t *testing.T) {
}
}
func TestPercentageThresholdsMet(t *testing.T) {
specifiecThresholds := []Threshold{
{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Percentage: 0.2,
},
},
{
Signal: SignalNodeFsAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Percentage: 0.3,
},
},
}
testCases := map[string]struct {
thresholds []Threshold
observations signalObservations
result []Threshold
}{
"BothMet": {
thresholds: specifiecThresholds,
observations: signalObservations{
SignalMemoryAvailable: signalObservation{
available: quantityMustParse("100Mi"),
capacity: quantityMustParse("1000Mi"),
},
SignalNodeFsAvailable: signalObservation{
available: quantityMustParse("100Gi"),
capacity: quantityMustParse("1000Gi"),
},
},
result: specifiecThresholds,
},
"NoneMet": {
thresholds: specifiecThresholds,
observations: signalObservations{
SignalMemoryAvailable: signalObservation{
available: quantityMustParse("300Mi"),
capacity: quantityMustParse("1000Mi"),
},
SignalNodeFsAvailable: signalObservation{
available: quantityMustParse("400Gi"),
capacity: quantityMustParse("1000Gi"),
},
},
result: []Threshold{},
},
"DiskMet": {
thresholds: specifiecThresholds,
observations: signalObservations{
SignalMemoryAvailable: signalObservation{
available: quantityMustParse("300Mi"),
capacity: quantityMustParse("1000Mi"),
},
SignalNodeFsAvailable: signalObservation{
available: quantityMustParse("100Gi"),
capacity: quantityMustParse("1000Gi"),
},
},
result: []Threshold{specifiecThresholds[1]},
},
"MemoryMet": {
thresholds: specifiecThresholds,
observations: signalObservations{
SignalMemoryAvailable: signalObservation{
available: quantityMustParse("100Mi"),
capacity: quantityMustParse("1000Mi"),
},
SignalNodeFsAvailable: signalObservation{
available: quantityMustParse("400Gi"),
capacity: quantityMustParse("1000Gi"),
},
},
result: []Threshold{specifiecThresholds[0]},
},
}
for testName, testCase := range testCases {
actual := thresholdsMet(testCase.thresholds, testCase.observations, false)
if !thresholdList(actual).Equal(thresholdList(testCase.result)) {
t.Errorf("Test case: %s, expected: %v, actual: %v", testName, testCase.result, actual)
}
}
}
func TestThresholdsFirstObservedAt(t *testing.T) {
hardThreshold := Threshold{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("1Gi"),
Value: ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
}
now := unversioned.Now()
oldTime := unversioned.NewTime(now.Time.Add(-1 * time.Minute))
@ -695,12 +900,16 @@ func TestThresholdsMetGracePeriod(t *testing.T) {
hardThreshold := Threshold{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("1Gi"),
Value: ThresholdValue{
Quantity: quantityMustParse("1Gi"),
},
}
softThreshold := Threshold{
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: quantityMustParse("2Gi"),
Signal: SignalMemoryAvailable,
Operator: OpLessThan,
Value: ThresholdValue{
Quantity: quantityMustParse("2Gi"),
},
GracePeriod: 1 * time.Minute,
}
oldTime := unversioned.NewTime(now.Time.Add(-2 * time.Minute))
@ -906,6 +1115,95 @@ func TestGetStarvedResources(t *testing.T) {
}
}
func testParsePercentage(t *testing.T) {
testCases := map[string]struct {
hasError bool
value float32
}{
"blah": {
hasError: true,
},
"25.5%": {
value: 0.255,
},
"foo%": {
hasError: true,
},
"12%345": {
hasError: true,
},
}
for input, expected := range testCases {
value, err := parsePercentage(input)
if (err != nil) != expected.hasError {
t.Errorf("Test case: %s, expected: %v, actual: %v", input, expected.hasError, err != nil)
}
if value != expected.value {
t.Errorf("Test case: %s, expected: %v, actual: %v", input, expected.value, value)
}
}
}
func testCompareThresholdValue(t *testing.T) {
testCases := []struct {
a, b ThresholdValue
equal bool
}{
{
a: ThresholdValue{
Quantity: resource.NewQuantity(123, resource.BinarySI),
},
b: ThresholdValue{
Quantity: resource.NewQuantity(123, resource.BinarySI),
},
equal: true,
},
{
a: ThresholdValue{
Quantity: resource.NewQuantity(123, resource.BinarySI),
},
b: ThresholdValue{
Quantity: resource.NewQuantity(456, resource.BinarySI),
},
equal: false,
},
{
a: ThresholdValue{
Quantity: resource.NewQuantity(123, resource.BinarySI),
},
b: ThresholdValue{
Percentage: 0.1,
},
equal: false,
},
{
a: ThresholdValue{
Percentage: 0.1,
},
b: ThresholdValue{
Percentage: 0.1,
},
equal: true,
},
{
a: ThresholdValue{
Percentage: 0.2,
},
b: ThresholdValue{
Percentage: 0.1,
},
equal: false,
},
}
for i, testCase := range testCases {
if compareThresholdValue(testCase.a, testCase.b) != testCase.equal ||
compareThresholdValue(testCase.b, testCase.a) != testCase.equal {
t.Errorf("Test case: %v failed", i)
}
}
}
// newPodDiskStats returns stats with specified usage amounts.
func newPodDiskStats(pod *api.Pod, rootFsUsed, logsUsed, perLocalVolumeUsed resource.Quantity) statsapi.PodStats {
result := statsapi.PodStats{

24
pkg/kubelet/eviction/types.go
View File

@ -66,14 +66,24 @@ type Config struct {
Thresholds []Threshold
}
// ThresholdValue is a value holder that abstracts literal versus percentage based quantity
type ThresholdValue struct {
// The following fields are exclusive. Only the topmost non-zero field is used.
// Quantity is a quantity associated with the signal that is evaluated against the specified operator.
Quantity *resource.Quantity
// Percentage represents the usage percentage over the total resource that is evaluated against the specified operator.
Percentage float32
}
// Threshold defines a metric for when eviction should occur.
type Threshold struct {
// Signal defines the entity that was measured.
Signal Signal
// Operator represents a relationship of a signal to a value.
Operator ThresholdOperator
// value is a quantity associated with the signal that is evaluated against the specified operator.
Value *resource.Quantity
// Value is the threshold the resource is evaluated against.
Value ThresholdValue
// GracePeriod represents the amount of time that a threshold must be met before eviction is triggered.
GracePeriod time.Duration
// MinReclaim represents the minimum amount of resource to reclaim if the threshold is met.
@ -122,8 +132,16 @@ type statsFunc func(pod *api.Pod) (statsapi.PodStats, bool)
// rankFunc sorts the pods in eviction order
type rankFunc func(pods []*api.Pod, stats statsFunc)
// signalObservation is the observed resource usage
type signalObservation struct {
// The resource capacity
capacity *resource.Quantity
// The available resource
available *resource.Quantity
}
// signalObservations maps a signal to an observed quantity
type signalObservations map[Signal]*resource.Quantity
type signalObservations map[Signal]signalObservation
// thresholdsObservedAt maps a threshold to a time that it was observed
type thresholdsObservedAt map[Threshold]time.Time