diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index c6fa59e0fa2..8395b3c4fa6 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -285,7 +285,7 @@ func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim api.ResourceNa glog.Errorf("eviction manager: unable to find value associated with signal %v", signal) continue } - value.Add(*reclaimed) + value.available.Add(*reclaimed) // evaluate all current thresholds to see if with adjusted observations, we think we have met min reclaim goals if len(thresholdsMet(m.thresholdsMet, observations, true)) == 0 { diff --git a/pkg/kubelet/eviction/eviction_manager_test.go b/pkg/kubelet/eviction/eviction_manager_test.go index 9870a654f19..1eb3716c63d 100644 --- a/pkg/kubelet/eviction/eviction_manager_test.go +++ b/pkg/kubelet/eviction/eviction_manager_test.go @@ -79,10 +79,12 @@ func TestMemoryPressure(t *testing.T) { summaryStatsMaker := func(nodeAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary { val := resource.MustParse(nodeAvailableBytes) availableBytes := uint64(val.Value()) + WorkingSetBytes := uint64(val.Value()) result := &statsapi.Summary{ Node: statsapi.NodeStats{ Memory: &statsapi.MemoryStats{ - AvailableBytes: &availableBytes, + AvailableBytes: &availableBytes, + WorkingSetBytes: &WorkingSetBytes, }, }, Pods: []statsapi.PodStats{}, @@ -129,12 +131,16 @@ func TestMemoryPressure(t *testing.T) { { Signal: SignalMemoryAvailable, Operator: OpLessThan, - Value: quantityMustParse("1Gi"), + Value: ThresholdValue{ + Quantity: quantityMustParse("1Gi"), + }, }, { - Signal: SignalMemoryAvailable, - Operator: OpLessThan, - Value: quantityMustParse("2Gi"), + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("2Gi"), + }, GracePeriod: time.Minute * 2, }, }, @@ -317,16 +323,20 @@ func TestDiskPressureNodeFs(t *testing.T) { summaryStatsMaker := func(rootFsAvailableBytes, imageFsAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary { rootFsVal := resource.MustParse(rootFsAvailableBytes) rootFsBytes := uint64(rootFsVal.Value()) + rootFsCapacityBytes := uint64(rootFsVal.Value() * 2) imageFsVal := resource.MustParse(imageFsAvailableBytes) imageFsBytes := uint64(imageFsVal.Value()) + imageFsCapacityBytes := uint64(imageFsVal.Value() * 2) result := &statsapi.Summary{ Node: statsapi.NodeStats{ Fs: &statsapi.FsStats{ AvailableBytes: &rootFsBytes, + CapacityBytes: &rootFsCapacityBytes, }, Runtime: &statsapi.RuntimeStats{ ImageFs: &statsapi.FsStats{ AvailableBytes: &imageFsBytes, + CapacityBytes: &imageFsCapacityBytes, }, }, }, @@ -376,12 +386,16 @@ func TestDiskPressureNodeFs(t *testing.T) { { Signal: SignalNodeFsAvailable, Operator: OpLessThan, - Value: quantityMustParse("1Gi"), + Value: ThresholdValue{ + Quantity: quantityMustParse("1Gi"), + }, }, { - Signal: SignalNodeFsAvailable, - Operator: OpLessThan, - Value: quantityMustParse("2Gi"), + Signal: SignalNodeFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("2Gi"), + }, GracePeriod: time.Minute * 2, }, }, @@ -544,10 +558,12 @@ func TestMinReclaim(t *testing.T) { summaryStatsMaker := func(nodeAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary { val := resource.MustParse(nodeAvailableBytes) availableBytes := uint64(val.Value()) + WorkingSetBytes := uint64(val.Value()) result := &statsapi.Summary{ Node: statsapi.NodeStats{ Memory: &statsapi.MemoryStats{ - AvailableBytes: &availableBytes, + AvailableBytes: &availableBytes, + WorkingSetBytes: &WorkingSetBytes, }, }, Pods: []statsapi.PodStats{}, @@ -592,9 +608,11 @@ func TestMinReclaim(t *testing.T) { PressureTransitionPeriod: time.Minute * 5, Thresholds: []Threshold{ { - Signal: SignalMemoryAvailable, - Operator: OpLessThan, - Value: quantityMustParse("1Gi"), + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("1Gi"), + }, MinReclaim: quantityMustParse("500Mi"), }, }, @@ -703,16 +721,20 @@ func TestNodeReclaimFuncs(t *testing.T) { summaryStatsMaker := func(rootFsAvailableBytes, imageFsAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary { rootFsVal := resource.MustParse(rootFsAvailableBytes) rootFsBytes := uint64(rootFsVal.Value()) + rootFsCapacityBytes := uint64(rootFsVal.Value() * 2) imageFsVal := resource.MustParse(imageFsAvailableBytes) imageFsBytes := uint64(imageFsVal.Value()) + imageFsCapacityBytes := uint64(imageFsVal.Value() * 2) result := &statsapi.Summary{ Node: statsapi.NodeStats{ Fs: &statsapi.FsStats{ AvailableBytes: &rootFsBytes, + CapacityBytes: &rootFsCapacityBytes, }, Runtime: &statsapi.RuntimeStats{ ImageFs: &statsapi.FsStats{ AvailableBytes: &imageFsBytes, + CapacityBytes: &imageFsCapacityBytes, }, }, }, @@ -761,9 +783,11 @@ func TestNodeReclaimFuncs(t *testing.T) { PressureTransitionPeriod: time.Minute * 5, Thresholds: []Threshold{ { - Signal: SignalNodeFsAvailable, - Operator: OpLessThan, - Value: quantityMustParse("1Gi"), + Signal: SignalNodeFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("1Gi"), + }, MinReclaim: quantityMustParse("500Mi"), }, }, diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index 6f04d600a8c..c2025f44ae7 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -19,11 +19,11 @@ package eviction import ( "fmt" "sort" + "strconv" "strings" "time" "github.com/golang/glog" - "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" statsapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats" @@ -169,18 +169,47 @@ func parseThresholdStatement(statement string) (Threshold, error) { return Threshold{}, fmt.Errorf(unsupportedEvictionSignal, signal) } - quantity, err := resource.ParseQuantity(parts[1]) + quantityValue := parts[1] + if strings.HasSuffix(quantityValue, "%") { + percentage, err := parsePercentage(quantityValue) + if err != nil { + return Threshold{}, err + } + if percentage <= 0 { + return Threshold{}, fmt.Errorf("eviction percentage threshold %v must be positive: %s", signal, quantityValue) + } + return Threshold{ + Signal: signal, + Operator: operator, + Value: ThresholdValue{ + Percentage: percentage, + }, + }, nil + } else { + quantity, err := resource.ParseQuantity(quantityValue) + if err != nil { + return Threshold{}, err + } + if quantity.Sign() < 0 || quantity.IsZero() { + return Threshold{}, fmt.Errorf("eviction threshold %v must be positive: %s", signal, &quantity) + } + return Threshold{ + Signal: signal, + Operator: operator, + Value: ThresholdValue{ + Quantity: &quantity, + }, + }, nil + } +} + +// parsePercentage parses a string representing a percentage value +func parsePercentage(input string) (float32, error) { + value, err := strconv.ParseFloat(strings.TrimRight(input, "%"), 32) if err != nil { - return Threshold{}, err + return 0, err } - if quantity.Sign() < 0 { - return Threshold{}, fmt.Errorf("eviction threshold %v cannot be negative: %s", signal, &quantity) - } - return Threshold{ - Signal: signal, - Operator: operator, - Value: &quantity, - }, nil + return float32(value) / 100, nil } // parseGracePeriods parses the grace period statements @@ -329,7 +358,15 @@ func podMemoryUsage(podStats statsapi.PodStats) (api.ResourceList, error) { // formatThreshold formats a threshold for logging. func formatThreshold(threshold Threshold) string { - return fmt.Sprintf("threshold(signal=%v, operator=%v, value=%v, gracePeriod=%v)", threshold.Signal, threshold.Value.String(), threshold.Operator, threshold.GracePeriod) + return fmt.Sprintf("threshold(signal=%v, operator=%v, value=%v, gracePeriod=%v)", threshold.Signal, formatThresholdValue(threshold.Value), threshold.Operator, threshold.GracePeriod) +} + +// formatThresholdValue formats a thresholdValue for logging. +func formatThresholdValue(value ThresholdValue) string { + if value.Quantity != nil { + return value.Quantity.String() + } + return fmt.Sprintf("%f%%", value.Percentage*float32(100)) } // cachedStatsFunc returns a statsFunc based on the provided pod stats. @@ -532,15 +569,24 @@ func makeSignalObservations(summaryProvider stats.SummaryProvider) (signalObserv // build an evaluation context for current eviction signals result := signalObservations{} - if memory := summary.Node.Memory; memory != nil && memory.AvailableBytes != nil { - result[SignalMemoryAvailable] = resource.NewQuantity(int64(*memory.AvailableBytes), resource.BinarySI) + if memory := summary.Node.Memory; memory != nil && memory.AvailableBytes != nil && memory.WorkingSetBytes != nil { + result[SignalMemoryAvailable] = signalObservation{ + available: resource.NewQuantity(int64(*memory.AvailableBytes), resource.BinarySI), + capacity: resource.NewQuantity(int64(*memory.AvailableBytes+*memory.WorkingSetBytes), resource.BinarySI), + } } - if nodeFs := summary.Node.Fs; nodeFs != nil && nodeFs.AvailableBytes != nil { - result[SignalNodeFsAvailable] = resource.NewQuantity(int64(*nodeFs.AvailableBytes), resource.BinarySI) + if nodeFs := summary.Node.Fs; nodeFs != nil && nodeFs.AvailableBytes != nil && nodeFs.CapacityBytes != nil { + result[SignalNodeFsAvailable] = signalObservation{ + available: resource.NewQuantity(int64(*nodeFs.AvailableBytes), resource.BinarySI), + capacity: resource.NewQuantity(int64(*nodeFs.CapacityBytes), resource.BinarySI), + } } if summary.Node.Runtime != nil { - if imageFs := summary.Node.Runtime.ImageFs; imageFs != nil && imageFs.AvailableBytes != nil { - result[SignalImageFsAvailable] = resource.NewQuantity(int64(*imageFs.AvailableBytes), resource.BinarySI) + if imageFs := summary.Node.Runtime.ImageFs; imageFs != nil && imageFs.AvailableBytes != nil && imageFs.CapacityBytes != nil { + result[SignalImageFsAvailable] = signalObservation{ + available: resource.NewQuantity(int64(*imageFs.AvailableBytes), resource.BinarySI), + capacity: resource.NewQuantity(int64(*imageFs.CapacityBytes), resource.BinarySI), + } } } return result, statsFunc, nil @@ -558,12 +604,12 @@ func thresholdsMet(thresholds []Threshold, observations signalObservations, enfo } // determine if we have met the specified threshold thresholdMet := false - quantity := threshold.Value.Copy() + quantity := getThresholdQuantity(threshold.Value, observed.capacity) // if enforceMinReclaim is specified, we compare relative to value - minreclaim if enforceMinReclaim && threshold.MinReclaim != nil { quantity.Add(*threshold.MinReclaim) } - thresholdResult := quantity.Cmp(*observed) + thresholdResult := quantity.Cmp(*observed.available) switch threshold.Operator { case OpLessThan: thresholdMet = thresholdResult > 0 @@ -575,6 +621,14 @@ func thresholdsMet(thresholds []Threshold, observations signalObservations, enfo return results } +// getThresholdQuantity returns the expected quantity value for a thresholdValue +func getThresholdQuantity(value ThresholdValue, capacity *resource.Quantity) *resource.Quantity { + if value.Quantity != nil { + return value.Quantity.Copy() + } + return resource.NewQuantity(int64(float64(capacity.Value())*float64(value.Percentage)), resource.BinarySI) +} + // thresholdsFirstObservedAt merges the input set of thresholds with the previous observation to determine when active set of thresholds were initially met. func thresholdsFirstObservedAt(thresholds []Threshold, lastObservedAt thresholdsObservedAt, now time.Time) thresholdsObservedAt { results := thresholdsObservedAt{} @@ -678,13 +732,27 @@ func mergeThresholds(inputsA []Threshold, inputsB []Threshold) []Threshold { // hasThreshold returns true if the threshold is in the input list func hasThreshold(inputs []Threshold, item Threshold) bool { for _, input := range inputs { - if input.GracePeriod == item.GracePeriod && input.Operator == item.Operator && input.Signal == item.Signal && input.Value.Cmp(*item.Value) == 0 { + if input.GracePeriod == item.GracePeriod && input.Operator == item.Operator && input.Signal == item.Signal && compareThresholdValue(input.Value, item.Value) { return true } } return false } +// compareThresholdValue returns true if the two thresholdValue objects are logically the same +func compareThresholdValue(a ThresholdValue, b ThresholdValue) bool { + if a.Quantity != nil { + if b.Quantity == nil { + return false + } + return a.Quantity.Cmp(*b.Quantity) == 0 + } + if b.Quantity != nil { + return false + } + return a.Percentage == b.Percentage +} + // getStarvedResources returns the set of resources that are starved based on thresholds met. func getStarvedResources(thresholds []Threshold) []api.ResourceName { results := []api.ResourceName{} diff --git a/pkg/kubelet/eviction/helpers_test.go b/pkg/kubelet/eviction/helpers_test.go index aeb6cd0ea09..fa2fbd418ea 100644 --- a/pkg/kubelet/eviction/helpers_test.go +++ b/pkg/kubelet/eviction/helpers_test.go @@ -61,15 +61,45 @@ func TestParseThresholdConfig(t *testing.T) { expectErr: false, expectThresholds: []Threshold{ { - Signal: SignalMemoryAvailable, - Operator: OpLessThan, - Value: quantityMustParse("150Mi"), + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("150Mi"), + }, MinReclaim: quantityMustParse("0"), }, { - Signal: SignalMemoryAvailable, - Operator: OpLessThan, - Value: quantityMustParse("300Mi"), + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("300Mi"), + }, + GracePeriod: gracePeriod, + MinReclaim: quantityMustParse("0"), + }, + }, + }, + "all flag values in percentages": { + evictionHard: "memory.available<10%", + evictionSoft: "memory.available<30%", + evictionSoftGracePeriod: "memory.available=30s", + evictionMinReclaim: "memory.available=0", + expectErr: false, + expectThresholds: []Threshold{ + { + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Percentage: 0.1, + }, + MinReclaim: quantityMustParse("0"), + }, + { + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Percentage: 0.3, + }, GracePeriod: gracePeriod, MinReclaim: quantityMustParse("0"), }, @@ -83,28 +113,79 @@ func TestParseThresholdConfig(t *testing.T) { expectErr: false, expectThresholds: []Threshold{ { - Signal: SignalImageFsAvailable, - Operator: OpLessThan, - Value: quantityMustParse("150Mi"), + Signal: SignalImageFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("150Mi"), + }, MinReclaim: quantityMustParse("2Gi"), }, { - Signal: SignalNodeFsAvailable, - Operator: OpLessThan, - Value: quantityMustParse("100Mi"), + Signal: SignalNodeFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("100Mi"), + }, MinReclaim: quantityMustParse("1Gi"), }, { - Signal: SignalImageFsAvailable, - Operator: OpLessThan, - Value: quantityMustParse("300Mi"), + Signal: SignalImageFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("300Mi"), + }, GracePeriod: gracePeriod, MinReclaim: quantityMustParse("2Gi"), }, { - Signal: SignalNodeFsAvailable, - Operator: OpLessThan, - Value: quantityMustParse("200Mi"), + Signal: SignalNodeFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("200Mi"), + }, + GracePeriod: gracePeriod, + MinReclaim: quantityMustParse("1Gi"), + }, + }, + }, + "disk flag values in percentages": { + evictionHard: "imagefs.available<15%,nodefs.available<10.5%", + evictionSoft: "imagefs.available<30%,nodefs.available<20.5%", + evictionSoftGracePeriod: "imagefs.available=30s,nodefs.available=30s", + evictionMinReclaim: "imagefs.available=2Gi,nodefs.available=1Gi", + expectErr: false, + expectThresholds: []Threshold{ + { + Signal: SignalImageFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Percentage: 0.15, + }, + MinReclaim: quantityMustParse("2Gi"), + }, + { + Signal: SignalNodeFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Percentage: 0.105, + }, + MinReclaim: quantityMustParse("1Gi"), + }, + { + Signal: SignalImageFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Percentage: 0.3, + }, + GracePeriod: gracePeriod, + MinReclaim: quantityMustParse("2Gi"), + }, + { + Signal: SignalNodeFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Percentage: 0.205, + }, GracePeriod: gracePeriod, MinReclaim: quantityMustParse("1Gi"), }, @@ -126,6 +207,14 @@ func TestParseThresholdConfig(t *testing.T) { expectErr: true, expectThresholds: []Threshold{}, }, + "hard-signal-negative-percentage": { + evictionHard: "memory.available<-15%", + evictionSoft: "", + evictionSoftGracePeriod: "", + evictionMinReclaim: "", + expectErr: true, + expectThresholds: []Threshold{}, + }, "soft-signal-negative": { evictionHard: "", evictionSoft: "memory.available<-150Mi", @@ -227,8 +316,8 @@ func thresholdEqual(a Threshold, b Threshold) bool { return a.GracePeriod == b.GracePeriod && a.Operator == b.Operator && a.Signal == b.Signal && - a.Value.Cmp(*b.Value) == 0 && - a.MinReclaim.Cmp(*b.MinReclaim) == 0 + a.MinReclaim.Cmp(*b.MinReclaim) == 0 && + compareThresholdValue(a.Value, b.Value) } // TestOrderedByQoS ensures we order BestEffort < Burstable < Guaranteed @@ -514,20 +603,26 @@ func TestMakeSignalObservations(t *testing.T) { return pod } nodeAvailableBytes := uint64(1024 * 1024 * 1024) + nodeWorkingSetBytes := uint64(1024 * 1024 * 1024) imageFsAvailableBytes := uint64(1024 * 1024) + imageFsCapacityBytes := uint64(1024 * 1024 * 2) nodeFsAvailableBytes := uint64(1024) + nodeFsCapacityBytes := uint64(1024 * 2) fakeStats := &statsapi.Summary{ Node: statsapi.NodeStats{ Memory: &statsapi.MemoryStats{ - AvailableBytes: &nodeAvailableBytes, + AvailableBytes: &nodeAvailableBytes, + WorkingSetBytes: &nodeWorkingSetBytes, }, Runtime: &statsapi.RuntimeStats{ ImageFs: &statsapi.FsStats{ AvailableBytes: &imageFsAvailableBytes, + CapacityBytes: &imageFsCapacityBytes, }, }, Fs: &statsapi.FsStats{ AvailableBytes: &nodeFsAvailableBytes, + CapacityBytes: &nodeFsCapacityBytes, }, }, Pods: []statsapi.PodStats{}, @@ -545,6 +640,7 @@ func TestMakeSignalObservations(t *testing.T) { fakeStats.Pods = append(fakeStats.Pods, newPodStats(pod, containerWorkingSetBytes)) } actualObservations, statsFunc, err := makeSignalObservations(provider) + if err != nil { t.Errorf("Unexpected err: %v", err) } @@ -552,22 +648,31 @@ func TestMakeSignalObservations(t *testing.T) { if !found { t.Errorf("Expected available memory observation: %v", err) } - if expectedBytes := int64(nodeAvailableBytes); memQuantity.Value() != expectedBytes { - t.Errorf("Expected %v, actual: %v", expectedBytes, memQuantity.Value()) + if expectedBytes := int64(nodeAvailableBytes); memQuantity.available.Value() != expectedBytes { + t.Errorf("Expected %v, actual: %v", expectedBytes, memQuantity.available.Value()) + } + if expectedBytes := int64(nodeWorkingSetBytes + nodeAvailableBytes); memQuantity.capacity.Value() != expectedBytes { + t.Errorf("Expected %v, actual: %v", expectedBytes, memQuantity.capacity.Value()) } nodeFsQuantity, found := actualObservations[SignalNodeFsAvailable] if !found { t.Errorf("Expected available nodefs observation: %v", err) } - if expectedBytes := int64(nodeFsAvailableBytes); nodeFsQuantity.Value() != expectedBytes { - t.Errorf("Expected %v, actual: %v", expectedBytes, nodeFsQuantity.Value()) + if expectedBytes := int64(nodeFsAvailableBytes); nodeFsQuantity.available.Value() != expectedBytes { + t.Errorf("Expected %v, actual: %v", expectedBytes, nodeFsQuantity.available.Value()) + } + if expectedBytes := int64(nodeFsCapacityBytes); nodeFsQuantity.capacity.Value() != expectedBytes { + t.Errorf("Expected %v, actual: %v", expectedBytes, nodeFsQuantity.capacity.Value()) } imageFsQuantity, found := actualObservations[SignalImageFsAvailable] if !found { t.Errorf("Expected available imagefs observation: %v", err) } - if expectedBytes := int64(imageFsAvailableBytes); imageFsQuantity.Value() != expectedBytes { - t.Errorf("Expected %v, actual: %v", expectedBytes, imageFsQuantity.Value()) + if expectedBytes := int64(imageFsAvailableBytes); imageFsQuantity.available.Value() != expectedBytes { + t.Errorf("Expected %v, actual: %v", expectedBytes, imageFsQuantity.available.Value()) + } + if expectedBytes := int64(imageFsCapacityBytes); imageFsQuantity.capacity.Value() != expectedBytes { + t.Errorf("Expected %v, actual: %v", expectedBytes, imageFsQuantity.capacity.Value()) } for _, pod := range pods { podStats, found := statsFunc(pod) @@ -585,9 +690,11 @@ func TestMakeSignalObservations(t *testing.T) { func TestThresholdsMet(t *testing.T) { hardThreshold := Threshold{ - Signal: SignalMemoryAvailable, - Operator: OpLessThan, - Value: quantityMustParse("1Gi"), + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("1Gi"), + }, MinReclaim: quantityMustParse("500Mi"), } testCases := map[string]struct { @@ -602,11 +709,13 @@ func TestThresholdsMet(t *testing.T) { observations: signalObservations{}, result: []Threshold{}, }, - "threshold-met": { + "threshold-met-memory": { enforceMinReclaim: false, thresholds: []Threshold{hardThreshold}, observations: signalObservations{ - SignalMemoryAvailable: quantityMustParse("500Mi"), + SignalMemoryAvailable: signalObservation{ + available: quantityMustParse("500Mi"), + }, }, result: []Threshold{hardThreshold}, }, @@ -614,7 +723,9 @@ func TestThresholdsMet(t *testing.T) { enforceMinReclaim: false, thresholds: []Threshold{hardThreshold}, observations: signalObservations{ - SignalMemoryAvailable: quantityMustParse("2Gi"), + SignalMemoryAvailable: signalObservation{ + available: quantityMustParse("2Gi"), + }, }, result: []Threshold{}, }, @@ -622,7 +733,9 @@ func TestThresholdsMet(t *testing.T) { enforceMinReclaim: true, thresholds: []Threshold{hardThreshold}, observations: signalObservations{ - SignalMemoryAvailable: quantityMustParse("1.05Gi"), + SignalMemoryAvailable: signalObservation{ + available: quantityMustParse("1.05Gi"), + }, }, result: []Threshold{hardThreshold}, }, @@ -630,7 +743,9 @@ func TestThresholdsMet(t *testing.T) { enforceMinReclaim: true, thresholds: []Threshold{hardThreshold}, observations: signalObservations{ - SignalMemoryAvailable: quantityMustParse("2Gi"), + SignalMemoryAvailable: signalObservation{ + available: quantityMustParse("2Gi"), + }, }, result: []Threshold{}, }, @@ -643,11 +758,101 @@ func TestThresholdsMet(t *testing.T) { } } +func TestPercentageThresholdsMet(t *testing.T) { + specifiecThresholds := []Threshold{ + { + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Percentage: 0.2, + }, + }, + { + Signal: SignalNodeFsAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Percentage: 0.3, + }, + }, + } + + testCases := map[string]struct { + thresholds []Threshold + observations signalObservations + result []Threshold + }{ + "BothMet": { + thresholds: specifiecThresholds, + observations: signalObservations{ + SignalMemoryAvailable: signalObservation{ + available: quantityMustParse("100Mi"), + capacity: quantityMustParse("1000Mi"), + }, + SignalNodeFsAvailable: signalObservation{ + available: quantityMustParse("100Gi"), + capacity: quantityMustParse("1000Gi"), + }, + }, + result: specifiecThresholds, + }, + "NoneMet": { + thresholds: specifiecThresholds, + observations: signalObservations{ + SignalMemoryAvailable: signalObservation{ + available: quantityMustParse("300Mi"), + capacity: quantityMustParse("1000Mi"), + }, + SignalNodeFsAvailable: signalObservation{ + available: quantityMustParse("400Gi"), + capacity: quantityMustParse("1000Gi"), + }, + }, + result: []Threshold{}, + }, + "DiskMet": { + thresholds: specifiecThresholds, + observations: signalObservations{ + SignalMemoryAvailable: signalObservation{ + available: quantityMustParse("300Mi"), + capacity: quantityMustParse("1000Mi"), + }, + SignalNodeFsAvailable: signalObservation{ + available: quantityMustParse("100Gi"), + capacity: quantityMustParse("1000Gi"), + }, + }, + result: []Threshold{specifiecThresholds[1]}, + }, + "MemoryMet": { + thresholds: specifiecThresholds, + observations: signalObservations{ + SignalMemoryAvailable: signalObservation{ + available: quantityMustParse("100Mi"), + capacity: quantityMustParse("1000Mi"), + }, + SignalNodeFsAvailable: signalObservation{ + available: quantityMustParse("400Gi"), + capacity: quantityMustParse("1000Gi"), + }, + }, + result: []Threshold{specifiecThresholds[0]}, + }, + } + for testName, testCase := range testCases { + actual := thresholdsMet(testCase.thresholds, testCase.observations, false) + if !thresholdList(actual).Equal(thresholdList(testCase.result)) { + t.Errorf("Test case: %s, expected: %v, actual: %v", testName, testCase.result, actual) + } + } +} + func TestThresholdsFirstObservedAt(t *testing.T) { hardThreshold := Threshold{ Signal: SignalMemoryAvailable, Operator: OpLessThan, - Value: quantityMustParse("1Gi"), + Value: ThresholdValue{ + Quantity: quantityMustParse("1Gi"), + }, } now := unversioned.Now() oldTime := unversioned.NewTime(now.Time.Add(-1 * time.Minute)) @@ -695,12 +900,16 @@ func TestThresholdsMetGracePeriod(t *testing.T) { hardThreshold := Threshold{ Signal: SignalMemoryAvailable, Operator: OpLessThan, - Value: quantityMustParse("1Gi"), + Value: ThresholdValue{ + Quantity: quantityMustParse("1Gi"), + }, } softThreshold := Threshold{ - Signal: SignalMemoryAvailable, - Operator: OpLessThan, - Value: quantityMustParse("2Gi"), + Signal: SignalMemoryAvailable, + Operator: OpLessThan, + Value: ThresholdValue{ + Quantity: quantityMustParse("2Gi"), + }, GracePeriod: 1 * time.Minute, } oldTime := unversioned.NewTime(now.Time.Add(-2 * time.Minute)) @@ -906,6 +1115,95 @@ func TestGetStarvedResources(t *testing.T) { } } +func testParsePercentage(t *testing.T) { + testCases := map[string]struct { + hasError bool + value float32 + }{ + "blah": { + hasError: true, + }, + "25.5%": { + value: 0.255, + }, + "foo%": { + hasError: true, + }, + "12%345": { + hasError: true, + }, + } + for input, expected := range testCases { + value, err := parsePercentage(input) + if (err != nil) != expected.hasError { + t.Errorf("Test case: %s, expected: %v, actual: %v", input, expected.hasError, err != nil) + } + if value != expected.value { + t.Errorf("Test case: %s, expected: %v, actual: %v", input, expected.value, value) + } + } +} + +func testCompareThresholdValue(t *testing.T) { + testCases := []struct { + a, b ThresholdValue + equal bool + }{ + { + a: ThresholdValue{ + Quantity: resource.NewQuantity(123, resource.BinarySI), + }, + b: ThresholdValue{ + Quantity: resource.NewQuantity(123, resource.BinarySI), + }, + equal: true, + }, + { + a: ThresholdValue{ + Quantity: resource.NewQuantity(123, resource.BinarySI), + }, + b: ThresholdValue{ + Quantity: resource.NewQuantity(456, resource.BinarySI), + }, + equal: false, + }, + { + a: ThresholdValue{ + Quantity: resource.NewQuantity(123, resource.BinarySI), + }, + b: ThresholdValue{ + Percentage: 0.1, + }, + equal: false, + }, + { + a: ThresholdValue{ + Percentage: 0.1, + }, + b: ThresholdValue{ + Percentage: 0.1, + }, + equal: true, + }, + { + a: ThresholdValue{ + Percentage: 0.2, + }, + b: ThresholdValue{ + Percentage: 0.1, + }, + equal: false, + }, + } + + for i, testCase := range testCases { + if compareThresholdValue(testCase.a, testCase.b) != testCase.equal || + compareThresholdValue(testCase.b, testCase.a) != testCase.equal { + t.Errorf("Test case: %v failed", i) + } + } +} + // newPodDiskStats returns stats with specified usage amounts. func newPodDiskStats(pod *api.Pod, rootFsUsed, logsUsed, perLocalVolumeUsed resource.Quantity) statsapi.PodStats { result := statsapi.PodStats{ diff --git a/pkg/kubelet/eviction/types.go b/pkg/kubelet/eviction/types.go index 897aae3e93d..6984148e194 100644 --- a/pkg/kubelet/eviction/types.go +++ b/pkg/kubelet/eviction/types.go @@ -66,14 +66,24 @@ type Config struct { Thresholds []Threshold } +// ThresholdValue is a value holder that abstracts literal versus percentage based quantity +type ThresholdValue struct { + // The following fields are exclusive. Only the topmost non-zero field is used. + + // Quantity is a quantity associated with the signal that is evaluated against the specified operator. + Quantity *resource.Quantity + // Percentage represents the usage percentage over the total resource that is evaluated against the specified operator. + Percentage float32 +} + // Threshold defines a metric for when eviction should occur. type Threshold struct { // Signal defines the entity that was measured. Signal Signal // Operator represents a relationship of a signal to a value. Operator ThresholdOperator - // value is a quantity associated with the signal that is evaluated against the specified operator. - Value *resource.Quantity + // Value is the threshold the resource is evaluated against. + Value ThresholdValue // GracePeriod represents the amount of time that a threshold must be met before eviction is triggered. GracePeriod time.Duration // MinReclaim represents the minimum amount of resource to reclaim if the threshold is met. @@ -122,8 +132,16 @@ type statsFunc func(pod *api.Pod) (statsapi.PodStats, bool) // rankFunc sorts the pods in eviction order type rankFunc func(pods []*api.Pod, stats statsFunc) +// signalObservation is the observed resource usage +type signalObservation struct { + // The resource capacity + capacity *resource.Quantity + // The available resource + available *resource.Quantity +} + // signalObservations maps a signal to an observed quantity -type signalObservations map[Signal]*resource.Quantity +type signalObservations map[Signal]signalObservation // thresholdsObservedAt maps a threshold to a time that it was observed type thresholdsObservedAt map[Threshold]time.Time