github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-01-13 11:25:19 +00:00
Code Issues Projects Releases Wiki Activity

Fixed forbidden window enforcement in horizontal pod autoscaler.

Browse Source 
Fixed forbidden window enforcement in horizontal pod autoscaler: time of the oldest report instead of now is taken into account. Fixes #17992.
This commit is contained in:
Jerzy Szczepkowski
2015-12-02 09:24:17 +01:00
parent 11c878e17c
commit 8cdda36b1e
3 changed files with 86 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
pkg/controller/podautoscaler/metrics/metrics_client.go
View File

@@ -43,10 +43,10 @@ var heapsterQueryStart = -5 * time.Minute
// MetricsClient is an interface for getting metrics for pods.
type MetricsClient interface {
// GetCPUUtilization returns average utilization over all pods
// represented as a percent of requested CPU, e.g. 70 means that
// an average pod uses 70% of the requested CPU.
GetCPUUtilization(namespace string, selector map[string]string) (*int, error)
// GetCPUUtilization returns the average utilization over all pods represented as a percent of requested CPU
// (e.g. 70 means that an average pod uses 70% of the requested CPU)
// and the time of generation of the oldest of utilization reports for pods.
GetCPUUtilization(namespace string, selector map[string]string) (*int, time.Time, error)
}
// ResourceConsumption specifies consumption of a particular resource.
@@ -55,9 +55,8 @@ type ResourceConsumption struct {
Quantity resource.Quantity
}
// Aggregates results into ResourceConsumption. Also returns number of
// pods included in the aggregation.
type metricAggregator func(heapster.MetricResultList) (ResourceConsumption, int)
// Aggregates results into ResourceConsumption. Also returns number of pods included in the aggregation.
type metricAggregator func(heapster.MetricResultList) (ResourceConsumption, int, time.Time)
type metricDefinition struct {
name string
@@ -76,23 +75,23 @@ type HeapsterMetricsClient struct {
var heapsterMetricDefinitions = map[api.ResourceName]metricDefinition{
api.ResourceCPU: {"cpu-usage",
func(metrics heapster.MetricResultList) (ResourceConsumption, int) {
sum, count := calculateSumFromLatestSample(metrics)
func(metrics heapster.MetricResultList) (ResourceConsumption, int, time.Time) {
sum, count, timestamp := calculateSumFromLatestSample(metrics)
value := "0"
if count > 0 {
// assumes that cpu usage is in millis
value = fmt.Sprintf("%dm", sum/uint64(count))
}
return ResourceConsumption{Resource: api.ResourceCPU, Quantity: resource.MustParse(value)}, count
return ResourceConsumption{Resource: api.ResourceCPU, Quantity: resource.MustParse(value)}, count, timestamp
}},
api.ResourceMemory: {"memory-usage",
func(metrics heapster.MetricResultList) (ResourceConsumption, int) {
sum, count := calculateSumFromLatestSample(metrics)
func(metrics heapster.MetricResultList) (ResourceConsumption, int, time.Time) {
sum, count, timestamp := calculateSumFromLatestSample(metrics)
value := int64(0)
if count > 0 {
value = int64(sum) / int64(count)
}
return ResourceConsumption{Resource: api.ResourceMemory, Quantity: *resource.NewQuantity(value, resource.DecimalSI)}, count
return ResourceConsumption{Resource: api.ResourceMemory, Quantity: *resource.NewQuantity(value, resource.DecimalSI)}, count, timestamp
}},
}
@@ -108,22 +107,22 @@ func NewHeapsterMetricsClient(client client.Interface, namespace, scheme, servic
}
}
func (h *HeapsterMetricsClient) GetCPUUtilization(namespace string, selector map[string]string) (*int, error) {
consumption, request, err := h.GetResourceConsumptionAndRequest(api.ResourceCPU, namespace, selector)
func (h *HeapsterMetricsClient) GetCPUUtilization(namespace string, selector map[string]string) (*int, time.Time, error) {
consumption, request, timestamp, err := h.GetResourceConsumptionAndRequest(api.ResourceCPU, namespace, selector)
if err != nil {
return nil, fmt.Errorf("failed to get CPU consumption and request: %v", err)
return nil, time.Time{}, fmt.Errorf("failed to get CPU consumption and request: %v", err)
}
utilization := new(int)
*utilization = int(float64(consumption.Quantity.MilliValue()) / float64(request.MilliValue()) * 100)
return utilization, nil
return utilization, timestamp, nil
}
func (h *HeapsterMetricsClient) GetResourceConsumptionAndRequest(resourceName api.ResourceName, namespace string, selector map[string]string) (consumption *ResourceConsumption, request *resource.Quantity, err error) {
func (h *HeapsterMetricsClient) GetResourceConsumptionAndRequest(resourceName api.ResourceName, namespace string, selector map[string]string) (consumption *ResourceConsumption, request *resource.Quantity, timestamp time.Time, err error) {
podList, err := h.client.Pods(namespace).
List(labels.SelectorFromSet(labels.Set(selector)), fields.Everything())
if err != nil {
return nil, nil, fmt.Errorf("failed to get pod list: %v", err)
return nil, nil, time.Time{}, fmt.Errorf("failed to get pod list: %v", err)
}
podNames := []string{}
sum := resource.MustParse("0")
@@ -140,22 +139,22 @@ func (h *HeapsterMetricsClient) GetResourceConsumptionAndRequest(resourceName ap
}
}
if missing || sum.Cmp(resource.MustParse("0")) == 0 {
return nil, nil, fmt.Errorf("some pods do not have request for %s", resourceName)
return nil, nil, time.Time{}, fmt.Errorf("some pods do not have request for %s", resourceName)
}
glog.Infof("Sum of %s requested: %v", resourceName, sum)
avg := resource.MustParse(fmt.Sprintf("%dm", sum.MilliValue()/int64(len(podList.Items))))
request = &avg
consumption, err = h.getForPods(resourceName, namespace, podNames)
consumption, timestamp, err = h.getForPods(resourceName, namespace, podNames)
if err != nil {
return nil, nil, err
return nil, nil, time.Time{}, err
}
return consumption, request, nil
return consumption, request, timestamp, nil
}
func (h *HeapsterMetricsClient) getForPods(resourceName api.ResourceName, namespace string, podNames []string) (*ResourceConsumption, error) {
func (h *HeapsterMetricsClient) getForPods(resourceName api.ResourceName, namespace string, podNames []string) (*ResourceConsumption, time.Time, error) {
metricSpec, metricDefined := h.resourceDefinitions[resourceName]
if !metricDefined {
return nil, fmt.Errorf("heapster metric not defined for %v", resourceName)
return nil, time.Time{}, fmt.Errorf("heapster metric not defined for %v", resourceName)
}
now := time.Now()
@@ -170,30 +169,33 @@ func (h *HeapsterMetricsClient) getForPods(resourceName api.ResourceName, namesp
DoRaw()
if err != nil {
return nil, fmt.Errorf("failed to get pods metrics: %v", err)
return nil, time.Time{}, fmt.Errorf("failed to get pods metrics: %v", err)
}
var metrics heapster.MetricResultList
err = json.Unmarshal(resultRaw, &metrics)
if err != nil {
return nil, fmt.Errorf("failed to unmarshall heapster response: %v", err)
return nil, time.Time{}, fmt.Errorf("failed to unmarshall heapster response: %v", err)
}
glog.Infof("Metrics available: %s", string(resultRaw))
currentConsumption, count := metricSpec.aggregator(metrics)
currentConsumption, count, timestamp := metricSpec.aggregator(metrics)
if count != len(podNames) {
return nil, fmt.Errorf("metrics obtained for %d/%d of pods", count, len(podNames))
return nil, time.Time{}, fmt.Errorf("metrics obtained for %d/%d of pods", count, len(podNames))
}
return &currentConsumption, nil
return &currentConsumption, timestamp, nil
}
func calculateSumFromLatestSample(metrics heapster.MetricResultList) (uint64, int) {
sum := uint64(0)
count := 0
func calculateSumFromLatestSample(metrics heapster.MetricResultList) (sum uint64, count int, timestamp time.Time) {
sum = uint64(0)
count = 0
timestamp = time.Time{}
var oldest *time.Time // creation time of the oldest of used samples across pods
oldest = nil
for _, metrics := range metrics.Items {
var newest *heapster.MetricPoint
var newest *heapster.MetricPoint // creation time of the newest sample for pod
newest = nil
for i, metricPoint := range metrics.Metrics {
if newest == nil || newest.Timestamp.Before(metricPoint.Timestamp) {
@@ -201,9 +203,15 @@ func calculateSumFromLatestSample(metrics heapster.MetricResultList) (uint64, in
}
}
if newest != nil {
if oldest == nil || newest.Timestamp.Before(*oldest) {
oldest = &newest.Timestamp
}
sum += newest.Value
count++
}
}
return sum, count
if oldest != nil {
timestamp = *oldest
}
return sum, count, timestamp
}