mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-08-09 03:57:41 +00:00
e2e: deflake hpa e2e custom metrics tests
a) add namespacing to metrics: fixes interference between `should scale up when one metric is missing (Pod and External metrics)` and `should not scale down when one metric is missing (Container Resource and External Metrics)` specs, cause of flakiness. b) replaces deployments containing unused exporters (metrics ignored) with deployments without any exporters: potential fix for often hitting a rate-limit on creating metrics descriptors (429 errors), also adds clarity. c) fixes metric types: some external metrics tests used non-average type while expecting the value to be constant regardless of the number of pods. However, queries resulting from metric specs don't filter by pods, so a sum of metrics for all the pods is the fetched metric value (https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects). Adding averaging back by the number of pods fixes a couple of specs where the tests were passing for the wrong reason (wanted d ifferent test conditions).
This commit is contained in:
Piotr Betkier
parent
6aa68d6a8b
commit
c99cf53a10
@ -32,6 +32,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/kubernetes/test/e2e/framework"
|
||||
e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
|
||||
e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
|
||||
"k8s.io/kubernetes/test/e2e/instrumentation/monitoring"
|
||||
admissionapi "k8s.io/pod-security-admission/api"
|
||||
@ -142,15 +143,14 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
metricValue := int64(100)
|
||||
metricTarget := 2 * metricValue
|
||||
metricSpecs := []autoscalingv2.MetricSpec{
|
||||
objectMetricSpecWithAverageValueTarget(metricTarget),
|
||||
objectMetricSpecWithValueTarget(metricTarget),
|
||||
}
|
||||
tc := CustomMetricTestCase{
|
||||
framework: f,
|
||||
kubeClient: f.ClientSet,
|
||||
initialReplicas: initialReplicas,
|
||||
scaledReplicas: 1,
|
||||
// Metric exported by deployment is ignored
|
||||
deployment: monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 0 /* ignored */),
|
||||
deployment: noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)),
|
||||
pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue),
|
||||
hpa: hpa("custom-metrics-objects-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs),
|
||||
}
|
||||
@ -163,15 +163,14 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
metricValue := int64(0)
|
||||
metricTarget := int64(200)
|
||||
metricSpecs := []autoscalingv2.MetricSpec{
|
||||
objectMetricSpecWithAverageValueTarget(metricTarget),
|
||||
objectMetricSpecWithValueTarget(metricTarget),
|
||||
}
|
||||
tc := CustomMetricTestCase{
|
||||
framework: f,
|
||||
kubeClient: f.ClientSet,
|
||||
initialReplicas: initialReplicas,
|
||||
scaledReplicas: 0,
|
||||
// Metric exported by deployment is ignored
|
||||
deployment: monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 0 /* ignored */),
|
||||
deployment: noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)),
|
||||
pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue),
|
||||
hpa: hpa("custom-metrics-objects-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 0, 3, metricSpecs),
|
||||
}
|
||||
@ -186,7 +185,7 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
metricValue := externalMetricValue
|
||||
metricTarget := 3 * metricValue
|
||||
metricSpecs := []autoscalingv2.MetricSpec{
|
||||
externalMetricSpecWithTarget("target", externalMetricTarget{
|
||||
externalMetricSpecWithTarget("target", f.Namespace.ObjectMeta.Name, externalMetricTarget{
|
||||
value: metricTarget,
|
||||
isAverage: false,
|
||||
}),
|
||||
@ -196,8 +195,7 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
kubeClient: f.ClientSet,
|
||||
initialReplicas: initialReplicas,
|
||||
scaledReplicas: 1,
|
||||
// Metric exported by deployment is ignored
|
||||
deployment: monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 0 /* ignored */),
|
||||
deployment: noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)),
|
||||
pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, "target", metricValue),
|
||||
hpa: hpa("custom-metrics-external-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs),
|
||||
}
|
||||
@ -210,7 +208,7 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
metricValue := externalMetricValue
|
||||
metricAverageTarget := 3 * metricValue
|
||||
metricSpecs := []autoscalingv2.MetricSpec{
|
||||
externalMetricSpecWithTarget("target_average", externalMetricTarget{
|
||||
externalMetricSpecWithTarget("target_average", f.Namespace.ObjectMeta.Name, externalMetricTarget{
|
||||
value: metricAverageTarget,
|
||||
isAverage: true,
|
||||
}),
|
||||
@ -220,8 +218,7 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
kubeClient: f.ClientSet,
|
||||
initialReplicas: initialReplicas,
|
||||
scaledReplicas: 1,
|
||||
// Metric exported by deployment is ignored
|
||||
deployment: monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 0 /* ignored */),
|
||||
deployment: noExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas)),
|
||||
pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, "target_average", externalMetricValue),
|
||||
hpa: hpa("custom-metrics-external-hpa", f.Namespace.ObjectMeta.Name, dummyDeploymentName, 1, 3, metricSpecs),
|
||||
}
|
||||
@ -237,13 +234,13 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
metric2Value := externalMetricValue
|
||||
metric2Target := int64(math.Ceil(0.5 * float64(metric2Value)))
|
||||
metricSpecs := []autoscalingv2.MetricSpec{
|
||||
externalMetricSpecWithTarget("external_metric_1", externalMetricTarget{
|
||||
externalMetricSpecWithTarget("external_metric_1", f.Namespace.ObjectMeta.Name, externalMetricTarget{
|
||||
value: metric1Target,
|
||||
isAverage: false,
|
||||
isAverage: true,
|
||||
}),
|
||||
externalMetricSpecWithTarget("external_metric_2", externalMetricTarget{
|
||||
externalMetricSpecWithTarget("external_metric_2", f.Namespace.ObjectMeta.Name, externalMetricTarget{
|
||||
value: metric2Target,
|
||||
isAverage: false,
|
||||
isAverage: true,
|
||||
}),
|
||||
}
|
||||
containers := []monitoring.CustomMetricContainerSpec{
|
||||
@ -277,9 +274,9 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
// Second metric is external metric which is present, it should cause scale up.
|
||||
metricSpecs := []autoscalingv2.MetricSpec{
|
||||
podMetricSpecWithAverageValueTarget(monitoring.CustomMetricName, 2*externalMetricValue),
|
||||
externalMetricSpecWithTarget("external_metric", externalMetricTarget{
|
||||
externalMetricSpecWithTarget("external_metric", f.Namespace.ObjectMeta.Name, externalMetricTarget{
|
||||
value: int64(math.Ceil(0.5 * float64(externalMetricValue))),
|
||||
isAverage: false,
|
||||
isAverage: true,
|
||||
}),
|
||||
}
|
||||
containers := []monitoring.CustomMetricContainerSpec{
|
||||
@ -307,7 +304,7 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
// Second metric is object metric which is present, it should cause scale up.
|
||||
metricSpecs := []autoscalingv2.MetricSpec{
|
||||
resourceMetricSpecWithAverageUtilizationTarget(50),
|
||||
objectMetricSpecWithAverageValueTarget(int64(math.Ceil(0.5 * float64(metricValue)))),
|
||||
objectMetricSpecWithValueTarget(int64(math.Ceil(0.5 * float64(metricValue)))),
|
||||
}
|
||||
tc := CustomMetricTestCase{
|
||||
framework: f,
|
||||
@ -326,9 +323,9 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
// Second metric is external metric which is present, it should cause scale down if the first metric wasn't missing.
|
||||
metricSpecs := []autoscalingv2.MetricSpec{
|
||||
containerResourceMetricSpecWithAverageUtilizationTarget("container-resource-metric", 50),
|
||||
externalMetricSpecWithTarget("external_metric", externalMetricTarget{
|
||||
externalMetricSpecWithTarget("external_metric", f.Namespace.ObjectMeta.Name, externalMetricTarget{
|
||||
value: 2 * externalMetricValue,
|
||||
isAverage: false,
|
||||
isAverage: true,
|
||||
}),
|
||||
}
|
||||
containers := []monitoring.CustomMetricContainerSpec{
|
||||
@ -356,7 +353,7 @@ var _ = SIGDescribe("[HPA] [Feature:CustomMetricsAutoscaling] Horizontal pod aut
|
||||
// First metric an object metric which is missing.
|
||||
// Second metric is pod metric which is present, it should cause scale down if the first metric wasn't missing.
|
||||
metricSpecs := []autoscalingv2.MetricSpec{
|
||||
objectMetricSpecWithAverageValueTarget(int64(math.Ceil(0.5 * float64(metricValue)))),
|
||||
objectMetricSpecWithValueTarget(int64(math.Ceil(0.5 * float64(metricValue)))),
|
||||
podMetricSpecWithAverageValueTarget("pod_metric", 2*metricValue),
|
||||
}
|
||||
containers := []monitoring.CustomMetricContainerSpec{
|
||||
@ -495,7 +492,7 @@ func podMetricSpecWithAverageValueTarget(metric string, targetValue int64) autos
|
||||
}
|
||||
}
|
||||
|
||||
func objectMetricSpecWithAverageValueTarget(targetValue int64) autoscalingv2.MetricSpec {
|
||||
func objectMetricSpecWithValueTarget(targetValue int64) autoscalingv2.MetricSpec {
|
||||
return autoscalingv2.MetricSpec{
|
||||
Type: autoscalingv2.ObjectMetricSourceType,
|
||||
Object: &autoscalingv2.ObjectMetricSource{
|
||||
@ -541,19 +538,17 @@ func containerResourceMetricSpecWithAverageUtilizationTarget(containerName strin
|
||||
}
|
||||
}
|
||||
|
||||
func externalMetricSpecWithTarget(metric string, target externalMetricTarget) autoscalingv2.MetricSpec {
|
||||
func externalMetricSpecWithTarget(metric string, namespace string, target externalMetricTarget) autoscalingv2.MetricSpec {
|
||||
selector := &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{"resource.type": "gke_container"},
|
||||
MatchLabels: map[string]string{"resource.type": "k8s_pod"},
|
||||
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||
{
|
||||
Key: "resource.labels.namespace_id",
|
||||
Key: "resource.labels.namespace_name",
|
||||
Operator: metav1.LabelSelectorOpIn,
|
||||
// TODO(bskiba): change default to real namespace name once it is available
|
||||
// from Stackdriver.
|
||||
Values: []string{"default", "dummy"},
|
||||
Values: []string{namespace},
|
||||
},
|
||||
{
|
||||
Key: "resource.labels.pod_id",
|
||||
Key: "resource.labels.pod_name",
|
||||
Operator: metav1.LabelSelectorOpExists,
|
||||
Values: []string{},
|
||||
},
|
||||
@ -637,3 +632,18 @@ func ensureDesiredReplicasInRange(ctx context.Context, deploymentName, namespace
|
||||
}
|
||||
framework.ExpectNoErrorWithOffset(1, err)
|
||||
}
|
||||
|
||||
func noExporterDeployment(name, namespace string, replicas int32) *appsv1.Deployment {
|
||||
d := e2edeployment.NewDeployment(name, replicas, map[string]string{"name": name}, "", "", appsv1.RollingUpdateDeploymentStrategyType)
|
||||
d.ObjectMeta.Namespace = namespace
|
||||
d.Spec.Template.Spec = v1.PodSpec{Containers: []v1.Container{
|
||||
{
|
||||
Name: "sleeper",
|
||||
Image: "k8s.gcr.io/ubuntu-slim:0.1",
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Command: []string{"/bin/sh"},
|
||||
Args: []string{"-c", "sleep 1d"}, // effectively forever
|
||||
},
|
||||
}}
|
||||
return d
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user