Add custom metrics e2e test with two metrics.

Tests a scenario where a pod is scaled based on two custom metrics.
This commit is contained in:
Beata Skiba 2017-12-06 10:14:58 +01:00
parent f302487942
commit b06a5a6027
3 changed files with 112 additions and 46 deletions

View File

@ -51,23 +51,58 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me
It("should scale down with Custom Metric of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
initialReplicas := 2
scaledReplicas := 1
deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas)
// metric should cause scale down
metricValue := int64(100)
metricTarget := 2 * metricValue
deployment := monitoring.SimpleStackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue)
customMetricTest(f, f.ClientSet, simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, nil, initialReplicas, scaledReplicas)
})
It("should scale down with Custom Metric of type Object from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
initialReplicas := 2
scaledReplicas := 1
deployment := monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.ObjectMeta.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100)
customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name), deployment, pod, initialReplicas, scaledReplicas)
// metric should cause scale down
metricValue := int64(100)
metricTarget := 2 * metricValue
deployment := monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue)
pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue)
customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, pod, initialReplicas, scaledReplicas)
})
It("should scale down with Custom Metric of type Pod from Stackdriver with Prometheus [Feature:CustomMetricsAutoscaling]", func() {
initialReplicas := 2
scaledReplicas := 1
deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas)
// metric should cause scale down
metricValue := int64(100)
metricTarget := 2 * metricValue
deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue)
customMetricTest(f, f.ClientSet, simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, nil, initialReplicas, scaledReplicas)
})
It("should scale up with two metrics of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
initialReplicas := 1
scaledReplicas := 3
// metric 1 would cause a scale down, if not for metric 2
metric1Value := int64(100)
metric1Target := 2 * metric1Value
// metric2 should cause a scale up
metric2Value := int64(200)
metric2Target := int64(0.5 * float64(metric2Value))
containers := []monitoring.CustomMetricContainerSpec{
{
Name: "stackdriver-exporter-metric1",
MetricName: "metric1",
MetricValue: metric1Value,
},
{
Name: "stackdriver-exporter-metric2",
MetricName: "metric2",
MetricValue: metric2Value,
},
}
metricTargets := map[string]int64{"metric1": metric1Target, "metric2": metric2Target}
deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers)
customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, metricTargets), deployment, nil, initialReplicas, scaledReplicas)
})
})
@ -153,35 +188,41 @@ func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface, d
}
}
func podsHPA(namespace string) *as.HorizontalPodAutoscaler {
func simplePodsHPA(namespace string, metricTarget int64) *as.HorizontalPodAutoscaler {
return podsHPA(namespace, stackdriverExporterDeployment, map[string]int64{monitoring.CustomMetricName: metricTarget})
}
func podsHPA(namespace string, deploymentName string, metricTargets map[string]int64) *as.HorizontalPodAutoscaler {
var minReplicas int32 = 1
metrics := []as.MetricSpec{}
for metric, target := range metricTargets {
metrics = append(metrics, as.MetricSpec{
Type: as.PodsMetricSourceType,
Pods: &as.PodsMetricSource{
MetricName: metric,
TargetAverageValue: *resource.NewQuantity(target, resource.DecimalSI),
},
})
}
return &as.HorizontalPodAutoscaler{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-metrics-pods-hpa",
Namespace: namespace,
},
Spec: as.HorizontalPodAutoscalerSpec{
Metrics: []as.MetricSpec{
{
Type: as.PodsMetricSourceType,
Pods: &as.PodsMetricSource{
MetricName: monitoring.CustomMetricName,
TargetAverageValue: *resource.NewQuantity(200, resource.DecimalSI),
},
},
},
Metrics: metrics,
MaxReplicas: 3,
MinReplicas: &minReplicas,
ScaleTargetRef: as.CrossVersionObjectReference{
APIVersion: "extensions/v1beta1",
Kind: "Deployment",
Name: stackdriverExporterDeployment,
Name: deploymentName,
},
},
}
}
func objectHPA(namespace string) *as.HorizontalPodAutoscaler {
func objectHPA(namespace string, metricTarget int64) *as.HorizontalPodAutoscaler {
var minReplicas int32 = 1
return &as.HorizontalPodAutoscaler{
ObjectMeta: metav1.ObjectMeta{
@ -198,7 +239,7 @@ func objectHPA(namespace string) *as.HorizontalPodAutoscaler {
Kind: "Pod",
Name: stackdriverExporterPod,
},
TargetValue: *resource.NewQuantity(200, resource.DecimalSI),
TargetValue: *resource.NewQuantity(metricTarget, resource.DecimalSI),
},
},
},

View File

@ -21,7 +21,6 @@ go_library(
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/metrics:go_default_library",
"//test/e2e/instrumentation/common:go_default_library",
"//test/utils/image:go_default_library",
"//vendor/github.com/influxdata/influxdb/client/v2:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",

View File

@ -25,14 +25,14 @@ import (
rbac "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
imageutils "k8s.io/kubernetes/test/utils/image"
)
var (
CustomMetricName = "foo"
UnusedMetricName = "unused"
CustomMetricValue = int64(448)
UnusedMetricValue = int64(446)
CustomMetricName = "foo"
UnusedMetricName = "unused"
CustomMetricValue = int64(448)
UnusedMetricValue = int64(446)
StackdriverExporter = "stackdriver-exporter"
// HPAPermissions is a ClusterRoleBinding that grants unauthenticated user permissions granted for
// HPA for testing purposes, i.e. it should grant permission to read custom metrics.
HPAPermissions = &rbac.ClusterRoleBinding{
@ -54,9 +54,37 @@ var (
}
)
// StackdriverExporterDeployment is a Deployment of simple application that exports a metric of
// CustomMetricContainerSpec allows to specify a config for StackdriverExporterDeployment
// with multiple containers exporting different metrics.
type CustomMetricContainerSpec struct {
Name string
MetricName string
MetricValue int64
}
// SimpleStackdriverExporterDeployment is a Deployment of simple application that exports a metric of
// fixed value to Stackdriver in a loop.
func StackdriverExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment {
func SimpleStackdriverExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment {
return StackdriverExporterDeployment(name, namespace, replicas,
[]CustomMetricContainerSpec{
{
Name: StackdriverExporter,
MetricName: CustomMetricName,
MetricValue: metricValue,
},
})
}
// StackdriverExporterDeployment is a Deployment of an application that can expose
// an arbitrary amount of metrics of fixed value to Stackdriver in a loop. Each metric
// is exposed by a different container in one pod.
// The metric names and values are configured via the containers parameter.
func StackdriverExporterDeployment(name, namespace string, replicas int32, containers []CustomMetricContainerSpec) *extensions.Deployment {
podSpec := corev1.PodSpec{Containers: []corev1.Container{}}
for _, containerSpec := range containers {
podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, containerSpec.MetricName, containerSpec.MetricValue))
}
return &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
@ -72,7 +100,7 @@ func StackdriverExporterDeployment(name, namespace string, replicas int32, metri
"name": name,
},
},
Spec: stackdriverExporterPodSpec(CustomMetricName, metricValue),
Spec: podSpec,
},
Replicas: &replicas,
},
@ -90,31 +118,29 @@ func StackdriverExporterPod(podName, namespace, podLabel, metricName string, met
"name": podLabel,
},
},
Spec: stackdriverExporterPodSpec(metricName, metricValue),
Spec: corev1.PodSpec{
Containers: []corev1.Container{stackdriverExporterContainerSpec(StackdriverExporter, metricName, metricValue)},
},
}
}
func stackdriverExporterPodSpec(metricName string, metricValue int64) corev1.PodSpec {
return corev1.PodSpec{
Containers: []corev1.Container{
func stackdriverExporterContainerSpec(name string, metricName string, metricValue int64) corev1.Container {
return corev1.Container{
Name: name,
Image: "gcr.io/google-containers/sd-dummy-exporter:v0.1.0",
ImagePullPolicy: corev1.PullPolicy("Always"),
Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)},
Env: []corev1.EnvVar{
{
Name: "stackdriver-exporter",
Image: imageutils.GetE2EImage(imageutils.SDDummyExporter),
ImagePullPolicy: corev1.PullPolicy("Always"),
Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)},
Env: []corev1.EnvVar{
{
Name: "POD_ID",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.uid",
},
},
Name: "POD_ID",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.uid",
},
},
Ports: []corev1.ContainerPort{{ContainerPort: 80}},
},
},
Ports: []corev1.ContainerPort{{ContainerPort: 80}},
}
}