From b06a5a60279355de1804cc360f28978a92c01ec8 Mon Sep 17 00:00:00 2001 From: Beata Skiba Date: Wed, 6 Dec 2017 10:14:58 +0100 Subject: [PATCH] Add custom metrics e2e test with two metrics. Tests a scenario where a pod is scaled based on two custom metrics. --- .../autoscaling/custom_metrics_autoscaling.go | 81 ++++++++++++++----- test/e2e/instrumentation/monitoring/BUILD | 1 - .../monitoring/custom_metrics_deployments.go | 76 +++++++++++------ 3 files changed, 112 insertions(+), 46 deletions(-) diff --git a/test/e2e/autoscaling/custom_metrics_autoscaling.go b/test/e2e/autoscaling/custom_metrics_autoscaling.go index 15ceadcc35d..5d002bf5055 100644 --- a/test/e2e/autoscaling/custom_metrics_autoscaling.go +++ b/test/e2e/autoscaling/custom_metrics_autoscaling.go @@ -51,23 +51,58 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me It("should scale down with Custom Metric of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 scaledReplicas := 1 - deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100) - customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas) + // metric should cause scale down + metricValue := int64(100) + metricTarget := 2 * metricValue + deployment := monitoring.SimpleStackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue) + customMetricTest(f, f.ClientSet, simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, nil, initialReplicas, scaledReplicas) }) It("should scale down with Custom Metric of type Object from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 scaledReplicas := 1 - deployment := monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100) - pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.ObjectMeta.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100) - customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name), deployment, pod, initialReplicas, scaledReplicas) + // metric should cause scale down + metricValue := int64(100) + metricTarget := 2 * metricValue + deployment := monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue) + pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue) + customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, pod, initialReplicas, scaledReplicas) }) It("should scale down with Custom Metric of type Pod from Stackdriver with Prometheus [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 scaledReplicas := 1 - deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100) - customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas) + // metric should cause scale down + metricValue := int64(100) + metricTarget := 2 * metricValue + deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue) + customMetricTest(f, f.ClientSet, simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, nil, initialReplicas, scaledReplicas) + }) + + It("should scale up with two metrics of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + initialReplicas := 1 + scaledReplicas := 3 + // metric 1 would cause a scale down, if not for metric 2 + metric1Value := int64(100) + metric1Target := 2 * metric1Value + // metric2 should cause a scale up + metric2Value := int64(200) + metric2Target := int64(0.5 * float64(metric2Value)) + containers := []monitoring.CustomMetricContainerSpec{ + { + Name: "stackdriver-exporter-metric1", + MetricName: "metric1", + MetricValue: metric1Value, + }, + { + Name: "stackdriver-exporter-metric2", + MetricName: "metric2", + MetricValue: metric2Value, + }, + } + metricTargets := map[string]int64{"metric1": metric1Target, "metric2": metric2Target} + deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers) + customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, metricTargets), deployment, nil, initialReplicas, scaledReplicas) }) }) @@ -153,35 +188,41 @@ func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface, d } } -func podsHPA(namespace string) *as.HorizontalPodAutoscaler { +func simplePodsHPA(namespace string, metricTarget int64) *as.HorizontalPodAutoscaler { + return podsHPA(namespace, stackdriverExporterDeployment, map[string]int64{monitoring.CustomMetricName: metricTarget}) +} + +func podsHPA(namespace string, deploymentName string, metricTargets map[string]int64) *as.HorizontalPodAutoscaler { var minReplicas int32 = 1 + metrics := []as.MetricSpec{} + for metric, target := range metricTargets { + metrics = append(metrics, as.MetricSpec{ + Type: as.PodsMetricSourceType, + Pods: &as.PodsMetricSource{ + MetricName: metric, + TargetAverageValue: *resource.NewQuantity(target, resource.DecimalSI), + }, + }) + } return &as.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-metrics-pods-hpa", Namespace: namespace, }, Spec: as.HorizontalPodAutoscalerSpec{ - Metrics: []as.MetricSpec{ - { - Type: as.PodsMetricSourceType, - Pods: &as.PodsMetricSource{ - MetricName: monitoring.CustomMetricName, - TargetAverageValue: *resource.NewQuantity(200, resource.DecimalSI), - }, - }, - }, + Metrics: metrics, MaxReplicas: 3, MinReplicas: &minReplicas, ScaleTargetRef: as.CrossVersionObjectReference{ APIVersion: "extensions/v1beta1", Kind: "Deployment", - Name: stackdriverExporterDeployment, + Name: deploymentName, }, }, } } -func objectHPA(namespace string) *as.HorizontalPodAutoscaler { +func objectHPA(namespace string, metricTarget int64) *as.HorizontalPodAutoscaler { var minReplicas int32 = 1 return &as.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ @@ -198,7 +239,7 @@ func objectHPA(namespace string) *as.HorizontalPodAutoscaler { Kind: "Pod", Name: stackdriverExporterPod, }, - TargetValue: *resource.NewQuantity(200, resource.DecimalSI), + TargetValue: *resource.NewQuantity(metricTarget, resource.DecimalSI), }, }, }, diff --git a/test/e2e/instrumentation/monitoring/BUILD b/test/e2e/instrumentation/monitoring/BUILD index 65f123e0e51..4a0523411c5 100644 --- a/test/e2e/instrumentation/monitoring/BUILD +++ b/test/e2e/instrumentation/monitoring/BUILD @@ -21,7 +21,6 @@ go_library( "//test/e2e/framework:go_default_library", "//test/e2e/framework/metrics:go_default_library", "//test/e2e/instrumentation/common:go_default_library", - "//test/utils/image:go_default_library", "//vendor/github.com/influxdata/influxdb/client/v2:go_default_library", "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go index b18a08dd5cd..b6e861575ee 100644 --- a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go +++ b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go @@ -25,14 +25,14 @@ import ( rbac "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" - imageutils "k8s.io/kubernetes/test/utils/image" ) var ( - CustomMetricName = "foo" - UnusedMetricName = "unused" - CustomMetricValue = int64(448) - UnusedMetricValue = int64(446) + CustomMetricName = "foo" + UnusedMetricName = "unused" + CustomMetricValue = int64(448) + UnusedMetricValue = int64(446) + StackdriverExporter = "stackdriver-exporter" // HPAPermissions is a ClusterRoleBinding that grants unauthenticated user permissions granted for // HPA for testing purposes, i.e. it should grant permission to read custom metrics. HPAPermissions = &rbac.ClusterRoleBinding{ @@ -54,9 +54,37 @@ var ( } ) -// StackdriverExporterDeployment is a Deployment of simple application that exports a metric of +// CustomMetricContainerSpec allows to specify a config for StackdriverExporterDeployment +// with multiple containers exporting different metrics. +type CustomMetricContainerSpec struct { + Name string + MetricName string + MetricValue int64 +} + +// SimpleStackdriverExporterDeployment is a Deployment of simple application that exports a metric of // fixed value to Stackdriver in a loop. -func StackdriverExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment { +func SimpleStackdriverExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment { + return StackdriverExporterDeployment(name, namespace, replicas, + []CustomMetricContainerSpec{ + { + Name: StackdriverExporter, + MetricName: CustomMetricName, + MetricValue: metricValue, + }, + }) +} + +// StackdriverExporterDeployment is a Deployment of an application that can expose +// an arbitrary amount of metrics of fixed value to Stackdriver in a loop. Each metric +// is exposed by a different container in one pod. +// The metric names and values are configured via the containers parameter. +func StackdriverExporterDeployment(name, namespace string, replicas int32, containers []CustomMetricContainerSpec) *extensions.Deployment { + podSpec := corev1.PodSpec{Containers: []corev1.Container{}} + for _, containerSpec := range containers { + podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, containerSpec.MetricName, containerSpec.MetricValue)) + } + return &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -72,7 +100,7 @@ func StackdriverExporterDeployment(name, namespace string, replicas int32, metri "name": name, }, }, - Spec: stackdriverExporterPodSpec(CustomMetricName, metricValue), + Spec: podSpec, }, Replicas: &replicas, }, @@ -90,31 +118,29 @@ func StackdriverExporterPod(podName, namespace, podLabel, metricName string, met "name": podLabel, }, }, - Spec: stackdriverExporterPodSpec(metricName, metricValue), + Spec: corev1.PodSpec{ + Containers: []corev1.Container{stackdriverExporterContainerSpec(StackdriverExporter, metricName, metricValue)}, + }, } } -func stackdriverExporterPodSpec(metricName string, metricValue int64) corev1.PodSpec { - return corev1.PodSpec{ - Containers: []corev1.Container{ +func stackdriverExporterContainerSpec(name string, metricName string, metricValue int64) corev1.Container { + return corev1.Container{ + Name: name, + Image: "gcr.io/google-containers/sd-dummy-exporter:v0.1.0", + ImagePullPolicy: corev1.PullPolicy("Always"), + Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)}, + Env: []corev1.EnvVar{ { - Name: "stackdriver-exporter", - Image: imageutils.GetE2EImage(imageutils.SDDummyExporter), - ImagePullPolicy: corev1.PullPolicy("Always"), - Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)}, - Env: []corev1.EnvVar{ - { - Name: "POD_ID", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.uid", - }, - }, + Name: "POD_ID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", }, }, - Ports: []corev1.ContainerPort{{ContainerPort: 80}}, }, }, + Ports: []corev1.ContainerPort{{ContainerPort: 80}}, } }