From 896cf2180e181308199f120902c48b8dfc7e406e Mon Sep 17 00:00:00 2001 From: Beata Skiba Date: Tue, 27 Mar 2018 11:26:40 +0200 Subject: [PATCH] Add e2e test for external metrics with Stackdriver Rename the file to note that these are Stackdriver tests in anticipation of tests with fake custom metrics apiserver. Refactor the tests to be more structured. --- test/e2e/autoscaling/BUILD | 2 +- ...custom_metrics_stackdriver_autoscaling.go} | 220 ++++++++++++++++-- 2 files changed, 201 insertions(+), 21 deletions(-) rename test/e2e/autoscaling/{custom_metrics_autoscaling.go => custom_metrics_stackdriver_autoscaling.go} (50%) diff --git a/test/e2e/autoscaling/BUILD b/test/e2e/autoscaling/BUILD index f80aeee6817..36f7f3f5cc3 100644 --- a/test/e2e/autoscaling/BUILD +++ b/test/e2e/autoscaling/BUILD @@ -11,7 +11,7 @@ go_library( "autoscaling_timer.go", "cluster_autoscaler_scalability.go", "cluster_size_autoscaling.go", - "custom_metrics_autoscaling.go", + "custom_metrics_stackdriver_autoscaling.go", "dns_autoscaling.go", "framework.go", "horizontal_pod_autoscaling.go", diff --git a/test/e2e/autoscaling/custom_metrics_autoscaling.go b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go similarity index 50% rename from test/e2e/autoscaling/custom_metrics_autoscaling.go rename to test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go index ce29b452b4f..5277cb9c5df 100644 --- a/test/e2e/autoscaling/custom_metrics_autoscaling.go +++ b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go @@ -18,6 +18,7 @@ package autoscaling import ( "context" + "math" "time" gcm "google.golang.org/api/monitoring/v3" @@ -39,6 +40,7 @@ const ( stackdriverExporterDeployment = "stackdriver-exporter-deployment" dummyDeploymentName = "dummy-deployment" stackdriverExporterPod = "stackdriver-exporter-pod" + externalMetricValue = int64(85) ) var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Metrics from Stackdriver)", func() { @@ -50,38 +52,99 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me It("should scale down with Custom Metric of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 - scaledReplicas := 1 // metric should cause scale down metricValue := int64(100) metricTarget := 2 * metricValue - deployment := monitoring.SimpleStackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue) - customMetricTest(f, f.ClientSet, simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, nil, initialReplicas, scaledReplicas) + tc := CustomMetricTestCase{ + framework: f, + kubeClient: f.ClientSet, + initialReplicas: initialReplicas, + scaledReplicas: 1, + deployment: monitoring.SimpleStackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue), + hpa: simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget)} + tc.Run() }) It("should scale down with Custom Metric of type Object from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 - scaledReplicas := 1 // metric should cause scale down metricValue := int64(100) metricTarget := 2 * metricValue - deployment := monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue) - pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue) - customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, pod, initialReplicas, scaledReplicas) + tc := CustomMetricTestCase{ + framework: f, + kubeClient: f.ClientSet, + initialReplicas: initialReplicas, + scaledReplicas: 1, + // Metric exported by deployment is ignored + deployment: monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 0 /* ignored */), + pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue), + hpa: objectHPA(f.Namespace.ObjectMeta.Name, metricTarget)} + tc.Run() + }) + + It("should scale down with External Metric with target value from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + initialReplicas := 2 + // metric should cause scale down + metricValue := externalMetricValue + metricTarget := 2 * metricValue + metricTargets := map[string]externalMetricTarget{ + "target": { + value: metricTarget, + isAverage: false, + }, + } + tc := CustomMetricTestCase{ + framework: f, + kubeClient: f.ClientSet, + initialReplicas: initialReplicas, + scaledReplicas: 1, + // Metric exported by deployment is ignored + deployment: monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 0 /* ignored */), + pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, "target", metricValue), + hpa: externalHPA(f.Namespace.ObjectMeta.Name, metricTargets)} + tc.Run() + }) + + It("should scale down with External Metric with target average value from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + initialReplicas := 2 + // metric should cause scale down + metricValue := externalMetricValue + metricAverageTarget := 2 * metricValue + metricTargets := map[string]externalMetricTarget{ + "target_average": { + value: metricAverageTarget, + isAverage: true, + }, + } + tc := CustomMetricTestCase{ + framework: f, + kubeClient: f.ClientSet, + initialReplicas: initialReplicas, + scaledReplicas: 1, + // Metric exported by deployment is ignored + deployment: monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 0 /* ignored */), + pod: monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, "target_average", externalMetricValue), + hpa: externalHPA(f.Namespace.ObjectMeta.Name, metricTargets)} + tc.Run() }) It("should scale down with Custom Metric of type Pod from Stackdriver with Prometheus [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 2 - scaledReplicas := 1 // metric should cause scale down metricValue := int64(100) metricTarget := 2 * metricValue - deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue) - customMetricTest(f, f.ClientSet, simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, nil, initialReplicas, scaledReplicas) + tc := CustomMetricTestCase{ + framework: f, + kubeClient: f.ClientSet, + initialReplicas: initialReplicas, + scaledReplicas: 1, + deployment: monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue), + hpa: simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget)} + tc.Run() }) It("should scale up with two metrics of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { initialReplicas := 1 - scaledReplicas := 3 // metric 1 would cause a scale down, if not for metric 2 metric1Value := int64(100) metric1Target := 2 * metric1Value @@ -101,13 +164,68 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me }, } metricTargets := map[string]int64{"metric1": metric1Target, "metric2": metric2Target} - deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers) - customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, metricTargets), deployment, nil, initialReplicas, scaledReplicas) + tc := CustomMetricTestCase{ + framework: f, + kubeClient: f.ClientSet, + initialReplicas: initialReplicas, + scaledReplicas: 3, + deployment: monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers), + hpa: podsHPA(f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, metricTargets)} + tc.Run() + }) + + It("should scale up with two External metrics from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + initialReplicas := 1 + // metric 1 would cause a scale down, if not for metric 2 + metric1Value := externalMetricValue + metric1Target := 2 * metric1Value + // metric2 should cause a scale up + metric2Value := externalMetricValue + metric2Target := int64(math.Ceil(0.5 * float64(metric2Value))) + metricTargets := map[string]externalMetricTarget{ + "external_metric_1": { + value: metric1Target, + isAverage: false, + }, + "external_metric_2": { + value: metric2Target, + isAverage: false, + }, + } + containers := []monitoring.CustomMetricContainerSpec{ + { + Name: "stackdriver-exporter-metric1", + MetricName: "external_metric_1", + MetricValue: metric1Value, + }, + { + Name: "stackdriver-exporter-metric2", + MetricName: "external_metric_2", + MetricValue: metric2Value, + }, + } + tc := CustomMetricTestCase{ + framework: f, + kubeClient: f.ClientSet, + initialReplicas: initialReplicas, + scaledReplicas: 3, + deployment: monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers), + hpa: externalHPA(f.Namespace.ObjectMeta.Name, metricTargets)} + tc.Run() }) }) -func customMetricTest(f *framework.Framework, kubeClient clientset.Interface, hpa *as.HorizontalPodAutoscaler, - deployment *extensions.Deployment, pod *corev1.Pod, initialReplicas, scaledReplicas int) { +type CustomMetricTestCase struct { + framework *framework.Framework + hpa *as.HorizontalPodAutoscaler + kubeClient clientset.Interface + deployment *extensions.Deployment + pod *corev1.Pod + initialReplicas int + scaledReplicas int +} + +func (tc *CustomMetricTestCase) Run() { projectId := framework.TestContext.CloudConfig.ProjectID ctx := context.Background() @@ -145,22 +263,23 @@ func customMetricTest(f *framework.Framework, kubeClient clientset.Interface, hp defer monitoring.CleanupAdapter(monitoring.AdapterDefault) // Run application that exports the metric - err = createDeploymentToScale(f, kubeClient, deployment, pod) + err = createDeploymentToScale(tc.framework, tc.kubeClient, tc.deployment, tc.pod) if err != nil { framework.Failf("Failed to create stackdriver-exporter pod: %v", err) } - defer cleanupDeploymentsToScale(f, kubeClient, deployment, pod) + defer cleanupDeploymentsToScale(tc.framework, tc.kubeClient, tc.deployment, tc.pod) // Wait for the deployment to run - waitForReplicas(deployment.ObjectMeta.Name, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, initialReplicas) + waitForReplicas(tc.deployment.ObjectMeta.Name, tc.framework.Namespace.ObjectMeta.Name, tc.kubeClient, 15*time.Minute, tc.initialReplicas) // Autoscale the deployment - _, err = kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(hpa) + _, err = tc.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(tc.framework.Namespace.ObjectMeta.Name).Create(tc.hpa) if err != nil { framework.Failf("Failed to create HPA: %v", err) } + defer tc.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(tc.framework.Namespace.ObjectMeta.Name).Delete(tc.hpa.ObjectMeta.Name, &metav1.DeleteOptions{}) - waitForReplicas(deployment.ObjectMeta.Name, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, scaledReplicas) + waitForReplicas(tc.deployment.ObjectMeta.Name, tc.framework.Namespace.ObjectMeta.Name, tc.kubeClient, 15*time.Minute, tc.scaledReplicas) } func createDeploymentToScale(f *framework.Framework, cs clientset.Interface, deployment *extensions.Deployment, pod *corev1.Pod) error { @@ -254,6 +373,67 @@ func objectHPA(namespace string, metricTarget int64) *as.HorizontalPodAutoscaler } } +type externalMetricTarget struct { + value int64 + isAverage bool +} + +func externalHPA(namespace string, metricTargets map[string]externalMetricTarget) *as.HorizontalPodAutoscaler { + var minReplicas int32 = 1 + metricSpecs := []as.MetricSpec{} + selector := &metav1.LabelSelector{ + MatchLabels: map[string]string{"resource.type": "gke_container"}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "resource.labels.namespace_id", + Operator: metav1.LabelSelectorOpIn, + // TODO(bskiba): change default to real namespace name once it is available + // from Stackdriver. + Values: []string{"default", "dummy"}, + }, + { + Key: "resource.labels.pod_id", + Operator: metav1.LabelSelectorOpExists, + Values: []string{}, + }, + }, + } + for metric, target := range metricTargets { + var metricSpec as.MetricSpec + metricSpec = as.MetricSpec{ + Type: as.ExternalMetricSourceType, + External: &as.ExternalMetricSource{ + MetricName: "custom.googleapis.com|" + metric, + MetricSelector: selector, + }, + } + if target.isAverage { + metricSpec.External.TargetAverageValue = resource.NewQuantity(target.value, resource.DecimalSI) + } else { + metricSpec.External.TargetValue = resource.NewQuantity(target.value, resource.DecimalSI) + } + metricSpecs = append(metricSpecs, metricSpec) + } + hpa := &as.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-metrics-external-hpa", + Namespace: namespace, + }, + Spec: as.HorizontalPodAutoscalerSpec{ + Metrics: metricSpecs, + MaxReplicas: 3, + MinReplicas: &minReplicas, + ScaleTargetRef: as.CrossVersionObjectReference{ + APIVersion: "extensions/v1beta1", + Kind: "Deployment", + Name: dummyDeploymentName, + }, + }, + } + + return hpa +} + func waitForReplicas(deploymentName, namespace string, cs clientset.Interface, timeout time.Duration, desiredReplicas int) { interval := 20 * time.Second err := wait.PollImmediate(interval, timeout, func() (bool, error) {