Add custom metrics e2e test with two metrics.

Tests a scenario where a pod is scaled based on two custom metrics.

test/e2e/autoscaling/custom_metrics_autoscaling.go

@@ -51,23 +51,58 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me
 	It("should scale down with Custom Metric of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
 		initialReplicas := 2
 		scaledReplicas := 1
-		deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
+		// metric should cause scale down
-		customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas)
+		metricValue := int64(100)
+		metricTarget := 2 * metricValue
+		deployment := monitoring.SimpleStackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue)
+		customMetricTest(f, f.ClientSet, simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, nil, initialReplicas, scaledReplicas)
 	})
 
 	It("should scale down with Custom Metric of type Object from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
 		initialReplicas := 2
 		scaledReplicas := 1
-		deployment := monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
+		// metric should cause scale down
-		pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.ObjectMeta.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100)
+		metricValue := int64(100)
-		customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name), deployment, pod, initialReplicas, scaledReplicas)
+		metricTarget := 2 * metricValue
+		deployment := monitoring.SimpleStackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue)
+		pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, metricValue)
+		customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, pod, initialReplicas, scaledReplicas)
 	})
 
 	It("should scale down with Custom Metric of type Pod from Stackdriver with Prometheus [Feature:CustomMetricsAutoscaling]", func() {
 		initialReplicas := 2
 		scaledReplicas := 1
-		deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
+		// metric should cause scale down
-		customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas)
+		metricValue := int64(100)
+		metricTarget := 2 * metricValue
+		deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), metricValue)
+		customMetricTest(f, f.ClientSet, simplePodsHPA(f.Namespace.ObjectMeta.Name, metricTarget), deployment, nil, initialReplicas, scaledReplicas)
+	})
+
+	It("should scale up with two metrics of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
+		initialReplicas := 1
+		scaledReplicas := 3
+		// metric 1 would cause a scale down, if not for metric 2
+		metric1Value := int64(100)
+		metric1Target := 2 * metric1Value
+		// metric2 should cause a scale up
+		metric2Value := int64(200)
+		metric2Target := int64(0.5 * float64(metric2Value))
+		containers := []monitoring.CustomMetricContainerSpec{
+			{
+				Name:        "stackdriver-exporter-metric1",
+				MetricName:  "metric1",
+				MetricValue: metric1Value,
+			},
+			{
+				Name:        "stackdriver-exporter-metric2",
+				MetricName:  "metric2",
+				MetricValue: metric2Value,
+			},
+		}
+		metricTargets := map[string]int64{"metric1": metric1Target, "metric2": metric2Target}
+		deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), containers)
+		customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name, stackdriverExporterDeployment, metricTargets), deployment, nil, initialReplicas, scaledReplicas)
 	})
 })
 
@@ -153,35 +188,41 @@ func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface, d
 	}
 }
 
-func podsHPA(namespace string) *as.HorizontalPodAutoscaler {
+func simplePodsHPA(namespace string, metricTarget int64) *as.HorizontalPodAutoscaler {
+	return podsHPA(namespace, stackdriverExporterDeployment, map[string]int64{monitoring.CustomMetricName: metricTarget})
+}
+
+func podsHPA(namespace string, deploymentName string, metricTargets map[string]int64) *as.HorizontalPodAutoscaler {
 	var minReplicas int32 = 1
+	metrics := []as.MetricSpec{}
+	for metric, target := range metricTargets {
+		metrics = append(metrics, as.MetricSpec{
+			Type: as.PodsMetricSourceType,
+			Pods: &as.PodsMetricSource{
+				MetricName:         metric,
+				TargetAverageValue: *resource.NewQuantity(target, resource.DecimalSI),
+			},
+		})
+	}
 	return &as.HorizontalPodAutoscaler{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "custom-metrics-pods-hpa",
 			Namespace: namespace,
 		},
 		Spec: as.HorizontalPodAutoscalerSpec{
-			Metrics: []as.MetricSpec{
+			Metrics:     metrics,
-				{
-					Type: as.PodsMetricSourceType,
-					Pods: &as.PodsMetricSource{
-						MetricName:         monitoring.CustomMetricName,
-						TargetAverageValue: *resource.NewQuantity(200, resource.DecimalSI),
-					},
-				},
-			},
 			MaxReplicas: 3,
 			MinReplicas: &minReplicas,
 			ScaleTargetRef: as.CrossVersionObjectReference{
 				APIVersion: "extensions/v1beta1",
 				Kind:       "Deployment",
-				Name:       stackdriverExporterDeployment,
+				Name:       deploymentName,
 			},
 		},
 	}
 }
 
-func objectHPA(namespace string) *as.HorizontalPodAutoscaler {
+func objectHPA(namespace string, metricTarget int64) *as.HorizontalPodAutoscaler {
 	var minReplicas int32 = 1
 	return &as.HorizontalPodAutoscaler{
 		ObjectMeta: metav1.ObjectMeta{
@@ -198,7 +239,7 @@ func objectHPA(namespace string) *as.HorizontalPodAutoscaler {
 							Kind: "Pod",
 							Name: stackdriverExporterPod,
 						},
-						TargetValue: *resource.NewQuantity(200, resource.DecimalSI),
+						TargetValue: *resource.NewQuantity(metricTarget, resource.DecimalSI),
 					},
 				},
 			},

test/e2e/instrumentation/monitoring/BUILD

@@ -21,7 +21,6 @@ go_library(
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
         "//test/e2e/instrumentation/common:go_default_library",
-        "//test/utils/image:go_default_library",
         "//vendor/github.com/influxdata/influxdb/client/v2:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",

test/e2e/instrumentation/monitoring/custom_metrics_deployments.go

@@ -25,7 +25,6 @@ import (
 	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
-	imageutils "k8s.io/kubernetes/test/utils/image"
 )
 
 var (
@@ -33,6 +32,7 @@ var (
 	UnusedMetricName    = "unused"
 	CustomMetricValue   = int64(448)
 	UnusedMetricValue   = int64(446)
+	StackdriverExporter = "stackdriver-exporter"
 	// HPAPermissions is a ClusterRoleBinding that grants unauthenticated user permissions granted for
 	// HPA for testing purposes, i.e. it should grant permission to read custom metrics.
 	HPAPermissions = &rbac.ClusterRoleBinding{
@@ -54,9 +54,37 @@ var (
 	}
 )
 
-// StackdriverExporterDeployment is a Deployment of simple application that exports a metric of
+// CustomMetricContainerSpec allows to specify a config for StackdriverExporterDeployment
+// with multiple containers exporting different metrics.
+type CustomMetricContainerSpec struct {
+	Name        string
+	MetricName  string
+	MetricValue int64
+}
+
+// SimpleStackdriverExporterDeployment is a Deployment of simple application that exports a metric of
 // fixed value to Stackdriver in a loop.
-func StackdriverExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment {
+func SimpleStackdriverExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment {
+	return StackdriverExporterDeployment(name, namespace, replicas,
+		[]CustomMetricContainerSpec{
+			{
+				Name:        StackdriverExporter,
+				MetricName:  CustomMetricName,
+				MetricValue: metricValue,
+			},
+		})
+}
+
+// StackdriverExporterDeployment is a Deployment of an application that can expose
+// an arbitrary amount of metrics of fixed value to Stackdriver in a loop. Each metric
+// is exposed by a different container in one pod.
+// The metric names and values are configured via the containers parameter.
+func StackdriverExporterDeployment(name, namespace string, replicas int32, containers []CustomMetricContainerSpec) *extensions.Deployment {
+	podSpec := corev1.PodSpec{Containers: []corev1.Container{}}
+	for _, containerSpec := range containers {
+		podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, containerSpec.MetricName, containerSpec.MetricValue))
+	}
+
 	return &extensions.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -72,7 +100,7 @@ func StackdriverExporterDeployment(name, namespace string, replicas int32, metri
 						"name": name,
 					},
 				},
-				Spec: stackdriverExporterPodSpec(CustomMetricName, metricValue),
+				Spec: podSpec,
 			},
 			Replicas: &replicas,
 		},
@@ -90,16 +118,16 @@ func StackdriverExporterPod(podName, namespace, podLabel, metricName string, met
 				"name": podLabel,
 			},
 		},
-		Spec: stackdriverExporterPodSpec(metricName, metricValue),
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{stackdriverExporterContainerSpec(StackdriverExporter, metricName, metricValue)},
+		},
 	}
 }
 
-func stackdriverExporterPodSpec(metricName string, metricValue int64) corev1.PodSpec {
+func stackdriverExporterContainerSpec(name string, metricName string, metricValue int64) corev1.Container {
-	return corev1.PodSpec{
+	return corev1.Container{
-		Containers: []corev1.Container{
+		Name:            name,
-			{
+		Image:           "gcr.io/google-containers/sd-dummy-exporter:v0.1.0",
-				Name:            "stackdriver-exporter",
-				Image:           imageutils.GetE2EImage(imageutils.SDDummyExporter),
 		ImagePullPolicy: corev1.PullPolicy("Always"),
 		Command:         []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)},
 		Env: []corev1.EnvVar{
@@ -113,8 +141,6 @@ func stackdriverExporterPodSpec(metricName string, metricValue int64) corev1.Pod
 			},
 		},
 		Ports: []corev1.ContainerPort{{ContainerPort: 80}},
-			},
-		},
 	}
 }