diff --git a/test/e2e/autoscaling/BUILD b/test/e2e/autoscaling/BUILD index ef75e7d9a68..f80aeee6817 100644 --- a/test/e2e/autoscaling/BUILD +++ b/test/e2e/autoscaling/BUILD @@ -31,6 +31,7 @@ go_library( "//vendor/google.golang.org/api/monitoring/v3:go_default_library", "//vendor/k8s.io/api/autoscaling/v2beta1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/api/extensions/v1beta1:go_default_library", "//vendor/k8s.io/api/policy/v1beta1:go_default_library", "//vendor/k8s.io/api/scheduling/v1alpha1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", diff --git a/test/e2e/autoscaling/custom_metrics_autoscaling.go b/test/e2e/autoscaling/custom_metrics_autoscaling.go index f5409ec5cd5..15ceadcc35d 100644 --- a/test/e2e/autoscaling/custom_metrics_autoscaling.go +++ b/test/e2e/autoscaling/custom_metrics_autoscaling.go @@ -20,18 +20,19 @@ import ( "context" "time" - "golang.org/x/oauth2/google" - clientset "k8s.io/client-go/kubernetes" - - . "github.com/onsi/ginkgo" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/test/e2e/framework" - gcm "google.golang.org/api/monitoring/v3" as "k8s.io/api/autoscaling/v2beta1" + corev1 "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/instrumentation/monitoring" + + . "github.com/onsi/ginkgo" + "golang.org/x/oauth2/google" ) const ( @@ -46,15 +47,32 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me }) f := framework.NewDefaultFramework("horizontal-pod-autoscaling") - var kubeClient clientset.Interface - It("should autoscale with Custom Metrics from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { - kubeClient = f.ClientSet - testHPA(f, kubeClient) + It("should scale down with Custom Metric of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + initialReplicas := 2 + scaledReplicas := 1 + deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100) + customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas) + }) + + It("should scale down with Custom Metric of type Object from Stackdriver [Feature:CustomMetricsAutoscaling]", func() { + initialReplicas := 2 + scaledReplicas := 1 + deployment := monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100) + pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.ObjectMeta.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100) + customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name), deployment, pod, initialReplicas, scaledReplicas) + }) + + It("should scale down with Custom Metric of type Pod from Stackdriver with Prometheus [Feature:CustomMetricsAutoscaling]", func() { + initialReplicas := 2 + scaledReplicas := 1 + deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100) + customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas) }) }) -func testHPA(f *framework.Framework, kubeClient clientset.Interface) { +func customMetricTest(f *framework.Framework, kubeClient clientset.Interface, hpa *as.HorizontalPodAutoscaler, + deployment *extensions.Deployment, pod *corev1.Pod, initialReplicas, scaledReplicas int) { projectId := framework.TestContext.CloudConfig.ProjectID ctx := context.Background() @@ -92,51 +110,55 @@ func testHPA(f *framework.Framework, kubeClient clientset.Interface) { defer monitoring.CleanupAdapter() // Run application that exports the metric - err = createDeploymentsToScale(f, kubeClient) + err = createDeploymentToScale(f, kubeClient, deployment, pod) if err != nil { framework.Failf("Failed to create stackdriver-exporter pod: %v", err) } - defer cleanupDeploymentsToScale(f, kubeClient) + defer cleanupDeploymentsToScale(f, kubeClient, deployment, pod) - // Autoscale the deployments - err = createPodsHPA(f, kubeClient) + // Wait for the deployment to run + waitForReplicas(deployment.ObjectMeta.Name, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, initialReplicas) + + // Autoscale the deployment + _, err = kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(hpa) if err != nil { - framework.Failf("Failed to create 'Pods' HPA: %v", err) - } - err = createObjectHPA(f, kubeClient) - if err != nil { - framework.Failf("Failed to create 'Objects' HPA: %v", err) + framework.Failf("Failed to create HPA: %v", err) } - waitForReplicas(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1) - waitForReplicas(dummyDeploymentName, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1) + waitForReplicas(deployment.ObjectMeta.Name, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, scaledReplicas) } -func createDeploymentsToScale(f *framework.Framework, cs clientset.Interface) error { - _, err := cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.Name, 2, 100)) - if err != nil { - return err +func createDeploymentToScale(f *framework.Framework, cs clientset.Interface, deployment *extensions.Deployment, pod *corev1.Pod) error { + if deployment != nil { + _, err := cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Create(deployment) + if err != nil { + return err + } } - _, err = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100)) - if err != nil { - return err + if pod != nil { + _, err := cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Create(pod) + if err != nil { + return err + } } - _, err = cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.Name, 2, 100)) - return err + return nil } -func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface) { - _ = cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterDeployment, &metav1.DeleteOptions{}) - _ = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterPod, &metav1.DeleteOptions{}) - _ = cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Delete(dummyDeploymentName, &metav1.DeleteOptions{}) +func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface, deployment *extensions.Deployment, pod *corev1.Pod) { + if deployment != nil { + _ = cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Delete(deployment.ObjectMeta.Name, &metav1.DeleteOptions{}) + } + if pod != nil { + _ = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Delete(pod.ObjectMeta.Name, &metav1.DeleteOptions{}) + } } -func createPodsHPA(f *framework.Framework, cs clientset.Interface) error { +func podsHPA(namespace string) *as.HorizontalPodAutoscaler { var minReplicas int32 = 1 - _, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{ + return &as.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-metrics-pods-hpa", - Namespace: f.Namespace.ObjectMeta.Name, + Namespace: namespace, }, Spec: as.HorizontalPodAutoscalerSpec{ Metrics: []as.MetricSpec{ @@ -156,16 +178,15 @@ func createPodsHPA(f *framework.Framework, cs clientset.Interface) error { Name: stackdriverExporterDeployment, }, }, - }) - return err + } } -func createObjectHPA(f *framework.Framework, cs clientset.Interface) error { +func objectHPA(namespace string) *as.HorizontalPodAutoscaler { var minReplicas int32 = 1 - _, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{ + return &as.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-metrics-objects-hpa", - Namespace: f.Namespace.ObjectMeta.Name, + Namespace: namespace, }, Spec: as.HorizontalPodAutoscalerSpec{ Metrics: []as.MetricSpec{ @@ -189,8 +210,7 @@ func createObjectHPA(f *framework.Framework, cs clientset.Interface) error { Name: dummyDeploymentName, }, }, - }) - return err + } } func waitForReplicas(deploymentName, namespace string, cs clientset.Interface, timeout time.Duration, desiredReplicas int) { diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go index 2adae0d4b6c..05b66946c48 100644 --- a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go +++ b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go @@ -18,6 +18,7 @@ package monitoring import ( "fmt" + gcm "google.golang.org/api/monitoring/v3" corev1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" @@ -27,8 +28,8 @@ import ( ) var ( - CustomMetricName = "foo-metric" - UnusedMetricName = "unused-metric" + CustomMetricName = "foo" + UnusedMetricName = "unused" CustomMetricValue = int64(448) UnusedMetricValue = int64(446) // HPAPermissions is a ClusterRoleBinding that grants unauthenticated user permissions granted for @@ -116,6 +117,72 @@ func stackdriverExporterPodSpec(metricName string, metricValue int64) corev1.Pod } } +// PrometheusExporterDeployment is a Deployment of simple application with two containers +// one exposing a metric in prometheus fromat and second a prometheus-to-sd container +// that scrapes the metric and pushes it to stackdriver. +func PrometheusExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment { + return &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: extensions.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"name": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": name, + }, + }, + Spec: prometheusExporterPodSpec(CustomMetricName, metricValue, 8080), + }, + Replicas: &replicas, + }, + } +} + +func prometheusExporterPodSpec(metricName string, metricValue int64, port int32) corev1.PodSpec { + return corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "prometheus-exporter", + Image: "gcr.io/google-containers/prometheus-dummy-exporter:v0.1.0", + ImagePullPolicy: corev1.PullPolicy("Always"), + Command: []string{"/prometheus_dummy_exporter", "--metric-name=" + metricName, + fmt.Sprintf("--metric-value=%v", metricValue), fmt.Sprintf("=--port=%d", port)}, + Ports: []corev1.ContainerPort{{ContainerPort: port}}, + }, + { + Name: "prometheus-to-sd", + Image: "gcr.io/google-containers/prometheus-to-sd:v0.2.3", + ImagePullPolicy: corev1.PullPolicy("Always"), + Command: []string{"/monitor", fmt.Sprintf("--source=:http://localhost:%d", port), + "--stackdriver-prefix=custom.googleapis.com", "--pod-id=$(POD_ID)", "--namespace-id=$(POD_NAMESPACE)"}, + Env: []corev1.EnvVar{ + { + Name: "POD_ID", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.uid", + }, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + }, + }, + }, + } +} + // CreateAdapter creates Custom Metrics - Stackdriver adapter. func CreateAdapter() error { stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml")