Add e2e test for Custom Metrics API with new Stackdriver resource model

and External Metrics API.

test/e2e/autoscaling/custom_metrics_autoscaling.go

@@ -138,11 +138,11 @@ func customMetricTest(f *framework.Framework, kubeClient clientset.Interface, hp
 	}
 	defer monitoring.CleanupDescriptors(gcmService, projectId)
 
-	err = monitoring.CreateAdapter()
+	err = monitoring.CreateAdapter(monitoring.AdapterDefault)
 	if err != nil {
 		framework.Failf("Failed to set up: %v", err)
 	}
-	defer monitoring.CleanupAdapter()
+	defer monitoring.CleanupAdapter(monitoring.AdapterDefault)
 
 	// Run application that exports the metric
 	err = createDeploymentToScale(f, kubeClient, deployment, pod)

test/e2e/instrumentation/monitoring/BUILD

@@ -42,6 +42,7 @@ go_library(
         "//vendor/k8s.io/client-go/discovery:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
+        "//vendor/k8s.io/metrics/pkg/client/external_metrics:go_default_library",
     ],
 )
 

test/e2e/instrumentation/monitoring/custom_metrics_deployments.go

@@ -18,6 +18,7 @@ package monitoring
 
 import (
 	"fmt"
+	"strings"
 
 	gcm "google.golang.org/api/monitoring/v3"
 	corev1 "k8s.io/api/core/v1"
@@ -25,6 +26,7 @@ import (
 	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"os/exec"
 )
 
 var (
@@ -52,6 +54,10 @@ var (
 			},
 		},
 	}
+	StagingDeploymentsLocation = "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/staging/"
+	AdapterForOldResourceModel = "adapter_old_resource_model.yaml"
+	AdapterForNewResourceModel = "adapter_new_resource_model.yaml"
+	AdapterDefault             = AdapterForOldResourceModel
 )
 
 // CustomMetricContainerSpec allows to specify a config for StackdriverExporterDeployment
@@ -82,7 +88,7 @@ func SimpleStackdriverExporterDeployment(name, namespace string, replicas int32,
 func StackdriverExporterDeployment(name, namespace string, replicas int32, containers []CustomMetricContainerSpec) *extensions.Deployment {
 	podSpec := corev1.PodSpec{Containers: []corev1.Container{}}
 	for _, containerSpec := range containers {
-		podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, containerSpec.MetricName, containerSpec.MetricValue))
+		podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, namespace, containerSpec.MetricName, containerSpec.MetricValue))
 	}
 
 	return &extensions.Deployment{
@@ -119,17 +125,30 @@ func StackdriverExporterPod(podName, namespace, podLabel, metricName string, met
 			},
 		},
 		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{stackdriverExporterContainerSpec(StackdriverExporter, metricName, metricValue)},
+			Containers: []corev1.Container{stackdriverExporterContainerSpec(StackdriverExporter, namespace, metricName, metricValue)},
 		},
 	}
 }
 
-func stackdriverExporterContainerSpec(name string, metricName string, metricValue int64) corev1.Container {
+func stackdriverExporterContainerSpec(name string, namespace string, metricName string, metricValue int64) corev1.Container {
 	return corev1.Container{
 		Name:            name,
-		Image:           "k8s.gcr.io/sd-dummy-exporter:v0.1.0",
+		Image:           "k8s.gcr.io/sd-dummy-exporter:v0.2.0",
 		ImagePullPolicy: corev1.PullPolicy("Always"),
-		Command:         []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)},
+		Command: []string{
+			"/bin/sh",
+			"-c",
+			strings.Join([]string{
+				"./sd_dummy_exporter",
+				"--pod-id=$(POD_ID)",
+				"--pod-name=$(POD_NAME)",
+				"--namespace=" + namespace,
+				"--metric-name=" + metricName,
+				fmt.Sprintf("--metric-value=%v", metricValue),
+				"--use-old-resource-model",
+				"--use-new-resource-model",
+			}, " "),
+		},
 		Env: []corev1.EnvVar{
 			{
 				Name: "POD_ID",
@@ -139,6 +158,14 @@ func stackdriverExporterContainerSpec(name string, metricName string, metricValu
 					},
 				},
 			},
+			{
+				Name: "POD_NAME",
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "metadata.name",
+					},
+				},
+			},
 		},
 		Ports: []corev1.ContainerPort{{ContainerPort: 80}},
 	}
@@ -210,9 +237,15 @@ func prometheusExporterPodSpec(metricName string, metricValue int64, port int32)
 	}
 }
 
-// CreateAdapter creates Custom Metrics - Stackdriver adapter.
+// CreateAdapter creates Custom Metrics - Stackdriver adapter
-func CreateAdapter() error {
+// adapterDeploymentFile should be a filename for adapter deployment located in StagingDeploymentLocation
-	stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml")
+func CreateAdapter(adapterDeploymentFile string) error {
+	adapterURL := StagingDeploymentsLocation + adapterDeploymentFile
+	err := exec.Command("wget", adapterURL).Run()
+	if err != nil {
+		return err
+	}
+	stat, err := framework.RunKubectl("create", "-f", adapterURL)
 	framework.Logf(stat)
 	return err
 }
@@ -251,8 +284,14 @@ func CleanupDescriptors(service *gcm.Service, projectId string) {
 }
 
 // CleanupAdapter deletes Custom Metrics - Stackdriver adapter deployments.
-func CleanupAdapter() error {
+func CleanupAdapter(adapterDeploymentFile string) {
-	stat, err := framework.RunKubectl("delete", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml")
+	stat, err := framework.RunKubectl("delete", "-f", adapterDeploymentFile)
 	framework.Logf(stat)
-	return err
+	if err != nil {
+		framework.Logf("Failed to delete adapter deployments: %s", err)
+	}
+	err = exec.Command("rm", adapterDeploymentFile).Run()
+	if err != nil {
+		framework.Logf("Failed to delete adapter deployment file: %s", err)
+	}
 }

test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/client-go/discovery"
 	"k8s.io/kubernetes/test/e2e/framework"
 	customclient "k8s.io/metrics/pkg/client/custom_metrics"
+	externalclient "k8s.io/metrics/pkg/client/external_metrics"
 )
 
 const (
@@ -48,41 +49,46 @@ var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() {
 	})
 
 	f := framework.NewDefaultFramework("stackdriver-monitoring")
-	var kubeClient clientset.Interface
-	var customMetricsClient customclient.CustomMetricsClient
-	var discoveryClient *discovery.DiscoveryClient
 
-	It("should run Custom Metrics - Stackdriver Adapter [Feature:StackdriverCustomMetrics]", func() {
+	It("should run Custom Metrics - Stackdriver Adapter for old resource model [Feature:StackdriverCustomMetrics]", func() {
-		kubeClient = f.ClientSet
+		kubeClient := f.ClientSet
 		config, err := framework.LoadConfig()
 		if err != nil {
 			framework.Failf("Failed to load config: %s", err)
 		}
-		customMetricsClient = customclient.NewForConfigOrDie(config)
+		customMetricsClient := customclient.NewForConfigOrDie(config)
-		discoveryClient = discovery.NewDiscoveryClientForConfigOrDie(config)
+		discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config)
-		testAdapter(f, kubeClient, customMetricsClient, discoveryClient)
+		testCustomMetrics(f, kubeClient, customMetricsClient, discoveryClient, AdapterForOldResourceModel)
+	})
+
+	It("should run Custom Metrics - Stackdriver Adapter for new resource model [Feature:StackdriverCustomMetrics]", func() {
+		kubeClient := f.ClientSet
+		config, err := framework.LoadConfig()
+		if err != nil {
+			framework.Failf("Failed to load config: %s", err)
+		}
+		customMetricsClient := customclient.NewForConfigOrDie(config)
+		discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config)
+		testCustomMetrics(f, kubeClient, customMetricsClient, discoveryClient, AdapterForNewResourceModel)
+	})
+
+	It("should run Custom Metrics - Stackdriver Adapter for external metrics [Feature:StackdriverExternalMetrics]", func() {
+		kubeClient := f.ClientSet
+		config, err := framework.LoadConfig()
+		if err != nil {
+			framework.Failf("Failed to load config: %s", err)
+		}
+		externalMetricsClient := externalclient.NewForConfigOrDie(config)
+		testExternalMetrics(f, kubeClient, externalMetricsClient)
 	})
 })
 
-func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) {
+func testCustomMetrics(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient, adapterDeployment string) {
 	projectId := framework.TestContext.CloudConfig.ProjectID
 
 	ctx := context.Background()
 	client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
 
-	// Hack for running tests locally, needed to authenticate in Stackdriver
-	// If this is your use case, create application default credentials:
-	// $ gcloud auth application-default login
-	// and uncomment following lines (comment out the two lines above):
-	/*
-		ts, err := google.DefaultTokenSource(oauth2.NoContext)
-		framework.Logf("Couldn't get application default credentials, %v", err)
-		if err != nil {
-			framework.Failf("Error accessing application default credentials, %v", err)
-		}
-		client := oauth2.NewClient(oauth2.NoContext, ts)
-	*/
-
 	gcmService, err := gcm.New(client)
 	if err != nil {
 		framework.Failf("Failed to create gcm service, %v", err)
@@ -95,11 +101,11 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM
 	}
 	defer CleanupDescriptors(gcmService, projectId)
 
-	err = CreateAdapter()
+	err = CreateAdapter(adapterDeployment)
 	if err != nil {
 		framework.Failf("Failed to set up: %s", err)
 	}
-	defer CleanupAdapter()
+	defer CleanupAdapter(adapterDeployment)
 
 	_, err = kubeClient.RbacV1().ClusterRoleBindings().Create(HPAPermissions)
 	defer kubeClient.RbacV1().ClusterRoleBindings().Delete("custom-metrics-reader", &metav1.DeleteOptions{})
@@ -116,16 +122,62 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM
 	//       i.e. pod creation, first time series exported
 	time.Sleep(60 * time.Second)
 
-	// Verify responses from Custom Metrics API
+	verifyResponsesFromCustomMetricsAPI(f, customMetricsClient, discoveryClient)
+}
+
+func testExternalMetrics(f *framework.Framework, kubeClient clientset.Interface, externalMetricsClient externalclient.ExternalMetricsClient) {
+	projectId := framework.TestContext.CloudConfig.ProjectID
+
+	ctx := context.Background()
+	client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
+
+	gcmService, err := gcm.New(client)
+	if err != nil {
+		framework.Failf("Failed to create gcm service, %v", err)
+	}
+
+	// Set up a cluster: create a custom metric and set up k8s-sd adapter
+	err = CreateDescriptors(gcmService, projectId)
+	if err != nil {
+		framework.Failf("Failed to create metric descriptor: %s", err)
+	}
+	defer CleanupDescriptors(gcmService, projectId)
+
+	// Both deployments - for old and new resource model - expose External Metrics API.
+	err = CreateAdapter(AdapterForNewResourceModel)
+	if err != nil {
+		framework.Failf("Failed to set up: %s", err)
+	}
+	defer CleanupAdapter(AdapterForNewResourceModel)
+
+	_, err = kubeClient.RbacV1().ClusterRoleBindings().Create(HPAPermissions)
+	defer kubeClient.RbacV1().ClusterRoleBindings().Delete("custom-metrics-reader", &metav1.DeleteOptions{})
+
+	// Run application that exports the metric
+	err = createSDExporterPods(f, kubeClient)
+	if err != nil {
+		framework.Failf("Failed to create stackdriver-exporter pod: %s", err)
+	}
+	defer cleanupSDExporterPod(f, kubeClient)
+
+	// Wait a short amount of time to create a pod and export some metrics
+	// TODO: add some events to wait for instead of fixed amount of time
+	//       i.e. pod creation, first time series exported
+	time.Sleep(60 * time.Second)
+
+	verifyResponseFromExternalMetricsAPI(f, externalMetricsClient)
+}
+
+func verifyResponsesFromCustomMetricsAPI(f *framework.Framework, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) {
 	resources, err := discoveryClient.ServerResourcesForGroupVersion("custom.metrics.k8s.io/v1beta1")
 	if err != nil {
 		framework.Failf("Failed to retrieve a list of supported metrics: %s", err)
 	}
 	gotCustomMetric, gotUnusedMetric := false, false
 	for _, resource := range resources.APIResources {
-		if resource.Name == "pods/"+CustomMetricName {
+		if resource.Name == "*/"+CustomMetricName {
 			gotCustomMetric = true
-		} else if resource.Name == "pods/"+UnusedMetricName {
+		} else if resource.Name == "*/"+UnusedMetricName {
 			gotUnusedMetric = true
 		} else {
 			framework.Failf("Unexpected metric %s. Only metric %s should be supported", resource.Name, CustomMetricName)
@@ -160,6 +212,31 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM
 	}
 }
 
+func verifyResponseFromExternalMetricsAPI(f *framework.Framework, externalMetricsClient externalclient.ExternalMetricsClient) {
+	req1, _ := labels.NewRequirement("resource.type", selection.Equals, []string{"k8s_pod"})
+	// It's important to filter out only metrics from the right namespace, since multiple e2e tests
+	// may run in the same project concurrently. "dummy" is added to test
+	req2, _ := labels.NewRequirement("resource.label.namespace_name", selection.In, []string{string(f.Namespace.Name), "dummy"})
+	req3, _ := labels.NewRequirement("resource.label.pod_name", selection.Exists, []string{})
+	req4, _ := labels.NewRequirement("resource.label.location", selection.NotEquals, []string{"dummy"})
+	req5, _ := labels.NewRequirement("resource.label.cluster_name", selection.NotIn, []string{"foo", "bar"})
+	values, err := externalMetricsClient.
+		NamespacedMetrics("dummy").
+		List("custom.googleapis.com|"+CustomMetricName, labels.NewSelector().Add(*req1, *req2, *req3, *req4, *req5))
+	if err != nil {
+		framework.Failf("Failed query: %s", err)
+	}
+	if len(values.Items) != 1 {
+		framework.Failf("Expected exactly one external metric value, but % values received", len(values.Items))
+	}
+	if values.Items[0].MetricName != "custom.googleapis.com|"+CustomMetricName ||
+		values.Items[0].Value.Value() != CustomMetricValue ||
+		// Check one label just to make sure labels are included
+		values.Items[0].MetricLabels["resource.label.namespace_name"] != string(f.Namespace.Name) {
+		framework.Failf("Unexpected result for metric %s: %v", CustomMetricName, values.Items[0])
+	}
+}
+
 func cleanupSDExporterPod(f *framework.Framework, cs clientset.Interface) {
 	err := cs.CoreV1().Pods(f.Namespace.Name).Delete(stackdriverExporterPod1, &metav1.DeleteOptions{})
 	if err != nil {