diff --git a/test/e2e/autoscaling/custom_metrics_autoscaling.go b/test/e2e/autoscaling/custom_metrics_autoscaling.go index 5d002bf5055..ce29b452b4f 100644 --- a/test/e2e/autoscaling/custom_metrics_autoscaling.go +++ b/test/e2e/autoscaling/custom_metrics_autoscaling.go @@ -138,11 +138,11 @@ func customMetricTest(f *framework.Framework, kubeClient clientset.Interface, hp } defer monitoring.CleanupDescriptors(gcmService, projectId) - err = monitoring.CreateAdapter() + err = monitoring.CreateAdapter(monitoring.AdapterDefault) if err != nil { framework.Failf("Failed to set up: %v", err) } - defer monitoring.CleanupAdapter() + defer monitoring.CleanupAdapter(monitoring.AdapterDefault) // Run application that exports the metric err = createDeploymentToScale(f, kubeClient, deployment, pod) diff --git a/test/e2e/instrumentation/monitoring/BUILD b/test/e2e/instrumentation/monitoring/BUILD index 321d00b0d7a..a817039f9c2 100644 --- a/test/e2e/instrumentation/monitoring/BUILD +++ b/test/e2e/instrumentation/monitoring/BUILD @@ -42,6 +42,7 @@ go_library( "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/metrics/pkg/client/custom_metrics:go_default_library", + "//vendor/k8s.io/metrics/pkg/client/external_metrics:go_default_library", ], ) diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go index f1d3e0926bf..793b3e425d8 100644 --- a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go +++ b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go @@ -18,6 +18,7 @@ package monitoring import ( "fmt" + "strings" gcm "google.golang.org/api/monitoring/v3" corev1 "k8s.io/api/core/v1" @@ -25,6 +26,7 @@ import ( rbac "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" + "os/exec" ) var ( @@ -52,6 +54,10 @@ var ( }, }, } + StagingDeploymentsLocation = "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/staging/" + AdapterForOldResourceModel = "adapter_old_resource_model.yaml" + AdapterForNewResourceModel = "adapter_new_resource_model.yaml" + AdapterDefault = AdapterForOldResourceModel ) // CustomMetricContainerSpec allows to specify a config for StackdriverExporterDeployment @@ -82,7 +88,7 @@ func SimpleStackdriverExporterDeployment(name, namespace string, replicas int32, func StackdriverExporterDeployment(name, namespace string, replicas int32, containers []CustomMetricContainerSpec) *extensions.Deployment { podSpec := corev1.PodSpec{Containers: []corev1.Container{}} for _, containerSpec := range containers { - podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, containerSpec.MetricName, containerSpec.MetricValue)) + podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, namespace, containerSpec.MetricName, containerSpec.MetricValue)) } return &extensions.Deployment{ @@ -119,17 +125,30 @@ func StackdriverExporterPod(podName, namespace, podLabel, metricName string, met }, }, Spec: corev1.PodSpec{ - Containers: []corev1.Container{stackdriverExporterContainerSpec(StackdriverExporter, metricName, metricValue)}, + Containers: []corev1.Container{stackdriverExporterContainerSpec(StackdriverExporter, namespace, metricName, metricValue)}, }, } } -func stackdriverExporterContainerSpec(name string, metricName string, metricValue int64) corev1.Container { +func stackdriverExporterContainerSpec(name string, namespace string, metricName string, metricValue int64) corev1.Container { return corev1.Container{ Name: name, - Image: "k8s.gcr.io/sd-dummy-exporter:v0.1.0", + Image: "k8s.gcr.io/sd-dummy-exporter:v0.2.0", ImagePullPolicy: corev1.PullPolicy("Always"), - Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)}, + Command: []string{ + "/bin/sh", + "-c", + strings.Join([]string{ + "./sd_dummy_exporter", + "--pod-id=$(POD_ID)", + "--pod-name=$(POD_NAME)", + "--namespace=" + namespace, + "--metric-name=" + metricName, + fmt.Sprintf("--metric-value=%v", metricValue), + "--use-old-resource-model", + "--use-new-resource-model", + }, " "), + }, Env: []corev1.EnvVar{ { Name: "POD_ID", @@ -139,6 +158,14 @@ func stackdriverExporterContainerSpec(name string, metricName string, metricValu }, }, }, + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, }, Ports: []corev1.ContainerPort{{ContainerPort: 80}}, } @@ -210,9 +237,15 @@ func prometheusExporterPodSpec(metricName string, metricValue int64, port int32) } } -// CreateAdapter creates Custom Metrics - Stackdriver adapter. -func CreateAdapter() error { - stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml") +// CreateAdapter creates Custom Metrics - Stackdriver adapter +// adapterDeploymentFile should be a filename for adapter deployment located in StagingDeploymentLocation +func CreateAdapter(adapterDeploymentFile string) error { + adapterURL := StagingDeploymentsLocation + adapterDeploymentFile + err := exec.Command("wget", adapterURL).Run() + if err != nil { + return err + } + stat, err := framework.RunKubectl("create", "-f", adapterURL) framework.Logf(stat) return err } @@ -251,8 +284,14 @@ func CleanupDescriptors(service *gcm.Service, projectId string) { } // CleanupAdapter deletes Custom Metrics - Stackdriver adapter deployments. -func CleanupAdapter() error { - stat, err := framework.RunKubectl("delete", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml") +func CleanupAdapter(adapterDeploymentFile string) { + stat, err := framework.RunKubectl("delete", "-f", adapterDeploymentFile) framework.Logf(stat) - return err + if err != nil { + framework.Logf("Failed to delete adapter deployments: %s", err) + } + err = exec.Command("rm", adapterDeploymentFile).Run() + if err != nil { + framework.Logf("Failed to delete adapter deployment file: %s", err) + } } diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go index 66ac0466a34..fe1ab44b1ae 100644 --- a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go +++ b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go @@ -34,6 +34,7 @@ import ( "k8s.io/client-go/discovery" "k8s.io/kubernetes/test/e2e/framework" customclient "k8s.io/metrics/pkg/client/custom_metrics" + externalclient "k8s.io/metrics/pkg/client/external_metrics" ) const ( @@ -48,41 +49,46 @@ var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() { }) f := framework.NewDefaultFramework("stackdriver-monitoring") - var kubeClient clientset.Interface - var customMetricsClient customclient.CustomMetricsClient - var discoveryClient *discovery.DiscoveryClient - It("should run Custom Metrics - Stackdriver Adapter [Feature:StackdriverCustomMetrics]", func() { - kubeClient = f.ClientSet + It("should run Custom Metrics - Stackdriver Adapter for old resource model [Feature:StackdriverCustomMetrics]", func() { + kubeClient := f.ClientSet config, err := framework.LoadConfig() if err != nil { framework.Failf("Failed to load config: %s", err) } - customMetricsClient = customclient.NewForConfigOrDie(config) - discoveryClient = discovery.NewDiscoveryClientForConfigOrDie(config) - testAdapter(f, kubeClient, customMetricsClient, discoveryClient) + customMetricsClient := customclient.NewForConfigOrDie(config) + discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config) + testCustomMetrics(f, kubeClient, customMetricsClient, discoveryClient, AdapterForOldResourceModel) + }) + + It("should run Custom Metrics - Stackdriver Adapter for new resource model [Feature:StackdriverCustomMetrics]", func() { + kubeClient := f.ClientSet + config, err := framework.LoadConfig() + if err != nil { + framework.Failf("Failed to load config: %s", err) + } + customMetricsClient := customclient.NewForConfigOrDie(config) + discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config) + testCustomMetrics(f, kubeClient, customMetricsClient, discoveryClient, AdapterForNewResourceModel) + }) + + It("should run Custom Metrics - Stackdriver Adapter for external metrics [Feature:StackdriverExternalMetrics]", func() { + kubeClient := f.ClientSet + config, err := framework.LoadConfig() + if err != nil { + framework.Failf("Failed to load config: %s", err) + } + externalMetricsClient := externalclient.NewForConfigOrDie(config) + testExternalMetrics(f, kubeClient, externalMetricsClient) }) }) -func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) { +func testCustomMetrics(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient, adapterDeployment string) { projectId := framework.TestContext.CloudConfig.ProjectID ctx := context.Background() client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) - // Hack for running tests locally, needed to authenticate in Stackdriver - // If this is your use case, create application default credentials: - // $ gcloud auth application-default login - // and uncomment following lines (comment out the two lines above): - /* - ts, err := google.DefaultTokenSource(oauth2.NoContext) - framework.Logf("Couldn't get application default credentials, %v", err) - if err != nil { - framework.Failf("Error accessing application default credentials, %v", err) - } - client := oauth2.NewClient(oauth2.NoContext, ts) - */ - gcmService, err := gcm.New(client) if err != nil { framework.Failf("Failed to create gcm service, %v", err) @@ -95,11 +101,11 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM } defer CleanupDescriptors(gcmService, projectId) - err = CreateAdapter() + err = CreateAdapter(adapterDeployment) if err != nil { framework.Failf("Failed to set up: %s", err) } - defer CleanupAdapter() + defer CleanupAdapter(adapterDeployment) _, err = kubeClient.RbacV1().ClusterRoleBindings().Create(HPAPermissions) defer kubeClient.RbacV1().ClusterRoleBindings().Delete("custom-metrics-reader", &metav1.DeleteOptions{}) @@ -116,16 +122,62 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM // i.e. pod creation, first time series exported time.Sleep(60 * time.Second) - // Verify responses from Custom Metrics API + verifyResponsesFromCustomMetricsAPI(f, customMetricsClient, discoveryClient) +} + +func testExternalMetrics(f *framework.Framework, kubeClient clientset.Interface, externalMetricsClient externalclient.ExternalMetricsClient) { + projectId := framework.TestContext.CloudConfig.ProjectID + + ctx := context.Background() + client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope) + + gcmService, err := gcm.New(client) + if err != nil { + framework.Failf("Failed to create gcm service, %v", err) + } + + // Set up a cluster: create a custom metric and set up k8s-sd adapter + err = CreateDescriptors(gcmService, projectId) + if err != nil { + framework.Failf("Failed to create metric descriptor: %s", err) + } + defer CleanupDescriptors(gcmService, projectId) + + // Both deployments - for old and new resource model - expose External Metrics API. + err = CreateAdapter(AdapterForNewResourceModel) + if err != nil { + framework.Failf("Failed to set up: %s", err) + } + defer CleanupAdapter(AdapterForNewResourceModel) + + _, err = kubeClient.RbacV1().ClusterRoleBindings().Create(HPAPermissions) + defer kubeClient.RbacV1().ClusterRoleBindings().Delete("custom-metrics-reader", &metav1.DeleteOptions{}) + + // Run application that exports the metric + err = createSDExporterPods(f, kubeClient) + if err != nil { + framework.Failf("Failed to create stackdriver-exporter pod: %s", err) + } + defer cleanupSDExporterPod(f, kubeClient) + + // Wait a short amount of time to create a pod and export some metrics + // TODO: add some events to wait for instead of fixed amount of time + // i.e. pod creation, first time series exported + time.Sleep(60 * time.Second) + + verifyResponseFromExternalMetricsAPI(f, externalMetricsClient) +} + +func verifyResponsesFromCustomMetricsAPI(f *framework.Framework, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) { resources, err := discoveryClient.ServerResourcesForGroupVersion("custom.metrics.k8s.io/v1beta1") if err != nil { framework.Failf("Failed to retrieve a list of supported metrics: %s", err) } gotCustomMetric, gotUnusedMetric := false, false for _, resource := range resources.APIResources { - if resource.Name == "pods/"+CustomMetricName { + if resource.Name == "*/"+CustomMetricName { gotCustomMetric = true - } else if resource.Name == "pods/"+UnusedMetricName { + } else if resource.Name == "*/"+UnusedMetricName { gotUnusedMetric = true } else { framework.Failf("Unexpected metric %s. Only metric %s should be supported", resource.Name, CustomMetricName) @@ -160,6 +212,31 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM } } +func verifyResponseFromExternalMetricsAPI(f *framework.Framework, externalMetricsClient externalclient.ExternalMetricsClient) { + req1, _ := labels.NewRequirement("resource.type", selection.Equals, []string{"k8s_pod"}) + // It's important to filter out only metrics from the right namespace, since multiple e2e tests + // may run in the same project concurrently. "dummy" is added to test + req2, _ := labels.NewRequirement("resource.label.namespace_name", selection.In, []string{string(f.Namespace.Name), "dummy"}) + req3, _ := labels.NewRequirement("resource.label.pod_name", selection.Exists, []string{}) + req4, _ := labels.NewRequirement("resource.label.location", selection.NotEquals, []string{"dummy"}) + req5, _ := labels.NewRequirement("resource.label.cluster_name", selection.NotIn, []string{"foo", "bar"}) + values, err := externalMetricsClient. + NamespacedMetrics("dummy"). + List("custom.googleapis.com|"+CustomMetricName, labels.NewSelector().Add(*req1, *req2, *req3, *req4, *req5)) + if err != nil { + framework.Failf("Failed query: %s", err) + } + if len(values.Items) != 1 { + framework.Failf("Expected exactly one external metric value, but % values received", len(values.Items)) + } + if values.Items[0].MetricName != "custom.googleapis.com|"+CustomMetricName || + values.Items[0].Value.Value() != CustomMetricValue || + // Check one label just to make sure labels are included + values.Items[0].MetricLabels["resource.label.namespace_name"] != string(f.Namespace.Name) { + framework.Failf("Unexpected result for metric %s: %v", CustomMetricName, values.Items[0]) + } +} + func cleanupSDExporterPod(f *framework.Framework, cs clientset.Interface) { err := cs.CoreV1().Pods(f.Namespace.Name).Delete(stackdriverExporterPod1, &metav1.DeleteOptions{}) if err != nil {