github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2025-07-22 19:31:44 +00:00
Code Issues Projects Releases Wiki Activity

Add e2e test for Custom Metrics API with new Stackdriver resource model

Browse Source 
and External Metrics API.
This commit is contained in:
Karol Wychowaniec 2018-03-14 12:16:33 +01:00
parent 0207a09074
commit 71f14cf335
4 changed files with 157 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/e2e/autoscaling/custom_metrics_autoscaling.go
View File

@ -138,11 +138,11 @@ func customMetricTest(f *framework.Framework, kubeClient clientset.Interface, hp
}
defer monitoring.CleanupDescriptors(gcmService, projectId)
err = monitoring.CreateAdapter()
err = monitoring.CreateAdapter(monitoring.AdapterDefault)
if err != nil {
framework.Failf("Failed to set up: %v", err)
}
defer monitoring.CleanupAdapter()
defer monitoring.CleanupAdapter(monitoring.AdapterDefault)
// Run application that exports the metric
err = createDeploymentToScale(f, kubeClient, deployment, pod)

1
test/e2e/instrumentation/monitoring/BUILD
View File

@ -42,6 +42,7 @@ go_library(
"//vendor/k8s.io/client-go/discovery:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
"//vendor/k8s.io/metrics/pkg/client/external_metrics:go_default_library",
],
)

61
test/e2e/instrumentation/monitoring/custom_metrics_deployments.go
View File

@ -18,6 +18,7 @@ package monitoring
import (
"fmt"
"strings"
gcm "google.golang.org/api/monitoring/v3"
corev1 "k8s.io/api/core/v1"
@ -25,6 +26,7 @@ import (
rbac "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
"os/exec"
)
var (
@ -52,6 +54,10 @@ var (
},
},
}
StagingDeploymentsLocation = "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/deploy/staging/"
AdapterForOldResourceModel = "adapter_old_resource_model.yaml"
AdapterForNewResourceModel = "adapter_new_resource_model.yaml"
AdapterDefault = AdapterForOldResourceModel
)
// CustomMetricContainerSpec allows to specify a config for StackdriverExporterDeployment
@ -82,7 +88,7 @@ func SimpleStackdriverExporterDeployment(name, namespace string, replicas int32,
func StackdriverExporterDeployment(name, namespace string, replicas int32, containers []CustomMetricContainerSpec) *extensions.Deployment {
podSpec := corev1.PodSpec{Containers: []corev1.Container{}}
for _, containerSpec := range containers {
podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, containerSpec.MetricName, containerSpec.MetricValue))
podSpec.Containers = append(podSpec.Containers, stackdriverExporterContainerSpec(containerSpec.Name, namespace, containerSpec.MetricName, containerSpec.MetricValue))
}
return &extensions.Deployment{
@ -119,17 +125,30 @@ func StackdriverExporterPod(podName, namespace, podLabel, metricName string, met
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{stackdriverExporterContainerSpec(StackdriverExporter, metricName, metricValue)},
Containers: []corev1.Container{stackdriverExporterContainerSpec(StackdriverExporter, namespace, metricName, metricValue)},
},
}
}
func stackdriverExporterContainerSpec(name string, metricName string, metricValue int64) corev1.Container {
func stackdriverExporterContainerSpec(name string, namespace string, metricName string, metricValue int64) corev1.Container {
return corev1.Container{
Name: name,
Image: "k8s.gcr.io/sd-dummy-exporter:v0.1.0",
Image: "k8s.gcr.io/sd-dummy-exporter:v0.2.0",
ImagePullPolicy: corev1.PullPolicy("Always"),
Command: []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)},
Command: []string{
"/bin/sh",
"-c",
strings.Join([]string{
"./sd_dummy_exporter",
"--pod-id=$(POD_ID)",
"--pod-name=$(POD_NAME)",
"--namespace=" + namespace,
"--metric-name=" + metricName,
fmt.Sprintf("--metric-value=%v", metricValue),
"--use-old-resource-model",
"--use-new-resource-model",
}, " "),
},
Env: []corev1.EnvVar{
{
Name: "POD_ID",
@ -139,6 +158,14 @@ func stackdriverExporterContainerSpec(name string, metricName string, metricValu
},
},
},
{
Name: "POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.name",
},
},
},
},
Ports: []corev1.ContainerPort{{ContainerPort: 80}},
}
@ -210,9 +237,15 @@ func prometheusExporterPodSpec(metricName string, metricValue int64, port int32)
}
}
// CreateAdapter creates Custom Metrics - Stackdriver adapter.
func CreateAdapter() error {
stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml")
// CreateAdapter creates Custom Metrics - Stackdriver adapter
// adapterDeploymentFile should be a filename for adapter deployment located in StagingDeploymentLocation
func CreateAdapter(adapterDeploymentFile string) error {
adapterURL := StagingDeploymentsLocation + adapterDeploymentFile
err := exec.Command("wget", adapterURL).Run()
if err != nil {
return err
}
stat, err := framework.RunKubectl("create", "-f", adapterURL)
framework.Logf(stat)
return err
}
@ -251,8 +284,14 @@ func CleanupDescriptors(service *gcm.Service, projectId string) {
}
// CleanupAdapter deletes Custom Metrics - Stackdriver adapter deployments.
func CleanupAdapter() error {
stat, err := framework.RunKubectl("delete", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml")
func CleanupAdapter(adapterDeploymentFile string) {
stat, err := framework.RunKubectl("delete", "-f", adapterDeploymentFile)
framework.Logf(stat)
return err
if err != nil {
framework.Logf("Failed to delete adapter deployments: %s", err)
}
err = exec.Command("rm", adapterDeploymentFile).Run()
if err != nil {
framework.Logf("Failed to delete adapter deployment file: %s", err)
}
}

131
test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go
View File

@ -34,6 +34,7 @@ import (
"k8s.io/client-go/discovery"
"k8s.io/kubernetes/test/e2e/framework"
customclient "k8s.io/metrics/pkg/client/custom_metrics"
externalclient "k8s.io/metrics/pkg/client/external_metrics"
)
const (
@ -48,41 +49,46 @@ var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() {
})
f := framework.NewDefaultFramework("stackdriver-monitoring")
var kubeClient clientset.Interface
var customMetricsClient customclient.CustomMetricsClient
var discoveryClient *discovery.DiscoveryClient
It("should run Custom Metrics - Stackdriver Adapter [Feature:StackdriverCustomMetrics]", func() {
kubeClient = f.ClientSet
It("should run Custom Metrics - Stackdriver Adapter for old resource model [Feature:StackdriverCustomMetrics]", func() {
kubeClient := f.ClientSet
config, err := framework.LoadConfig()
if err != nil {
framework.Failf("Failed to load config: %s", err)
}
customMetricsClient = customclient.NewForConfigOrDie(config)
discoveryClient = discovery.NewDiscoveryClientForConfigOrDie(config)
testAdapter(f, kubeClient, customMetricsClient, discoveryClient)
customMetricsClient := customclient.NewForConfigOrDie(config)
discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config)
testCustomMetrics(f, kubeClient, customMetricsClient, discoveryClient, AdapterForOldResourceModel)
})
It("should run Custom Metrics - Stackdriver Adapter for new resource model [Feature:StackdriverCustomMetrics]", func() {
kubeClient := f.ClientSet
config, err := framework.LoadConfig()
if err != nil {
framework.Failf("Failed to load config: %s", err)
}
customMetricsClient := customclient.NewForConfigOrDie(config)
discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(config)
testCustomMetrics(f, kubeClient, customMetricsClient, discoveryClient, AdapterForNewResourceModel)
})
It("should run Custom Metrics - Stackdriver Adapter for external metrics [Feature:StackdriverExternalMetrics]", func() {
kubeClient := f.ClientSet
config, err := framework.LoadConfig()
if err != nil {
framework.Failf("Failed to load config: %s", err)
}
externalMetricsClient := externalclient.NewForConfigOrDie(config)
testExternalMetrics(f, kubeClient, externalMetricsClient)
})
})
func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) {
func testCustomMetrics(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient, adapterDeployment string) {
projectId := framework.TestContext.CloudConfig.ProjectID
ctx := context.Background()
client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
// Hack for running tests locally, needed to authenticate in Stackdriver
// If this is your use case, create application default credentials:
// $ gcloud auth application-default login
// and uncomment following lines (comment out the two lines above):
/*
ts, err := google.DefaultTokenSource(oauth2.NoContext)
framework.Logf("Couldn't get application default credentials, %v", err)
if err != nil {
framework.Failf("Error accessing application default credentials, %v", err)
}
client := oauth2.NewClient(oauth2.NoContext, ts)
*/
gcmService, err := gcm.New(client)
if err != nil {
framework.Failf("Failed to create gcm service, %v", err)
@ -95,11 +101,11 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM
}
defer CleanupDescriptors(gcmService, projectId)
err = CreateAdapter()
err = CreateAdapter(adapterDeployment)
if err != nil {
framework.Failf("Failed to set up: %s", err)
}
defer CleanupAdapter()
defer CleanupAdapter(adapterDeployment)
_, err = kubeClient.RbacV1().ClusterRoleBindings().Create(HPAPermissions)
defer kubeClient.RbacV1().ClusterRoleBindings().Delete("custom-metrics-reader", &metav1.DeleteOptions{})
@ -116,16 +122,62 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM
// i.e. pod creation, first time series exported
time.Sleep(60 * time.Second)
// Verify responses from Custom Metrics API
verifyResponsesFromCustomMetricsAPI(f, customMetricsClient, discoveryClient)
}
func testExternalMetrics(f *framework.Framework, kubeClient clientset.Interface, externalMetricsClient externalclient.ExternalMetricsClient) {
projectId := framework.TestContext.CloudConfig.ProjectID
ctx := context.Background()
client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
gcmService, err := gcm.New(client)
if err != nil {
framework.Failf("Failed to create gcm service, %v", err)
}
// Set up a cluster: create a custom metric and set up k8s-sd adapter
err = CreateDescriptors(gcmService, projectId)
if err != nil {
framework.Failf("Failed to create metric descriptor: %s", err)
}
defer CleanupDescriptors(gcmService, projectId)
// Both deployments - for old and new resource model - expose External Metrics API.
err = CreateAdapter(AdapterForNewResourceModel)
if err != nil {
framework.Failf("Failed to set up: %s", err)
}
defer CleanupAdapter(AdapterForNewResourceModel)
_, err = kubeClient.RbacV1().ClusterRoleBindings().Create(HPAPermissions)
defer kubeClient.RbacV1().ClusterRoleBindings().Delete("custom-metrics-reader", &metav1.DeleteOptions{})
// Run application that exports the metric
err = createSDExporterPods(f, kubeClient)
if err != nil {
framework.Failf("Failed to create stackdriver-exporter pod: %s", err)
}
defer cleanupSDExporterPod(f, kubeClient)
// Wait a short amount of time to create a pod and export some metrics
// TODO: add some events to wait for instead of fixed amount of time
// i.e. pod creation, first time series exported
time.Sleep(60 * time.Second)
verifyResponseFromExternalMetricsAPI(f, externalMetricsClient)
}
func verifyResponsesFromCustomMetricsAPI(f *framework.Framework, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) {
resources, err := discoveryClient.ServerResourcesForGroupVersion("custom.metrics.k8s.io/v1beta1")
if err != nil {
framework.Failf("Failed to retrieve a list of supported metrics: %s", err)
}
gotCustomMetric, gotUnusedMetric := false, false
for _, resource := range resources.APIResources {
if resource.Name == "pods/"+CustomMetricName {
if resource.Name == "*/"+CustomMetricName {
gotCustomMetric = true
} else if resource.Name == "pods/"+UnusedMetricName {
} else if resource.Name == "*/"+UnusedMetricName {
gotUnusedMetric = true
} else {
framework.Failf("Unexpected metric %s. Only metric %s should be supported", resource.Name, CustomMetricName)
@ -160,6 +212,31 @@ func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customM
}
}
func verifyResponseFromExternalMetricsAPI(f *framework.Framework, externalMetricsClient externalclient.ExternalMetricsClient) {
req1, _ := labels.NewRequirement("resource.type", selection.Equals, []string{"k8s_pod"})
// It's important to filter out only metrics from the right namespace, since multiple e2e tests
// may run in the same project concurrently. "dummy" is added to test
req2, _ := labels.NewRequirement("resource.label.namespace_name", selection.In, []string{string(f.Namespace.Name), "dummy"})
req3, _ := labels.NewRequirement("resource.label.pod_name", selection.Exists, []string{})
req4, _ := labels.NewRequirement("resource.label.location", selection.NotEquals, []string{"dummy"})
req5, _ := labels.NewRequirement("resource.label.cluster_name", selection.NotIn, []string{"foo", "bar"})
values, err := externalMetricsClient.
NamespacedMetrics("dummy").
List("custom.googleapis.com|"+CustomMetricName, labels.NewSelector().Add(*req1, *req2, *req3, *req4, *req5))
if err != nil {
framework.Failf("Failed query: %s", err)
}
if len(values.Items) != 1 {
framework.Failf("Expected exactly one external metric value, but % values received", len(values.Items))
}
if values.Items[0].MetricName != "custom.googleapis.com|"+CustomMetricName ||
values.Items[0].Value.Value() != CustomMetricValue ||
// Check one label just to make sure labels are included
values.Items[0].MetricLabels["resource.label.namespace_name"] != string(f.Namespace.Name) {
framework.Failf("Unexpected result for metric %s: %v", CustomMetricName, values.Items[0])
}
}
func cleanupSDExporterPod(f *framework.Framework, cs clientset.Interface) {
err := cs.CoreV1().Pods(f.Namespace.Name).Delete(stackdriverExporterPod1, &metav1.DeleteOptions{})
if err != nil {