From 93530ce8d2a4701e7c7cf1a80427c4dcf5bdad34 Mon Sep 17 00:00:00 2001 From: Karol Wychowaniec Date: Thu, 1 Feb 2018 14:02:31 +0100 Subject: [PATCH 1/3] Bump default Metadata Agent version --- cluster/gce/config-default.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index c3e59d0babb..fa1961ac1fe 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -157,7 +157,7 @@ ENABLE_METRICS_SERVER="${KUBE_ENABLE_METRICS_SERVER:-true}" ENABLE_METADATA_AGENT="${KUBE_ENABLE_METADATA_AGENT:-none}" # Version tag of metadata agent -METADATA_AGENT_VERSION="${KUBE_METADATA_AGENT_VERSION:-0.2-0.0.13-5-watch}" +METADATA_AGENT_VERSION="${KUBE_METADATA_AGENT_VERSION:-0.2-0.0.16-1}" # One special node out of NUM_NODES would be created of this type if specified. # Useful for scheduling heapster in large clusters with nodes of small size. From b7c8281ba9bf6236f5c5453ad0df3e1cb8527b0b Mon Sep 17 00:00:00 2001 From: Karol Wychowaniec Date: Fri, 16 Feb 2018 16:05:53 +0100 Subject: [PATCH 2/3] Allow Metadata Agent to get and list resources --- .../addons/metadata-agent/stackdriver/metadata-agent-rbac.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster/addons/metadata-agent/stackdriver/metadata-agent-rbac.yaml b/cluster/addons/metadata-agent/stackdriver/metadata-agent-rbac.yaml index dbe999c4b3b..b82b55c06b8 100644 --- a/cluster/addons/metadata-agent/stackdriver/metadata-agent-rbac.yaml +++ b/cluster/addons/metadata-agent/stackdriver/metadata-agent-rbac.yaml @@ -14,6 +14,8 @@ rules: - "*" verbs: - watch + - get + - list --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From 16ffe42a16a302a3b83c11c305cec10b36b298b9 Mon Sep 17 00:00:00 2001 From: Karol Wychowaniec Date: Wed, 20 Dec 2017 14:35:41 +0100 Subject: [PATCH 3/3] Introduce e2e test for Metadata Agent --- cluster/gce/config-test.sh | 10 ++ test/e2e/instrumentation/monitoring/BUILD | 1 + .../monitoring/stackdriver_metadata_agent.go | 169 ++++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 93d254a1b87..69c73538971 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -145,6 +145,16 @@ ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}" # TODO(piosz) remove this option once Metrics Server became a stable thing. ENABLE_METRICS_SERVER="${KUBE_ENABLE_METRICS_SERVER:-true}" +# Optional: Metadata agent to setup as part of the cluster bring up: +# none - No metadata agent +# stackdriver - Stackdriver metadata agent +# Metadata agent is a daemon set that provides metadata of kubernetes objects +# running on the same node for exporting metrics and logs. +ENABLE_METADATA_AGENT="${KUBE_ENABLE_METADATA_AGENT:-none}" + +# Version tag of metadata agent +METADATA_AGENT_VERSION="${KUBE_METADATA_AGENT_VERSION:-0.2-0.0.16-1}" + # One special node out of NUM_NODES would be created of this type if specified. # Useful for scheduling heapster in large clusters with nodes of small size. HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}" diff --git a/test/e2e/instrumentation/monitoring/BUILD b/test/e2e/instrumentation/monitoring/BUILD index 535098bcfb2..259464452cf 100644 --- a/test/e2e/instrumentation/monitoring/BUILD +++ b/test/e2e/instrumentation/monitoring/BUILD @@ -15,6 +15,7 @@ go_library( "influxdb.go", "metrics_grabber.go", "stackdriver.go", + "stackdriver_metadata_agent.go", ], importpath = "k8s.io/kubernetes/test/e2e/instrumentation/monitoring", deps = [ diff --git a/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go b/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go new file mode 100644 index 00000000000..cca75616888 --- /dev/null +++ b/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go @@ -0,0 +1,169 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package monitoring + +import ( + "time" + + "golang.org/x/oauth2/google" + clientset "k8s.io/client-go/kubernetes" + + "context" + "encoding/json" + "fmt" + . "github.com/onsi/ginkgo" + "io/ioutil" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common" + "reflect" +) + +const ( + // Time to wait after a pod creation for it's metadata to be exported + metadataWaitTime = 120 * time.Second + + // Scope for Stackdriver Metadata API + MonitoringScope = "https://www.googleapis.com/auth/monitoring" +) + +var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() { + BeforeEach(func() { + framework.SkipUnlessProviderIs("gce", "gke") + }) + + f := framework.NewDefaultFramework("stackdriver-monitoring") + var kubeClient clientset.Interface + + It("should run Stackdriver Metadata Agent [Feature:StackdriverMetadataAgent]", func() { + kubeClient = f.ClientSet + testAgent(f, kubeClient) + }) +}) + +func testAgent(f *framework.Framework, kubeClient clientset.Interface) { + projectId := framework.TestContext.CloudConfig.ProjectID + resourceType := "k8s_container" + uniqueContainerName := fmt.Sprintf("test-container-%v", time.Now().Unix()) + endpoint := fmt.Sprintf( + "https://stackdriver.googleapis.com/v1beta2/projects/%v/resourceMetadata?filter=resource.type%%3D%v+AND+resource.label.container_name%%3D%v", + projectId, + resourceType, + uniqueContainerName) + + oauthClient, err := google.DefaultClient(context.Background(), MonitoringScope) + if err != nil { + framework.Failf("Failed to create oauth client: %s", err) + } + + // Create test pod with unique name. + framework.CreateExecPodOrFail(kubeClient, f.Namespace.Name, uniqueContainerName, func(pod *v1.Pod) { + pod.Spec.Containers[0].Name = uniqueContainerName + }) + defer kubeClient.CoreV1().Pods(f.Namespace.Name).Delete(uniqueContainerName, &metav1.DeleteOptions{}) + + // Wait a short amount of time for Metadata Agent to be created and metadata to be exported + time.Sleep(metadataWaitTime) + + resp, err := oauthClient.Get(endpoint) + if err != nil { + framework.Failf("Failed to call Stackdriver Metadata API %s", err) + } + if resp.StatusCode != 200 { + framework.Failf("Stackdriver Metadata API returned error status: %s", resp.Status) + } + metadataAPIResponse, err := ioutil.ReadAll(resp.Body) + if err != nil { + framework.Failf("Failed to read response from Stackdriver Metadata API: %s", err) + } + + exists, err := verifyPodExists(metadataAPIResponse, uniqueContainerName) + if err != nil { + framework.Failf("Failed to process response from Stackdriver Metadata API: %s", err) + } + if !exists { + framework.Failf("Missing Metadata for container %q", uniqueContainerName) + } +} + +type Metadata struct { + Results []map[string]interface{} +} + +type Resource struct { + resourceType string + resourceLabels map[string]string +} + +func verifyPodExists(response []byte, containerName string) (bool, error) { + var metadata Metadata + err := json.Unmarshal(response, &metadata) + if err != nil { + return false, fmt.Errorf("Failed to unmarshall: %s", err) + } + + for _, result := range metadata.Results { + rawResource, ok := result["resource"] + if !ok { + return false, fmt.Errorf("No resource entry in response from Stackdriver Metadata API") + } + resource, err := parseResource(rawResource) + if err != nil { + return false, fmt.Errorf("No 'resource' label: %s", err) + } + if resource.resourceType == "k8s_container" && + resource.resourceLabels["container_name"] == containerName { + return true, nil + } + } + return false, nil +} + +func parseResource(resource interface{}) (*Resource, error) { + var labels map[string]string = map[string]string{} + resourceMap, ok := resource.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("Resource entry is of type %s, expected map[string]interface{}", reflect.TypeOf(resource)) + } + resourceType, ok := resourceMap["type"] + if !ok { + return nil, fmt.Errorf("Resource entry doesn't have a type specified") + } + resourceTypeName, ok := resourceType.(string) + if !ok { + return nil, fmt.Errorf("Resource type is of type %s, expected string", reflect.TypeOf(resourceType)) + } + resourceLabels, ok := resourceMap["labels"] + if !ok { + return nil, fmt.Errorf("Resource entry doesn't have any labels specified") + } + resourceLabelMap, ok := resourceLabels.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("Resource labels entry is of type %s, expected map[string]interface{}", reflect.TypeOf(resourceLabels)) + } + for label, val := range resourceLabelMap { + labels[label], ok = val.(string) + if !ok { + return nil, fmt.Errorf("Resource label %q is of type %s, expected string", label, reflect.TypeOf(val)) + } + } + return &Resource{ + resourceType: resourceTypeName, + resourceLabels: labels, + }, nil +}