Kubelet Metrics Summary Api Implementation

test/e2e_node/e2e_node_suite_test.go

@@ -15,7 +15,8 @@ limitations under the License.
 */
 
 // To run tests in this suite
-// `$ ginkgo -- --node-name node-e2e-test-1  --api-server-address <serveraddress> --logtostderr`
+// Local: `$ ginkgo -- --logtostderr -v 2`
+// Remote: `$ ginkgo -- --node-name <hostname> --api-server-address=<hostname:api_port> --kubelet-address=<hostname=kubelet_port> --logtostderr -v 2`
 package e2e_node
 
 import (
@@ -26,9 +27,9 @@ import (
 	"testing"
 )
 
-var kubeletAddress = flag.String("kubelet-address", "localhost:10250", "Host and port of the kubelet")
-var apiServerAddress = flag.String("api-server-address", "localhost:8080", "Host and port of the api server")
-var nodeName = flag.String("node-name", "", "Name of the node")
+var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet")
+var apiServerAddress = flag.String("api-server-address", "http://127.0.0.1:8080", "Host and port of the api server")
+var nodeName = flag.String("node-name", "127.0.0.1", "Name of the node")
 
 func TestE2eNode(t *testing.T) {
 	flag.Parse()

test/e2e_node/kubelet_test.go

@@ -18,13 +18,18 @@ package e2e_node
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"io/ioutil"
 	"k8s.io/kubernetes/pkg/api"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/kubelet/server/stats"
 )
 
 var _ = Describe("Kubelet", func() {
@@ -49,10 +54,9 @@ var _ = Describe("Kubelet", func() {
 						RestartPolicy: api.RestartPolicyNever,
 						Containers: []api.Container{
 							{
-								Image:           "gcr.io/google_containers/busybox",
-								Name:            "busybox",
-								Command:         []string{"echo", "'Hello World'"},
-								ImagePullPolicy: api.PullIfNotPresent,
+								Image:   "gcr.io/google_containers/busybox",
+								Name:    "busybox",
+								Command: []string{"echo", "'Hello World'"},
 							},
 						},
 					},
@@ -84,4 +88,125 @@ var _ = Describe("Kubelet", func() {
 			})
 		})
 	})
+
+	Describe("metrics api", func() {
+		statsPrefix := "stats-busybox-"
+		podNames := []string{}
+		podCount := 2
+		for i := 0; i < podCount; i++ {
+			podNames = append(podNames, fmt.Sprintf("%s%v", statsPrefix, i))
+		}
+		BeforeEach(func() {
+			for _, podName := range podNames {
+				createPod(cl, podName, []api.Container{
+					{
+						Image:   "gcr.io/google_containers/busybox",
+						Command: []string{"sh", "-c", "echo 'Hello World' | tee ~/file | tee -a ~/file | tee /test-empty-dir | sleep 60"},
+						Name:    podName + containerSuffix,
+					},
+				})
+			}
+
+			// Sleep long enough for cadvisor to see the pod and calculate all of its metrics
+			time.Sleep(60 * time.Second)
+		})
+
+		Context("when querying /stats/summary", func() {
+			It("it should report resource usage through the stats api", func() {
+				resp, err := http.Get(*kubeletAddress + "/stats/summary")
+				now := time.Now()
+				Expect(err).To(BeNil(), fmt.Sprintf("Failed to get /stats/summary"))
+				summary := stats.Summary{}
+				contentsBytes, err := ioutil.ReadAll(resp.Body)
+				Expect(err).To(BeNil(), fmt.Sprintf("Failed to read /stats/summary: %+v", resp))
+				contents := string(contentsBytes)
+				decoder := json.NewDecoder(strings.NewReader(contents))
+				err = decoder.Decode(&summary)
+				Expect(err).To(BeNil(), fmt.Sprintf("Failed to parse /stats/summary to go struct: %+v", resp))
+
+				// Verify Misc Stats
+				Expect(summary.Time.Time).To(BeTemporally("~", now, 20*time.Second))
+
+				// Verify Node Stats are present
+				Expect(summary.Node.NodeName).To(Equal(*nodeName))
+				Expect(summary.Node.CPU.UsageCoreNanoSeconds).NotTo(BeZero())
+				Expect(summary.Node.Memory.UsageBytes).NotTo(BeZero())
+				Expect(summary.Node.Memory.WorkingSetBytes).NotTo(BeZero())
+				Expect(summary.Node.Fs.UsedBytes).NotTo(BeZero())
+				Expect(summary.Node.Fs.CapacityBytes).NotTo(BeZero())
+				Expect(summary.Node.Fs.AvailableBytes).NotTo(BeZero())
+
+				sysContainers := map[string]stats.ContainerStats{}
+				sysContainersList := []string{}
+				for _, container := range summary.Node.SystemContainers {
+					sysContainers[container.Name] = container
+					sysContainersList = append(sysContainersList, container.Name)
+					Expect(container.CPU.UsageCoreNanoSeconds).NotTo(BeZero())
+					// TODO: Test Network
+					Expect(container.Memory.UsageBytes).NotTo(BeZero())
+					Expect(container.Memory.WorkingSetBytes).NotTo(BeZero())
+					Expect(container.Rootfs.CapacityBytes).NotTo(BeZero())
+					Expect(container.Rootfs.AvailableBytes).NotTo(BeZero())
+					Expect(container.Logs.CapacityBytes).NotTo(BeZero())
+					Expect(container.Logs.AvailableBytes).NotTo(BeZero())
+				}
+				Expect(sysContainersList).To(ConsistOf("kubelet", "runtime"))
+
+				// Verify Pods Stats are present
+				podsList := []string{}
+				for _, pod := range summary.Pods {
+					if !strings.HasPrefix(pod.PodRef.Name, statsPrefix) {
+						// Ignore pods created outside this test
+						continue
+
+					}
+					// TODO: Test network
+
+					podsList = append(podsList, pod.PodRef.Name)
+					Expect(pod.Containers).To(HaveLen(1))
+					container := pod.Containers[0]
+					Expect(container.Name).To(Equal(pod.PodRef.Name + containerSuffix))
+					Expect(container.CPU.UsageCoreNanoSeconds).NotTo(BeZero())
+					Expect(container.Memory.UsageBytes).NotTo(BeZero())
+					Expect(container.Memory.WorkingSetBytes).NotTo(BeZero())
+					Expect(container.Rootfs.CapacityBytes).NotTo(BeZero())
+					Expect(container.Rootfs.AvailableBytes).NotTo(BeZero())
+					Expect(*container.Rootfs.UsedBytes).NotTo(BeZero(), contents)
+					Expect(container.Logs.CapacityBytes).NotTo(BeZero())
+					Expect(container.Logs.AvailableBytes).NotTo(BeZero())
+					Expect(*container.Logs.UsedBytes).NotTo(BeZero(), contents)
+				}
+				Expect(podsList).To(ConsistOf(podNames))
+			})
+		})
+
+		AfterEach(func() {
+			for _, podName := range podNames {
+				err := cl.Pods(api.NamespaceDefault).Delete(podName, &api.DeleteOptions{})
+				Expect(err).To(BeNil(), fmt.Sprintf("Error deleting Pod %v", podName))
+			}
+		})
+	})
 })
+
+const (
+	containerSuffix = "-c"
+)
+
+func createPod(cl *client.Client, podName string, containers []api.Container) {
+	pod := &api.Pod{
+		ObjectMeta: api.ObjectMeta{
+			Name:      podName,
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: api.PodSpec{
+			// Force the Pod to schedule to the node without a scheduler running
+			NodeName: *nodeName,
+			// Don't restart the Pod since it is expected to exit
+			RestartPolicy: api.RestartPolicyNever,
+			Containers:    containers,
+		},
+	}
+	_, err := cl.Pods(api.NamespaceDefault).Create(pod)
+	Expect(err).To(BeNil(), fmt.Sprintf("Error creating Pod %v", err))
+}