From 1c22fd39ff34165cccb1f627b33a3d5403738997 Mon Sep 17 00:00:00 2001 From: gmarek Date: Thu, 16 Jul 2015 16:06:02 +0200 Subject: [PATCH] Add e2e test monitoring current container resource usage.wq --- hack/jenkins/e2e.sh | 1 + test/e2e/kubelet_stats.go | 4 ++ test/e2e/monitor_resources.go | 132 ++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 test/e2e/monitor_resources.go diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 95e4c70ec30..4815e92b039 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -105,6 +105,7 @@ GCE_PARALLEL_SKIP_TESTS=( "Nodes\sNetwork" "Nodes\sResize" "MaxPods" + "ResourceUsage" "SchedulerPredicates" "Services.*restarting" "Shell.*services" diff --git a/test/e2e/kubelet_stats.go b/test/e2e/kubelet_stats.go index 014265ad5d4..53353c1b4dd 100644 --- a/test/e2e/kubelet_stats.go +++ b/test/e2e/kubelet_stats.go @@ -195,6 +195,10 @@ type containerResourceUsage struct { CPUInterval time.Duration } +func (r *containerResourceUsage) isStrictlyGreaterThan(rhs *containerResourceUsage) bool { + return r.CPUUsageInCores > rhs.CPUUsageInCores && r.MemoryUsageInBytes > rhs.MemoryUsageInBytes && r.MemoryWorkingSetInBytes > rhs.MemoryWorkingSetInBytes +} + // getOneTimeResourceUsageOnNode queries the node's /stats/container endpoint // and returns the resource usage of targetContainers for the past // cpuInterval. diff --git a/test/e2e/monitor_resources.go b/test/e2e/monitor_resources.go new file mode 100644 index 00000000000..88ef6156aab --- /dev/null +++ b/test/e2e/monitor_resources.go @@ -0,0 +1,132 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "time" + + "k8s.io/kubernetes/pkg/client" + "k8s.io/kubernetes/pkg/fields" + "k8s.io/kubernetes/pkg/labels" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const datapointAmount = 5 + +type resourceUsagePerContainer map[string]*containerResourceUsage + +//TODO tweak those values. +var allowedUsage = resourceUsagePerContainer{ + "/": &containerResourceUsage{ + CPUUsageInCores: 0.3, + MemoryUsageInBytes: 4000000000, + MemoryWorkingSetInBytes: 2000000000, + }, + "/docker-daemon": &containerResourceUsage{ + CPUUsageInCores: 0.05, + MemoryUsageInBytes: 2500000000, + MemoryWorkingSetInBytes: 1500000000, + }, + "/kubelet": &containerResourceUsage{ + CPUUsageInCores: 0.1, + MemoryUsageInBytes: 80000000, + MemoryWorkingSetInBytes: 80000000, + }, + "/kube-proxy": &containerResourceUsage{ + CPUUsageInCores: 0.02, + MemoryUsageInBytes: 12000000, + MemoryWorkingSetInBytes: 12000000, + }, + "/system": &containerResourceUsage{ + CPUUsageInCores: 0.01, + MemoryUsageInBytes: 70000000, + MemoryWorkingSetInBytes: 35000000, + }, +} + +func computeAverage(sliceOfUsages []resourceUsagePerContainer) (result resourceUsagePerContainer) { + result = make(resourceUsagePerContainer) + for _, container := range targetContainers { + result[container] = &containerResourceUsage{} + } + for _, usage := range sliceOfUsages { + for _, container := range targetContainers { + singleResult := &containerResourceUsage{ + CPUUsageInCores: result[container].CPUUsageInCores + usage[container].CPUUsageInCores, + MemoryUsageInBytes: result[container].MemoryUsageInBytes + usage[container].MemoryUsageInBytes, + MemoryWorkingSetInBytes: result[container].MemoryWorkingSetInBytes + usage[container].MemoryWorkingSetInBytes, + } + result[container] = singleResult + } + } + for _, container := range targetContainers { + singleResult := &containerResourceUsage{ + CPUUsageInCores: result[container].CPUUsageInCores / float64(len(sliceOfUsages)), + MemoryUsageInBytes: result[container].MemoryUsageInBytes / int64(len(sliceOfUsages)), + MemoryWorkingSetInBytes: result[container].MemoryWorkingSetInBytes / int64(len(sliceOfUsages)), + } + result[container] = singleResult + } + return +} + +// This tests does nothing except checking current resource usage of containers defained in kubelet_stats targetContainers variable. +// Test fails if an average container resource consumption over datapointAmount tries exceeds amount defined in allowedUsage. +var _ = Describe("ResourceUsage", func() { + var c *client.Client + BeforeEach(func() { + var err error + c, err = loadClient() + expectNoError(err) + }) + + It("should not exceed expected amount.", func() { + By("Getting ResourceConsumption on all nodes") + nodeList, err := c.Nodes().List(labels.Everything(), fields.Everything()) + expectNoError(err) + + resourceUsagePerNode := make(map[string][]resourceUsagePerContainer) + + for i := 0; i < datapointAmount; i++ { + for _, node := range nodeList.Items { + resourceUsage, err := getOneTimeResourceUsageOnNode(c, node.Name, 5*time.Second) + expectNoError(err) + resourceUsagePerNode[node.Name] = append(resourceUsagePerNode[node.Name], resourceUsage) + } + time.Sleep(3 * time.Second) + } + + averageResourceUsagePerNode := make(map[string]resourceUsagePerContainer) + for _, node := range nodeList.Items { + averageResourceUsagePerNode[node.Name] = computeAverage(resourceUsagePerNode[node.Name]) + } + + violating := make(map[string]resourceUsagePerContainer) + for node, usage := range averageResourceUsagePerNode { + for container, cUsage := range usage { + Logf("%v on %v usage: %#v", container, node, cUsage) + if !allowedUsage[container].isStrictlyGreaterThan(cUsage) { + violating[node] = usage + } + } + } + + Expect(violating).To(BeEmpty()) + }) +})