From 736f1cb7c3071eeca137c615cc8cc4080a3751c5 Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Tue, 31 May 2016 15:35:39 -0700 Subject: [PATCH] Node e2e memory eviction test This test creates three pods with QoS of besteffort, burstable, and guaranteed, respectively, which each contain a container that tries to consume almost all the available memory at a rate of about 12Mi/10sec. The expectation is that eviction will be initiated when the hard memory.available<250Mi threshold is triggered, and that eviction will proceed in the order of besteffort, then burstable. Since guaranteed pods should only be evicted if something charged to the host uses more resources than were reserved for it, we currently end the test when besteffort and burstable have both been evicted. Note that this commit also sets --eviction-hard=memory.available<250Mi to enable eviction during tests. --- test/e2e/framework/test_context.go | 2 +- test/e2e_node/memory_eviction_test.go | 153 ++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 test/e2e_node/memory_eviction_test.go diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 5d9a5589349..5dcb3c1017b 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -152,5 +152,5 @@ func RegisterClusterFlags() { func RegisterNodeFlags() { flag.StringVar(&TestContext.NodeName, "node-name", "", "Name of the node to run tests on (node e2e suite only).") flag.BoolVar(&TestContext.CgroupsPerQOS, "cgroups-per-qos", false, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.") - flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.") + flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "memory.available<250Mi", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.") } diff --git a/test/e2e_node/memory_eviction_test.go b/test/e2e_node/memory_eviction_test.go new file mode 100644 index 00000000000..61739d1f414 --- /dev/null +++ b/test/e2e_node/memory_eviction_test.go @@ -0,0 +1,153 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_node + +import ( + "fmt" + "strconv" + "time" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/test/e2e/framework" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +// Eviction Policy is described here: +// https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/kubelet-eviction.md + +var _ = framework.KubeDescribe("MemoryEviction [Slow] [Serial]", func() { + f := framework.NewDefaultFramework("eviction-test") + + Context("When there is memory pressure", func() { + It("It should evict pods in the correct order (besteffort first, then burstable, then guaranteed)", func() { + By("Creating a guaranteed pod, a burstable pod, and a besteffort pod.") + + // A pod is guaranteed only when requests and limits are specified for all the containers and they are equal. + guaranteed := createMemhogPod(f, "guaranteed-", "guaranteed", api.ResourceRequirements{ + Requests: api.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("100Mi"), + }, + Limits: api.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("100Mi"), + }}) + + // A pod is burstable if limits and requests do not match across all containers. + burstable := createMemhogPod(f, "burstable-", "burstable", api.ResourceRequirements{ + Requests: api.ResourceList{ + "cpu": resource.MustParse("100m"), + "memory": resource.MustParse("100Mi"), + }}) + + // A pod is besteffort if none of its containers have specified any requests or limits. + besteffort := createMemhogPod(f, "besteffort-", "besteffort", api.ResourceRequirements{}) + + // We poll until timeout or all pods are killed. + // Inside the func, we check that all pods are in a valid phase with + // respect to the eviction order of best effort, then burstable, then guaranteed. + By("Polling the Status.Phase of each pod and checking for violations of the eviction order.") + Eventually(func() bool { + + gteed, gtErr := f.Client.Pods(f.Namespace.Name).Get(guaranteed.Name) + framework.ExpectNoError(gtErr, fmt.Sprintf("getting pod %s", guaranteed.Name)) + gteedPh := gteed.Status.Phase + + burst, buErr := f.Client.Pods(f.Namespace.Name).Get(burstable.Name) + framework.ExpectNoError(buErr, fmt.Sprintf("getting pod %s", burstable.Name)) + burstPh := burst.Status.Phase + + best, beErr := f.Client.Pods(f.Namespace.Name).Get(besteffort.Name) + framework.ExpectNoError(beErr, fmt.Sprintf("getting pod %s", besteffort.Name)) + bestPh := best.Status.Phase + + glog.Infof("Pod phase: guaranteed: %v, burstable: %v, besteffort: %v", gteedPh, burstPh, bestPh) + + if bestPh == api.PodRunning { + Expect(burstPh).NotTo(Equal(api.PodFailed), "Burstable pod failed before best effort pod") + Expect(gteedPh).NotTo(Equal(api.PodFailed), "Guaranteed pod failed before best effort pod") + } else if burstPh == api.PodRunning { + Expect(gteedPh).NotTo(Equal(api.PodFailed), "Guaranteed pod failed before burstable pod") + } + + // When both besteffort and burstable have been evicted, return true, else false + if bestPh == api.PodFailed && burstPh == api.PodFailed { + return true + } + return false + + }, 60*time.Minute, 5*time.Second).Should(Equal(true)) + + }) + }) + +}) + +func createMemhogPod(f *framework.Framework, genName string, ctnName string, res api.ResourceRequirements) *api.Pod { + env := []api.EnvVar{ + { + Name: "MEMORY_LIMIT", + ValueFrom: &api.EnvVarSource{ + ResourceFieldRef: &api.ResourceFieldSelector{ + Resource: "limits.memory", + }, + }, + }, + } + + // If there is a limit specified, pass 80% of it for -mem-total, otherwise use the downward API + // to pass limits.memory, which will be the total memory available. + // This helps prevent a guaranteed pod from triggering an OOM kill due to it's low memory limit, + // which will cause the test to fail inappropriately. + var memLimit string + if limit, ok := res.Limits["memory"]; ok { + memLimit = strconv.Itoa(int( + float64(limit.Value()) * 0.8)) + } else { + memLimit = "$(MEMORY_LIMIT)" + } + + pod := &api.Pod{ + ObjectMeta: api.ObjectMeta{ + GenerateName: genName, + }, + Spec: api.PodSpec{ + RestartPolicy: api.RestartPolicyNever, + Containers: []api.Container{ + { + Name: ctnName, + Image: "gcr.io/google-containers/stress:v1", + ImagePullPolicy: "Always", + Env: env, + // 60 min timeout * 60s / tick per 10s = 360 ticks before timeout => ~11.11Mi/tick + // to fill ~4Gi of memory, so initial ballpark 12Mi/tick. + // We might see flakes due to timeout if the total memory on the nodes increases. + Args: []string{"-mem-alloc-size", "12Mi", "-mem-alloc-sleep", "10s", "-mem-total", memLimit}, + Resources: res, + }, + }, + }, + } + // The generated pod.Name will be on the pod spec returned by CreateSync + pod = f.PodClient().CreateSync(pod) + glog.Infof("pod created with name: %s", pod.Name) + return pod +}