From a227ce42f2ca063c6a0b8204d280f4f4ff83c4bd Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Thu, 11 Aug 2016 10:48:09 -0700 Subject: [PATCH] Wait for memory to be reclaimed after node_e2e MemoryEviction test This helps prevent interference with other tests that run immediately after the MemoryEviction test. --- test/e2e_node/memory_eviction_test.go | 59 +++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/test/e2e_node/memory_eviction_test.go b/test/e2e_node/memory_eviction_test.go index 61739d1f414..dfac6182b47 100644 --- a/test/e2e_node/memory_eviction_test.go +++ b/test/e2e_node/memory_eviction_test.go @@ -17,13 +17,18 @@ limitations under the License. package e2e_node import ( + "encoding/json" "fmt" + "io/ioutil" + "net/http" "strconv" + "strings" "time" "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" @@ -96,6 +101,60 @@ var _ = framework.KubeDescribe("MemoryEviction [Slow] [Serial]", func() { }, 60*time.Minute, 5*time.Second).Should(Equal(true)) + // Wait for available memory to decrease to a reasonable level before ending the test. + // This prevents interference with tests that start immediately after this one. + Eventually(func() bool { + glog.Infof("Waiting for available memory to decrease to a reasonable level before ending the test.") + + summary := stats.Summary{} + client := &http.Client{} + req, err := http.NewRequest("GET", "http://localhost:10255/stats/summary", nil) + if err != nil { + glog.Warningf("Failed to build http request: %v", err) + return false + } + req.Header.Add("Accept", "application/json") + resp, err := client.Do(req) + if err != nil { + glog.Warningf("Failed to get /stats/summary: %v", err) + return false + } + contentsBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + glog.Warningf("Failed to read /stats/summary: %+v", resp) + return false + } + contents := string(contentsBytes) + decoder := json.NewDecoder(strings.NewReader(contents)) + err = decoder.Decode(&summary) + if err != nil { + glog.Warningf("Failed to parse /stats/summary to go struct: %+v", resp) + return false + } + if summary.Node.Memory.AvailableBytes == nil { + glog.Warningf("summary.Node.Memory.AvailableBytes was nil, cannot get memory stats.") + return false + } + if summary.Node.Memory.WorkingSetBytes == nil { + glog.Warningf("summary.Node.Memory.WorkingSetBytes was nil, cannot get memory stats.") + return false + } + avail := *summary.Node.Memory.AvailableBytes + wset := *summary.Node.Memory.WorkingSetBytes + + // memory limit = avail + wset + limit := avail + wset + halflimit := limit / 2 + + // Wait for at least half of memory limit to be available + glog.Infof("Current available memory is: %d bytes. Waiting for at least %d bytes available.", avail, halflimit) + if avail >= halflimit { + return true + } + + return false + }, 5*time.Minute, 5*time.Second).Should(Equal(true)) + }) })