diff --git a/test/e2e_node/disk_eviction_test.go b/test/e2e_node/disk_eviction_test.go index 2415dd158e8..3992ca8f1fd 100644 --- a/test/e2e_node/disk_eviction_test.go +++ b/test/e2e_node/disk_eviction_test.go @@ -110,6 +110,12 @@ var _ = framework.KubeDescribe("Kubelet Eviction Manager [Serial] [Disruptive]", nodeDiskPressureCondition := false podRescheduleable := false Eventually(func() error { + // Avoid the test using up all the disk space + err := checkDiskUsage(0.05) + if err != nil { + return err + } + // The pod should be evicted. if !evictionOccurred { podData, err := podClient.Get(busyPodName) @@ -227,3 +233,29 @@ func isImageSupported() bool { // instead of skipping images the eviction thresholds should be adjusted based on the images. return strings.Contains(framework.TestContext.NodeName, "-gci-dev-") } + +// checkDiskUsage verifies that the available bytes on disk are above the limit. +func checkDiskUsage(limit float64) error { + summary, err := getNodeSummary() + if err != nil { + return err + } + + if nodeFs := summary.Node.Fs; nodeFs != nil { + if nodeFs.AvailableBytes != nil && nodeFs.CapacityBytes != nil { + if float64(*nodeFs.CapacityBytes)*limit > float64(*nodeFs.AvailableBytes) { + return fmt.Errorf("available nodefs byte is less than %v%%", int(limit)*100) + } + } + } + + if summary.Node.Runtime != nil { + if imageFs := summary.Node.Runtime.ImageFs; imageFs != nil { + if float64(*imageFs.CapacityBytes)*limit > float64(*imageFs.AvailableBytes) { + return fmt.Errorf("available imagefs byte is less than %v%%", int(limit)*100) + } + } + } + + return nil +} diff --git a/test/e2e_node/kubelet_test.go b/test/e2e_node/kubelet_test.go index 6f8657e9315..aaeec9c1b98 100644 --- a/test/e2e_node/kubelet_test.go +++ b/test/e2e_node/kubelet_test.go @@ -18,10 +18,7 @@ package e2e_node import ( "bytes" - "encoding/json" "fmt" - "io/ioutil" - "net/http" "strings" "time" @@ -118,31 +115,20 @@ var _ = framework.KubeDescribe("Kubelet", func() { volumeNamePrefix := "test-empty-dir" podNames, volumes := createSummaryTestPods(f.PodClient(), podNamePrefix, 2, volumeNamePrefix) By("Returning stats summary") - summary := stats.Summary{} Eventually(func() error { - resp, err := http.Get(*kubeletAddress + "/stats/summary") + summary, err := getNodeSummary() if err != nil { - return fmt.Errorf("Failed to get /stats/summary - %v", err) + return fmt.Errorf("failed to get node summary: %v", err) } - contentsBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("Failed to read /stats/summary - %+v", resp) - } - contents := string(contentsBytes) - decoder := json.NewDecoder(strings.NewReader(contents)) - err = decoder.Decode(&summary) - if err != nil { - return fmt.Errorf("Failed to parse /stats/summary to go struct: %+v", resp) - } - missingPods := podsMissingFromSummary(summary, podNames) + missingPods := podsMissingFromSummary(*summary, podNames) if missingPods.Len() != 0 { return fmt.Errorf("expected pods not found. Following pods are missing - %v", missingPods) } - missingVolumes := volumesMissingFromSummary(summary, volumes) + missingVolumes := volumesMissingFromSummary(*summary, volumes) if missingVolumes.Len() != 0 { return fmt.Errorf("expected volumes not found. Following volumes are missing - %v", missingVolumes) } - if err := testSummaryMetrics(summary, podNamePrefix); err != nil { + if err := testSummaryMetrics(*summary, podNamePrefix); err != nil { return err } return nil diff --git a/test/e2e_node/memory_eviction_test.go b/test/e2e_node/memory_eviction_test.go index 457528be183..1847ba7a96c 100644 --- a/test/e2e_node/memory_eviction_test.go +++ b/test/e2e_node/memory_eviction_test.go @@ -17,18 +17,13 @@ limitations under the License. package e2e_node import ( - "encoding/json" "fmt" - "io/ioutil" - "net/http" "strconv" - "strings" "time" "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" - "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats" "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo" @@ -106,29 +101,8 @@ var _ = framework.KubeDescribe("MemoryEviction [Slow] [Serial] [Disruptive]", fu Eventually(func() bool { glog.Infof("Waiting for available memory to decrease to a reasonable level before ending the test.") - summary := stats.Summary{} - client := &http.Client{} - req, err := http.NewRequest("GET", "http://localhost:10255/stats/summary", nil) + summary, err := getNodeSummary() if err != nil { - glog.Warningf("Failed to build http request: %v", err) - return false - } - req.Header.Add("Accept", "application/json") - resp, err := client.Do(req) - if err != nil { - glog.Warningf("Failed to get /stats/summary: %v", err) - return false - } - contentsBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - glog.Warningf("Failed to read /stats/summary: %+v", resp) - return false - } - contents := string(contentsBytes) - decoder := json.NewDecoder(strings.NewReader(contents)) - err = decoder.Decode(&summary) - if err != nil { - glog.Warningf("Failed to parse /stats/summary to go struct: %+v", resp) return false } if summary.Node.Memory.AvailableBytes == nil { diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index ebb9f141917..91dcfeb22a4 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -17,7 +17,15 @@ limitations under the License. package e2e_node import ( + "encoding/json" "flag" + "io/ioutil" + "net/http" + "strings" + + "github.com/golang/glog" + + "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats" ) // TODO(random-liu): Get this automatically from kubelet flag. @@ -25,3 +33,35 @@ var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "H var startServices = flag.Bool("start-services", true, "If true, start local node services") var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests") + +func getNodeSummary() (*stats.Summary, error) { + req, err := http.NewRequest("GET", *kubeletAddress+"/stats/summary", nil) + if err != nil { + glog.Warningf("Failed to build http request: %v", err) + return nil, err + } + req.Header.Add("Accept", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + glog.Warningf("Failed to get /stats/summary: %v", err) + return nil, err + } + + defer resp.Body.Close() + contentsBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + glog.Warningf("Failed to read /stats/summary: %+v", resp) + return nil, err + } + + decoder := json.NewDecoder(strings.NewReader(string(contentsBytes))) + summary := stats.Summary{} + err = decoder.Decode(&summary) + if err != nil { + glog.Warningf("Failed to parse /stats/summary to go struct: %+v", resp) + return nil, err + } + return &summary, nil +}