Merge pull request #31391 from ronnielai/container-gc

Automatic merge from submit-queue

Avoid disk eviction node e2e test using up all the disk space

test/e2e_node/disk_eviction_test.go

@@ -110,6 +110,12 @@ var _ = framework.KubeDescribe("Kubelet Eviction Manager [Serial] [Disruptive]",
 				nodeDiskPressureCondition := false
 				podRescheduleable := false
 				Eventually(func() error {
+					// Avoid the test using up all the disk space
+					err := checkDiskUsage(0.05)
+					if err != nil {
+						return err
+					}
+
 					// The pod should be evicted.
 					if !evictionOccurred {
 						podData, err := podClient.Get(busyPodName)
@@ -227,3 +233,29 @@ func isImageSupported() bool {
 	// instead of skipping images the eviction thresholds should be adjusted based on the images.
 	return strings.Contains(framework.TestContext.NodeName, "-gci-dev-")
 }
+
+// checkDiskUsage verifies that the available bytes on disk are above the limit.
+func checkDiskUsage(limit float64) error {
+	summary, err := getNodeSummary()
+	if err != nil {
+		return err
+	}
+
+	if nodeFs := summary.Node.Fs; nodeFs != nil {
+		if nodeFs.AvailableBytes != nil && nodeFs.CapacityBytes != nil {
+			if float64(*nodeFs.CapacityBytes)*limit > float64(*nodeFs.AvailableBytes) {
+				return fmt.Errorf("available nodefs byte is less than %v%%", int(limit)*100)
+			}
+		}
+	}
+
+	if summary.Node.Runtime != nil {
+		if imageFs := summary.Node.Runtime.ImageFs; imageFs != nil {
+			if float64(*imageFs.CapacityBytes)*limit > float64(*imageFs.AvailableBytes) {
+				return fmt.Errorf("available imagefs byte is less than %v%%", int(limit)*100)
+			}
+		}
+	}
+
+	return nil
+}

test/e2e_node/kubelet_test.go

@@ -18,10 +18,7 @@ package e2e_node
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"net/http"
 	"strings"
 	"time"
 
@@ -118,31 +115,20 @@ var _ = framework.KubeDescribe("Kubelet", func() {
 				volumeNamePrefix := "test-empty-dir"
 				podNames, volumes := createSummaryTestPods(f.PodClient(), podNamePrefix, 2, volumeNamePrefix)
 				By("Returning stats summary")
-				summary := stats.Summary{}
 				Eventually(func() error {
-					resp, err := http.Get(*kubeletAddress + "/stats/summary")
+					summary, err := getNodeSummary()
 					if err != nil {
-						return fmt.Errorf("Failed to get /stats/summary - %v", err)
+						return fmt.Errorf("failed to get node summary: %v", err)
 					}
-					contentsBytes, err := ioutil.ReadAll(resp.Body)
+					missingPods := podsMissingFromSummary(*summary, podNames)
-					if err != nil {
-						return fmt.Errorf("Failed to read /stats/summary - %+v", resp)
-					}
-					contents := string(contentsBytes)
-					decoder := json.NewDecoder(strings.NewReader(contents))
-					err = decoder.Decode(&summary)
-					if err != nil {
-						return fmt.Errorf("Failed to parse /stats/summary to go struct: %+v", resp)
-					}
-					missingPods := podsMissingFromSummary(summary, podNames)
 					if missingPods.Len() != 0 {
 						return fmt.Errorf("expected pods not found. Following pods are missing - %v", missingPods)
 					}
-					missingVolumes := volumesMissingFromSummary(summary, volumes)
+					missingVolumes := volumesMissingFromSummary(*summary, volumes)
 					if missingVolumes.Len() != 0 {
 						return fmt.Errorf("expected volumes not found. Following volumes are missing - %v", missingVolumes)
 					}
-					if err := testSummaryMetrics(summary, podNamePrefix); err != nil {
+					if err := testSummaryMetrics(*summary, podNamePrefix); err != nil {
 						return err
 					}
 					return nil

test/e2e_node/memory_eviction_test.go

@@ -17,18 +17,13 @@ limitations under the License.
 package e2e_node
 
 import (
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"net/http"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
-	"k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
@@ -106,29 +101,8 @@ var _ = framework.KubeDescribe("MemoryEviction [Slow] [Serial] [Disruptive]", fu
 			Eventually(func() bool {
 				glog.Infof("Waiting for available memory to decrease to a reasonable level before ending the test.")
 
-				summary := stats.Summary{}
+				summary, err := getNodeSummary()
-				client := &http.Client{}
-				req, err := http.NewRequest("GET", "http://localhost:10255/stats/summary", nil)
 				if err != nil {
-					glog.Warningf("Failed to build http request: %v", err)
-					return false
-				}
-				req.Header.Add("Accept", "application/json")
-				resp, err := client.Do(req)
-				if err != nil {
-					glog.Warningf("Failed to get /stats/summary: %v", err)
-					return false
-				}
-				contentsBytes, err := ioutil.ReadAll(resp.Body)
-				if err != nil {
-					glog.Warningf("Failed to read /stats/summary: %+v", resp)
-					return false
-				}
-				contents := string(contentsBytes)
-				decoder := json.NewDecoder(strings.NewReader(contents))
-				err = decoder.Decode(&summary)
-				if err != nil {
-					glog.Warningf("Failed to parse /stats/summary to go struct: %+v", resp)
 					return false
 				}
 				if summary.Node.Memory.AvailableBytes == nil {

test/e2e_node/util.go

@@ -17,7 +17,15 @@ limitations under the License.
 package e2e_node
 
 import (
+	"encoding/json"
 	"flag"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/golang/glog"
+
+	"k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
 )
 
 // TODO(random-liu): Get this automatically from kubelet flag.
@@ -25,3 +33,35 @@ var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "H
 
 var startServices = flag.Bool("start-services", true, "If true, start local node services")
 var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")
+
+func getNodeSummary() (*stats.Summary, error) {
+	req, err := http.NewRequest("GET", *kubeletAddress+"/stats/summary", nil)
+	if err != nil {
+		glog.Warningf("Failed to build http request: %v", err)
+		return nil, err
+	}
+	req.Header.Add("Accept", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		glog.Warningf("Failed to get /stats/summary: %v", err)
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	contentsBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		glog.Warningf("Failed to read /stats/summary: %+v", resp)
+		return nil, err
+	}
+
+	decoder := json.NewDecoder(strings.NewReader(string(contentsBytes)))
+	summary := stats.Summary{}
+	err = decoder.Decode(&summary)
+	if err != nil {
+		glog.Warningf("Failed to parse /stats/summary to go struct: %+v", resp)
+		return nil, err
+	}
+	return &summary, nil
+}