From 16a495158715b0ddb09a0102f12f730f76a92e8b Mon Sep 17 00:00:00 2001
From: Jing Xu <jinxu@google.com>
Date: Tue, 11 Oct 2016 13:22:06 -0700
Subject: [PATCH] Add a retry when reading a file content from a container

To avoid temporal failure in reading the file content, add a retry
process in function verifyPDContentsViaContainer
---
 test/e2e/pd.go | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/test/e2e/pd.go b/test/e2e/pd.go
index a71608e554c..d61149c6ebd 100644
--- a/test/e2e/pd.go
+++ b/test/e2e/pd.go
@@ -49,6 +49,7 @@ const (
 	nodeStatusPollTime  = 1 * time.Second
 	gcePDRetryTimeout   = 5 * time.Minute
 	gcePDRetryPollTime  = 5 * time.Second
+	maxReadRetry        = 3
 )
 
 var _ = framework.KubeDescribe("Pod Disks", func() {
@@ -450,18 +451,25 @@ func deletePDWithRetry(diskName string) {
 
 func verifyPDContentsViaContainer(f *framework.Framework, podName, containerName string, fileAndContentToVerify map[string]string) {
 	for filePath, expectedContents := range fileAndContentToVerify {
-		v, err := f.ReadFileViaContainer(podName, containerName, filePath)
-		if err != nil {
-			framework.Logf("Error reading file: %v", err)
-		}
-		framework.ExpectNoError(err)
-		framework.Logf("Read file %q with content: %v", filePath, v)
-		if strings.TrimSpace(v) != strings.TrimSpace(expectedContents) {
-			size, err := f.CheckFileSizeViaContainer(podName, containerName, filePath)
+		var v string
+		// Add a retry to avoid temporal failure in reading the content
+		for i := 0; i < maxReadRetry; i++ {
+			v, err := f.ReadFileViaContainer(podName, containerName, filePath)
 			if err != nil {
 				framework.Logf("Error reading file: %v", err)
 			}
-			framework.Logf("Check file %q size: %q", filePath, size)
+			framework.ExpectNoError(err)
+			framework.Logf("Read file %q with content: %v", filePath, v)
+			if strings.TrimSpace(v) != strings.TrimSpace(expectedContents) {
+				framework.Logf("Warning: read content <%q> does not match execpted content <%q>.", v, expectedContents)
+				size, err := f.CheckFileSizeViaContainer(podName, containerName, filePath)
+				if err != nil {
+					framework.Logf("Error checking file size: %v", err)
+				}
+				framework.Logf("Check file %q size: %q", filePath, size)
+			} else {
+				break
+			}
 		}
 		Expect(strings.TrimSpace(v)).To(Equal(strings.TrimSpace(expectedContents)))
 	}