e2e: abstract access to additional files

The new test/e2e/framework/testfiles package makes it possible to
write tests that do not depend on a specific way of providing
additional test files at runtime. Such tests and the framework are
then more easily reused in other test suites.

In the test/e2e suite file access is enabled based on the existing
"repo-root" command line parameter and the built-in bindata. Tests
using the new API will first check for files under "repo-root" and
then fall back to the builtin data. This way, users of a test binary
can modify those files without having to rebuild the binary.

"repo-root" is still needed because at least some tests check for
additional files (secret.yaml, via ingress_utils.go) that are not part
of the upstream source code and thus may or may not be built into a
test binary.

Tests using bindata or repo-root directly get modified to use the new
API, or removed when they are obsolete: test/e2e/examples.go depended
on files that were removed in
https://github.com/kubernetes/kubernetes/pull/61246 and thus can no
longer be run in Kubernetes. Moving the tests to kubernetes/examples
is tracked in https://github.com/kubernetes/examples/issues/214.

The file removal did not break the automated E2E testing probably
because the tests are under the Feature:Example tag and thus not
enabled during normal CI runs.

Removing also the obsolete tests makes it simpler to rework the
"repo-root" setting because less code uses it.

Related-to: #66649 and #23987

test/e2e/e2e_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	"k8s.io/kubernetes/test/e2e/generated"
 
 	// test sources
 	_ "k8s.io/kubernetes/test/e2e/apimachinery"
@@ -42,6 +44,21 @@ import (
 
 func init() {
 	framework.ViperizeFlags()
+
+	// TODO: Deprecating repo-root over time... instead just use gobindata_util.go , see #23987.
+	// Right now it is still needed, for example by
+	// test/e2e/framework/ingress/ingress_utils.go
+	// for providing the optional secret.yaml file and by
+	// test/e2e/framework/util.go for cluster/log-dump.
+	if framework.TestContext.RepoRoot != "" {
+		testfiles.AddFileSource(testfiles.RootFileSource{Root: framework.TestContext.RepoRoot})
+	}
+
+	// Enable bindata file lookup as fallback.
+	testfiles.AddFileSource(testfiles.BindataFileSource{
+		Asset:      generated.Asset,
+		AssetNames: generated.AssetNames,
+	})
 }
 
 func TestE2E(t *testing.T) {

test/e2e/examples.go

@@ -17,28 +17,19 @@ limitations under the License.
 package e2e
 
 import (
-	"context"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"path/filepath"
-	"regexp"
-	"strings"
 	"sync"
 	"time"
 
-	"k8s.io/api/core/v1"
 	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	clientset "k8s.io/client-go/kubernetes"
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	"k8s.io/kubernetes/test/e2e/framework"
-	"k8s.io/kubernetes/test/e2e/generated"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
-	testutils "k8s.io/kubernetes/test/utils"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -51,19 +42,6 @@ const (
 var _ = framework.KubeDescribe("[Feature:Example]", func() {
 	f := framework.NewDefaultFramework("examples")
 
-	// Reusable cluster state function.  This won't be adversly affected by lazy initialization of framework.
-	clusterState := func(selectorKey string, selectorValue string) *framework.ClusterVerification {
-		return f.NewClusterVerification(
-			f.Namespace,
-			framework.PodStateVerification{
-				Selectors:   map[string]string{selectorKey: selectorValue},
-				ValidPhases: []v1.PodPhase{v1.PodRunning},
-			})
-	}
-	// Customized ForEach wrapper for this test.
-	forEachPod := func(selectorKey string, selectorValue string, fn func(v1.Pod)) {
-		clusterState(selectorKey, selectorValue).ForEach(fn)
-	}
 	var c clientset.Interface
 	var ns string
 	BeforeEach(func() {
@@ -81,330 +59,15 @@ var _ = framework.KubeDescribe("[Feature:Example]", func() {
 		framework.ExpectNoError(err)
 	})
 
-	framework.KubeDescribe("Redis", func() {
-		It("should create and stop redis servers", func() {
-			mkpath := func(file string) string {
-				return filepath.Join(framework.TestContext.RepoRoot, "examples/storage/redis", file)
-			}
-			bootstrapYaml := mkpath("redis-master.yaml")
-			sentinelServiceYaml := mkpath("redis-sentinel-service.yaml")
-			sentinelControllerYaml := mkpath("redis-sentinel-controller.yaml")
-			controllerYaml := mkpath("redis-controller.yaml")
-
-			bootstrapPodName := "redis-master"
-			redisRC := "redis"
-			sentinelRC := "redis-sentinel"
-			nsFlag := fmt.Sprintf("--namespace=%v", ns)
-			expectedOnServer := "The server is now ready to accept connections"
-			expectedOnSentinel := "+monitor master"
-
-			By("starting redis bootstrap")
-			framework.RunKubectlOrDie("create", "-f", bootstrapYaml, nsFlag)
-			err := framework.WaitForPodNameRunningInNamespace(c, bootstrapPodName, ns)
-			Expect(err).NotTo(HaveOccurred())
-
-			_, err = framework.LookForStringInLog(ns, bootstrapPodName, "master", expectedOnServer, serverStartTimeout)
-			Expect(err).NotTo(HaveOccurred())
-			_, err = framework.LookForStringInLog(ns, bootstrapPodName, "sentinel", expectedOnSentinel, serverStartTimeout)
-			Expect(err).NotTo(HaveOccurred())
-
-			By("setting up services and controllers")
-			framework.RunKubectlOrDie("create", "-f", sentinelServiceYaml, nsFlag)
-			framework.RunKubectlOrDie("create", "-f", sentinelControllerYaml, nsFlag)
-			framework.RunKubectlOrDie("create", "-f", controllerYaml, nsFlag)
-			label := labels.SelectorFromSet(labels.Set(map[string]string{sentinelRC: "true"}))
-			err = testutils.WaitForPodsWithLabelRunning(c, ns, label)
-			Expect(err).NotTo(HaveOccurred())
-			label = labels.SelectorFromSet(labels.Set(map[string]string{"name": redisRC}))
-			err = testutils.WaitForPodsWithLabelRunning(c, ns, label)
-			Expect(err).NotTo(HaveOccurred())
-
-			By("scaling up the deployment")
-			framework.RunKubectlOrDie("scale", "rc", redisRC, "--replicas=3", nsFlag)
-			framework.RunKubectlOrDie("scale", "rc", sentinelRC, "--replicas=3", nsFlag)
-			framework.WaitForRCToStabilize(c, ns, redisRC, framework.PodReadyBeforeTimeout)
-			framework.WaitForRCToStabilize(c, ns, sentinelRC, framework.PodReadyBeforeTimeout)
-
-			By("checking up the services")
-			checkAllLogs := func() {
-				selectorKey, selectorValue := "name", redisRC
-				label := labels.SelectorFromSet(labels.Set(map[string]string{selectorKey: selectorValue}))
-				err = testutils.WaitForPodsWithLabelRunning(c, ns, label)
-				Expect(err).NotTo(HaveOccurred())
-				forEachPod(selectorKey, selectorValue, func(pod v1.Pod) {
-					if pod.Name != bootstrapPodName {
-						_, err := framework.LookForStringInLog(ns, pod.Name, "redis", expectedOnServer, serverStartTimeout)
-						Expect(err).NotTo(HaveOccurred())
-					}
-				})
-				selectorKey, selectorValue = sentinelRC, "true"
-				label = labels.SelectorFromSet(labels.Set(map[string]string{selectorKey: selectorValue}))
-				err = testutils.WaitForPodsWithLabelRunning(c, ns, label)
-				Expect(err).NotTo(HaveOccurred())
-				forEachPod(selectorKey, selectorValue, func(pod v1.Pod) {
-					if pod.Name != bootstrapPodName {
-						_, err := framework.LookForStringInLog(ns, pod.Name, "sentinel", expectedOnSentinel, serverStartTimeout)
-						Expect(err).NotTo(HaveOccurred())
-					}
-				})
-			}
-			checkAllLogs()
-
-			By("turning down bootstrap")
-			framework.RunKubectlOrDie("delete", "-f", bootstrapYaml, nsFlag)
-			err = framework.WaitForRCPodToDisappear(c, ns, redisRC, bootstrapPodName)
-			Expect(err).NotTo(HaveOccurred())
-			By("waiting for the new master election")
-			checkAllLogs()
-		})
-	})
-
-	framework.KubeDescribe("Spark", func() {
-		It("should start spark master, driver and workers", func() {
-			mkpath := func(file string) string {
-				return filepath.Join(framework.TestContext.RepoRoot, "examples/spark", file)
-			}
-
-			// TODO: Add Zepplin and Web UI to this example.
-			serviceYaml := mkpath("spark-master-service.yaml")
-			masterYaml := mkpath("spark-master-controller.yaml")
-			workerControllerYaml := mkpath("spark-worker-controller.yaml")
-			nsFlag := fmt.Sprintf("--namespace=%v", ns)
-
-			master := func() {
-				By("starting master")
-				framework.RunKubectlOrDie("create", "-f", serviceYaml, nsFlag)
-				framework.RunKubectlOrDie("create", "-f", masterYaml, nsFlag)
-				selectorKey, selectorValue := "component", "spark-master"
-				label := labels.SelectorFromSet(labels.Set(map[string]string{selectorKey: selectorValue}))
-				err := testutils.WaitForPodsWithLabelRunning(c, ns, label)
-				Expect(err).NotTo(HaveOccurred())
-
-				framework.Logf("Now polling for Master startup...")
-				// Only one master pod: But its a natural way to look up pod names.
-				forEachPod(selectorKey, selectorValue, func(pod v1.Pod) {
-					framework.Logf("Now waiting for master to startup in %v", pod.Name)
-					_, err := framework.LookForStringInLog(ns, pod.Name, "spark-master", "Starting Spark master at", serverStartTimeout)
-					Expect(err).NotTo(HaveOccurred())
-				})
-
-				By("waiting for master endpoint")
-				err = framework.WaitForEndpoint(c, ns, "spark-master")
-				Expect(err).NotTo(HaveOccurred())
-				forEachPod(selectorKey, selectorValue, func(pod v1.Pod) {
-					_, maErr := framework.LookForStringInLog(f.Namespace.Name, pod.Name, "spark-master", "Starting Spark master at", serverStartTimeout)
-					if maErr != nil {
-						framework.Failf("Didn't find target string. error: %v", maErr)
-					}
-				})
-			}
-			worker := func() {
-				By("starting workers")
-				framework.Logf("Now starting Workers")
-				framework.RunKubectlOrDie("create", "-f", workerControllerYaml, nsFlag)
-				selectorKey, selectorValue := "component", "spark-worker"
-				label := labels.SelectorFromSet(labels.Set(map[string]string{selectorKey: selectorValue}))
-				err := testutils.WaitForPodsWithLabelRunning(c, ns, label)
-				Expect(err).NotTo(HaveOccurred())
-
-				// For now, scaling is orthogonal to the core test.
-				// framework.ScaleRC(c, ns, "spark-worker-controller", 2, true)
-
-				framework.Logf("Now polling for worker startup...")
-				forEachPod(selectorKey, selectorValue,
-					func(pod v1.Pod) {
-						_, slaveErr := framework.LookForStringInLog(ns, pod.Name, "spark-worker", "Successfully registered with master", serverStartTimeout)
-						Expect(slaveErr).NotTo(HaveOccurred())
-					})
-			}
-			// Run the worker verification after we turn up the master.
-			defer worker()
-			master()
-		})
-	})
-
-	framework.KubeDescribe("Cassandra", func() {
-		It("should create and scale cassandra", func() {
-			mkpath := func(file string) string {
-				return filepath.Join(framework.TestContext.RepoRoot, "examples/storage/cassandra", file)
-			}
-			serviceYaml := mkpath("cassandra-service.yaml")
-			controllerYaml := mkpath("cassandra-controller.yaml")
-			nsFlag := fmt.Sprintf("--namespace=%v", ns)
-
-			By("Starting the cassandra service")
-			framework.RunKubectlOrDie("create", "-f", serviceYaml, nsFlag)
-			framework.Logf("wait for service")
-			err := framework.WaitForService(c, ns, "cassandra", true, framework.Poll, framework.ServiceRespondingTimeout)
-			Expect(err).NotTo(HaveOccurred())
-
-			// Create an RC with n nodes in it.  Each node will then be verified.
-			By("Creating a Cassandra RC")
-			framework.RunKubectlOrDie("create", "-f", controllerYaml, nsFlag)
-			label := labels.SelectorFromSet(labels.Set(map[string]string{"app": "cassandra"}))
-			err = testutils.WaitForPodsWithLabelRunning(c, ns, label)
-			Expect(err).NotTo(HaveOccurred())
-			forEachPod("app", "cassandra", func(pod v1.Pod) {
-				framework.Logf("Verifying pod %v ", pod.Name)
-				// TODO how do we do this better?  Ready Probe?
-				_, err = framework.LookForStringInLog(ns, pod.Name, "cassandra", "Starting listening for CQL clients", serverStartTimeout)
-				Expect(err).NotTo(HaveOccurred())
-			})
-
-			By("Finding each node in the nodetool status lines")
-			forEachPod("app", "cassandra", func(pod v1.Pod) {
-				output := framework.RunKubectlOrDie("exec", pod.Name, nsFlag, "--", "nodetool", "status")
-				matched, _ := regexp.MatchString("UN.*"+pod.Status.PodIP, output)
-				if matched != true {
-					framework.Failf("Cassandra pod ip %s is not reporting Up and Normal 'UN' via nodetool status", pod.Status.PodIP)
-				}
-			})
-		})
-	})
-
-	framework.KubeDescribe("CassandraStatefulSet", func() {
-		It("should create statefulset", func() {
-			mkpath := func(file string) string {
-				return filepath.Join(framework.TestContext.RepoRoot, "examples/storage/cassandra", file)
-			}
-			serviceYaml := mkpath("cassandra-service.yaml")
-			nsFlag := fmt.Sprintf("--namespace=%v", ns)
-
-			// have to change dns prefix because of the dynamic namespace
-			input := generated.ReadOrDie(mkpath("cassandra-statefulset.yaml"))
-
-			output := strings.Replace(string(input), "cassandra-0.cassandra.default.svc.cluster.local", "cassandra-0.cassandra."+ns+".svc.cluster.local", -1)
-
-			statefulsetYaml := "/tmp/cassandra-statefulset.yaml"
-
-			err := ioutil.WriteFile(statefulsetYaml, []byte(output), 0644)
-			Expect(err).NotTo(HaveOccurred())
-
-			By("Starting the cassandra service")
-			framework.RunKubectlOrDie("create", "-f", serviceYaml, nsFlag)
-			framework.Logf("wait for service")
-			err = framework.WaitForService(c, ns, "cassandra", true, framework.Poll, framework.ServiceRespondingTimeout)
-			Expect(err).NotTo(HaveOccurred())
-
-			// Create an StatefulSet with n nodes in it.  Each node will then be verified.
-			By("Creating a Cassandra StatefulSet")
-
-			framework.RunKubectlOrDie("create", "-f", statefulsetYaml, nsFlag)
-
-			statefulsetPoll := 30 * time.Second
-			statefulsetTimeout := 10 * time.Minute
-			// TODO - parse this number out of the yaml
-			numPets := 3
-			label := labels.SelectorFromSet(labels.Set(map[string]string{"app": "cassandra"}))
-			err = wait.PollImmediate(statefulsetPoll, statefulsetTimeout,
-				func() (bool, error) {
-					podList, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: label.String()})
-					if err != nil {
-						return false, fmt.Errorf("Unable to get list of pods in statefulset %s", label)
-					}
-					framework.ExpectNoError(err)
-					if len(podList.Items) < numPets {
-						framework.Logf("Found %d pets, waiting for %d", len(podList.Items), numPets)
-						return false, nil
-					}
-					if len(podList.Items) > numPets {
-						return false, fmt.Errorf("Too many pods scheduled, expected %d got %d", numPets, len(podList.Items))
-					}
-					for _, p := range podList.Items {
-						isReady := podutil.IsPodReady(&p)
-						if p.Status.Phase != v1.PodRunning || !isReady {
-							framework.Logf("Waiting for pod %v to enter %v - Ready=True, currently %v - Ready=%v", p.Name, v1.PodRunning, p.Status.Phase, isReady)
-							return false, nil
-						}
-					}
-					return true, nil
-				})
-			Expect(err).NotTo(HaveOccurred())
-
-			By("Finding each node in the nodetool status lines")
-			forEachPod("app", "cassandra", func(pod v1.Pod) {
-				output := framework.RunKubectlOrDie("exec", pod.Name, nsFlag, "--", "nodetool", "status")
-				matched, _ := regexp.MatchString("UN.*"+pod.Status.PodIP, output)
-				if matched != true {
-					framework.Failf("Cassandra pod ip %s is not reporting Up and Normal 'UN' via nodetool status", pod.Status.PodIP)
-				}
-			})
-			// using out of statefulset e2e as deleting pvc is a pain
-			framework.DeleteAllStatefulSets(c, ns)
-		})
-	})
-
-	framework.KubeDescribe("Storm", func() {
-		It("should create and stop Zookeeper, Nimbus and Storm worker servers", func() {
-			mkpath := func(file string) string {
-				return filepath.Join(framework.TestContext.RepoRoot, "examples/storm", file)
-			}
-			zookeeperServiceJson := mkpath("zookeeper-service.json")
-			zookeeperPodJson := mkpath("zookeeper.json")
-			nimbusServiceJson := mkpath("storm-nimbus-service.json")
-			nimbusPodJson := mkpath("storm-nimbus.json")
-			workerControllerJson := mkpath("storm-worker-controller.json")
-			nsFlag := fmt.Sprintf("--namespace=%v", ns)
-			zookeeperPod := "zookeeper"
-			nimbusPod := "nimbus"
-
-			By("starting Zookeeper")
-			framework.RunKubectlOrDie("create", "-f", zookeeperPodJson, nsFlag)
-			framework.RunKubectlOrDie("create", "-f", zookeeperServiceJson, nsFlag)
-			err := f.WaitForPodRunningSlow(zookeeperPod)
-			Expect(err).NotTo(HaveOccurred())
-
-			By("checking if zookeeper is up and running")
-			_, err = framework.LookForStringInLog(ns, zookeeperPod, "zookeeper", "binding to port", serverStartTimeout)
-			Expect(err).NotTo(HaveOccurred())
-			err = framework.WaitForEndpoint(c, ns, "zookeeper")
-			Expect(err).NotTo(HaveOccurred())
-
-			By("starting Nimbus")
-			framework.RunKubectlOrDie("create", "-f", nimbusPodJson, nsFlag)
-			framework.RunKubectlOrDie("create", "-f", nimbusServiceJson, nsFlag)
-			err = f.WaitForPodRunningSlow(nimbusPod)
-			Expect(err).NotTo(HaveOccurred())
-
-			err = framework.WaitForEndpoint(c, ns, "nimbus")
-			Expect(err).NotTo(HaveOccurred())
-
-			By("starting workers")
-			framework.RunKubectlOrDie("create", "-f", workerControllerJson, nsFlag)
-			label := labels.SelectorFromSet(labels.Set(map[string]string{"name": "storm-worker"}))
-			err = testutils.WaitForPodsWithLabelRunning(c, ns, label)
-			Expect(err).NotTo(HaveOccurred())
-			forEachPod("name", "storm-worker", func(pod v1.Pod) {
-				//do nothing, just wait for the pod to be running
-			})
-			// TODO: Add logging configuration to nimbus & workers images and then
-			// look for a string instead of sleeping.
-			time.Sleep(20 * time.Second)
-
-			By("checking if there are established connections to Zookeeper")
-			_, err = framework.LookForStringInLog(ns, zookeeperPod, "zookeeper", "Established session", serverStartTimeout)
-			Expect(err).NotTo(HaveOccurred())
-
-			By("checking if Nimbus responds to requests")
-			framework.LookForString("No topologies running.", time.Minute, func() string {
-				return framework.RunKubectlOrDie("exec", "nimbus", nsFlag, "--", "bin/storm", "list")
-			})
-		})
-	})
-
 	framework.KubeDescribe("Liveness", func() {
 		It("liveness pods should be automatically restarted", func() {
-			mkpath := func(file string) string {
+			test := "test/fixtures/doc-yaml/user-guide/liveness"
-				path := filepath.Join("test/fixtures/doc-yaml/user-guide/liveness", file)
+			execYaml := readFile(test, "exec-liveness.yaml")
-				framework.ExpectNoError(createFileForGoBinData(path, path))
+			httpYaml := readFile(test, "http-liveness.yaml")
-				return path
-			}
-			execYaml := mkpath("exec-liveness.yaml")
-			httpYaml := mkpath("http-liveness.yaml")
 			nsFlag := fmt.Sprintf("--namespace=%v", ns)
 
-			framework.RunKubectlOrDie("create", "-f", filepath.Join(framework.TestContext.OutputDir, execYaml), nsFlag)
+			framework.RunKubectlOrDieInput(execYaml, "create", "-f", "-", nsFlag)
-			framework.RunKubectlOrDie("create", "-f", filepath.Join(framework.TestContext.OutputDir, httpYaml), nsFlag)
+			framework.RunKubectlOrDieInput(httpYaml, "create", "-f", "-", nsFlag)
 
 			// Since both containers start rapidly, we can easily run this test in parallel.
 			var wg sync.WaitGroup
@@ -446,20 +109,16 @@ var _ = framework.KubeDescribe("[Feature:Example]", func() {
 
 	framework.KubeDescribe("Secret", func() {
 		It("should create a pod that reads a secret", func() {
-			mkpath := func(file string) string {
+			test := "test/fixtures/doc-yaml/user-guide/secrets"
-				path := filepath.Join("test/fixtures/doc-yaml/user-guide/secrets", file)
+			secretYaml := readFile(test, "secret.yaml")
-				framework.ExpectNoError(createFileForGoBinData(path, path))
+			podYaml := readFile(test, "secret-pod.yaml")
-				return path
-			}
-			secretYaml := mkpath("secret.yaml")
-			podYaml := mkpath("secret-pod.yaml")
 
 			nsFlag := fmt.Sprintf("--namespace=%v", ns)
 			podName := "secret-test-pod"
 
 			By("creating secret and pod")
-			framework.RunKubectlOrDie("create", "-f", filepath.Join(framework.TestContext.OutputDir, secretYaml), nsFlag)
+			framework.RunKubectlOrDieInput(secretYaml, "create", "-f", "-", nsFlag)
-			framework.RunKubectlOrDie("create", "-f", filepath.Join(framework.TestContext.OutputDir, podYaml), nsFlag)
+			framework.RunKubectlOrDieInput(podYaml, "create", "-f", "-", nsFlag)
 			err := framework.WaitForPodNoLongerRunningInNamespace(c, podName, ns)
 			Expect(err).NotTo(HaveOccurred())
 
@@ -471,17 +130,13 @@ var _ = framework.KubeDescribe("[Feature:Example]", func() {
 
 	framework.KubeDescribe("Downward API", func() {
 		It("should create a pod that prints his name and namespace", func() {
-			mkpath := func(file string) string {
+			test := "test/fixtures/doc-yaml/user-guide/downward-api"
-				path := filepath.Join("test/fixtures/doc-yaml/user-guide/downward-api", file)
+			podYaml := readFile(test, "dapi-pod.yaml")
-				framework.ExpectNoError(createFileForGoBinData(path, path))
-				return path
-			}
-			podYaml := mkpath("dapi-pod.yaml")
 			nsFlag := fmt.Sprintf("--namespace=%v", ns)
 			podName := "dapi-test-pod"
 
 			By("creating the pod")
-			framework.RunKubectlOrDie("create", "-f", filepath.Join(framework.TestContext.OutputDir, podYaml), nsFlag)
+			framework.RunKubectlOrDieInput(podYaml, "create", "-f", "-", nsFlag)
 			err := framework.WaitForPodNoLongerRunningInNamespace(c, podName, ns)
 			Expect(err).NotTo(HaveOccurred())
 
@@ -492,125 +147,9 @@ var _ = framework.KubeDescribe("[Feature:Example]", func() {
 			Expect(err).NotTo(HaveOccurred())
 		})
 	})
-
-	framework.KubeDescribe("RethinkDB", func() {
-		It("should create and stop rethinkdb servers", func() {
-			mkpath := func(file string) string {
-				return filepath.Join(framework.TestContext.RepoRoot, "examples/storage/rethinkdb", file)
-			}
-			driverServiceYaml := mkpath("driver-service.yaml")
-			rethinkDbControllerYaml := mkpath("rc.yaml")
-			adminPodYaml := mkpath("admin-pod.yaml")
-			adminServiceYaml := mkpath("admin-service.yaml")
-			nsFlag := fmt.Sprintf("--namespace=%v", ns)
-
-			By("starting rethinkdb")
-			framework.RunKubectlOrDie("create", "-f", driverServiceYaml, nsFlag)
-			framework.RunKubectlOrDie("create", "-f", rethinkDbControllerYaml, nsFlag)
-			label := labels.SelectorFromSet(labels.Set(map[string]string{"db": "rethinkdb"}))
-			err := testutils.WaitForPodsWithLabelRunning(c, ns, label)
-			Expect(err).NotTo(HaveOccurred())
-			checkDbInstances := func() {
-				forEachPod("db", "rethinkdb", func(pod v1.Pod) {
-					_, err = framework.LookForStringInLog(ns, pod.Name, "rethinkdb", "Server ready", serverStartTimeout)
-					Expect(err).NotTo(HaveOccurred())
-				})
-			}
-			checkDbInstances()
-			err = framework.WaitForEndpoint(c, ns, "rethinkdb-driver")
-			Expect(err).NotTo(HaveOccurred())
-
-			By("scaling rethinkdb")
-			framework.ScaleRC(f.ClientSet, f.ScalesGetter, ns, "rethinkdb-rc", 2, true)
-			checkDbInstances()
-
-			By("starting admin")
-			framework.RunKubectlOrDie("create", "-f", adminServiceYaml, nsFlag)
-			framework.RunKubectlOrDie("create", "-f", adminPodYaml, nsFlag)
-			err = framework.WaitForPodNameRunningInNamespace(c, "rethinkdb-admin", ns)
-			Expect(err).NotTo(HaveOccurred())
-			checkDbInstances()
-			content, err := makeHttpRequestToService(c, ns, "rethinkdb-admin", "/", framework.EndpointRegisterTimeout)
-			Expect(err).NotTo(HaveOccurred())
-			if !strings.Contains(content, "<title>RethinkDB Administration Console</title>") {
-				framework.Failf("RethinkDB console is not running")
-			}
-		})
-	})
-
-	framework.KubeDescribe("Hazelcast", func() {
-		It("should create and scale hazelcast", func() {
-			mkpath := func(file string) string {
-				return filepath.Join(framework.TestContext.RepoRoot, "examples/storage/hazelcast", file)
-			}
-			serviceYaml := mkpath("hazelcast-service.yaml")
-			deploymentYaml := mkpath("hazelcast-deployment.yaml")
-			nsFlag := fmt.Sprintf("--namespace=%v", ns)
-
-			By("starting hazelcast")
-			framework.RunKubectlOrDie("create", "-f", serviceYaml, nsFlag)
-			framework.RunKubectlOrDie("create", "-f", deploymentYaml, nsFlag)
-			label := labels.SelectorFromSet(labels.Set(map[string]string{"name": "hazelcast"}))
-			err := testutils.WaitForPodsWithLabelRunning(c, ns, label)
-			Expect(err).NotTo(HaveOccurred())
-			forEachPod("name", "hazelcast", func(pod v1.Pod) {
-				_, err := framework.LookForStringInLog(ns, pod.Name, "hazelcast", "Members [1]", serverStartTimeout)
-				Expect(err).NotTo(HaveOccurred())
-				_, err = framework.LookForStringInLog(ns, pod.Name, "hazelcast", "is STARTED", serverStartTimeout)
-				Expect(err).NotTo(HaveOccurred())
-			})
-
-			err = framework.WaitForEndpoint(c, ns, "hazelcast")
-			Expect(err).NotTo(HaveOccurred())
-
-			By("scaling hazelcast")
-			framework.ScaleRC(f.ClientSet, f.ScalesGetter, ns, "hazelcast", 2, true)
-			forEachPod("name", "hazelcast", func(pod v1.Pod) {
-				_, err := framework.LookForStringInLog(ns, pod.Name, "hazelcast", "Members [2]", serverStartTimeout)
-				Expect(err).NotTo(HaveOccurred())
-			})
-		})
-	})
 })
 
-func makeHttpRequestToService(c clientset.Interface, ns, service, path string, timeout time.Duration) (string, error) {
+func readFile(test, file string) string {
-	var result []byte
+	from := filepath.Join(test, file)
-	var err error
+	return string(testfiles.ReadOrDie(from, Fail))
-	for t := time.Now(); time.Since(t) < timeout; time.Sleep(framework.Poll) {
-		proxyRequest, errProxy := framework.GetServicesProxyRequest(c, c.CoreV1().RESTClient().Get())
-		if errProxy != nil {
-			break
-		}
-
-		ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout)
-		defer cancel()
-
-		result, err = proxyRequest.Namespace(ns).
-			Context(ctx).
-			Name(service).
-			Suffix(path).
-			Do().
-			Raw()
-		if err != nil {
-			break
-		}
-	}
-	return string(result), err
-}
-
-func createFileForGoBinData(gobindataPath, outputFilename string) error {
-	data := generated.ReadOrDie(gobindataPath)
-	if len(data) == 0 {
-		return fmt.Errorf("Failed to read gobindata from %v", gobindataPath)
-	}
-	fullPath := filepath.Join(framework.TestContext.OutputDir, outputFilename)
-	err := os.MkdirAll(filepath.Dir(fullPath), 0777)
-	if err != nil {
-		return fmt.Errorf("Error while creating directory %v: %v", filepath.Dir(fullPath), err)
-	}
-	err = ioutil.WriteFile(fullPath, data, 0644)
-	if err != nil {
-		return fmt.Errorf("Error while trying to write to file %v: %v", fullPath, err)
-	}
-	return nil
 }

test/e2e/framework/ingress_utils.go

@@ -52,7 +52,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
-	utilfile "k8s.io/kubernetes/pkg/util/file"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 	"k8s.io/kubernetes/test/e2e/manifest"
 	testutils "k8s.io/kubernetes/test/utils"
 	utilexec "k8s.io/utils/exec"
@@ -1166,15 +1166,18 @@ func NewIngressTestJig(c clientset.Interface) *IngressTestJig {
 // If svcAnnotations is specified it will overwrite any annotations in svc.yaml
 func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string, svcAnnotations map[string]string) {
 	var err error
-	mkpath := func(file string) string {
+	read := func(file string) string {
-		return filepath.Join(TestContext.RepoRoot, manifestPath, file)
+		return string(testfiles.ReadOrDie(filepath.Join(manifestPath, file), Fail))
+	}
+	exists := func(file string) bool {
+		return testfiles.Exists(filepath.Join(manifestPath, file), Fail)
 	}
 
 	j.Logger.Infof("creating replication controller")
-	RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", ns))
+	RunKubectlOrDieInput(read("rc.yaml"), "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns))
 
 	j.Logger.Infof("creating service")
-	RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns))
+	RunKubectlOrDieInput(read("svc.yaml"), "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns))
 	if len(svcAnnotations) > 0 {
 		svcList, err := j.Client.CoreV1().Services(ns).List(metav1.ListOptions{})
 		ExpectNoError(err)
@@ -1185,9 +1188,9 @@ func (j *IngressTestJig) CreateIngress(manifestPath, ns string, ingAnnotations m
 		}
 	}
 
-	if exists, _ := utilfile.FileExists(mkpath("secret.yaml")); exists {
+	if exists("secret.yaml") {
 		j.Logger.Infof("creating secret")
-		RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns))
+		RunKubectlOrDieInput(read("secret.yaml"), "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns))
 	}
 	j.Logger.Infof("Parsing ingress from %v", filepath.Join(manifestPath, "ing.yaml"))
 
@@ -1608,11 +1611,11 @@ type NginxIngressController struct {
 
 // Init initializes the NginxIngressController
 func (cont *NginxIngressController) Init() {
-	mkpath := func(file string) string {
+	read := func(file string) string {
-		return filepath.Join(TestContext.RepoRoot, IngressManifestPath, "nginx", file)
+		return string(testfiles.ReadOrDie(filepath.Join(IngressManifestPath, "nginx", file), Fail))
 	}
 	Logf("initializing nginx ingress controller")
-	RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", cont.Ns))
+	RunKubectlOrDieInput(read("rc.yaml"), "create", "-f", "-", fmt.Sprintf("--namespace=%v", cont.Ns))
 
 	rc, err := cont.Client.CoreV1().ReplicationControllers(cont.Ns).Get("nginx-ingress-controller", metav1.GetOptions{})
 	ExpectNoError(err)

test/e2e/framework/testfiles/testfiles.go

@@ -0,0 +1,189 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package testfiles provides a wrapper around various optional ways
+// of retrieving additional files needed during a test run:
+// - builtin bindata
+// - filesystem access
+//
+// Because it is a is self-contained package, it can be used by
+// test/e2e/framework and test/e2e/manifest without creating
+// a circular dependency.
+package testfiles
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+var filesources []FileSource
+
+// AddFileSource registers another provider for files that may be
+// needed at runtime. Should be called during initialization of a test
+// binary.
+func AddFileSource(filesource FileSource) {
+	filesources = append(filesources, filesource)
+}
+
+// FileSource implements one way of retrieving test file content.  For
+// example, one file source could read from the original source code
+// file tree, another from bindata compiled into a test executable.
+type FileSource interface {
+	// ReadTestFile retrieves the content of a file that gets maintained
+	// alongside a test's source code. Files are identified by the
+	// relative path inside the repository containing the tests, for
+	// example "cluster/gce/upgrade.sh" inside kubernetes/kubernetes.
+	//
+	// When the file is not found, a nil slice is returned. An error is
+	// returned for all fatal errors.
+	ReadTestFile(filePath string) ([]byte, error)
+
+	// DescribeFiles returns a multi-line description of which
+	// files are available via this source. It is meant to be
+	// used as part of the error message when a file cannot be
+	// found.
+	DescribeFiles() string
+}
+
+// Fail is an error handler function with the same prototype and
+// semantic as ginkgo.Fail. Typically ginkgo.Fail is what callers
+// of ReadOrDie and Exists will pass. This way this package
+// avoids depending on Ginkgo.
+type Fail func(failure string, callerSkip ...int)
+
+// ReadOrDie tries to retrieve the desired file content from
+// one of the registered file sources. In contrast to FileSource, it
+// will either return a valid slice or abort the test by calling the fatal function,
+// i.e. the caller doesn't have to implement error checking.
+func ReadOrDie(filePath string, fail Fail) []byte {
+	data, err := Read(filePath)
+	if err != nil {
+		fail(err.Error(), 1)
+	}
+	return data
+}
+
+// Read tries to retrieve the desired file content from
+// one of the registered file sources.
+func Read(filePath string) ([]byte, error) {
+	if len(filesources) == 0 {
+		return nil, fmt.Errorf("no file sources registered (yet?), cannot retrieve test file %s", filePath)
+	}
+	for _, filesource := range filesources {
+		data, err := filesource.ReadTestFile(filePath)
+		if err != nil {
+			return nil, fmt.Errorf("fatal error retrieving test file %s: %s", filePath, err)
+		}
+		if data != nil {
+			return data, nil
+		}
+	}
+	// Here we try to generate an error that points test authors
+	// or users in the right direction for resolving the problem.
+	error := fmt.Sprintf("Test file %q was not found.\n", filePath)
+	for _, filesource := range filesources {
+		error += filesource.DescribeFiles()
+		error += "\n"
+	}
+	return nil, errors.New(error)
+}
+
+// Exists checks whether a file could be read. Unexpected errors
+// are handled by calling the fail function, which then should
+// abort the current test.
+func Exists(filePath string, fail Fail) bool {
+	for _, filesource := range filesources {
+		data, err := filesource.ReadTestFile(filePath)
+		if err != nil {
+			fail(fmt.Sprintf("fatal error looking for test file %s: %s", filePath, err), 1)
+		}
+		if data != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// RootFileSource looks for files relative to a root directory.
+type RootFileSource struct {
+	Root string
+}
+
+// ReadTestFile looks looks for the file relative to the configured
+// root directory.
+func (r RootFileSource) ReadTestFile(filePath string) ([]byte, error) {
+	fullPath := filepath.Join(r.Root, filePath)
+	data, err := ioutil.ReadFile(fullPath)
+	if os.IsNotExist(err) {
+		// Not an error (yet), some other provider may have the file.
+		return nil, nil
+	}
+	return data, err
+}
+
+// DescribeFiles explains that it looks for files inside a certain
+// root directory.
+func (r RootFileSource) DescribeFiles() string {
+	description := fmt.Sprintf("Test files are expected in %q", r.Root)
+	if !path.IsAbs(r.Root) {
+		// The default in test_context.go is the relative path
+		// ../../, which doesn't really help locating the
+		// actual location. Therefore we add also the absolute
+		// path if necessary.
+		abs, err := filepath.Abs(r.Root)
+		if err == nil {
+			description += fmt.Sprintf(" = %q", abs)
+		}
+	}
+	description += "."
+	return description
+}
+
+// BindataFileSource handles files stored in a package generated with bindata.
+type BindataFileSource struct {
+	Asset      func(string) ([]byte, error)
+	AssetNames func() []string
+}
+
+// ReadTestFile looks for an asset with the given path.
+func (b BindataFileSource) ReadTestFile(filePath string) ([]byte, error) {
+	fileBytes, err := b.Asset(filePath)
+	if err != nil {
+		// It would be nice to have a better way to detect
+		// "not found" errors :-/
+		if strings.HasSuffix(err.Error(), "not found") {
+			return nil, nil
+		}
+	}
+	return fileBytes, nil
+}
+
+// DescribeFiles explains about gobindata and then lists all available files.
+func (b BindataFileSource) DescribeFiles() string {
+	var lines []string
+	lines = append(lines, "The following files are built into the test executable via gobindata. For questions on maintaining gobindata, contact the sig-testing group.")
+	assets := b.AssetNames()
+	sort.Strings(assets)
+	lines = append(lines, assets...)
+	description := strings.Join(lines, "\n   ")
+	return description
+}

test/e2e/kubectl/kubectl.go

@@ -58,7 +58,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/test/e2e/framework"
-	"k8s.io/kubernetes/test/e2e/generated"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 	"k8s.io/kubernetes/test/e2e/scheduling"
 	testutils "k8s.io/kubernetes/test/utils"
 	uexec "k8s.io/utils/exec"
@@ -150,7 +150,7 @@ func substituteImageName(content string) string {
 }
 
 func readTestFileOrDie(file string) []byte {
-	return generated.ReadOrDie(path.Join(kubeCtlManifestPath, file))
+	return testfiles.ReadOrDie(path.Join(kubeCtlManifestPath, file), Fail)
 }
 
 func runKubectlRetryOrDie(args ...string) string {
@@ -292,8 +292,8 @@ var _ = SIGDescribe("Kubectl client", func() {
 		var nautilus, kitten string
 		BeforeEach(func() {
 			updateDemoRoot := "test/fixtures/doc-yaml/user-guide/update-demo"
-			nautilus = substituteImageName(string(generated.ReadOrDie(filepath.Join(updateDemoRoot, "nautilus-rc.yaml.in"))))
+			nautilus = substituteImageName(string(testfiles.ReadOrDie(filepath.Join(updateDemoRoot, "nautilus-rc.yaml.in"), Fail)))
-			kitten = substituteImageName(string(generated.ReadOrDie(filepath.Join(updateDemoRoot, "kitten-rc.yaml.in"))))
+			kitten = substituteImageName(string(testfiles.ReadOrDie(filepath.Join(updateDemoRoot, "kitten-rc.yaml.in"), Fail)))
 		})
 		/*
 			Release : v1.9
@@ -357,7 +357,7 @@ var _ = SIGDescribe("Kubectl client", func() {
 				"redis-master-deployment.yaml.in",
 				"redis-slave-deployment.yaml.in",
 			} {
-				contents := substituteImageName(string(generated.ReadOrDie(filepath.Join(guestbookRoot, gbAppFile))))
+				contents := substituteImageName(string(testfiles.ReadOrDie(filepath.Join(guestbookRoot, gbAppFile), Fail)))
 				run(contents)
 			}
 		}
@@ -383,9 +383,10 @@ var _ = SIGDescribe("Kubectl client", func() {
 	})
 
 	framework.KubeDescribe("Simple pod", func() {
-		podYaml := substituteImageName(string(readTestFileOrDie("pod-with-readiness-probe.yaml.in")))
+		var podYaml string
 		BeforeEach(func() {
 			By(fmt.Sprintf("creating the pod from %v", podYaml))
+			podYaml = substituteImageName(string(readTestFileOrDie("pod-with-readiness-probe.yaml.in")))
 			framework.RunKubectlOrDieInput(podYaml, "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns))
 			Expect(framework.CheckPodsRunningReady(c, ns, []string{simplePodName}, framework.PodStartTimeout)).To(BeTrue())
 		})
@@ -1037,10 +1038,11 @@ metadata:
 	})
 
 	framework.KubeDescribe("Kubectl label", func() {
-		podYaml := substituteImageName(string(readTestFileOrDie("pause-pod.yaml.in")))
+		var podYaml string
 		var nsFlag string
 		BeforeEach(func() {
 			By("creating the pod")
+			podYaml = substituteImageName(string(readTestFileOrDie("pause-pod.yaml.in")))
 			nsFlag = fmt.Sprintf("--namespace=%v", ns)
 			framework.RunKubectlOrDieInput(podYaml, "create", "-f", "-", nsFlag)
 			Expect(framework.CheckPodsRunningReady(c, ns, []string{pausePodName}, framework.PodStartTimeout)).To(BeTrue())
@@ -1078,10 +1080,11 @@ metadata:
 
 	framework.KubeDescribe("Kubectl logs", func() {
 		var nsFlag string
-		rc := substituteImageName(string(readTestFileOrDie(redisControllerFilename)))
+		var rc string
 		containerName := "redis-master"
 		BeforeEach(func() {
 			By("creating an rc")
+			rc = substituteImageName(string(readTestFileOrDie(redisControllerFilename)))
 			nsFlag = fmt.Sprintf("--namespace=%v", ns)
 			framework.RunKubectlOrDieInput(rc, "create", "-f", "-", nsFlag)
 		})

test/e2e/manifest/manifest.go

@@ -29,13 +29,16 @@ import (
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
-	"k8s.io/kubernetes/test/e2e/generated"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 )
 
 // PodFromManifest reads a .json/yaml file and returns the pod in it.
 func PodFromManifest(filename string) (*v1.Pod, error) {
 	var pod v1.Pod
-	data := generated.ReadOrDie(filename)
+	data, err := testfiles.Read(filename)
+	if err != nil {
+		return nil, err
+	}
 
 	json, err := utilyaml.ToJSON(data)
 	if err != nil {
@@ -50,7 +53,10 @@ func PodFromManifest(filename string) (*v1.Pod, error) {
 // RcFromManifest reads a .json/yaml file and returns the rc in it.
 func RcFromManifest(fileName string) (*v1.ReplicationController, error) {
 	var controller v1.ReplicationController
-	data := generated.ReadOrDie(fileName)
+	data, err := testfiles.Read(fileName)
+	if err != nil {
+		return nil, err
+	}
 
 	json, err := utilyaml.ToJSON(data)
 	if err != nil {
@@ -65,7 +71,10 @@ func RcFromManifest(fileName string) (*v1.ReplicationController, error) {
 // SvcFromManifest reads a .json/yaml file and returns the service in it.
 func SvcFromManifest(fileName string) (*v1.Service, error) {
 	var svc v1.Service
-	data := generated.ReadOrDie(fileName)
+	data, err := testfiles.Read(fileName)
+	if err != nil {
+		return nil, err
+	}
 
 	json, err := utilyaml.ToJSON(data)
 	if err != nil {
@@ -80,7 +89,10 @@ func SvcFromManifest(fileName string) (*v1.Service, error) {
 // IngressFromManifest reads a .json/yaml file and returns the ingress in it.
 func IngressFromManifest(fileName string) (*extensions.Ingress, error) {
 	var ing extensions.Ingress
-	data := generated.ReadOrDie(fileName)
+	data, err := testfiles.Read(fileName)
+	if err != nil {
+		return nil, err
+	}
 
 	json, err := utilyaml.ToJSON(data)
 	if err != nil {
@@ -109,7 +121,10 @@ func IngressToManifest(ing *extensions.Ingress, path string) error {
 // StatefulSetFromManifest returns a StatefulSet from a manifest stored in fileName in the Namespace indicated by ns.
 func StatefulSetFromManifest(fileName, ns string) (*apps.StatefulSet, error) {
 	var ss apps.StatefulSet
-	data := generated.ReadOrDie(fileName)
+	data, err := testfiles.Read(fileName)
+	if err != nil {
+		return nil, err
+	}
 
 	json, err := utilyaml.ToJSON(data)
 	if err != nil {
@@ -130,7 +145,10 @@ func StatefulSetFromManifest(fileName, ns string) (*apps.StatefulSet, error) {
 // DaemonSetFromManifest returns a DaemonSet from a manifest stored in fileName in the Namespace indicated by ns.
 func DaemonSetFromManifest(fileName, ns string) (*apps.DaemonSet, error) {
 	var ds apps.DaemonSet
-	data := generated.ReadOrDie(fileName)
+	data, err := testfiles.Read(fileName)
+	if err != nil {
+		return nil, err
+	}
 
 	json, err := utilyaml.ToJSON(data)
 	if err != nil {
@@ -147,7 +165,7 @@ func DaemonSetFromManifest(fileName, ns string) (*apps.DaemonSet, error) {
 // RoleFromManifest returns a Role from a manifest stored in fileName in the Namespace indicated by ns.
 func RoleFromManifest(fileName, ns string) (*rbac.Role, error) {
 	var role rbac.Role
-	data := generated.ReadOrDie(fileName)
+	data, err := testfiles.Read(fileName)
 
 	json, err := utilyaml.ToJSON(data)
 	if err != nil {

test/e2e/storage/flexvolume.go

@@ -32,7 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/version"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
-	"k8s.io/kubernetes/test/e2e/generated"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
 )
 
@@ -95,7 +95,7 @@ func installFlex(c clientset.Interface, node *v1.Node, vendor, driver, filePath
 	cmd := fmt.Sprintf("sudo mkdir -p %s", flexDir)
 	sshAndLog(cmd, host, true /*failOnError*/)
 
-	data := generated.ReadOrDie(filePath)
+	data := testfiles.ReadOrDie(filePath, Fail)
 	cmd = fmt.Sprintf("sudo tee <<'EOF' %s\n%s\nEOF", flexFile, string(data))
 	sshAndLog(cmd, host, true /*failOnError*/)
 

test/e2e/upgrades/cassandra.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 )
 
 const cassandraManifestPath = "test/e2e/testing-manifests/statefulset/cassandra"
@@ -59,8 +60,8 @@ func (CassandraUpgradeTest) Skip(upgCtx UpgradeContext) bool {
 }
 
 func cassandraKubectlCreate(ns, file string) {
-	path := filepath.Join(framework.TestContext.RepoRoot, cassandraManifestPath, file)
+	input := string(testfiles.ReadOrDie(filepath.Join(cassandraManifestPath, file), Fail))
-	framework.RunKubectlOrDie("create", "-f", path, fmt.Sprintf("--namespace=%s", ns))
+	framework.RunKubectlOrDieInput(input, "create", "-f", "-", fmt.Sprintf("--namespace=%s", ns))
 }
 
 // Setup creates a Cassandra StatefulSet and a PDB. It also brings up a tester

test/e2e/upgrades/etcd.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 )
 
 const manifestPath = "test/e2e/testing-manifests/statefulset/etcd"
@@ -55,8 +56,8 @@ func (EtcdUpgradeTest) Skip(upgCtx UpgradeContext) bool {
 }
 
 func kubectlCreate(ns, file string) {
-	path := filepath.Join(framework.TestContext.RepoRoot, manifestPath, file)
+	input := string(testfiles.ReadOrDie(filepath.Join(manifestPath, file), Fail))
-	framework.RunKubectlOrDie("create", "-f", path, fmt.Sprintf("--namespace=%s", ns))
+	framework.RunKubectlOrDieInput(input, "create", "-f", "-", fmt.Sprintf("--namespace=%s", ns))
 }
 
 func (t *EtcdUpgradeTest) Setup(f *framework.Framework) {

test/e2e/upgrades/mysql.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
 )
 
 const mysqlManifestPath = "test/e2e/testing-manifests/statefulset/mysql-upgrade"
@@ -58,8 +59,8 @@ func (MySqlUpgradeTest) Skip(upgCtx UpgradeContext) bool {
 }
 
 func mysqlKubectlCreate(ns, file string) {
-	path := filepath.Join(framework.TestContext.RepoRoot, mysqlManifestPath, file)
+	input := string(testfiles.ReadOrDie(filepath.Join(mysqlManifestPath, file), Fail))
-	framework.RunKubectlOrDie("create", "-f", path, fmt.Sprintf("--namespace=%s", ns))
+	framework.RunKubectlOrDieInput(input, "create", "-f", "-", fmt.Sprintf("--namespace=%s", ns))
 }
 
 func (t *MySqlUpgradeTest) getServiceIP(f *framework.Framework, ns, svcName string) string {