Merge pull request #37094 from sjug/reshuffle_gobindata_dep

Automatic merge from submit-queue (batch tested with PRs 37094, 37663, 37442, 37808, 37826)

Moved gobindata, refactored ReadOrDie refs

**What this PR does / why we need it**: Having gobindata inside of test/e2e/framework prevents external projects from importing the framework. Moving it out and managing refs fixes this problem.

**Which issue this PR fixes**: fixes #37007

hack/lib/golang.sh

@@ -168,7 +168,7 @@ readonly KUBE_STATIC_LIBRARIES=(
 
 # Add any files with those //generate annotations in the array below.
 readonly KUBE_BINDATAS=(
-  test/e2e/framework/gobindata_util.go
+  test/e2e/generated/gobindata_util.go
 )
 
 kube::golang::is_statically_linked_library() {

test/e2e/BUILD

@@ -186,6 +186,7 @@ go_library(
         "//test/e2e/chaosmonkey:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
+        "//test/e2e/generated:go_default_library",
         "//test/images/net/nat:go_default_library",
         "//test/utils:go_default_library",
         "//vendor:github.com/aws/aws-sdk-go/aws",

test/e2e/e2e.go

@@ -32,11 +32,16 @@ import (
 
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
 	gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
+	"k8s.io/kubernetes/pkg/runtime"
 	"k8s.io/kubernetes/pkg/util/logs"
-	"k8s.io/kubernetes/pkg/util/runtime"
+	runtimeutils "k8s.io/kubernetes/pkg/util/runtime"
+	utilyaml "k8s.io/kubernetes/pkg/util/yaml"
 	commontest "k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/generated"
+	testutils "k8s.io/kubernetes/test/utils"
 )
 
 const (
@@ -126,7 +131,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	if err := framework.WaitForPodsRunningReady(c, api.NamespaceSystem, int32(framework.TestContext.MinStartupPods), podStartupTimeout, framework.ImagePullerLabels); err != nil {
 		framework.DumpAllNamespaceInfo(c, api.NamespaceSystem)
 		framework.LogFailedContainers(c, api.NamespaceSystem, framework.Logf)
-		framework.RunKubernetesServiceTestContainer(c, v1.NamespaceDefault)
+		runKubernetesServiceTestContainer(c, v1.NamespaceDefault)
 		framework.Failf("Error waiting for all pods to be running and ready: %v", err)
 	}
 
@@ -220,7 +225,7 @@ var _ = ginkgo.SynchronizedAfterSuite(func() {
 // generated in this directory, and cluster logs will also be saved.
 // This function is called on each Ginkgo node in parallel mode.
 func RunE2ETests(t *testing.T) {
-	runtime.ReallyCrash = true
+	runtimeutils.ReallyCrash = true
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
@@ -245,3 +250,49 @@ func RunE2ETests(t *testing.T) {
 
 	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r)
 }
+
+func podFromManifest(filename string) (*v1.Pod, error) {
+	var pod v1.Pod
+	framework.Logf("Parsing pod from %v", filename)
+	data := generated.ReadOrDie(filename)
+	json, err := utilyaml.ToJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &pod); err != nil {
+		return nil, err
+	}
+	return &pod, nil
+}
+
+// Run a test container to try and contact the Kubernetes api-server from a pod, wait for it
+// to flip to Ready, log its output and delete it.
+func runKubernetesServiceTestContainer(c clientset.Interface, ns string) {
+	path := "test/images/clusterapi-tester/pod.yaml"
+	p, err := podFromManifest(path)
+	if err != nil {
+		framework.Logf("Failed to parse clusterapi-tester from manifest %v: %v", path, err)
+		return
+	}
+	p.Namespace = ns
+	if _, err := c.Core().Pods(ns).Create(p); err != nil {
+		framework.Logf("Failed to create %v: %v", p.Name, err)
+		return
+	}
+	defer func() {
+		if err := c.Core().Pods(ns).Delete(p.Name, nil); err != nil {
+			framework.Logf("Failed to delete pod %v: %v", p.Name, err)
+		}
+	}()
+	timeout := 5 * time.Minute
+	if err := framework.WaitForPodCondition(c, ns, p.Name, "clusterapi-tester", timeout, testutils.PodRunningReady); err != nil {
+		framework.Logf("Pod %v took longer than %v to enter running/ready: %v", p.Name, timeout, err)
+		return
+	}
+	logs, err := framework.GetPodLogs(c, ns, p.Name, p.Spec.Containers[0].Name)
+	if err != nil {
+		framework.Logf("Failed to retrieve logs from %v: %v", p.Name, err)
+	} else {
+		framework.Logf("Output of clusterapi-tester:\n%v", logs)
+	}
+}

test/e2e/examples.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/kubernetes/pkg/labels"
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/generated"
 	testutils "k8s.io/kubernetes/test/utils"
 
 	. "github.com/onsi/ginkgo"
@@ -379,7 +380,7 @@ var _ = framework.KubeDescribe("[Feature:Example]", func() {
 		It("liveness pods should be automatically restarted", func() {
 			mkpath := func(file string) string {
 				path := filepath.Join("test/fixtures/doc-yaml/user-guide/liveness", file)
-				ExpectNoError(framework.CreateFileForGoBinData(path, path))
+				ExpectNoError(createFileForGoBinData(path, path))
 				return path
 			}
 			execYaml := mkpath("exec-liveness.yaml")
@@ -431,7 +432,7 @@ var _ = framework.KubeDescribe("[Feature:Example]", func() {
 		It("should create a pod that reads a secret", func() {
 			mkpath := func(file string) string {
 				path := filepath.Join("test/fixtures/doc-yaml/user-guide/secrets", file)
-				ExpectNoError(framework.CreateFileForGoBinData(path, path))
+				ExpectNoError(createFileForGoBinData(path, path))
 				return path
 			}
 			secretYaml := mkpath("secret.yaml")
@@ -456,7 +457,7 @@ var _ = framework.KubeDescribe("[Feature:Example]", func() {
 		It("should create a pod that prints his name and namespace", func() {
 			mkpath := func(file string) string {
 				path := filepath.Join("test/fixtures/doc-yaml/user-guide/downward-api", file)
-				ExpectNoError(framework.CreateFileForGoBinData(path, path))
+				ExpectNoError(createFileForGoBinData(path, path))
 				return path
 			}
 			podYaml := mkpath("dapi-pod.yaml")
@@ -586,3 +587,20 @@ func prepareResourceWithReplacedString(inputFile, old, new string) string {
 	podYaml := strings.Replace(string(data), old, new, 1)
 	return podYaml
 }
+
+func createFileForGoBinData(gobindataPath, outputFilename string) error {
+	data := generated.ReadOrDie(gobindataPath)
+	if len(data) == 0 {
+		return fmt.Errorf("Failed to read gobindata from %v", gobindataPath)
+	}
+	fullPath := filepath.Join(framework.TestContext.OutputDir, outputFilename)
+	err := os.MkdirAll(filepath.Dir(fullPath), 0777)
+	if err != nil {
+		return fmt.Errorf("Error while creating directory %v: %v", filepath.Dir(fullPath), err)
+	}
+	err = ioutil.WriteFile(fullPath, data, 0644)
+	if err != nil {
+		return fmt.Errorf("Error while trying to write to file %v: %v", fullPath, err)
+	}
+	return nil
+}

test/e2e/framework/BUILD

@@ -17,7 +17,6 @@ go_library(
         "exec_util.go",
         "federation_util.go",
         "framework.go",
-        "gobindata_util.go",
         "kubelet_stats.go",
         "log_size_monitoring.go",
         "metrics_util.go",
@@ -82,12 +81,10 @@ go_library(
         "//pkg/util/uuid:go_default_library",
         "//pkg/util/validation:go_default_library",
         "//pkg/util/wait:go_default_library",
-        "//pkg/util/yaml:go_default_library",
         "//pkg/version:go_default_library",
         "//pkg/watch:go_default_library",
         "//plugin/pkg/scheduler/algorithm/predicates:go_default_library",
         "//plugin/pkg/scheduler/schedulercache:go_default_library",
-        "//test/e2e/generated:go_default_library",
         "//test/e2e/perftype:go_default_library",
         "//test/utils:go_default_library",
         "//vendor:github.com/blang/semver",

test/e2e/framework/pods.go

@@ -161,7 +161,7 @@ func (c *PodClient) mungeSpec(pod *v1.Pod) {
 // WaitForSuccess waits for pod to success.
 func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
 	f := c.f
-	Expect(waitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout,
+	Expect(WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout,
 		func(pod *v1.Pod) (bool, error) {
 			switch pod.Status.Phase {
 			case v1.PodFailed:

test/e2e/framework/util.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -78,7 +77,6 @@ import (
 	"k8s.io/kubernetes/pkg/util/system"
 	"k8s.io/kubernetes/pkg/util/uuid"
 	"k8s.io/kubernetes/pkg/util/wait"
-	utilyaml "k8s.io/kubernetes/pkg/util/yaml"
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/pkg/watch"
 	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
@@ -617,52 +615,6 @@ func WaitForPodsRunningReady(c clientset.Interface, ns string, minPods int32, ti
 	return nil
 }
 
-func podFromManifest(filename string) (*v1.Pod, error) {
-	var pod v1.Pod
-	Logf("Parsing pod from %v", filename)
-	data := ReadOrDie(filename)
-	json, err := utilyaml.ToJSON(data)
-	if err != nil {
-		return nil, err
-	}
-	if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), json, &pod); err != nil {
-		return nil, err
-	}
-	return &pod, nil
-}
-
-// Run a test container to try and contact the Kubernetes api-server from a pod, wait for it
-// to flip to Ready, log its output and delete it.
-func RunKubernetesServiceTestContainer(c clientset.Interface, ns string) {
-	path := "test/images/clusterapi-tester/pod.yaml"
-	p, err := podFromManifest(path)
-	if err != nil {
-		Logf("Failed to parse clusterapi-tester from manifest %v: %v", path, err)
-		return
-	}
-	p.Namespace = ns
-	if _, err := c.Core().Pods(ns).Create(p); err != nil {
-		Logf("Failed to create %v: %v", p.Name, err)
-		return
-	}
-	defer func() {
-		if err := c.Core().Pods(ns).Delete(p.Name, nil); err != nil {
-			Logf("Failed to delete pod %v: %v", p.Name, err)
-		}
-	}()
-	timeout := 5 * time.Minute
-	if err := waitForPodCondition(c, ns, p.Name, "clusterapi-tester", timeout, testutils.PodRunningReady); err != nil {
-		Logf("Pod %v took longer than %v to enter running/ready: %v", p.Name, timeout, err)
-		return
-	}
-	logs, err := GetPodLogs(c, ns, p.Name, p.Spec.Containers[0].Name)
-	if err != nil {
-		Logf("Failed to retrieve logs from %v: %v", p.Name, err)
-	} else {
-		Logf("Output of clusterapi-tester:\n%v", logs)
-	}
-}
-
 func kubectlLogPod(c clientset.Interface, pod v1.Pod, containerNameSubstr string, logFunc func(ftm string, args ...interface{})) {
 	for _, container := range pod.Spec.Containers {
 		if strings.Contains(container.Name, containerNameSubstr) {
@@ -791,7 +743,7 @@ func waitForServiceAccountInNamespace(c clientset.Interface, ns, serviceAccountN
 	return err
 }
 
-func waitForPodCondition(c clientset.Interface, ns, podName, desc string, timeout time.Duration, condition podCondition) error {
+func WaitForPodCondition(c clientset.Interface, ns, podName, desc string, timeout time.Duration, condition podCondition) error {
 	Logf("Waiting up to %[1]v for pod %[2]s status to be %[3]s", timeout, podName, desc)
 	for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) {
 		pod, err := c.Core().Pods(ns).Get(podName)
@@ -1354,7 +1306,7 @@ func WaitForPodNotPending(c clientset.Interface, ns, podName, resourceVersion st
 // waitForPodTerminatedInNamespace returns an error if it took too long for the pod
 // to terminate or if the pod terminated with an unexpected reason.
 func waitForPodTerminatedInNamespace(c clientset.Interface, podName, reason, namespace string) error {
-	return waitForPodCondition(c, namespace, podName, "terminated due to deadline exceeded", PodStartTimeout, func(pod *v1.Pod) (bool, error) {
+	return WaitForPodCondition(c, namespace, podName, "terminated due to deadline exceeded", PodStartTimeout, func(pod *v1.Pod) (bool, error) {
 		if pod.Status.Phase == v1.PodFailed {
 			if pod.Status.Reason == reason {
 				return true, nil
@@ -1369,7 +1321,7 @@ func waitForPodTerminatedInNamespace(c clientset.Interface, podName, reason, nam
 
 // waitForPodSuccessInNamespaceTimeout returns nil if the pod reached state success, or an error if it reached failure or ran too long.
 func waitForPodSuccessInNamespaceTimeout(c clientset.Interface, podName string, namespace string, timeout time.Duration) error {
-	return waitForPodCondition(c, namespace, podName, "success or failure", timeout, func(pod *v1.Pod) (bool, error) {
+	return WaitForPodCondition(c, namespace, podName, "success or failure", timeout, func(pod *v1.Pod) (bool, error) {
 		if pod.Spec.RestartPolicy == v1.RestartPolicyAlways {
 			return true, fmt.Errorf("pod %q will never terminate with a succeeded state since its restart policy is Always", podName)
 		}
@@ -3703,7 +3655,7 @@ func CheckPodsCondition(c clientset.Interface, ns string, podNames []string, tim
 	for ix := range podNames {
 		// Launch off pod readiness checkers.
 		go func(name string) {
-			err := waitForPodCondition(c, ns, name, desc, timeout, condition)
+			err := WaitForPodCondition(c, ns, name, desc, timeout, condition)
 			result <- err == nil
 		}(podNames[ix])
 	}
@@ -4786,23 +4738,6 @@ func GetMasterAndWorkerNodesOrDie(c clientset.Interface) (sets.String, *v1.NodeL
 	return masters, nodes
 }
 
-func CreateFileForGoBinData(gobindataPath, outputFilename string) error {
-	data := ReadOrDie(gobindataPath)
-	if len(data) == 0 {
-		return fmt.Errorf("Failed to read gobindata from %v", gobindataPath)
-	}
-	fullPath := filepath.Join(TestContext.OutputDir, outputFilename)
-	err := os.MkdirAll(filepath.Dir(fullPath), 0777)
-	if err != nil {
-		return fmt.Errorf("Error while creating directory %v: %v", filepath.Dir(fullPath), err)
-	}
-	err = ioutil.WriteFile(fullPath, data, 0644)
-	if err != nil {
-		return fmt.Errorf("Error while trying to write to file %v: %v", fullPath, err)
-	}
-	return nil
-}
-
 func ListNamespaceEvents(c clientset.Interface, ns string) error {
 	ls, err := c.Core().Events(ns).List(v1.ListOptions{})
 	if err != nil {

test/e2e/generated/BUILD

@@ -13,9 +13,13 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "gobindata_util.go",
         "main.go",
         ":bindata",
     ],
+    deps = [
+        "//vendor:github.com/golang/glog",
+    ],
 )
 
 genrule(

test/e2e/generated/gobindata_util.go

@@ -14,12 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package framework
+package generated
 
 //go:generate ../../../hack/generate-bindata.sh
 
-// See https://github.com/kubernetes/kubernetes/issues/23987
+import "github.com/golang/glog"
-import "k8s.io/kubernetes/test/e2e/generated"
 
 /*
 ReadOrDie reads a file from gobindata.
@@ -27,11 +26,11 @@ Relies heavily on the successful generation of build artifacts as per the go:gen
 */
 func ReadOrDie(filePath string) []byte {
 
-	fileBytes, err := generated.Asset(filePath)
+	fileBytes, err := Asset(filePath)
 	if err != nil {
-		gobindata_msg := "An error occured, possibly gobindata doesn't know about the file you're opening. For questions on maintaining gobindata, contact the sig-testing group."
+		gobindataMsg := "An error occured, possibly gobindata doesn't know about the file you're opening. For questions on maintaining gobindata, contact the sig-testing group."
-		Logf("Available gobindata files: %v ", generated.AssetNames())
+		glog.Infof("Available gobindata files: %v ", AssetNames())
-		Failf("Failed opening %v , with error %v.  %v.", filePath, err, gobindata_msg)
+		glog.Fatalf("Failed opening %v , with error %v.  %v.", filePath, err, gobindataMsg)
 	}
 	return fileBytes
 }

test/e2e/kubectl.go

@@ -57,6 +57,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/version"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/generated"
 	testutils "k8s.io/kubernetes/test/utils"
 
 	. "github.com/onsi/ginkgo"
@@ -150,7 +151,7 @@ func cleanupKubectlInputs(fileContents string, ns string, selectors ...string) {
 }
 
 func readTestFileOrDie(file string) []byte {
-	return framework.ReadOrDie(path.Join(kubeCtlManifestPath, file))
+	return generated.ReadOrDie(path.Join(kubeCtlManifestPath, file))
 }
 
 func runKubectlRetryOrDie(args ...string) string {
@@ -298,8 +299,8 @@ var _ = framework.KubeDescribe("Kubectl client", func() {
 		var nautilus, kitten []byte
 		BeforeEach(func() {
 			updateDemoRoot := "test/fixtures/doc-yaml/user-guide/update-demo"
-			nautilus = framework.ReadOrDie(filepath.Join(updateDemoRoot, "nautilus-rc.yaml"))
+			nautilus = generated.ReadOrDie(filepath.Join(updateDemoRoot, "nautilus-rc.yaml"))
-			kitten = framework.ReadOrDie(filepath.Join(updateDemoRoot, "kitten-rc.yaml"))
+			kitten = generated.ReadOrDie(filepath.Join(updateDemoRoot, "kitten-rc.yaml"))
 		})
 		It("should create and stop a replication controller [Conformance]", func() {
 			defer cleanupKubectlInputs(string(nautilus), ns, updateDemoSelector)
@@ -344,7 +345,7 @@ var _ = framework.KubeDescribe("Kubectl client", func() {
 				"examples/guestbook/redis-slave-deployment.yaml",
 				"examples/guestbook/redis-slave-service.yaml",
 			} {
-				contents := framework.ReadOrDie(gbAppFile)
+				contents := generated.ReadOrDie(gbAppFile)
 				run(string(contents))
 			}
 		}
@@ -370,7 +371,7 @@ var _ = framework.KubeDescribe("Kubectl client", func() {
 		var podPath []byte
 
 		BeforeEach(func() {
-			podPath = framework.ReadOrDie(path.Join(kubeCtlManifestPath, "pod-with-readiness-probe.yaml"))
+			podPath = generated.ReadOrDie(path.Join(kubeCtlManifestPath, "pod-with-readiness-probe.yaml"))
 			By(fmt.Sprintf("creating the pod from %v", string(podPath)))
 			framework.RunKubectlOrDieInput(string(podPath[:]), "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns))
 			Expect(framework.CheckPodsRunningReady(c, ns, []string{simplePodName}, framework.PodStartTimeout)).To(BeTrue())

test/e2e/serviceloadbalancers.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/wait"
 	utilyaml "k8s.io/kubernetes/pkg/util/yaml"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/generated"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -266,7 +267,7 @@ func simpleGET(c *http.Client, url, host string) (string, error) {
 func rcFromManifest(fileName string) *v1.ReplicationController {
 	var controller v1.ReplicationController
 	framework.Logf("Parsing rc from %v", fileName)
-	data := framework.ReadOrDie(fileName)
+	data := generated.ReadOrDie(fileName)
 
 	json, err := utilyaml.ToJSON(data)
 	Expect(err).NotTo(HaveOccurred())
@@ -279,7 +280,7 @@ func rcFromManifest(fileName string) *v1.ReplicationController {
 func svcFromManifest(fileName string) *v1.Service {
 	var svc v1.Service
 	framework.Logf("Parsing service from %v", fileName)
-	data := framework.ReadOrDie(fileName)
+	data := generated.ReadOrDie(fileName)
 
 	json, err := utilyaml.ToJSON(data)
 	Expect(err).NotTo(HaveOccurred())