Make integration test work with go test.

The EtcdMain function in integration tests is designed to launch an etcd
instance only when running in a bazel test. For non-bazel, we rely on
hack/make-rules/test-integration.sh to bring up the etcd instance.

This patch fixes the following in EtcdMain:
1. If etcd is not found in ${RUNFILES_DIR} then look in ${PATH}.
2. Try to connect to the etcd started by `make test-integraion`; if it
   is up, then don't start etcd.
3. Gracefully shut down etcd after tests.
4. Get a port from the OS instead of deriving it from argv[0].
5. Don't use sync.Once.

The benefit of this change is that integration tests work with `go test`
as well as `make test-integration` without users needing to do anything
special. That makes it much easier to pass go testing flags to tests and
integrate with IDEs.

test/integration/framework/etcd.go

@@ -17,52 +17,86 @@ limitations under the License.
 package framework
 
 import (
+	"context"
 	"fmt"
-	"hash/adler32"
-	"io"
 	"io/ioutil"
-	"math/rand"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"sync"
+	"strings"
 
 	"github.com/golang/glog"
 
 	"k8s.io/kubernetes/pkg/util/env"
 )
 
-var (
+var etcdURL = ""
-	etcdSetup sync.Once
-	etcdURL   = ""
-)
 
-func setupETCD() {
+const installEtcd = `
-	etcdSetup.Do(func() {
+Cannot find etcd, cannot run integration tests
-		if os.Getenv("RUNFILES_DIR") == "" {
+Please see https://github.com/kubernetes/community/blob/master/contributors/devel/testing.md#install-etcd-dependency for instructions.
-			etcdURL = env.GetEnvAsStringOrFallback("KUBE_INTEGRATION_ETCD_URL", "http://127.0.0.1:2379")
+
-			return
+You can use 'hack/install-etcd.sh' to install a copy in third_party/.
+
+`
+
+// getEtcdPath returns a path to an etcd executable.
+func getEtcdPath() (string, error) {
+	bazelPath := filepath.Join(os.Getenv("RUNFILES_DIR"), "com_coreos_etcd/etcd")
+	p, err := exec.LookPath(bazelPath)
+	if err == nil {
+		return p, nil
 	}
-		etcdPath := filepath.Join(os.Getenv("RUNFILES_DIR"), "com_coreos_etcd/etcd")
+	return exec.LookPath("etcd")
-		// give every test the same random port each run
+}
-		etcdPort := 20000 + rand.New(rand.NewSource(int64(adler32.Checksum([]byte(os.Args[0]))))).Intn(5000)
-		etcdURL = fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
 
-		info, err := os.Stat(etcdPath)
+// getAvailablePort returns a TCP port that is available for binding.
+func getAvailablePort() (int, error) {
+	l, err := net.Listen("tcp", ":0")
 	if err != nil {
-			glog.Fatalf("Unable to stat etcd: %v", err)
+		return 0, fmt.Errorf("could not bind to a port: %v", err)
 	}
-		if info.IsDir() {
+	// It is possible but unlikely that someone else will bind this port before we
-			glog.Fatalf("Did not expect %q to be a directory", etcdPath)
+	// get a chance to use it.
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port, nil
+}
+
+// startEtcd executes an etcd instance. The returned function will signal the
+// etcd process and wait for it to exit.
+func startEtcd() (func(), error) {
+	etcdURL = env.GetEnvAsStringOrFallback("KUBE_INTEGRATION_ETCD_URL", "http://127.0.0.1:2379")
+	conn, err := net.Dial("tcp", strings.TrimPrefix(etcdURL, "http://"))
+	if err == nil {
+		glog.Infof("etcd already running at %s", etcdURL)
+		conn.Close()
+		return func() {}, nil
 	}
+	glog.V(1).Infof("could not connect to etcd: %v", err)
+
+	// TODO: Check for valid etcd version.
+	etcdPath, err := getEtcdPath()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, installEtcd)
+		return nil, fmt.Errorf("could not find etcd in PATH: %v", err)
+	}
+	etcdPort, err := getAvailablePort()
+	if err != nil {
+		return nil, fmt.Errorf("could not get a port: %v", err)
+	}
+	etcdURL = fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
+	glog.Infof("starting etcd on %s", etcdURL)
 
 	etcdDataDir, err := ioutil.TempDir(os.TempDir(), "integration_test_etcd_data")
 	if err != nil {
-			glog.Fatalf("Unable to make temp etcd data dir: %v", err)
+		return nil, fmt.Errorf("unable to make temp etcd data dir: %v", err)
 	}
 	glog.Infof("storing etcd data in: %v", etcdDataDir)
 
-		etcdCmd := exec.Command(
+	ctx, cancel := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(
+		ctx,
 		etcdPath,
 		"--data-dir",
 		etcdDataDir,
@@ -73,37 +107,36 @@ func setupETCD() {
 		"--listen-peer-urls",
 		"http://127.0.0.1:0",
 	)
-
+	cmd.Stdout = os.Stdout
-		stdout, err := etcdCmd.StdoutPipe()
+	cmd.Stderr = os.Stderr
+	stop := func() {
+		cancel()
+		err := cmd.Wait()
+		glog.Infof("etcd exit status: %v", err)
+		err = os.RemoveAll(etcdDataDir)
 		if err != nil {
-			glog.Fatalf("Failed to run etcd: %v", err)
+			glog.Warningf("error during etcd cleanup: %v", err)
 		}
-		stderr, err := etcdCmd.StderrPipe()
-		if err != nil {
-			glog.Fatalf("Failed to run etcd: %v", err)
-		}
-		if err := etcdCmd.Start(); err != nil {
-			glog.Fatalf("Failed to run etcd: %v", err)
 	}
 
-		go io.Copy(os.Stdout, stdout)
+	if err := cmd.Start(); err != nil {
-		go io.Copy(os.Stderr, stderr)
+		return nil, fmt.Errorf("failed to run etcd: %v", err)
-
-		go func() {
-			if err := etcdCmd.Wait(); err != nil {
-				glog.Fatalf("Failed to run etcd: %v", err)
 	}
-			glog.Fatalf("etcd should not have succeeded")
+	return stop, nil
-		}()
-	})
 }
 
+// EtcdMain starts an etcd instance before running tests.
 func EtcdMain(tests func() int) {
-	setupETCD()
+	stop, err := startEtcd()
-	os.Exit(tests())
+	if err != nil {
+		glog.Fatalf("cannot run integration tests: unable to start etcd: %v", err)
+	}
+	result := tests()
+	stop() // Don't defer this. See os.Exit documentation.
+	os.Exit(result)
 }
 
-// return the EtcdURL
+// GetEtcdURL returns the URL of the etcd instance started by EtcdMain.
 func GetEtcdURL() string {
 	return etcdURL
 }