test-e2e-node: support pure SSH mode

Right now, `run_remote.go` only supports GCE instances. But actually
running the tests is completely independent of GCE and could work just
as well on any SSH-accessible machine.

This patch adds a new `--mode` switch, which defaults to `gce` for
backwards compatibility, but can be set to `ssh`. In that mode, the GCE
API is not used at all, and we simply connect to the hosts given via
`--hosts`.

This is still better than `run_local.go` because the latter mixes build
environment with test environment, which doesn't fit well with
container-optimized operating systems.

This is part of an effort to setup the e2e node tests on Fedora CoreOS
(see https://github.com/coreos/fedora-coreos-tracker/issues/990).

Patch best viewed with whitespace ignored.

build/root/Makefile

@@ -214,8 +214,10 @@ define TEST_E2E_NODE_HELP_INFO
 #    Defaults to "".
 #  RUN_UNTIL_FAILURE: If true, pass --untilItFails to ginkgo so tests are run
 #    repeatedly until they fail.  Defaults to false.
-#  REMOTE: If true, run the tests on a remote host instance on GCE.  Defaults
-#    to false.
+#  REMOTE: If true, run the tests on a remote host.  Defaults to false.
+#  REMOTE_MODE: For REMOTE=true only.  Mode for remote execution (eg. gce, ssh).
+#    If set to "gce", an instance can be provisioned or reused from GCE. If set
+#    to "ssh", provided `HOSTS` must be IPs or resolvable.  Defaults to "gce".
 #  ARTIFACTS: Local directory to scp test artifacts into from the remote hosts
 #    for REMOTE=true. Local directory to write juntil xml results into for REMOTE=false.
 #    Defaults to "/tmp/_artifacts/$$(date +%y%m%dT%H%M%S)".

hack/make-rules/test-e2e-node.sh

@@ -38,6 +38,7 @@ skip=${SKIP-"\[Flaky\]|\[Slow\]|\[Serial\]"}
 parallelism=${PARALLELISM:-8}
 artifacts="${ARTIFACTS:-"/tmp/_artifacts/$(date +%y%m%dT%H%M%S)"}"
 remote=${REMOTE:-"false"}
+remote_mode=${REMOTE_MODE:-"gce"}
 runtime=${RUNTIME:-"docker"}
 container_runtime_endpoint=${CONTAINER_RUNTIME_ENDPOINT:-""}
 image_service_endpoint=${IMAGE_SERVICE_ENDPOINT:-""}
@@ -87,8 +88,8 @@ if [[ ${runtime} == "remote" ]] ; then
 fi
 
 
-if [ "${remote}" = true ] ; then
-  # The following options are only valid in remote run.
+if [ "${remote}" = true ] && [ "${remote_mode}" = gce ] ; then
+  # The following options are only valid in remote GCE run.
   images=${IMAGES:-""}
   hosts=${HOSTS:-""}
   image_project=${IMAGE_PROJECT:-"kubernetes-node-e2e-images"}
@@ -185,6 +186,28 @@ if [ "${remote}" = true ] ; then
     2>&1 | tee -i "${artifacts}/build-log.txt"
   exit $?
 
+elif [ "${remote}" = true ] && [ "${remote_mode}" = ssh ] ; then
+  hosts=${HOSTS:-""}
+  test_suite=${TEST_SUITE:-"default"}
+  if [[ -n "${TIMEOUT:-}" ]] ; then
+    timeout_arg="--test-timeout=${TIMEOUT}"
+  fi
+
+  # Use cluster.local as default dns-domain
+  test_args='--dns-domain="'${KUBE_DNS_DOMAIN:-cluster.local}'" '${test_args}
+  test_args='--kubelet-flags="--cluster-domain='${KUBE_DNS_DOMAIN:-cluster.local}'" '${test_args}
+
+  # Invoke the runner
+  go run test/e2e_node/runner/remote/run_remote.go  --mode="ssh" --logtostderr --vmodule=*=4 \
+    --hosts="${hosts}" --results-dir="${artifacts}" --ginkgo-flags="${ginkgoflags}" \
+    --test_args="${test_args}" --system-spec-name="${system_spec_name}" \
+    --runtime-config="${runtime_config}" \
+    --ssh-user="${ssh_user}" --ssh-key="${ssh_key}" --ssh-options="${ssh_options}" \
+    --extra-envs="${extra_envs}" --test-suite="${test_suite}" \
+    "${timeout_arg}" \
+    2>&1 | tee -i "${artifacts}/build-log.txt"
+  exit $?
+
 else
   # Refresh sudo credentials if needed
   if ping -c 1 -q metadata.google.internal &> /dev/null; then

test/e2e_node/runner/remote/run_remote.go

@@ -49,6 +49,7 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+var mode = flag.String("mode", "gce", "Mode to operate in. One of gce|ssh. Defaults to gce")
 var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
 var testSuite = flag.String("test-suite", "default", "Test suite the runner initializes with. Currently support default|cadvisor|conformance")
 var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names")
@@ -223,22 +224,22 @@ func main() {
 		return
 	}
 
-	if *hosts == "" && *imageConfigFile == "" && *images == "" {
-		klog.Fatalf("Must specify one of --image-config-file, --hosts, --images.")
-	}
-	var err error
-	computeService, err = getComputeClient()
-	if err != nil {
-		klog.Fatalf("Unable to create gcloud compute service using defaults.  Make sure you are authenticated. %v", err)
-	}
-
 	var gceImages *internalImageConfig
-	if gceImages, err = prepareGceImages(); err != nil {
-		klog.Fatalf("While preparing GCE images: %v", err)
-	}
-
-	if *instanceNamePrefix == "" {
-		*instanceNamePrefix = "tmp-node-e2e-" + uuid.New().String()[:8]
+	if *mode == "gce" {
+		if *hosts == "" && *imageConfigFile == "" && *images == "" {
+			klog.Fatalf("Must specify one of --image-config-file, --hosts, --images.")
+		}
+		var err error
+		computeService, err = getComputeClient()
+		if err != nil {
+			klog.Fatalf("Unable to create gcloud compute service using defaults.  Make sure you are authenticated. %v", err)
+		}
+		if gceImages, err = prepareGceImages(); err != nil {
+			klog.Fatalf("While preparing GCE images: %v", err)
+		}
+		if *instanceNamePrefix == "" {
+			*instanceNamePrefix = "tmp-node-e2e-" + uuid.New().String()[:8]
+		}
 	}
 
 	// Setup coloring