Merge pull request #825 from imikushin/qemu-arm

Run tests on qemu-system-aarch64

Dockerfile.dapper

@@ -39,6 +39,7 @@ RUN if [ "${TOOLCHAIN}" != "" ] && ! which ${TOOLCHAIN}-gcc; then \
     ;fi
 
 RUN if [ "${TOOLCHAIN}" != "" ]; then \
+        apt-get update && \
         cd /usr/local/src && \
         for i in libselinux libsepol pcre3 util-linux; do \
             apt-get build-dep -y $i && \
@@ -75,3 +76,6 @@ RUN if [ "${TOOLCHAIN}" != "" ]; then \
         make -j$(nproc) && \
         make install \
     ;fi
+
+RUN apt-get update && \
+    apt-get -y install dosfstools

compose/project.go

@@ -177,7 +177,7 @@ func newCoreServiceProject(cfg *config.CloudConfig, network bool) (*project.Proj
 
 	go func() {
 		for event := range projectEvents {
-			if event.EventType == project.EventContainerStarted && event.ServiceName == "network" {
+			if event.EventType == project.EventContainerStarted && event.ServiceName == "ntp" {
 				network = true
 			}
 		}

os-config.tpl.yml

@@ -105,7 +105,7 @@ rancher:
         io.rancher.os.detach: "false"
         io.rancher.os.reloadconfig: "true"
         io.rancher.os.scope: system
-        io.rancher.os.after: cloud-init-pre, wait-for-network
+        io.rancher.os.after: cloud-init-pre, wait-for-network, ntp
       net: host
       uts: host
       privileged: true
@@ -208,7 +208,7 @@ rancher:
       image: {{.OS_IMAGES_ROOT}}/os-ntp:{{.VERSION}}{{.SUFFIX}}
       labels:
         io.rancher.os.scope: system
-        io.rancher.os.after: cloud-init, wait-for-network
+        io.rancher.os.after: wait-for-network
       net: host
       uts: host
       privileged: true

scripts/run

@@ -1,6 +1,5 @@
 #!/bin/bash
 set -e
-set -x
 
 cd $(dirname $0)/..
 
@@ -23,10 +22,18 @@ hd_amd64() {
     echo "-drive if=virtio,file=$1"
 }
 hd_arm() {
-    echo "-drive if=none,file=$1,id=hd0 -device virtio-blk-device,drive=hd0"
+    echo "-drive if=none,id=hd0,file=$1 -device virtio-blk-device,drive=hd0"
 }
 declare -A hd=( ["amd64"]="hd_amd64" ["arm"]="hd_arm" ["arm64"]="hd_arm" )
 
+cd_amd64() {
+    echo "-hdc $1"
+}
+cd_arm() {
+    echo "-drive if=none,id=cd0,file=$1 -device virtio-blk-device,drive=cd0"
+}
+declare -A cd=( ["amd64"]="cd_amd64" ["arm"]="cd_arm" ["arm64"]="cd_arm" )
+
 BASE=$(pwd)
 UNAME=$(uname)
 
@@ -56,12 +63,6 @@ while [ "$#" -gt 0 ]; do
                 exit 1
             fi
             ;;
-        --xhyve)
-            if [ -x $(which xhyve) ]; then
-                XHYVE=1
-                QEMU=0
-            fi
-            ;;
         --arch)
             shift 1
             ARCH="$1"
@@ -152,18 +153,28 @@ if [ "$KVM" == "" ] && [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; th
     KVM=1
 fi
 
-if [ "$XHYVE" == "1" ] || [ "$QEMU" == "1" ]; then
+if [ "$QEMU" == "1" ]; then
 
     HD=${BASE}/state/hd.img
     [ "$FRESH" == "1" ] && rm -f ${HD} >/dev/null 2>&1 || :
     if [ ! -e ${HD} ]; then
         mkdir -p $(dirname ${HD})
-        if [ "$XHYVE" == "1" ]; then qemu-img create -f raw -o size=10G ${HD}
-        else qemu-img create -f qcow2 -o size=10G ${HD}
-        fi
+        qemu-img create -f qcow2 -o size=10G ${HD}
     fi
 
-    USER_DATA=${BUILD}/cloud-config/openstack/latest/user_data
+
+    CCIMG=${BUILD}/cc.img
+    CCDEV=$(losetup -f)
+    CCROOT=${BUILD}/cloud-config
+    dd if=/dev/zero of=${CCIMG} bs=1024 count=512
+    losetup ${CCDEV} ${CCIMG}
+    mkfs.vfat -n "config-2" ${CCDEV}
+
+    rm -rf ${CCROOT}
+    mkdir -p ${CCROOT}
+    mount ${CCDEV} ${CCROOT}
+
+    USER_DATA=${CCROOT}/openstack/latest/user_data
     mkdir -p $(dirname ${USER_DATA})
     rm -f ${USER_DATA} || :
     if [ -n "$CLOUD_CONFIG" ]; then
@@ -174,51 +185,34 @@ if [ "$XHYVE" == "1" ] || [ "$QEMU" == "1" ]; then
         echo "- $(<${BASE}/assets/rancher.key.pub)" >> ${USER_DATA}
     fi
 
-    CLOUD_CONFIG_ISO="${BUILD}/cloud-config.iso"
-    rm -rf ${CLOUD_CONFIG_ISO}
-    mkisofs -R -V config-2 -o "${CLOUD_CONFIG_ISO}" "$BUILD/cloud-config"
+    umount ${CCROOT}
+    losetup -d ${CCDEV}
 
-    if [ "$XHYVE" == "1" ]; then
-
-        exec sudo -n xhyve -A -H -P -u \
-            -m 4G \
-            -s 0:0,hostbridge -s 31,lpc \
-            -l com1,stdio \
-            -s 2:0,virtio-net \
-            -s 3,ahci-cd,${CLOUD_CONFIG_ISO} \
-            -s 4,virtio-blk,${HD} \
-            -U a01fb25c-3a19-4759-a47a-2e353e51807d \
-            -f kexec,${KERNEL},${INITRD},"earlyprintk=serial elevator=noop ${KERNEL_ARGS}" \
-            "${@}"
-
-    elif [ "$QEMU" == "1" ]; then
 
     CPU=${cpu["$ARCH"]}
     if [ "$KVM" == "1" ] && [ "$ARCH" == "$HOST_ARCH" ]; then
         KVM_ENABLE="-enable-kvm"
         CPU="-cpu host"
     fi
+    set -x
     exec qemu-system-${QEMUARCH} -serial stdio \
+        -rtc base=utc,clock=host \
         ${KVM_ENABLE} \
         ${CPU} \
         ${machine["$ARCH"]} \
         -kernel ${KERNEL} \
         -initrd ${INITRD} \
-            -m 1024 \
+        -m 2048 \
         ${network["$ARCH"]} \
+        $(eval "${cd["$ARCH"]} ${CCIMG}") \
         $(eval "${hd["$ARCH"]} ${HD}") \
         -smp 1 \
-            -cdrom ${CLOUD_CONFIG_ISO} \
         -append "${KERNEL_ARGS}" \
         -nographic \
         -display none \
         ${QEMU_ARGS} \
         "${@}"
 
-    else
-        exit 42
-    fi
-
 elif [ "$QIND" == "1" ]; then
 
     NAME=${NAME:-ros-qind}
@@ -246,6 +240,7 @@ elif [ "$QIND" == "1" ]; then
             SSH_PUB_ENABLE="--ssh-pub /stuff/rancher.key.pub"
         fi
 
+        set -x
         docker create --name=${NAME} -it ${KVM_ENABLE} \
                     rancher/qind \
                     --hostname "rancher-dev" \

scripts/shell.sh

@@ -4,4 +4,4 @@ set -e
 cd $(dirname $0)/..
 . ./scripts/dapper-common
 
-exec dapper -s
+exec dapper -d -s

scripts/ssh

@@ -18,10 +18,6 @@ while [ "$#" -gt 0 ]; do
         --notty)
             INTERACTIVE=""
             ;;
-        --xhyve)
-            XHYVE=1
-            QEMU=0
-            ;;
         --qemu)
             QEMU=1
             ;;
@@ -40,10 +36,7 @@ while [ "$#" -gt 0 ]; do
     shift 1
 done
 
-if [ "$XHYVE" == "1" ]; then
-    HOST=192.168.64.2  #  consult `/var/db/dhcpd_leases` or delete it
-    exec ssh -F ./assets/scripts_ssh_config -i ${KEY:-./assets/rancher.key} rancher@${HOST} "$@"
-elif [ "$QEMU" == "1" ]; then
+if [ "$QEMU" == "1" ]; then
     exec ssh -p 2222 -F ./assets/scripts_ssh_config -i ${KEY:-./assets/rancher.key} rancher@localhost "$@"
 elif [ "$QIND" == "1" ]; then
     NAME=${NAME:-ros-qind}

tests/integration/assets/test_01/cloud-config.yml

@@ -9,7 +9,7 @@ rancher:
     interfaces:
       eth*:
         dhcp: true
-      eth1:
+      "mac=52:54:00:12:34:59":
         address: 10.10.2.17/24
         gateway: 10.10.2.2
         mtu: 1500

tests/integration/rostest/test_00_system.py

@@ -18,12 +18,15 @@ def test_system_boot(qemu):
     u.flush_out(qemu.stdout, 'RancherOS {v} started'.format(v=version))
 
 
+busybox = {'amd64': 'busybox', 'arm': 'armhf/busybox', 'arm64': 'aarch64/busybox'}
+
+
 @pytest.mark.timeout(60)
 def test_run_system_container(qemu):
     u.wait_for_ssh(qemu)
 
     ssh = subprocess.Popen(
-        './scripts/ssh --qemu sudo system-docker run --rm busybox /bin/true',
+        './scripts/ssh --qemu sudo system-docker run --rm ' + busybox[u.arch] + ' /bin/true',
         shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
 
     for ln in u.iter_lines(ssh.stdout):

tests/integration/rostest/test_01_cloud_config.py

@@ -1,5 +1,4 @@
 import string
-import subprocess
 
 import pytest
 import rostest.util as u
@@ -10,10 +9,16 @@ ssh_command = ['./scripts/ssh', '--qemu', '--key', './tests/integration/assets/t
 cloud_config_path = './tests/integration/assets/test_01/cloud-config.yml'
 
 
+net_args = {'amd64': ['-net', 'nic,vlan=1,model=virtio,macaddr=52:54:00:12:34:59',
+                      '-net', 'user,vlan=1,net=10.10.2.0/24'],
+            'arm64': ['-netdev', 'user,id=net1,net=10.10.2.0/24',
+                      '-device', 'virtio-net-device,netdev=net1,mac=52:54:00:12:34:59']}
+net_args['arm'] = net_args['arm64']
+
+
 @pytest.fixture(scope="module")
 def qemu(request):
-    q = u.run_qemu(request, ['--cloud-config', cloud_config_path,
-                             '-net', 'nic,vlan=1,model=virtio', '-net', 'user,vlan=1,net=10.10.2.0/24'])
+    q = u.run_qemu(request, ['--cloud-config', cloud_config_path] + net_args[u.arch])
     u.flush_out(q.stdout)
     return q
 
@@ -31,22 +36,18 @@ def test_ssh_authorized_keys(qemu):
 
 @pytest.mark.timeout(40)
 def test_rancher_environment(qemu, cloud_config):
-    u.wait_for_ssh(qemu, ssh_command)
-
-    v = subprocess.check_output(
-        ssh_command + ['sudo', 'ros', 'env', 'printenv', 'FLANNEL_NETWORK'],
-        stderr=subprocess.STDOUT, universal_newlines=True)
+    v = SSH(qemu, ssh_command).check_output('''
+sudo ros env printenv FLANNEL_NETWORK
+    '''.strip())
 
     assert v.strip() == cloud_config['rancher']['environment']['FLANNEL_NETWORK']
 
 
 @pytest.mark.timeout(40)
 def test_docker_args(qemu, cloud_config):
-    u.wait_for_ssh(qemu, ssh_command)
-
-    v = subprocess.check_output(
-        ssh_command + ['sh', '-c', 'ps -ef | grep docker'],
-        stderr=subprocess.STDOUT, universal_newlines=True)
+    v = SSH(qemu, ssh_command).check_output('''
+ps -ef | grep docker
+    '''.strip())
 
     expected = string.join(cloud_config['rancher']['docker']['args'])
 
@@ -55,11 +56,9 @@ def test_docker_args(qemu, cloud_config):
 
 @pytest.mark.timeout(40)
 def test_dhcpcd(qemu, cloud_config):
-    u.wait_for_ssh(qemu, ssh_command)
-
-    v = subprocess.check_output(
-        ssh_command + ['sh', '-c', 'ps -ef | grep dhcpcd'],
-        stderr=subprocess.STDOUT, universal_newlines=True)
+    v = SSH(qemu, ssh_command).check_output('''
+ps -ef | grep dhcpcd
+    '''.strip())
 
     assert v.find('dhcpcd -M') != -1
 
@@ -71,33 +70,28 @@ def test_services_include(qemu, cloud_config):
 
 @pytest.mark.timeout(40)
 def test_docker_tls_args(qemu, cloud_config):
-    u.wait_for_ssh(qemu, ssh_command)
-
-    subprocess.check_call(
-        ssh_command + ['sudo', 'ros', 'tls', 'gen'],
-        stderr=subprocess.STDOUT, universal_newlines=True)
-
-    subprocess.check_call(
-        ssh_command + ['docker', '--tlsverify', 'version'],
-        stderr=subprocess.STDOUT, universal_newlines=True)
+    SSH(qemu, ssh_command).check_call('''
+set -e -x
+sudo ros tls gen
+sleep 5
+docker --tlsverify version
+    '''.strip())
 
 
 @pytest.mark.timeout(40)
 def test_rancher_network(qemu, cloud_config):
-    u.wait_for_ssh(qemu, ssh_command)
+    v = SSH(qemu, ssh_command).check_output('''
+ip route get to 10.10.2.120
+    '''.strip())
 
-    v = subprocess.check_output(
-        ssh_command + ['ip', 'route', 'get', 'to', '10.10.2.120'],
-        stderr=subprocess.STDOUT, universal_newlines=True)
-
-    assert v.split(' ')[2] == 'eth1'
-    assert v.split(' ')[5] + '/24' == cloud_config['rancher']['network']['interfaces']['eth1']['address']
+    assert v.split(' ')[5] + '/24' == \
+        cloud_config['rancher']['network']['interfaces']['mac=52:54:00:12:34:59']['address']
 
 
 def test_docker_not_pid_one(qemu):
-    SSH(qemu, ssh_command=ssh_command).check_call('bash', '-c', '''
-    set -e -x
-    for i in $(pidof docker); do
+    SSH(qemu, ssh_command).check_call('''
+set -e -x
+for i in $(pidof docker); do
     [ $i != 1 ]
-    done
+done
     '''.strip())

tests/integration/rostest/test_03_docker_in_persistent_console.py

@@ -1,7 +1,8 @@
-import subprocess
+import time
 
 import pytest
 import rostest.util as u
+from rostest.util import SSH
 
 ssh_command = ['./scripts/ssh', '--qemu', '--key', './tests/integration/assets/test.key']
 cloud_config_path = './tests/integration/assets/test_03/cloud-config.yml'
@@ -14,17 +15,24 @@ def qemu(request):
     return q
 
 
+nginx = {'amd64': 'nginx', 'arm': 'armhfbuild/nginx', 'arm64': 'armhfbuild/nginx'}
+
+
 @pytest.mark.timeout(40)
 def test_reboot_with_container_running(qemu):
-    u.wait_for_ssh(qemu, ssh_command)
-    subprocess.check_call(ssh_command + ['docker', 'run', '-d', '--restart=always', 'nginx'],
-                          stderr=subprocess.STDOUT, universal_newlines=True)
+    try:
+        SSH(qemu, ssh_command).check_call('''
+set -ex
+docker run -d --restart=always %(image)s
+sudo reboot
+    '''.strip() % {'image': nginx[u.arch]})
+    except:
+        pass
 
-    subprocess.call(ssh_command + ['sudo', 'reboot'],
-                    stderr=subprocess.STDOUT, universal_newlines=True)
+    time.sleep(3)
 
-    u.wait_for_ssh(qemu, ssh_command)
-    v = subprocess.check_output(ssh_command + ['docker', 'ps', '-f', 'status=running'],
-                                stderr=subprocess.STDOUT, universal_newlines=True)
+    v = SSH(qemu, ssh_command).check_output('''
+docker ps -f status=running
+    '''.strip())
 
     assert v.find('nginx') != -1

tests/integration/rostest/test_04_ros_install.py

@@ -20,7 +20,7 @@ def test_ros_install_on_formatted_disk(qemu):
                           stderr=subprocess.STDOUT, universal_newlines=True)
 
     subprocess.check_call(ssh_command + ['sudo', 'ros', 'install', '-f', '--no-reboot', '-d', '/dev/vda',
-                                         '-i', 'rancher/os:v0.4.1'],
+                                         '-i', 'rancher/os:v0.4.4-dev' + u.suffix],
                           stderr=subprocess.STDOUT, universal_newlines=True)
 
     subprocess.call(ssh_command + ['sudo', 'reboot'],

tests/integration/rostest/test_05_custom_docker_in_persistent_console.py

@@ -13,18 +13,21 @@ def qemu(request):
     u.flush_out(q.stdout)
     return q
 
+docker_url = {'amd64': 'https://experimental.docker.com/builds/Linux/x86_64/docker-1.10.0-dev',
+              'arm': 'https://github.com/rancher/docker/releases/download/v1.10.3-arm/docker-1.10.3_arm',
+              'arm64': 'https://github.com/rancher/docker/releases/download/v1.10.3-arm/docker-1.10.3_arm64'}
+
 
 @pytest.mark.timeout(40)
 def test_system_docker_survives_custom_docker_install(qemu):
     u.wait_for_ssh(qemu, ssh_command)
-    subprocess.check_call(ssh_command + ['curl', '-OL',
-                                         'https://experimental.docker.com/builds/Linux/x86_64/docker-1.10.0-dev'],
+    subprocess.check_call(ssh_command + ['curl', '-Lfo', './docker', docker_url[u.arch]],
                           stderr=subprocess.STDOUT, universal_newlines=True)
 
-    subprocess.check_call(ssh_command + ['chmod', '+x', '/home/rancher/docker-1.10.0-dev'],
+    subprocess.check_call(ssh_command + ['chmod', '+x', '/home/rancher/docker'],
                           stderr=subprocess.STDOUT, universal_newlines=True)
 
-    subprocess.check_call(ssh_command + ['sudo', 'ln', '-sf', '/home/rancher/docker-1.10.0-dev', '/usr/bin/docker'],
+    subprocess.check_call(ssh_command + ['sudo', 'ln', '-sf', '/home/rancher/docker', '/usr/bin/docker'],
                           stderr=subprocess.STDOUT, universal_newlines=True)
 
     subprocess.check_call(ssh_command + ['sudo', 'system-docker', 'restart', 'docker'],

tests/integration/rostest/test_06_subdir.py

@@ -11,9 +11,9 @@ def qemu(request):
 
 
 def test_system_docker_survives_custom_docker_install(qemu):
-    SSH(qemu).check_call('bash', '-c', '''
-        set -x -e
-        mkdir x
-        sudo mount $(sudo ros dev LABEL=RANCHER_STATE) x
-        [ -d x/ros_subdir/home/rancher ]
+    SSH(qemu).check_call('''
+set -x -e
+mkdir x
+sudo mount $(sudo ros dev LABEL=RANCHER_STATE) x
+[ -d x/ros_subdir/home/rancher ]
     '''.strip())

tests/integration/rostest/test_07_oem.py

@@ -12,12 +12,13 @@ def qemu(request):
 
 
 def test_oem(qemu):
-    SSH(qemu).check_call('sudo', 'bash', '-c', '''
+    try:
+        SSH(qemu).check_call('''
 set -x
 set -e
 sudo mkfs.ext4 -L RANCHER_OEM /dev/vda
 sudo mount /dev/vda /mnt
-cat > /tmp/oem-config.yml << "EOF"
+cat > /tmp/oem-config.yml << EOF
 #cloud-config
 rancher:
   upgrade:
@@ -25,11 +26,14 @@ rancher:
 EOF
 sudo cp /tmp/oem-config.yml /mnt
 sudo umount /mnt
-sudo reboot >/dev/null 2>&1 &'''.strip())
+sudo reboot
+        '''.strip())
+    except:
+        pass
 
-    time.sleep(1)
+    time.sleep(3)
 
-    SSH(qemu).check_call('bash', '-c', '''
+    SSH(qemu).check_call('''
 set -x
 if [ ! -e /usr/share/ros/oem/oem-config.yml ]; then
     echo Failed to find /usr/share/ros/oem/oem-config.yml

tests/integration/rostest/test_08_ssh_key_merge.py

@@ -11,7 +11,7 @@ def qemu(request):
 
 
 def test_ssh_key_merging(qemu):
-    SSH(qemu).check_call('bash', '-c', '''cat > test-merge << "SCRIPT"
+    SSH(qemu).check_call('''cat > test-merge << "SCRIPT"
 set -x -e
 rm /var/lib/rancher/conf/cloud-config.yml
 

tests/integration/rostest/test_09_network.py

@@ -2,26 +2,27 @@ import pytest
 import rostest.util as u
 from rostest.util import SSH
 
+pytestmark = pytest.mark.skipif(u.arch != 'amd64', reason='amd64 network setup impossible to replicate for arm64')
 
 cloud_config_path = './tests/integration/assets/test_09/cloud-config.yml'
 
+net_args_arch = {'amd64': ['-net', 'nic,vlan=0,model=virtio'],
+                 'arm64': ['-device', 'virtio-net-device']}
+net_args_arch['arm'] = net_args_arch['arm64']
+net_args = net_args_arch[u.arch]
+
 
 @pytest.fixture(scope="module")
 def qemu(request):
-    q = u.run_qemu(request, run_args=['--cloud-config', cloud_config_path,
-                                      '-net', 'nic,vlan=0,model=virtio',
-                                      '-net', 'nic,vlan=0,model=virtio',
-                                      '-net', 'nic,vlan=0,model=virtio',
-                                      '-net', 'nic,vlan=0,model=virtio',
-                                      '-net', 'nic,vlan=0,model=virtio',
-                                      '-net', 'nic,vlan=0,model=virtio',
-                                      '-net', 'nic,vlan=0,model=virtio'])
+    q = u.run_qemu(request,
+                   run_args=['--cloud-config', cloud_config_path] +
+                   net_args + net_args + net_args + net_args + net_args + net_args + net_args)
     u.flush_out(q.stdout)
     return q
 
 
 def test_network_conf(qemu):
-    SSH(qemu).check_call('bash', '-c', '''cat > test-merge << "SCRIPT"
+    SSH(qemu).check_call('''cat > test-merge << "SCRIPT"
 set -x -e
 
 ip link show dev br0
@@ -37,7 +38,7 @@ sudo bash test-merge
 
 
 def test_network_dns_ros_set(qemu):
-    SSH(qemu).check_call('bash', '-c', '''
+    SSH(qemu).check_call('''
 set -x -e
 
 sudo ros config set rancher.network.dns.search '[a,b]'

tests/integration/rostest/test_sample.py

@@ -1,6 +0,0 @@
-def func(x):
-    return x + 1
-
-
-def test_answer():
-    assert func(3) == 4

tests/integration/rostest/util.py

@@ -1,12 +1,18 @@
 from __future__ import print_function
 
 import itertools as it
+import os
 import subprocess
 import time
 
 import pytest
 
 ros_test = 'ros-test'
+arch = os.environ['ARCH']
+
+suffix = ''
+if arch != 'amd64':
+    suffix = '_' + arch
 
 
 def iter_lines(s):
@@ -84,8 +90,8 @@ def wait_for_ssh(qemu, ssh_command=['./scripts/ssh', '--qemu'], command=['docker
         i += 1
         print('\nWaiting for ssh and docker... ' + str(i))
         time.sleep(1)
-        if i > 60:
-            raise 'Failed to connect to SSH'
+        if i > 150:
+            raise AssertionError('Failed to connect to SSH')
         assert qemu.returncode is None
 
 
@@ -95,11 +101,19 @@ class SSH:
         self._ssh_command = ssh_command
         self._waited = False
 
-    def check_call(self, *args, **kw):
+    def wait(self):
         if not self._waited:
             wait_for_ssh(self._qemu, ssh_command=self._ssh_command)
             self._waited = True
 
+    def check_call(self, *args, **kw):
+        self.wait()
         kw['stderr'] = subprocess.STDOUT
         kw['universal_newlines'] = True
         return subprocess.check_call(self._ssh_command + list(args), **kw)
+
+    def check_output(self, *args, **kw):
+        self.wait()
+        kw['stderr'] = subprocess.STDOUT
+        kw['universal_newlines'] = True
+        return subprocess.check_output(self._ssh_command + list(args), **kw)

util/util.go

@@ -294,6 +294,16 @@ func DirLs(dir string) ([]interface{}, error) {
 	return result, nil
 }
 
+func retryHttp(f func() (*http.Response, error), times int) (resp *http.Response, err error) {
+	for i := 0; i < times; i++ {
+		if resp, err = f(); err == nil {
+			return
+		}
+		log.Warnf("Error making HTTP request: %s. Retrying", err)
+	}
+	return
+}
+
 func LoadResource(location string, network bool, urls []string) ([]byte, error) {
 	var bytes []byte
 	err := ErrNotFound
@@ -302,7 +312,7 @@ func LoadResource(location string, network bool, urls []string) ([]byte, error)
 		if !network {
 			return nil, ErrNoNetwork
 		}
-		resp, err := http.Get(location)
+		resp, err := retryHttp(func() (*http.Response, error) { return http.Get(location) }, 8)
 		if err != nil {
 			return nil, err
 		}