From 29aff3d807de10e8ecefc7f8efd1dbc20c1d4eb4 Mon Sep 17 00:00:00 2001 From: Ivan Mikushin Date: Wed, 23 Mar 2016 15:31:33 -0700 Subject: [PATCH 1/7] Dockerfile.dapper: make sure to update apt repos before fetching build deps and sources Also, use dapper -d flag in scripts/shell.sh for better visibility --- Dockerfile.dapper | 1 + scripts/shell.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 81dd4eee..da3803be 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -39,6 +39,7 @@ RUN if [ "${TOOLCHAIN}" != "" ] && ! which ${TOOLCHAIN}-gcc; then \ ;fi RUN if [ "${TOOLCHAIN}" != "" ]; then \ + apt-get update && \ cd /usr/local/src && \ for i in libselinux libsepol pcre3 util-linux; do \ apt-get build-dep -y $i && \ diff --git a/scripts/shell.sh b/scripts/shell.sh index a11fc635..f171e10e 100755 --- a/scripts/shell.sh +++ b/scripts/shell.sh @@ -4,4 +4,4 @@ set -e cd $(dirname $0)/.. . ./scripts/dapper-common -exec dapper -s +exec dapper -d -s From 99c3207b14cbbeee14af9e76b8e624bd92db7317 Mon Sep 17 00:00:00 2001 From: Ivan Mikushin Date: Wed, 23 Mar 2016 13:28:05 -0700 Subject: [PATCH 2/7] Hook cloud-config into qemu-system-aarch46 --- Dockerfile.dapper | 3 ++ scripts/run | 99 ++++++++++++++++++++++------------------------- 2 files changed, 49 insertions(+), 53 deletions(-) diff --git a/Dockerfile.dapper b/Dockerfile.dapper index da3803be..0a98e613 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -76,3 +76,6 @@ RUN if [ "${TOOLCHAIN}" != "" ]; then \ make -j$(nproc) && \ make install \ ;fi + +RUN apt-get update && \ + apt-get -y install dosfstools diff --git a/scripts/run b/scripts/run index ae5b6f31..0619c38f 100755 --- a/scripts/run +++ b/scripts/run @@ -23,10 +23,18 @@ hd_amd64() { echo "-drive if=virtio,file=$1" } hd_arm() { - echo "-drive if=none,file=$1,id=hd0 -device virtio-blk-device,drive=hd0" + echo "-drive if=none,id=hd0,file=$1 -device virtio-blk-device,drive=hd0" } declare -A hd=( ["amd64"]="hd_amd64" ["arm"]="hd_arm" ["arm64"]="hd_arm" ) +cd_amd64() { + echo "-hdc $1" +} +cd_arm() { + echo "-drive if=none,id=cd0,file=$1 -device virtio-blk-device,drive=cd0" +} +declare -A cd=( ["amd64"]="cd_amd64" ["arm"]="cd_arm" ["arm64"]="cd_arm" ) + BASE=$(pwd) UNAME=$(uname) @@ -56,12 +64,6 @@ while [ "$#" -gt 0 ]; do exit 1 fi ;; - --xhyve) - if [ -x $(which xhyve) ]; then - XHYVE=1 - QEMU=0 - fi - ;; --arch) shift 1 ARCH="$1" @@ -152,18 +154,28 @@ if [ "$KVM" == "" ] && [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; th KVM=1 fi -if [ "$XHYVE" == "1" ] || [ "$QEMU" == "1" ]; then +if [ "$QEMU" == "1" ]; then HD=${BASE}/state/hd.img [ "$FRESH" == "1" ] && rm -f ${HD} >/dev/null 2>&1 || : if [ ! -e ${HD} ]; then mkdir -p $(dirname ${HD}) - if [ "$XHYVE" == "1" ]; then qemu-img create -f raw -o size=10G ${HD} - else qemu-img create -f qcow2 -o size=10G ${HD} - fi + qemu-img create -f qcow2 -o size=10G ${HD} fi - USER_DATA=${BUILD}/cloud-config/openstack/latest/user_data + + CCIMG=${BUILD}/cc.img + CCDEV=$(losetup -f) + CCROOT=${BUILD}/cloud-config + dd if=/dev/zero of=${CCIMG} bs=1024 count=512 + losetup ${CCDEV} ${CCIMG} + mkfs.vfat -n "config-2" ${CCDEV} + + rm -rf ${CCROOT} + mkdir -p ${CCROOT} + mount ${CCDEV} ${CCROOT} + + USER_DATA=${CCROOT}/openstack/latest/user_data mkdir -p $(dirname ${USER_DATA}) rm -f ${USER_DATA} || : if [ -n "$CLOUD_CONFIG" ]; then @@ -174,50 +186,31 @@ if [ "$XHYVE" == "1" ] || [ "$QEMU" == "1" ]; then echo "- $(<${BASE}/assets/rancher.key.pub)" >> ${USER_DATA} fi - CLOUD_CONFIG_ISO="${BUILD}/cloud-config.iso" - rm -rf ${CLOUD_CONFIG_ISO} - mkisofs -R -V config-2 -o "${CLOUD_CONFIG_ISO}" "$BUILD/cloud-config" + umount ${CCROOT} + losetup -d ${CCDEV} - if [ "$XHYVE" == "1" ]; then - exec sudo -n xhyve -A -H -P -u \ - -m 4G \ - -s 0:0,hostbridge -s 31,lpc \ - -l com1,stdio \ - -s 2:0,virtio-net \ - -s 3,ahci-cd,${CLOUD_CONFIG_ISO} \ - -s 4,virtio-blk,${HD} \ - -U a01fb25c-3a19-4759-a47a-2e353e51807d \ - -f kexec,${KERNEL},${INITRD},"earlyprintk=serial elevator=noop ${KERNEL_ARGS}" \ - "${@}" - - elif [ "$QEMU" == "1" ]; then - - CPU=${cpu["$ARCH"]} - if [ "$KVM" == "1" ] && [ "$ARCH" == "$HOST_ARCH" ]; then - KVM_ENABLE="-enable-kvm" - CPU="-cpu host" - fi - exec qemu-system-${QEMUARCH} -serial stdio \ - ${KVM_ENABLE} \ - ${CPU} \ - ${machine["$ARCH"]} \ - -kernel ${KERNEL} \ - -initrd ${INITRD} \ - -m 1024 \ - ${network["$ARCH"]} \ - $(eval "${hd["$ARCH"]} ${HD}") \ - -smp 1 \ - -cdrom ${CLOUD_CONFIG_ISO} \ - -append "${KERNEL_ARGS}" \ - -nographic \ - -display none \ - ${QEMU_ARGS} \ - "${@}" - - else - exit 42 + CPU=${cpu["$ARCH"]} + if [ "$KVM" == "1" ] && [ "$ARCH" == "$HOST_ARCH" ]; then + KVM_ENABLE="-enable-kvm" + CPU="-cpu host" fi + exec qemu-system-${QEMUARCH} -serial stdio \ + ${KVM_ENABLE} \ + ${CPU} \ + ${machine["$ARCH"]} \ + -kernel ${KERNEL} \ + -initrd ${INITRD} \ + -m 1024 \ + ${network["$ARCH"]} \ + $(eval "${cd["$ARCH"]} ${CCIMG}") \ + $(eval "${hd["$ARCH"]} ${HD}") \ + -smp 1 \ + -append "${KERNEL_ARGS}" \ + -nographic \ + -display none \ + ${QEMU_ARGS} \ + "${@}" elif [ "$QIND" == "1" ]; then From 95d24ac67d59ddc2e9ffaabec58f40b12b444b13 Mon Sep 17 00:00:00 2001 From: Ivan Mikushin Date: Wed, 30 Mar 2016 17:48:31 -0700 Subject: [PATCH 3/7] Modify the tests to work with multiple architectures --- tests/integration/rostest/test_00_system.py | 5 ++++- tests/integration/rostest/test_01_cloud_config.py | 8 ++++++-- .../rostest/test_03_docker_in_persistent_console.py | 5 ++++- tests/integration/rostest/test_04_ros_install.py | 2 +- .../test_05_custom_docker_in_persistent_console.py | 11 +++++++---- tests/integration/rostest/test_sample.py | 6 ------ tests/integration/rostest/util.py | 6 ++++++ 7 files changed, 28 insertions(+), 15 deletions(-) delete mode 100644 tests/integration/rostest/test_sample.py diff --git a/tests/integration/rostest/test_00_system.py b/tests/integration/rostest/test_00_system.py index 690370a6..23cefb1f 100644 --- a/tests/integration/rostest/test_00_system.py +++ b/tests/integration/rostest/test_00_system.py @@ -18,12 +18,15 @@ def test_system_boot(qemu): u.flush_out(qemu.stdout, 'RancherOS {v} started'.format(v=version)) +busybox = {'amd64': 'busybox', 'arm': 'armhf/busybox', 'arm64': 'aarch64/busybox'} + + @pytest.mark.timeout(60) def test_run_system_container(qemu): u.wait_for_ssh(qemu) ssh = subprocess.Popen( - './scripts/ssh --qemu sudo system-docker run --rm busybox /bin/true', + './scripts/ssh --qemu sudo system-docker run --rm ' + busybox[u.arch] + ' /bin/true', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) for ln in u.iter_lines(ssh.stdout): diff --git a/tests/integration/rostest/test_01_cloud_config.py b/tests/integration/rostest/test_01_cloud_config.py index 60ce2aa5..18fc5ed3 100644 --- a/tests/integration/rostest/test_01_cloud_config.py +++ b/tests/integration/rostest/test_01_cloud_config.py @@ -10,10 +10,14 @@ ssh_command = ['./scripts/ssh', '--qemu', '--key', './tests/integration/assets/t cloud_config_path = './tests/integration/assets/test_01/cloud-config.yml' +net_args = {'amd64': ['-net', 'nic,vlan=1,model=virtio', '-net', 'user,vlan=1,net=10.10.2.0/24'], + 'arm64': ['-netdev', 'user,id=net1,net=10.10.2.0/24', '-device', 'virtio-net-device,netdev=net1']} +net_args['arm'] = net_args['arm64'] + + @pytest.fixture(scope="module") def qemu(request): - q = u.run_qemu(request, ['--cloud-config', cloud_config_path, - '-net', 'nic,vlan=1,model=virtio', '-net', 'user,vlan=1,net=10.10.2.0/24']) + q = u.run_qemu(request, ['--cloud-config', cloud_config_path] + net_args[u.arch]) u.flush_out(q.stdout) return q diff --git a/tests/integration/rostest/test_03_docker_in_persistent_console.py b/tests/integration/rostest/test_03_docker_in_persistent_console.py index 47b03fb1..49326a0c 100644 --- a/tests/integration/rostest/test_03_docker_in_persistent_console.py +++ b/tests/integration/rostest/test_03_docker_in_persistent_console.py @@ -14,10 +14,13 @@ def qemu(request): return q +nginx = {'amd64': 'nginx', 'arm': 'armhfbuild/nginx', 'arm64': 'armhfbuild/nginx'} + + @pytest.mark.timeout(40) def test_reboot_with_container_running(qemu): u.wait_for_ssh(qemu, ssh_command) - subprocess.check_call(ssh_command + ['docker', 'run', '-d', '--restart=always', 'nginx'], + subprocess.check_call(ssh_command + ['docker', 'run', '-d', '--restart=always', nginx[u.arch]], stderr=subprocess.STDOUT, universal_newlines=True) subprocess.call(ssh_command + ['sudo', 'reboot'], diff --git a/tests/integration/rostest/test_04_ros_install.py b/tests/integration/rostest/test_04_ros_install.py index 69e71da0..65b2a938 100644 --- a/tests/integration/rostest/test_04_ros_install.py +++ b/tests/integration/rostest/test_04_ros_install.py @@ -20,7 +20,7 @@ def test_ros_install_on_formatted_disk(qemu): stderr=subprocess.STDOUT, universal_newlines=True) subprocess.check_call(ssh_command + ['sudo', 'ros', 'install', '-f', '--no-reboot', '-d', '/dev/vda', - '-i', 'rancher/os:v0.4.1'], + '-i', 'rancher/os:v0.4.4-dev' + u.suffix], stderr=subprocess.STDOUT, universal_newlines=True) subprocess.call(ssh_command + ['sudo', 'reboot'], diff --git a/tests/integration/rostest/test_05_custom_docker_in_persistent_console.py b/tests/integration/rostest/test_05_custom_docker_in_persistent_console.py index 42f90ea3..dd906433 100644 --- a/tests/integration/rostest/test_05_custom_docker_in_persistent_console.py +++ b/tests/integration/rostest/test_05_custom_docker_in_persistent_console.py @@ -13,18 +13,21 @@ def qemu(request): u.flush_out(q.stdout) return q +docker_url = {'amd64': 'https://experimental.docker.com/builds/Linux/x86_64/docker-1.10.0-dev', + 'arm': 'https://github.com/rancher/docker/releases/download/v1.10.3-arm/docker-1.10.3_arm', + 'arm64': 'https://github.com/rancher/docker/releases/download/v1.10.3-arm/docker-1.10.3_arm64'} + @pytest.mark.timeout(40) def test_system_docker_survives_custom_docker_install(qemu): u.wait_for_ssh(qemu, ssh_command) - subprocess.check_call(ssh_command + ['curl', '-OL', - 'https://experimental.docker.com/builds/Linux/x86_64/docker-1.10.0-dev'], + subprocess.check_call(ssh_command + ['curl', '-Lfo', './docker', docker_url[u.arch]], stderr=subprocess.STDOUT, universal_newlines=True) - subprocess.check_call(ssh_command + ['chmod', '+x', '/home/rancher/docker-1.10.0-dev'], + subprocess.check_call(ssh_command + ['chmod', '+x', '/home/rancher/docker'], stderr=subprocess.STDOUT, universal_newlines=True) - subprocess.check_call(ssh_command + ['sudo', 'ln', '-sf', '/home/rancher/docker-1.10.0-dev', '/usr/bin/docker'], + subprocess.check_call(ssh_command + ['sudo', 'ln', '-sf', '/home/rancher/docker', '/usr/bin/docker'], stderr=subprocess.STDOUT, universal_newlines=True) subprocess.check_call(ssh_command + ['sudo', 'system-docker', 'restart', 'docker'], diff --git a/tests/integration/rostest/test_sample.py b/tests/integration/rostest/test_sample.py deleted file mode 100644 index 1030a2e3..00000000 --- a/tests/integration/rostest/test_sample.py +++ /dev/null @@ -1,6 +0,0 @@ -def func(x): - return x + 1 - - -def test_answer(): - assert func(3) == 4 diff --git a/tests/integration/rostest/util.py b/tests/integration/rostest/util.py index 0ef01eed..acfa2718 100644 --- a/tests/integration/rostest/util.py +++ b/tests/integration/rostest/util.py @@ -1,12 +1,18 @@ from __future__ import print_function import itertools as it +import os import subprocess import time import pytest ros_test = 'ros-test' +arch = os.environ['ARCH'] + +suffix = '' +if arch != 'amd64': + suffix = '_' + arch def iter_lines(s): From 60e5ff7ede1f314cfb3477118bcb08dc6467db21 Mon Sep 17 00:00:00 2001 From: Ivan Mikushin Date: Thu, 31 Mar 2016 11:07:35 -0700 Subject: [PATCH 4/7] Fix problems loading external services on boot (like ubuntu-console or kernel-headers) On arm64 external services would not load on boot because of DNS i/o timeouts or valid TLS certificates appearing expired or not issued yet (I know that sounds weird). --- compose/project.go | 2 +- os-config.tpl.yml | 4 ++-- scripts/run | 1 + util/util.go | 12 +++++++++++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/compose/project.go b/compose/project.go index 14552a38..e64a2058 100644 --- a/compose/project.go +++ b/compose/project.go @@ -177,7 +177,7 @@ func newCoreServiceProject(cfg *config.CloudConfig, network bool) (*project.Proj go func() { for event := range projectEvents { - if event.EventType == project.EventContainerStarted && event.ServiceName == "network" { + if event.EventType == project.EventContainerStarted && event.ServiceName == "ntp" { network = true } } diff --git a/os-config.tpl.yml b/os-config.tpl.yml index 54438df3..2346cac0 100644 --- a/os-config.tpl.yml +++ b/os-config.tpl.yml @@ -105,7 +105,7 @@ rancher: io.rancher.os.detach: "false" io.rancher.os.reloadconfig: "true" io.rancher.os.scope: system - io.rancher.os.after: cloud-init-pre, wait-for-network + io.rancher.os.after: cloud-init-pre, wait-for-network, ntp net: host uts: host privileged: true @@ -207,7 +207,7 @@ rancher: image: {{.OS_IMAGES_ROOT}}/os-ntp:{{.VERSION}}{{.SUFFIX}} labels: io.rancher.os.scope: system - io.rancher.os.after: cloud-init, wait-for-network + io.rancher.os.after: wait-for-network net: host uts: host privileged: true diff --git a/scripts/run b/scripts/run index 0619c38f..96caa7b9 100755 --- a/scripts/run +++ b/scripts/run @@ -196,6 +196,7 @@ if [ "$QEMU" == "1" ]; then CPU="-cpu host" fi exec qemu-system-${QEMUARCH} -serial stdio \ + -rtc base=utc,clock=host \ ${KVM_ENABLE} \ ${CPU} \ ${machine["$ARCH"]} \ diff --git a/util/util.go b/util/util.go index f903e470..0048097c 100644 --- a/util/util.go +++ b/util/util.go @@ -274,6 +274,16 @@ func DirLs(dir string) ([]interface{}, error) { return result, nil } +func retryHttp(f func() (*http.Response, error), times int) (resp *http.Response, err error) { + for i := 0; i < times; i++ { + if resp, err = f(); err == nil { + return + } + log.Warnf("Error making HTTP request: %s. Retrying", err) + } + return +} + func LoadResource(location string, network bool, urls []string) ([]byte, error) { var bytes []byte err := ErrNotFound @@ -282,7 +292,7 @@ func LoadResource(location string, network bool, urls []string) ([]byte, error) if !network { return nil, ErrNoNetwork } - resp, err := http.Get(location) + resp, err := retryHttp(func() (*http.Response, error) { return http.Get(location) }, 8) if err != nil { return nil, err } From 2dd57c5f3af54c4e0bb948b1c794132645cb5028 Mon Sep 17 00:00:00 2001 From: Ivan Mikushin Date: Thu, 31 Mar 2016 16:10:38 -0700 Subject: [PATCH 5/7] remove an unused option --xhyve from ./scripts/run --- scripts/ssh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/ssh b/scripts/ssh index 933cab05..b537f56a 100755 --- a/scripts/ssh +++ b/scripts/ssh @@ -18,10 +18,6 @@ while [ "$#" -gt 0 ]; do --notty) INTERACTIVE="" ;; - --xhyve) - XHYVE=1 - QEMU=0 - ;; --qemu) QEMU=1 ;; @@ -40,10 +36,7 @@ while [ "$#" -gt 0 ]; do shift 1 done -if [ "$XHYVE" == "1" ]; then - HOST=192.168.64.2 # consult `/var/db/dhcpd_leases` or delete it - exec ssh -F ./assets/scripts_ssh_config -i ${KEY:-./assets/rancher.key} rancher@${HOST} "$@" -elif [ "$QEMU" == "1" ]; then +if [ "$QEMU" == "1" ]; then exec ssh -p 2222 -F ./assets/scripts_ssh_config -i ${KEY:-./assets/rancher.key} rancher@localhost "$@" elif [ "$QIND" == "1" ]; then NAME=${NAME:-ros-qind} From e2f3ac78a72f21aac48e409a6ce99d13643b614e Mon Sep 17 00:00:00 2001 From: Ivan Mikushin Date: Thu, 31 Mar 2016 18:56:18 -0700 Subject: [PATCH 6/7] cleanup tests using SSH --- scripts/run | 5 ++-- .../rostest/test_01_cloud_config.py | 25 ++++++++----------- tests/integration/rostest/test_06_subdir.py | 10 ++++---- tests/integration/rostest/test_07_oem.py | 14 +++++++---- .../rostest/test_08_ssh_key_merge.py | 2 +- tests/integration/rostest/test_09_network.py | 4 +-- 6 files changed, 31 insertions(+), 29 deletions(-) diff --git a/scripts/run b/scripts/run index 96caa7b9..6632c6e1 100755 --- a/scripts/run +++ b/scripts/run @@ -1,6 +1,5 @@ #!/bin/bash set -e -set -x cd $(dirname $0)/.. @@ -195,6 +194,7 @@ if [ "$QEMU" == "1" ]; then KVM_ENABLE="-enable-kvm" CPU="-cpu host" fi + set -x exec qemu-system-${QEMUARCH} -serial stdio \ -rtc base=utc,clock=host \ ${KVM_ENABLE} \ @@ -202,7 +202,7 @@ if [ "$QEMU" == "1" ]; then ${machine["$ARCH"]} \ -kernel ${KERNEL} \ -initrd ${INITRD} \ - -m 1024 \ + -m 2048 \ ${network["$ARCH"]} \ $(eval "${cd["$ARCH"]} ${CCIMG}") \ $(eval "${hd["$ARCH"]} ${HD}") \ @@ -240,6 +240,7 @@ elif [ "$QIND" == "1" ]; then SSH_PUB_ENABLE="--ssh-pub /stuff/rancher.key.pub" fi + set -x docker create --name=${NAME} -it ${KVM_ENABLE} \ rancher/qind \ --hostname "rancher-dev" \ diff --git a/tests/integration/rostest/test_01_cloud_config.py b/tests/integration/rostest/test_01_cloud_config.py index 18fc5ed3..35c1c7e8 100644 --- a/tests/integration/rostest/test_01_cloud_config.py +++ b/tests/integration/rostest/test_01_cloud_config.py @@ -75,15 +75,12 @@ def test_services_include(qemu, cloud_config): @pytest.mark.timeout(40) def test_docker_tls_args(qemu, cloud_config): - u.wait_for_ssh(qemu, ssh_command) - - subprocess.check_call( - ssh_command + ['sudo', 'ros', 'tls', 'gen'], - stderr=subprocess.STDOUT, universal_newlines=True) - - subprocess.check_call( - ssh_command + ['docker', '--tlsverify', 'version'], - stderr=subprocess.STDOUT, universal_newlines=True) + SSH(qemu, ssh_command).check_call(''' +set -e -x +sudo ros tls gen +sleep 3 +docker --tlsverify version + '''.strip()) @pytest.mark.timeout(40) @@ -99,9 +96,9 @@ def test_rancher_network(qemu, cloud_config): def test_docker_not_pid_one(qemu): - SSH(qemu, ssh_command=ssh_command).check_call('bash', '-c', ''' - set -e -x - for i in $(pidof docker); do - [ $i != 1 ] - done + SSH(qemu, ssh_command).check_call(''' +set -e -x +for i in $(pidof docker); do + [ $i != 1 ] +done '''.strip()) diff --git a/tests/integration/rostest/test_06_subdir.py b/tests/integration/rostest/test_06_subdir.py index 078f7640..4d8aad40 100644 --- a/tests/integration/rostest/test_06_subdir.py +++ b/tests/integration/rostest/test_06_subdir.py @@ -11,9 +11,9 @@ def qemu(request): def test_system_docker_survives_custom_docker_install(qemu): - SSH(qemu).check_call('bash', '-c', ''' - set -x -e - mkdir x - sudo mount $(sudo ros dev LABEL=RANCHER_STATE) x - [ -d x/ros_subdir/home/rancher ] + SSH(qemu).check_call(''' +set -x -e +mkdir x +sudo mount $(sudo ros dev LABEL=RANCHER_STATE) x +[ -d x/ros_subdir/home/rancher ] '''.strip()) diff --git a/tests/integration/rostest/test_07_oem.py b/tests/integration/rostest/test_07_oem.py index 33723333..aea8767e 100644 --- a/tests/integration/rostest/test_07_oem.py +++ b/tests/integration/rostest/test_07_oem.py @@ -12,12 +12,13 @@ def qemu(request): def test_oem(qemu): - SSH(qemu).check_call('sudo', 'bash', '-c', ''' + try: + SSH(qemu).check_call(''' set -x set -e sudo mkfs.ext4 -L RANCHER_OEM /dev/vda sudo mount /dev/vda /mnt -cat > /tmp/oem-config.yml << "EOF" +cat > /tmp/oem-config.yml << EOF #cloud-config rancher: upgrade: @@ -25,11 +26,14 @@ rancher: EOF sudo cp /tmp/oem-config.yml /mnt sudo umount /mnt -sudo reboot >/dev/null 2>&1 &'''.strip()) +sudo reboot + '''.strip()) + except: + pass - time.sleep(1) + time.sleep(3) - SSH(qemu).check_call('bash', '-c', ''' + SSH(qemu).check_call(''' set -x if [ ! -e /usr/share/ros/oem/oem-config.yml ]; then echo Failed to find /usr/share/ros/oem/oem-config.yml diff --git a/tests/integration/rostest/test_08_ssh_key_merge.py b/tests/integration/rostest/test_08_ssh_key_merge.py index e3d6f9c9..0e49d365 100644 --- a/tests/integration/rostest/test_08_ssh_key_merge.py +++ b/tests/integration/rostest/test_08_ssh_key_merge.py @@ -11,7 +11,7 @@ def qemu(request): def test_ssh_key_merging(qemu): - SSH(qemu).check_call('bash', '-c', '''cat > test-merge << "SCRIPT" + SSH(qemu).check_call('''cat > test-merge << "SCRIPT" set -x -e rm /var/lib/rancher/conf/cloud-config.yml diff --git a/tests/integration/rostest/test_09_network.py b/tests/integration/rostest/test_09_network.py index 8ab98857..a4f17948 100644 --- a/tests/integration/rostest/test_09_network.py +++ b/tests/integration/rostest/test_09_network.py @@ -21,7 +21,7 @@ def qemu(request): def test_network_conf(qemu): - SSH(qemu).check_call('bash', '-c', '''cat > test-merge << "SCRIPT" + SSH(qemu).check_call('''cat > test-merge << "SCRIPT" set -x -e ip link show dev br0 @@ -37,7 +37,7 @@ sudo bash test-merge def test_network_dns_ros_set(qemu): - SSH(qemu).check_call('bash', '-c', ''' + SSH(qemu).check_call(''' set -x -e sudo ros config set rancher.network.dns.search '[a,b]' From 0f6cf07786d6b5675be252d55164bf59be1102a9 Mon Sep 17 00:00:00 2001 From: Ivan Mikushin Date: Fri, 1 Apr 2016 11:00:42 -0700 Subject: [PATCH 7/7] make tests run on arm64 --- .../assets/test_01/cloud-config.yml | 2 +- .../rostest/test_01_cloud_config.py | 45 ++++++++----------- .../test_03_docker_in_persistent_console.py | 23 ++++++---- tests/integration/rostest/test_09_network.py | 17 +++---- tests/integration/rostest/util.py | 14 ++++-- 5 files changed, 54 insertions(+), 47 deletions(-) diff --git a/tests/integration/assets/test_01/cloud-config.yml b/tests/integration/assets/test_01/cloud-config.yml index 853223d4..cf05068d 100644 --- a/tests/integration/assets/test_01/cloud-config.yml +++ b/tests/integration/assets/test_01/cloud-config.yml @@ -9,7 +9,7 @@ rancher: interfaces: eth*: dhcp: true - eth1: + "mac=52:54:00:12:34:59": address: 10.10.2.17/24 gateway: 10.10.2.2 mtu: 1500 diff --git a/tests/integration/rostest/test_01_cloud_config.py b/tests/integration/rostest/test_01_cloud_config.py index 35c1c7e8..87ed91d5 100644 --- a/tests/integration/rostest/test_01_cloud_config.py +++ b/tests/integration/rostest/test_01_cloud_config.py @@ -1,5 +1,4 @@ import string -import subprocess import pytest import rostest.util as u @@ -10,8 +9,10 @@ ssh_command = ['./scripts/ssh', '--qemu', '--key', './tests/integration/assets/t cloud_config_path = './tests/integration/assets/test_01/cloud-config.yml' -net_args = {'amd64': ['-net', 'nic,vlan=1,model=virtio', '-net', 'user,vlan=1,net=10.10.2.0/24'], - 'arm64': ['-netdev', 'user,id=net1,net=10.10.2.0/24', '-device', 'virtio-net-device,netdev=net1']} +net_args = {'amd64': ['-net', 'nic,vlan=1,model=virtio,macaddr=52:54:00:12:34:59', + '-net', 'user,vlan=1,net=10.10.2.0/24'], + 'arm64': ['-netdev', 'user,id=net1,net=10.10.2.0/24', + '-device', 'virtio-net-device,netdev=net1,mac=52:54:00:12:34:59']} net_args['arm'] = net_args['arm64'] @@ -35,22 +36,18 @@ def test_ssh_authorized_keys(qemu): @pytest.mark.timeout(40) def test_rancher_environment(qemu, cloud_config): - u.wait_for_ssh(qemu, ssh_command) - - v = subprocess.check_output( - ssh_command + ['sudo', 'ros', 'env', 'printenv', 'FLANNEL_NETWORK'], - stderr=subprocess.STDOUT, universal_newlines=True) + v = SSH(qemu, ssh_command).check_output(''' +sudo ros env printenv FLANNEL_NETWORK + '''.strip()) assert v.strip() == cloud_config['rancher']['environment']['FLANNEL_NETWORK'] @pytest.mark.timeout(40) def test_docker_args(qemu, cloud_config): - u.wait_for_ssh(qemu, ssh_command) - - v = subprocess.check_output( - ssh_command + ['sh', '-c', 'ps -ef | grep docker'], - stderr=subprocess.STDOUT, universal_newlines=True) + v = SSH(qemu, ssh_command).check_output(''' +ps -ef | grep docker + '''.strip()) expected = string.join(cloud_config['rancher']['docker']['args']) @@ -59,11 +56,9 @@ def test_docker_args(qemu, cloud_config): @pytest.mark.timeout(40) def test_dhcpcd(qemu, cloud_config): - u.wait_for_ssh(qemu, ssh_command) - - v = subprocess.check_output( - ssh_command + ['sh', '-c', 'ps -ef | grep dhcpcd'], - stderr=subprocess.STDOUT, universal_newlines=True) + v = SSH(qemu, ssh_command).check_output(''' +ps -ef | grep dhcpcd + '''.strip()) assert v.find('dhcpcd -M') != -1 @@ -78,21 +73,19 @@ def test_docker_tls_args(qemu, cloud_config): SSH(qemu, ssh_command).check_call(''' set -e -x sudo ros tls gen -sleep 3 +sleep 5 docker --tlsverify version '''.strip()) @pytest.mark.timeout(40) def test_rancher_network(qemu, cloud_config): - u.wait_for_ssh(qemu, ssh_command) + v = SSH(qemu, ssh_command).check_output(''' +ip route get to 10.10.2.120 + '''.strip()) - v = subprocess.check_output( - ssh_command + ['ip', 'route', 'get', 'to', '10.10.2.120'], - stderr=subprocess.STDOUT, universal_newlines=True) - - assert v.split(' ')[2] == 'eth1' - assert v.split(' ')[5] + '/24' == cloud_config['rancher']['network']['interfaces']['eth1']['address'] + assert v.split(' ')[5] + '/24' == \ + cloud_config['rancher']['network']['interfaces']['mac=52:54:00:12:34:59']['address'] def test_docker_not_pid_one(qemu): diff --git a/tests/integration/rostest/test_03_docker_in_persistent_console.py b/tests/integration/rostest/test_03_docker_in_persistent_console.py index 49326a0c..b492cc2b 100644 --- a/tests/integration/rostest/test_03_docker_in_persistent_console.py +++ b/tests/integration/rostest/test_03_docker_in_persistent_console.py @@ -1,7 +1,8 @@ -import subprocess +import time import pytest import rostest.util as u +from rostest.util import SSH ssh_command = ['./scripts/ssh', '--qemu', '--key', './tests/integration/assets/test.key'] cloud_config_path = './tests/integration/assets/test_03/cloud-config.yml' @@ -19,15 +20,19 @@ nginx = {'amd64': 'nginx', 'arm': 'armhfbuild/nginx', 'arm64': 'armhfbuild/nginx @pytest.mark.timeout(40) def test_reboot_with_container_running(qemu): - u.wait_for_ssh(qemu, ssh_command) - subprocess.check_call(ssh_command + ['docker', 'run', '-d', '--restart=always', nginx[u.arch]], - stderr=subprocess.STDOUT, universal_newlines=True) + try: + SSH(qemu, ssh_command).check_call(''' +set -ex +docker run -d --restart=always %(image)s +sudo reboot + '''.strip() % {'image': nginx[u.arch]}) + except: + pass - subprocess.call(ssh_command + ['sudo', 'reboot'], - stderr=subprocess.STDOUT, universal_newlines=True) + time.sleep(3) - u.wait_for_ssh(qemu, ssh_command) - v = subprocess.check_output(ssh_command + ['docker', 'ps', '-f', 'status=running'], - stderr=subprocess.STDOUT, universal_newlines=True) + v = SSH(qemu, ssh_command).check_output(''' +docker ps -f status=running + '''.strip()) assert v.find('nginx') != -1 diff --git a/tests/integration/rostest/test_09_network.py b/tests/integration/rostest/test_09_network.py index a4f17948..260ec253 100644 --- a/tests/integration/rostest/test_09_network.py +++ b/tests/integration/rostest/test_09_network.py @@ -2,20 +2,21 @@ import pytest import rostest.util as u from rostest.util import SSH +pytestmark = pytest.mark.skipif(u.arch != 'amd64', reason='amd64 network setup impossible to replicate for arm64') cloud_config_path = './tests/integration/assets/test_09/cloud-config.yml' +net_args_arch = {'amd64': ['-net', 'nic,vlan=0,model=virtio'], + 'arm64': ['-device', 'virtio-net-device']} +net_args_arch['arm'] = net_args_arch['arm64'] +net_args = net_args_arch[u.arch] + @pytest.fixture(scope="module") def qemu(request): - q = u.run_qemu(request, run_args=['--cloud-config', cloud_config_path, - '-net', 'nic,vlan=0,model=virtio', - '-net', 'nic,vlan=0,model=virtio', - '-net', 'nic,vlan=0,model=virtio', - '-net', 'nic,vlan=0,model=virtio', - '-net', 'nic,vlan=0,model=virtio', - '-net', 'nic,vlan=0,model=virtio', - '-net', 'nic,vlan=0,model=virtio']) + q = u.run_qemu(request, + run_args=['--cloud-config', cloud_config_path] + + net_args + net_args + net_args + net_args + net_args + net_args + net_args) u.flush_out(q.stdout) return q diff --git a/tests/integration/rostest/util.py b/tests/integration/rostest/util.py index acfa2718..e9be1a52 100644 --- a/tests/integration/rostest/util.py +++ b/tests/integration/rostest/util.py @@ -90,8 +90,8 @@ def wait_for_ssh(qemu, ssh_command=['./scripts/ssh', '--qemu'], command=['docker i += 1 print('\nWaiting for ssh and docker... ' + str(i)) time.sleep(1) - if i > 60: - raise 'Failed to connect to SSH' + if i > 150: + raise AssertionError('Failed to connect to SSH') assert qemu.returncode is None @@ -101,11 +101,19 @@ class SSH: self._ssh_command = ssh_command self._waited = False - def check_call(self, *args, **kw): + def wait(self): if not self._waited: wait_for_ssh(self._qemu, ssh_command=self._ssh_command) self._waited = True + def check_call(self, *args, **kw): + self.wait() kw['stderr'] = subprocess.STDOUT kw['universal_newlines'] = True return subprocess.check_call(self._ssh_command + list(args), **kw) + + def check_output(self, *args, **kw): + self.wait() + kw['stderr'] = subprocess.STDOUT + kw['universal_newlines'] = True + return subprocess.check_output(self._ssh_command + list(args), **kw)