Merge pull request #689 from imikushin/die-qemu-die

Integration tests: make sure qemu exits in fixture finalizer

tests/integration/rostest/conftest.py

@@ -1,6 +1,7 @@
+import subprocess
+
 import os
 import pytest
-import subprocess
 
 
 @pytest.fixture(scope="session", autouse=True)

tests/integration/rostest/test_00_system.py

@@ -1,7 +1,8 @@
 from __future__ import print_function
-import itertools as it
+
-import pytest
 import subprocess
+
+import pytest
 import rostest.util as u
 
 
@@ -14,21 +15,12 @@ def qemu(request):
 def test_system_boot(qemu):
     version = u.rancheros_version('./build.conf')
     print('parsed version: ' + version)
-
+    u.flush_out(qemu.stdout, 'RancherOS {v} started'.format(v=version))
-    def has_ros_started_substr(s):
-        return str.find(s, 'RancherOS {v} started'.format(v=version)) > -1
-
-    for _ in it.ifilter(has_ros_started_substr,
-                        it.imap(u.with_effect(print), u.iter_lines(qemu.stdout))):
-        assert True
-        return
-    assert False
 
 
 @pytest.mark.timeout(60)
 def test_run_system_container(qemu):
-    assert qemu is not None
+    u.wait_for_ssh(qemu)
-    u.wait_for_ssh()
 
     ssh = subprocess.Popen(
         './scripts/ssh --qemu sudo system-docker run --rm busybox /bin/true',

tests/integration/rostest/test_01_cloud_config.py

@@ -1,9 +1,9 @@
-import pytest
-import rostest.util as u
 import string
 import subprocess
-import yaml
 
+import pytest
+import rostest.util as u
+import yaml
 
 ssh_command = ['./scripts/ssh', '--qemu', '--key', './tests/integration/assets/test.key']
 cloud_config_path = './tests/integration/assets/test_01/cloud-config.yml'
@@ -11,8 +11,10 @@ cloud_config_path = './tests/integration/assets/test_01/cloud-config.yml'
 
 @pytest.fixture(scope="module")
 def qemu(request):
-    return u.run_qemu(request, ['--cloud-config', cloud_config_path,
+    q = u.run_qemu(request, ['--cloud-config', cloud_config_path,
-                                '-net', 'nic,vlan=1,model=virtio', '-net', 'user,vlan=1,net=10.10.2.0/24'])
+                             '-net', 'nic,vlan=1,model=virtio', '-net', 'user,vlan=1,net=10.10.2.0/24'])
+    u.flush_out(q.stdout)
+    return q
 
 
 @pytest.fixture(scope="module")
@@ -22,15 +24,13 @@ def cloud_config():
 
 @pytest.mark.timeout(40)
 def test_ssh_authorized_keys(qemu):
-    assert qemu is not None
+    u.wait_for_ssh(qemu, ssh_command)
-    u.wait_for_ssh(ssh_command)
     assert True
 
 
 @pytest.mark.timeout(40)
 def test_rancher_environment(qemu, cloud_config):
-    assert qemu is not None
+    u.wait_for_ssh(qemu, ssh_command)
-    u.wait_for_ssh(ssh_command)
 
     v = subprocess.check_output(
         ssh_command + ['sudo', 'ros', 'env', 'printenv', 'FLANNEL_NETWORK'],
@@ -41,8 +41,7 @@ def test_rancher_environment(qemu, cloud_config):
 
 @pytest.mark.timeout(40)
 def test_docker_args(qemu, cloud_config):
-    assert qemu is not None
+    u.wait_for_ssh(qemu, ssh_command)
-    u.wait_for_ssh(ssh_command)
 
     v = subprocess.check_output(
         ssh_command + ['sh', '-c', 'ps -ef | grep docker'],
@@ -55,8 +54,7 @@ def test_docker_args(qemu, cloud_config):
 
 @pytest.mark.timeout(40)
 def test_dhcpcd(qemu, cloud_config):
-    assert qemu is not None
+    u.wait_for_ssh(qemu, ssh_command)
-    u.wait_for_ssh(ssh_command)
 
     v = subprocess.check_output(
         ssh_command + ['sh', '-c', 'ps -ef | grep dhcpcd'],
@@ -67,8 +65,7 @@ def test_dhcpcd(qemu, cloud_config):
 
 @pytest.mark.timeout(40)
 def test_docker_tls_args(qemu, cloud_config):
-    assert qemu is not None
+    u.wait_for_ssh(qemu, ssh_command)
-    u.wait_for_ssh(ssh_command)
 
     subprocess.check_call(
         ssh_command + ['sudo', 'ros', 'tls', 'gen'],
@@ -81,8 +78,7 @@ def test_docker_tls_args(qemu, cloud_config):
 
 @pytest.mark.timeout(40)
 def test_rancher_network(qemu, cloud_config):
-    assert qemu is not None
+    u.wait_for_ssh(qemu, ssh_command)
-    u.wait_for_ssh(ssh_command)
 
     v = subprocess.check_output(
         ssh_command + ['ip', 'route', 'get', 'to', '10.10.2.120'],

tests/integration/rostest/test_02_custom_kernel.py

@@ -1,7 +1,8 @@
 from __future__ import print_function
-import itertools as it
+
-import pytest
 import subprocess
+
+import pytest
 import rostest.util as u
 
 
@@ -15,6 +16,7 @@ def build_and_run(request):
     def fin():
         print('\nTerminating docker-run test-custom-kernel')
         p.terminate()
+        p.wait()
 
     request.addfinalizer(fin)
     return p
@@ -25,11 +27,4 @@ def test_system_boot(build_and_run):
     version = u.rancheros_version('./tests/integration/assets/test_02/build.conf')
     print('parsed version: ' + version)
 
-    def has_ros_started_substr(s):
+    u.flush_out(build_and_run.stdout, 'RancherOS {v} started'.format(v=version))
-        return str.find(s, 'RancherOS {v} started'.format(v=version)) > -1
-
-    for _ in it.ifilter(has_ros_started_substr,
-                        it.imap(u.with_effect(print), u.iter_lines(build_and_run.stdout))):
-        assert True
-        return
-    assert False

tests/integration/rostest/test_03_docker_in_persistent_console.py

@@ -1,7 +1,7 @@
-import pytest
-import rostest.util as u
 import subprocess
 
+import pytest
+import rostest.util as u
 
 ssh_command = ['./scripts/ssh', '--qemu', '--key', './tests/integration/assets/test.key']
 cloud_config_path = './tests/integration/assets/test_03/cloud-config.yml'
@@ -9,20 +9,21 @@ cloud_config_path = './tests/integration/assets/test_03/cloud-config.yml'
 
 @pytest.fixture(scope="module")
 def qemu(request):
-    return u.run_qemu(request, ['--cloud-config', cloud_config_path])
+    q = u.run_qemu(request, ['--cloud-config', cloud_config_path])
+    u.flush_out(q.stdout)
+    return q
 
 
 @pytest.mark.timeout(40)
 def test_reboot_with_container_running(qemu):
-    assert qemu is not None
+    u.wait_for_ssh(qemu, ssh_command)
-    u.wait_for_ssh(ssh_command)
     subprocess.check_call(ssh_command + ['docker', 'run', '-d', '--restart=always', 'nginx'],
                           stderr=subprocess.STDOUT, universal_newlines=True)
 
     subprocess.call(ssh_command + ['sudo', 'reboot'],
                     stderr=subprocess.STDOUT, universal_newlines=True)
 
-    u.wait_for_ssh(ssh_command)
+    u.wait_for_ssh(qemu, ssh_command)
     v = subprocess.check_output(ssh_command + ['docker', 'ps', '-f', 'status=running'],
                                 stderr=subprocess.STDOUT, universal_newlines=True)
 

tests/integration/rostest/test_04_ros_install.py

@@ -1,20 +1,21 @@
-import pytest
-import rostest.util as u
 import subprocess
 
+import pytest
+import rostest.util as u
 
 ssh_command = ['./scripts/ssh', '--qemu']
 
 
 @pytest.fixture(scope="module")
 def qemu(request):
-    return u.run_qemu(request, ['--no-format'])
+    q = u.run_qemu(request, ['--no-format'])
+    u.flush_out(q.stdout)
+    return q
 
 
 @pytest.mark.timeout(40)
 def test_ros_install_on_formatted_disk(qemu):
-    assert qemu is not None
+    u.wait_for_ssh(qemu, ssh_command)
-    u.wait_for_ssh(ssh_command)
     subprocess.check_call(ssh_command + ['sudo', 'mkfs.ext4', '/dev/vda'],
                           stderr=subprocess.STDOUT, universal_newlines=True)
 
@@ -25,4 +26,4 @@ def test_ros_install_on_formatted_disk(qemu):
     subprocess.call(ssh_command + ['sudo', 'reboot'],
                     stderr=subprocess.STDOUT, universal_newlines=True)
 
-    u.wait_for_ssh(ssh_command)
+    u.wait_for_ssh(qemu, ssh_command)

tests/integration/rostest/test_05_custom_docker_in_persistent_console.py

@@ -1,7 +1,7 @@
-import pytest
-import rostest.util as u
 import subprocess
 
+import pytest
+import rostest.util as u
 
 ssh_command = ['./scripts/ssh', '--qemu', '--key', './tests/integration/assets/test.key']
 cloud_config_path = './tests/integration/assets/test_05/cloud-config.yml'
@@ -9,13 +9,14 @@ cloud_config_path = './tests/integration/assets/test_05/cloud-config.yml'
 
 @pytest.fixture(scope="module")
 def qemu(request):
-    return u.run_qemu(request, ['--cloud-config', cloud_config_path])
+    q = u.run_qemu(request, ['--cloud-config', cloud_config_path])
+    u.flush_out(q.stdout)
+    return q
 
 
 @pytest.mark.timeout(40)
 def test_system_docker_survives_custom_docker_install(qemu):
-    assert qemu is not None
+    u.wait_for_ssh(qemu, ssh_command)
-    u.wait_for_ssh(ssh_command)
     subprocess.check_call(ssh_command + ['curl', '-OL',
                                          'https://experimental.docker.com/builds/Linux/x86_64/docker-1.10.0-dev'],
                           stderr=subprocess.STDOUT, universal_newlines=True)
@@ -32,4 +33,4 @@ def test_system_docker_survives_custom_docker_install(qemu):
     subprocess.check_call(ssh_command + ['sudo', 'system-docker', 'version'],
                           stderr=subprocess.STDOUT, universal_newlines=True)
 
-    u.wait_for_ssh(ssh_command)
+    u.wait_for_ssh(qemu, ssh_command)

tests/integration/rostest/util.py

@@ -1,8 +1,10 @@
+from __future__ import print_function
+
 import itertools as it
-import pytest
 import subprocess
 import time
 
+import pytest
 
 ros_test = 'ros-test'
 
@@ -26,6 +28,7 @@ def parse_value(var):
         if k == var:
             return v
         return ''
+
     return get_value
 
 
@@ -33,6 +36,7 @@ def with_effect(p):
     def effect(s):
         p(s)
         return s
+
     return effect
 
 
@@ -50,20 +54,34 @@ def run_qemu(request, run_args=[]):
     print('\nStarting QEMU')
     p = subprocess.Popen(['./scripts/run', '--qemu', '--no-rebuild', '--no-rm-usr', '--fresh'] + run_args,
                          stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+    assert p.returncode is None
 
     def fin():
         print('\nTerminating QEMU')
         p.terminate()
+        p.wait()
 
     request.addfinalizer(fin)
     return p
 
 
+def has_substr(token):
+    return lambda s: str.find(s, token) > -1
+
+
+def flush_out(stdout, substr='RancherOS '):
+    for _ in it.ifilter(has_substr(substr),
+                        it.imap(with_effect(print), iter_lines(stdout))):
+        return
+
+
 @pytest.mark.timeout(10)
-def wait_for_ssh(ssh_command=['./scripts/ssh', '--qemu']):
+def wait_for_ssh(qemu, ssh_command=['./scripts/ssh', '--qemu']):
     i = 0
+    assert qemu.returncode is None
     print('\nWaiting for ssh and docker... ' + str(i))
     while subprocess.call(ssh_command + ['docker version >/dev/null 2>&1']) != 0:
         i += 1
         print('\nWaiting for ssh and docker... ' + str(i))
         time.sleep(1)
+        assert qemu.returncode is None