diff --git a/.travis.yml b/.travis.yml
index d4953371..fe37c22f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,14 @@ install:
     - sudo apt-get --force-yes install g++-4.8
     - sudo apt-get install rpm linux-headers-$(uname -r)
     - git clone https://github.com/draios/sysdig.git ../sysdig
+    - sudo apt-get install -y python-pip libvirt-dev jq
+    - cd ..
+    - curl -Lo avocado-36.0-tar.gz https://github.com/avocado-framework/avocado/archive/36.0lts.tar.gz
+    - tar -zxvf avocado-36.0-tar.gz
+    - cd avocado-36.0lts
+    - sudo pip install -r requirements-travis.txt
+    - sudo python setup.py install
+    - cd ../falco
 before_script:
     - export KERNELDIR=/lib/modules/$(ls /lib/modules | sort | head -1)/build
 script:
@@ -28,7 +36,7 @@ script:
     - make VERBOSE=1
     - make package
     - cd ..
-    - sudo test/falco_trace_regression.sh build/userspace/falco/falco
+    - sudo test/run_regression_tests.sh
 notifications:
   webhooks:
     urls:
diff --git a/test/falco_test.py b/test/falco_test.py
new file mode 100644
index 00000000..72875c1c
--- /dev/null
+++ b/test/falco_test.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+import os
+import re
+
+from avocado import Test
+from avocado.utils import process
+from avocado.utils import linux_modules
+
+class FalcoTest(Test):
+
+    def setUp(self):
+        """
+        Load the sysdig kernel module if not already loaded.
+        """
+        self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build'))
+
+        self.should_detect = self.params.get('detect', '*')
+        self.trace_file = self.params.get('trace_file', '*')
+
+        # Doing this in 2 steps instead of simply using
+        # module_is_loaded to avoid logging lsmod output to the log.
+        lsmod_output = process.system_output("lsmod", verbose=False)
+
+        if linux_modules.parse_lsmod_for_module(lsmod_output, 'sysdig_probe') == {}:
+            self.log.debug("Loading sysdig kernel module")
+            process.run('sudo insmod {}/driver/sysdig-probe.ko'.format(self.falcodir))
+
+        self.str_variant = self.trace_file
+
+    def test(self):
+        self.log.info("Trace file %s", self.trace_file)
+
+        # Run the provided trace file though falco
+        cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {}'.format(
+            self.falcodir, self.falcodir, self.falcodir, self.trace_file)
+
+        self.falco_proc = process.SubProcess(cmd)
+
+        res = self.falco_proc.run(timeout=60, sig=9)
+
+        if res.exit_status != 0:
+            self.error("Falco command \"{}\" exited with non-zero return value {}".format(
+                cmd, res.exit_status))
+
+        # Get the number of events detected.
+        res = re.search('Events detected: (\d+)', res.stdout)
+        if res is None:
+            self.fail("Could not find a line 'Events detected: <count>' in falco output")
+
+        events_detected = int(res.group(1))
+
+        if not self.should_detect and events_detected > 0:
+            self.fail("Detected {} events when should have detected none".format(events_detected))
+
+        if self.should_detect and events_detected == 0:
+            self.fail("Detected {} events when should have detected > 0".format(events_detected))
+
+        pass
+
+
+if __name__ == "__main__":
+    main()
diff --git a/test/falco_trace_regression.sh b/test/falco_trace_regression.sh
deleted file mode 100755
index a4b3498c..00000000
--- a/test/falco_trace_regression.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-set -eu
-
-SCRIPT=$(readlink -f $0)
-BASEDIR=$(dirname $SCRIPT)
-
-FALCO=$1
-BUILDDIR=$(dirname $FALCO)
-
-# Load the built kernel module by hand
-insmod $BUILDDIR/../../driver/sysdig-probe.ko
-
-# For now, simply ensure that falco can run without errors.
-FALCO_CMDLINE="$FALCO -c $BASEDIR/../falco.yaml -r $BASEDIR/../rules/falco_rules.yaml"
-echo "Running falco: $FALCO_CMDLINE"
-$FALCO_CMDLINE > $BASEDIR/falco.log 2>&1 &
-FALCO_PID=$!
-echo "Falco started, pid $FALCO_PID"
-sleep 10
-if kill -0 $FALCO_PID > /dev/null 2>&1; then
-    echo "Falco ran successfully"
-    kill $FALCO_PID
-    ret=0
-else
-    echo "Falco did not start successfully. Full program output:"
-    cat $BASEDIR/falco.log
-    ret=1
-fi
-
-exit $ret
diff --git a/test/run_regression_tests.sh b/test/run_regression_tests.sh
new file mode 100755
index 00000000..9f6b2a28
--- /dev/null
+++ b/test/run_regression_tests.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+SCRIPT=$(readlink -f $0)
+SCRIPTDIR=$(dirname $SCRIPT)
+MULT_FILE=$SCRIPTDIR/falco_tests.yaml
+
+function download_trace_files() {
+    for TRACE in traces-positive traces-negative ; do
+	curl -so $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip &&
+	unzip -d $SCRIPTDIR $SCRIPTDIR/$TRACE.zip &&
+	rm -rf $SCRIPTDIR/$TRACE.zip
+    done
+}
+
+function prepare_multiplex_file() {
+    echo "trace_files: !mux" > $MULT_FILE
+
+    for trace in $SCRIPTDIR/traces-positive/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+	cat << EOF >> $MULT_FILE
+  $NAME:
+    detect: True
+    trace_file: $trace
+EOF
+    done
+
+    for trace in $SCRIPTDIR/traces-negative/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+	cat << EOF >> $MULT_FILE
+  $NAME:
+    detect: False
+    trace_file: $trace
+EOF
+    done
+
+    echo "Contents of $MULT_FILE:"
+    cat $MULT_FILE
+}
+
+function run_tests() {
+    CMD="avocado run --multiplex $MULT_FILE --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
+    echo "Running: $CMD"
+    $CMD
+    TEST_RC=$?
+}
+
+
+function print_test_failure_details() {
+    echo "Showing full job logs for any tests that failed:"
+    jq '.tests[] | select(.status != "PASS") | .logfile' $SCRIPTDIR/job-results/latest/results.json  | xargs cat
+}
+
+download_trace_files
+prepare_multiplex_file
+run_tests
+if [ $TEST_RC -ne 0 ]; then
+   print_test_failure_details
+fi
+
+exit $TEST_RC
diff --git a/userspace/falco/lua/rule_loader.lua b/userspace/falco/lua/rule_loader.lua
index 0e041f7e..7a9774a7 100644
--- a/userspace/falco/lua/rule_loader.lua
+++ b/userspace/falco/lua/rule_loader.lua
@@ -230,7 +230,7 @@ function describe_rule(name)
    end
 end
 
-local rule_output_counts = {by_level={}, by_name={}}
+local rule_output_counts = {total=0, by_level={}, by_name={}}
 
 for idx, level in ipairs(output.levels) do
    rule_output_counts[level] = 0
@@ -242,6 +242,7 @@ function on_event(evt_, rule_id)
       error ("rule_loader.on_event(): event with invalid rule_id: ", rule_id)
    end
 
+   rule_output_counts.total = rule_output_counts.total + 1
    local rule = state.rules_by_idx[rule_id]
 
    if rule_output_counts.by_level[rule.level] == nil then
@@ -260,6 +261,7 @@ function on_event(evt_, rule_id)
 end
 
 function print_stats()
+   print("Events detected: "..rule_output_counts.total)
    print("Rule counts by severity:")
    for idx, level in ipairs(output.levels) do
       -- To keep the output concise, we only print 0 counts for error, warning, and info levels