Add correctness tests using Avocado

Start using the Avocado framework for automated regression
testing. Create a test FalcoTest in falco_test.py which can run on a
collection of trace files. The script test/run_regression_tests.sh is
responsible for pulling zip files containing the positive (falco should
detect) and negative (falco should not detect) trace files, creating a
Avocado multiplex file that defines all the tests (one for each trace
file), running avocado on all the trace files, and showing full logs for
any test that didn't pass.

The old regression script, which simply ran falco, has been removed.

Modify falco's stats output to show the total number of events detected
for use in the tests.

In travis.yml, pull a known stable version of avocado and build it,
including installing any dependencies, as a part of the build process.

.travis.yml

@@ -9,6 +9,14 @@ install:
     - sudo apt-get --force-yes install g++-4.8
     - sudo apt-get install rpm linux-headers-$(uname -r)
     - git clone https://github.com/draios/sysdig.git ../sysdig
+    - sudo apt-get install -y python-pip libvirt-dev jq
+    - cd ..
+    - curl -Lo avocado-36.0-tar.gz https://github.com/avocado-framework/avocado/archive/36.0lts.tar.gz
+    - tar -zxvf avocado-36.0-tar.gz
+    - cd avocado-36.0lts
+    - sudo pip install -r requirements-travis.txt
+    - sudo python setup.py install
+    - cd ../falco
 before_script:
     - export KERNELDIR=/lib/modules/$(ls /lib/modules | sort | head -1)/build
 script:
@@ -28,7 +36,7 @@ script:
     - make VERBOSE=1
     - make package
     - cd ..
-    - sudo test/falco_trace_regression.sh build/userspace/falco/falco
+    - sudo test/run_regression_tests.sh
 notifications:
   webhooks:
     urls:

test/falco_test.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+import os
+import re
+
+from avocado import Test
+from avocado.utils import process
+from avocado.utils import linux_modules
+
+class FalcoTest(Test):
+
+    def setUp(self):
+        """
+        Load the sysdig kernel module if not already loaded.
+        """
+        self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build'))
+
+        self.should_detect = self.params.get('detect', '*')
+        self.trace_file = self.params.get('trace_file', '*')
+
+        # Doing this in 2 steps instead of simply using
+        # module_is_loaded to avoid logging lsmod output to the log.
+        lsmod_output = process.system_output("lsmod", verbose=False)
+
+        if linux_modules.parse_lsmod_for_module(lsmod_output, 'sysdig_probe') == {}:
+            self.log.debug("Loading sysdig kernel module")
+            process.run('sudo insmod {}/driver/sysdig-probe.ko'.format(self.falcodir))
+
+        self.str_variant = self.trace_file
+
+    def test(self):
+        self.log.info("Trace file %s", self.trace_file)
+
+        # Run the provided trace file though falco
+        cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {}'.format(
+            self.falcodir, self.falcodir, self.falcodir, self.trace_file)
+
+        self.falco_proc = process.SubProcess(cmd)
+
+        res = self.falco_proc.run(timeout=60, sig=9)
+
+        if res.exit_status != 0:
+            self.error("Falco command \"{}\" exited with non-zero return value {}".format(
+                cmd, res.exit_status))
+
+        # Get the number of events detected.
+        res = re.search('Events detected: (\d+)', res.stdout)
+        if res is None:
+            self.fail("Could not find a line 'Events detected: <count>' in falco output")
+
+        events_detected = int(res.group(1))
+
+        if not self.should_detect and events_detected > 0:
+            self.fail("Detected {} events when should have detected none".format(events_detected))
+
+        if self.should_detect and events_detected == 0:
+            self.fail("Detected {} events when should have detected > 0".format(events_detected))
+
+        pass
+
+
+if __name__ == "__main__":
+    main()

test/falco_trace_regression.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-set -eu
-
-SCRIPT=$(readlink -f $0)
-BASEDIR=$(dirname $SCRIPT)
-
-FALCO=$1
-BUILDDIR=$(dirname $FALCO)
-
-# Load the built kernel module by hand
-insmod $BUILDDIR/../../driver/sysdig-probe.ko
-
-# For now, simply ensure that falco can run without errors.
-FALCO_CMDLINE="$FALCO -c $BASEDIR/../falco.yaml -r $BASEDIR/../rules/falco_rules.yaml"
-echo "Running falco: $FALCO_CMDLINE"
-$FALCO_CMDLINE > $BASEDIR/falco.log 2>&1 &
-FALCO_PID=$!
-echo "Falco started, pid $FALCO_PID"
-sleep 10
-if kill -0 $FALCO_PID > /dev/null 2>&1; then
-    echo "Falco ran successfully"
-    kill $FALCO_PID
-    ret=0
-else
-    echo "Falco did not start successfully. Full program output:"
-    cat $BASEDIR/falco.log
-    ret=1
-fi
-
-exit $ret

test/run_regression_tests.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+SCRIPT=$(readlink -f $0)
+SCRIPTDIR=$(dirname $SCRIPT)
+MULT_FILE=$SCRIPTDIR/falco_tests.yaml
+
+function download_trace_files() {
+    for TRACE in traces-positive traces-negative ; do
+	curl -so $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip &&
+	unzip -d $SCRIPTDIR $SCRIPTDIR/$TRACE.zip &&
+	rm -rf $SCRIPTDIR/$TRACE.zip
+    done
+}
+
+function prepare_multiplex_file() {
+    echo "trace_files: !mux" > $MULT_FILE
+
+    for trace in $SCRIPTDIR/traces-positive/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+	cat << EOF >> $MULT_FILE
+  $NAME:
+    detect: True
+    trace_file: $trace
+EOF
+    done
+
+    for trace in $SCRIPTDIR/traces-negative/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+	cat << EOF >> $MULT_FILE
+  $NAME:
+    detect: False
+    trace_file: $trace
+EOF
+    done
+
+    echo "Contents of $MULT_FILE:"
+    cat $MULT_FILE
+}
+
+function run_tests() {
+    CMD="avocado run --multiplex $MULT_FILE --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
+    echo "Running: $CMD"
+    $CMD
+    TEST_RC=$?
+}
+
+
+function print_test_failure_details() {
+    echo "Showing full job logs for any tests that failed:"
+    jq '.tests[] | select(.status != "PASS") | .logfile' $SCRIPTDIR/job-results/latest/results.json  | xargs cat
+}
+
+download_trace_files
+prepare_multiplex_file
+run_tests
+if [ $TEST_RC -ne 0 ]; then
+   print_test_failure_details
+fi
+
+exit $TEST_RC

userspace/falco/lua/rule_loader.lua

@@ -230,7 +230,7 @@ function describe_rule(name)
    end
 end
 
-local rule_output_counts = {by_level={}, by_name={}}
+local rule_output_counts = {total=0, by_level={}, by_name={}}
 
 for idx, level in ipairs(output.levels) do
    rule_output_counts[level] = 0
@@ -242,6 +242,7 @@ function on_event(evt_, rule_id)
       error ("rule_loader.on_event(): event with invalid rule_id: ", rule_id)
    end
 
+   rule_output_counts.total = rule_output_counts.total + 1
    local rule = state.rules_by_idx[rule_id]
 
    if rule_output_counts.by_level[rule.level] == nil then
@@ -260,6 +261,7 @@ function on_event(evt_, rule_id)
 end
 
 function print_stats()
+   print("Events detected: "..rule_output_counts.total)
    print("Rule counts by severity:")
    for idx, level in ipairs(output.levels) do
       -- To keep the output concise, we only print 0 counts for error, warning, and info levels