From 4751546c033660b35fb174af5399fc7abee8f964 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 18 May 2016 17:08:01 -0700 Subject: [PATCH] Add correctness tests using Avocado Start using the Avocado framework for automated regression testing. Create a test FalcoTest in falco_test.py which can run on a collection of trace files. The script test/run_regression_tests.sh is responsible for pulling zip files containing the positive (falco should detect) and negative (falco should not detect) trace files, creating a Avocado multiplex file that defines all the tests (one for each trace file), running avocado on all the trace files, and showing full logs for any test that didn't pass. The old regression script, which simply ran falco, has been removed. Modify falco's stats output to show the total number of events detected for use in the tests. In travis.yml, pull a known stable version of avocado and build it, including installing any dependencies, as a part of the build process. --- .travis.yml | 10 ++++- test/falco_test.py | 63 +++++++++++++++++++++++++++++ test/falco_trace_regression.sh | 30 -------------- test/run_regression_tests.sh | 62 ++++++++++++++++++++++++++++ userspace/falco/lua/rule_loader.lua | 4 +- 5 files changed, 137 insertions(+), 32 deletions(-) create mode 100644 test/falco_test.py delete mode 100755 test/falco_trace_regression.sh create mode 100755 test/run_regression_tests.sh diff --git a/.travis.yml b/.travis.yml index d4953371..fe37c22f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,14 @@ install: - sudo apt-get --force-yes install g++-4.8 - sudo apt-get install rpm linux-headers-$(uname -r) - git clone https://github.com/draios/sysdig.git ../sysdig + - sudo apt-get install -y python-pip libvirt-dev jq + - cd .. + - curl -Lo avocado-36.0-tar.gz https://github.com/avocado-framework/avocado/archive/36.0lts.tar.gz + - tar -zxvf avocado-36.0-tar.gz + - cd avocado-36.0lts + - sudo pip install -r requirements-travis.txt + - sudo python setup.py install + - cd ../falco before_script: - export KERNELDIR=/lib/modules/$(ls /lib/modules | sort | head -1)/build script: @@ -28,7 +36,7 @@ script: - make VERBOSE=1 - make package - cd .. - - sudo test/falco_trace_regression.sh build/userspace/falco/falco + - sudo test/run_regression_tests.sh notifications: webhooks: urls: diff --git a/test/falco_test.py b/test/falco_test.py new file mode 100644 index 00000000..72875c1c --- /dev/null +++ b/test/falco_test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import os +import re + +from avocado import Test +from avocado.utils import process +from avocado.utils import linux_modules + +class FalcoTest(Test): + + def setUp(self): + """ + Load the sysdig kernel module if not already loaded. + """ + self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build')) + + self.should_detect = self.params.get('detect', '*') + self.trace_file = self.params.get('trace_file', '*') + + # Doing this in 2 steps instead of simply using + # module_is_loaded to avoid logging lsmod output to the log. + lsmod_output = process.system_output("lsmod", verbose=False) + + if linux_modules.parse_lsmod_for_module(lsmod_output, 'sysdig_probe') == {}: + self.log.debug("Loading sysdig kernel module") + process.run('sudo insmod {}/driver/sysdig-probe.ko'.format(self.falcodir)) + + self.str_variant = self.trace_file + + def test(self): + self.log.info("Trace file %s", self.trace_file) + + # Run the provided trace file though falco + cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {}'.format( + self.falcodir, self.falcodir, self.falcodir, self.trace_file) + + self.falco_proc = process.SubProcess(cmd) + + res = self.falco_proc.run(timeout=60, sig=9) + + if res.exit_status != 0: + self.error("Falco command \"{}\" exited with non-zero return value {}".format( + cmd, res.exit_status)) + + # Get the number of events detected. + res = re.search('Events detected: (\d+)', res.stdout) + if res is None: + self.fail("Could not find a line 'Events detected: ' in falco output") + + events_detected = int(res.group(1)) + + if not self.should_detect and events_detected > 0: + self.fail("Detected {} events when should have detected none".format(events_detected)) + + if self.should_detect and events_detected == 0: + self.fail("Detected {} events when should have detected > 0".format(events_detected)) + + pass + + +if __name__ == "__main__": + main() diff --git a/test/falco_trace_regression.sh b/test/falco_trace_regression.sh deleted file mode 100755 index a4b3498c..00000000 --- a/test/falco_trace_regression.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -eu - -SCRIPT=$(readlink -f $0) -BASEDIR=$(dirname $SCRIPT) - -FALCO=$1 -BUILDDIR=$(dirname $FALCO) - -# Load the built kernel module by hand -insmod $BUILDDIR/../../driver/sysdig-probe.ko - -# For now, simply ensure that falco can run without errors. -FALCO_CMDLINE="$FALCO -c $BASEDIR/../falco.yaml -r $BASEDIR/../rules/falco_rules.yaml" -echo "Running falco: $FALCO_CMDLINE" -$FALCO_CMDLINE > $BASEDIR/falco.log 2>&1 & -FALCO_PID=$! -echo "Falco started, pid $FALCO_PID" -sleep 10 -if kill -0 $FALCO_PID > /dev/null 2>&1; then - echo "Falco ran successfully" - kill $FALCO_PID - ret=0 -else - echo "Falco did not start successfully. Full program output:" - cat $BASEDIR/falco.log - ret=1 -fi - -exit $ret diff --git a/test/run_regression_tests.sh b/test/run_regression_tests.sh new file mode 100755 index 00000000..9f6b2a28 --- /dev/null +++ b/test/run_regression_tests.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +SCRIPT=$(readlink -f $0) +SCRIPTDIR=$(dirname $SCRIPT) +MULT_FILE=$SCRIPTDIR/falco_tests.yaml + +function download_trace_files() { + for TRACE in traces-positive traces-negative ; do + curl -so $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip && + unzip -d $SCRIPTDIR $SCRIPTDIR/$TRACE.zip && + rm -rf $SCRIPTDIR/$TRACE.zip + done +} + +function prepare_multiplex_file() { + echo "trace_files: !mux" > $MULT_FILE + + for trace in $SCRIPTDIR/traces-positive/*.scap ; do + [ -e "$trace" ] || continue + NAME=`basename $trace .scap` + cat << EOF >> $MULT_FILE + $NAME: + detect: True + trace_file: $trace +EOF + done + + for trace in $SCRIPTDIR/traces-negative/*.scap ; do + [ -e "$trace" ] || continue + NAME=`basename $trace .scap` + cat << EOF >> $MULT_FILE + $NAME: + detect: False + trace_file: $trace +EOF + done + + echo "Contents of $MULT_FILE:" + cat $MULT_FILE +} + +function run_tests() { + CMD="avocado run --multiplex $MULT_FILE --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py" + echo "Running: $CMD" + $CMD + TEST_RC=$? +} + + +function print_test_failure_details() { + echo "Showing full job logs for any tests that failed:" + jq '.tests[] | select(.status != "PASS") | .logfile' $SCRIPTDIR/job-results/latest/results.json | xargs cat +} + +download_trace_files +prepare_multiplex_file +run_tests +if [ $TEST_RC -ne 0 ]; then + print_test_failure_details +fi + +exit $TEST_RC diff --git a/userspace/falco/lua/rule_loader.lua b/userspace/falco/lua/rule_loader.lua index 0e041f7e..7a9774a7 100644 --- a/userspace/falco/lua/rule_loader.lua +++ b/userspace/falco/lua/rule_loader.lua @@ -230,7 +230,7 @@ function describe_rule(name) end end -local rule_output_counts = {by_level={}, by_name={}} +local rule_output_counts = {total=0, by_level={}, by_name={}} for idx, level in ipairs(output.levels) do rule_output_counts[level] = 0 @@ -242,6 +242,7 @@ function on_event(evt_, rule_id) error ("rule_loader.on_event(): event with invalid rule_id: ", rule_id) end + rule_output_counts.total = rule_output_counts.total + 1 local rule = state.rules_by_idx[rule_id] if rule_output_counts.by_level[rule.level] == nil then @@ -260,6 +261,7 @@ function on_event(evt_, rule_id) end function print_stats() + print("Events detected: "..rule_output_counts.total) print("Rule counts by severity:") for idx, level in ipairs(output.levels) do -- To keep the output concise, we only print 0 counts for error, warning, and info levels