Add regression tests for json output.

Modify falco_test.py to look for a boolean multiplex attribute
'json_output'. If true, examine the lines of the output and for any line
that begins with '{', parse it as json and ensure it has the 4
attributes we expect.

Modify run_regression_tests to have a utility function
prepare_multiplex_fileset that does the work of looping over files in a
directory, along with detect, level, and json output arguments. The
appropriate multiplex attributes are added for each file.

Use that utility function to test json output for the positive and
informational  directories along with non-json output. The negative
directory is only tested once.

test/falco_test.py

@@ -2,6 +2,7 @@
 
 import os
 import re
+import json
 
 from avocado import Test
 from avocado.utils import process
@@ -17,6 +18,7 @@ class FalcoTest(Test):
 
         self.should_detect = self.params.get('detect', '*')
         self.trace_file = self.params.get('trace_file', '*')
+        self.json_output = self.params.get('json_output', '*')
 
         if self.should_detect:
             self.detect_level = self.params.get('detect_level', '*')
@@ -35,8 +37,8 @@ class FalcoTest(Test):
         self.log.info("Trace file %s", self.trace_file)
 
         # Run the provided trace file though falco
-        cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {}'.format(
-            self.falcodir, self.falcodir, self.falcodir, self.trace_file)
+        cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {} -o json_output={}'.format(
+            self.falcodir, self.falcodir, self.falcodir, self.trace_file, self.json_output)
 
         self.falco_proc = process.SubProcess(cmd)
 
@@ -71,6 +73,15 @@ class FalcoTest(Test):
             if not events_detected > 0:
                 self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, self.detect_level))
 
+        if self.json_output:
+            # Just verify that any lines starting with '{' are valid json objects.
+            # Doesn't do any deep inspection of the contents.
+            for line in res.stdout.splitlines():
+                if line.startswith('{'):
+                    obj = json.loads(line)
+                    for attr in ['time', 'rule', 'priority', 'output']:
+                        if not attr in obj:
+                            self.fail("Falco JSON object {} does not contain property \"{}\"".format(line, attr))
         pass
 
 

test/run_regression_tests.sh

@@ -13,40 +13,35 @@ function download_trace_files() {
     done
 }
 
+function prepare_multiplex_fileset() {
+
+    dir=$1
+    detect=$2
+    detect_level=$3
+    json_output=$4
+
+    for trace in $SCRIPTDIR/$dir/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+	cat << EOF >> $MULT_FILE
+  $NAME-detect-$detect-json-$json_output:
+    detect: $detect
+    detect_level: $detect_level
+    trace_file: $trace
+    json_output: $json_output
+EOF
+    done
+}
+
 function prepare_multiplex_file() {
     echo "trace_files: !mux" > $MULT_FILE
 
-    for trace in $SCRIPTDIR/traces-positive/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
-	cat << EOF >> $MULT_FILE
-  $NAME:
-    detect: True
-    detect_level: Warning
-    trace_file: $trace
-EOF
-    done
+    prepare_multiplex_fileset traces-positive True Warning False
+    prepare_multiplex_fileset traces-negative False Warning True
+    prepare_multiplex_fileset traces-info True Informational False
 
-    for trace in $SCRIPTDIR/traces-negative/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
-	cat << EOF >> $MULT_FILE
-  $NAME:
-    detect: False
-    trace_file: $trace
-EOF
-    done
-
-    for trace in $SCRIPTDIR/traces-info/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
-	cat << EOF >> $MULT_FILE
-  $NAME:
-    detect: True
-    detect_level: Informational
-    trace_file: $trace
-EOF
-    done
+    prepare_multiplex_fileset traces-positive True Warning True
+    prepare_multiplex_fileset traces-info True Informational True
 
     echo "Contents of $MULT_FILE:"
     cat $MULT_FILE