From 995e61210e1e6f4e36ee070d8a10aa1e52b3b2c9 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@sysdig.com>
Date: Tue, 7 Jun 2016 13:35:27 -0700
Subject: [PATCH] Add regression tests for json output.

Modify falco_test.py to look for a boolean multiplex attribute
'json_output'. If true, examine the lines of the output and for any line
that begins with '{', parse it as json and ensure it has the 4
attributes we expect.

Modify run_regression_tests to have a utility function
prepare_multiplex_fileset that does the work of looping over files in a
directory, along with detect, level, and json output arguments. The
appropriate multiplex attributes are added for each file.

Use that utility function to test json output for the positive and
informational  directories along with non-json output. The negative
directory is only tested once.
---
 test/falco_test.py           | 15 ++++++++--
 test/run_regression_tests.sh | 55 ++++++++++++++++--------------------
 2 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/test/falco_test.py b/test/falco_test.py
index a2ff0847..adb35767 100644
--- a/test/falco_test.py
+++ b/test/falco_test.py
@@ -2,6 +2,7 @@
 
 import os
 import re
+import json
 
 from avocado import Test
 from avocado.utils import process
@@ -17,6 +18,7 @@ class FalcoTest(Test):
 
         self.should_detect = self.params.get('detect', '*')
         self.trace_file = self.params.get('trace_file', '*')
+        self.json_output = self.params.get('json_output', '*')
 
         if self.should_detect:
             self.detect_level = self.params.get('detect_level', '*')
@@ -35,8 +37,8 @@ class FalcoTest(Test):
         self.log.info("Trace file %s", self.trace_file)
 
         # Run the provided trace file though falco
-        cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {}'.format(
-            self.falcodir, self.falcodir, self.falcodir, self.trace_file)
+        cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {} -o json_output={}'.format(
+            self.falcodir, self.falcodir, self.falcodir, self.trace_file, self.json_output)
 
         self.falco_proc = process.SubProcess(cmd)
 
@@ -71,6 +73,15 @@ class FalcoTest(Test):
             if not events_detected > 0:
                 self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, self.detect_level))
 
+        if self.json_output:
+            # Just verify that any lines starting with '{' are valid json objects.
+            # Doesn't do any deep inspection of the contents.
+            for line in res.stdout.splitlines():
+                if line.startswith('{'):
+                    obj = json.loads(line)
+                    for attr in ['time', 'rule', 'priority', 'output']:
+                        if not attr in obj:
+                            self.fail("Falco JSON object {} does not contain property \"{}\"".format(line, attr))
         pass
 
 
diff --git a/test/run_regression_tests.sh b/test/run_regression_tests.sh
index 1057ab61..b46646a1 100755
--- a/test/run_regression_tests.sh
+++ b/test/run_regression_tests.sh
@@ -13,40 +13,35 @@ function download_trace_files() {
     done
 }
 
+function prepare_multiplex_fileset() {
+
+    dir=$1
+    detect=$2
+    detect_level=$3
+    json_output=$4
+
+    for trace in $SCRIPTDIR/$dir/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+	cat << EOF >> $MULT_FILE
+  $NAME-detect-$detect-json-$json_output:
+    detect: $detect
+    detect_level: $detect_level
+    trace_file: $trace
+    json_output: $json_output
+EOF
+    done
+}
+
 function prepare_multiplex_file() {
     echo "trace_files: !mux" > $MULT_FILE
 
-    for trace in $SCRIPTDIR/traces-positive/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
-	cat << EOF >> $MULT_FILE
-  $NAME:
-    detect: True
-    detect_level: Warning
-    trace_file: $trace
-EOF
-    done
+    prepare_multiplex_fileset traces-positive True Warning False
+    prepare_multiplex_fileset traces-negative False Warning True
+    prepare_multiplex_fileset traces-info True Informational False
 
-    for trace in $SCRIPTDIR/traces-negative/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
-	cat << EOF >> $MULT_FILE
-  $NAME:
-    detect: False
-    trace_file: $trace
-EOF
-    done
-
-    for trace in $SCRIPTDIR/traces-info/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
-	cat << EOF >> $MULT_FILE
-  $NAME:
-    detect: True
-    detect_level: Informational
-    trace_file: $trace
-EOF
-    done
+    prepare_multiplex_fileset traces-positive True Warning True
+    prepare_multiplex_fileset traces-info True Informational True
 
     echo "Contents of $MULT_FILE:"
     cat $MULT_FILE