Add unit test for rule with invalid output.

Add the ability to check falco's return code with exit_status and to
generally match stderr with stderr_contains in a test.

Use those to create a test that has an invalid output expression using
%not_a_real_field. It expects falco to exit with 1 and the output to
contain a message about the invalid output.

test/falco_test.py

@@ -17,6 +17,8 @@ class FalcoTest(Test):
         """
         self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build'))
 
+        self.stderr_contains = self.params.get('stderr_contains', '*', default='')
+        self.exit_status = self.params.get('exit_status', '*', default=0)
         self.should_detect = self.params.get('detect', '*', default=False)
         self.trace_file = self.params.get('trace_file', '*')
 
@@ -197,9 +199,18 @@ class FalcoTest(Test):
 
         res = self.falco_proc.run(timeout=180, sig=9)
 
+        if self.stderr_contains != '':
+            match = re.search(self.stderr_contains, res.stderr)
+            if match is None:
+                self.fail("Stderr of falco process did not contain content matching {}".format(self.stderr_contains))
+
+        if res.exit_status != self.exit_status:
+            self.error("Falco command \"{}\" exited with unexpected return value {} (!= {})".format(
+                cmd, res.exit_status, self.exit_status))
+
+        # No need to check any outputs if the falco process exited abnormally.
         if res.exit_status != 0:
-            self.error("Falco command \"{}\" exited with non-zero return value {}".format(
+            return
-                cmd, res.exit_status))
 
         self.check_rules_warnings(res)
         if len(self.rules_events) > 0:

test/falco_tests.yaml.in

@@ -95,6 +95,13 @@ trace_files: !mux
       - rules/double_rule.yaml
     trace_file: trace_files/cat_write.scap
 
+  invalid_rule_output:
+    exit_status: 1
+    stderr_contains: "Runtime error: Error loading rules:.* Invalid output format 'An open was seen %not_a_real_field': 'invalid formatting token not_a_real_field'. Exiting."
+    rules_file:
+      - rules/invalid_rule_output.yaml
+    trace_file: trace_files/cat_write.scap
+
   disabled_rules:
     detect: False
     rules_file:

test/rules/invalid_rule_output.yaml

@@ -0,0 +1,5 @@
+- rule: rule_with_invalid_output
+  desc: A rule with an invalid output field
+  condition: evt.type=open
+  output: "An open was seen %not_a_real_field"
+  priority: WARNING