Add tests for multiple files, disabled rules.

Add test that cover reading from multiple sets of rule files and
disabling rules. Specific changes:

 - Modify falco to allow multiple -r arguments to read from multiple
   files.
 - In the test multiplex file, add a disabled_rules attribute,
   containing a sequence of rules to disable. Result in -D arguments
   when running falco.
 - In the test multiplex file, 'rules_file' can be a sequence. It
   results in multiple -r arguments when running falco.
 - In the test multiplex file, 'detect_level' can be a squence of
   multiple severity levels. All levels will be checked for in the
   output.
 - Move all test rules files to a rules subdirectory and all trace files
   to a traces subdirectory.
 - Add a small trace file for a simple cat of /dev/null. Used by the
   new tests.
 - Add the following new tests:
     - Reading from multiple files, with the first file being
       empty. Ensure that the rules from the second file are properly
       loaded.
     - Reading from multiple files with the last being empty. Ensures
       that the empty file doesn't overwrite anything from the first
       file.
     - Reading from multiple files with varying severity levels for each
       rule. Ensures that both files are properly read.
     - Disabling rules from a rules file, both with full rule names
       and regexes. Will result in not detecting anything.

test/falco_test.py

@@ -26,8 +26,28 @@ class FalcoTest(Test):
         self.json_output = self.params.get('json_output', '*', default=False)
         self.rules_file = self.params.get('rules_file', '*', default=os.path.join(self.basedir, '../rules/falco_rules.yaml'))
 
-        if not os.path.isabs(self.rules_file):
-            self.rules_file = os.path.join(self.basedir, self.rules_file)
+        if not isinstance(self.rules_file, list):
+            self.rules_file = [self.rules_file]
+
+        self.rules_args = ""
+
+        for file in self.rules_file:
+            if not os.path.isabs(file):
+                file = os.path.join(self.basedir, file)
+            self.rules_args = self.rules_args + "-r " + file + " "
+
+        self.disabled_rules = self.params.get('disabled_rules', '*', default='')
+
+        if self.disabled_rules == '':
+            self.disabled_rules = []
+
+        if not isinstance(self.disabled_rules, list):
+            self.disabled_rules = [self.disabled_rules]
+
+        self.disabled_args = ""
+
+        for rule in self.disabled_rules:
+            self.disabled_args = self.disabled_args + "-D " + rule + " "
 
         self.rules_warning = self.params.get('rules_warning', '*', default=False)
         if self.rules_warning == False:
@@ -49,6 +69,9 @@ class FalcoTest(Test):
         if self.should_detect:
             self.detect_level = self.params.get('detect_level', '*')
 
+            if not isinstance(self.detect_level, list):
+                self.detect_level = [self.detect_level]
+
         # Doing this in 2 steps instead of simply using
         # module_is_loaded to avoid logging lsmod output to the log.
         lsmod_output = process.system_output("lsmod", verbose=False)
@@ -105,16 +128,17 @@ class FalcoTest(Test):
             if events_detected == 0:
                 self.fail("Detected {} events when should have detected > 0".format(events_detected))
 
-            level_line = '(?i){}: (\d+)'.format(self.detect_level)
-            match = re.search(level_line, res.stdout)
+            for level in self.detect_level:
+                level_line = '(?i){}: (\d+)'.format(level)
+                match = re.search(level_line, res.stdout)
 
-            if match is None:
-                self.fail("Could not find a line '{}: <count>' in falco output".format(self.detect_level))
+                if match is None:
+                    self.fail("Could not find a line '{}: <count>' in falco output".format(level))
 
-            events_detected = int(match.group(1))
+                    events_detected = int(match.group(1))
 
-            if not events_detected > 0:
-                self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, self.detect_level))
+                    if not events_detected > 0:
+                        self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, level))
 
     def check_json_output(self, res):
         if self.json_output:
@@ -131,8 +155,8 @@ class FalcoTest(Test):
         self.log.info("Trace file %s", self.trace_file)
 
         # Run the provided trace file though falco
-        cmd = '{}/userspace/falco/falco -r {} -c {}/../falco.yaml -e {} -o json_output={} -v'.format(
-            self.falcodir, self.rules_file, self.falcodir, self.trace_file, self.json_output)
+        cmd = '{}/userspace/falco/falco {} {} -c {}/../falco.yaml -e {} -o json_output={} -v'.format(
+            self.falcodir, self.rules_args, self.disabled_args, self.falcodir, self.trace_file, self.json_output)
 
         self.falco_proc = process.SubProcess(cmd)
 

test/falco_tests.yaml.in

@@ -1,13 +1,13 @@
 trace_files: !mux
   builtin_rules_no_warnings:
     detect: False
-    trace_file: empty.scap
+    trace_file: trace_files/empty.scap
     rules_warning: False
 
   test_warnings:
     detect: False
-    trace_file: empty.scap
-    rules_file: falco_rules_warnings.yaml
+    trace_file: trace_files/empty.scap
+    rules_file: rules/falco_rules_warnings.yaml
     rules_warning:
       - no_evttype
       - evttype_not_equals
@@ -60,3 +60,48 @@ trace_files: !mux
       - repeated_evttypes_with_in: [open]
       - repeated_evttypes_with_separate_in: [open]
       - repeated_evttypes_with_mix: [open]
+
+  multiple_rules_first_empty:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules_last_empty:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/empty_rules.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules:
+    detect: True
+    detect_level:
+      - WARNING
+      - INFO
+      - ERROR
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/double_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  disabled_rules:
+    detect: False
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    disabled_rules:
+      - open_from_cat
+    trace_file: trace_files/cat_write.scap
+
+  disabled_rules_using_regex:
+    detect: False
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    disabled_rules:
+      - "open.*"
+    trace_file: trace_files/cat_write.scap

test/rules/double_rule.yaml

@@ -0,0 +1,13 @@
+# This ruleset depends on the is_cat macro defined in single_rule.yaml
+
+- rule: exec_from_cat
+  desc: A process named cat does execve
+  condition: evt.type=execve and is_cat
+  output: "An exec was seen (command=%proc.cmdline)"
+  priority: ERROR
+
+- rule: access_from_cat
+  desc: A process named cat does an access
+  condition: evt.type=access and is_cat
+  output: "An access was seen (command=%proc.cmdline)"
+  priority: INFO

test/rules/single_rule.yaml

@@ -0,0 +1,8 @@
+- macro: is_cat
+  condition: proc.name=cat
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING