Add automated tests for tagged rules

Add automated tests that verify the ability to tag sets of rules,
disable them with -T, and run them with -t, works:

 - New test option disable_tags adds -T <tag> arguments to the falco
   command line, and run_tags adds -t <tag> arguments to the falco command
   line.
 - A new trace file open-multiple-files.scap opens 13 different files,
   and a new rules file has 13 different rules with all combinations of
   the tags a, b, c (both forward and backward), a rule with an empty
   list of tags, a rule with no tags field, and a rule with a completely
   different tag d.

Using the above, add tests for:

 - Both disabling all combations of a, b, c using disable_tags as well as
   run all combinations of a, b, c, using run_tags.
 - Specifying both disabled (-T/-D) and enabled (-t) rules. Not allowed.
 - Specifying a ruleset while having tagged rules enabled, rules based
   on a name disabled, and no particular rules enabled or disabled.

test/falco_test.py

@@ -23,6 +23,8 @@ class FalcoTest(Test):
         self.should_detect = self.params.get('detect', '*', default=False)
         self.trace_file = self.params.get('trace_file', '*')
 
+        self.env = {}
+
         if not os.path.isabs(self.trace_file):
             self.trace_file = os.path.join(self.basedir, self.trace_file)
 
@@ -113,6 +115,20 @@ class FalcoTest(Test):
                     outputs.append(output)
             self.outputs = outputs
 
+        self.disable_tags = self.params.get('disable_tags', '*', default='')
+
+        if self.disable_tags == '':
+            self.disable_tags=[]
+
+        self.run_tags = self.params.get('run_tags', '*', default='')
+
+        if self.run_tags == '':
+            self.run_tags=[]
+
+        self.ruleset = self.params.get('ruleset', '*', default='')
+        if self.ruleset != '':
+            self.env["FALCO_RULESET"] = self.ruleset
+
     def check_rules_warnings(self, res):
 
         found_warning = sets.Set()
@@ -180,13 +196,18 @@ class FalcoTest(Test):
         triggered_rules = match.group(1)
 
         for rule, count in self.detect_counts.iteritems():
-            expected_line = '{}: {}'.format(rule, count)
+            expected = '{}: (\d+)'.format(rule)
-            match = re.search(expected_line, triggered_rules)
+            match = re.search(expected, triggered_rules)
 
             if match is None:
-                self.fail("Could not find a line '{}' in triggered rule counts '{}'".format(expected_line, triggered_rules))
+                actual_count = 0
             else:
-                self.log.debug("Found expected count for {}: {}".format(rule, match.group()))
+                actual_count = int(match.group(1))
+
+            if actual_count != count:
+                self.fail("Different counts for rule {}: expected={}, actual={}".format(rule, count, actual_count))
+            else:
+                self.log.debug("Found expected count for rule {}: {}".format(rule, count))
 
     def check_outputs(self):
         for output in self.outputs:
@@ -223,7 +244,13 @@ class FalcoTest(Test):
         cmd = '{}/userspace/falco/falco {} {} -c {} -e {} -o json_output={} -v'.format(
             self.falcodir, self.rules_args, self.disabled_args, self.conf_file, self.trace_file, self.json_output)
 
-        self.falco_proc = process.SubProcess(cmd)
+        for tag in self.disable_tags:
+            cmd += ' -T {}'.format(tag)
+
+        for tag in self.run_tags:
+            cmd += ' -t {}'.format(tag)
+
+        self.falco_proc = process.SubProcess(cmd, env=self.env)
 
         res = self.falco_proc.run(timeout=180, sig=9)
 

test/falco_tests.yaml.in

@@ -154,6 +154,25 @@ trace_files: !mux
       - rules/single_rule_enabled_flag.yaml
     trace_file: trace_files/cat_write.scap
 
+  disabled_and_enabled_rules_1:
+    exit_status: 1
+    stderr_contains: "Runtime error: You can not specify both disabled .-D/-T. and enabled .-t. rules. Exiting."
+    disable_tags: [a]
+    run_tags: [a]
+    rules_file:
+      - rules/single_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  disabled_and_enabled_rules_2:
+    exit_status: 1
+    stderr_contains: "Runtime error: You can not specify both disabled .-D/-T. and enabled .-t. rules. Exiting."
+    disabled_rules:
+      - "open.*"
+    run_tags: [a]
+    rules_file:
+      - rules/single_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
   null_output_field:
     detect: True
     detect_level: WARNING
@@ -200,3 +219,339 @@ trace_files: !mux
       - "Create files below dev": 1
       - "Modify binary dirs": 2
       - "Change thread namespace": 2
+
+  disabled_tags_a:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [a]
+    detect_counts:
+      - open_1: 0
+      - open_2: 1
+      - open_3: 1
+      - open_4: 0
+      - open_5: 0
+      - open_6: 1
+      - open_7: 0
+      - open_8: 0
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  disabled_tags_b:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [b]
+    detect_counts:
+      - open_1: 1
+      - open_2: 0
+      - open_3: 1
+      - open_4: 0
+      - open_5: 1
+      - open_6: 0
+      - open_7: 0
+      - open_8: 0
+      - open_9: 1
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  disabled_tags_c:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [c]
+    detect_counts:
+      - open_1: 1
+      - open_2: 1
+      - open_3: 0
+      - open_4: 1
+      - open_5: 0
+      - open_6: 0
+      - open_7: 0
+      - open_8: 1
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  disabled_tags_ab:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [a, b]
+    detect_counts:
+      - open_1: 0
+      - open_2: 0
+      - open_3: 1
+      - open_4: 0
+      - open_5: 0
+      - open_6: 0
+      - open_7: 0
+      - open_8: 0
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  disabled_tags_abc:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [a, b, c]
+    detect_counts:
+      - open_1: 0
+      - open_2: 0
+      - open_3: 0
+      - open_4: 0
+      - open_5: 0
+      - open_6: 0
+      - open_7: 0
+      - open_8: 0
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  run_tags_a:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [a]
+    detect_counts:
+      - open_1: 1
+      - open_2: 0
+      - open_3: 0
+      - open_4: 1
+      - open_5: 1
+      - open_6: 0
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_b:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [b]
+    detect_counts:
+      - open_1: 0
+      - open_2: 1
+      - open_3: 0
+      - open_4: 1
+      - open_5: 0
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 0
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_c:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [c]
+    detect_counts:
+      - open_1: 0
+      - open_2: 0
+      - open_3: 1
+      - open_4: 0
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 0
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_ab:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [a, b]
+    detect_counts:
+      - open_1: 1
+      - open_2: 1
+      - open_3: 0
+      - open_4: 1
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_bc:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [b, c]
+    detect_counts:
+      - open_1: 0
+      - open_2: 1
+      - open_3: 1
+      - open_4: 1
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_abc:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [a, b, c]
+    detect_counts:
+      - open_1: 1
+      - open_2: 1
+      - open_3: 1
+      - open_4: 1
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_d:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [d]
+    detect_counts:
+      - open_1: 0
+      - open_2: 0
+      - open_3: 0
+      - open_4: 0
+      - open_5: 0
+      - open_6: 0
+      - open_7: 0
+      - open_8: 0
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 0
+      - open_13: 0
+
+  # Trying one of the combinations with a ruleset specified. Should get same output and no errors.
+  run_tags_c_ruleset:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [c]
+    ruleset: my-ruleset
+    detect_counts:
+      - open_1: 0
+      - open_2: 0
+      - open_3: 1
+      - open_4: 0
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 0
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  # Disabling one rule with a ruleset specified. Should get that one rule missing and no errors.
+  disabled_with_ruleset:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disabled_rules:
+      - "open_4"
+    ruleset: my-ruleset
+    detect_counts:
+      - open_1: 1
+      - open_2: 1
+      - open_3: 1
+      - open_4: 0
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  # Enabling all rules with a ruleset specified. Should get all rules matching.
+  default_with_ruleset:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    ruleset: my-ruleset
+    detect_counts:
+      - open_1: 1
+      - open_2: 1
+      - open_3: 1
+      - open_4: 1
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1

test/rules/tagged_rules.yaml

@@ -0,0 +1,93 @@
+- macro: open_read
+  condition: (evt.type=open or evt.type=openat) and evt.is_open_read=true and fd.typechar='f'
+
+- rule: open_1
+  desc: open one
+  condition: open_read and fd.name=/tmp/file-1
+  output: Open one (file=%fd.name)
+  priority: WARNING
+  tags: [a]
+
+- rule: open_2
+  desc: open two
+  condition: open_read and fd.name=/tmp/file-2
+  output: Open two (file=%fd.name)
+  priority: WARNING
+  tags: [b]
+
+- rule: open_3
+  desc: open three
+  condition: open_read and fd.name=/tmp/file-3
+  output: Open three (file=%fd.name)
+  priority: WARNING
+  tags: [c]
+
+- rule: open_4
+  desc: open four
+  condition: open_read and fd.name=/tmp/file-4
+  output: Open four (file=%fd.name)
+  priority: WARNING
+  tags: [a, b]
+
+- rule: open_5
+  desc: open file
+  condition: open_read and fd.name=/tmp/file-5
+  output: Open file (file=%fd.name)
+  priority: WARNING
+  tags: [a, c]
+
+- rule: open_6
+  desc: open six
+  condition: open_read and fd.name=/tmp/file-6
+  output: Open six (file=%fd.name)
+  priority: WARNING
+  tags: [b, c]
+
+- rule: open_7
+  desc: open seven
+  condition: open_read and fd.name=/tmp/file-7
+  output: Open seven (file=%fd.name)
+  priority: WARNING
+  tags: [a, b, c]
+
+- rule: open_8
+  desc: open eight
+  condition: open_read and fd.name=/tmp/file-8
+  output: Open eight (file=%fd.name)
+  priority: WARNING
+  tags: [b, a]
+
+- rule: open_9
+  desc: open nine
+  condition: open_read and fd.name=/tmp/file-9
+  output: Open nine (file=%fd.name)
+  priority: WARNING
+  tags: [c, a]
+
+- rule: open_10
+  desc: open ten
+  condition: open_read and fd.name=/tmp/file-10
+  output: Open ten (file=%fd.name)
+  priority: WARNING
+  tags: [b, c, a]
+
+- rule: open_11
+  desc: open eleven
+  condition: open_read and fd.name=/tmp/file-11
+  output: Open eleven (file=%fd.name)
+  priority: WARNING
+  tags: [d]
+
+- rule: open_12
+  desc: open twelve
+  condition: open_read and fd.name=/tmp/file-12
+  output: Open twelve (file=%fd.name)
+  priority: WARNING
+  tags: []
+
+- rule: open_13
+  desc: open thirteen
+  condition: open_read and fd.name=/tmp/file-13
+  output: Open thirteen (file=%fd.name)
+  priority: WARNING
+