diff --git a/test/empty.scap b/test/empty.scap new file mode 100644 index 00000000..9e0651e3 Binary files /dev/null and b/test/empty.scap differ diff --git a/test/falco_rules_warnings.yaml b/test/falco_rules_warnings.yaml new file mode 100644 index 00000000..476ca3ad --- /dev/null +++ b/test/falco_rules_warnings.yaml @@ -0,0 +1,186 @@ +- rule: no_warnings + desc: Rule with no warnings + condition: evt.type=execve + output: "None" + priority: WARNING + +- rule: no_evttype + desc: No evttype at all + condition: proc.name=foo + output: "None" + priority: WARNING + +- rule: evttype_not_equals + desc: Using != for event type + condition: evt.type!=execve + output: "None" + priority: WARNING + +- rule: leading_not + desc: condition starts with not + condition: not evt.type=execve + output: "None" + priority: WARNING + +- rule: not_equals_after_evttype + desc: != after evt.type, not affecting results + condition: evt.type=execve and proc.name!=foo + output: "None" + priority: WARNING + +- rule: not_after_evttype + desc: not operator after evt.type, not affecting results + condition: evt.type=execve and not proc.name=foo + output: "None" + priority: WARNING + +- rule: leading_trailing_evttypes + desc: evttype at beginning and end + condition: evt.type=execve and proc.name=foo or evt.type=open + output: "None" + priority: WARNING + +- rule: leading_multtrailing_evttypes + desc: one evttype at beginning, multiple at end + condition: evt.type=execve and proc.name=foo or evt.type=open or evt.type=connect + output: "None" + priority: WARNING + +- rule: leading_multtrailing_evttypes_using_in + desc: one evttype at beginning, multiple at end, using in + condition: evt.type=execve and proc.name=foo or evt.type in (open, connect) + output: "None" + priority: WARNING + +- rule: not_equals_at_end + desc: not_equals at final evttype + condition: evt.type=execve and proc.name=foo or evt.type=open or evt.type!=connect + output: "None" + priority: WARNING + +- rule: not_at_end + desc: not operator for final evttype + condition: evt.type=execve and proc.name=foo or evt.type=open or not evt.type=connect + output: "None" + priority: WARNING + +- rule: not_before_trailing_evttype + desc: a not before a trailing event type + condition: evt.type=execve and not proc.name=foo or evt.type=open + output: "None" + priority: WARNING + +- rule: not_equals_before_trailing_evttype + desc: a != before a trailing event type + condition: evt.type=execve and proc.name!=foo or evt.type=open + output: "None" + priority: WARNING + +- rule: not_equals_and_not + desc: both != and not before event types + condition: evt.type=execve and proc.name!=foo or evt.type=open or not evt.type=connect + output: "None" + priority: WARNING + +- rule: not_equals_before_in + desc: != before an in with event types + condition: evt.type=execve and proc.name!=foo or evt.type in (open, connect) + output: "None" + priority: WARNING + +- rule: not_before_in + desc: a not before an in with event types + condition: evt.type=execve and not proc.name=foo or evt.type in (open, connect) + output: "None" + priority: WARNING + +- rule: not_in_before_in + desc: a not with in before an in with event types + condition: evt.type=execve and not proc.name in (foo, bar) or evt.type in (open, connect) + output: "None" + priority: WARNING + +- rule: evttype_in + desc: using in for event types + condition: evt.type in (execve, open) + output: "None" + priority: WARNING + +- rule: evttype_in_plus_trailing + desc: using in for event types and a trailing evttype + condition: evt.type in (execve, open) and proc.name=foo or evt.type=connect + output: "None" + priority: WARNING + +- rule: leading_in_not_equals_before_evttype + desc: initial in() for event types, then a != before an additional event type + condition: evt.type in (execve, open) and proc.name!=foo or evt.type=connect + output: "None" + priority: WARNING + +- rule: leading_in_not_equals_at_evttype + desc: initial in() for event types, then a != with an additional event type + condition: evt.type in (execve, open) or evt.type!=connect + output: "None" + priority: WARNING + +- rule: not_with_evttypes + desc: not in for event types + condition: not evt.type in (execve, open) + output: "None" + priority: WARNING + +- rule: not_with_evttypes_addl + desc: not in for event types, and an additional event type + condition: not evt.type in (execve, open) or evt.type=connect + output: "None" + priority: WARNING + +- rule: not_equals_before_evttype + desc: != before any event type + condition: proc.name!=foo and evt.type=execve + output: "None" + priority: WARNING + +- rule: not_equals_before_in_evttype + desc: != before any event type using in + condition: proc.name!=foo and evt.type in (execve, open) + output: "None" + priority: WARNING + +- rule: not_before_evttype + desc: not operator before any event type + condition: not proc.name=foo and evt.type=execve + output: "None" + priority: WARNING + +- rule: not_before_evttype_using_in + desc: not operator before any event type using in + condition: not proc.name=foo and evt.type in (execve, open) + output: "None" + priority: WARNING + +- rule: repeated_evttypes + desc: event types appearing multiple times + condition: evt.type=open or evt.type=open + output: "None" + priority: WARNING + +- rule: repeated_evttypes_with_in + desc: event types appearing multiple times with in + condition: evt.type in (open, open) + output: "None" + priority: WARNING + +- rule: repeated_evttypes_with_separate_in + desc: event types appearing multiple times with separate ins + condition: evt.type in (open) or evt.type in (open, open) + output: "None" + priority: WARNING + +- rule: repeated_evttypes_with_mix + desc: event types appearing multiple times with mix of = and in + condition: evt.type=open or evt.type in (open, open) + output: "None" + priority: WARNING + diff --git a/test/falco_test.py b/test/falco_test.py index b9358e17..8c4cf9f7 100644 --- a/test/falco_test.py +++ b/test/falco_test.py @@ -3,6 +3,7 @@ import os import re import json +import sets from avocado import Test from avocado.utils import process @@ -16,9 +17,34 @@ class FalcoTest(Test): """ self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build')) - self.should_detect = self.params.get('detect', '*') + self.should_detect = self.params.get('detect', '*', default=False) self.trace_file = self.params.get('trace_file', '*') - self.json_output = self.params.get('json_output', '*') + + if not os.path.isabs(self.trace_file): + self.trace_file = os.path.join(self.basedir, self.trace_file) + + self.json_output = self.params.get('json_output', '*', default=False) + self.rules_file = self.params.get('rules_file', '*', default=os.path.join(self.basedir, '../rules/falco_rules.yaml')) + + if not os.path.isabs(self.rules_file): + self.rules_file = os.path.join(self.basedir, self.rules_file) + + self.rules_warning = self.params.get('rules_warning', '*', default=False) + if self.rules_warning == False: + self.rules_warning = sets.Set() + else: + self.rules_warning = sets.Set(self.rules_warning) + + # Maps from rule name to set of evttypes + self.rules_events = self.params.get('rules_events', '*', default=False) + if self.rules_events == False: + self.rules_events = {} + else: + events = {} + for item in self.rules_events: + for item2 in item: + events[item2[0]] = sets.Set(item2[1]) + self.rules_events = events if self.should_detect: self.detect_level = self.params.get('detect_level', '*') @@ -33,21 +59,38 @@ class FalcoTest(Test): self.str_variant = self.trace_file - def test(self): - self.log.info("Trace file %s", self.trace_file) + def check_rules_warnings(self, res): - # Run the provided trace file though falco - cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {} -o json_output={}'.format( - self.falcodir, self.falcodir, self.falcodir, self.trace_file, self.json_output) + found_warning = sets.Set() - self.falco_proc = process.SubProcess(cmd) + for match in re.finditer('Rule ([^:]+): warning \(([^)]+)\):', res.stderr): + rule = match.group(1) + warning = match.group(2) + found_warning.add(rule) - res = self.falco_proc.run(timeout=180, sig=9) + self.log.debug("Expected warning rules: {}".format(self.rules_warning)) + self.log.debug("Actual warning rules: {}".format(found_warning)) - if res.exit_status != 0: - self.error("Falco command \"{}\" exited with non-zero return value {}".format( - cmd, res.exit_status)) + if found_warning != self.rules_warning: + self.fail("Expected rules with warnings {} does not match actual rules with warnings {}".format(self.rules_warning, found_warning)) + def check_rules_events(self, res): + + found_events = {} + + for match in re.finditer('Event types for rule ([^:]+): (\S+)', res.stderr): + rule = match.group(1) + events = sets.Set(match.group(2).split(",")) + found_events[rule] = events + + self.log.debug("Expected events for rules: {}".format(self.rules_events)) + self.log.debug("Actual events for rules: {}".format(found_events)) + + for rule in found_events.keys(): + if found_events.get(rule) != self.rules_events.get(rule): + self.fail("rule {}: expected events {} differs from actual events {}".format(rule, self.rules_events.get(rule), found_events.get(rule))) + + def check_detections(self, res): # Get the number of events detected. match = re.search('Events detected: (\d+)', res.stdout) if match is None: @@ -73,6 +116,7 @@ class FalcoTest(Test): if not events_detected > 0: self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, self.detect_level)) + def check_json_output(self, res): if self.json_output: # Just verify that any lines starting with '{' are valid json objects. # Doesn't do any deep inspection of the contents. @@ -82,6 +126,27 @@ class FalcoTest(Test): for attr in ['time', 'rule', 'priority', 'output']: if not attr in obj: self.fail("Falco JSON object {} does not contain property \"{}\"".format(line, attr)) + + def test(self): + self.log.info("Trace file %s", self.trace_file) + + # Run the provided trace file though falco + cmd = '{}/userspace/falco/falco -r {} -c {}/../falco.yaml -e {} -o json_output={} -v'.format( + self.falcodir, self.rules_file, self.falcodir, self.trace_file, self.json_output) + + self.falco_proc = process.SubProcess(cmd) + + res = self.falco_proc.run(timeout=180, sig=9) + + if res.exit_status != 0: + self.error("Falco command \"{}\" exited with non-zero return value {}".format( + cmd, res.exit_status)) + + self.check_rules_warnings(res) + if len(self.rules_events) > 0: + self.check_rules_events(res) + self.check_detections(res) + self.check_json_output(res) pass diff --git a/test/falco_tests.yaml.in b/test/falco_tests.yaml.in new file mode 100644 index 00000000..bb1eb511 --- /dev/null +++ b/test/falco_tests.yaml.in @@ -0,0 +1,62 @@ +trace_files: !mux + builtin_rules_no_warnings: + detect: False + trace_file: empty.scap + rules_warning: False + + test_warnings: + detect: False + trace_file: empty.scap + rules_file: falco_rules_warnings.yaml + rules_warning: + - no_evttype + - evttype_not_equals + - leading_not + - not_equals_at_end + - not_at_end + - not_before_trailing_evttype + - not_equals_before_trailing_evttype + - not_equals_and_not + - not_equals_before_in + - not_before_in + - not_in_before_in + - leading_in_not_equals_before_evttype + - leading_in_not_equals_at_evttype + - not_with_evttypes + - not_with_evttypes_addl + - not_equals_before_evttype + - not_equals_before_in_evttype + - not_before_evttype + - not_before_evttype_using_in + rules_events: + - no_warnings: [execve] + - no_evttype: [all] + - evttype_not_equals: [all] + - leading_not: [all] + - not_equals_after_evttype: [execve] + - not_after_evttype: [execve] + - leading_trailing_evttypes: [execve,open] + - leading_multtrailing_evttypes: [connect,execve,open] + - leading_multtrailing_evttypes_using_in: [connect,execve,open] + - not_equals_at_end: [all] + - not_at_end: [all] + - not_before_trailing_evttype: [all] + - not_equals_before_trailing_evttype: [all] + - not_equals_and_not: [all] + - not_equals_before_in: [all] + - not_before_in: [all] + - not_in_before_in: [all] + - evttype_in: [execve,open] + - evttype_in_plus_trailing: [connect,execve,open] + - leading_in_not_equals_before_evttype: [all] + - leading_in_not_equals_at_evttype: [all] + - not_with_evttypes: [all] + - not_with_evttypes_addl: [all] + - not_equals_before_evttype: [all] + - not_equals_before_in_evttype: [all] + - not_before_evttype: [all] + - not_before_evttype_using_in: [all] + - repeated_evttypes: [open] + - repeated_evttypes_with_in: [open] + - repeated_evttypes_with_separate_in: [open] + - repeated_evttypes_with_mix: [open] diff --git a/test/run_regression_tests.sh b/test/run_regression_tests.sh index 8d63073b..efc40034 100755 --- a/test/run_regression_tests.sh +++ b/test/run_regression_tests.sh @@ -36,7 +36,7 @@ EOF } function prepare_multiplex_file() { - echo "trace_files: !mux" > $MULT_FILE + cp $SCRIPTDIR/falco_tests.yaml.in $MULT_FILE prepare_multiplex_fileset traces-positive True Warning False prepare_multiplex_fileset traces-negative False Warning True