github/falco
1
0
Fork 0
mirror of https://github.com/falcosecurity/falco.git synced 2026-01-24 14:32:34 +00:00
Code Issues Projects Releases Wiki Activity

Update tests to use result struct + json-based validation

Browse Source 
Update tests that validated rules files (typically looking for
errors/warnings) to use the new result struct + json based validation:

- When validating rules files, always use json output.

- In test cases, instead of parsing stderr/stdout, use new test
  properties "validate_ok", "validate_errors",
  "validate_warnings". These parse the json output and look for
  specific tuples of (error code, error message, item type, item name)
  in the output.

- There were a few tests that were actually validation tests but using
  the -r argument to load rules. Convert them to validation tests. In
  one case, split the test into two separate tests--one for
  validation, one ensuring that the rule doesn't match anything.

- There were a couple of tests that were duplicates of existing
  validation tests, just checking for the error in a different
  way. Remove them.

Signed-off-by: Mark Stemm <mark.stemm@gmail.com>
This commit is contained in:
Mark Stemm
2022-06-28 17:12:58 -07:00
committed by poiana
parent 550cdbd176
commit a37e2252b2
5 changed files with 358 additions and 456 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
test/falco_test.py
View File

@@ -77,6 +77,10 @@ class FalcoTest(Test):
else:
self.stderr_not_contains = [self.stderr_not_contains]
self.validate_ok = self.params.get('validate_ok', '*', default='')
self.validate_warnings = self.params.get('validate_warnings', '*', default='')
self.validate_errors = self.params.get('validate_errors', '*', default='')
self.exit_status = self.params.get('exit_status', '*', default=0)
self.should_detect = self.params.get('detect', '*', default=False)
self.check_detection_counts = self.params.get('check_detection_counts', '*', default=True)
@@ -105,6 +109,9 @@ class FalcoTest(Test):
if self.validate_rules_file == False:
self.validate_rules_file = []
else:
# Always enable json output when validating rules
# files. Makes parsing errors/warnings easier
self.json_output = True
if not isinstance(self.validate_rules_file, list):
self.validate_rules_file = [self.validate_rules_file]
@@ -153,13 +160,6 @@ class FalcoTest(Test):
detect_counts[key] = value
self.detect_counts = detect_counts
self.rules_warning = self.params.get(
'rules_warning', '*', default=False)
if self.rules_warning == False:
self.rules_warning = set()
else:
self.rules_warning = set(self.rules_warning)
# Maps from rule name to set of evttypes
self.rules_events = self.params.get('rules_events', '*', default=False)
if self.rules_events == False:
@@ -265,22 +265,6 @@ class FalcoTest(Test):
if self.package != 'None':
self.uninstall_package()
def check_rules_warnings(self, res):
found_warning = set()
for match in re.finditer('Rule ([^:]+): warning \(([^)]+)\):', res.stderr.decode("utf-8")):
rule = match.group(1)
warning = match.group(2)
found_warning.add(rule)
self.log.debug("Expected warning rules: {}".format(self.rules_warning))
self.log.debug("Actual warning rules: {}".format(found_warning))
if found_warning != self.rules_warning:
self.fail("Expected rules with warnings {} does not match actual rules with warnings {}".format(
self.rules_warning, found_warning))
def check_rules_events(self, res):
found_events = {}
@@ -376,7 +360,68 @@ class FalcoTest(Test):
return True
def check_json_output(self, res):
def get_validate_json(self, res):
if self.validate_json is None:
# The first line of stdout should be the validation result as json
self.validate_json = json.loads(res.stdout.decode("utf-8").partition('\n')[0])
return self.validate_json
def check_validate_ok(self, res):
if self.validate_ok != '':
vobj = self.get_validate_json(res)
for expected in self.validate_ok:
found = False
for vres in vobj["falco_load_results"]:
if vres["successful"] and os.path.basename(vres["name"]) == expected:
found = True
break
if not found:
self.fail("Validation json did not contain a successful result for file '{}'".format(expected))
def check_validate_warnings(self, res):
if self.validate_warnings != '':
vobj = self.get_validate_json(res)
for warnobj in self.validate_warnings:
found = False
for vres in vobj["falco_load_results"]:
for warning in vres["warnings"]:
if warning["code"] == warnobj["code"]:
if ("message" in warnobj and warning["message"] == warnobj["message"]) or ("message_contains" in warnobj and warnobj["message_contains"] in warning["message"]):
for loc in warning["context"]["locations"]:
if loc["item_type"] == warnobj["item_type"] and loc["item_name"] == warnobj["item_name"]:
found = True
break
if not found:
if "message" in warnobj:
self.fail("Validation json did not contain a warning '{}' for '{}' '{}' with message '{}'".format(
warnobj["code"], warnobj["item_type"], warnobj["item_name"], warnobj["message"]))
else:
self.fail("Validation json did not contain a warning '{}' for '{}' '{}' with message containing '{}'".format(
warnobj["code"], warnobj["item_type"], warnobj["item_name"], warnobj["message_contains"]))
def check_validate_errors(self, res):
if self.validate_errors != '':
vobj = self.get_validate_json(res)
for errobj in self.validate_errors:
found = False
for vres in vobj["falco_load_results"]:
for error in vres["errors"]:
if error["code"] == errobj["code"]:
if ("message" in errobj and error["message"] == errobj["message"]) or ("message_contains" in errobj and errobj["message_contains"] in error["message"]):
for loc in error["context"]["locations"]:
if loc["item_type"] == errobj["item_type"] and loc["item_name"] == errobj["item_name"]:
found = True
break
if not found:
if "message" in errobj:
self.fail("Validation json did not contain a error '{}' for '{}' '{}' with message '{}'".format(
errobj["code"], errobj["item_type"], errobj["item_name"], errobj["message"]))
else:
self.fail("Validation json did not contain a error '{}' for '{}' '{}' with message containing '{}'".format(
errobj["code"], errobj["item_type"], errobj["item_name"], errobj["message_contains"]))
def check_json_event_output(self, res):
if self.json_output:
# Just verify that any lines starting with '{' are valid json objects.
# Doesn't do any deep inspection of the contents.
@@ -578,6 +623,8 @@ class FalcoTest(Test):
# This sets falco_binary_path as a side-effect.
self.install_package()
self.validate_json = None
trace_arg = self.trace_file
if self.trace_file:
@@ -644,18 +691,22 @@ class FalcoTest(Test):
self.error("Falco command \"{}\" exited with unexpected return value {} (!= {})".format(
cmd, res.exit_status, self.exit_status))
self.check_validate_ok(res)
self.check_validate_errors(res)
self.check_validate_warnings(res)
# No need to check any outputs if the falco process exited abnormally.
if res.exit_status != 0:
return
self.check_rules_warnings(res)
if len(self.rules_events) > 0:
self.check_rules_events(res)
if len(self.validate_rules_file) == 0 and self.check_detection_counts:
self.check_detections(res)
if len(self.detect_counts) > 0:
self.check_detections_by_rule(res)
self.check_json_output(res)
if not self.validate_rules_file:
self.check_json_event_output(res)
self.check_outputs()
self.check_output_strictly_contains(res)
self.check_grpc()

480
test/falco_tests.yaml
View File

@@ -20,22 +20,55 @@ trace_files: !mux
builtin_rules_no_warnings:
detect: False
trace_file: trace_files/empty.scap
rules_warning: False
# The rules_events part of this test was mistakenly disabled when
# generic events (e.g. k8s_audit support) was added (#1715).
# The implementation no longer prints messages of the form:
# "Event types for rule (<RULE>): (<EVENT TYPES>)
# And without that output, none of the checks below rules_events
# are considered.
# XXX/mstemm add it back
test_warnings:
detect: False
trace_file: trace_files/empty.scap
rules_file: rules/falco_rules_warnings.yaml
rules_warning:
- no_evttype
- evttype_not_equals
- leading_not
- not_equals_at_end
- not_at_end
- not_equals_and_not
- leading_in_not_equals_at_evttype
- not_with_evttypes
- not_with_evttypes_addl
validate_rules_file: rules/falco_rules_warnings.yaml
validate_warnings:
- item_type: rule
item_name: no_evttype
code: LOAD_NO_EVTTYPE
message: "Rule matches too many evt.type values. This has a significant performance penalty."
- item_type: rule
item_name: evttype_not_equals
code: LOAD_NO_EVTTYPE
message: "Rule matches too many evt.type values. This has a significant performance penalty."
- item_type: rule
item_name: leading_not
code: LOAD_NO_EVTTYPE
message: "Rule matches too many evt.type values. This has a significant performance penalty."
- item_type: rule
item_name: not_equals_at_end
code: LOAD_NO_EVTTYPE
message: "Rule matches too many evt.type values. This has a significant performance penalty."
- item_type: rule
item_name: not_at_end
code: LOAD_NO_EVTTYPE
message: "Rule matches too many evt.type values. This has a significant performance penalty."
- item_type: rule
item_name: not_equals_and_not
code: LOAD_NO_EVTTYPE
message: "Rule matches too many evt.type values. This has a significant performance penalty."
- item_type: rule
item_name: leading_in_not_equals_at_evttype
code: LOAD_NO_EVTTYPE
message: "Rule matches too many evt.type values. This has a significant performance penalty."
- item_type: rule
item_name: not_with_evttypes
code: LOAD_NO_EVTTYPE
message: "Rule matches too many evt.type values. This has a significant performance penalty."
- item_type: rule
item_name: not_with_evttypes_addl
code: LOAD_NO_EVTTYPE
message: "Rule matches too many evt.type values. This has a significant performance penalty."
rules_events:
- no_warnings: [execve]
- no_evttype: [all]
@@ -251,159 +284,149 @@ trace_files: !mux
invalid_not_yaml:
exit_status: 1
stdout_is: |+
1 errors:
Rules content is not yaml
validate_errors:
- item_type: file
item_name: ""
code: LOAD_ERR_YAML_VALIDATE
message: "Rules content is not yaml"
validate_rules_file:
- rules/invalid_not_yaml.yaml
trace_file: trace_files/cat_write.scap
invalid_not_array:
exit_status: 1
stdout_is: |+
1 errors:
Rules content is not yaml array of objects
validate_errors:
- item_type: file
item_name: ""
code: LOAD_ERR_YAML_VALIDATE
message: "Rules content is not yaml array of objects"
validate_rules_file:
- rules/invalid_not_array.yaml
trace_file: trace_files/cat_write.scap
invalid_array_item_not_object:
exit_status: 1
stdout_is: |+
1 errors:
Unexpected element type. Each element should be a yaml associative array.
---
- foo
---
validate_errors:
- item_type: item
item_name: ""
code: LOAD_ERR_YAML_VALIDATE
message: "Unexpected element type. Each element should be a yaml associative array."
validate_rules_file:
- rules/invalid_array_item_not_object.yaml
trace_file: trace_files/cat_write.scap
invalid_engine_version_not_number:
exit_status: 1
stdout_is: |+
1 errors:
Value of required_engine_version must be a number
---
- required_engine_version: not-a-number
---
validate_errors:
- item_type: required_engine_version
item_name: ""
code: LOAD_ERR_YAML_VALIDATE
message: "Can't decode YAML scalar value"
validate_rules_file:
- rules/invalid_engine_version_not_number.yaml
trace_file: trace_files/cat_write.scap
invalid_yaml_parse_error:
exit_status: 1
validate_errors:
- item_type: file
item_name: ""
code: LOAD_ERR_YAML_PARSE
message: "yaml-cpp: error at line 1, column 11: illegal map value"
validate_rules_file:
- rules/invalid_yaml_parse_error.yaml
trace_file: trace_files/cat_write.scap
invalid_list_without_items:
exit_status: 1
stdout_is: |+
1 errors:
List must have property items
---
- list: bad_list
no_items: foo
---
validate_errors:
- item_type: list
item_name: bad_list
code: LOAD_ERR_YAML_VALIDATE
message: "Item has no mapping for key 'items'"
validate_rules_file:
- rules/invalid_list_without_items.yaml
trace_file: trace_files/cat_write.scap
invalid_macro_without_condition:
exit_status: 1
stdout_is: |+
1 errors:
Macro must have property condition
---
- macro: bad_macro
nope: 1
---
validate_errors:
- item_type: macro
item_name: bad_macro
code: LOAD_ERR_YAML_VALIDATE
message: "Item has no mapping for key 'condition'"
validate_rules_file:
- rules/invalid_macro_without_condition.yaml
trace_file: trace_files/cat_write.scap
invalid_rule_without_output:
exit_status: 1
stdout_is: |+
1 errors:
Rule must have properties 'condition', 'output', 'desc', and 'priority'
---
- rule: no output rule
desc: some desc
condition: evt.type=fork
priority: INFO
---
validate_errors:
- item_type: rule
item_name: no output rule
code: LOAD_ERR_YAML_VALIDATE
message: "Item has no mapping for key 'output'"
validate_rules_file:
- rules/invalid_rule_without_output.yaml
trace_file: trace_files/cat_write.scap
invalid_append_rule_without_condition:
exit_status: 1
stdout_is: |+
1 errors:
Appended rule must have exceptions or condition property
---
- rule: no condition rule
append: true
---
validate_errors:
- item_type: rule
item_name: no condition rule
code: LOAD_ERR_VALIDATE
message: "Appended rule must have exceptions or condition property"
validate_rules_file:
- rules/invalid_append_rule_without_condition.yaml
trace_file: trace_files/cat_write.scap
invalid_append_macro_dangling:
exit_status: 1
stdout_is: |+
1 errors:
Macro dangling append has 'append' key but no macro by that name already exists
---
- macro: dangling append
condition: and evt.type=execve
append: true
---
validate_errors:
- item_type: macro
item_name: dangling append
code: LOAD_ERR_VALIDATE
message: "Macro has 'append' key but no macro by that name already exists"
validate_rules_file:
- rules/invalid_append_macro_dangling.yaml
trace_file: trace_files/cat_write.scap
invalid_list_append_dangling:
exit_status: 1
stdout_is: |+
1 errors:
List my_list has 'append' key but no list by that name already exists
---
- list: my_list
items: [not-cat]
append: true
---
validate_errors:
- item_type: list
item_name: my_list
code: LOAD_ERR_VALIDATE
message: "List has 'append' key but no list by that name already exists"
validate_rules_file:
- rules/list_append_failure.yaml
trace_file: trace_files/cat_write.scap
invalid_rule_append_dangling:
exit_status: 1
stdout_is: |+
1 errors:
Rule my_rule has 'append' key but no rule by that name already exists
---
- rule: my_rule
condition: evt.type=open
append: true
---
validate_errors:
- item_type: rule
item_name: my_rule
code: LOAD_ERR_VALIDATE
message: "Rule has 'append' key but no rule by that name already exists"
validate_rules_file:
- rules/rule_append_failure.yaml
trace_file: trace_files/cat_write.scap
invalid_overwrite_macro:
exit_status: 1
stdout_contains: |+
.*invalid_base_macro.yaml: Ok
.*invalid_overwrite_macro.yaml: 1 errors:
Compilation error when compiling "foo": Undefined macro 'foo' used in filter.
---
- macro: some macro
condition: foo
append: false
---
validate_ok: [invalid_base_macro.yaml]
validate_errors:
- item_type: macro
item_name: some macro
code: LOAD_ERR_VALIDATE
message: "Undefined macro 'foo' used in filter."
validate_warnings:
- item_type: macro
item_name: some macro
code: LOAD_UNUSED_MACRO
message: "Macro not referred to by any other rule/macro"
validate_rules_file:
- rules/invalid_base_macro.yaml
- rules/invalid_overwrite_macro.yaml
@@ -411,18 +434,17 @@ trace_files: !mux
invalid_append_macro:
exit_status: 1
stdout_contains: |+
.*invalid_base_macro.yaml: Ok
.*invalid_append_macro.yaml: 1 errors:
Compilation error when compiling "evt.type=execve foo": 17: unexpected token after 'execve', expecting 'or', 'and'
---
- macro: some macro
condition: evt.type=execve
- macro: some macro
condition: foo
append: true
---
validate_ok: [invalid_base_macro.yaml]
validate_errors:
- item_type: macro
item_name: some macro
code: LOAD_ERR_COMPILE_CONDITION
message: "unexpected token after 'execve', expecting 'or', 'and'"
validate_warnings:
- item_type: macro
item_name: some macro
code: LOAD_UNUSED_MACRO
message: "Macro not referred to by any other rule/macro"
validate_rules_file:
- rules/invalid_base_macro.yaml
- rules/invalid_append_macro.yaml
@@ -430,49 +452,39 @@ trace_files: !mux
invalid_overwrite_macro_multiple_docs:
exit_status: 1
stdout_is: |+
1 errors:
Compilation error when compiling "foo": Undefined macro 'foo' used in filter.
---
- macro: some macro
condition: foo
append: false
---
validate_errors:
- item_type: macro
item_name: some macro
code: LOAD_ERR_VALIDATE
message: "Undefined macro 'foo' used in filter."
validate_warnings:
- item_type: macro
item_name: some macro
code: LOAD_UNUSED_MACRO
message: "Macro not referred to by any other rule/macro"
validate_rules_file:
- rules/invalid_overwrite_macro_multiple_docs.yaml
trace_file: trace_files/cat_write.scap
invalid_append_macro_multiple_docs:
exit_status: 1
stdout_is: |+
1 errors:
Compilation error when compiling "evt.type=execve foo": 17: unexpected token after 'execve', expecting 'or', 'and'
---
- macro: some macro
condition: evt.type=execve
- macro: some macro
condition: foo
append: true
---
validate_errors:
- item_type: macro
item_name: some macro
code: LOAD_ERR_COMPILE_CONDITION
message: "unexpected token after 'execve', expecting 'or', 'and'"
validate_rules_file:
- rules/invalid_append_macro_multiple_docs.yaml
trace_file: trace_files/cat_write.scap
invalid_overwrite_rule:
exit_status: 1
stdout_contains: |+
.*invalid_base_rule.yaml: Ok
.*invalid_overwrite_rule.yaml: 1 errors:
Undefined macro 'bar' used in filter.
---
- rule: some rule
desc: some desc
condition: bar
output: some output
priority: INFO
append: false
---
validate_ok: [invalid_base_rule.yaml]
validate_errors:
- item_type: rule
item_name: some rule
code: LOAD_ERR_VALIDATE
message: "Undefined macro 'bar' used in filter."
validate_rules_file:
- rules/invalid_base_rule.yaml
- rules/invalid_overwrite_rule.yaml
@@ -480,24 +492,12 @@ trace_files: !mux
invalid_append_rule:
exit_status: 1
stdout_contains: |+
.*invalid_base_rule.yaml: Ok
.*invalid_append_rule.yaml: 1 errors:
Compilation error when compiling "evt.type=open bar": 15: unexpected token after 'open', expecting 'or', 'and'
---
- rule: some rule
desc: some desc
condition: evt.type=open
output: some output
priority: INFO
- rule: some rule
desc: some desc
condition: bar
output: some output
priority: INFO
append: true
---
validate_ok: [invalid_base_rule.yaml]
validate_errors:
- item_type: rule
item_name: some rule
code: LOAD_ERR_COMPILE_CONDITION
message: "unexpected token after 'open', expecting 'or', 'and'"
validate_rules_file:
- rules/invalid_base_rule.yaml
- rules/invalid_append_rule.yaml
@@ -505,96 +505,66 @@ trace_files: !mux
invalid_overwrite_rule_multiple_docs:
exit_status: 1
stdout_is: |+
1 errors:
Undefined macro 'bar' used in filter.
---
- rule: some rule
desc: some desc
condition: bar
output: some output
priority: INFO
append: false
---
validate_errors:
- item_type: rule
item_name: some rule
code: LOAD_ERR_VALIDATE
message: "Undefined macro 'bar' used in filter."
validate_rules_file:
- rules/invalid_overwrite_rule_multiple_docs.yaml
trace_file: trace_files/cat_write.scap
invalid_append_rule_multiple_docs:
exit_status: 1
stdout_contains: |+
Compilation error when compiling "evt.type=open bar": 15: unexpected token after 'open', expecting 'or', 'and'
---
- rule: some rule
desc: some desc
condition: evt.type=open
output: some output
priority: INFO
- rule: some rule
desc: some desc
condition: bar
output: some output
priority: INFO
append: true
---
validate_errors:
- item_type: rule
item_name: some rule
code: LOAD_ERR_COMPILE_CONDITION
message: "unexpected token after 'open', expecting 'or', 'and'"
validate_rules_file:
- rules/invalid_append_rule_multiple_docs.yaml
trace_file: trace_files/cat_write.scap
invalid_missing_rule_name:
exit_status: 1
stdout_is: |+
1 errors:
Rule name is empty
---
- rule:
desc: some desc
condition: evt.type=execve
output: some output
---
validate_errors:
- item_type: rule
item_name: ""
code: LOAD_ERR_YAML_VALIDATE
message: "Mapping for key 'rule' is empty"
validate_rules_file:
- rules/invalid_missing_rule_name.yaml
trace_file: trace_files/cat_write.scap
invalid_missing_list_name:
exit_status: 1
stdout_is: |+
1 errors:
List name is empty
---
- list:
items: [foo]
---
validate_errors:
- item_type: list
item_name: ""
code: LOAD_ERR_YAML_VALIDATE
message: "Mapping for key 'list' is empty"
validate_rules_file:
- rules/invalid_missing_list_name.yaml
trace_file: trace_files/cat_write.scap
invalid_missing_macro_name:
exit_status: 1
stdout_is: |+
1 errors:
Macro name is empty
---
- macro:
condition: evt.type=execve
---
validate_errors:
- item_type: macro
item_name: ""
code: LOAD_ERR_YAML_VALIDATE
message: "Mapping for key 'macro' is empty"
validate_rules_file:
- rules/invalid_missing_macro_name.yaml
trace_file: trace_files/cat_write.scap
invalid_rule_output:
exit_status: 1
stdout_is: |+
1 errors:
Invalid output format 'An open was seen %not_a_real_field': 'invalid formatting token not_a_real_field'
---
- rule: rule_with_invalid_output
desc: A rule with an invalid output field
condition: evt.type=open
output: "An open was seen %not_a_real_field"
priority: WARNING
---
validate_errors:
- item_type: rule
item_name: rule_with_invalid_output
code: LOAD_ERR_COMPILE_OUTPUT
message: "invalid formatting token not_a_real_field"
validate_rules_file:
- rules/invalid_rule_output.yaml
trace_file: trace_files/cat_write.scap
@@ -622,13 +592,13 @@ trace_files: !mux
rules_file:
- rules/single_rule_enabled_flag.yaml
trace_file: trace_files/cat_write.scap
disabled_rule_using_false_enabled_flag_only:
detect: False
rules_file:
- rules/disabled_rule_using_enabled_flag_only.yaml
trace_file: trace_files/cat_write.scap
enabled_rule_using_false_enabled_flag_only:
detect: True
detect_level: WARNING
@@ -1025,13 +995,6 @@ trace_files: !mux
- open_12: 0
- open_13: 0
list_append_failure:
exit_status: 1
stderr_contains: "List my_list has 'append' key but no list by that name already exists"
rules_file:
- rules/list_append_failure.yaml
trace_file: trace_files/cat_write.scap
list_append:
detect: True
detect_level: WARNING
@@ -1045,13 +1008,6 @@ trace_files: !mux
- rules/list_append_false.yaml
trace_file: trace_files/cat_write.scap
macro_append_failure:
exit_status: 1
stderr_contains: "Macro my_macro has 'append' key but no macro by that name already exists"
rules_file:
- rules/macro_append_failure.yaml
trace_file: trace_files/cat_write.scap
macro_append:
detect: True
detect_level: WARNING
@@ -1065,13 +1021,6 @@ trace_files: !mux
- rules/macro_append_false.yaml
trace_file: trace_files/cat_write.scap
rule_append_failure:
exit_status: 1
stderr_contains: "Rule my_rule has 'append' key but no rule by that name already exists"
rules_file:
- rules/rule_append_failure.yaml
trace_file: trace_files/cat_write.scap
rule_append_skipped:
detect: False
priority: ERROR
@@ -1152,10 +1101,18 @@ trace_files: !mux
dev_null: 0
trace_file: trace_files/cat_write.scap
skip_unknown_noevt:
validate_skip_unknown_noevt:
validate_warnings:
- item_type: rule
item_name: "Contains Unknown Event And Skipping"
code: LOAD_UNKNOWN_FIELD
message: "filter_check called with nonexistent field proc.nobody"
validate_rules_file:
- rules/skip_unknown_evt.yaml
trace_file: trace_files/cat_write.scap
detect_skip_unknown_noevt:
detect: False
rules_warning:
- Contains Unknown Event And Skipping
rules_file:
- rules/skip_unknown_evt.yaml
trace_file: trace_files/cat_write.scap
@@ -1168,41 +1125,34 @@ trace_files: !mux
skip_unknown_error:
exit_status: 1
stderr_contains: |+
Could not load rules file.*skip_unknown_error.yaml: 1 errors:
Rule Contains Unknown Event And Not Skipping: error filter_check called with nonexistent field proc.nobody
---
- rule: Contains Unknown Event And Not Skipping
desc: Contains an unknown event
condition: proc.nobody=cat
output: Never
skip-if-unknown-filter: false
priority: INFO
---
rules_file:
validate_errors:
- item_type: rule
item_name: "Contains Unknown Event And Not Skipping"
code: LOAD_ERR_COMPILE_CONDITION
message: "filter_check called with nonexistent field proc.nobody"
validate_rules_file:
- rules/skip_unknown_error.yaml
trace_file: trace_files/cat_write.scap
skip_unknown_unspec_error:
exit_status: 1
stderr_contains: |+
Could not load rules file .*skip_unknown_unspec.yaml: 1 errors:
Rule Contains Unknown Event And Unspecified: error filter_check called with nonexistent field proc.nobody
---
- rule: Contains Unknown Event And Unspecified
desc: Contains an unknown event
condition: proc.nobody=cat
output: Never
priority: INFO
---
rules_file:
validate_errors:
- item_type: rule
item_name: "Contains Unknown Event And Unspecified"
code: LOAD_ERR_COMPILE_CONDITION
message: "filter_check called with nonexistent field proc.nobody"
validate_rules_file:
- rules/skip_unknown_unspec.yaml
trace_file: trace_files/cat_write.scap
engine_version_mismatch:
exit_status: 1
stderr_contains: Rules require engine version 9999999, but engine version is
rules_file:
validate_errors:
- item_type: required_engine_version
item_name: ""
code: LOAD_ERR_VALIDATE
message_contains: "Rules require engine version 9999999, but engine version is"
validate_rules_file:
- rules/engine_version_mismatch.yaml
trace_file: trace_files/cat_write.scap

185
test/falco_tests_exceptions.yaml
View File

@@ -20,175 +20,99 @@ trace_files: !mux
rule_exception_no_fields:
exit_status: 1
stdout_is: |+
1 errors:
Rule exception item ex1: must have fields property with a list of fields
---
- rule: My Rule
desc: Some desc
condition: evt.type=open and proc.name=cat
output: Some output
exceptions:
- name: ex1
priority: error
---
validate_errors:
- item_type: exception
item_name: ex1
code: LOAD_ERR_YAML_VALIDATE
message: "Item has no mapping for key 'fields'"
validate_rules_file:
- rules/exceptions/item_no_fields.yaml
trace_file: trace_files/cat_write.scap
rule_exception_no_name:
exit_status: 1
stdout_is: |+
1 errors:
Rule exception item must have name property
---
- rule: My Rule
desc: Some desc
condition: evt.type=open and proc.name=cat
output: Some output
exceptions:
- fields: [proc.name, fd.filename]
priority: error
---
validate_errors:
- item_type: exception
item_name: ""
code: LOAD_ERR_YAML_VALIDATE
message: "Item has no mapping for key 'name'"
validate_rules_file:
- rules/exceptions/item_no_name.yaml
trace_file: trace_files/cat_write.scap
rule_exception_append_no_name:
exit_status: 1
stdout_is: |+
1 errors:
Rule exception item must have name property
---
- rule: My Rule
exceptions:
- values:
- [nginx, /tmp/foo]
append: true
---
validate_errors:
- item_type: exception
item_name: ""
code: LOAD_ERR_YAML_VALIDATE
message: "Item has no mapping for key 'name'"
validate_rules_file:
- rules/exceptions/append_item_no_name.yaml
trace_file: trace_files/cat_write.scap
rule_exception_unknown_fields:
exit_status: 1
stdout_is: |+
1 errors:
Rule exception item ex1: field name not.exist is not a supported filter field
---
- rule: My Rule
desc: Some desc
condition: evt.type=open and proc.name=cat
output: Some output
exceptions:
- name: ex1
fields: [not.exist]
priority: error
---
validate_errors:
- item_type: exception
item_name: ex1
code: LOAD_ERR_VALIDATE
message: "'not.exist' is not a supported filter field"
validate_rules_file:
- rules/exceptions/item_unknown_fields.yaml
trace_file: trace_files/cat_write.scap
rule_exception_comps_fields_len_mismatch:
exit_status: 1
stdout_is: |+
1 errors:
Rule exception item ex1: fields and comps lists must have equal length
---
- rule: My Rule
desc: Some desc
condition: evt.type=open and proc.name=cat
output: Some output
exceptions:
- name: ex1
fields: [proc.name, fd.filename]
comps: [=]
priority: error
---
validate_errors:
- item_type: exception
item_name: ex1
code: LOAD_ERR_VALIDATE
message: "Fields and comps lists must have equal length"
validate_rules_file:
- rules/exceptions/item_comps_fields_len_mismatch.yaml
trace_file: trace_files/cat_write.scap
rule_exception_unknown_comp:
exit_status: 1
stdout_is: |+
1 errors:
Rule exception item ex1: comparison operator no-comp is not a supported comparison operator
---
- rule: My Rule
desc: Some desc
condition: evt.type=open and proc.name=cat
output: Some output
exceptions:
- name: ex1
fields: [proc.name, fd.filename]
comps: [=, no-comp]
priority: error
---
validate_errors:
- item_type: exception
item_name: ex1
code: LOAD_ERR_VALIDATE
message: "'no-comp' is not a supported comparison operator"
validate_rules_file:
- rules/exceptions/item_unknown_comp.yaml
trace_file: trace_files/cat_write.scap
rule_exception_fields_values_len_mismatch:
exit_status: 1
stdout_is: |+
1 errors:
Exception item ex1: fields and values lists must have equal length
---
- rule: My Rule
desc: Some desc
condition: evt.type=open and proc.name=cat
output: Some output
exceptions:
- name: ex1
fields: [proc.name, fd.filename]
values:
- [nginx]
priority: error
---
validate_errors:
- item_type: exception
item_name: ex1
code: LOAD_ERR_VALIDATE
message: "Fields and values lists must have equal length"
validate_rules_file:
- rules/exceptions/item_fields_values_len_mismatch.yaml
trace_file: trace_files/cat_write.scap
rule_exception_append_fields_values_len_mismatch:
exit_status: 1
stdout_is: |+
1 errors:
Exception item ex1: fields and values lists must have equal length
---
- rule: My Rule
desc: Some desc
condition: evt.type=open and proc.name=cat
output: Some output
exceptions:
- name: ex1
fields: [proc.name, fd.filename]
priority: error
- rule: My Rule
exceptions:
- name: ex1
values:
- [nginx]
append: true
---
validate_errors:
- item_type: exception
item_name: ex1
code: LOAD_ERR_VALIDATE
message: "Fields and values lists must have equal length"
validate_rules_file:
- rules/exceptions/append_item_fields_values_len_mismatch.yaml
trace_file: trace_files/cat_write.scap
rule_exception_append_item_not_in_rule:
exit_status: 1
stdout_is: |+
1 errors:
Rule exception new item ex2: must have fields property with a list of fields
---
- rule: My Rule
exceptions:
- name: ex2
values:
- [apache, /tmp]
append: true
---
validate_errors:
- item_type: exception
item_name: ex2
code: LOAD_ERR_VALIDATE
message: "Rule exception must have fields property with a list of fields"
validate_rules_file:
- rules/exceptions/append_item_not_in_rule.yaml
trace_file: trace_files/cat_write.scap
@@ -325,7 +249,7 @@ trace_files: !mux
rules_file:
- rules/exceptions/rule_exception_new_single_field_append.yaml
trace_file: trace_files/cat_write.scap
rule_exception_new_second_field_append:
detect: False
detect_level: WARNING
@@ -335,18 +259,11 @@ trace_files: !mux
rule_exception_new_append_no_field:
exit_status: 1
stdout_is: |+
1 errors:
Rule exception new item proc_cmdline: must have fields property with a list of fields
---
- rule: Open From Cat
exceptions:
- name: proc_cmdline
comps: in
values:
- "cat /dev/null"
append: true
---
validate_errors:
- item_type: exception
item_name: proc_cmdline
code: LOAD_ERR_VALIDATE
message: "Rule exception must have fields property with a list of fields"
validate_rules_file:
- rules/exceptions/rule_exception_new_no_field_append.yaml
trace_file: trace_files/cat_write.scap

27
test/falco_tests_plugins.yaml
View File

@@ -95,21 +95,24 @@ trace_files: !mux
- rules/plugins/cloudtrail_incompat_plugin_version.yaml
no_plugins_unknown_source:
detect: False
rules_file:
exit_status: 0
validate_warnings:
- item_type: rule
item_name: Cloudtrail Create Instance
code: LOAD_UNKNOWN_SOURCE
message: "Unknown source aws_cloudtrail, skipping"
validate_rules_file:
- rules/plugins/cloudtrail_create_instances.yaml
trace_file: trace_files/empty.scap
rules_warning:
- Cloudtrail Create Instance
stderr_contains: "Rule Cloudtrail Create Instance: warning .unknown-source.: unknown source aws_cloudtrail, skipping"
no_plugins_unknown_source_rule_exception:
detect: False
rules_file:
exit_status: 0
validate_warnings:
- item_type: rule
item_name: Cloudtrail Create Instance
code: LOAD_UNKNOWN_SOURCE
message: "Unknown source aws_cloudtrail, skipping"
validate_rules_file:
- rules/plugins/cloudtrail_create_instances_exceptions.yaml
trace_file: trace_files/empty.scap
rules_warning:
- Cloudtrail Create Instance
stderr_contains: "Rule Cloudtrail Create Instance: warning .unknown-source.: unknown source aws_cloudtrail, skipping"

19
test/rules/macro_append_failure.yaml
View File

@@ -1,19 +0,0 @@
#
# Copyright (C) 2019 The Falco Authors.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
- macro: my_macro
condition: proc.name=not-cat
append: true