From 01f65e3baeaa1e2b75d0bdd73d9c8f2e686f6092 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Fri, 5 Jul 2019 15:42:52 -0700 Subject: [PATCH] Add new tests for validating rules files Add a bunch of additional test cases for validating rules files. Each has a specific kind of parse failure and checks for the appropriate error info on stdout. Signed-off-by: Mark Stemm --- test/falco_tests.yaml | 199 +++++++++++++++++- test/rules/invalid_append_macro_dangling.yaml | 3 + ...invalid_append_rule_without_condition.yaml | 2 + test/rules/invalid_array_item_not_object.yaml | 1 + test/rules/invalid_condition_not_rule.yaml | 5 + .../invalid_engine_version_not_number.yaml | 34 +++ test/rules/invalid_list_without_items.yaml | 5 + test/rules/invalid_macro_comple_error.yaml | 2 + .../invalid_macro_without_condition.yaml | 6 + test/rules/invalid_missing_list_name.yaml | 2 + test/rules/invalid_missing_macro_name.yaml | 2 + test/rules/invalid_missing_rule_name.yaml | 4 + test/rules/invalid_not_array.yaml | 1 + test/rules/invalid_not_yaml.yaml | 1 + test/rules/invalid_rule_without_output.yaml | 4 + test/rules/invalid_unexpected_object.yaml | 1 + test/rules/invalid_yaml_parse_error.yaml | 1 + 17 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 test/rules/invalid_append_macro_dangling.yaml create mode 100644 test/rules/invalid_append_rule_without_condition.yaml create mode 100644 test/rules/invalid_array_item_not_object.yaml create mode 100644 test/rules/invalid_condition_not_rule.yaml create mode 100644 test/rules/invalid_engine_version_not_number.yaml create mode 100644 test/rules/invalid_list_without_items.yaml create mode 100644 test/rules/invalid_macro_comple_error.yaml create mode 100644 test/rules/invalid_macro_without_condition.yaml create mode 100644 test/rules/invalid_missing_list_name.yaml create mode 100644 test/rules/invalid_missing_macro_name.yaml create mode 100644 test/rules/invalid_missing_rule_name.yaml create mode 100644 test/rules/invalid_not_array.yaml create mode 100644 test/rules/invalid_not_yaml.yaml create mode 100644 test/rules/invalid_rule_without_output.yaml create mode 100644 test/rules/invalid_unexpected_object.yaml create mode 100644 test/rules/invalid_yaml_parse_error.yaml diff --git a/test/falco_tests.yaml b/test/falco_tests.yaml index 5a7c0598..aecc1a9a 100644 --- a/test/falco_tests.yaml +++ b/test/falco_tests.yaml @@ -238,6 +238,199 @@ trace_files: !mux - rules/endswith.yaml trace_file: trace_files/cat_write.scap + invalid_not_yaml: + exit_status: 1 + stdout_is: |+ + Rules content is not yaml + --- + This is not yaml + --- + validate_rules_file: + - rules/invalid_not_yaml.yaml + trace_file: trace_files/cat_write.scap + + invalid_not_array: + exit_status: 1 + stdout_is: |+ + Rules content is not yaml array of objects + --- + foo: bar + --- + validate_rules_file: + - rules/invalid_not_array.yaml + trace_file: trace_files/cat_write.scap + + invalid_array_item_not_object: + exit_status: 1 + stdout_is: |+ + Unexpected element of type string. Each element should be a yaml associative array. + --- + - foo + --- + validate_rules_file: + - rules/invalid_array_item_not_object.yaml + trace_file: trace_files/cat_write.scap + + invalid_unexpected object: + exit_status: 1 + stdout_is: |+ + Unknown rule object: {foo="bar"} + --- + - foo: bar + --- + validate_rules_file: + - rules/invalid_unexpected_object.yaml + trace_file: trace_files/cat_write.scap + + invalid_engine_version_not_number: + exit_status: 1 + stdout_is: |+ + Value of required_engine_version must be a number + --- + - required_engine_version: not-a-number + --- + validate_rules_file: + - rules/invalid_engine_version_not_number.yaml + trace_file: trace_files/cat_write.scap + + invalid_yaml_parse_error: + exit_status: 1 + stdout_is: |+ + mapping values are not allowed in this context + --- + this : is : not : yaml + --- + validate_rules_file: + - rules/invalid_yaml_parse_error.yaml + trace_file: trace_files/cat_write.scap + + invalid_list_without_items: + exit_status: 1 + stdout_is: |+ + List must have property items + --- + - list: bad_list + no_items: foo + --- + validate_rules_file: + - rules/invalid_list_without_items.yaml + trace_file: trace_files/cat_write.scap + + invalid_macro_without_condition: + exit_status: 1 + stdout_is: |+ + Macro must have property condition + --- + - macro: bad_macro + nope: 1 + --- + validate_rules_file: + - rules/invalid_macro_without_condition.yaml + trace_file: trace_files/cat_write.scap + + invalid_rule_without_output: + exit_status: 1 + stdout_is: |+ + Rule must have property output + --- + - rule: no output rule + desc: some desc + condition: evt.type=fork + priority: INFO + --- + validate_rules_file: + - rules/invalid_rule_without_output.yaml + trace_file: trace_files/cat_write.scap + + invalid_append_rule_without_condition: + exit_status: 1 + stdout_is: |+ + Rule must have property condition + --- + - rule: no condition rule + append: true + --- + validate_rules_file: + - rules/invalid_append_rule_without_condition.yaml + trace_file: trace_files/cat_write.scap + + invalid_append_macro_dangling: + exit_status: 1 + stdout_is: |+ + Macro dangling append has 'append' key but no macro by that name already exists + --- + - macro: dangling append + condition: and evt.type=execve + append: true + --- + validate_rules_file: + - rules/invalid_append_macro_dangling.yaml + trace_file: trace_files/cat_write.scap + + invalid_list_append_dangling: + exit_status: 1 + stdout_is: |+ + List my_list has 'append' key but no list by that name already exists + --- + - list: my_list + items: [not-cat] + append: true + --- + validate_rules_file: + - rules/list_append_failure.yaml + trace_file: trace_files/cat_write.scap + + invalid_rule_append_dangling: + exit_status: 1 + stdout_is: |+ + Rule my_rule has 'append' key but no rule by that name already exists + --- + - rule: my_rule + condition: evt.type=open + append: true + --- + validate_rules_file: + - rules/rule_append_failure.yaml + trace_file: trace_files/cat_write.scap + + invalid_missing_rule_name: + exit_status: 1 + stdout_is: |+ + Rule name is empty + --- + - rule: + desc: some desc + condition: evt.type=execve + output: some output + --- + validate_rules_file: + - rules/invalid_missing_rule_name.yaml + trace_file: trace_files/cat_write.scap + + invalid_missing_list_name: + exit_status: 1 + stdout_is: |+ + List name is empty + --- + - list: + items: [foo] + --- + validate_rules_file: + - rules/invalid_missing_list_name.yaml + trace_file: trace_files/cat_write.scap + + invalid_missing_macro_name: + exit_status: 1 + stdout_is: |+ + Macro name is empty + --- + - macro: + condition: evt.type=execve + --- + validate_rules_file: + - rules/invalid_missing_macro_name.yaml + trace_file: trace_files/cat_write.scap + invalid_rule_output: exit_status: 1 stderr_contains: "Runtime error: Error loading rules:.* Invalid output format 'An open was seen %not_a_real_field': 'invalid formatting token not_a_real_field'. Exiting." @@ -601,7 +794,7 @@ trace_files: !mux list_append_failure: exit_status: 1 - stderr_contains: "List my_list has 'append' key but no list by that name already exists. Exiting" + stderr_contains: "List my_list has 'append' key but no list by that name already exists" rules_file: - rules/list_append_failure.yaml trace_file: trace_files/cat_write.scap @@ -621,7 +814,7 @@ trace_files: !mux macro_append_failure: exit_status: 1 - stderr_contains: "Macro my_macro has 'append' key but no macro by that name already exists. Exiting" + stderr_contains: "Macro my_macro has 'append' key but no macro by that name already exists" rules_file: - rules/macro_append_failure.yaml trace_file: trace_files/cat_write.scap @@ -641,7 +834,7 @@ trace_files: !mux rule_append_failure: exit_status: 1 - stderr_contains: "Rule my_rule has 'append' key but no rule by that name already exists. Exiting" + stderr_contains: "Rule my_rule has 'append' key but no rule by that name already exists" rules_file: - rules/rule_append_failure.yaml trace_file: trace_files/cat_write.scap diff --git a/test/rules/invalid_append_macro_dangling.yaml b/test/rules/invalid_append_macro_dangling.yaml new file mode 100644 index 00000000..3348a7f1 --- /dev/null +++ b/test/rules/invalid_append_macro_dangling.yaml @@ -0,0 +1,3 @@ +- macro: dangling append + condition: and evt.type=execve + append: true \ No newline at end of file diff --git a/test/rules/invalid_append_rule_without_condition.yaml b/test/rules/invalid_append_rule_without_condition.yaml new file mode 100644 index 00000000..b9a4011f --- /dev/null +++ b/test/rules/invalid_append_rule_without_condition.yaml @@ -0,0 +1,2 @@ +- rule: no condition rule + append: true \ No newline at end of file diff --git a/test/rules/invalid_array_item_not_object.yaml b/test/rules/invalid_array_item_not_object.yaml new file mode 100644 index 00000000..a4b65cdd --- /dev/null +++ b/test/rules/invalid_array_item_not_object.yaml @@ -0,0 +1 @@ +- foo diff --git a/test/rules/invalid_condition_not_rule.yaml b/test/rules/invalid_condition_not_rule.yaml new file mode 100644 index 00000000..158eff61 --- /dev/null +++ b/test/rules/invalid_condition_not_rule.yaml @@ -0,0 +1,5 @@ +- rule: condition not rule + condition: + desc: some desc + output: some output + priority: INFO diff --git a/test/rules/invalid_engine_version_not_number.yaml b/test/rules/invalid_engine_version_not_number.yaml new file mode 100644 index 00000000..fa77b4a5 --- /dev/null +++ b/test/rules/invalid_engine_version_not_number.yaml @@ -0,0 +1,34 @@ +# +# Copyright (C) 2016-2018 Draios Inc dba Sysdig. +# +# This file is part of falco. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +- required_engine_version: not-a-number + +- list: cat_binaries + items: [cat] + +- list: cat_capable_binaries + items: [cat_binaries] + +- macro: is_cat + condition: proc.name in (cat_capable_binaries) + +- rule: open_from_cat + desc: A process named cat does an open + condition: evt.type=open and is_cat + output: "An open was seen (command=%proc.cmdline)" + priority: WARNING \ No newline at end of file diff --git a/test/rules/invalid_list_without_items.yaml b/test/rules/invalid_list_without_items.yaml new file mode 100644 index 00000000..e3d21438 --- /dev/null +++ b/test/rules/invalid_list_without_items.yaml @@ -0,0 +1,5 @@ +- list: good_list + items: [foo] + +- list: bad_list + no_items: foo \ No newline at end of file diff --git a/test/rules/invalid_macro_comple_error.yaml b/test/rules/invalid_macro_comple_error.yaml new file mode 100644 index 00000000..57bf69ad --- /dev/null +++ b/test/rules/invalid_macro_comple_error.yaml @@ -0,0 +1,2 @@ +- macro: macro with comp error + condition: gak diff --git a/test/rules/invalid_macro_without_condition.yaml b/test/rules/invalid_macro_without_condition.yaml new file mode 100644 index 00000000..5b528dc3 --- /dev/null +++ b/test/rules/invalid_macro_without_condition.yaml @@ -0,0 +1,6 @@ +- macro: bad_macro + nope: 1 + +- macro: good_macro + condition: evt.type=execve + diff --git a/test/rules/invalid_missing_list_name.yaml b/test/rules/invalid_missing_list_name.yaml new file mode 100644 index 00000000..90ed15f0 --- /dev/null +++ b/test/rules/invalid_missing_list_name.yaml @@ -0,0 +1,2 @@ +- list: + items: [foo] \ No newline at end of file diff --git a/test/rules/invalid_missing_macro_name.yaml b/test/rules/invalid_missing_macro_name.yaml new file mode 100644 index 00000000..77f102ab --- /dev/null +++ b/test/rules/invalid_missing_macro_name.yaml @@ -0,0 +1,2 @@ +- macro: + condition: evt.type=execve diff --git a/test/rules/invalid_missing_rule_name.yaml b/test/rules/invalid_missing_rule_name.yaml new file mode 100644 index 00000000..238f563f --- /dev/null +++ b/test/rules/invalid_missing_rule_name.yaml @@ -0,0 +1,4 @@ +- rule: + desc: some desc + condition: evt.type=execve + output: some output diff --git a/test/rules/invalid_not_array.yaml b/test/rules/invalid_not_array.yaml new file mode 100644 index 00000000..7daacd5d --- /dev/null +++ b/test/rules/invalid_not_array.yaml @@ -0,0 +1 @@ +foo: bar \ No newline at end of file diff --git a/test/rules/invalid_not_yaml.yaml b/test/rules/invalid_not_yaml.yaml new file mode 100644 index 00000000..2626d0a1 --- /dev/null +++ b/test/rules/invalid_not_yaml.yaml @@ -0,0 +1 @@ +This is not yaml \ No newline at end of file diff --git a/test/rules/invalid_rule_without_output.yaml b/test/rules/invalid_rule_without_output.yaml new file mode 100644 index 00000000..f337b1ad --- /dev/null +++ b/test/rules/invalid_rule_without_output.yaml @@ -0,0 +1,4 @@ +- rule: no output rule + desc: some desc + condition: evt.type=fork + priority: INFO diff --git a/test/rules/invalid_unexpected_object.yaml b/test/rules/invalid_unexpected_object.yaml new file mode 100644 index 00000000..3867010c --- /dev/null +++ b/test/rules/invalid_unexpected_object.yaml @@ -0,0 +1 @@ +- foo: bar diff --git a/test/rules/invalid_yaml_parse_error.yaml b/test/rules/invalid_yaml_parse_error.yaml new file mode 100644 index 00000000..210c5555 --- /dev/null +++ b/test/rules/invalid_yaml_parse_error.yaml @@ -0,0 +1 @@ +this : is : not : yaml \ No newline at end of file