From a582599778e4afda2a121d30d11d0317a21bcb8e Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 17 Sep 2020 18:21:00 -0700 Subject: [PATCH] Support exceptions properties on rules Support exceptions properties on rules as described in https://github.com/falcosecurity/falco/pull/1376. - When parsing rules, add an empty exceptions table if not specified. - If exceptions are specified, they must contain names and lists of fields, and optionally can contain lists of comps and lists of lists of values. - If comps are not specified, = is used. - If a rule has exceptions and append:true, add values to the original rule's exception values with the matching name. - It's a warning but not an error to have exception values with a name not matching any fields. After loading all rules, build the exception condition string based on any exceptions: - If an exception has a single value for the "fields" property, values are combined into a single set to build a condition string like "field cmp (val1, val2, ...)". - Otherwise, iterate through each rule's exception values, finding the matching field names (field1, field2, ...) and comp operators (cmp1, cmp2, ...), then iterating over the list of field values (val1a, val1b, ...), (val2a, val2b, ...), building up a string of the form: and not ((field1 cmp1 val1a and field2 cmp2 val1b and ...) or (field1 cmp1 val2a and field2 cmp2 val2b and ...)... )" - If a value is not already quoted and contains a space, quote it in the string. Signed-off-by: Mark Stemm --- userspace/engine/formats.cpp | 21 +- userspace/engine/lua/rule_loader.lua | 400 ++++++++++++++++++++++++--- 2 files changed, 369 insertions(+), 52 deletions(-) diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index 3a8d3889..33fb8bbe 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -60,25 +60,32 @@ int falco_formats::lua_formatter(lua_State *ls) { sinsp_evt_formatter *formatter; formatter = new sinsp_evt_formatter(s_inspector, format); + lua_pushnil(ls); lua_pushlightuserdata(ls, formatter); } else { json_event_formatter *formatter; formatter = new json_event_formatter(s_engine->json_factory(), format); + lua_pushnil(ls); lua_pushlightuserdata(ls, formatter); } } - catch(sinsp_exception &e) + catch(exception &e) { - luaL_error(ls, "Invalid output format '%s': '%s'", format.c_str(), e.what()); - } - catch(falco_exception &e) - { - luaL_error(ls, "Invalid output format '%s': '%s'", format.c_str(), e.what()); + std::ostringstream os; + + os << "Invalid output format '" + << format + << "': '" + << e.what() + << "'"; + + lua_pushstring(ls, os.str().c_str()); + lua_pushnil(ls); } - return 1; + return 2; } int falco_formats::lua_free_formatter(lua_State *ls) diff --git a/userspace/engine/lua/rule_loader.lua b/userspace/engine/lua/rule_loader.lua index 14cec263..c213bed8 100644 --- a/userspace/engine/lua/rule_loader.lua +++ b/userspace/engine/lua/rule_loader.lua @@ -126,6 +126,31 @@ function set_output(output_format, state) end end +-- This should be keep in sync with parser.lua +defined_comp_operators = { + ["="]=1, + ["=="] = 1, + ["!"] = 1, + ["<="] = 1, + [">="] = 1, + ["<"] = 1, + [">"] = 1, + ["contains"] = 1, + ["icontains"] = 1, + ["glob"] = 1, + ["startswith"] = 1, + ["endswith"] = 1, + ["in"] = 1, + ["intersects"] = 1, + ["pmatch"] = 1 +} + +defined_list_comp_operators = { + ["in"] = 1, + ["intersects"] = 1, + ["pmatch"] = 1 +} + -- Note that the rules_by_name and rules_by_idx refer to the same rule -- object. The by_name index is used for things like describing rules, -- and the by_idx index is used to map the relational node index back @@ -253,6 +278,27 @@ function get_lines(rules_lines, row, num_lines) return ret end +function quote_item(item) + + -- Add quotes if the string contains spaces and doesn't start/end + -- w/ quotes + if string.find(item, " ") then + if string.sub(item, 1, 1) ~= "'" and string.sub(item, 1, 1) ~= '"' then + item = "\""..item.."\"" + end + end + + return item +end + +function paren_item(item) + if string.sub(item, 1, 1) ~= "(" then + item = "("..item..")" + end + + return item +end + function build_error(rules_lines, row, num_lines, err) local ret = err.."\n---\n"..get_lines(rules_lines, row, num_lines).."---" @@ -264,6 +310,90 @@ function build_error_with_context(ctx, err) return {ret} end +function validate_exception_item_multi_fields(eitem, context) + + local name = eitem['name'] + local fields = eitem['fields'] + local values = eitem['values'] + local comps = eitem['comps'] + + if comps == nil then + comps = {} + for c=1,#fields do + table.insert(comps, "=") + end + eitem['comps'] = comps + else + if #fields ~= #comps then + return false, build_error_with_context(context, "Rule exception item "..name..": fields and comps lists must have equal length"), warnings + end + end + for k, fname in ipairs(fields) do + if not is_defined_filter(fname) then + return false, build_error_with_context(context, "Rule exception item "..name..": field name "..fname.." is not a supported filter field"), warnings + end + end + for k, comp in ipairs(comps) do + if defined_comp_operators[comp] == nil then + return false, build_error_with_context(context, "Rule exception item "..name..": comparison operator "..comp.." is not a supported comparison operator"), warnings + end + end +end + +function validate_exception_item_single_field(eitem, context) + + local name = eitem['name'] + local fields = eitem['fields'] + local values = eitem['values'] + local comps = eitem['comps'] + + if comps == nil then + eitem['comps'] = "in" + comps = eitem['comps'] + else + if type(fields) ~= "string" or type(comps) ~= "string" then + return false, build_error_with_context(context, "Rule exception item "..name..": fields and comps must both be strings"), warnings + end + end + if not is_defined_filter(fields) then + return false, build_error_with_context(context, "Rule exception item "..name..": field name "..fields.." is not a supported filter field"), warnings + end + if defined_comp_operators[comps] == nil then + return false, build_error_with_context(context, "Rule exception item "..name..": comparison operator "..comps.." is not a supported comparison operator"), warnings + end +end + +function is_defined_filter(filter) + if defined_noarg_filters[filter] ~= nil then + return true + else + bracket_idx = string.find(filter, "[", 1, true) + + if bracket_idx ~= nil then + subfilter = string.sub(filter, 1, bracket_idx-1) + + if defined_arg_filters[subfilter] ~= nil then + return true + end + end + + dot_idx = string.find(filter, ".", 1, true) + + while dot_idx ~= nil do + subfilter = string.sub(filter, 1, dot_idx-1) + + if defined_arg_filters[subfilter] ~= nil then + return true + end + + dot_idx = string.find(filter, ".", dot_idx+1, true) + end + end + + return false +end + + function load_rules_doc(rules_mgr, doc, load_state) local warnings = {} @@ -378,6 +508,10 @@ function load_rules_doc(rules_mgr, doc, load_state) return false, build_error_with_context(v['context'], "Rule name is empty"), warnings end + if (v['condition'] == nil and v['exceptions'] == nil) then + return false, build_error_with_context(v['context'], "Rule must have exceptions or condition property"), warnings + end + -- By default, if a rule's condition refers to an unknown -- filter like evt.type, etc the loader throws an error. if v['skip-if-unknown-filter'] == nil then @@ -388,6 +522,13 @@ function load_rules_doc(rules_mgr, doc, load_state) v['source'] = "syscall" end + -- Add an empty exceptions property to the rule if not + -- defined, but add a warning about defining one + if v['exceptions'] == nil then + warnings[#warnings + 1] = "Rule "..v['rule']..": consider adding an exceptions property to define supported exceptions fields" + v['exceptions'] = {} + end + -- Possibly append to the condition field of an existing rule append = false @@ -395,21 +536,95 @@ function load_rules_doc(rules_mgr, doc, load_state) append = v['append'] end - if append then + -- Validate the contents of the rule exception + if next(v['exceptions']) ~= nil then - -- For append rules, all you need is the condition - for j, field in ipairs({'condition'}) do - if (v[field] == nil) then - return false, build_error_with_context(v['context'], "Rule must have property "..field), warnings + -- This validation only applies if append=false. append=true validation is handled below + if append == false then + + for _, eitem in ipairs(v['exceptions']) do + + if eitem['name'] == nil then + return false, build_error_with_context(v['context'], "Rule exception item must have name property"), warnings + end + + if eitem['fields'] == nil then + return false, build_error_with_context(v['context'], "Rule exception item "..eitem['name']..": must have fields property with a list of fields"), warnings + end + + if eitem['values'] == nil then + -- An empty values array is okay + eitem['values'] = {} + end + + -- Different handling if the fields property is a single item vs a list + local valid, err + if type(eitem['fields']) == "table" then + valid, err = validate_exception_item_multi_fields(eitem, v['context']) + else + valid, err = validate_exception_item_single_field(eitem, v['context']) + end + + if valid == false then + return valid, err + end end end + end + + if append then if state.rules_by_name[v['rule']] == nil then if state.skipped_rules_by_name[v['rule']] == nil then return false, build_error_with_context(v['context'], "Rule " ..v['rule'].. " has 'append' key but no rule by that name already exists"), warnings end else - state.rules_by_name[v['rule']]['condition'] = state.rules_by_name[v['rule']]['condition'] .. " " .. v['condition'] + + if next(v['exceptions']) ~= nil then + + for _, eitem in ipairs(v['exceptions']) do + local name = eitem['name'] + local fields = eitem['fields'] + local comps = eitem['comps'] + + if name == nil then + return false, build_error_with_context(v['context'], "Rule exception item must have name property"), warnings + end + + -- You can't append exception fields or comps to a rule + if fields ~= nil then + return false, build_error_with_context(v['context'], "Can not append exception fields to existing rule, only values"), warnings + end + + if comps ~= nil then + return false, build_error_with_context(v['context'], "Can not append exception comps to existing rule, only values"), warnings + end + + -- You can append values. They are added to the + -- corresponding name, if it exists. If no + -- exception with that name exists, add a + -- warning. + if eitem['values'] ~= nil then + local found=false + for _, reitem in ipairs(state.rules_by_name[v['rule']]['exceptions']) do + if reitem['name'] == eitem['name'] then + found=true + for _, values in ipairs(eitem['values']) do + reitem['values'][#reitem['values'] + 1] = values + end + end + end + + if found == false then + warnings[#warnings + 1] = "Rule "..v['rule'].." with append=true: no set of fields matching name "..eitem['name'] + end + end + end + end + + if v['condition'] ~= nil then + state.rules_by_name[v['rule']]['condition'] = state.rules_by_name[v['rule']]['condition'] .. " " .. v['condition'] + end -- Add the current object to the context of the base rule state.rules_by_name[v['rule']]['context'] = state.rules_by_name[v['rule']]['context'].."\n"..v['context'] @@ -458,6 +673,97 @@ function load_rules_doc(rules_mgr, doc, load_state) return true, {}, warnings end +-- cond and not ((proc.name=apk and fd.directory=/usr/lib/alpine) or (proc.name=npm and fd.directory=/usr/node/bin) or ...) +function build_exception_condition_string_multi_fields(eitem) + + local fields = eitem['fields'] + local comps = eitem['comps'] + + local icond = "(" + + for i, values in ipairs(eitem['values']) do + + if #fields ~= #values then + return nil, "Exception item "..eitem['name']..": fields and values lists must have equal length" + end + + if icond ~= "(" then + icond=icond.." or " + end + + icond=icond.."(" + + for k=1,#fields do + if k > 1 then + icond=icond.." and " + end + local ival = values[k] + local istr = "" + + -- If ival is a table, express it as (titem1, titem2, etc) + if type(ival) == "table" then + istr = "(" + for _, item in ipairs(ival) do + if istr ~= "(" then + istr = istr..", " + end + istr = istr..quote_item(item) + end + istr = istr..")" + else + -- If the corresponding operator is one that works on lists, possibly add surrounding parentheses. + if defined_list_comp_operators[comps[k]] then + istr = paren_item(ival) + else + -- Quote the value if not already quoted + istr = quote_item(ival) + end + end + + icond = icond..fields[k].." "..comps[k].." "..istr + end + + icond=icond..")" + end + + icond = icond..")" + + -- Don't return a trivially empty condition string + if icond == "()" then + icond = "" + end + + return icond, nil + +end + +function build_exception_condition_string_single_field(eitem) + + local icond = "" + + for i, value in ipairs(eitem['values']) do + + if type(value) ~= "string" then + return "", "Expected values array for item "..eitem['name'].." to contain a list of strings" + end + + if icond == "" then + icond = "("..eitem['fields'].." "..eitem['comps'].." (" + else + icond = icond..", " + end + + icond = icond..quote_item(value) + end + + if icond ~= "" then + icond = icond.."))" + end + + return icond, nil + +end + -- Returns: -- - Load Result: bool -- - required engine version. will be nil when load result is false @@ -553,7 +859,7 @@ function load_rules(sinsp_lua_parser, -- the items and expand any references to the items in the list for i, item in ipairs(v['items']) do if (state.lists[item] == nil) then - items[#items+1] = item + items[#items+1] = quote_item(item) else for i, exp_item in ipairs(state.lists[item].items) do items[#items+1] = exp_item @@ -587,12 +893,40 @@ function load_rules(sinsp_lua_parser, local v = state.rules_by_name[name] + local econd = "" + + -- Turn exceptions into condition strings and add them to each + -- rule's condition + for _, eitem in ipairs(v['exceptions']) do + + local icond, err + if type(eitem['fields']) == "table" then + icond, err = build_exception_condition_string_multi_fields(eitem) + else + icond, err = build_exception_condition_string_single_field(eitem) + end + + if err ~= nil then + return false, nil, build_error_with_context(v['context'], err), warnings + end + + if icond ~= "" then + econd = econd.." and not "..icond + end + end + + if econd ~= "" then + state.rules_by_name[name]['compile_condition'] = "("..state.rules_by_name[name]['condition']..") "..econd + else + state.rules_by_name[name]['compile_condition'] = state.rules_by_name[name]['condition'] + end + warn_evttypes = true if v['warn_evttypes'] ~= nil then warn_evttypes = v['warn_evttypes'] end - local status, filter_ast, filters = compiler.compile_filter(v['rule'], v['condition'], + local status, filter_ast, filters = compiler.compile_filter(v['rule'], v['compile_condition'], state.macros, state.lists) if status == false then @@ -607,50 +941,22 @@ function load_rules(sinsp_lua_parser, sinsp_rule_utils.check_for_ignored_syscalls_events(filter_ast, 'rule', v['rule']) end - evttypes, syscallnums = sinsp_rule_utils.get_evttypes_syscalls(name, filter_ast, v['condition'], warn_evttypes, verbose) + evttypes, syscallnums = sinsp_rule_utils.get_evttypes_syscalls(name, filter_ast, v['compile_condition'], warn_evttypes, verbose) end -- If a filter in the rule doesn't exist, either skip the rule -- or raise an error, depending on the value of -- skip-if-unknown-filter. for filter, _ in pairs(filters) do - found = false - - if defined_noarg_filters[filter] ~= nil then - found = true - else - bracket_idx = string.find(filter, "[", 1, true) - - if bracket_idx ~= nil then - subfilter = string.sub(filter, 1, bracket_idx-1) - - if defined_arg_filters[subfilter] ~= nil then - found = true - end - end - - if not found then - dot_idx = string.find(filter, ".", 1, true) - - while dot_idx ~= nil do - subfilter = string.sub(filter, 1, dot_idx-1) - - if defined_arg_filters[subfilter] ~= nil then - found = true - break - end - - dot_idx = string.find(filter, ".", dot_idx+1, true) - end - end - end - - if not found then - msg = "rule \""..v['rule'].."\" contains unknown filter "..filter + if not is_defined_filter(filter) then + msg = "rule \""..v['rule'].."\": contains unknown filter "..filter warnings[#warnings + 1] = msg if not v['skip-if-unknown-filter'] then - error("Rule \""..v['rule'].."\" contains unknown filter "..filter) + return false, nil, build_error_with_context(v['context'], msg), warnings + else + print("Skipping "..msg) + goto next_rule end end end @@ -729,8 +1035,12 @@ function load_rules(sinsp_lua_parser, -- Ensure that the output field is properly formatted by -- creating a formatter from it. Any error will be thrown -- up to the top level. - formatter = formats.formatter(v['source'], v['output']) - formats.free_formatter(v['source'], formatter) + local err, formatter = formats.formatter(v['source'], v['output']) + if err == nil then + formats.free_formatter(v['source'], formatter) + else + return false, nil, build_error_with_context(v['context'], err), warnings + end else return false, nil, build_error_with_context(v['context'], "Unexpected type in load_rule: "..filter_ast.type), warnings end