Allow any macro/list/rule to be overridden

Allow any list/macro/rule to be overridden by a subsequent file. The
persistent state that lives across invocations of load_rules are the 3
arrays ordered_{list,macro,rule}_names, which have the
lists/macros/rules in the order in which they first appear, and tables
{rules,macros,lists}_by_name, which maps from a name to a yaml object.

With each call to load_rules, the set of loaded rules is reset and the
state of expanded lists, compiled macros, compiled rules, and rule
metadata are recreated from scratch, using the ordered_*_names arrays
and *_by_name tables. That way, any list/macro/rule can be redefined in
a subsequent file with new values.

userspace/engine/lua/rule_loader.lua

@@ -121,7 +121,16 @@ end
 -- object. The by_name index is used for things like describing rules,
 -- and the by_idx index is used to map the relational node index back
 -- to a rule.
-local state = {macros={}, lists={}, filter_ast=nil, rules_by_name={}, n_rules=0, rules_by_idx={}}
+local state = {macros={}, lists={}, filter_ast=nil, rules_by_name={}, macros_by_name={}, lists_by_name={},
+	       n_rules=0, rules_by_idx={}, ordered_rule_names={}, ordered_macro_names={}, ordered_list_names={}}
+
+local function reset_rules(rules_mgr)
+   falco_rules.clear_filters(rules_mgr)
+   state.n_rules = 0
+   state.rules_by_idx = {}
+   state.macros = {}
+   state.lists = {}
+end
 
 -- From http://lua-users.org/wiki/TableUtils
 --
@@ -178,17 +187,79 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
       error("Rules content \""..rules_content.."\" is not yaml")
    end
 
-   for i,v in ipairs(rules) do -- iterate over yaml list
+   -- Iterate over yaml list. In this pass, all we're doing is
+   -- populating the set of rules, macros, and lists. We're not
+   -- expanding/compiling anything yet. All that will happen in a
+   -- second pass
+   for i,v in ipairs(rules) do
 
       if (not (type(v) == "table")) then
 	 error ("Unexpected element of type " ..type(v)..". Each element should be a yaml associative array.")
       end
 
       if (v['macro']) then
-	 local ast = compiler.compile_macro(v['condition'], state.lists)
-	 state.macros[v['macro']] = ast.filter.value
+	 if state.macros_by_name[v['macro']] == nil then
+	    state.ordered_macro_names[#state.ordered_macro_names+1] = v['macro']
+	 end
+
+	 for i, field in ipairs({'condition'}) do
+	    if (v[field] == nil) then
+	       error ("Missing "..field.." in macro with name "..v['macro'])
+	    end
+	 end
+
+	 state.macros_by_name[v['macro']] = v
 
       elseif (v['list']) then
+
+	 if state.lists_by_name[v['list']] == nil then
+	    state.ordered_list_names[#state.ordered_list_names+1] = v['list']
+	 end
+
+	 for i, field in ipairs({'items'}) do
+	    if (v[field] == nil) then
+	       error ("Missing "..field.." in list with name "..v['list'])
+	    end
+	 end
+
+	 state.lists_by_name[v['list']] = v
+
+      elseif (v['rule']) then
+
+	 if (v['rule'] == nil or type(v['rule']) == "table") then
+	    error ("Missing name in rule")
+	 end
+
+	 for i, field in ipairs({'condition', 'output', 'desc', 'priority'}) do
+	    if (v[field] == nil) then
+	       error ("Missing "..field.." in rule with name "..v['rule'])
+	    end
+	 end
+
+	 -- Note that we can overwrite rules, but the rules are still
+	 -- loaded in the order in which they first appeared,
+	 -- potentially across multiple files.
+	 if state.rules_by_name[v['rule']] == nil then
+	    state.ordered_rule_names[#state.ordered_rule_names+1] = v['rule']
+	 end
+
+	 state.rules_by_name[v['rule']] = v
+
+      else
+	 error ("Unknown rule object: "..table.tostring(v))
+      end
+   end
+
+   -- We've now loaded all the rules, macros, and list. Now
+   -- compile/expand the rules, macros, and lists. We use
+   -- ordered_rule_{lists,macros,names} to compile them in the order
+   -- in which they appeared in the file(s).
+   reset_rules(rules_mgr)
+
+   for i, name in ipairs(state.ordered_list_names) do
+
+      local v = state.lists_by_name[name]
+
       -- list items are represented in yaml as a native list, so no
       -- parsing necessary
       local items = {}
@@ -206,20 +277,19 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
       end
 
       state.lists[v['list']] = items
-
-      elseif (v['rule']) then
-
-	 if (v['rule'] == nil or type(v['rule']) == "table") then
-	    error ("Missing name in rule")
    end
 
-	 for i, field in ipairs({'condition', 'output', 'desc', 'priority'}) do
-	    if (v[field] == nil) then
-	       error ("Missing "..field.." in rule with name "..v['rule'])
-	    end
+   for i, name in ipairs(state.ordered_macro_names) do
+
+      local v = state.macros_by_name[name]
+
+      local ast = compiler.compile_macro(v['condition'], state.lists)
+      state.macros[v['macro']] = ast.filter.value
    end
 
-	 state.rules_by_name[v['rule']] = v
+   for i, name in ipairs(state.ordered_rule_names) do
+
+      local v = state.rules_by_name[name]
 
       local filter_ast, evttypes = compiler.compile_filter(v['rule'], v['condition'],
 							   state.macros, state.lists)
@@ -290,9 +360,6 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
       else
 	 error ("Unexpected type in load_rule: "..filter_ast.type)
       end
-      else
-	 error ("Unknown rule object: "..table.tostring(v))
-      end
    end
 
    io.flush()