Support new yaml format for rules

Uses yaml parsing lib to parse a yaml file comprising of a list of
macros and rules, like:

- macro: bin_dir
  condition: fd.directory in (/bin, /sbin, /usr/bin, /usr/sbin)

- macro: core_binaries
  condition: proc.name in (ls, mkdir, cat, less, ps)

- condition: (fd.typechar = 4 or fd.typechar=6) and core_binaries
  output: "%evt.time: %proc.name network with %fd.l4proto"

- condition: evt.type = write and bin_dir
  output: "%evt.time: System binary modified (file '%fd.filename' written by process %proc.name)"

- condition: container.id != host and proc.name = bash
  output: "%evt.time: Shell running in container (%proc.name, %container.id)"

userspace/falco/lua/compiler.lua

@@ -155,17 +155,6 @@ end
 
 -- grammar
 
-local function normalize_level(level)
-   valid_levels = {"emergency", "alert", "critical", "error", "warning", "notice", "informational", "debug"}
-   level = string.lower(level)
-   for i,v in ipairs(valid_levels) do
-      if (string.find(v, "^"..level)) then
-	 return i - 1 -- (syslog levels start at 0, lua indices start at 1)
-      end
-   end
-   error("Invalid severity level: "..level)
-end
-
 
 local function filter(e)
    return {type = "Filter", value=e}
@@ -417,10 +406,7 @@ function compiler.parser.parse_filter (subject)
 end
 
 
---[[
+function compiler.compile_macro(line)
-   Compiles a single line from a falco ruleset and updates the passed-in macros table. Returns the AST of the line.
---]]
-function compiler.compile_line(line, macro_defs)
    local ast, error_msg = compiler.parser.parse_filter(line)
 
    if (error_msg) then
@@ -428,24 +414,22 @@ function compiler.compile_line(line, macro_defs)
       error(error_msg)
    end
 
-   if (type(ast) == "number") then
+   return ast
-      -- hack: we get a number (# of matched chars) V"Skip" back if this line
+end
-      -- only contained whitespace. (According to docs 'v"Skip" / 0' in Start
+
-      -- rule should not capture anything but it doesn't seem to work that
+--[[
-      -- way...)
+   Parses a single filter, then expands macros using passed-in table of definitions. Returns resulting AST.
-      return {}
+--]]
+function compiler.compile_filter(source, macro_defs)
+   local ast, error_msg = compiler.parser.parse_filter(source)
+
+   if (error_msg) then
+      print ("Compilation error: ", error_msg)
+      error(error_msg)
    end
 
-   if (ast.type == "MacroDef") then
+   if (ast.type == "Rule") then
-      -- Parsed line is a macro definition, so update our dictionary of macros and
+      -- Line is a filter, so expand macro references
-      -- return
-      macro_defs[ast.name] = ast.value
-      return ast
-
-   elseif (ast.type == "Rule") then
-      -- Line is a filter, so expand macro references then
-      -- stitch it into global ast
-
       repeat
 	 expanded  = expand_macros(ast, macro_defs, false)
       until expanded == false

userspace/falco/lua/rule_loader.lua

@@ -6,8 +6,11 @@
 --]]
 
 local DEFAULT_OUTPUT_FORMAT = "%evt.time: %evt.num %evt.cpu %proc.name (%thread.tid) %evt.dir %evt.type %evt.args"
+local DEFAULT_PRIORITY = "WARNING"
+
 
 local compiler = require "compiler"
+local yaml = require"lyaml"
 
 --[[
    Traverse AST, adding the passed-in 'index' to each node that contains a relational expression
@@ -89,78 +92,90 @@ local function install_filter(node, parent_bool_op)
    end
 end
 
-local state
+function set_output(output_format, state)
-
---[[
-   Sets up compiler state and returns it.
-
-   It holds state such as macro definitions that must be kept across calls
-   to the line-oriented compiler.
---]]
-local function init()
-   return {macros={}, filter_ast=nil, n_rules=0, outputs={}}
-end
-
-
-function set_output(output_ast)
 
    if(output_ast.type == "OutputFormat") then
 
       local format
-      if output_ast.value==nil then
-	 format = DEFAULT_OUTPUT_FORMAT
-      else
-	 format = output_ast.value
-      end
-
-      state.outputs[state.n_rules] = {format=format, level = output_ast.level}
 
    else
       error ("Unexpected type in set_output: ".. output_ast.type)
    end
 end
 
-function load_rule(r)
+local function priority(s)
-   if (state == nil) then
+   valid_levels = {"emergency", "alert", "critical", "error", "warning", "notice", "informational", "debug"}
-      state = init()
+   s = string.lower(s)
+   for i,v in ipairs(valid_levels) do
+      if (string.find(v, "^"..s)) then
+	 return i - 1 -- (syslog levels start at 0, lua indices start at 1)
       end
-   local line_ast = compiler.compile_line(r, state.macros)
+   end
+   error("Invalid severity level: "..level)
+end
 
-   if (line_ast.type == nil) then -- blank line
+local state = {macros={}, filter_ast=nil, n_rules=0, outputs={}}
-      return
+
-   elseif (line_ast.type == "MacroDef") then
+function load_rules(filename)
-      return
+
-   elseif (not (line_ast.type == "Rule")) then
+   local f = assert(io.open(filename, "r"))
-      error ("Unexpected type in load_rule: "..line_ast.type)
+   local s = f:read("*all")
+   f:close()
+   local rules = yaml.load(s)
+
+   for i,v in ipairs(rules) do -- iterate over yaml list
+
+      if (not (type(v) == "table")) then
+	 error ("Unexpected element of type " ..type(v)..". Each element should be a yaml associative array.")
       end
 
+      if (v['macro']) then
+	 local ast = compiler.compile_macro(v['condition'])
+	 state.macros[v['macro']] = ast.filter.value
+
+      else -- filter
+
+	 if (v['condition'] == nil) then
+	    error ("Missing condition in rule")
+	 end
+
+	 if (v['output'] == nil) then
+	    error ("Missing output in rule with condition"..v['condition'])
+	 end
+
+	 local filter_ast = compiler.compile_filter(v['condition'], state.macros)
+
+	 if (filter_ast.type == "Rule") then
 	    state.n_rules = state.n_rules + 1
 
-   set_output(line_ast.output)
+	    state.outputs[state.n_rules] = {format=v['output'] or DEFAULT_OUTPUT_FORMAT,
+					    level=priority(v['priority'] or DEFAULT_PRIORITY)}
 
 	    -- Store the index of this formatter in each relational expression that
 	    -- this rule contains.
 	    -- This index will eventually be stamped in events passing this rule, and
 	    -- we'll use it later to determine which output to display when we get an
 	    -- event.
-   mark_relational_nodes(line_ast.filter.value, state.n_rules)
+	    mark_relational_nodes(filter_ast.filter.value, state.n_rules)
 
 	    -- Rule ASTs are merged together into one big AST, with "OR" between each
 	    -- rule.
 	    if (state.filter_ast == nil) then
-      state.filter_ast = line_ast.filter.value
+	       state.filter_ast = filter_ast.filter.value
 	    else
-      state.filter_ast = { type = "BinaryBoolOp", operator = "or", left = state.filter_ast, right = line_ast.filter.value }
+	       state.filter_ast = { type = "BinaryBoolOp", operator = "or", left = state.filter_ast, right = filter_ast.filter.value }
+	    end
+	 else
+	    error ("Unexpected type in load_rule: "..filter_ast.type)
+	 end
+      end
    end
-end
 
-function on_done()
    install_filter(state.filter_ast)
    io.flush()
 end
 
 local output_functions = require('output')
-
 outputs = {}
 
 function add_output(output_name, config)

userspace/falco/rules.cpp

@@ -41,48 +41,19 @@ void falco_rules::load_compiler(string lua_main_filename)
 
 void falco_rules::load_rules(string rules_filename)
 {
-	ifstream is;
+	lua_getglobal(m_ls, m_lua_load_rules.c_str());
-	is.open(rules_filename);
-	if(!is.is_open())
-	{
-		throw sinsp_exception("Can't open file " + rules_filename + ". Try setting file location in config file or use '-r' flag.");
-	}
-
-	lua_getglobal(m_ls, m_lua_load_rule.c_str());
 	if(lua_isfunction(m_ls, -1))
 	{
-		lua_pop(m_ls, 1);
+		lua_pushstring(m_ls, rules_filename.c_str());
-	} else {
-		throw sinsp_exception("No function " + m_lua_load_rule + " found in lua compiler module");
-	}
-
-	std::string line;
-	while (std::getline(is, line))
-	{
-		lua_getglobal(m_ls, m_lua_load_rule.c_str());
-		lua_pushstring(m_ls, line.c_str());
-
 		if(lua_pcall(m_ls, 1, 0, 0) != 0)
 		{
 			const char* lerr = lua_tostring(m_ls, -1);
-			string err = "Error loading rule '" + line + "':" + string(lerr);
+			string err = "Error loading rules:" + string(lerr);
-			throw sinsp_exception(err);
-		}
-	}
-
-	lua_getglobal(m_ls, m_lua_on_done.c_str());
-	if(lua_isfunction(m_ls, -1))
-	{
-		if(lua_pcall(m_ls, 0, 0, 0) != 0)
-		{
-			const char* lerr = lua_tostring(m_ls, -1);
-			string err = "Error installing rules: " + string(lerr);
 			throw sinsp_exception(err);
 		}
 	} else {
-		throw sinsp_exception("No function " + m_lua_on_done + " found in lua compiler module");
+		throw sinsp_exception("No function " + m_lua_load_rules + " found in lua compiler module");
 	}
-
 }
 
 sinsp_filter* falco_rules::get_filter()

userspace/falco/rules.h

@@ -17,7 +17,6 @@ class falco_rules
 	lua_parser* m_lua_parser;
 	lua_State* m_ls;
 
-	string m_lua_load_rule = "load_rule";
+	string m_lua_load_rules = "load_rules";
-	string m_lua_on_done = "on_done";
 	string m_lua_on_event = "on_event";
 };