Rules loading changes for plugins

Rules loading changes for plugins:

 - parse required_engine_versions from yaml and pass up to rules
   loader as a lua table as an additional return value from load_rules().
 - c++ rules loader converts to map: plugin -> list of required plugin
   versions
 - support is_source_valid callback from lua, calls engine method. If
   a source is not valid, skip any rules for that source and add a warning.

Co-authored-by: Leonardo Grasso <me@leonardograsso.com>
Co-authored-by: Loris Degioanni <loris@sysdig.com>
Signed-off-by: Mark Stemm <mark.stemm@gmail.com>

userspace/engine/lua/rule_loader.lua

@@ -437,6 +437,29 @@ function load_rules_doc(rules_mgr, doc, load_state)
 	    return false, build_error_with_context(v['context'], "Rules require engine version "..v['required_engine_version']..", but engine version is "..falco_rules.engine_version(rules_mgr)), warnings
 	 end
 
+      elseif (v['required_plugin_versions']) then
+
+	 for _, vobj in ipairs(v['required_plugin_versions']) do
+	    if vobj['name'] == nil then
+	       return false, build_error_with_context(v['context'], "required_plugin_versions item must have name property"), warnings
+	    end
+
+	    if vobj['version'] == nil then
+	       return false, build_error_with_context(v['context'], "required_plugin_versions item must have version property"), warnings
+	    end
+
+	    -- In the rules yaml, it's a name + version. But it's
+	    -- possible, although unlikely, that a single yaml blob
+	    -- contains multiple docs, withe each doc having its own
+	    -- required_engine_version entry. So populate a map plugin
+	    -- name -> list of required plugin versions.
+	    if load_state.required_plugin_versions[vobj['name']] == nil then
+	       load_state.required_plugin_versions[vobj['name']] = {}
+	    end
+
+	    table.insert(load_state.required_plugin_versions[vobj['name']], vobj['version'])
+	 end
+
       elseif (v['macro']) then
 
 	 if (v['macro'] == nil or type(v['macro']) == "table") then
@@ -780,6 +803,7 @@ end
 -- Returns:
 -- - Load Result: bool
 -- - required engine version. will be nil when load result is false
+-- - required_plugin_versions. will be nil when load_result is false
 -- - List of Errors
 -- - List of Warnings
 function load_rules(rules_content,
@@ -792,7 +816,7 @@ function load_rules(rules_content,
 
    local warnings = {}
 
-   local load_state = {lines={}, indices={}, cur_item_idx=0, min_priority=min_priority, required_engine_version=0}
+   local load_state = {lines={}, indices={}, cur_item_idx=0, min_priority=min_priority, required_engine_version=0, required_plugin_versions={}}
 
    load_state.lines, load_state.indices = split_lines(rules_content)
 
@@ -813,29 +837,29 @@ function load_rules(rules_content,
       row = tonumber(row)
       col = tonumber(col)
 
-      return false, nil, build_error(load_state.lines, row, 3, docs), warnings
+      return false, nil, nil, build_error(load_state.lines, row, 3, docs), warnings
    end
 
    if docs == nil then
       -- An empty rules file is acceptable
-      return true, load_state.required_engine_version, {}, warnings
+      return true, load_state.required_engine_version, {}, {}, warnings
    end
 
    if type(docs) ~= "table" then
-      return false, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml"), warnings
+      return false, nil, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml"), warnings
    end
 
    for docidx, doc in ipairs(docs) do
 
       if type(doc) ~= "table" then
-	 return false, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml"), warnings
+	 return false, nil, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml"), warnings
       end
 
       -- Look for non-numeric indices--implies that document is not array
       -- of objects.
       for key, val in pairs(doc) do
 	 if type(key) ~= "number" then
-	    return false, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml array of objects"), warnings
+	    return false, nil, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml array of objects"), warnings
 	 end
       end
 
@@ -848,7 +872,7 @@ function load_rules(rules_content,
       end
 
       if not res then
-	 return res, nil, errors, warnings
+	 return res, nil, nil, errors, warnings
       end
    end
 
@@ -889,7 +913,7 @@ function load_rules(rules_content,
       local status, ast = compiler.compile_macro(v['condition'], state.macros, state.lists)
 
       if status == false then
-	 return false, nil, build_error_with_context(v['context'], ast), warnings
+	 return false, nil, nil, build_error_with_context(v['context'], ast), warnings
       end
 
       state.macros[v['macro']] = {["ast"] = ast.filter.value, ["used"] = false}
@@ -915,7 +939,7 @@ function load_rules(rules_content,
 	 end
 
 	 if err ~= nil then
-	    return false, nil, build_error_with_context(v['context'], err), warnings
+	    return false, nil, nil, build_error_with_context(v['context'], err), warnings
 	 end
 
 	 if icond ~= "" then
@@ -940,10 +964,19 @@ function load_rules(rules_content,
 							 state.macros, state.lists)
 
       if status == false then
-	 return false, nil, build_error_with_context(v['context'], filter_ast), warnings
+	 return false, nil, nil, build_error_with_context(v['context'], filter_ast), warnings
       end
 
       if (filter_ast.type == "Rule") then
+
+	 valid = falco_rules.is_source_valid(rules_mgr, v['source'])
+
+	 if valid == false then
+	    msg = "Rule "..v['rule']..": warning (unknown-source): unknown source "..v['source']..", skipping"
+	    warnings[#warnings + 1] = msg
+	    goto next_rule
+	 end
+
 	 state.n_rules = state.n_rules + 1
 
 	 state.rules_by_idx[state.n_rules] = v
@@ -971,7 +1004,7 @@ function load_rules(rules_content,
 	       warnings[#warnings + 1] = msg
 	    else
 	       msg = "Rule "..v['rule']..": error "..err
-	       return false, nil, build_error_with_context(v['context'], msg), warnings
+	       return false, nil, nil, build_error_with_context(v['context'], msg), warnings
 	    end
 
 	 else
@@ -1034,10 +1067,10 @@ function load_rules(rules_content,
 	 -- up to the top level.
 	 local err = falco_rules.is_format_valid(rules_mgr, v['source'], v['output'])
 	 if err ~= nil then
-	    return false, nil, build_error_with_context(v['context'], err), warnings
+	    return false, nil, nil, build_error_with_context(v['context'], err), warnings
 	 end
       else
-	 return false, nil, build_error_with_context(v['context'], "Unexpected type in load_rule: "..filter_ast.type), warnings
+	 return false, nil, nil, build_error_with_context(v['context'], "Unexpected type in load_rule: "..filter_ast.type), warnings
       end
 
       ::next_rule::
@@ -1060,7 +1093,7 @@ function load_rules(rules_content,
 
    io.flush()
 
-   return true, load_state.required_engine_version, {}, warnings
+   return true, load_state.required_engine_version, load_state.required_plugin_versions, {}, warnings
 end
 
 local rule_fmt = "%-50s %s"

userspace/engine/rules.cpp

@@ -34,6 +34,7 @@ const static struct luaL_Reg ll_falco_rules[] =
 		{"add_filter", &falco_rules::add_filter},
 		{"enable_rule", &falco_rules::enable_rule},
 		{"engine_version", &falco_rules::engine_version},
+		{"is_source_valid", &falco_rules::is_source_valid},
 		{"is_format_valid", &falco_rules::is_format_valid},
 		{"is_defined_field", &falco_rules::is_defined_field},
 		{NULL, NULL}};
@@ -205,6 +206,30 @@ int falco_rules::engine_version(lua_State *ls)
 	return 1;
 }
 
+bool falco_rules::is_source_valid(const std::string &source)
+{
+	return m_engine->is_source_valid(source);
+}
+
+int falco_rules::is_source_valid(lua_State *ls)
+{
+	if (! lua_islightuserdata(ls, -2) ||
+	    ! lua_isstring(ls, -1))
+	{
+		lua_pushstring(ls, "Invalid arguments passed to is_source_valid");
+		lua_error(ls);
+	}
+
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -2);
+	string source = luaL_checkstring(ls, -1);
+
+	bool ret = rules->is_source_valid(source);
+
+	lua_pushboolean(ls, (ret ? 1 : 0));
+
+	return 1;
+}
+
 int falco_rules::is_format_valid(lua_State *ls)
 {
 	if (! lua_islightuserdata(ls, -3) ||
@@ -328,11 +353,48 @@ static std::list<std::string> get_lua_table_values(lua_State *ls, int idx)
 	return ret;
 }
 
+static void get_lua_table_list_values(lua_State *ls,
+				      int idx,
+				      std::map<std::string, std::list<std::string>> &required_plugin_versions)
+{
+	if (lua_isnil(ls, idx)) {
+		return;
+	}
+
+	lua_pushnil(ls);  /* first key */
+	while (lua_next(ls, idx-1) != 0) {
+                // key is at index -2, table of values is at index -1.
+		if (! lua_isstring(ls, -2)) {
+			std::string err = "Non-string key in table of strings";
+			throw falco_exception(err);
+		}
+
+		std::string key = string(lua_tostring(ls, -2));
+		std::list<std::string> vals = get_lua_table_values(ls, -1);
+
+		if (required_plugin_versions.find(key) == required_plugin_versions.end())
+		{
+			required_plugin_versions[key] = vals;
+		}
+		else
+		{
+			required_plugin_versions[key].insert(required_plugin_versions[key].end(),
+							     vals.begin(),
+							     vals.end());
+		}
+
+		// Remove value, keep key for next iteration
+		lua_pop(ls, 1);
+	}
+}
+
+
 void falco_rules::load_rules(const string &rules_content,
 			     bool verbose, bool all_events,
 			     string &extra, bool replace_container_info,
 			     falco_common::priority_type min_priority,
-			     uint64_t &required_engine_version)
+			     uint64_t &required_engine_version,
+			     std::map<std::string, std::list<std::string>> &required_plugin_versions)
 {
 	lua_getglobal(m_ls, m_lua_load_rules.c_str());
 	if(lua_isfunction(m_ls, -1))
@@ -344,7 +406,7 @@ void falco_rules::load_rules(const string &rules_content,
 		lua_pushstring(m_ls, extra.c_str());
 		lua_pushboolean(m_ls, (replace_container_info ? 1 : 0));
 		lua_pushnumber(m_ls, min_priority);
-		if(lua_pcall(m_ls, 7, 4, 0) != 0)
+		if(lua_pcall(m_ls, 7, 5, 0) != 0)
 		{
 			const char* lerr = lua_tostring(m_ls, -1);
 
@@ -356,10 +418,12 @@ void falco_rules::load_rules(const string &rules_content,
 		// Returns:
 		// Load result: bool
 		// required engine version: will be nil when load result is false
+		// required_plugin_versions: will be nil when load result is false
 		// array of errors
 		// array of warnings
-		bool successful = lua_toboolean(m_ls, -4);
-		required_engine_version = lua_tonumber(m_ls, -3);
+		bool successful = lua_toboolean(m_ls, -5);
+		required_engine_version = lua_tonumber(m_ls, -4);
+		get_lua_table_list_values(m_ls, -3, required_plugin_versions);
 		std::list<std::string> errors = get_lua_table_values(m_ls, -2);
 		std::list<std::string> warnings = get_lua_table_values(m_ls, -1);
 

userspace/engine/rules.h

@@ -42,9 +42,12 @@ class falco_rules
 	void load_rules(const string &rules_content, bool verbose, bool all_events,
 			std::string &extra, bool replace_container_info,
 			falco_common::priority_type min_priority,
-			uint64_t &required_engine_version);
+			uint64_t &required_engine_version,
+			std::map<std::string, std::list<std::string>> &required_plugin_versions);
 	void describe_rule(string *rule);
 
+	bool is_source_valid(const std::string &source);
+
 	bool is_format_valid(const std::string &source, const std::string &format, std::string &errstr);
 
 	bool is_defined_field(const std::string &source, const std::string &field);
@@ -56,6 +59,8 @@ class falco_rules
 	static int enable_rule(lua_State *ls);
 	static int engine_version(lua_State *ls);
 
+	static int is_source_valid(lua_State *ls);
+
 	// err = falco_rules.is_format_valid(source, format_string)
 	static int is_format_valid(lua_State *ls);