refactor(userspace/engine): re-implement the rule loader in C++

Signed-off-by: Jason Dellaluce <jasondellaluce@gmail.com>

userspace/engine/CMakeLists.txt

@@ -18,7 +18,8 @@ set(FALCO_ENGINE_SOURCE_FILES
     ruleset.cpp
     formats.cpp
     filter_macro_resolver.cpp
-    lua_filter_helper.cpp)
+    rule_loader.cpp
+    stats_manager.cpp)
 
 add_library(falco_engine STATIC ${FALCO_ENGINE_SOURCE_FILES})
 add_dependencies(falco_engine njson string-view-lite)

userspace/engine/falco_engine.cpp

@@ -41,14 +41,6 @@ falco_engine::falco_engine(bool seed_rng)
 	  m_sampling_ratio(1), m_sampling_multiplier(0),
 	  m_replace_container_info(false)
 {
-	luaopen_yaml(m_ls);
-
-	falco_common::init();
-	falco_rules::init(m_ls);
-	lua_filter_helper::init(m_ls);
-
-	m_required_plugin_versions.clear();
-
 	if(seed_rng)
 	{
 		srandom((unsigned) getpid());
@@ -59,6 +51,8 @@ falco_engine::falco_engine(bool seed_rng)
 
 falco_engine::~falco_engine()
 {
+	m_rule_loader.clear();
+	m_rule_stats_manager.clear();
 }
 
 uint32_t falco_engine::engine_version()
@@ -154,18 +148,35 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al
 
 void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version)
 {
-	if(!m_rules)
+	std::vector<std::string> warnings;
+	std::vector<std::string> errors;
+	m_rule_loader.configure(m_min_priority, m_replace_container_info, m_extra);
+	bool success = m_rule_loader.load(rules_content, this, warnings, errors);
+	std::ostringstream os;
+	if (!errors.empty())
 	{
-		m_rules.reset(new falco_rules(this,
-					      m_ls));
-
-		for(auto const &it : m_filter_factories)
+		os << errors.size() << " errors:" << std::endl;
+		for(auto &err : errors)
 		{
-			m_rules->add_filter_factory(it.first, it.second);
+			os << err << std::endl;
 		}
 	}
-
-	m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info, m_min_priority, required_engine_version, m_required_plugin_versions);
+	if (!warnings.empty())
+	{
+		os << warnings.size() << " warnings:" << std::endl;
+		for(auto &warn : warnings)
+		{
+			os << warn << std::endl;
+		}
+	}
+	if(!success)
+	{
+		throw falco_exception(os.str());
+	}
+	if (verbose && os.str() != "") {
+		// todo(jasondellaluce): introduce a logging callback in Falco
+		fprintf(stderr, "When reading rules content: %s", os.str().c_str());
+	}
 }
 
 void falco_engine::load_rules_file(const string &rules_filename, bool verbose, bool all_events)
@@ -302,9 +313,8 @@ unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::size_t so
 		}
 
 		unique_ptr<struct rule_result> res(new rule_result());
-		res->source = r.source;
-
 		populate_rule_result(res, ev);
+		m_rule_stats_manager.on_event(m_rule_loader.rules(), ev->get_check_id());
 
 		return res;
 	}
@@ -333,82 +343,65 @@ std::size_t falco_engine::add_source(const std::string &source,
 	return idx;
 }
 
+std::shared_ptr<gen_event_filter_factory> falco_engine::get_filter_factory(
+	const std::string &source)
+{
+	auto it = m_filter_factories.find(source);
+	if(it == m_filter_factories.end())
+	{
+		throw falco_exception(string("unknown event source: ") + source);
+	}
+	return it->second;
+}
+
 void falco_engine::populate_rule_result(unique_ptr<struct rule_result> &res, gen_event *ev)
 {
-	std::lock_guard<std::mutex> guard(m_ls_semaphore);
-	lua_getglobal(m_ls, lua_on_event.c_str());
-	if(lua_isfunction(m_ls, -1))
+	res->evt = ev;
+	auto rule = m_rule_loader.rules().at(ev->get_check_id());
+	if (!rule)
 	{
-		lua_pushnumber(m_ls, ev->get_check_id());
-		if(lua_pcall(m_ls, 1, 5, 0) != 0)
-		{
-			const char* lerr = lua_tostring(m_ls, -1);
-			string err = "Error invoking function output: " + string(lerr);
-			throw falco_exception(err);
-		}
-		const char *p =  lua_tostring(m_ls, -5);
-		res->rule = p;
-		res->evt = ev;
-		res->priority_num = (falco_common::priority_type) lua_tonumber(m_ls, -4);
-		res->format = lua_tostring(m_ls, -3);
-
-		// Tags are passed back as a table, and is on the top of the stack
-		lua_pushnil(m_ls);  /* first key */
-		while (lua_next(m_ls, -2) != 0) {
-			// key is at index -2, value is at index
-			// -1. We want the value.
-			res->tags.insert(luaL_checkstring(m_ls, -1));
-
-			// Remove value, keep key for next iteration
-			lua_pop(m_ls, 1);
-		}
-		lua_pop(m_ls, 1); // Clean table leftover
-
-		// Exception fields are passed back as a table
-		lua_pushnil(m_ls);  /* first key */
-		while (lua_next(m_ls, -2) != 0) {
-			// key is at index -2, value is at index
-			// -1. We want the keys.
-			res->exception_fields.insert(luaL_checkstring(m_ls, -2));
-
-			// Remove value, keep key for next iteration
-			lua_pop(m_ls, 1);
-		}
-
-		lua_pop(m_ls, 4);
-	}
-	else
-	{
-		throw falco_exception("No function " + lua_on_event + " found in lua compiler module");
+		throw falco_exception("populate_rule_result error: unknown rule id "
+				+ to_string(ev->get_check_id()));
 	}
+	res->rule = rule->name;
+	res->source = rule->source;
+	res->format = rule->output;
+	res->priority_num = rule->priority;
+	res->tags = rule->tags;
+	res->exception_fields = rule->exception_fields;
 }
 
 void falco_engine::describe_rule(string *rule)
 {
-	return m_rules->describe_rule(rule);
-}
-
-// Print statistics on the rules that triggered
-void falco_engine::print_stats()
-{
-	lua_getglobal(m_ls, lua_print_stats.c_str());
-
-	if(lua_isfunction(m_ls, -1))
+	static const char* rule_fmt = "%-50s %s\n";
+	fprintf(stdout, rule_fmt, "Rule", "Description");
+	fprintf(stdout, rule_fmt, "----",  "-----------");
+	if (!rule)
 	{
-		if(lua_pcall(m_ls, 0, 0, 0) != 0)
+		for (uint32_t id = 0; id < m_rule_loader.rules().size(); id++)
 		{
-			const char* lerr = lua_tostring(m_ls, -1);
-			string err = "Error invoking function print_stats: " + string(lerr);
-			throw falco_exception(err);
+			auto r = m_rule_loader.rules().at(id);
+			auto wrapped = falco::utils::wrap_text(r->description, 51, 110);
+			fprintf(stdout, rule_fmt, r->name.c_str(), wrapped.c_str());
 		}
 	}
 	else
 	{
-		throw falco_exception("No function " + lua_print_stats + " found in lua rule loader module");
+		auto r = m_rule_loader.rules().at(*rule);
+		auto wrapped = falco::utils::wrap_text(r->description, 51, 110);
+		fprintf(stdout, rule_fmt, r->name.c_str(), wrapped.c_str());
 	}
 
 }
 
+void falco_engine::print_stats()
+{
+	string out;
+	m_rule_stats_manager.format_stats(m_rule_loader.rules(), out);
+	// todo(jasondellaluce): introduce a logging callback in Falco
+	fprintf(stdout, "%s", out.c_str());
+}
+
 void falco_engine::add_filter(std::shared_ptr<gen_event_filter> filter,
 			      std::string &rule,
 			      std::string &source,
@@ -433,30 +426,7 @@ bool falco_engine::is_plugin_compatible(const std::string &name,
 					const std::string &version,
 					std::string &required_version)
 {
-	sinsp_plugin::version plugin_version(version);
-
-	if(!plugin_version.m_valid)
-	{
-		throw falco_exception(string("Plugin version string ") + version + " not valid");
-	}
-
-	if(m_required_plugin_versions.find(name) == m_required_plugin_versions.end())
-	{
-		// No required engine versions, so no restrictions. Compatible.
-		return true;
-	}
-
-	for(auto &rversion : m_required_plugin_versions[name])
-	{
-		sinsp_plugin::version req_version(rversion);
-		if (!plugin_version.check(req_version))
-		{
-			required_version = rversion;
-			return false;
-		}
-	}
-
-	return true;
+	return m_rule_loader.is_plugin_compatible(name, version, required_version);
 }
 
 void falco_engine::clear_filters()
@@ -465,8 +435,6 @@ void falco_engine::clear_filters()
 	{
 		it.ruleset.reset(new falco_ruleset);
 	}
-
-	m_required_plugin_versions.clear();
 }
 
 void falco_engine::set_sampling_ratio(uint32_t sampling_ratio)

userspace/engine/falco_engine.h

@@ -30,7 +30,8 @@ limitations under the License.
 
 #include "gen_filter.h"
 #include "ruleset.h"
-
+#include "rule_loader.h"
+#include "stats_manager.h"
 #include "falco_common.h"
 
 //
@@ -178,6 +179,11 @@ public:
 			       std::shared_ptr<gen_event_filter_factory> filter_factory,
 			       std::shared_ptr<gen_event_formatter_factory> formatter_factory);
 
+	// todo(jasondellaluce): this is here for internal use, and
+	// will possibly be removed in the future
+	std::shared_ptr<gen_event_filter_factory> get_filter_factory(
+		const std::string &source);
+
 	// Return whether or not there is a valid filter/formatter
 	// factory for this source.
 	bool is_source_valid(const std::string &source);
@@ -241,15 +247,13 @@ private:
 	// Maps from event source to the set of rules for that event source
 	std::vector<ruleset_node> m_rulesets;
 
-	std::unique_ptr<falco_rules> m_rules;
+	rule_loader m_rule_loader;
+	stats_manager m_rule_stats_manager;
+
 	uint16_t m_next_ruleset_id;
 	std::map<string, uint16_t> m_known_rulesets;
 	falco_common::priority_type m_min_priority;
 
-	// Maps from plugin to a list of required plugin versions
-	// found in any loaded rules files.
-	std::map<std::string, std::list<std::string>> m_required_plugin_versions;
-
 	void populate_rule_result(unique_ptr<struct rule_result> &res, gen_event *ev);
 
 	//

userspace/engine/rule_loader.h

@@ -0,0 +1,130 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+#include <yaml-cpp/yaml.h>
+#include "falco_rule.h"
+#include "indexed_vector.h"
+
+// todo(jasondellaluce): remove this cyclic dependency
+class falco_engine;
+
+
+/*!
+	\brief Ruleset loader of the falco engine
+*/
+class rule_loader
+{
+public:
+
+	/*!
+		\brief Erases the internal states and all the loaded rules
+	*/
+	virtual void clear();
+
+	/*!
+		\brief Returns the rules loaded after the last invocation of load()
+	*/
+	virtual indexed_vector<falco_rule>& rules();
+
+	/*!
+		\brief Configures the loader. The changes will influence the next
+		invocation of load().
+		\param min_priority The minimum priority below which rules are skipped
+		by the loader
+		\param extra Text to be appended/substituted in the output of all rules
+		\param replace_container_info If true, the extra string is used to
+		replace the "%container.info" token in rules outputs. If false, the
+		"%container.info" token is substituted with a default text and the
+		extra string is appended at the end of the rule output. If a rule
+		output does not contain "%container.info", then this flag has no effect
+		and the extra string is appended at the end of the rule output anyways.
+	*/
+	virtual void configure(falco_common::priority_type min_priority,
+		bool replace_container_info, const std::string& extra);
+
+	/*!
+		\brief Returns true if the given plugin name and version are compatible
+		with the loaded rulesets. If false is returned, required_version is
+		filled with the required plugin version that didn't match.
+	*/
+	virtual bool is_plugin_compatible(const std::string& name,
+		const std::string& version, std::string& required_version);
+
+	/*!
+		\brief Parses the content of a ruleset. This should be called multiple
+		times to load different ruleset files. The internal state (e.g. loaded
+		rules, plugin version requirements, etc...) gets updated at each
+		invocation of the load() method.
+		\param rules_content The contents of the ruleset file
+		\param engine The instance of falco_engine used to add rule filters
+		\param warnings Filled-out with warnings
+		\param warnings Filled-out with errors
+		\return true if the ruleset content is loaded successfully
+	*/
+	virtual bool load(const std::string& rules_content, falco_engine* engine,
+		std::vector<std::string>& warnings, std::vector<std::string>& errors);
+
+private:
+	bool read(
+		const std::string& content, falco_engine* engine,
+		std::vector<std::string>& warnings, std::vector<std::string>& errors);
+	void read_item(
+		falco_engine* engine, YAML::Node& item, vector<string>& warn);
+	void read_required_engine_version(
+		falco_engine* engine, YAML::Node& item, vector<string>& warn);
+	void read_required_plugin_versions(
+		falco_engine* engine, YAML::Node& item, vector<string>& warn);
+	void read_macro(
+		falco_engine* engine, YAML::Node& item, vector<string>& warn);
+	void read_list(
+		falco_engine* engine, YAML::Node& item, vector<string>& warn);
+	void read_rule(
+		falco_engine* engine, YAML::Node& item, vector<string>& warn);
+	void read_rule_exceptions(
+		falco_engine* engine, YAML::Node& item, bool append);
+	bool expand(falco_engine* engine,
+		std::vector<std::string>& warnings, std::vector<std::string>& errors);
+	void expand_list_infos(
+		std::map<string, bool>& used, indexed_vector<YAML::Node>& out);
+	void expand_macro_infos(
+		indexed_vector<YAML::Node>& lists,
+		std::map<string, bool>& used_lists,
+		std::map<string, bool>& used_macros,
+		indexed_vector<pair<YAML::Node,shared_ptr<libsinsp::filter::ast::expr>>>& out);
+	void expand_rule_infos(
+		falco_engine* engine,
+		indexed_vector<YAML::Node>& lists,
+		indexed_vector<pair<YAML::Node,shared_ptr<libsinsp::filter::ast::expr>>>& macros,
+		std::map<string, bool>& used_lists,
+		std::map<string, bool>& used_macros,
+		vector<string>& warnings);
+	void apply_output_replacements(std::string& output);
+
+	uint32_t m_cur_index;
+	std::string m_extra;
+	bool m_replace_container_info;
+	falco_common::priority_type m_min_priority;
+	indexed_vector<falco_rule> m_rules;
+	indexed_vector<YAML::Node> m_rule_infos;
+	indexed_vector<YAML::Node> m_macro_infos;
+	indexed_vector<YAML::Node> m_list_infos;
+	std::map<std::string, std::set<std::string>> m_required_plugin_versions;
+};