refactor(userspace/engine): refactor engine interface and internals

This updates the engine to comply and work properly with the newly-introduced
interface design.

Signed-off-by: Jason Dellaluce <jasondellaluce@gmail.com>

userspace/engine/falco_engine.cpp

@@ -38,7 +38,8 @@ const std::string falco_engine::s_default_ruleset = "falco-default-ruleset";
 using namespace std;
 
 falco_engine::falco_engine(bool seed_rng)
-	: m_min_priority(falco_common::PRIORITY_DEBUG),
+	: m_next_ruleset_id(0),
+	  m_min_priority(falco_common::PRIORITY_DEBUG),
 	  m_sampling_ratio(1), m_sampling_multiplier(0),
 	  m_replace_container_info(false)
 {
@@ -46,6 +47,8 @@ falco_engine::falco_engine(bool seed_rng)
 	{
 		srandom((unsigned) getpid());
 	}
+
+	m_default_ruleset_id = find_ruleset_id(s_default_ruleset);
 }
 
 falco_engine::~falco_engine()
@@ -53,6 +56,7 @@ falco_engine::~falco_engine()
 	m_rules.clear();
 	m_rule_loader.clear();
 	m_rule_stats_manager.clear();
+	m_sources.clear();
 }
 
 uint32_t falco_engine::engine_version()
@@ -60,24 +64,24 @@ uint32_t falco_engine::engine_version()
 	return (uint32_t) FALCO_ENGINE_VERSION;
 }
 
-falco_source& falco_engine::find_source(const std::string& name)
+falco_source* falco_engine::find_source(const std::string& name)
 {
 	auto ret = m_sources.at(name);
 	if(!ret)
 	{
 		throw falco_exception("Unknown event source " + name);
 	}
-	return *ret;
+	return ret;
 }
 
-falco_source& falco_engine::find_source(std::size_t index)
+falco_source* falco_engine::find_source(std::size_t index)
 {
 	auto ret = m_sources.at(index);
 	if(!ret)
 	{
 		throw falco_exception("Unknown event source index " + to_string(index));
 	}
-	return *ret;
+	return ret;
 }
 
 // Return a key that uniquely represents a field class.
@@ -168,20 +172,20 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al
 
 void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version)
 {
-	rule_loader::configuration cfg(rules_content);
+	rule_loader::configuration cfg(rules_content, m_sources);
-	cfg.sources = m_sources;
 	cfg.min_priority = m_min_priority;
 	cfg.output_extra = m_extra;
 	cfg.replace_output_container_info = m_replace_container_info;
+	cfg.default_ruleset_id = m_default_ruleset_id;
 
 	std::ostringstream os;
 	rule_reader reader;
 	bool success = reader.load(cfg, m_rule_loader);
 	if (success)
 	{
-		for (auto &s : m_sources)
+		for (auto &src : m_sources)
 		{
-			s.ruleset->clear();
+			src.ruleset = src.ruleset_factory->new_ruleset();
 		}
 		m_rules.clear();
 		success = m_rule_loader.compile(cfg, m_rules);
@@ -244,7 +248,14 @@ void falco_engine::enable_rule(const string &substring, bool enabled, const stri
 
 	for(auto &it : m_sources)
 	{
-		it.ruleset->enable(substring, match_exact, enabled, ruleset_id);
+		if(enabled)
+		{
+			it.ruleset->enable(substring, match_exact, ruleset_id);
+		}
+		else
+		{
+			it.ruleset->disable(substring, match_exact, ruleset_id);
+		}
 	}
 }
 
@@ -255,7 +266,14 @@ void falco_engine::enable_rule_exact(const string &rule_name, bool enabled, cons
 
 	for(auto &it : m_sources)
 	{
-		it.ruleset->enable(rule_name, match_exact, enabled, ruleset_id);
+		if(enabled)
+		{
+			it.ruleset->enable(rule_name, match_exact, ruleset_id);
+		}
+		else
+		{
+			it.ruleset->disable(rule_name, match_exact, ruleset_id);
+		}
 	}
 }
 
@@ -265,7 +283,14 @@ void falco_engine::enable_rule_by_tag(const set<string> &tags, bool enabled, con
 
 	for(auto &it : m_sources)
 	{
-		it.ruleset->enable_tags(tags, enabled, ruleset_id);
+		if(enabled)
+		{
+			it.ruleset->enable_tags(tags, ruleset_id);
+		}
+		else
+		{
+			it.ruleset->disable_tags(tags, ruleset_id);
+		}
 	}
 }
 
@@ -274,43 +299,43 @@ void falco_engine::set_min_priority(falco_common::priority_type priority)
 	m_min_priority = priority;
 }
 
-uint16_t falco_engine::find_ruleset_id(
+uint16_t falco_engine::find_ruleset_id(const std::string &ruleset)
-	const std::string &ruleset, const std::string &source)
 {
-	return find_source(source).ruleset->ruleset_id(ruleset);
+	auto it = m_known_rulesets.lower_bound(ruleset);
+	if(it == m_known_rulesets.end() || it->first != ruleset)
+	{
+		it = m_known_rulesets.emplace_hint(it,
+						   std::make_pair(ruleset, m_next_ruleset_id++));
+	}
+	return it->second;
 }
 
 uint64_t falco_engine::num_rules_for_ruleset(const std::string &ruleset)
 {
+	uint16_t ruleset_id = find_ruleset_id(ruleset);
 	uint64_t ret = 0;
 	for (auto &src : m_sources)
 	{
-		ret += src.ruleset->enabled_count(src.ruleset->ruleset_id(ruleset));
+		ret += src.ruleset->enabled_count(ruleset_id);
 	}
 	return ret;
 }
 
 void falco_engine::evttypes_for_ruleset(std::string &source, std::set<uint16_t> &evttypes, const std::string &ruleset)
 {
-	auto src = find_source(source);
+	find_source(source)->ruleset->enabled_evttypes(evttypes, find_ruleset_id(ruleset));
-	src.ruleset->enabled_evttypes(evttypes, src.ruleset->ruleset_id(ruleset));
 }
 
 std::shared_ptr<gen_event_formatter> falco_engine::create_formatter(const std::string &source,
 								    const std::string &output)
 {
-	return find_source(source).formatter_factory->create_formatter(output);
+	return find_source(source)->formatter_factory->create_formatter(output);
 }
 
 unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::size_t source_idx, gen_event *ev, uint16_t ruleset_id)
 {
-	if(should_drop_evt())
-	{
-		return unique_ptr<struct rule_result>();
-	}
-
 	falco_rule rule;
-	if(!find_source(source_idx).ruleset->run(ev, rule, ruleset_id))
+	if(should_drop_evt() || !find_source(source_idx)->ruleset->run(ev, rule, ruleset_id))
 	{
 		return unique_ptr<struct rule_result>();
 	}
@@ -323,13 +348,13 @@ unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::size_t so
 	res->priority_num = rule.priority;
 	res->tags = rule.tags;
 	res->exception_fields = rule.exception_fields;
-	m_rule_stats_manager.on_event(m_rules, rule.id);
+	m_rule_stats_manager.on_event(rule);
 	return res;
 }
 
 unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::size_t source_idx, gen_event *ev)
 {
-	return process_event(source_idx, ev, find_source(source_idx).default_ruleset_id);
+	return process_event(source_idx, ev, m_default_ruleset_id);
 }
 
 std::size_t falco_engine::add_source(const std::string &source,
@@ -353,7 +378,6 @@ std::size_t falco_engine::add_source(const std::string &source,
 	src.formatter_factory = formatter_factory;
 	src.ruleset_factory = ruleset_factory;
 	src.ruleset = ruleset_factory->new_ruleset();
-	src.default_ruleset_id = src.ruleset->ruleset_id(s_default_ruleset);
 	return m_sources.insert(src, source);
 }
 
@@ -434,20 +458,14 @@ bool falco_engine::check_plugin_requirements(
 	return true;
 }
 
-void falco_engine::clear_filters()
+void falco_engine::complete_rule_loading()
 {
-	for(auto &src : m_sources)
+	for (auto &src : m_sources)
 	{
-		src.ruleset = src.ruleset_factory->new_ruleset();
+		src.ruleset->on_loading_complete();
-		src.default_ruleset_id = src.ruleset->ruleset_id(s_default_ruleset);
 	}
 }
 
-void falco_engine::clear_loader()
-{
-	m_rule_loader.clear();
-}
-
 void falco_engine::set_sampling_ratio(uint32_t sampling_ratio)
 {
 	m_sampling_ratio = sampling_ratio;

userspace/engine/falco_engine.h

@@ -90,6 +90,16 @@ public:
 	//
 	void enable_rule_by_tag(const std::set<std::string> &tags, bool enabled, const std::string &ruleset = s_default_ruleset);
 
+	//
+	// Must be called after the engine has been configured and all rulesets
+	// have been loaded and enabled/disabled.
+	// This does not change the engine configuration nor the loaded/enabled rule
+	// setup, and does not affect the functional behavior.
+	// Internally, this can be used to release unused resources before starting
+	// processing events with process_event().
+	//
+	void complete_rule_loading();
+
 	// Only load rules having this priority or more severe.
 	void set_min_priority(falco_common::priority_type priority);
 
@@ -99,9 +109,7 @@ public:
 	// to enable_rule/enable_rule_by_tag(), you should look up the
 	// ruleset id and pass it to process_event().
 	//
-	uint16_t find_ruleset_id(
+	uint16_t find_ruleset_id(const std::string &ruleset);
-		const std::string &ruleset,
-		const std::string &source = falco_common::syscall_source);
 
 	//
 	// Return the number of falco rules enabled for the provided ruleset
@@ -120,18 +128,6 @@ public:
 	//
 	void print_stats();
 
-	// Clear all existing filters.
-	void clear_filters();
-
-	//
-	// Clear all the definitions of the internal rule loader (e.g. defined
-	// rules, macros, lists, engine/plugin version requirements). This is meant
-	// to be used to free-up memory at runtime when the definitions are not
-	// used anymore. Calling this between successive invocations of load_rules
-	// or load_rules_file can cause failures of features like appending.
-	//
-	void clear_loader();
-
 	//
 	// Set the sampling ratio, which can affect which events are
 	// matched against the set of rules.
@@ -239,8 +235,8 @@ public:
 private:
 	indexed_vector<falco_source> m_sources;
 
-	falco_source& find_source(std::size_t index);
+	falco_source* find_source(std::size_t index);
-	falco_source& find_source(const std::string& name);
+	falco_source* find_source(const std::string& name);
 
 	//
 	// Determine whether the given event should be matched at all
@@ -253,6 +249,8 @@ private:
 	indexed_vector<falco_rule> m_rules;
 	stats_manager m_rule_stats_manager;
 
+	uint16_t m_next_ruleset_id;
+	std::map<string, uint16_t> m_known_rulesets;
 	falco_common::priority_type m_min_priority;
 
 	//
@@ -279,6 +277,7 @@ private:
 	double m_sampling_multiplier;
 
 	static const std::string s_default_ruleset;
+	uint32_t m_default_ruleset_id;
 
 	std::string m_extra;
 	bool m_replace_container_info;

userspace/engine/falco_rule.h

@@ -20,9 +20,13 @@ limitations under the License.
 #include <string>
 #include "falco_common.h"
 
+/*!
+	\brief Represents a rule in the Falco Engine.
+	The rule ID must be unique across all the rules loaded in the engine.
+*/
 struct falco_rule
 {
-	size_t id;
+	std::size_t id;
 	std::string source;
 	std::string name;
 	std::string description;

userspace/engine/falco_source.h

@@ -19,10 +19,14 @@ limitations under the License.
 #include <string>
 #include "ruleset.h"
 
+/*!
+	\brief Represents a given data source used by the engine.
+	The ruleset of a source should be created through the ruleset factory
+	of the same data source.
+*/
 struct falco_source
 {
 	std::string name;
-	uint16_t default_ruleset_id;
 	std::shared_ptr<filter_ruleset> ruleset;
 	std::shared_ptr<filter_ruleset_factory> ruleset_factory;
 	std::shared_ptr<gen_event_filter_factory> filter_factory;

userspace/engine/rule_loader.cpp

@@ -648,7 +648,7 @@ void rule_loader::compile_rule_infos(
 				apply_output_substitutions(cfg, rule.output);
 			}
 
-			THROW(!is_format_valid(cfg.engine, r.source, rule.output, err),
+			THROW(!is_format_valid(*cfg.sources.at(r.source), rule.output, err),
 				"Invalid output format '" + rule.output + "': '" + err + "'");
 
 			// construct rule definition and compile it to a filter
@@ -662,7 +662,16 @@ void rule_loader::compile_rule_infos(
 				auto rule_id = out.insert(rule, rule.name);
 				out.at(rule_id)->id = rule_id;
 				source->ruleset->add(*out.at(rule_id), ast);
-				source->ruleset->enable(rule.name, false, r.enabled);
+
+				// By default rules are enabled/disabled for the default ruleset
+				if(r.enabled)
+				{
+					source->ruleset->enable(rule.name, true, cfg.default_ruleset_id);
+				}
+				else
+				{
+					source->ruleset->disable(rule.name, true, cfg.default_ruleset_id);
+				}
 			}
 			catch (falco_exception& e)
 			{

userspace/engine/rule_loader.h

@@ -66,14 +66,19 @@ public:
 	*/
 	struct configuration
 	{
-		explicit configuration(const std::string& cont): content(cont) {}
+		explicit configuration(
+			const std::string& cont,
+			const indexed_vector<falco_source>& srcs)
+				: content(cont), sources(srcs) {}
+
 		const std::string& content;
+		const indexed_vector<falco_source>& sources;
+		std::vector<std::string> errors;
+		std::vector<std::string> warnings;
 		std::string output_extra;
+		uint16_t default_ruleset_id;
 		bool replace_output_container_info;
 		falco_common::priority_type min_priority;
-		indexed_vector<falco_source> sources;
-		std::vector<std::string> warnings;
-		std::vector<std::string> errors;
 	};
 
 	/*!

userspace/engine/ruleset.h

@@ -23,9 +23,8 @@ limitations under the License.
 #include <gen_filter.h>
 
 /*!
-	\brief Represents a manager for rulesets.A ruleset represents a set of
+	\brief Manages a set of rulesets. A ruleset is a set of
-	enabled rules that is able to process events and find potential rule
+	enabled rules that is able to process events and find matches for those rules.
-	matches. By convention, the ruleset with id = 0 is the default one.
 */
 class filter_ruleset
 {