From 0abd7eaa28093609472fc5f338936d82cfad8dc9 Mon Sep 17 00:00:00 2001 From: Jason Dellaluce Date: Mon, 23 May 2022 13:49:36 +0000 Subject: [PATCH] refactor(userspace/engine): refactor engine interface and internals This updates the engine to comply and work properly with the newly-introduced interface design. Signed-off-by: Jason Dellaluce --- userspace/engine/falco_engine.cpp | 96 ++++++++++++++++++------------- userspace/engine/falco_engine.h | 33 ++++++----- userspace/engine/falco_rule.h | 6 +- userspace/engine/falco_source.h | 8 ++- userspace/engine/rule_loader.cpp | 13 ++++- userspace/engine/rule_loader.h | 13 +++-- userspace/engine/ruleset.h | 5 +- 7 files changed, 106 insertions(+), 68 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 44aeb53b..da1b234f 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -38,7 +38,8 @@ const std::string falco_engine::s_default_ruleset = "falco-default-ruleset"; using namespace std; falco_engine::falco_engine(bool seed_rng) - : m_min_priority(falco_common::PRIORITY_DEBUG), + : m_next_ruleset_id(0), + m_min_priority(falco_common::PRIORITY_DEBUG), m_sampling_ratio(1), m_sampling_multiplier(0), m_replace_container_info(false) { @@ -46,6 +47,8 @@ falco_engine::falco_engine(bool seed_rng) { srandom((unsigned) getpid()); } + + m_default_ruleset_id = find_ruleset_id(s_default_ruleset); } falco_engine::~falco_engine() @@ -53,6 +56,7 @@ falco_engine::~falco_engine() m_rules.clear(); m_rule_loader.clear(); m_rule_stats_manager.clear(); + m_sources.clear(); } uint32_t falco_engine::engine_version() @@ -60,24 +64,24 @@ uint32_t falco_engine::engine_version() return (uint32_t) FALCO_ENGINE_VERSION; } -falco_source& falco_engine::find_source(const std::string& name) +falco_source* falco_engine::find_source(const std::string& name) { auto ret = m_sources.at(name); if(!ret) { throw falco_exception("Unknown event source " + name); } - return *ret; + return ret; } -falco_source& falco_engine::find_source(std::size_t index) +falco_source* falco_engine::find_source(std::size_t index) { auto ret = m_sources.at(index); if(!ret) { throw falco_exception("Unknown event source index " + to_string(index)); } - return *ret; + return ret; } // Return a key that uniquely represents a field class. @@ -168,20 +172,20 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version) { - rule_loader::configuration cfg(rules_content); - cfg.sources = m_sources; + rule_loader::configuration cfg(rules_content, m_sources); cfg.min_priority = m_min_priority; cfg.output_extra = m_extra; cfg.replace_output_container_info = m_replace_container_info; + cfg.default_ruleset_id = m_default_ruleset_id; std::ostringstream os; rule_reader reader; bool success = reader.load(cfg, m_rule_loader); if (success) { - for (auto &s : m_sources) + for (auto &src : m_sources) { - s.ruleset->clear(); + src.ruleset = src.ruleset_factory->new_ruleset(); } m_rules.clear(); success = m_rule_loader.compile(cfg, m_rules); @@ -244,7 +248,14 @@ void falco_engine::enable_rule(const string &substring, bool enabled, const stri for(auto &it : m_sources) { - it.ruleset->enable(substring, match_exact, enabled, ruleset_id); + if(enabled) + { + it.ruleset->enable(substring, match_exact, ruleset_id); + } + else + { + it.ruleset->disable(substring, match_exact, ruleset_id); + } } } @@ -255,7 +266,14 @@ void falco_engine::enable_rule_exact(const string &rule_name, bool enabled, cons for(auto &it : m_sources) { - it.ruleset->enable(rule_name, match_exact, enabled, ruleset_id); + if(enabled) + { + it.ruleset->enable(rule_name, match_exact, ruleset_id); + } + else + { + it.ruleset->disable(rule_name, match_exact, ruleset_id); + } } } @@ -265,7 +283,14 @@ void falco_engine::enable_rule_by_tag(const set &tags, bool enabled, con for(auto &it : m_sources) { - it.ruleset->enable_tags(tags, enabled, ruleset_id); + if(enabled) + { + it.ruleset->enable_tags(tags, ruleset_id); + } + else + { + it.ruleset->disable_tags(tags, ruleset_id); + } } } @@ -274,47 +299,47 @@ void falco_engine::set_min_priority(falco_common::priority_type priority) m_min_priority = priority; } -uint16_t falco_engine::find_ruleset_id( - const std::string &ruleset, const std::string &source) +uint16_t falco_engine::find_ruleset_id(const std::string &ruleset) { - return find_source(source).ruleset->ruleset_id(ruleset); + auto it = m_known_rulesets.lower_bound(ruleset); + if(it == m_known_rulesets.end() || it->first != ruleset) + { + it = m_known_rulesets.emplace_hint(it, + std::make_pair(ruleset, m_next_ruleset_id++)); + } + return it->second; } uint64_t falco_engine::num_rules_for_ruleset(const std::string &ruleset) { + uint16_t ruleset_id = find_ruleset_id(ruleset); uint64_t ret = 0; for (auto &src : m_sources) { - ret += src.ruleset->enabled_count(src.ruleset->ruleset_id(ruleset)); + ret += src.ruleset->enabled_count(ruleset_id); } return ret; } void falco_engine::evttypes_for_ruleset(std::string &source, std::set &evttypes, const std::string &ruleset) { - auto src = find_source(source); - src.ruleset->enabled_evttypes(evttypes, src.ruleset->ruleset_id(ruleset)); + find_source(source)->ruleset->enabled_evttypes(evttypes, find_ruleset_id(ruleset)); } std::shared_ptr falco_engine::create_formatter(const std::string &source, const std::string &output) { - return find_source(source).formatter_factory->create_formatter(output); + return find_source(source)->formatter_factory->create_formatter(output); } unique_ptr falco_engine::process_event(std::size_t source_idx, gen_event *ev, uint16_t ruleset_id) { - if(should_drop_evt()) - { - return unique_ptr(); - } - falco_rule rule; - if(!find_source(source_idx).ruleset->run(ev, rule, ruleset_id)) + if(should_drop_evt() || !find_source(source_idx)->ruleset->run(ev, rule, ruleset_id)) { return unique_ptr(); } - + unique_ptr res(new rule_result()); res->evt = ev; res->rule = rule.name; @@ -323,19 +348,19 @@ unique_ptr falco_engine::process_event(std::size_t so res->priority_num = rule.priority; res->tags = rule.tags; res->exception_fields = rule.exception_fields; - m_rule_stats_manager.on_event(m_rules, rule.id); + m_rule_stats_manager.on_event(rule); return res; } unique_ptr falco_engine::process_event(std::size_t source_idx, gen_event *ev) { - return process_event(source_idx, ev, find_source(source_idx).default_ruleset_id); + return process_event(source_idx, ev, m_default_ruleset_id); } std::size_t falco_engine::add_source(const std::string &source, std::shared_ptr filter_factory, std::shared_ptr formatter_factory) -{ +{ // evttype_index_ruleset is the default ruleset implementation std::shared_ptr ruleset_factory( new evttype_index_ruleset_factory(filter_factory)); @@ -353,7 +378,6 @@ std::size_t falco_engine::add_source(const std::string &source, src.formatter_factory = formatter_factory; src.ruleset_factory = ruleset_factory; src.ruleset = ruleset_factory->new_ruleset(); - src.default_ruleset_id = src.ruleset->ruleset_id(s_default_ruleset); return m_sources.insert(src, source); } @@ -434,20 +458,14 @@ bool falco_engine::check_plugin_requirements( return true; } -void falco_engine::clear_filters() +void falco_engine::complete_rule_loading() { - for(auto &src : m_sources) + for (auto &src : m_sources) { - src.ruleset = src.ruleset_factory->new_ruleset(); - src.default_ruleset_id = src.ruleset->ruleset_id(s_default_ruleset); + src.ruleset->on_loading_complete(); } } -void falco_engine::clear_loader() -{ - m_rule_loader.clear(); -} - void falco_engine::set_sampling_ratio(uint32_t sampling_ratio) { m_sampling_ratio = sampling_ratio; diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 8df24fce..40782a9e 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -90,6 +90,16 @@ public: // void enable_rule_by_tag(const std::set &tags, bool enabled, const std::string &ruleset = s_default_ruleset); + // + // Must be called after the engine has been configured and all rulesets + // have been loaded and enabled/disabled. + // This does not change the engine configuration nor the loaded/enabled rule + // setup, and does not affect the functional behavior. + // Internally, this can be used to release unused resources before starting + // processing events with process_event(). + // + void complete_rule_loading(); + // Only load rules having this priority or more severe. void set_min_priority(falco_common::priority_type priority); @@ -99,9 +109,7 @@ public: // to enable_rule/enable_rule_by_tag(), you should look up the // ruleset id and pass it to process_event(). // - uint16_t find_ruleset_id( - const std::string &ruleset, - const std::string &source = falco_common::syscall_source); + uint16_t find_ruleset_id(const std::string &ruleset); // // Return the number of falco rules enabled for the provided ruleset @@ -120,18 +128,6 @@ public: // void print_stats(); - // Clear all existing filters. - void clear_filters(); - - // - // Clear all the definitions of the internal rule loader (e.g. defined - // rules, macros, lists, engine/plugin version requirements). This is meant - // to be used to free-up memory at runtime when the definitions are not - // used anymore. Calling this between successive invocations of load_rules - // or load_rules_file can cause failures of features like appending. - // - void clear_loader(); - // // Set the sampling ratio, which can affect which events are // matched against the set of rules. @@ -239,8 +235,8 @@ public: private: indexed_vector m_sources; - falco_source& find_source(std::size_t index); - falco_source& find_source(const std::string& name); + falco_source* find_source(std::size_t index); + falco_source* find_source(const std::string& name); // // Determine whether the given event should be matched at all @@ -253,6 +249,8 @@ private: indexed_vector m_rules; stats_manager m_rule_stats_manager; + uint16_t m_next_ruleset_id; + std::map m_known_rulesets; falco_common::priority_type m_min_priority; // @@ -279,6 +277,7 @@ private: double m_sampling_multiplier; static const std::string s_default_ruleset; + uint32_t m_default_ruleset_id; std::string m_extra; bool m_replace_container_info; diff --git a/userspace/engine/falco_rule.h b/userspace/engine/falco_rule.h index 77aa8c79..7cdbcf8a 100644 --- a/userspace/engine/falco_rule.h +++ b/userspace/engine/falco_rule.h @@ -20,9 +20,13 @@ limitations under the License. #include #include "falco_common.h" +/*! + \brief Represents a rule in the Falco Engine. + The rule ID must be unique across all the rules loaded in the engine. +*/ struct falco_rule { - size_t id; + std::size_t id; std::string source; std::string name; std::string description; diff --git a/userspace/engine/falco_source.h b/userspace/engine/falco_source.h index ca70db94..826f2783 100644 --- a/userspace/engine/falco_source.h +++ b/userspace/engine/falco_source.h @@ -19,14 +19,18 @@ limitations under the License. #include #include "ruleset.h" +/*! + \brief Represents a given data source used by the engine. + The ruleset of a source should be created through the ruleset factory + of the same data source. +*/ struct falco_source { std::string name; - uint16_t default_ruleset_id; std::shared_ptr ruleset; std::shared_ptr ruleset_factory; std::shared_ptr filter_factory; - std::shared_ptr formatter_factory; + std::shared_ptr formatter_factory; inline bool is_field_defined(std::string field) const { diff --git a/userspace/engine/rule_loader.cpp b/userspace/engine/rule_loader.cpp index 103aacfc..ba7f686d 100644 --- a/userspace/engine/rule_loader.cpp +++ b/userspace/engine/rule_loader.cpp @@ -648,7 +648,7 @@ void rule_loader::compile_rule_infos( apply_output_substitutions(cfg, rule.output); } - THROW(!is_format_valid(cfg.engine, r.source, rule.output, err), + THROW(!is_format_valid(*cfg.sources.at(r.source), rule.output, err), "Invalid output format '" + rule.output + "': '" + err + "'"); // construct rule definition and compile it to a filter @@ -662,7 +662,16 @@ void rule_loader::compile_rule_infos( auto rule_id = out.insert(rule, rule.name); out.at(rule_id)->id = rule_id; source->ruleset->add(*out.at(rule_id), ast); - source->ruleset->enable(rule.name, false, r.enabled); + + // By default rules are enabled/disabled for the default ruleset + if(r.enabled) + { + source->ruleset->enable(rule.name, true, cfg.default_ruleset_id); + } + else + { + source->ruleset->disable(rule.name, true, cfg.default_ruleset_id); + } } catch (falco_exception& e) { diff --git a/userspace/engine/rule_loader.h b/userspace/engine/rule_loader.h index 57964da3..47bb2c27 100644 --- a/userspace/engine/rule_loader.h +++ b/userspace/engine/rule_loader.h @@ -66,14 +66,19 @@ public: */ struct configuration { - explicit configuration(const std::string& cont): content(cont) {} + explicit configuration( + const std::string& cont, + const indexed_vector& srcs) + : content(cont), sources(srcs) {} + const std::string& content; + const indexed_vector& sources; + std::vector errors; + std::vector warnings; std::string output_extra; + uint16_t default_ruleset_id; bool replace_output_container_info; falco_common::priority_type min_priority; - indexed_vector sources; - std::vector warnings; - std::vector errors; }; /*! diff --git a/userspace/engine/ruleset.h b/userspace/engine/ruleset.h index 8ae5475e..82001c44 100644 --- a/userspace/engine/ruleset.h +++ b/userspace/engine/ruleset.h @@ -23,9 +23,8 @@ limitations under the License. #include /*! - \brief Represents a manager for rulesets.A ruleset represents a set of - enabled rules that is able to process events and find potential rule - matches. By convention, the ruleset with id = 0 is the default one. + \brief Manages a set of rulesets. A ruleset is a set of + enabled rules that is able to process events and find matches for those rules. */ class filter_ruleset {