diff --git a/userspace/engine/CMakeLists.txt b/userspace/engine/CMakeLists.txt index 72e55599..0cdea720 100644 --- a/userspace/engine/CMakeLists.txt +++ b/userspace/engine/CMakeLists.txt @@ -18,7 +18,8 @@ set(FALCO_ENGINE_SOURCE_FILES ruleset.cpp formats.cpp filter_macro_resolver.cpp - lua_filter_helper.cpp) + rule_loader.cpp + stats_manager.cpp) add_library(falco_engine STATIC ${FALCO_ENGINE_SOURCE_FILES}) add_dependencies(falco_engine njson string-view-lite) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index fa620516..a52095f9 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -41,14 +41,6 @@ falco_engine::falco_engine(bool seed_rng) m_sampling_ratio(1), m_sampling_multiplier(0), m_replace_container_info(false) { - luaopen_yaml(m_ls); - - falco_common::init(); - falco_rules::init(m_ls); - lua_filter_helper::init(m_ls); - - m_required_plugin_versions.clear(); - if(seed_rng) { srandom((unsigned) getpid()); @@ -59,6 +51,8 @@ falco_engine::falco_engine(bool seed_rng) falco_engine::~falco_engine() { + m_rule_loader.clear(); + m_rule_stats_manager.clear(); } uint32_t falco_engine::engine_version() @@ -154,18 +148,35 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version) { - if(!m_rules) + std::vector warnings; + std::vector errors; + m_rule_loader.configure(m_min_priority, m_replace_container_info, m_extra); + bool success = m_rule_loader.load(rules_content, this, warnings, errors); + std::ostringstream os; + if (!errors.empty()) { - m_rules.reset(new falco_rules(this, - m_ls)); - - for(auto const &it : m_filter_factories) + os << errors.size() << " errors:" << std::endl; + for(auto &err : errors) { - m_rules->add_filter_factory(it.first, it.second); + os << err << std::endl; } } - - m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info, m_min_priority, required_engine_version, m_required_plugin_versions); + if (!warnings.empty()) + { + os << warnings.size() << " warnings:" << std::endl; + for(auto &warn : warnings) + { + os << warn << std::endl; + } + } + if(!success) + { + throw falco_exception(os.str()); + } + if (verbose && os.str() != "") { + // todo(jasondellaluce): introduce a logging callback in Falco + fprintf(stderr, "When reading rules content: %s", os.str().c_str()); + } } void falco_engine::load_rules_file(const string &rules_filename, bool verbose, bool all_events) @@ -302,9 +313,8 @@ unique_ptr falco_engine::process_event(std::size_t so } unique_ptr res(new rule_result()); - res->source = r.source; - populate_rule_result(res, ev); + m_rule_stats_manager.on_event(m_rule_loader.rules(), ev->get_check_id()); return res; } @@ -333,82 +343,65 @@ std::size_t falco_engine::add_source(const std::string &source, return idx; } +std::shared_ptr falco_engine::get_filter_factory( + const std::string &source) +{ + auto it = m_filter_factories.find(source); + if(it == m_filter_factories.end()) + { + throw falco_exception(string("unknown event source: ") + source); + } + return it->second; +} + void falco_engine::populate_rule_result(unique_ptr &res, gen_event *ev) { - std::lock_guard guard(m_ls_semaphore); - lua_getglobal(m_ls, lua_on_event.c_str()); - if(lua_isfunction(m_ls, -1)) + res->evt = ev; + auto rule = m_rule_loader.rules().at(ev->get_check_id()); + if (!rule) { - lua_pushnumber(m_ls, ev->get_check_id()); - if(lua_pcall(m_ls, 1, 5, 0) != 0) - { - const char* lerr = lua_tostring(m_ls, -1); - string err = "Error invoking function output: " + string(lerr); - throw falco_exception(err); - } - const char *p = lua_tostring(m_ls, -5); - res->rule = p; - res->evt = ev; - res->priority_num = (falco_common::priority_type) lua_tonumber(m_ls, -4); - res->format = lua_tostring(m_ls, -3); - - // Tags are passed back as a table, and is on the top of the stack - lua_pushnil(m_ls); /* first key */ - while (lua_next(m_ls, -2) != 0) { - // key is at index -2, value is at index - // -1. We want the value. - res->tags.insert(luaL_checkstring(m_ls, -1)); - - // Remove value, keep key for next iteration - lua_pop(m_ls, 1); - } - lua_pop(m_ls, 1); // Clean table leftover - - // Exception fields are passed back as a table - lua_pushnil(m_ls); /* first key */ - while (lua_next(m_ls, -2) != 0) { - // key is at index -2, value is at index - // -1. We want the keys. - res->exception_fields.insert(luaL_checkstring(m_ls, -2)); - - // Remove value, keep key for next iteration - lua_pop(m_ls, 1); - } - - lua_pop(m_ls, 4); - } - else - { - throw falco_exception("No function " + lua_on_event + " found in lua compiler module"); + throw falco_exception("populate_rule_result error: unknown rule id " + + to_string(ev->get_check_id())); } + res->rule = rule->name; + res->source = rule->source; + res->format = rule->output; + res->priority_num = rule->priority; + res->tags = rule->tags; + res->exception_fields = rule->exception_fields; } void falco_engine::describe_rule(string *rule) { - return m_rules->describe_rule(rule); -} - -// Print statistics on the rules that triggered -void falco_engine::print_stats() -{ - lua_getglobal(m_ls, lua_print_stats.c_str()); - - if(lua_isfunction(m_ls, -1)) + static const char* rule_fmt = "%-50s %s\n"; + fprintf(stdout, rule_fmt, "Rule", "Description"); + fprintf(stdout, rule_fmt, "----", "-----------"); + if (!rule) { - if(lua_pcall(m_ls, 0, 0, 0) != 0) + for (uint32_t id = 0; id < m_rule_loader.rules().size(); id++) { - const char* lerr = lua_tostring(m_ls, -1); - string err = "Error invoking function print_stats: " + string(lerr); - throw falco_exception(err); + auto r = m_rule_loader.rules().at(id); + auto wrapped = falco::utils::wrap_text(r->description, 51, 110); + fprintf(stdout, rule_fmt, r->name.c_str(), wrapped.c_str()); } } else { - throw falco_exception("No function " + lua_print_stats + " found in lua rule loader module"); + auto r = m_rule_loader.rules().at(*rule); + auto wrapped = falco::utils::wrap_text(r->description, 51, 110); + fprintf(stdout, rule_fmt, r->name.c_str(), wrapped.c_str()); } } +void falco_engine::print_stats() +{ + string out; + m_rule_stats_manager.format_stats(m_rule_loader.rules(), out); + // todo(jasondellaluce): introduce a logging callback in Falco + fprintf(stdout, "%s", out.c_str()); +} + void falco_engine::add_filter(std::shared_ptr filter, std::string &rule, std::string &source, @@ -433,30 +426,7 @@ bool falco_engine::is_plugin_compatible(const std::string &name, const std::string &version, std::string &required_version) { - sinsp_plugin::version plugin_version(version); - - if(!plugin_version.m_valid) - { - throw falco_exception(string("Plugin version string ") + version + " not valid"); - } - - if(m_required_plugin_versions.find(name) == m_required_plugin_versions.end()) - { - // No required engine versions, so no restrictions. Compatible. - return true; - } - - for(auto &rversion : m_required_plugin_versions[name]) - { - sinsp_plugin::version req_version(rversion); - if (!plugin_version.check(req_version)) - { - required_version = rversion; - return false; - } - } - - return true; + return m_rule_loader.is_plugin_compatible(name, version, required_version); } void falco_engine::clear_filters() @@ -465,8 +435,6 @@ void falco_engine::clear_filters() { it.ruleset.reset(new falco_ruleset); } - - m_required_plugin_versions.clear(); } void falco_engine::set_sampling_ratio(uint32_t sampling_ratio) diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 2a6e5ec8..e74c5184 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -30,7 +30,8 @@ limitations under the License. #include "gen_filter.h" #include "ruleset.h" - +#include "rule_loader.h" +#include "stats_manager.h" #include "falco_common.h" // @@ -178,6 +179,11 @@ public: std::shared_ptr filter_factory, std::shared_ptr formatter_factory); + // todo(jasondellaluce): this is here for internal use, and + // will possibly be removed in the future + std::shared_ptr get_filter_factory( + const std::string &source); + // Return whether or not there is a valid filter/formatter // factory for this source. bool is_source_valid(const std::string &source); @@ -241,15 +247,13 @@ private: // Maps from event source to the set of rules for that event source std::vector m_rulesets; - std::unique_ptr m_rules; + rule_loader m_rule_loader; + stats_manager m_rule_stats_manager; + uint16_t m_next_ruleset_id; std::map m_known_rulesets; falco_common::priority_type m_min_priority; - // Maps from plugin to a list of required plugin versions - // found in any loaded rules files. - std::map> m_required_plugin_versions; - void populate_rule_result(unique_ptr &res, gen_event *ev); // diff --git a/userspace/engine/rule_loader.cpp b/userspace/engine/rule_loader.cpp new file mode 100644 index 00000000..e6c288f6 --- /dev/null +++ b/userspace/engine/rule_loader.cpp @@ -0,0 +1,1095 @@ +/* +Copyright (C) 2022 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "falco_engine.h" +#include "rule_loader.h" +#include "filter_macro_resolver.h" + +#define MAX_VISIBILITY ((uint32_t) -1) +#define THROW(cond, err) { if (cond) { throw falco_exception(err); } } + +static string s_container_info_fmt = "%container.info"; +static string s_default_extra_fmt = "%container.name (id=%container.id)"; + +using namespace std; +using namespace libsinsp::filter; + +string ctxerr(std::string ctx, std::string e) +{ + e += "\n---\n"; + e += trim(ctx); + e += "\n---"; + return e; +} + +// todo(jasondellaluce): this breaks string escaping in lists and exceptions +static void quote_item(string& e) +{ + if (e.find(" ") != string::npos && e[0] != '"' && e[0] != '\'') + { + e = '"' + e + '"'; + } +} + +static void paren_item(std::string& e) +{ + if(e[0] != '(') + { + e = '(' + e + ')'; + } +} + +static bool is_field_defined(falco_engine *engine, string source, string field) +{ + auto factory = engine->get_filter_factory(source); + if(factory) + { + auto *chk = factory->new_filtercheck(field.c_str()); + if (chk) + { + delete(chk); + return true; + } + } + return false; +} + +// todo: this should be in libsinsp +static bool is_operator_defined(std::string op) +{ + static vector ops = {"=", "==", "!=", "<=", ">=", "<", ">", + "contains", "icontains", "bcontains", "glob", "bstartswith", + "startswith", "endswith", "in", "intersects", "pmatch"}; + return find(ops.begin(), ops.end(), op) != ops.end(); +} + +// todo: this should be in libsinsp +static bool is_operator_for_list(std::string op) +{ + return op == "in" || op == "intersects" || op == "pmatch"; +} + +static bool is_format_valid(falco_engine* e, string src, string fmt, string& err) +{ + try + { + shared_ptr formatter; + formatter = e->create_formatter(src, fmt); + return true; + } + catch(exception &e) + { + err = e.what(); + return false; + } +} + +static string yaml_format_object( + const string& content, + const vector& docs, + vector::iterator doc, + YAML::iterator node) +{ + YAML::Node item = *node++; + YAML::Node cur_doc = *doc++; + // include the "- " sequence mark + size_t from = item.Mark().pos - 2; + size_t to = 0; + if (node != cur_doc.end()) + { + // end of item is beginning of next item + to = node->Mark().pos - 2; + } + else if (doc != docs.end()) + { + // end of item is beginning of next doc + to = doc->Mark().pos - 4; + } + else + { + // end of item is end of file contents + to = content.length(); + } + string obj = content.substr(from, to - from); + return trim(obj); +} + +template +static bool yaml_is_type(const YAML::Node& v) +{ + T t; + return v.IsDefined() && v.IsScalar() && YAML::convert::decode(v, t); +} + +template +static void yaml_decode_seq(const YAML::Node& item, vector& out) +{ + T value; + for(const YAML::Node& v : item) + { + THROW(!v.IsScalar() || !YAML::convert::decode(v, value), + "Can't decode YAML sequence value"); + out.push_back(value); + } +} + +template +static void yaml_decode_seq(const YAML::Node& item, set& out) +{ + T value; + for(const YAML::Node& v : item) + { + THROW(!v.IsScalar() || !YAML::convert::decode(v, value), + "Can't decode YAML sequence value"); + out.insert(value); + } +} + +// todo(jasondellaluce): this breaks string escaping in lists +static bool resolve_list(string& cnd, YAML::Node& list) +{ + static string blanks = " \t\n\r"; + static string delims = blanks + "(),="; + string new_cnd; + size_t start, end; + bool used = false; + start = cnd.find(list["list"].as()); + while (start != string::npos) + { + // the characters surrounding the name must + // be delims of beginning/end of string + end = start + list["list"].as().length(); + if ((start == 0 || delims.find(cnd[start - 1]) != string::npos) + && (end >= cnd.length() || delims.find(cnd[end]) != string::npos)) + { + // shift pointers to consume all whitespaces + while (start > 0 + && blanks.find(cnd[start - 1]) != string::npos) + { + start--; + } + while (end < cnd.length() + && blanks.find(cnd[end]) != string::npos) + { + end++; + } + // create substitution string by concatenating all values + string sub = ""; + for (auto v : list["items"]) + { + if (!sub.empty()) + { + sub += ", "; + } + sub += v.as(); + } + // if substituted list is empty, we need to + // remove a comma from the left or the right + if (sub.empty()) + { + if (start > 0 && cnd[start - 1] == ',') + { + start--; + } + else if (end < cnd.length() && cnd[end] == ',') + { + end++; + } + } + // compose new string with substitution + new_cnd = ""; + if (start > 0) + { + new_cnd += cnd.substr(0, start) + " "; + } + new_cnd += sub + " "; + if (end <= cnd.length()) + { + new_cnd += cnd.substr(end); + } + cnd = new_cnd; + start += sub.length() + 1; + used = true; + } + start = cnd.find(list["list"].as(), start + 1); + } + return used; +} + +static void resolve_macros( + indexed_vector>>& macros, + map& used_macros, + shared_ptr& ast, + uint32_t index_visibility, + string on_unknown_err_prefix) +{ + filter_macro_resolver macro_resolver; + + for (auto &ref : macros) + { + if (ref.first["index"].as() < index_visibility) + { + macro_resolver.set_macro(ref.first["macro"].as(), ref.second); + } + } + macro_resolver.run(ast); + THROW(!macro_resolver.get_unknown_macros().empty(), on_unknown_err_prefix + + "Undefined macro '" + *macro_resolver.get_unknown_macros().begin() + + "' used in filter."); + for (auto &resolved : macro_resolver.get_resolved_macros()) + { + used_macros[resolved] = true; + } +} + +// note: there is no visibility order between filter conditions and lists +static shared_ptr parse_condition(string cnd, + std::map& used_lists, indexed_vector& lists) +{ + for (auto &l : lists) + { + if (resolve_list(cnd, l)) + { + used_lists[l["list"].as()] = true; + } + } + libsinsp::filter::parser p(cnd); + p.set_max_depth(1000); + try + { + shared_ptr res_ptr(p.parse()); + return res_ptr; + } + catch (const sinsp_exception& e) + { + throw falco_exception("Compilation error when compiling \"" + + cnd + "\": " + to_string(p.get_pos().col) + ": " + e.what()); + } +} + +static shared_ptr compile_condition( + falco_engine* engine, uint32_t id, shared_ptr cnd, + string src, string& err) +{ + try + { + auto factory = engine->get_filter_factory(src); + sinsp_filter_compiler compiler(factory, cnd.get()); + compiler.set_check_id(id); + shared_ptr ret(compiler.compile()); + return ret; + } + catch (const sinsp_exception& e) + { + err = e.what(); + } + catch (const falco_exception& e) + { + err = e.what(); + } + return nullptr; +} + +static void define_info(indexed_vector& infos, + YAML::Node& item, string name, uint32_t id) +{ + auto prev = infos.at(name); + if (prev) + { + item["index"] = (*prev)["index"]; + item["index_visibility"] = id; + (*prev) = item; + } + else + { + item["index"] = id; + item["index_visibility"] = id; + infos.insert(item, name); + } +} + +static void append_infos(YAML::Node& item, YAML::Node& append, uint32_t id) +{ + item["index_visibility"] = id; + item["context"] = item["context"].as() + + "\n\n" + append["context"].as(); +} + +static void validate_rule_exception( + falco_engine* engine, YAML::Node& ex, string source) +{ + switch (ex["fields"].Type()) + { + case YAML::NodeType::Scalar: + if (!ex["comps"].IsDefined()) + { + ex["comps"] = "in"; + } + else + { + THROW(!yaml_is_type(ex["fields"]) + || !yaml_is_type(ex["comps"]), + "Rule exception item " + ex["name"].as() + + ": fields and comps must both be strings"); + } + THROW(!is_field_defined( + engine, source, ex["fields"].as()), + "Rule exception item " + ex["name"].as() + + ": field name " + ex["fields"].as() + + " is not a supported filter field"); + THROW(!is_operator_defined(ex["comps"].as()), + "Rule exception item " + ex["name"].as() + + ": comparison operator " + ex["comps"].as() + + " is not a supported comparison operator"); + break; + case YAML::NodeType::Sequence: + if (!ex["comps"].IsDefined()) + { + ex["comps"] = vector(); + for (size_t i = 0; i < ex["fields"].size(); i++) + { + ex["comps"].push_back("="); + } + } + else + { + THROW(ex["fields"].size() != ex["comps"].size(), + "Rule exception item " + ex["name"].as() + + ": fields and comps lists must have equal length"); + } + for (auto field : ex["fields"]) + { + THROW(!yaml_is_type(field), + "Rule exception item " + ex["name"].as() + ": fields must strings "); + THROW(!is_field_defined(engine, source, field.as()), + "Rule exception item " + ex["name"].as() + ": field name " + + field.as() + " is not a supported filter field"); + } + for (auto comp : ex["comps"]) + { + THROW(!yaml_is_type(comp), + "Rule exception item " + ex["name"].as() + + ": comps must strings "); + THROW(!is_operator_defined(comp.as()), + "Rule exception item " + ex["name"].as() + + ": comparison operator " + comp.as() + + " is not a supported comparison operator"); + } + break; + default: + throw falco_exception( + "Rule exception fields must be a sequence or a string"); + } +} + +static void build_rule_exception_infos( + YAML::Node exceptions, set& exception_fields, string& condition) +{ + for (auto ex : exceptions) + { + string icond; + string value; + string exname = ex["name"].as(); + if(ex["fields"].IsScalar()) + { + for (auto val : ex["values"]) + { + THROW(!yaml_is_type(val), + "Expected values array for item " + + exname + " to contain a list of strings"); + icond += icond.empty() + ? "(" + ex["fields"].as() + " " + + ex["comps"].as() + " (" + : ", "; + exception_fields.insert(ex["fields"].as()); + value = val.as(); + quote_item(value); + icond += value; + } + icond += icond.empty() ? "" : "))"; + } + else + { + icond = "("; + for (auto values : ex["values"]) + { + THROW(ex["fields"].size() != values.size(), + "Exception item " + exname + + ": fields and values lists must have equal length"); + icond += icond == "(" ? "" : " or "; + icond += "("; + uint32_t k = 0; + string istr; + for (auto field : ex["fields"]) + { + icond += k == 0 ? "" : " and "; + if (values[k].IsSequence()) + { + istr = "("; + for (auto v : values[k]) + { + value = v.as(); + quote_item(value); + istr += istr == "(" ? "" : ", "; + istr += value; + } + istr += ")"; + } + else + { + istr = values[k].as(); + if(is_operator_for_list(ex["comps"][k].as())) + { + paren_item(istr); + } + else + { + quote_item(istr); + } + } + icond += " " + ex["fields"][k].as() + + " " + ex["comps"][k].as() + + " " + istr; + exception_fields.insert(ex["fields"][k].as()); + k++; + } + icond += ")"; + } + icond += ")"; + if (icond == "()") + { + icond = ""; + } + } + condition += icond.empty() ? "" : " and not " + icond; + } +} + +void rule_loader::clear() +{ + m_cur_index = 0; + m_rules.clear(); + m_rule_infos.clear(); + m_list_infos.clear(); + m_macro_infos.clear(); + m_required_plugin_versions.clear(); +} + +indexed_vector& rule_loader::rules() +{ + return m_rules; +} + +void rule_loader::configure( + falco_common::priority_type min_priority, + bool replace_container_info, + const string& extra) +{ + m_extra = extra; + m_min_priority = min_priority; + m_replace_container_info = replace_container_info; +} + +bool rule_loader::is_plugin_compatible( + const string &name, + const string &version, + string &required_version) +{ + set required_plugin_versions; + sinsp_plugin::version plugin_version(version); + if(!plugin_version.m_valid) + { + throw falco_exception( + string("Plugin version string ") + version + " not valid"); + } + auto it = m_required_plugin_versions.find(name); + if (it != m_required_plugin_versions.end()) + { + for(auto &rversion : it->second) + { + sinsp_plugin::version req_version(rversion); + if (!plugin_version.check(req_version)) + { + required_version = rversion; + return false; + } + } + } + return true; +} + +void rule_loader::apply_output_replacements(std::string& out) +{ + if (out.find(s_container_info_fmt) != string::npos) + { + if (m_replace_container_info) + { + out = replace(out, s_container_info_fmt, m_extra); + return; + } + out = replace(out, s_container_info_fmt, s_default_extra_fmt); + } + out += m_extra.empty() ? "" : " " + m_extra; +} + +bool rule_loader::load(const string &content, falco_engine* engine, + vector& warnings, vector& errors) +{ + if (read(content, engine, warnings, errors)) + { + m_rules.clear(); + engine->clear_filters(); + return expand(engine, warnings, errors); + } + return false; +} + +bool rule_loader::read(const string &content, falco_engine* engine, + vector& warnings, vector& errors) +{ + std::vector docs; + try + { + docs = YAML::LoadAll(content); + } + catch(const exception& e) + { + errors.push_back("Could not load YAML file: " + string(e.what())); + return false; + } + + for (auto doc = docs.begin(); doc != docs.end(); doc++) + { + if (doc->IsDefined() && !doc->IsNull()) + { + if(!doc->IsMap() && !doc->IsSequence()) + { + errors.push_back("Rules content is not yaml"); + return false; + } + if(!doc->IsSequence()) + { + errors.push_back("Rules content is not yaml array of objects"); + return false; + } + for(auto it = doc->begin(); it != doc->end(); it++) + { + string ctx = yaml_format_object(content, docs, doc, it); + YAML::Node item = *it; + try + { + THROW(!item.IsMap(), "Unexpected element type. " + "Each element should be a yaml associative array."); + item["context"] = ctx; + read_item(engine, item, warnings); + } + catch(const exception& e) + { + errors.push_back(ctxerr(ctx, e.what())); + return false; + } + } + } + } + return true; +} + +void rule_loader::read_item( + falco_engine* engine, YAML::Node& item, vector& warn) +{ + if (item["required_engine_version"].IsDefined()) + { + read_required_engine_version(engine, item, warn); + } + else if(item["required_plugin_versions"].IsDefined()) + { + read_required_plugin_versions(engine, item, warn); + } + else if(item["macro"].IsDefined()) + { + read_macro(engine, item, warn); + } + else if(item["list"].IsDefined()) + { + read_list(engine, item, warn); + } + else if(item["rule"].IsDefined()) + { + read_rule(engine, item, warn); + } + else + { + warn.push_back("Unknown top level object"); + } +} + +void rule_loader::read_required_engine_version( + falco_engine* engine, YAML::Node& item, vector& warn) +{ + uint32_t v = 0; + THROW(!YAML::convert::decode(item["required_engine_version"], v), + "Value of required_engine_version must be a number"); + auto engine_ver = falco_engine::engine_version(); + THROW(engine_ver < v, "Rules require engine version " + to_string(v) + + ", but engine version is " + to_string(engine_ver)); +} + +void rule_loader::read_required_plugin_versions( + falco_engine* engine, YAML::Node& item, vector& warn) +{ + string name, ver; + THROW(!item["required_plugin_versions"].IsSequence(), + "Value of required_plugin_versions must be a sequence"); + for(const YAML::Node& plugin : item["required_plugin_versions"]) + { + THROW(!plugin["name"].IsDefined() + || !YAML::convert::decode(plugin["name"], name) + || name.empty(), + "required_plugin_versions item must have name property"); + THROW(!plugin["version"].IsDefined() + || !YAML::convert::decode(plugin["version"], ver) + || ver.empty(), + "required_plugin_versions item must have version property"); + m_required_plugin_versions[name].insert(ver); + } +} + +void rule_loader::read_list( + falco_engine* engine, YAML::Node& item, vector& warn) +{ + string name; + THROW(!YAML::convert::decode(item["list"], name) || name.empty(), + "List name is empty"); + + THROW(!item["items"].IsDefined() || !item["items"].IsSequence(), + "List must have property items"); + + if(item["append"].IsDefined() && item["append"].as()) + { + auto prev = m_list_infos.at(name); + THROW(!prev, "List " + name + + " has 'append' key but no list by that name already exists"); + for (auto val : item["items"]) + { + (*prev)["items"].push_back(val.as()); + } + append_infos(*prev, item, m_cur_index++); + return; + } + define_info(m_list_infos, item, name, m_cur_index++); +} + +void rule_loader::read_macro( + falco_engine* engine, YAML::Node& item, vector& warn) +{ + string name, cnd; + THROW(!YAML::convert::decode(item["macro"], name) || name.empty(), + "Macro name is empty"); + + THROW(!item["condition"].IsDefined() + || !YAML::convert::decode(item["condition"], cnd) + || cnd.empty(), + "Macro must have property condition"); + + if (!yaml_is_type(item["source"]) + || item["source"].as().empty()) + { + item["source"] = falco_common::syscall_source; + } + if (!engine->is_source_valid(item["source"].as())) + { + warn.push_back("Macro " + name + + ": warning (unknown-source): unknown source " + + item["source"].as() + ", skipping"); + return; + } + + if(item["append"].IsDefined() && item["append"].as()) + { + auto prev = m_macro_infos.at(name); + THROW(!prev, "Macro " + name + + " has 'append' key but no macro by that name already exists"); + (*prev)["condition"] = (*prev)["condition"].as() + " " + cnd; + append_infos(*prev, item, m_cur_index++); + return; + } + define_info(m_macro_infos, item, name, m_cur_index++); +} + +void rule_loader::read_rule( + falco_engine* engine, YAML::Node& item, vector& warn) +{ + string name; + falco_common::priority_type priority; + THROW(!YAML::convert::decode(item["rule"], name) || name.empty(), + "Rule name is empty"); + + auto prev = m_rule_infos.at(name); + + if (!yaml_is_type(item["skip-if-unknown-filter"])) + { + item["skip-if-unknown-filter"] = false; + } + if (!yaml_is_type(item["warn_evttypes"])) + { + item["warn_evttypes"] = true; + } + if (!yaml_is_type(item["append"])) + { + item["append"] = false; + } + + if (!yaml_is_type(item["source"]) + || item["source"].as().empty()) + { + item["source"] = falco_common::syscall_source; + } + if (!engine->is_source_valid(item["source"].as())) + { + warn.push_back("Rule " + name + + ": warning (unknown-source): unknown source " + + item["source"].as() + ", skipping"); + return; + } + THROW(prev && (*prev)["source"].as() != item["source"].as(), + "Rule " + name + " has been re-defined with a different source"); + + if (item["append"].as()) + { + THROW(!prev, "Rule " + name + + " has 'append' key but no rule by that name already exists"); + THROW(!item["condition"].IsDefined() && !item["exceptions"].IsDefined(), + "Appended rule must have exceptions or condition property"); + + if (item["exceptions"].IsDefined()) + { + read_rule_exceptions(engine, item, true); + } + + if (item["condition"].IsDefined()) + { + (*prev)["condition"] = (*prev)["condition"].as() + + " " + item["condition"].as(); + } + append_infos(*prev, item, m_cur_index++); + return; + } + + if (!item["condition"].IsDefined() || !item["output"].IsDefined() + || !item["desc"].IsDefined() || !item["priority"].IsDefined()) + { + // we support enabled-only rules + THROW(!yaml_is_type(item["enabled"]), + "Rule must have properties 'condition', 'output', 'desc', and 'priority'"); + auto prev = m_rule_infos.at(name); + THROW(!prev, "Rule " + name + + " has 'enabled' key but no rule by that name already exists"); + (*prev)["enabled"] = item["enabled"].as(); + return; + } + + if (!yaml_is_type(item["enabled"])) + { + item["enabled"] = true; + } + + THROW(!yaml_is_type(item["priority"]) + || !falco_common::parse_priority(item["priority"].as(), priority), + "Invalid priority"); + item["priority_num"] = (uint32_t) priority; + + string output = item["output"].as(); + item["output"] = trim(output); + + if (item["exceptions"].IsDefined()) + { + read_rule_exceptions(engine, item, false); + } + + define_info(m_rule_infos, item, name, m_cur_index++); +} + +void rule_loader::read_rule_exceptions( + falco_engine* engine, YAML::Node& item, bool append) +{ + string exname; + string rule = item["rule"].as(); + THROW(!item["exceptions"].IsSequence(), "Rule exceptions must be a sequence"); + for (auto ex : item["exceptions"]) + { + THROW(!YAML::convert::decode(ex["name"], exname) + || exname.empty(), "Rule exception item must have name property"); + + if(!ex["values"].IsDefined()) + { + ex["values"] = vector({}); + } + + if (append) + { + bool is_new = true; + auto prev = m_rule_infos.at(rule); + YAML::Node prev_ex; + for (YAML::Node e : (*prev)["exceptions"]) + { + if (is_new && e["name"].as() == exname) + { + prev_ex = e; + is_new = false; + } + } + if (is_new) + { + THROW(!ex["fields"].IsDefined(), + "Rule exception new item " + exname + + ": must have fields property with a list of fields"); + THROW(!ex["values"].IsDefined(), + "Rule exception new item " + exname + + ": must have fields property with a list of values"); + validate_rule_exception(engine, ex, item["source"].as()); + (*prev)["exceptions"].push_back(ex); + } + else + { + THROW(ex["fields"].IsDefined(), + "Can not append exception fields to existing rule, only values"); + THROW(ex["comps"].IsDefined(), + "Can not append exception comps to existing rule, only values"); + for (auto vals : ex["values"]) + { + prev_ex["values"].push_back(vals); + } + } + } + else + { + THROW(!ex["fields"].IsDefined(), + "Rule exception item " + exname + + ": must have fields property with a list of fields"); + validate_rule_exception(engine, ex, item["source"].as()); + } + } +} + +bool rule_loader::expand(falco_engine* engine, + vector& warnings, vector& errors) +{ + indexed_vector lists; + indexed_vector // todo: maybe remove pair + >> macros; + map used_lists; + map used_macros; + + // expand all lists, macros, and rules + try + { + expand_list_infos(used_lists, lists); + expand_macro_infos(lists, used_lists, used_macros, macros); + expand_rule_infos(engine, lists, macros, used_lists, used_macros, warnings); + } + catch (exception& e) + { + errors.push_back(e.what()); + return false; + } + + // print info on any dangling lists or macros that were not used anywhere + for (auto &m : macros) + { + if (!used_macros[m.first["macro"].as()]) + { + warnings.push_back("macro " + m.first["macro"].as() + + " not referred to by any rule/macro"); + } + } + for (auto &l : lists) + { + if (!used_lists[l["list"].as()]) + { + warnings.push_back("list " + l["list"].as() + + " not referred to by any rule/macro/list"); + } + } + return true; +} + +// note: there is a visibility ordering between lists +void rule_loader::expand_list_infos( + map& used, indexed_vector& out) +{ + string value; + vector values; + for (auto l : m_list_infos) + { + try + { + values.clear(); + for (auto item : l["items"]) + { + value = item.as(); + auto ref = m_list_infos.at(value); + if (ref && (*ref)["index"].as() < l["index_visibility"].as()) + { + used[value] = true; + for (auto val : (*ref)["items"]) + { + value = val.as(); + quote_item(value); + values.push_back(value); + } + } + else + { + quote_item(value); + values.push_back(value); + } + } + auto new_list = YAML::Clone(l); + new_list["items"] = values; + out.insert(new_list, new_list["list"].as()); + } + catch (exception& e) + { + throw falco_exception(ctxerr(l["context"].as(), e.what())); + } + } +} + +// note: there is a visibility ordering between macros +void rule_loader::expand_macro_infos( + indexed_vector& lists, + map& used_lists, + map& used_macros, + indexed_vector>>& out) +{ + for (auto m : m_macro_infos) + { + try + { + auto ast = parse_condition(m["condition"].as(), used_lists, lists); + auto pair = make_pair(m, ast); + out.insert(pair, m["macro"].as()); + } + catch (exception& e) + { + throw falco_exception(ctxerr(m["context"].as(), e.what())); + } + } + for (auto &m : out) + { + try + { + resolve_macros(out, used_macros, m.second, + m.first["index_visibility"].as(), + "Compilation error when compiling \"" + + m.first["condition"].as() + "\": "); + } + catch (exception& e) + { + throw falco_exception( + ctxerr(m.first["context"].as(), e.what())); + } + } +} + +void rule_loader::expand_rule_infos( + falco_engine* engine, + indexed_vector& lists, + indexed_vector>>& macros, + map& used_lists, + map& used_macros, + vector& warn) +{ + string err; + for (auto r : m_rule_infos) + { + try + { + uint32_t priority = r["priority_num"].as(); + if ((falco_common::priority_type) priority > m_min_priority) + { + continue; + } + + set exception_fields; + string condition = r["condition"].as(); + if (r["exceptions"].IsDefined()) + { + build_rule_exception_infos( + r["exceptions"], exception_fields, condition); + } + + auto ast = parse_condition(condition, used_lists, lists); + + resolve_macros(macros, used_macros, ast, MAX_VISIBILITY, ""); + + string output = r["output"].as(); + if (r["source"].as() == falco_common::syscall_source) + { + apply_output_replacements(output); + } + + THROW(!is_format_valid(engine, r["source"].as(), output, err), + "Invalid output format '" + output + "': '" + err + "'"); + + falco_rule rule; + rule.name = r["rule"].as(); + rule.source = r["source"].as(); + rule.description = r["desc"].as(); + rule.output = output; + rule.priority = (falco_common::priority_type) priority; + rule.exception_fields = exception_fields; + yaml_decode_seq(r["tags"], rule.tags); + + auto rule_id = m_rules.insert(rule, rule.name); + auto filter = compile_condition(engine, rule_id, ast, rule.source, err); + if (!filter) + { + if (r["skip-if-unknown-filter"].as() + && err.find("nonexistent field") != string::npos) + { + warn.push_back( + "Rule " + rule.name + ": warning (unknown-field):"); + continue; + } + else + { + throw falco_exception("Rule " + rule.name + ": error " + err); + } + } + engine->add_filter(filter, rule.name, rule.source, rule.tags); + if (rule.source == falco_common::syscall_source + && r["warn_evttypes"].as()) + { + auto evttypes = filter->evttypes(); + if (evttypes.size() == 0 || evttypes.size() > 100) + { + warn.push_back( + "Rule " + rule.name + ": warning (no-evttype):\n" + + + " matches too many evt.type values.\n" + + " This has a significant performance penalty."); + } + } + engine->enable_rule(rule.name, r["enabled"].as()); + } + catch (exception& e) + { + throw falco_exception(ctxerr(r["context"].as(), e.what())); + } + } +} \ No newline at end of file diff --git a/userspace/engine/rule_loader.h b/userspace/engine/rule_loader.h new file mode 100644 index 00000000..a396f4d1 --- /dev/null +++ b/userspace/engine/rule_loader.h @@ -0,0 +1,130 @@ +/* +Copyright (C) 2022 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +#include +#include +#include +#include +#include "falco_rule.h" +#include "indexed_vector.h" + +// todo(jasondellaluce): remove this cyclic dependency +class falco_engine; + + +/*! + \brief Ruleset loader of the falco engine +*/ +class rule_loader +{ +public: + + /*! + \brief Erases the internal states and all the loaded rules + */ + virtual void clear(); + + /*! + \brief Returns the rules loaded after the last invocation of load() + */ + virtual indexed_vector& rules(); + + /*! + \brief Configures the loader. The changes will influence the next + invocation of load(). + \param min_priority The minimum priority below which rules are skipped + by the loader + \param extra Text to be appended/substituted in the output of all rules + \param replace_container_info If true, the extra string is used to + replace the "%container.info" token in rules outputs. If false, the + "%container.info" token is substituted with a default text and the + extra string is appended at the end of the rule output. If a rule + output does not contain "%container.info", then this flag has no effect + and the extra string is appended at the end of the rule output anyways. + */ + virtual void configure(falco_common::priority_type min_priority, + bool replace_container_info, const std::string& extra); + + /*! + \brief Returns true if the given plugin name and version are compatible + with the loaded rulesets. If false is returned, required_version is + filled with the required plugin version that didn't match. + */ + virtual bool is_plugin_compatible(const std::string& name, + const std::string& version, std::string& required_version); + + /*! + \brief Parses the content of a ruleset. This should be called multiple + times to load different ruleset files. The internal state (e.g. loaded + rules, plugin version requirements, etc...) gets updated at each + invocation of the load() method. + \param rules_content The contents of the ruleset file + \param engine The instance of falco_engine used to add rule filters + \param warnings Filled-out with warnings + \param warnings Filled-out with errors + \return true if the ruleset content is loaded successfully + */ + virtual bool load(const std::string& rules_content, falco_engine* engine, + std::vector& warnings, std::vector& errors); + +private: + bool read( + const std::string& content, falco_engine* engine, + std::vector& warnings, std::vector& errors); + void read_item( + falco_engine* engine, YAML::Node& item, vector& warn); + void read_required_engine_version( + falco_engine* engine, YAML::Node& item, vector& warn); + void read_required_plugin_versions( + falco_engine* engine, YAML::Node& item, vector& warn); + void read_macro( + falco_engine* engine, YAML::Node& item, vector& warn); + void read_list( + falco_engine* engine, YAML::Node& item, vector& warn); + void read_rule( + falco_engine* engine, YAML::Node& item, vector& warn); + void read_rule_exceptions( + falco_engine* engine, YAML::Node& item, bool append); + bool expand(falco_engine* engine, + std::vector& warnings, std::vector& errors); + void expand_list_infos( + std::map& used, indexed_vector& out); + void expand_macro_infos( + indexed_vector& lists, + std::map& used_lists, + std::map& used_macros, + indexed_vector>>& out); + void expand_rule_infos( + falco_engine* engine, + indexed_vector& lists, + indexed_vector>>& macros, + std::map& used_lists, + std::map& used_macros, + vector& warnings); + void apply_output_replacements(std::string& output); + + uint32_t m_cur_index; + std::string m_extra; + bool m_replace_container_info; + falco_common::priority_type m_min_priority; + indexed_vector m_rules; + indexed_vector m_rule_infos; + indexed_vector m_macro_infos; + indexed_vector m_list_infos; + std::map> m_required_plugin_versions; +};