From 39f55f4b5ce70e31c2ead6998d677f3ef45c52c6 Mon Sep 17 00:00:00 2001 From: Federico Di Pierro Date: Fri, 13 May 2022 13:01:29 +0200 Subject: [PATCH] update(userspace): split filterchecks list for each source idx. Signed-off-by: Federico Di Pierro --- userspace/engine/rule_loader.cpp | 5 +- .../falco/app_actions/init_falco_engine.cpp | 6 +- userspace/falco/app_actions/list_plugins.cpp | 28 ++- userspace/falco/app_actions/load_plugins.cpp | 191 ++++++++++-------- .../falco/app_actions/load_rules_files.cpp | 10 +- .../falco/app_actions/open_inspector.cpp | 47 ++--- .../falco/app_actions/process_events.cpp | 28 ++- userspace/falco/app_cmdline_options.cpp | 4 +- 8 files changed, 182 insertions(+), 137 deletions(-) diff --git a/userspace/engine/rule_loader.cpp b/userspace/engine/rule_loader.cpp index 85ec4f80..70d95ca6 100644 --- a/userspace/engine/rule_loader.cpp +++ b/userspace/engine/rule_loader.cpp @@ -20,6 +20,7 @@ limitations under the License. #include "filter_macro_resolver.h" #include "filter_evttype_resolver.h" #include "filter_warning_resolver.h" +#include #define MAX_VISIBILITY ((uint32_t) -1) #define THROW(cond, err) { if (cond) { throw falco_exception(err); } } @@ -426,7 +427,7 @@ bool rule_loader::is_plugin_compatible( string &required_version) { set required_plugin_versions; - sinsp_plugin::version plugin_version(version); + sinsp_version plugin_version(version); if(!plugin_version.m_valid) { throw falco_exception( @@ -437,7 +438,7 @@ bool rule_loader::is_plugin_compatible( { for (auto &rversion : it->second) { - sinsp_plugin::version req_version(rversion); + sinsp_version req_version(rversion); if (!plugin_version.check(req_version)) { required_version = rversion; diff --git a/userspace/falco/app_actions/init_falco_engine.cpp b/userspace/falco/app_actions/init_falco_engine.cpp index def756a4..42b85032 100644 --- a/userspace/falco/app_actions/init_falco_engine.cpp +++ b/userspace/falco/app_actions/init_falco_engine.cpp @@ -64,7 +64,7 @@ application::run_result application::init_falco_engine() // libs requires raw pointer, we should modify libs to use reference/shared_ptr std::shared_ptr syscall_formatter_factory(new sinsp_evt_formatter_factory(m_state->inspector.get())); - m_state->syscall_source_idx = m_state->engine->add_source(application::s_syscall_source, syscall_filter_factory, syscall_formatter_factory); + m_state->syscall_source_idx = m_state->engine->add_source(falco_common::syscall_source, syscall_filter_factory, syscall_formatter_factory); if(m_state->config->m_json_output) { @@ -73,6 +73,10 @@ application::run_result application::init_falco_engine() for(const auto &src : m_options.disable_sources) { + if (m_state->enabled_sources.find(src) == m_state->enabled_sources.end()) + { + throw std::invalid_argument("Attempted disabling unknown event source: " + src); + } m_state->enabled_sources.erase(src); } diff --git a/userspace/falco/app_actions/list_plugins.cpp b/userspace/falco/app_actions/list_plugins.cpp index 7cfa4e7a..5c6e70cd 100644 --- a/userspace/falco/app_actions/list_plugins.cpp +++ b/userspace/falco/app_actions/list_plugins.cpp @@ -15,6 +15,7 @@ limitations under the License. */ #include "application.h" +#include using namespace falco::app; @@ -25,23 +26,28 @@ application::run_result application::list_plugins() if(m_options.list_plugins) { std::ostringstream os; - - for(auto &info : m_state->plugin_infos) + const auto &plugins = m_state->inspector->get_plugin_manager()->plugins(); + for (auto &p : plugins) { - os << "Name: " << info.name << std::endl; - os << "Description: " << info.description << std::endl; - os << "Contact: " << info.contact << std::endl; - os << "Version: " << info.plugin_version.as_string() << std::endl; - os << "Capabilities: " << info.caps << std::endl; - - if(info.caps & CAP_SOURCING) + os << "Name: " << p->name() << std::endl; + os << "Description: " << p->description() << std::endl; + os << "Contact: " << p->contact() << std::endl; + os << "Version: " << p->plugin_version().as_string() << std::endl; + os << "Capabilities: " << std::endl; + if(p->caps() & CAP_SOURCING) { - os << "ID: " << info.id << std::endl; + os << " - Event Sourcing: (ID=" << p->id(); + os << ", source='" << p->event_source() << "')" << std::endl; } + if(p->caps() & CAP_EXTRACTION) + { + os << " - Field Extraction" << std::endl; + } + os << std::endl; } - printf("%lu Plugins Loaded:\n\n%s\n", m_state->plugin_infos.size(), os.str().c_str()); + printf("%lu Plugins Loaded:\n\n%s\n", plugins.size(), os.str().c_str()); ret.proceed = false; } diff --git a/userspace/falco/app_actions/load_plugins.cpp b/userspace/falco/app_actions/load_plugins.cpp index a7dca031..08c492d0 100644 --- a/userspace/falco/app_actions/load_plugins.cpp +++ b/userspace/falco/app_actions/load_plugins.cpp @@ -15,6 +15,7 @@ limitations under the License. */ #include "application.h" +#include using namespace falco::app; @@ -22,103 +23,131 @@ application::run_result application::load_plugins() { run_result ret; - // The event source is syscall by default. If an input - // plugin was found, the source is the source of that - // plugin. - std::string event_source = s_syscall_source; - m_state->event_source_idx = m_state->syscall_source_idx; - - // Factories that can create filters/formatters for - // the (single) source supported by the (single) input plugin. - // libs requires raw pointer, we should modify libs to use reference/shared_ptr - std::shared_ptr plugin_filter_factory(new sinsp_filter_factory(m_state->inspector.get(), m_state->plugin_filter_checks)); - std::shared_ptr plugin_formatter_factory(new sinsp_evt_formatter_factory(m_state->inspector.get(), m_state->plugin_filter_checks)); - - if(m_state->config->m_json_output) - { - plugin_formatter_factory->set_output_format(gen_event_formatter::OF_JSON); - } - - std::shared_ptr input_plugin; - std::list> extractor_plugins; - for(auto &p : m_state->config->m_plugins) - { - std::shared_ptr plugin; #ifdef MUSL_OPTIMIZED + if (!m_state->config->m_plugins.empty()) + { ret.success = ret.proceed = false; ret.errstr = "Can not load/use plugins with musl optimized build"; return ret; -#else - falco_logger::log(LOG_INFO, "Loading plugin (" + p.m_name + ") from file " + p.m_library_path + "\n"); - - // libs requires raw pointer, we should modify libs to use reference/shared_ptr - plugin = m_state->inspector->register_plugin(p.m_library_path, - (p.m_init_config.empty() ? nullptr : (char *)p.m_init_config.c_str()), - m_state->plugin_filter_checks); + } #endif + // The only enabled event source is syscall by default + m_state->enabled_sources = {falco_common::syscall_source}; + + std::shared_ptr loaded_plugin = nullptr; + for(auto &p : m_state->config->m_plugins) + { + falco_logger::log(LOG_INFO, "Loading plugin (" + p.m_name + ") from file " + p.m_library_path + "\n"); + auto plugin = m_state->inspector->register_plugin(p.m_library_path, p.m_init_config); + if(plugin->caps() & CAP_SOURCING) { - if(input_plugin) + if (!is_capture_mode()) { - ret.success = false; - ret.errstr = string("Can not load multiple source plugins. ") + input_plugin->name() + " already loaded"; - ret.proceed = false; - return ret; - } - - input_plugin = plugin; - event_source = plugin->event_source(); - - m_state->inspector->set_input_plugin(p.m_name); - if(!p.m_open_params.empty()) - { - m_state->inspector->set_input_plugin_open_params(p.m_open_params); - } - - m_state->event_source_idx = m_state->engine->add_source(event_source, plugin_filter_factory, plugin_formatter_factory); - - } - - if(plugin->caps() & CAP_EXTRACTION) - { - extractor_plugins.push_back(plugin); - } - } - - // Ensure that extractor plugins are compatible with the event source. - // Also, ensure that extractor plugins don't have overlapping compatible event sources. - std::set compat_sources_seen; - for(const auto& eplugin : extractor_plugins) - { - // If the extractor plugin names compatible sources, - // ensure that the input plugin's source is in the list - // of compatible sources. - const std::set &compat_sources = eplugin->extract_event_sources(); - if(input_plugin && - !compat_sources.empty()) - { - if (compat_sources.find(event_source) == compat_sources.end()) - { - ret.success = ret.proceed = false; - ret.errstr = string("Extractor plugin not compatible with event source ") + event_source; - return ret; - } - - for(const auto &compat_source : compat_sources) - { - if(compat_sources_seen.find(compat_source) != compat_sources_seen.end()) + // todo(jasondellaluce): change this once we support multiple enabled event sources + if(loaded_plugin) { - ret.success = ret.proceed = false; - ret.errstr = string("Extractor plugins have overlapping compatible event source ") + compat_source; + ret.success = false; + ret.errstr = "Can not load multiple plugins with event sourcing capability: '" + + loaded_plugin->name() + + "' already loaded"; + ret.proceed = false; return ret; } - compat_sources_seen.insert(compat_source); + loaded_plugin = plugin; + m_state->enabled_sources = {plugin->event_source()}; + m_state->inspector->set_input_plugin(p.m_name, p.m_open_params); + } + + // Init filtercheck list for the plugin's source and add the + // event-generic filterchecks + auto &filterchecks = m_state->plugin_filter_checks[plugin->event_source()]; + filterchecks.add_filter_check(m_state->inspector->new_generic_filtercheck()); + + // Factories that can create filters/formatters for the event source of the plugin. + std::shared_ptr filter_factory(new sinsp_filter_factory(m_state->inspector.get(), filterchecks)); + std::shared_ptr formatter_factory(new sinsp_evt_formatter_factory(m_state->inspector.get(), filterchecks)); + if(m_state->config->m_json_output) + { + formatter_factory->set_output_format(gen_event_formatter::OF_JSON); + } + + // note: here we assume that the source index will be the same in + // both the falco engine and the sinsp plugin manager. This assumption + // stands because the plugin manager stores sources in a vector, and + // the syscall source is appended in the engine *after* the sources + // coming from plugins. Since this is an implementation-based + // assumption, we check this and throw an exception to spot + // regressions in the future. We keep it like this for to avoid the + // overhead of additional mappings at runtime, but we may consider + // mapping the two indexes under something like std::unordered_map in the future. + bool added = false; + auto source_idx = m_state->inspector->get_plugin_manager()->source_idx_by_plugin_id(plugin->id(), added); + auto source_idx_engine = m_state->engine->add_source(plugin->event_source(), filter_factory, formatter_factory); + if (!added || source_idx != source_idx_engine) + { + ret.success = ret.proceed = false; + ret.errstr = "Could not add event source in the engine: " + plugin->event_source(); + return ret; } } } - m_state->plugin_infos = m_state->inspector->plugin_infos(); + // Iterate over the plugins with extractor capability and add them to the + // filtercheck list of their compatible sources + std::vector filtercheck_info; + for(const auto& p : m_state->inspector->get_plugin_manager()->plugins()) + { + if (!(p->caps() & CAP_EXTRACTION)) + { + continue; + } + + bool used = false; + for (auto &it : m_state->plugin_filter_checks) + { + // check if the event source is compatible with this plugin + if (p->is_source_compatible(it.first)) + { + // check if some fields are overlapping on this event sources + filtercheck_info.clear(); + it.second.get_all_fields(filtercheck_info); + for (auto &info : filtercheck_info) + { + for (int32_t i = 0; i < info->m_nfields; i++) + { + // check if one of the fields extractable by the plugin + // is already provided by another filtercheck for this source + std::string fname = info->m_fields[i].m_name; + for (auto &f : p->fields()) + { + if (std::string(f.m_name) == fname) + { + ret.success = ret.proceed = false; + ret.errstr = + "Plugin '" + p->name() + + "' supports extraction of field '" + fname + + "' that is overlapping for source '" + it.first + "'"; + return ret; + } + } + } + } + + // add plugin filterchecks to the event source + it.second.add_filter_check(sinsp_plugin::new_filtercheck(p)); + used = true; + } + } + if (!used) + { + ret.success = ret.proceed = false; + ret.errstr = "Plugin '" + p->name() + + "' has field extraction capability but is not compatible with any enabled event source"; + return ret; + } + } return ret; } diff --git a/userspace/falco/app_actions/load_rules_files.cpp b/userspace/falco/app_actions/load_rules_files.cpp index 83fb026f..a665aebc 100644 --- a/userspace/falco/app_actions/load_rules_files.cpp +++ b/userspace/falco/app_actions/load_rules_files.cpp @@ -15,6 +15,7 @@ limitations under the License. */ #include "application.h" +#include using namespace falco::app; @@ -24,7 +25,8 @@ void application::check_for_ignored_events() sinsp_evttables* einfo = m_state->inspector->get_event_info_tables(); const struct ppm_event_info* etable = einfo->m_event_info; - m_state->engine->evttypes_for_ruleset(application::s_syscall_source, evttypes); + std::string source = falco_common::syscall_source; + m_state->engine->evttypes_for_ruleset(source, evttypes); // Save event names so we don't warn for both the enter and exit event. std::set warn_event_names; @@ -115,14 +117,14 @@ application::run_result application::load_rules_files() } // Ensure that all plugins are compatible with the loaded set of rules - for(auto &info : m_state->plugin_infos) + for(const auto &plugin : m_state->inspector->get_plugin_manager()->plugins()) { std::string required_version; - if(!m_state->engine->is_plugin_compatible(info.name, info.plugin_version.as_string(), required_version)) + if(!m_state->engine->is_plugin_compatible(plugin->name(), plugin->plugin_version().as_string(), required_version)) { ret.success = false; - ret.errstr = std::string("Plugin ") + info.name + " version " + info.plugin_version.as_string() + " not compatible with required plugin version " + required_version; + ret.errstr = "Plugin " + plugin->name() + " version " + plugin->plugin_version().as_string() + " not compatible with required plugin version " + required_version; ret.proceed = false; } } diff --git a/userspace/falco/app_actions/open_inspector.cpp b/userspace/falco/app_actions/open_inspector.cpp index 55266ea5..dc1b58bd 100644 --- a/userspace/falco/app_actions/open_inspector.cpp +++ b/userspace/falco/app_actions/open_inspector.cpp @@ -28,7 +28,7 @@ application::run_result application::open_inspector() { run_result ret; - if(m_options.trace_filename.size()) + if(is_capture_mode()) { // Try to open the trace file as a // capture file first. @@ -46,49 +46,32 @@ application::run_result application::open_inspector() } else { - open_t open_cb = [this](std::shared_ptr inspector) - { - if(m_options.userspace) - { - // open_udig() is the underlying method used in the capture code to parse userspace events from the kernel. - // - // Falco uses a ptrace(2) based userspace implementation. - // Regardless of the implementation, the underlying method remains the same. - inspector->open_udig(); - return; - } - inspector->open(); - }; - open_t open_nodriver_cb = [](std::shared_ptr inspector) { - inspector->open_nodriver(); - }; - open_t open_f; - - // Default mode: both event sources enabled - if (m_state->enabled_sources.find(application::s_syscall_source) != m_state->enabled_sources.end()) - { - open_f = open_cb; - } - else - { - open_f = open_nodriver_cb; - } - try { - open_f(m_state->inspector); + // open_udig() is the underlying method used in the capture code to parse userspace events from the kernel. + // + // Falco uses a ptrace(2) based userspace implementation. + // Regardless of the implementation, the underlying method remains the same. + if(m_options.userspace) + { + m_state->inspector->open_udig(); + } + else + { + m_state->inspector->open(); + } } catch(sinsp_exception &e) { // If syscall input source is enabled and not through userspace instrumentation - if (m_state->enabled_sources.find(application::s_syscall_source) != m_state->enabled_sources.end() && !m_options.userspace) + if (is_syscall_source_enabled() && !m_options.userspace) { // Try to insert the Falco kernel module if(system("modprobe " DRIVER_NAME " > /dev/null 2> /dev/null")) { falco_logger::log(LOG_ERR, "Unable to load the driver.\n"); } - open_f(m_state->inspector); + m_state->inspector->open(); } else { diff --git a/userspace/falco/app_actions/process_events.cpp b/userspace/falco/app_actions/process_events.cpp index eb183e27..20e0b55b 100644 --- a/userspace/falco/app_actions/process_events.cpp +++ b/userspace/falco/app_actions/process_events.cpp @@ -29,6 +29,8 @@ limitations under the License. #include "statsfilewriter.h" #include "application.h" +#include + using namespace falco::app; // @@ -44,6 +46,8 @@ uint64_t application::do_inspect(syscall_evt_drop_mgr &sdropmgr, StatsFileWriter writer; uint64_t duration_start = 0; uint32_t timeouts_since_last_success_or_msg = 0; + std::size_t source_idx; + bool source_idx_found = false; sdropmgr.init(m_state->inspector, m_state->outputs, @@ -95,8 +99,8 @@ uint64_t application::do_inspect(syscall_evt_drop_mgr &sdropmgr, if(unlikely(ev == nullptr)) { timeouts_since_last_success_or_msg++; - if(m_state->event_source_idx == m_state->syscall_source_idx && - (timeouts_since_last_success_or_msg > m_state->config->m_syscall_evt_timeout_max_consecutives)) + if(timeouts_since_last_success_or_msg > m_state->config->m_syscall_evt_timeout_max_consecutives + && is_syscall_source_enabled()) { std::string rule = "Falco internal: timeouts notification"; std::string msg = rule + ". " + std::to_string(m_state->config->m_syscall_evt_timeout_max_consecutives) + " consecutive timeouts without event."; @@ -157,12 +161,28 @@ uint64_t application::do_inspect(syscall_evt_drop_mgr &sdropmgr, continue; } + source_idx = m_state->syscall_source_idx; + if (ev->get_type() == PPME_PLUGINEVENT_E) + { + // note: here we can assume that the source index will be the same + // in both the falco engine and the sinsp plugin manager. See the + // comment in load_plugins.cpp for more details. + source_idx = m_state->inspector->get_plugin_manager()->source_idx_by_plugin_id(*(int32_t *)ev->get_param(0)->m_val, source_idx_found); + if (!source_idx_found) + { + result.success = false; + result.errstr = "Unknown plugin ID in inspector: " + std::to_string(*(int32_t *)ev->get_param(0)->m_val); + result.proceed = false; + break; + } + } + // As the inspector has no filter at its level, all // events are returned here. Pass them to the falco // engine, which will match the event against the set // of rules. If a match is found, pass the event to // the outputs. - unique_ptr res = m_state->engine->process_event(m_state->event_source_idx, ev); + unique_ptr res = m_state->engine->process_event(source_idx, ev); if(res) { m_state->outputs->handle_event(res->evt, res->rule, res->source, res->priority_num, res->format, res->tags); @@ -208,7 +228,7 @@ application::run_result application::process_events() // Honor -M also when using a trace file. // Since inspection stops as soon as all events have been consumed // just await the given duration is reached, if needed. - if(!m_options.trace_filename.empty() && m_options.duration_to_tot>0) + if(is_capture_mode() && m_options.duration_to_tot > 0) { std::this_thread::sleep_for(std::chrono::seconds(m_options.duration_to_tot)); } diff --git a/userspace/falco/app_cmdline_options.cpp b/userspace/falco/app_cmdline_options.cpp index ea701d25..7f028ea6 100644 --- a/userspace/falco/app_cmdline_options.cpp +++ b/userspace/falco/app_cmdline_options.cpp @@ -159,7 +159,7 @@ void cmdline_options::define() ("cri", "Path to CRI socket for container metadata. Use the specified socket to fetch data from a CRI-compatible runtime. If not specified, uses libs default. It can be passed multiple times to specify socket to be tried until a successful one is found.", cxxopts::value(cri_socket_paths), "") ("d,daemon", "Run as a daemon.", cxxopts::value(daemon)->default_value("false")) ("disable-cri-async", "Disable asynchronous CRI metadata fetching. This is useful to let the input event wait for the container metadata fetch to finish before moving forward. Async fetching, in some environments leads to empty fields for container metadata when the fetch is not fast enough to be completed asynchronously. This can have a performance penalty on your environment depending on the number of containers and the frequency at which they are created/started/stopped.", cxxopts::value(disable_cri_async)->default_value("false")) - ("disable-source", "Disable a specific event source. Available event sources are: syscall or any source from a configured source plugin. It can be passed multiple times. Can not disable all event sources.", cxxopts::value(disable_sources), "") + ("disable-source", "Disable a specific event source. Available event sources are: syscall or any source from a configured plugin with event sourcing capability. It can be passed multiple times. Can not disable all event sources.", cxxopts::value(disable_sources), "") ("D", "Disable any rules with names having the substring . Can be specified multiple times. Can not be specified with -t.", cxxopts::value(disabled_rule_substrings), "") ("e", "Read the events from in .scap format instead of tapping into live.", cxxopts::value(trace_filename), "") ("i", "Print all events that are ignored by default (i.e. without the -A flag) and exit.", cxxopts::value(print_ignored_events)->default_value("false")) @@ -170,7 +170,7 @@ void cmdline_options::define() #endif ("L", "Show the name and description of all rules and exit.", cxxopts::value(describe_all_rules)->default_value("false")) ("l", "Show the name and description of the rule with name and exit.", cxxopts::value(describe_rule), "") - ("list", "List all defined fields. If is provided, only list those fields for the source . Current values for are \"syscall\" or any source from a configured source plugin.", cxxopts::value(list_source_fields)->implicit_value(""), "") + ("list", "List all defined fields. If is provided, only list those fields for the source . Current values for are \"syscall\" or any source from a configured plugin with event sourcing capability.", cxxopts::value(list_source_fields)->implicit_value(""), "") ("list-syscall-events", "List all defined system call events.", cxxopts::value(list_syscall_events)) #ifndef MUSL_OPTIMIZED ("list-plugins", "Print info on all loaded plugins and exit.", cxxopts::value(list_plugins)->default_value("false"))