wip

Signed-off-by: Federico Di Pierro <nierro92@gmail.com>

.github/workflows/release.yaml

@@ -147,7 +147,7 @@ jobs:
           sed -i s/FALCOVER/${{ github.event.release.tag_name }}/g release-body.md
           
       - name: Generate release notes
-        uses: leodido/rn2md@1a17f0e75758c15128a5146e8af5ca3a47209b3f
+        uses: leodido/rn2md@0669e5f3b21492c11c2db43cd6e267566f5880f3
         with:
           milestone: ${{ github.event.release.tag_name }}
           output: ./notes.md

CHANGELOG.md

@@ -3,7 +3,7 @@
 
 ## v0.36.0
 
-Released on 2023-09-25
+Released on 2023-09-26
 
 ### Breaking Changes
 
@@ -22,7 +22,7 @@ Released on 2023-09-25
 
 
 * new(falco-driver-loader): --source-only now prints the values as env vars [[#2353](https://github.com/falcosecurity/falco/pull/2353)] - [@steakunderscore](https://github.com/steakunderscore)
-* new(docker): allow passing options to falco-driver-loader from the driver loader container [[#2781](https://github.com/falcosecurity/falco/pull/2781)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* new(docker): allow passing options to falco-driver-loader from the driver loader cointainer [[#2781](https://github.com/falcosecurity/falco/pull/2781)] - [@LucaGuerra](https://github.com/LucaGuerra)
 * new(docker): add experimental falco-distroless image based on Wolfi [[#2768](https://github.com/falcosecurity/falco/pull/2768)] - [@LucaGuerra](https://github.com/LucaGuerra)
 * new: the legacy falco image is available as driver-loader-legacy [[#2718](https://github.com/falcosecurity/falco/pull/2718)] - [@LucaGuerra](https://github.com/LucaGuerra)
 * new: added option to enable/disable echoing of server answer to stdout (disabled by default) when using HTTP output [[#2602](https://github.com/falcosecurity/falco/pull/2602)] - [@FedeDP](https://github.com/FedeDP)
@@ -1086,7 +1086,7 @@ Released on 2021-01-18
 ### Minor Changes
 
 * build: bump b64 to v2.0.0.1 [[#1441](https://github.com/falcosecurity/falco/pull/1441)] - [@fntlnz](https://github.com/fntlnz)
-* rules(macro container_started): reuse `spawned_process` macro inside `container_started` macro [[#1449](https://github.com/falcosecurity/falco/pull/1449)] - [@leodido](https://github.com/leodido)
+* rules(macro container_started): re-use `spawned_process` macro inside `container_started` macro [[#1449](https://github.com/falcosecurity/falco/pull/1449)] - [@leodido](https://github.com/leodido)
 * docs: reach out documentation [[#1472](https://github.com/falcosecurity/falco/pull/1472)] - [@fntlnz](https://github.com/fntlnz)
 * docs: Broken outputs.proto link [[#1493](https://github.com/falcosecurity/falco/pull/1493)] - [@deepskyblue86](https://github.com/deepskyblue86)
 * docs(README.md): correct broken links [[#1506](https://github.com/falcosecurity/falco/pull/1506)] - [@leogr](https://github.com/leogr)

cmake/modules/falcosecurity-libs.cmake

@@ -28,15 +28,15 @@ else()
   # In case you want to test against a fork of falcosecurity/libs just pass the variable -
   # ie., `cmake -DFALCOSECURITY_LIBS_REPO=<your-gh-handle>/libs ..` 
   if (NOT FALCOSECURITY_LIBS_REPO)
-    set(FALCOSECURITY_LIBS_REPO "Andreagit97/libs")
+    set(FALCOSECURITY_LIBS_REPO "falcosecurity/libs")
   endif()
 
   # FALCOSECURITY_LIBS_VERSION accepts a git reference (branch name, commit hash, or tag) to the falcosecurity/libs repository.
   # In case you want to test against another falcosecurity/libs version (or branch, or commit) just pass the variable -
   # ie., `cmake -DFALCOSECURITY_LIBS_VERSION=dev ..` 
   if(NOT FALCOSECURITY_LIBS_VERSION)
-    set(FALCOSECURITY_LIBS_VERSION "8b3375596fec7b02ac403dc4158c5bcdf037cf65")
-    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=587ac097e1ef35ba70ddd3403e569d377a0c4a5935cb17c9ee8f9521dac4140c")
+    set(FALCOSECURITY_LIBS_VERSION "ebd17a1cfb5935d774681aa6a4696deb6561d965")
+    # set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=2be42a27be3ffe6bd7e53eaa5d8358cab05a0dca821819c6e9059e51b9786219")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

falco.yaml

@@ -331,17 +331,24 @@ rule_matching: first
 # If it does, it is most likely happening due to the entire event flow being too slow,
 # indicating that the server is under heavy load.
 # 
+# Lowering the number of items can prevent memory from steadily increasing until the OOM
+# killer stops the Falco process. We provide recovery actions to self-limit or self-kill
+# in order to handle this situation earlier, similar to how we expose the kernel buffer size
+# as a parameter. However, it will not address the root cause of the event pipe not keeping up.
+# 
 # `capacity`: the maximum number of items allowed in the queue is determined by this value. 
 # Setting the value to 0 (which is the default) is equivalent to keeping the queue unbounded. 
-# In other words, when this configuration is set to 0, the number of allowed items is 
-# effectively set to the largest possible long value, disabling this setting.
+# In other words, when this configuration is set to 0, the number of allowed items is effectively 
+# set to the largest possible long value, disabling this setting.
 # 
-# In the case of an unbounded queue, if the available memory on the system is consumed, 
-# the Falco process would be OOM killed. When using this option and setting the capacity, 
-# the current event would be dropped, and the event loop would continue. This behavior mirrors 
-# kernel-side event drops when the buffer between kernel space and user space is full.
+# `recovery`: strategy to follow when the queue becomes filled up. It applies only when the
+# queue is bounded and there is still available system memory. In the case of an unbounded
+# queue, if the available memory on the system is consumed, the Falco process would be
+# OOM killed. The value `exit` is the default, `continue` does nothing special and `empty` 
+# empties the queue and then continues.
 outputs_queue:
   capacity: 0
+  recovery: exit
 
 
 ##########################

proposals/20221129-artifacts-distribution.md

@@ -69,7 +69,7 @@ The allowed publishing channels are:
 Both channels are equivalent and may publish the same artifacts. However, for historical reasons and to avoid confusion, the **`docker.io` registry should only be used for container images** and not for other kinds of artifacts (e.g., plugins, rules, etc.).
 
 
-Mirrors are allowed and encouraged if they facilitate artifacts consumption by our users. This proposal recommends to enable mirrors on the major public OCI registry, such as [Amazon ECR](https://gallery.ecr.aws/) (which is already implentend in our infra at the time of writing).
+Mirrors are allowed and encouraged if they facilitate artifacts consumption by our users. This proposal reccomends to enable mirrors on the major public OCI registry, such as [Amazon ECR](https://gallery.ecr.aws/) (which is already implentend in our infra at the time of writing).
 
 
 Official **channels and mirrors must be listed at [falco.org](https://falco.org/)**. 

userspace/engine/falco_common.cpp

@@ -33,6 +33,12 @@ static std::vector<std::string> rule_matching_names = {
 	"all"
 };
 
+static std::vector<std::string> outputs_queue_recovery_names = {
+		"continue",
+		"exit",
+		"empty",
+};
+
 bool falco_common::parse_priority(std::string v, priority_type& out)
 {
 	for (size_t i = 0; i < priority_names.size(); i++)
@@ -60,6 +66,19 @@ falco_common::priority_type falco_common::parse_priority(std::string v)
 	return out;
 }
 
+bool falco_common::parse_queue_recovery(std::string v, outputs_queue_recovery_type& out)
+{
+	for (size_t i = 0; i < outputs_queue_recovery_names.size(); i++)
+	{
+		if (!strcasecmp(v.c_str(), outputs_queue_recovery_names[i].c_str()))
+		{
+			out = (outputs_queue_recovery_type) i;
+			return true;
+		}
+	}
+	return false;
+}
+
 bool falco_common::format_priority(priority_type v, std::string& out, bool shortfmt)
 {
 	if ((size_t) v < priority_names.size())

userspace/engine/falco_common.h

@@ -60,6 +60,12 @@ struct falco_exception : std::exception
 namespace falco_common
 {
 
+	enum outputs_queue_recovery_type {
+		RECOVERY_CONTINUE = 0,  /* outputs_queue_capacity recovery strategy of continuing on. */
+		RECOVERY_EXIT = 1,  /* outputs_queue_capacity recovery strategy of exiting, self OOM kill. */
+		RECOVERY_EMPTY = 2,  /* outputs_queue_capacity recovery strategy of emptying queue then continuing. */
+	};
+
 	const std::string syscall_source = sinsp_syscall_event_source_name;
 
 	// Same as numbers/indices into the above vector
@@ -77,6 +83,7 @@ namespace falco_common
 	
 	bool parse_priority(std::string v, priority_type& out);
 	priority_type parse_priority(std::string v);
+	bool parse_queue_recovery(std::string v, outputs_queue_recovery_type& out);
 	bool format_priority(priority_type v, std::string& out, bool shortfmt=false);
 	std::string format_priority(priority_type v, bool shortfmt=false);
 

userspace/engine/falco_engine.cpp

@@ -46,7 +46,6 @@ limitations under the License.
 #include "utils.h"
 #include "banned.h" // This raises a compilation error when certain functions are used
 #include "evttype_index_ruleset.h"
-#include "filter_details_resolver.h"
 
 const std::string falco_engine::s_default_ruleset = "falco-default-ruleset";
 
@@ -191,22 +190,60 @@ void falco_engine::load_rules(const std::string &rules_content, bool verbose, bo
 std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_content, const std::string &name)
 {
 	rule_loader::configuration cfg(rules_content, m_sources, name);
-	cfg.min_priority = m_min_priority;
 	cfg.output_extra = m_extra;
 	cfg.replace_output_container_info = m_replace_container_info;
-	cfg.default_ruleset_id = m_default_ruleset_id;
 
+	// read rules YAML file and collect its definitions
 	rule_loader::reader reader;
 	if (reader.read(cfg, m_rule_collector))
-	{
+	{	
+		// compile the definitions (resolve macro/list refs, exceptions, ...)
+		rule_loader::compiler::compile_output out;
+		rule_loader::compiler().compile(cfg, m_rule_collector, out);
+
+		// clear the rules known by the engine and each ruleset
+		m_rules.clear();
 		for (auto &src : m_sources)
 		{
 			src.ruleset = src.ruleset_factory->new_ruleset();
 		}
 
-		rule_loader::compiler compiler;
-		m_rules.clear();
-		compiler.compile(cfg, m_rule_collector, m_rules);
+		// add rules to the engine and the rulesets
+		for (const auto& rule : out.rules)
+		{
+			// skip the rule if below the minimum priority
+			if (rule.priority > m_min_priority)
+			{
+				continue;
+			}
+
+			auto info = m_rule_collector.rules().at(rule.name);
+			if (!info)
+			{
+				// this is just defensive, it should never happen
+				throw falco_exception("can't find internal rule info at name: " + name);
+			}
+
+			// the rule is ok, we can add it to the engine and the rulesets
+			// note: the compiler should guarantee that the rule's condition
+			// is a valid sinsp filter
+			auto source = find_source(rule.source);
+			std::shared_ptr<gen_event_filter> filter(
+				sinsp_filter_compiler(source->filter_factory, rule.condition.get()).compile());
+			auto rule_id = m_rules.insert(rule, rule.name);
+			m_rules.at(rule_id)->id = rule_id;
+			source->ruleset->add(rule, filter, rule.condition);
+
+			// By default rules are enabled/disabled for the default ruleset
+			if(info->enabled)
+			{
+				source->ruleset->enable(rule.name, true, m_default_ruleset_id);
+			}
+			else
+			{
+				source->ruleset->disable(rule.name, true, m_default_ruleset_id);
+			}
+		}
 	}
 
 	if (cfg.res->successful())
@@ -469,7 +506,17 @@ std::size_t falco_engine::add_source(const std::string &source,
 	return m_sources.insert(src, source);
 }
 
-void falco_engine::describe_rule(std::string *rule, bool json) const
+template <typename T> inline Json::Value sequence_to_json_array(const T& seq)
+{
+	Json::Value ret = Json::arrayValue;
+	for (auto it = seq.begin(); it != seq.end(); it++)
+	{
+		ret.append(*it);
+	}
+	return ret;
+}
+
+void falco_engine::describe_rule(std::string *rule, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins, bool json) const
 {
 	if(!json)
 	{
@@ -498,10 +545,20 @@ void falco_engine::describe_rule(std::string *rule, bool json) const
 		return;
 	}
 
-	std::unique_ptr<sinsp> insp(new sinsp());
+	// use previously-loaded collector definitions to obtain a compiled
+	// output of rules, macros, and lists.
+	// note: we ignore the loading result (errors, warnings), as they should have
+	// already been checked when previously-loading the rules files. Thus, we
+	// assume that the definitions will give no compilation error.
+	rule_loader::configuration cfg("", m_sources, "");
+	cfg.output_extra = m_extra;
+	cfg.replace_output_container_info = m_replace_container_info;
+	rule_loader::compiler::compile_output compiled;
+	rule_loader::compiler().compile(cfg, m_rule_collector, compiled);
+
+	// use collected and compiled info to print a json output
 	Json::FastWriter writer;
 	std::string json_str;
-
 	if(!rule)
 	{
 		// In this case we build json information about
@@ -537,33 +594,33 @@ void falco_engine::describe_rule(std::string *rule, bool json) const
 
 		// Store information about rules
 		Json::Value rules_array = Json::arrayValue;
-		for(const auto& r : m_rules)
+		for(const auto& r : compiled.rules)
 		{
-			auto ri = m_rule_collector.rules().at(r.name);
+			auto info = m_rule_collector.rules().at(r.name);
 			Json::Value rule;
-			get_json_details(r, *ri, insp.get(), rule);
-
-			// Append to rule array
+			get_json_details(rule, r, *info, plugins);
 			rules_array.append(rule);
 		}
 		output["rules"] = rules_array;
 		
 		// Store information about macros
-		Json::Value macros_array;
-		for(const auto &m : m_rule_collector.macros())
+		Json::Value macros_array = Json::arrayValue;
+		for(const auto &m : compiled.macros)
 		{
+			auto info = m_rule_collector.macros().at(m.name);
 			Json::Value macro;
-			get_json_details(m, macro);
+			get_json_details(macro, m, *info, plugins);
 			macros_array.append(macro);
 		}
 		output["macros"] = macros_array;
 
 		// Store information about lists 
 		Json::Value lists_array = Json::arrayValue;
-		for(const auto &l : m_rule_collector.lists())
+		for(const auto &l : compiled.lists)
 		{
+			auto info = m_rule_collector.lists().at(l.name);
 			Json::Value list;
-			get_json_details(l, list);
+			get_json_details(list, l, *info, plugins);
 			lists_array.append(list);			
 		}
 		output["lists"] = lists_array;
@@ -580,68 +637,73 @@ void falco_engine::describe_rule(std::string *rule, bool json) const
 		}
 		auto r = m_rules.at(ri->name);
 		Json::Value rule; 
-		get_json_details(*r, *ri, insp.get(), rule);
+		get_json_details(rule, *r, *ri, plugins);
 		json_str = writer.write(rule);
 	}
 
 	fprintf(stdout, "%s", json_str.c_str());
 }
 
-void falco_engine::get_json_details(const falco_rule &r,
-	const rule_loader::rule_info &ri,
-	sinsp *insp,
-	Json::Value &rule) const
+void falco_engine::get_json_details(
+	Json::Value &out,
+	const falco_rule &r,
+	const rule_loader::rule_info &info,
+	const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
 	Json::Value rule_info;
 
 	// Fill general rule information
 	rule_info["name"] = r.name;
-	rule_info["condition"] = ri.cond;
+	rule_info["condition"] = info.cond;
 	rule_info["priority"] = format_priority(r.priority, false);
-	rule_info["output"] = r.output;
+	rule_info["output"] = info.output;
 	rule_info["description"] = r.description;
-	rule_info["enabled"] = ri.enabled;
+	rule_info["enabled"] = info.enabled;
 	rule_info["source"] = r.source;
-	Json::Value tags = Json::arrayValue;
-	for(const auto &t : ri.tags)
-	{
-		tags.append(t);
-	}
-	rule_info["tags"] = tags;
-	rule["info"] = rule_info;
+	rule_info["tags"] = sequence_to_json_array(info.tags);
+	out["info"] = rule_info;
 
-	// Parse rule condition and build the AST
-	// Assumption: no exception because rules have already been loaded.
-	auto ast = libsinsp::filter::parser(ri.cond).parse();
+	// Parse rule condition and build the non-compiled AST
+	// Assumption: no error because rules have already been loaded.
+	auto ast = libsinsp::filter::parser(info.cond).parse();
+
+	// get details related to the condition's filter
+	filter_details details;
+	filter_details compiled_details;
 	Json::Value json_details;
-	get_json_details(ast.get(), json_details);
-	rule["details"] = json_details;
+	for(const auto &m : m_rule_collector.macros())
+	{
+		details.known_macros.insert(m.name);
+		compiled_details.known_macros.insert(m.name);
+	}
+	for(const auto &l : m_rule_collector.lists())
+	{
+		details.known_lists.insert(l.name);
+		compiled_details.known_lists.insert(l.name);
+	}
+	filter_details_resolver().run(ast.get(), details);
+	filter_details_resolver().run(r.condition.get(), compiled_details);
+
+	out["details"]["macros"] = sequence_to_json_array(details.macros);
+	out["details"]["lists"] = sequence_to_json_array(details.lists);
+	out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators);
+	out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields);
 
 	// Get fields from output string
 	auto fmt = create_formatter(r.source, r.output);
 	std::vector<std::string> out_fields;
 	fmt->get_field_names(out_fields);
-	Json::Value outputFields = Json::arrayValue;
-	for(const auto &of : out_fields)
-	{
-		outputFields.append(of);
-	}
-	rule["details"]["output_fields"] = outputFields;
+	out["details"]["output_fields"] = sequence_to_json_array(out_fields);
 
 	// Get fields from exceptions
-	Json::Value exception_fields = Json::arrayValue;
-	for(const auto &f : r.exception_fields)
-	{
-		exception_fields.append(f);
-	}
-	rule["details"]["exception_fields"] = exception_fields;
+	out["details"]["exception_fields"] = sequence_to_json_array(r.exception_fields);
 
 	// Get names and operators from exceptions
-	Json::Value exception_names = Json::arrayValue;
-	Json::Value exception_operators = Json::arrayValue;
-	for(const auto &e : ri.exceptions)
+	std::unordered_set<std::string> exception_names;
+	std::unordered_set<std::string> exception_operators;
+	for(const auto &e : info.exceptions)
 	{
-		exception_names.append(e.name);
+		exception_names.insert(e.name);
 		if(e.comps.is_list)
 		{
 			for(const auto& c : e.comps.items)
@@ -651,140 +713,236 @@ void falco_engine::get_json_details(const falco_rule &r,
 					// considering max two levels of lists
 					for(const auto& i : c.items)
 					{
-						exception_operators.append(i.item);
+						exception_operators.insert(i.item);
 					}
 				}
 				else
 				{
-					exception_operators.append(c.item);
+					exception_operators.insert(c.item);
 				}
 			}
 		}
 		else
 		{
-			exception_operators.append(e.comps.item);
+			exception_operators.insert(e.comps.item);
 		}	
 	}
-	rule["details"]["exceptions"] = exception_names;
-	rule["details"]["exception_operators"] = exception_operators;
+	out["details"]["exception_names"] = sequence_to_json_array(exception_names);
+	out["details"]["exception_operators"] = sequence_to_json_array(exception_operators);
 
-	if(ri.source == falco_common::syscall_source)
-	{
-		// Store event types
-		Json::Value events;
-		get_json_evt_types(ast.get(), events);
-		rule["details"]["events"] = events;
-	}
+	// Store event types
+	Json::Value events;
+	get_json_evt_types(events, info.source, r.condition.get());
+	out["details"]["events"] = events;
+
+	// Store compiled condition and output
+	out["details"]["condition_compiled"] = libsinsp::filter::ast::as_string(r.condition.get());
+	out["details"]["output_compiled"] = r.output;
+
+	// Compute the plugins that are actually used by this rule. This is involves:
+	// - The rule's event source, that can be implemented by a plugin
+	// - The fields used in the rule's condition, output, and exceptions
+	// - The evt types used in the rule's condition checks, that can potentially
+	//   match plugin-provided async events
+	Json::Value used_plugins;
+	// note: making a union of conditions's and output's fields
+	// note: the condition's AST accounts for all the resolved refs and exceptions
+	compiled_details.fields.insert(out_fields.begin(), out_fields.end());
+	get_json_used_plugins(used_plugins, info.source, compiled_details.evtnames, compiled_details.fields, plugins);
+	out["details"]["plugins"] = used_plugins;
 }
 
-void falco_engine::get_json_details(const rule_loader::macro_info& m,
-	Json::Value& macro) const
+void falco_engine::get_json_details(
+	Json::Value& out,
+	const falco_macro& m,
+	const rule_loader::macro_info& info,
+	const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
 	Json::Value macro_info;
 
 	macro_info["name"] = m.name;
-	macro_info["condition"] = m.cond;
-	macro["info"] = macro_info;
+	macro_info["condition"] = info.cond;
+	out["info"] = macro_info;
 
+	// Parse the macro condition and build the non-compiled AST
 	// Assumption: no exception because rules have already been loaded.
-	auto ast = libsinsp::filter::parser(m.cond).parse();
+	auto ast = libsinsp::filter::parser(info.cond).parse();
 
+	// get details related to the condition's filter
+	filter_details details;
+	filter_details compiled_details;
 	Json::Value json_details;
-	get_json_details(ast.get(), json_details);
-	macro["details"] = json_details;
+	for(const auto &m : m_rule_collector.macros())
+	{
+		details.known_macros.insert(m.name);
+		compiled_details.known_macros.insert(m.name);
+	}
+	for(const auto &l : m_rule_collector.lists())
+	{
+		details.known_lists.insert(l.name);
+		compiled_details.known_lists.insert(l.name);
+	}
+	filter_details_resolver().run(ast.get(), details);
+	filter_details_resolver().run(m.condition.get(), compiled_details);
+
+	out["details"]["used"] = m.used;
+	out["details"]["macros"] = sequence_to_json_array(details.macros);
+	out["details"]["lists"] = sequence_to_json_array(details.lists);
+	out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators);
+	out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields);
 
 	// Store event types
 	Json::Value events;
-	get_json_evt_types(ast.get(), events);
-	macro["details"]["events"] = events;
+	get_json_evt_types(events, "", m.condition.get());
+	out["details"]["events"] = events;
+
+	// Store compiled condition
+	out["details"]["condition_compiled"] = libsinsp::filter::ast::as_string(m.condition.get());
+
+	// Compute the plugins that are actually used by this macro.
+	// Note: macros have no specific source, we need to set an empty list of used
+	// plugins because we can't be certain about their actual usage. For example,
+	// if a macro uses a plugin's field, we can't be sure which plugin actually
+	// is used until we resolve the macro ref in a rule providing a source for
+	// disambiguation.
+	out["details"]["plugins"] = Json::arrayValue;
 }
 
-void falco_engine::get_json_details(const rule_loader::list_info& l, 
-	Json::Value& list) const
+void falco_engine::get_json_details(
+	Json::Value& out,
+	const falco_list& l,
+	const rule_loader::list_info& info,
+	const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
 	Json::Value list_info;
 	list_info["name"] = l.name;
 
+	// note: the syntactic definitions still has the list refs unresolved
 	Json::Value items = Json::arrayValue;
-	Json::Value lists = Json::arrayValue;
-	for(const auto &i : l.items)
+	std::unordered_set<std::string> lists;
+	for(const auto &i : info.items)
 	{
-		if(m_rule_collector.lists().at(i) != nullptr)
+		// if an item is present in the syntactic def of a list, but not
+		// on the compiled_items of the same list, then we can assume it
+		// being a resolved list ref
+		if(std::find(l.items.begin(), l.items.end(), i) == l.items.end())
 		{
-			lists.append(i);
+			lists.insert(i);
 			continue;
 		}
 		items.append(i);
 	}
 
 	list_info["items"] = items;
-	list["info"] = list_info;
-	list["details"]["lists"] = lists;
+	out["info"] = list_info;
+	out["details"]["used"] = l.used;
+	out["details"]["lists"] = sequence_to_json_array(lists);
+	out["details"]["items_compiled"] = sequence_to_json_array(l.items);
+	out["details"]["plugins"] = Json::arrayValue; // always empty
 }
 
-void falco_engine::get_json_details(libsinsp::filter::ast::expr* ast,
-	Json::Value& output) const
+void falco_engine::get_json_evt_types(
+	Json::Value& out,
+	const std::string& source,
+	libsinsp::filter::ast::expr* ast) const
 {
-	filter_details details;
-	for(const auto &m : m_rule_collector.macros())
+	// note: this duplicates part of the logic of evttype_index_ruleset,
+	// not good but it's our best option for now
+	if (source.empty() || source == falco_common::syscall_source)
 	{
-		details.known_macros.insert(m.name);
+		auto evtcodes = libsinsp::filter::ast::ppm_event_codes(ast);
+		evtcodes.insert(ppm_event_code::PPME_ASYNCEVENT_E);
+		auto syscodes = libsinsp::filter::ast::ppm_sc_codes(ast);
+		auto syscodes_to_evt_names = libsinsp::events::sc_set_to_event_names(syscodes);
+		auto evtcodes_to_evt_names = libsinsp::events::event_set_to_names(evtcodes, false);
+		out = sequence_to_json_array(unordered_set_union(syscodes_to_evt_names, evtcodes_to_evt_names));
 	}
-
-	for(const auto &l : m_rule_collector.lists())
+	else
 	{
-		details.known_lists.insert(l.name);
+		out = sequence_to_json_array(libsinsp::events::event_set_to_names(
+			{ppm_event_code::PPME_PLUGINEVENT_E, ppm_event_code::PPME_ASYNCEVENT_E}));
 	}
-
-	// Resolve the AST details
-	filter_details_resolver resolver;
-	resolver.run(ast, details);
-
-	Json::Value macros = Json::arrayValue;
-	for(const auto &m : details.macros)
-	{
-		macros.append(m);
-	}
-	output["macros"] = macros;
-
-	Json::Value operators = Json::arrayValue;
-	for(const auto &o : details.operators)
-	{
-		operators.append(o);
-	}
-	output["operators"] = operators;
-
-	Json::Value condition_fields = Json::arrayValue;
-	for(const auto &f : details.fields)
-	{
-		condition_fields.append(f);
-	}
-	output["condition_fields"] = condition_fields;
-
-	Json::Value lists = Json::arrayValue;
-	for(const auto &l : details.lists)
-	{
-		lists.append(l);
-	}
-	output["lists"] = lists;
-	
-	details.reset();
 }
 
-void falco_engine::get_json_evt_types(libsinsp::filter::ast::expr* ast,
-					Json::Value& output) const
+void falco_engine::get_json_used_plugins(
+	Json::Value& out,
+	const std::string& source,
+	const std::unordered_set<std::string>& evtnames,
+	const std::unordered_set<std::string>& fields,
+	const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
-	output = Json::arrayValue;
-	auto evtcodes = libsinsp::filter::ast::ppm_event_codes(ast);
-	auto syscodes = libsinsp::filter::ast::ppm_sc_codes(ast);
-	auto syscodes_to_evt_names = libsinsp::events::sc_set_to_event_names(syscodes);
-	auto evtcodes_to_evt_names = libsinsp::events::event_set_to_names(evtcodes, false);
-	for (const auto& n : unordered_set_union(syscodes_to_evt_names, evtcodes_to_evt_names))
+	// note: condition and output fields may have an argument, so
+	// we need to isolate the field names
+	std::unordered_set<std::string> fieldnames;
+	for (auto f: fields)
 	{
-		output.append(n);
+		auto argpos = f.find('[');
+		if (argpos != std::string::npos)
+		{
+			f = f.substr(0, argpos);
+		}
+		fieldnames.insert(f);
 	}
-}
 
+	std::unordered_set<std::string> used_plugins;
+	for (const auto& p : plugins)
+	{
+		bool used = false;
+		if (p->caps() & CAP_SOURCING)
+		{
+			// The rule's source is implemented by a plugin with event
+			// sourcing capability.
+			// Note: if Falco loads two plugins implementing the same source,
+			// they will both be included in the list.
+			if (!used && p->event_source() == source)
+			{
+				used_plugins.insert(p->name());
+				used = true;
+			}
+		}
+		if (!used && p->caps() & CAP_EXTRACTION)
+		{
+			// The rule uses a field implemented by a plugin with field
+			// extraction capability that is compatible with the rule's source.
+			// Note: here we're assuming that Falco will prevent loading
+			// plugins implementing fields with the same name for the same
+			// event source (implemented in init_inspectors app action).
+			if (sinsp_plugin::is_source_compatible(p->extract_event_sources(), source))
+			{
+				for (const auto &f : p->fields())
+				{
+					if (!used && fieldnames.find(f.m_name) != fieldnames.end())
+					{
+						used_plugins.insert(p->name());
+						used = true;
+						break;
+					}
+				}
+			}
+		}
+		if (!used && p->caps() & CAP_ASYNC)
+		{
+			// The rule matches an event type implemented by a plugin with
+			// async events capability that is compatible with the rule's source.
+			// Note: if Falco loads two plugins implementing async events with
+			// the same name, they will both be included in the list.
+			if (sinsp_plugin::is_source_compatible(p->async_event_sources(), source))
+			{
+				for (const auto &n : p->async_event_names())
+				{
+					if (!used && evtnames.find(n) != evtnames.end())
+					{
+						used_plugins.insert(p->name());
+						used = true;
+						break;
+					}
+				}
+			}
+		}
+	}
+
+	out = sequence_to_json_array(used_plugins);
+}
 
 void falco_engine::print_stats() const
 {

userspace/engine/falco_engine.h

@@ -125,7 +125,7 @@ public:
 	// Print details on the given rule. If rule is NULL, print
 	// details on all rules.
 	//
-	void describe_rule(std::string *rule, bool json) const;
+	void describe_rule(std::string *rule, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins, bool json) const;
 
 	//
 	// Print statistics on how many events matched each rule.
@@ -303,18 +303,31 @@ private:
 	inline bool should_drop_evt() const;
 
 	// Retrieve json details from rules, macros, lists
-	void get_json_details(const falco_rule& r,
-					const rule_loader::rule_info& ri,
-					sinsp* insp,
-					Json::Value& rule) const;
-	void get_json_details(const rule_loader::macro_info& m,
-					Json::Value& macro) const;
-	void get_json_details(const rule_loader::list_info& l,
-					Json::Value& list) const;
-	void get_json_details(libsinsp::filter::ast::expr* ast,
-					Json::Value& output) const;
-	void get_json_evt_types(libsinsp::filter::ast::expr* ast,
-					Json::Value& output) const;
+	void get_json_details(
+		Json::Value& out,
+		const falco_rule& r,
+		const rule_loader::rule_info& info,
+		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
+	void get_json_details(
+		Json::Value& out,
+		const falco_macro& m,
+		const rule_loader::macro_info& info,
+		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
+	void get_json_details(
+		Json::Value& out,
+		const falco_list& l,
+		const rule_loader::list_info& info,
+		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
+	void get_json_evt_types(
+		Json::Value& out,
+		const std::string& source,
+		libsinsp::filter::ast::expr* ast) const;
+	void get_json_used_plugins(
+		Json::Value& out,
+		const std::string& source,
+		const std::unordered_set<std::string>& evttypes,
+		const std::unordered_set<std::string>& fields,
+		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
 
 	rule_loader::collector m_rule_collector;
 	indexed_vector<falco_rule> m_rules;

userspace/engine/falco_rule.h

@@ -21,6 +21,46 @@ limitations under the License.
 #include <string>
 #include "falco_common.h"
 
+#include <filter/ast.h>
+
+/*!
+	\brief Represents a list in the Falco Engine.
+	The rule ID must be unique across all the lists loaded in the engine.
+*/
+struct falco_list
+{
+	falco_list(): used(false), id(0) { }
+	falco_list(falco_list&&) = default;
+	falco_list& operator = (falco_list&&) = default;
+	falco_list(const falco_list&) = default;
+	falco_list& operator = (const falco_list&) = default;
+	~falco_list() = default;
+
+	bool used;
+	std::size_t id;
+	std::string name;
+	std::vector<std::string> items;
+};
+
+/*!
+	\brief Represents a macro in the Falco Engine.
+	The rule ID must be unique across all the macros loaded in the engine.
+*/
+struct falco_macro
+{
+	falco_macro(): used(false), id(0) { }
+	falco_macro(falco_macro&&) = default;
+	falco_macro& operator = (falco_macro&&) = default;
+	falco_macro(const falco_macro&) = default;
+	falco_macro& operator = (const falco_macro&) = default;
+	~falco_macro() = default;
+
+	bool used;
+	std::size_t id;
+	std::string name;
+	std::shared_ptr<libsinsp::filter::ast::expr> condition;
+};
+
 /*!
 	\brief Represents a rule in the Falco Engine.
 	The rule ID must be unique across all the rules loaded in the engine.
@@ -32,6 +72,7 @@ struct falco_rule
 	falco_rule& operator = (falco_rule&&) = default;
 	falco_rule(const falco_rule&) = default;
 	falco_rule& operator = (const falco_rule&) = default;
+	~falco_rule() = default;
 
 	std::size_t id;
 	std::string source;
@@ -41,4 +82,5 @@ struct falco_rule
 	std::set<std::string> tags;
 	std::set<std::string> exception_fields;
 	falco_common::priority_type priority;
+	std::shared_ptr<libsinsp::filter::ast::expr> condition;
 };

userspace/engine/filter_details_resolver.cpp

@@ -19,12 +19,23 @@ limitations under the License.
 
 using namespace libsinsp::filter;
 
+std::string get_field_name(const std::string& name, const std::string& arg)
+{
+	std::string fld = name;
+	if (!arg.empty())
+	{
+		fld += "[" + arg + "]";
+	}
+	return fld;
+}
+
 void filter_details::reset()
 {
 	fields.clear();
 	macros.clear();
 	operators.clear();
 	lists.clear();
+	evtnames.clear();
 }
 
 void filter_details_resolver::run(ast::expr* filter, filter_details& details)
@@ -35,6 +46,7 @@ void filter_details_resolver::run(ast::expr* filter, filter_details& details)
 
 void filter_details_resolver::visitor::visit(ast::and_expr* e)
 {
+	m_expect_macro = false;
 	for(size_t i = 0; i < e->children.size(); i++)
 	{
 		m_expect_macro = true;
@@ -45,6 +57,7 @@ void filter_details_resolver::visitor::visit(ast::and_expr* e)
 
 void filter_details_resolver::visitor::visit(ast::or_expr* e)
 {
+	m_expect_macro = false;
 	for(size_t i = 0; i < e->children.size(); i++)
 	{
 		m_expect_macro = true;
@@ -70,35 +83,50 @@ void filter_details_resolver::visitor::visit(ast::list_expr* e)
 			}
 		}
 	}
+	if (m_expect_evtname)
+	{
+		for(const auto& item : e->values)
+		{
+			if(m_details.known_lists.find(item) == m_details.known_lists.end())
+			{
+				m_details.evtnames.insert(item);
+			}
+		}
+	}
 }
 
 void filter_details_resolver::visitor::visit(ast::binary_check_expr* e)
 {
 	m_expect_macro = false;
-	m_details.fields.insert(e->field);
+	m_details.fields.insert(get_field_name(e->field, e->arg));
 	m_details.operators.insert(e->op);
 	m_expect_list = true;
+	m_expect_evtname = e->field == "evt.type" || e->field == "evt.asynctype";
 	e->value->accept(this);
+	m_expect_evtname = false;
 	m_expect_list = false;
 }
 
 void filter_details_resolver::visitor::visit(ast::unary_check_expr* e)
 {
 	m_expect_macro = false;
-	m_details.fields.insert(e->field);
+	m_details.fields.insert(get_field_name(e->field, e->arg));
 	m_details.operators.insert(e->op);
 }
 
 void filter_details_resolver::visitor::visit(ast::value_expr* e)
 {
-	if(m_expect_macro)
+	if (m_expect_macro)
 	{
-		auto it = m_details.known_macros.find(e->value);
-		if(it == m_details.known_macros.end())
+		if(m_details.known_macros.find(e->value) != m_details.known_macros.end())
 		{
-			return;
+			m_details.macros.insert(e->value);
 		}
-
-		m_details.macros.insert(e->value);
+		// todo(jasondellaluce): should we throw an error if we
+		// encounter an unknown macro?
+	}
+	else if (m_expect_evtname)
+	{
+		m_details.evtnames.insert(e->value);
 	}
 }

userspace/engine/filter_details_resolver.h

@@ -33,6 +33,7 @@ struct filter_details
 	std::unordered_set<std::string> macros;
 	std::unordered_set<std::string> operators;
 	std::unordered_set<std::string> lists;
+	std::unordered_set<std::string> evtnames;
 
 	void reset();
 };
@@ -59,7 +60,8 @@ private:
 		visitor(filter_details& details) : 
 			m_details(details),
 			m_expect_list(false),
-			m_expect_macro(false) {}
+			m_expect_macro(false),
+			m_expect_evtname(false) {}
 		visitor(visitor&&) = default;
 		visitor& operator = (visitor&&) = default;
 		visitor(const visitor&) = delete;
@@ -76,5 +78,6 @@ private:
 		filter_details& m_details;
 		bool m_expect_list;
 		bool m_expect_macro;
+		bool m_expect_evtname;
 	};
 };

userspace/engine/rule_loader.cpp

@@ -532,12 +532,12 @@ rule_loader::plugin_version_info::plugin_version_info(context &ctx)
 }
 
 rule_loader::list_info::list_info(context &ctx)
-	: ctx(ctx), used(false), index(0), visibility(0)
+	: ctx(ctx), index(0), visibility(0)
 {
 }
 
 rule_loader::macro_info::macro_info(context &ctx)
-	: ctx(ctx), cond_ctx(ctx), used(false), index(0), visibility(0)
+	: ctx(ctx), cond_ctx(ctx), index(0), visibility(0)
 {
 }
 

userspace/engine/rule_loader.h

@@ -273,8 +273,7 @@ namespace rule_loader
 			const indexed_vector<falco_source>& srcs,
 			const std::string& name)
 				: content(cont), sources(srcs), name(name),
-				  default_ruleset_id(0), replace_output_container_info(false),
-				  min_priority(falco_common::PRIORITY_DEBUG)
+				  output_extra(), replace_output_container_info(false)
 			{
 				res.reset(new result(name));
 			}
@@ -283,14 +282,15 @@ namespace rule_loader
 		configuration(const configuration&) = delete;
 		configuration& operator = (const configuration&) = delete;
 
+		// inputs
 		const std::string& content;
 		const indexed_vector<falco_source>& sources;
 		std::string name;
-		std::unique_ptr<result> res;
 		std::string output_extra;
-		uint16_t default_ruleset_id;
 		bool replace_output_container_info;
-		falco_common::priority_type min_priority;
+
+		// outputs
+		std::unique_ptr<result> res;
 	};
 
 	/*!
@@ -359,7 +359,6 @@ namespace rule_loader
 		list_info& operator = (const list_info&) = default;
 
 		context ctx;
-		bool used;
 		size_t index;
 		size_t visibility;
 		std::string name;
@@ -380,12 +379,10 @@ namespace rule_loader
 
 		context ctx;
 		context cond_ctx;
-		bool used;
 		size_t index;
 		size_t visibility;
 		std::string name;
 		std::string cond;
-		std::shared_ptr<libsinsp::filter::ast::expr> cond_ast;
 	};
 
 	/*!

userspace/engine/rule_loader_compiler.cpp

@@ -160,8 +160,30 @@ static void build_rule_exception_infos(
 	}
 }
 
+static inline rule_loader::list_info* list_info_from_name(
+	const rule_loader::collector& c, const std::string& name)
+{
+	auto ret = c.lists().at(name);
+	if (!ret)
+	{
+		throw falco_exception("can't find internal list info at name: " + name);
+	}
+	return ret;
+}
+
+static inline rule_loader::macro_info* macro_info_from_name(
+	const rule_loader::collector& c, const std::string& name)
+{
+	auto ret = c.macros().at(name);
+	if (!ret)
+	{
+		throw falco_exception("can't find internal macro info at name: " + name);
+	}
+	return ret;
+}
+
 // todo(jasondellaluce): this breaks string escaping in lists
-static bool resolve_list(std::string& cnd, const rule_loader::list_info& list)
+static bool resolve_list(std::string& cnd, const falco_list& list)
 {
 	static std::string blanks = " \t\n\r";
 	static std::string delims = blanks + "(),=";
@@ -232,18 +254,20 @@ static bool resolve_list(std::string& cnd, const rule_loader::list_info& list)
 }
 
 static void resolve_macros(
-	indexed_vector<rule_loader::macro_info>& macros,
+	const indexed_vector<rule_loader::macro_info>& infos,
+	indexed_vector<falco_macro>& macros,
 	std::shared_ptr<ast::expr>& ast,
 	const std::string& condition,
 	uint32_t visibility,
 	const rule_loader::context &ctx)
 {
 	filter_macro_resolver macro_resolver;
-	for (auto &m : macros)
+	for (auto &m : infos)
 	{
 		if (m.index < visibility)
 		{
-			macro_resolver.set_macro(m.name, m.cond_ast);
+			auto macro = macros.at(m.name);
+			macro_resolver.set_macro(m.name, macro->condition);
 		}
 	}
 	macro_resolver.run(ast);
@@ -272,7 +296,7 @@ static void resolve_macros(
 // note: there is no visibility order between filter conditions and lists
 static std::shared_ptr<ast::expr> parse_condition(
 	std::string condition,
-	indexed_vector<rule_loader::list_info>& lists,
+	indexed_vector<falco_list>& lists,
 	const rule_loader::context &ctx)
 {
 	for (auto &l : lists)
@@ -319,13 +343,14 @@ static void apply_output_substitutions(
 void rule_loader::compiler::compile_list_infos(
 		configuration& cfg,
 		const collector& col,
-		indexed_vector<list_info>& out) const
+		indexed_vector<falco_list>& out) const
 {
 	std::string tmp;
 	std::vector<std::string> used;
 	for (auto &list : col.lists())
 	{
-		list_info v = list;
+		falco_list v;
+		v.name = list.name;
 		v.items.clear();
 		for (auto &item : list.items)
 		{
@@ -347,7 +372,8 @@ void rule_loader::compiler::compile_list_infos(
 			}
 		}
 		v.used = false;
-		out.insert(v, v.name);
+		auto list_id = out.insert(v, v.name);
+		out.at(list_id)->id = list_id;
 	}
 	for (auto &v : used)
 	{
@@ -359,20 +385,23 @@ void rule_loader::compiler::compile_list_infos(
 void rule_loader::compiler::compile_macros_infos(
 		configuration& cfg,
 		const collector& col,
-		indexed_vector<list_info>& lists,
-		indexed_vector<macro_info>& out) const
+		indexed_vector<falco_list>& lists,
+		indexed_vector<falco_macro>& out) const
 {
 	for (auto &m : col.macros())
 	{
-		macro_info entry = m;
-		entry.cond_ast = parse_condition(m.cond, lists, m.cond_ctx);
+		falco_macro entry;
+		entry.name = m.name;
+		entry.condition = parse_condition(m.cond, lists, m.cond_ctx);
 		entry.used = false;
-		out.insert(entry, m.name);
+		auto macro_id = out.insert(entry, m.name);
+		out.at(macro_id)->id = macro_id;
 	}
 
 	for (auto &m : out)
 	{
-		resolve_macros(out, m.cond_ast, m.cond, m.visibility, m.ctx);
+		auto info = macro_info_from_name(col, m.name);
+		resolve_macros(col.macros(), out, m.condition, info->cond, info->visibility, info->ctx);
 	}
 }
 
@@ -386,8 +415,8 @@ static bool err_is_unknown_type_or_field(const std::string& err)
 void rule_loader::compiler::compile_rule_infos(
 		configuration& cfg,
 		const collector& col,
-		indexed_vector<list_info>& lists,
-		indexed_vector<macro_info>& macros,
+		indexed_vector<falco_list>& lists,
+		indexed_vector<falco_macro>& macros,
 		indexed_vector<falco_rule>& out) const
 {
 	std::string err, condition;
@@ -401,12 +430,6 @@ void rule_loader::compiler::compile_rule_infos(
 			continue;
 		}
 
-		// skip the rule if below the minimum priority
-		if (r.priority > cfg.min_priority)
-		{
-			continue;
-		}
-
 		// note: this should not be nullptr if the source is not unknown
 		auto source = cfg.sources.at(r.source);
 		THROW(!source,
@@ -423,12 +446,12 @@ void rule_loader::compiler::compile_rule_infos(
 			build_rule_exception_infos(
 				r.exceptions, rule.exception_fields, condition);
 		}
-		auto ast = parse_condition(condition, lists, r.cond_ctx);
-		resolve_macros(macros, ast, condition, MAX_VISIBILITY, r.ctx);
+		rule.condition = parse_condition(condition, lists, r.cond_ctx);
+		resolve_macros(col.macros(), macros, rule.condition, condition, MAX_VISIBILITY, r.ctx);
 
 		// check for warnings in the filtering condition
 		warn_codes.clear();
-		if (warn_resolver.run(ast.get(), warn_codes))
+		if (warn_resolver.run(rule.condition.get(), warn_codes))
 		{
 			for (auto &w : warn_codes)
 			{
@@ -443,8 +466,11 @@ void rule_loader::compiler::compile_rule_infos(
 			apply_output_substitutions(cfg, rule.output);
 		}
 
+		// validate the rule's output
 		if(!is_format_valid(*cfg.sources.at(r.source), rule.output, err))
 		{
+			// skip the rule silently if skip_if_unknown_filter is true and
+			// we encountered some specific kind of errors
 			if (err_is_unknown_type_or_field(err) && r.skip_if_unknown_filter)
 			{
 				cfg.res->add_warning(
@@ -459,30 +485,18 @@ void rule_loader::compiler::compile_rule_infos(
 				r.output_ctx);
 		}
 
-		// construct rule definition and compile it to a filter
-		rule.name = r.name;
-		rule.source = r.source;
-		rule.description = r.desc;
-		rule.priority = r.priority;
-		rule.tags = r.tags;
-
-		auto rule_id = out.insert(rule, rule.name);
-		out.at(rule_id)->id = rule_id;
-
-		// This also compiles the filter, and might throw a
-		// falco_exception with details on the compilation
-		// failure.
-		sinsp_filter_compiler compiler(cfg.sources.at(r.source)->filter_factory, ast.get());
-		try {
-			std::shared_ptr<gen_event_filter> filter(compiler.compile());
-			source->ruleset->add(*out.at(rule_id), filter, ast);
+		// validate the rule's condition: we compile it into a sinsp filter
+		// on-the-fly and we throw an exception with details on failure
+		sinsp_filter_compiler compiler(cfg.sources.at(r.source)->filter_factory, rule.condition.get());
+		try
+		{
+			compiler.compile();
 		}
 		catch (const sinsp_exception& e)
 		{
-			// Allow errors containing "nonexistent field" if
-			// skip_if_unknown_filter is true
+			// skip the rule silently if skip_if_unknown_filter is true and
+			// we encountered some specific kind of errors
 			std::string err = e.what();
-
 			if (err_is_unknown_type_or_field(err) && r.skip_if_unknown_filter)
 			{
 				cfg.res->add_warning(
@@ -491,7 +505,6 @@ void rule_loader::compiler::compile_rule_infos(
 					r.cond_ctx);
 				continue;
 			}
-
 			rule_loader::context ctx(compiler.get_pos(), condition, r.cond_ctx);
 			throw rule_loader::rule_load_exception(
 				falco::load_result::load_result::LOAD_ERR_COMPILE_CONDITION,
@@ -499,20 +512,10 @@ void rule_loader::compiler::compile_rule_infos(
 				ctx);
 		}
 
-		// By default rules are enabled/disabled for the default ruleset
-		if(r.enabled)
-		{
-			source->ruleset->enable(rule.name, true, cfg.default_ruleset_id);
-		}
-		else
-		{
-			source->ruleset->disable(rule.name, true, cfg.default_ruleset_id);
-		}
-
 		// populate set of event types and emit an special warning
-		if(rule.source == falco_common::syscall_source)
+		if(r.source == falco_common::syscall_source)
 		{
-			auto evttypes = libsinsp::filter::ast::ppm_event_codes(ast.get());
+			auto evttypes = libsinsp::filter::ast::ppm_event_codes(rule.condition.get());
 			if ((evttypes.empty() || evttypes.size() > 100) && r.warn_evttypes)
 			{
 				cfg.res->add_warning(
@@ -521,23 +524,29 @@ void rule_loader::compiler::compile_rule_infos(
 					r.ctx);
 			}
 		}
+
+		// finalize the rule definition and add it to output
+		rule.name = r.name;
+		rule.source = r.source;
+		rule.description = r.desc;
+		rule.priority = r.priority;
+		rule.tags = r.tags;
+		auto rule_id = out.insert(rule, rule.name);
+		out.at(rule_id)->id = rule_id;
 	}
 }
 
 void rule_loader::compiler::compile(
 		configuration& cfg,
 		const collector& col,
-		indexed_vector<falco_rule>& out) const
+		compile_output& out) const
 {
-	indexed_vector<list_info> lists;
-	indexed_vector<macro_info> macros;
-
 	// expand all lists, macros, and rules
 	try
 	{
-		compile_list_infos(cfg, col, lists);
-		compile_macros_infos(cfg, col, lists, macros);
-		compile_rule_infos(cfg, col, lists, macros, out);
+		compile_list_infos(cfg, col, out.lists);
+		compile_macros_infos(cfg, col, out.lists, out.macros);
+		compile_rule_infos(cfg, col, out.lists, out.macros, out.rules);
 	}
 	catch(rule_load_exception &e)
 	{
@@ -546,24 +555,24 @@ void rule_loader::compiler::compile(
 	}
 
 	// print info on any dangling lists or macros that were not used anywhere
-	for (auto &m : macros)
+	for (auto &m : out.macros)
 	{
 		if (!m.used)
 		{
 			cfg.res->add_warning(
 				falco::load_result::load_result::LOAD_UNUSED_MACRO,
 				"Macro not referred to by any other rule/macro",
-				m.ctx);
+				macro_info_from_name(col, m.name)->ctx);
 		}
 	}
-	for (auto &l : lists)
+	for (auto &l : out.lists)
 	{
 		if (!l.used)
 		{
 			cfg.res->add_warning(
 				falco::load_result::LOAD_UNUSED_LIST,
 				"List not referred to by any other rule/macro",
-				l.ctx);
+				list_info_from_name(col, l.name)->ctx);
 		}
 	}
 }

userspace/engine/rule_loader_compiler.h

@@ -31,6 +31,23 @@ namespace rule_loader
 class compiler
 {
 public:
+	/*!
+		\brief The output of a compilation.
+	*/
+	struct compile_output
+	{
+		compile_output() = default;
+		virtual ~compile_output() = default;
+		compile_output(compile_output&&) = default;
+		compile_output& operator = (compile_output&&) = default;
+		compile_output(const compile_output&) = default;
+		compile_output& operator = (const compile_output&) = default;
+
+		indexed_vector<falco_list> lists;
+		indexed_vector<falco_macro> macros;
+		indexed_vector<falco_rule> rules;
+	};
+
 	compiler() = default;
 	virtual ~compiler() = default;
 	compiler(compiler&&) = default;
@@ -44,25 +61,25 @@ public:
 	virtual void compile(
 		configuration& cfg,
 		const collector& col,
-		indexed_vector<falco_rule>& out) const;
+		compile_output& out) const;
 
 private:
 	void compile_list_infos(
 		configuration& cfg,
 		const collector& col,
-		indexed_vector<list_info>& out) const;
+		indexed_vector<falco_list>& out) const;
 
 	void compile_macros_infos(
 		configuration& cfg,
 		const collector& col,
-		indexed_vector<list_info>& lists,
-		indexed_vector<macro_info>& out) const;
+		indexed_vector<falco_list>& lists,
+		indexed_vector<falco_macro>& out) const;
 
 	void compile_rule_infos(
 		configuration& cfg,
 		const collector& col,
-		indexed_vector<list_info>& lists,
-		indexed_vector<macro_info>& macros,
+		indexed_vector<falco_list>& lists,
+		indexed_vector<falco_macro>& macros,
 		indexed_vector<falco_rule>& out) const;
 };
 

userspace/falco/app/actions/helpers_inspector.cpp

@@ -86,15 +86,6 @@ falco::app::run_result falco::app::actions::open_live_inspector(
 			falco_logger::log(LOG_INFO, "Opening '" + source + "' source with no driver\n");
 			inspector->open_nodriver();
 		}
-		else if (s.options.userspace) /* udig engine. */
-		{
-			// open_udig() is the underlying method used in the capture code to parse userspace events from the kernel.
-			//
-			// Falco uses a ptrace(2) based userspace implementation.
-			// Regardless of the implementation, the underlying method remains the same.
-			falco_logger::log(LOG_WARNING, "The udig engine is deprecated and will be removed in Falco 0.37. Opening '" + source + "' source with udig\n");
-			inspector->open_udig();
-		}
 		else if(s.is_gvisor_enabled()) /* gvisor engine. */
 		{
 			falco_logger::log(LOG_INFO, "Opening '" + source + "' source with gVisor. Configuration path: " + s.options.gvisor_config);

userspace/falco/app/actions/init_outputs.cpp

@@ -65,6 +65,7 @@ falco::app::run_result falco::app::actions::init_outputs(falco::app::state& s)
 		s.config->m_output_timeout,
 		s.config->m_buffered_outputs,
 		s.config->m_outputs_queue_capacity,
+		s.config->m_outputs_queue_recovery,
 		s.config->m_time_format_iso_8601,
 		hostname));
 

userspace/falco/app/actions/load_rules_files.cpp

@@ -157,13 +157,15 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 
 	if (s.options.describe_all_rules)
 	{
-		s.engine->describe_rule(NULL, s.config->m_json_output);
+		const auto& plugins = s.offline_inspector->get_plugin_manager()->plugins();
+		s.engine->describe_rule(NULL, plugins, s.config->m_json_output);
 		return run_result::exit();
 	}
 
 	if (!s.options.describe_rule.empty())
 	{
-		s.engine->describe_rule(&(s.options.describe_rule), s.config->m_json_output);
+		const auto& plugins = s.offline_inspector->get_plugin_manager()->plugins();
+		s.engine->describe_rule(&(s.options.describe_rule), plugins, s.config->m_json_output);
 		return run_result::exit();
 	}
 

userspace/falco/app/options.cpp

@@ -37,7 +37,6 @@ options::options()
 	  list_plugins(false),
 	  list_syscall_events(false),
 	  markdown(false),
-	  userspace(false),
 	  modern_bpf(false),
 	  dry_run(false),
 	  nodriver(false)
@@ -148,14 +147,13 @@ bool options::parse(int argc, char **argv, std::string &errstr)
 
 	int open_modes = 0;
 	open_modes += !trace_filename.empty();
-	open_modes += userspace;
 	open_modes += !gvisor_config.empty();
 	open_modes += modern_bpf;
 	open_modes += getenv("FALCO_BPF_PROBE") != NULL;
 	open_modes += nodriver;
 	if (open_modes > 1)
 	{
-		errstr = std::string("You can not specify more than one of -e, -u (--userspace), -g (--gvisor-config), --modern-bpf, --nodriver, and the FALCO_BPF_PROBE env var");
+		errstr = std::string("You can not specify more than one of -e, -g (--gvisor-config), --modern-bpf, --nodriver, and the FALCO_BPF_PROBE env var");
 		return false;
 	}
 
@@ -220,9 +218,6 @@ void options::define(cxxopts::Options& opts)
 		("T",                             "Turn off any rules with a tag=<tag>. This option can be passed multiple times. This option can not be mixed with -t.", cxxopts::value<std::vector<std::string>>(), "<tag>")
 		("t",                             "Only enable those rules with a tag=<tag>. This option can be passed multiple times. This option can not be mixed with -T/-D.", cxxopts::value<std::vector<std::string>>(), "<tag>")
 		("U,unbuffered",                  "Turn off output buffering for configured outputs. This causes every single line emitted by Falco to be flushed, which generates higher CPU usage but is useful when piping those outputs into another process or a script.", cxxopts::value(unbuffered_outputs)->default_value("false"))
-#if !defined(_WIN32) && !defined(__EMSCRIPTEN__) && !defined(MINIMAL_BUILD)
-		("u,userspace",                   "[DEPRECATED: this option will be removed in Falco 0.37] Use a userspace driver to collect 'syscall' events. To be used in conjunction with the ptrace(2) based driver (pdig).", cxxopts::value(userspace)->default_value("false"))
-#endif
 		("V,validate",                    "Read the contents of the specified <rules_file> file(s), validate the loaded rules, and exit. This option can be passed multiple times to validate multiple files.", cxxopts::value(validate_rules_filenames), "<rules_file>")
 		("v",                             "Enable verbose output.", cxxopts::value(verbose)->default_value("false"))
 		("version",                       "Print version information and exit.", cxxopts::value(print_version_info)->default_value("false"))

userspace/falco/app/options.h

@@ -75,7 +75,6 @@ public:
 	std::set<std::string> disabled_rule_tags;
 	std::set<std::string> enabled_rule_tags;
 	bool unbuffered_outputs;
-	bool userspace;
 	std::vector<std::string> validate_rules_filenames;
 	bool verbose;
 	bool print_version_info;

userspace/falco/atomic_signal_handler.h

@@ -87,9 +87,9 @@ namespace falco
         /**
          * @brief If a signal is triggered, performs an handler action.
          * The action function will be invoked exactly once among all the
-         * simultaneous calls. The action will not be performed if the
+         * simultaneus calls. The action will not be performed if the
          * signal is not triggered, or if the triggered has already been
-         * handled. When an action is being performed, all the simultaneous
+         * handled. When an action is being performed, all the simultaneus
          * callers will wait and be blocked up until its execution is finished.
          * If the handler action throws an exception, it will be considered
          * performed. After the first handler has been performed, every

userspace/falco/configuration.cpp

@@ -42,6 +42,7 @@ falco_configuration::falco_configuration():
 	m_watch_config_files(true),
 	m_buffered_outputs(false),
 	m_outputs_queue_capacity(DEFAULT_OUTPUTS_QUEUE_CAPACITY_UNBOUNDED_MAX_LONG_VALUE),
+	m_outputs_queue_recovery(falco_common::RECOVERY_EXIT),
 	m_time_format_iso_8601(false),
 	m_output_timeout(2000),
 	m_grpc_enabled(false),
@@ -289,6 +290,11 @@ void falco_configuration::load_yaml(const std::string& config_name, const yaml_h
 	{
 		m_outputs_queue_capacity = DEFAULT_OUTPUTS_QUEUE_CAPACITY_UNBOUNDED_MAX_LONG_VALUE;
 	}
+	std::string recovery = config.get_scalar<std::string>("outputs_queue.recovery", "exit");
+	if (!falco_common::parse_queue_recovery(recovery, m_outputs_queue_recovery))
+	{
+		throw std::logic_error("Unknown recovery \"" + recovery + "\"--must be one of exit, continue, empty");
+	}
 
 	m_time_format_iso_8601 = config.get_scalar<bool>("time_format_iso_8601", false);
 

userspace/falco/configuration.h

@@ -74,6 +74,7 @@ public:
 	bool m_watch_config_files;
 	bool m_buffered_outputs;
 	size_t m_outputs_queue_capacity;
+	falco_common::outputs_queue_recovery_type m_outputs_queue_recovery;
 	bool m_time_format_iso_8601;
 	uint32_t m_output_timeout;
 

userspace/falco/falco_outputs.cpp

@@ -48,6 +48,7 @@ falco_outputs::falco_outputs(
 	uint32_t timeout,
 	bool buffered,
 	size_t outputs_queue_capacity,
+	falco_common::outputs_queue_recovery_type outputs_queue_recovery,
 	bool time_format_iso_8601,
 	const std::string& hostname)
 {
@@ -66,6 +67,7 @@ falco_outputs::falco_outputs(
 		add_output(output);
 	}
 	m_outputs_queue_num_drops = {0};
+	m_outputs_queue_recovery = outputs_queue_recovery;
 #ifndef __EMSCRIPTEN__
 	m_queue.set_capacity(outputs_queue_capacity);
 	m_worker_thread = std::thread(&falco_outputs::worker, this);
@@ -285,11 +287,29 @@ inline void falco_outputs::push(const ctrl_msg& cmsg)
 #ifndef __EMSCRIPTEN__
 	if (!m_queue.try_push(cmsg))
 	{
-		if(m_outputs_queue_num_drops.load() == 0)
+		switch (m_outputs_queue_recovery)
 		{
-			falco_logger::log(LOG_ERR, "Outputs queue out of memory. Drop event and continue on ...");
+		case falco_common::RECOVERY_EXIT:
+			throw falco_exception("Fatal error: Output queue out of memory. Exiting ...");
+		case falco_common::RECOVERY_EMPTY:
+			/* Print a log just the first time */
+			if(m_outputs_queue_num_drops.load() == 0)
+			{
+				falco_logger::log(LOG_ERR, "Output queue out of memory. Drop event plus events in queue due to emptying the queue; continue on ...");
+			}
+			m_outputs_queue_num_drops += m_queue.size() + 1;
+			m_queue.clear();
+			break;
+		case falco_common::RECOVERY_CONTINUE:
+			if(m_outputs_queue_num_drops.load() == 0)
+			{
+				falco_logger::log(LOG_ERR, "Output queue out of memory. Drop event and continue on ...");
+			}
+			m_outputs_queue_num_drops++;
+			break;
+		default:
+			throw falco_exception("Fatal error: strategy unknown. Exiting ...");
 		}
-		m_outputs_queue_num_drops++;
 	}
 #else
 	for (auto o : m_outputs)

userspace/falco/falco_outputs.h

@@ -36,7 +36,7 @@ limitations under the License.
 
 	All methods in this class are thread-safe. The output framework supports
 	a multi-producer model where messages are stored in a queue and consumed
-	by each configured output asynchronously.
+	by each configured output asynchrounously.
 */
 class falco_outputs
 {
@@ -50,6 +50,7 @@ public:
 		uint32_t timeout,
 		bool buffered,
 		size_t outputs_queue_capacity,
+		falco_common::outputs_queue_recovery_type outputs_queue_recovery,
 		bool time_format_iso_8601,
 		const std::string& hostname);
 
@@ -86,8 +87,8 @@ public:
 	void reopen_outputs();
 
 	/*!
-		\brief Return the number of events currently dropped due to failed push 
-		attempts into the outputs queue
+		\brief Return the number of currently dropped events as a result of failed push attempts
+		into the outputs queue when using `continue` or `empty` recovery strategies.
 	*/
 	uint64_t get_outputs_queue_num_drops();
 
@@ -120,6 +121,7 @@ private:
 	falco_outputs_cbq m_queue;
 #endif
 
+	falco_common::outputs_queue_recovery_type m_outputs_queue_recovery;
 	std::atomic<uint64_t> m_outputs_queue_num_drops;
 	std::thread m_worker_thread;
 	inline void push(const ctrl_msg& cmsg);

userspace/falco/stats_writer.cpp

@@ -34,12 +34,6 @@ limitations under the License.
 // check that this value changed since their last observation.
 static std::atomic<stats_writer::ticker_t> s_timer((stats_writer::ticker_t) 0);
 static timer_t s_timerid;
-// note: Workaround for older GLIBC versions (< 2.35), where calling timer_delete() 
-// with an invalid timer ID not returned by timer_create() causes a segfault because of 
-// a bug in GLIBC (https://sourceware.org/bugzilla/show_bug.cgi?id=28257).
-// Just performing a nullptr check is not enough as even after creating the timer, s_timerid
-// remains a nullptr somehow.
-bool s_timerid_exists = false;
 
 static void timer_handler(int signum)
 {
@@ -66,31 +60,18 @@ bool stats_writer::init_ticker(uint32_t interval_msec, std::string &err)
 	sev.sigev_value.sival_ptr = &s_timerid;
 #ifndef __EMSCRIPTEN__
 	// delete any previously set timer
-	if (s_timerid_exists)
-	{
-		if (timer_delete(s_timerid) == -1) 
-		{
-			err = std::string("Could not delete previous timer: ") + strerror(errno);
-			return false;
-		}
-		s_timerid_exists = false;
-	}
-
-	if (timer_create(CLOCK_MONOTONIC, &sev, &s_timerid) == -1) 
-	{
+	timer_delete(s_timerid);
+	if (timer_create(CLOCK_MONOTONIC, &sev, &s_timerid) == -1) {
 		err = std::string("Could not create periodic timer: ") + strerror(errno);
 		return false;
 	}
-	s_timerid_exists = true;
-
 #endif
 	timer.it_value.tv_sec = interval_msec / 1000;
 	timer.it_value.tv_nsec = (interval_msec % 1000) * 1000 * 1000;
 	timer.it_interval = timer.it_value;
 
 #ifndef __EMSCRIPTEN__
-	if (timer_settime(s_timerid, 0, &timer, NULL) == -1) 
-	{
+	if (timer_settime(s_timerid, 0, &timer, NULL) == -1) {
 		err = std::string("Could not set up periodic timer: ") + strerror(errno);
 		return false;
 	}
@@ -132,7 +113,8 @@ stats_writer::stats_writer(
 	if (m_initialized)
 	{
 #ifndef __EMSCRIPTEN__
-		// Adopt capacity for completeness, even if it's likely not relevant
+		// capacity and controls should not be relevant for stats outputs, adopt capacity
+		// for completeness, but do not implement config recovery strategies.
 		m_queue.set_capacity(config->m_outputs_queue_capacity);
 		m_worker = std::thread(&stats_writer::worker, this);
 #endif
@@ -152,11 +134,7 @@ stats_writer::~stats_writer()
 		}
 		// delete timerID and reset timer
 #ifndef __EMSCRIPTEN__
-		if (s_timerid_exists)
-		{
-			timer_delete(s_timerid);
-			s_timerid_exists = false;
-		}
+		timer_delete(s_timerid);
 #endif
 	}
 }
@@ -251,7 +229,7 @@ void stats_writer::collector::get_metrics_output_fields_wrapper(
 {
 	static const char* all_driver_engines[] = {
 		BPF_ENGINE, KMOD_ENGINE, MODERN_BPF_ENGINE,
-		SOURCE_PLUGIN_ENGINE, NODRIVER_ENGINE, UDIG_ENGINE, GVISOR_ENGINE };
+		SOURCE_PLUGIN_ENGINE, NODRIVER_ENGINE, GVISOR_ENGINE };
 	const scap_agent_info* agent_info = inspector->get_agent_info();
 	const scap_machine_info* machine_info = inspector->get_machine_info();