new(metrics): enable plugins metrics

Signed-off-by: Gianmatteo Palmieri <mail@gian.im>
Co-authored-by: Melissa Kilby <melissa.kilby.oss@gmail.com>

cmake/modules/falcosecurity-libs.cmake

@@ -35,8 +35,8 @@ else()
   # In case you want to test against another falcosecurity/libs version (or branch, or commit) just pass the variable -
   # ie., `cmake -DFALCOSECURITY_LIBS_VERSION=dev ..`
   if(NOT FALCOSECURITY_LIBS_VERSION)
-    set(FALCOSECURITY_LIBS_VERSION "0.17.1")
-    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=d252aa689f7d3eafe9470d553ed4e42e0c58fd2c90141f77ca3e6f7de10c2b14")
+    set(FALCOSECURITY_LIBS_VERSION "0.17.2")
+    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=5c4f0c987272b7d5236f6ab2bbe3906ffdaf76b59817b63cf90cc8c387ab5b15")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

falco.yaml

@@ -1037,6 +1037,11 @@ syscall_event_drops:
 # beneficial for exploring the data schema and ensuring that fields with empty
 # values are included in the output.
 #
+# `plugins_metrics_enabled`: Falco can now expose your custom plugins' 
+# metrics. Please note that if the respective plugin has no metrics implemented, 
+# there will be no metrics available. In other words, there are no default or 
+# generic plugin metrics at this time. This may be subject to change.
+#
 # If metrics are enabled, the web server can be configured to activate the
 # corresponding Prometheus endpoint using `webserver.prometheus_metrics_enabled`.
 # Prometheus output can be used in combination with the other output options.
@@ -1055,6 +1060,7 @@ metrics:
   state_counters_enabled: true
   kernel_event_counters_enabled: true
   libbpf_stats_enabled: true
+  plugins_metrics_enabled: true
   convert_memory_to_mb: true
   include_empty_values: false
 

userspace/engine/rule_loader_compiler.cpp

@@ -509,7 +509,15 @@ void rule_loader::compiler::compile_rule_infos(
 
 		// build rule output message
 		rule.output = r.output;
-		apply_output_substitutions(cfg, rule.output);
+
+		// plugins sources do not have any container info and so we won't apply -pk, -pc, etc.
+		// on the other hand, when using plugins you might want to append custom output based on the plugin
+		// TODO: this is not flexible enough (esp. if you mix plugin with syscalls),
+		// it would be better to add configuration options to control the output.
+		if (!cfg.replace_output_container_info || r.source == falco_common::syscall_source)
+		{
+			apply_output_substitutions(cfg, rule.output);
+		}
 
 		// validate the rule's output
 		if(!is_format_valid(*cfg.sources.at(r.source), rule.output, err))

userspace/falco/app/actions/process_events.cpp

@@ -413,9 +413,9 @@ static falco::app::run_result init_stats_writer(
 		return falco::app::run_result::fatal("Metrics interval was passed as numeric value without Prometheus time unit. Please specify a time unit");
 	}
 
-	if (config->m_metrics_enabled && !sw->has_output())
+	if (config->m_metrics_enabled && !(sw->has_output() || config->m_webserver_config.m_prometheus_metrics_enabled))
 	{
-		return falco::app::run_result::fatal("Metrics are enabled with no output configured. Please enable at least one output channel");
+		return falco::app::run_result::fatal("Metrics are enabled with no output configured. Please enable at least one output channel ('metrics.output_rule', 'metrics.output_file' or 'webserver.prometheus_metrics_enabled')");
 	}
 
 	falco_logger::log(falco_logger::level::INFO, "Setting metrics interval to " + config->m_metrics_interval_str + ", equivalent to " + std::to_string(config->m_metrics_interval) + " (ms)\n");

userspace/falco/app/options.cpp

@@ -172,7 +172,7 @@ void options::define(cxxopts::Options& opts)
 		("N",                             "Only print field names when used in conjunction with the --list option. It has no effect when used with other options.", cxxopts::value(names_only)->default_value("false"))
 		("o,option",                      "Set the value of option <opt> to <val>. Overrides values in the configuration file. <opt> can be identified using its location in the configuration file using dot notation. Elements of list entries can be accessed via square brackets [].\n    E.g. base.id = val\n         base.subvalue.subvalue2 = val\n         base.list[1]=val", cxxopts::value(cmdline_config_options), "<opt>=<val>")
 		("plugin-info",                   "Print info for the plugin specified by <plugin_name> and exit.\nThis includes all descriptive information like name and author, along with the\nschema format for the init configuration and a list of suggested open parameters.\n<plugin_name> can be the plugin's name or its configured 'library_path'.", cxxopts::value(print_plugin_info), "<plugin_name>")
-		("p,print",                       "Print (or replace) additional information in the rule's output.\nUse -pc or -pcontainer to append container details.\nUse -pk or -pkubernetes to add both container and Kubernetes details.\nIf using gVisor, choose -pcg or -pkg variants (or -pcontainer-gvisor and -pkubernetes-gvisor, respectively).\nIf a rule's output contains %container.info, it will be replaced with the corresponding details. Otherwise, these details will be directly appended to the rule's output.\nAlternatively, use -p <output_format> for a custom format. In this case, the given <output_format> will be appended to the rule's output without any replacement.", cxxopts::value(print_additional), "<output_format>")
+		("p,print",                       "Print (or replace) additional information in the rule's output.\nUse -pc or -pcontainer to append container details to syscall events.\nUse -pk or -pkubernetes to add both container and Kubernetes details to syscall events.\nIf using gVisor, choose -pcg or -pkg variants (or -pcontainer-gvisor and -pkubernetes-gvisor, respectively).\nIf a syscall rule's output contains %container.info, it will be replaced with the corresponding details. Otherwise, these details will be directly appended to the rule's output.\nAlternatively, use -p <output_format> for a custom format. In this case, the given <output_format> will be appended to the rule's output without any replacement to all events, including plugin events.", cxxopts::value(print_additional), "<output_format>")
 		("P,pidfile",                     "Write PID to specified <pid_file> path. By default, no PID file is created.", cxxopts::value(pidfilename)->default_value(""), "<pid_file>")
 		("r",                             "Rules file or directory to be loaded. This option can be passed multiple times. Falco defaults to the values in the configuration file when this option is not specified.", cxxopts::value<std::vector<std::string>>(), "<rules_file>")
 		("S,snaplen",                     "Collect only the first <len> bytes of each I/O buffer for 'syscall' events. By default, the first 80 bytes are collected by the driver and sent to the user space for processing. Use this option with caution since it can have a strong performance impact.", cxxopts::value(snaplen)->default_value("0"), "<len>")

userspace/falco/configuration.cpp

@@ -72,7 +72,7 @@ falco_configuration::falco_configuration():
 	m_metrics_interval(5000),
 	m_metrics_stats_rule_enabled(false),
 	m_metrics_output_file(""),
-	m_metrics_flags((METRICS_V2_KERNEL_COUNTERS | METRICS_V2_LIBBPF_STATS | METRICS_V2_RESOURCE_UTILIZATION | METRICS_V2_STATE_COUNTERS | METRICS_V2_RULE_COUNTERS)),
+	m_metrics_flags(0),
 	m_metrics_convert_memory_to_mb(true),
 	m_metrics_include_empty_values(false)
 {
@@ -555,6 +555,10 @@ void falco_configuration::load_yaml(const std::string& config_name)
 	{
 		m_metrics_flags |= METRICS_V2_LIBBPF_STATS;
 	}
+	if (config.get_scalar<bool>("metrics.plugins_metrics_enabled", true))
+	{
+		m_metrics_flags |= METRICS_V2_PLUGINS;
+	}
 
 	m_metrics_convert_memory_to_mb = config.get_scalar<bool>("metrics.convert_memory_to_mb", true);
 	m_metrics_include_empty_values = config.get_scalar<bool>("metrics.include_empty_values", false);

userspace/falco/falco_metrics.cpp

@@ -53,20 +53,20 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		BPF_ENGINE, KMOD_ENGINE, MODERN_BPF_ENGINE,
 		SOURCE_PLUGIN_ENGINE, NODRIVER_ENGINE, GVISOR_ENGINE };
 
-	std::vector<sinsp*> inspectors;
+	std::vector<std::shared_ptr<sinsp>> inspectors;
 	std::vector<libs::metrics::libs_metrics_collector> metrics_collectors;
 
-	for (const auto& source_info: state.source_infos)
+	for (const auto& source: state.enabled_sources)
 	{
-		sinsp *source_inspector = source_info.inspector.get();
+		auto source_info = state.source_infos.at(source);
+		auto source_inspector = source_info->inspector;
 		inspectors.emplace_back(source_inspector);
-		metrics_collectors.emplace_back(libs::metrics::libs_metrics_collector(source_inspector, state.config->m_metrics_flags));
+		metrics_collectors.emplace_back(libs::metrics::libs_metrics_collector(source_inspector.get(), state.config->m_metrics_flags));
 	}
-
 	libs::metrics::prometheus_metrics_converter prometheus_metrics_converter;
 	std::string prometheus_text;
 
-	for (auto* inspector: inspectors)
+	for (auto inspector: inspectors)
 	{
 		// Falco wrapper metrics
 		//
@@ -79,21 +79,28 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 			}
 		}
 
+
 		const scap_agent_info* agent_info = inspector->get_agent_info();
 		const scap_machine_info* machine_info = inspector->get_machine_info();
-
-		libs::metrics::libs_metrics_collector libs_metrics_collector(inspector, 0);
-
+		libs::metrics::libs_metrics_collector libs_metrics_collector(inspector.get(), 0);
 		prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("version", "falcosecurity", "falco", {{"version", FALCO_VERSION}});
-		prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("kernel_release", "falcosecurity", "falco", {{"kernel_release", agent_info->uname_r}});
-		prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("hostname", "falcosecurity", "evt", {{"hostname", machine_info->hostname}});
+
+		// Not all scap engines report agent and machine infos.
+		if (agent_info)
+		{
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("kernel_release", "falcosecurity", "falco", {{"kernel_release", agent_info->uname_r}});
+		}
+		if (machine_info)
+		{
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("hostname", "falcosecurity", "evt", {{"hostname", machine_info->hostname}});
+		}
 
 #if defined(__linux__) and !defined(MINIMAL_BUILD) and !defined(__EMSCRIPTEN__)
 		for (const auto& item : state.config.get()->m_loaded_rules_filenames_sha256sum)
 		{
 			fs::path fs_path = item.first;
 			std::string metric_name_file_sha256 = fs_path.filename().stem();
-			metric_name_file_sha256 = "falco.sha256_rules_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
+			metric_name_file_sha256 = "falco_sha256_rules_file_" + falco::utils::sanitize_metric_name(metric_name_file_sha256);
 			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric_name_file_sha256, "falcosecurity", "falco", {{metric_name_file_sha256, item.second}});
 		}
 
@@ -101,7 +108,7 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		{
 			fs::path fs_path = item.first;
 			std::string metric_name_file_sha256 = fs_path.filename().stem();
-			metric_name_file_sha256 = "falco.sha256_config_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
+			metric_name_file_sha256 = "falco_sha256_config_file_" + falco::utils::sanitize_metric_name(metric_name_file_sha256);
 			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric_name_file_sha256, "falcosecurity", "falco", {{metric_name_file_sha256, item.second}});
 		}
 #endif
@@ -112,39 +119,47 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		}
 		std::vector<metrics_v2> additional_wrapper_metrics;
 
-		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("start_ts",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
-																	METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
-																	agent_info->start_ts_epoch));
-		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_boot_ts",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
-																	METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
-																	machine_info->boot_ts_epoch));
-		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_num_cpus",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U32,
-																	METRIC_VALUE_UNIT_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
-																	machine_info->num_cpus));
+		if (agent_info)
+		{
+			additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("start_ts",
+												  METRICS_V2_MISC,
+												  METRIC_VALUE_TYPE_U64,
+												  METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
+												  METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
+												  agent_info->start_ts_epoch));
+		}
+		if (machine_info)
+		{
+			additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_boot_ts",
+												  METRICS_V2_MISC,
+												  METRIC_VALUE_TYPE_U64,
+												  METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
+												  METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
+												  machine_info->boot_ts_epoch));
+			additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_num_cpus",
+												  METRICS_V2_MISC,
+												  METRIC_VALUE_TYPE_U32,
+												  METRIC_VALUE_UNIT_COUNT,
+												  METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
+												  machine_info->num_cpus));
+		}
 		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("outputs_queue_num_drops",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
-																	state.outputs->get_outputs_queue_num_drops()));
+												METRICS_V2_MISC,
+												METRIC_VALUE_TYPE_U64,
+												METRIC_VALUE_UNIT_COUNT,
+												METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+												state.outputs->get_outputs_queue_num_drops()));
 
-		auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
-
-		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("duration_sec",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_TIME_S_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
-																	(uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS)));
+		if (agent_info)
+		{
+			auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+			additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("duration_sec",
+												  METRICS_V2_MISC,
+												  METRIC_VALUE_TYPE_U64,
+												  METRIC_VALUE_UNIT_TIME_S_COUNT,
+												  METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+												  (uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS)));
+		}
 
 		for (auto metric: additional_wrapper_metrics)
 		{
@@ -160,11 +175,11 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 			const stats_manager& rule_stats_manager = state.engine->get_rule_stats_manager();
 			const indexed_vector<falco_rule>& rules = state.engine->get_rules();
 			auto metric = libs_metrics_collector.new_metric("rules.matches_total",
-																	METRICS_V2_RULE_COUNTERS,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
-																	rule_stats_manager.get_total().load());
+										METRICS_V2_RULE_COUNTERS,
+										METRIC_VALUE_TYPE_U64,
+										METRIC_VALUE_UNIT_COUNT,
+										METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+										rule_stats_manager.get_total().load());
 
 			prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
 			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", "falco");
@@ -175,11 +190,11 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 				std::string rules_metric_name = "rules." + falco::utils::sanitize_metric_name(rule->name);
 				// Separate processing of rules counter metrics given we add extra tags
 				auto metric = libs_metrics_collector.new_metric(rules_metric_name.c_str(),
-																	METRICS_V2_RULE_COUNTERS,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
-																	rules_by_id[i]->load());
+											METRICS_V2_RULE_COUNTERS,
+											METRIC_VALUE_TYPE_U64,
+											METRIC_VALUE_UNIT_COUNT,
+											METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+											rules_by_id[i]->load());
 				prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
 				const std::map<std::string, std::string>& const_labels = {
 					{"rule", rule->name},
@@ -207,10 +222,17 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		{
 			prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
 			std::string namespace_name = "scap";
+			
 			if (metric.flags & METRICS_V2_RESOURCE_UTILIZATION || metric.flags & METRICS_V2_KERNEL_COUNTERS)
 			{
 				namespace_name = "falco";
 			}
+
+			if (metric.flags & METRICS_V2_PLUGINS)
+			{
+				namespace_name = "plugins";
+			}
+
 			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", namespace_name);
 		}
 

userspace/falco/stats_writer.cpp

@@ -327,12 +327,18 @@ void stats_writer::collector::get_metrics_output_fields_wrapper(
 	/* Wrapper fields useful for statistical analyses and attributions. Always enabled. */
 	output_fields["evt.time"] = now; /* Some ETLs may prefer a consistent timestamp within output_fields. */
 	output_fields["falco.version"] = FALCO_VERSION;
-	output_fields["falco.start_ts"] = agent_info->start_ts_epoch;
-	output_fields["falco.duration_sec"] = (uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS);
-	output_fields["falco.kernel_release"] = agent_info->uname_r;
-	output_fields["evt.hostname"] = machine_info->hostname; /* Explicitly add hostname to log msg in case hostname rule output field is disabled. */
-	output_fields["falco.host_boot_ts"] = machine_info->boot_ts_epoch;
-	output_fields["falco.host_num_cpus"] = machine_info->num_cpus;
+	if (agent_info)
+	{
+		output_fields["falco.start_ts"] = agent_info->start_ts_epoch;
+		output_fields["falco.duration_sec"] = (uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS);
+		output_fields["falco.kernel_release"] = agent_info->uname_r;
+	}
+	if (machine_info)
+	{
+		output_fields["evt.hostname"] = machine_info->hostname; /* Explicitly add hostname to log msg in case hostname rule output field is disabled. */
+		output_fields["falco.host_boot_ts"] = machine_info->boot_ts_epoch;
+		output_fields["falco.host_num_cpus"] = machine_info->num_cpus;
+	}
 	output_fields["falco.outputs_queue_num_drops"] = m_writer->m_outputs->get_outputs_queue_num_drops();
 
 #if defined(__linux__) and !defined(MINIMAL_BUILD) and !defined(__EMSCRIPTEN__)
@@ -434,6 +440,10 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 			{
 				strlcpy(metric_name, "scap.", sizeof(metric_name));
 			}
+			if(metric.flags & METRICS_V2_PLUGINS)
+			{
+				strlcpy(metric_name, "plugins.", sizeof(metric_name));
+			}
 			strlcat(metric_name, metric.name, sizeof(metric_name));
 
 			switch (metric.type)
@@ -445,6 +455,13 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 				}
 				output_fields[metric_name] = metric.value.u32;
 				break;
+			case METRIC_VALUE_TYPE_S32:
+				if (metric.value.s32 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				output_fields[metric_name] = metric.value.s32;
+				break;
 			case METRIC_VALUE_TYPE_U64:
 				if (strncmp(metric.name, "n_evts", 7) == 0)
 				{
@@ -486,6 +503,13 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 				}
 				output_fields[metric_name] = metric.value.u64;
 				break;
+			case METRIC_VALUE_TYPE_S64:
+				if (metric.value.s64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				output_fields[metric_name] = metric.value.s64;
+				break;
 			case METRIC_VALUE_TYPE_D:
 				if (metric.value.d == 0 && !m_writer->m_config->m_metrics_include_empty_values)
 				{
@@ -493,6 +517,20 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 				}
 				output_fields[metric_name] = metric.value.d;
 				break;
+			case METRIC_VALUE_TYPE_F:
+				if (metric.value.f == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				output_fields[metric_name] = metric.value.f;
+				break;
+			case METRIC_VALUE_TYPE_I:
+				if (metric.value.i == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				output_fields[metric_name] = metric.value.i;
+				break;
 			default:
 				break;
 			}