new(app): add append_output configuration option with fields and format

Signed-off-by: Luca Guerra <luca@guerra.sh>

userspace/engine/falco_engine.cpp

@@ -57,8 +57,7 @@ falco_engine::falco_engine(bool seed_rng)
 	  m_rule_compiler(std::make_shared<rule_loader::compiler>()),
 	  m_next_ruleset_id(0),
 	  m_min_priority(falco_common::PRIORITY_DEBUG),
-	  m_sampling_ratio(1), m_sampling_multiplier(0),
-	  m_replace_container_info(false)
+	  m_sampling_ratio(1), m_sampling_multiplier(0)
 {
 	if(seed_rng)
 	{
@@ -76,6 +75,7 @@ falco_engine::~falco_engine()
 	m_rule_collector->clear();
 	m_rule_stats_manager.clear();
 	m_sources.clear();
+	m_extra_output_format.clear();
 }
 
 sinsp_version falco_engine::engine_version()
@@ -194,8 +194,8 @@ void falco_engine::list_fields(const std::string &source, bool verbose, bool nam
 std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_content, const std::string &name)
 {
 	rule_loader::configuration cfg(rules_content, m_sources, name);
-	cfg.output_extra = m_extra;
-	cfg.replace_output_container_info = m_replace_container_info;
+	cfg.extra_output_format = m_extra_output_format;
+	cfg.extra_output_fields = m_extra_output_fields;
 
 	// read rules YAML file and collect its definitions
 	if(m_rule_reader->read(cfg, *m_rule_collector))
@@ -455,6 +455,7 @@ std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_ev
 		rule_result.priority_num = rule.priority;
 		rule_result.tags = rule.tags;
 		rule_result.exception_fields = rule.exception_fields;
+		rule_result.extra_output_fields = rule.extra_output_fields;
 		m_rule_stats_manager.on_event(rule);
 		res->push_back(rule_result);
 	}
@@ -646,9 +647,22 @@ void falco_engine::get_json_details(
 	out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators);
 	out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields);
 
+	// Get extra requested fields
+	std::vector<std::string> out_fields;
+
+	for(auto const& f : r.extra_output_fields)
+	{
+		// add all the field keys
+		out_fields.emplace_back(f.second.first);
+
+		if (!f.second.second) // formatted field
+		{
+			out["details"]["extra_output_formatted_fields"][f.first] = f.second.first;
+		}
+	}
+
 	// Get fields from output string
 	auto fmt = create_formatter(r.source, r.output);
-	std::vector<std::string> out_fields;
 	fmt->get_field_names(out_fields);
 	out["details"]["output_fields"] = sequence_to_json_array(out_fields);
 
@@ -1082,10 +1096,37 @@ void falco_engine::set_sampling_multiplier(double sampling_multiplier)
 	m_sampling_multiplier = sampling_multiplier;
 }
 
-void falco_engine::set_extra(const std::string &extra, bool replace_container_info)
+void falco_engine::add_extra_output_format(
+	const std::string &format,
+	const std::string &source,
+	const std::string &tag,
+	const std::string &rule,
+	bool replace_container_info
+)
 {
-	m_extra = extra;
-	m_replace_container_info = replace_container_info;
+	m_extra_output_format.push_back({format, source, tag, rule, replace_container_info});
+}
+
+void falco_engine::add_extra_output_formatted_field(
+	const std::string &key,
+	const std::string &format,
+	const std::string &source,
+	const std::string &tag,
+	const std::string &rule
+)
+{
+	m_extra_output_fields.push_back({key, format, source, tag, rule, false});
+}
+
+void falco_engine::add_extra_output_raw_field(
+	const std::string &key,
+	const std::string &source,
+	const std::string &tag,
+	const std::string &rule
+)
+{
+	std::string format = "%" + key;
+	m_extra_output_fields.push_back({key, format, source, tag, rule, true});
 }
 
 inline bool falco_engine::should_drop_evt() const

userspace/engine/falco_engine.h

@@ -176,15 +176,40 @@ public:
 	//
 	void set_sampling_multiplier(double sampling_multiplier);
 
-	//
-	// You can optionally add "extra" formatting fields to the end
+	// You can optionally add "extra" output to the end
 	// of all output expressions. You can also choose to replace
 	// %container.info with the extra information or add it to the
 	// end of the expression. This is used in open source falco to
 	// add k8s/container information to outputs when
 	// available.
 	//
-	void set_extra(const std::string &extra, bool replace_container_info);
+	void add_extra_output_format(
+		const std::string &format,
+		const std::string &source,
+		const std::string &tag,
+		const std::string &rule,
+		bool replace_container_info
+	);
+
+	// You can optionally add fields that will only show up in the object
+	// output (e.g. json, gRPC) alongside other output_fields
+	// and not in the text message output.
+	// You can add two types of fields: formatted which will act like
+	// an additional output format that appears in the output field
+	void add_extra_output_formatted_field(
+		const std::string &key,
+		const std::string &format,
+		const std::string &source,
+		const std::string &tag,
+		const std::string &rule
+	);
+
+	void add_extra_output_raw_field(
+		const std::string &key,
+		const std::string &source,
+		const std::string &tag,
+		const std::string &rule
+	);
 
 	// Represents the result of matching an event against a set of
 	// rules.
@@ -196,6 +221,7 @@ public:
 		std::string format;
 		std::set<std::string> exception_fields;
 		std::set<std::string> tags;
+		std::unordered_map<std::string, std::pair<std::string, bool>> extra_output_fields;
 	};
 
 	//
@@ -461,6 +487,6 @@ private:
 	static const std::string s_default_ruleset;
 	uint32_t m_default_ruleset_id;
 
-	std::string m_extra;
-	bool m_replace_container_info;
+	std::vector<rule_loader::extra_output_format_conf> m_extra_output_format;
+	std::vector<rule_loader::extra_output_field_conf> m_extra_output_fields;
 };

userspace/engine/falco_rule.h

@@ -79,6 +79,7 @@ struct falco_rule
 	std::string name;
 	std::string description;
 	std::string output;
+	std::unordered_map<std::string, std::pair<std::string, bool>> extra_output_fields;
 	std::set<std::string> tags;
 	std::set<std::string> exception_fields;
 	falco_common::priority_type priority;

userspace/engine/formats.cpp

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include <json/json.h>
+#include <nlohmann/json.hpp>
 
 #include "formats.h"
 #include "falco_engine.h"
@@ -35,7 +35,7 @@ falco_formats::~falco_formats()
 
 std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
 				   const std::string &level, const std::string &format, const std::set<std::string> &tags,
-				   const std::string &hostname) const
+				   const std::string &hostname, const std::unordered_map<std::string, std::pair<std::string, bool>> &extra_fields) const
 {
 	std::string line;
 
@@ -48,27 +48,17 @@ std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule,
 
 	if(formatter->get_output_format() == sinsp_evt_formatter::OF_JSON)
 	{
-		std::string json_line;
+		std::string json_fields;
 
 		// Format the event into a json object with all fields resolved
-		formatter->tostring(evt, json_line);
-
-		// The formatted string might have a leading newline. If it does, remove it.
-		if(json_line[0] == '\n')
-		{
-			json_line.erase(0, 1);
-		}
+		formatter->tostring(evt, json_fields);
 
 		// For JSON output, the formatter returned a json-as-text
 		// object containing all the fields in the original format
 		// message as well as the event time in ns. Use this to build
 		// a more detailed object containing the event time, rule,
 		// severity, full output, and fields.
-		Json::Value event;
-		Json::Value rule_tags;
-		Json::FastWriter writer;
-		std::string full_line;
-		unsigned int rule_tags_idx = 0;
+		nlohmann::json event;
 
 		// Convert the time-as-nanoseconds to a more json-friendly ISO8601.
 		time_t evttime = evt->get_ts() / 1000000000;
@@ -94,43 +84,54 @@ std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule,
 
 		if(m_json_include_tags_property)
 		{
-			if (tags.size() == 0)
-			{
-				// This sets an empty array
-				rule_tags = Json::arrayValue;
-			}
-			else
-			{
-				for (const auto &tag : tags)
-				{
-					rule_tags[rule_tags_idx++] = tag;
-				}
-			}
-			event["tags"] = rule_tags;
+			event["tags"] = tags;
 		}
 
-		full_line = writer.write(event);
+		event["output_fields"] = nlohmann::json::parse(json_fields);
 
-		// Json::FastWriter may add a trailing newline. If it
-		// does, remove it.
-		if(full_line[full_line.length() - 1] == '\n')
+		for (auto const& ef : extra_fields)
 		{
-			full_line.resize(full_line.length() - 1);
+			std::string fformat = ef.second.first;
+			if (fformat.size() == 0)
+			{
+				continue;
+			}
+
+			if (!(fformat[0] == '*'))
+			{
+				fformat = "*" + fformat;
+			}
+
+			if(ef.second.second) // raw field
+			{
+				std::string json_field_map;
+				formatter = m_falco_engine->create_formatter(source, fformat);
+				formatter->tostring_withformat(evt, json_field_map, sinsp_evt_formatter::OF_JSON);
+				auto json_obj = nlohmann::json::parse(json_field_map);
+				event["output_fields"][ef.first] = json_obj[ef.first];
+			} else
+			{
+				event["output_fields"][ef.first] = format_string(evt, fformat, source);
+			}
 		}
 
-		// Cheat-graft the output from the formatter into this
-		// string. Avoids an unnecessary json parse just to
-		// merge the formatted fields at the object level.
-		full_line.pop_back();
-		full_line.append(", \"output_fields\": ");
-		full_line.append(json_line);
-		full_line.append("}");
-		line = full_line;
+		line = event.dump();
 	}
 
 	return line;
 }
 
+std::string falco_formats::format_string(sinsp_evt *evt, const std::string &format, const std::string &source) const
+{
+	std::string line;
+	std::shared_ptr<sinsp_evt_formatter> formatter;
+
+	formatter = m_falco_engine->create_formatter(source, format);
+	formatter->tostring_withformat(evt, line, sinsp_evt_formatter::OF_NORMAL);
+
+	return line;
+}
+
 std::map<std::string, std::string> falco_formats::get_field_values(sinsp_evt *evt, const std::string &source,
 						    const std::string &format) const
 {

userspace/engine/formats.h

@@ -31,7 +31,9 @@ public:
 
 	std::string format_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
 				 const std::string &level, const std::string &format, const std::set<std::string> &tags,
-				 const std::string &hostname) const;
+				 const std::string &hostname, const std::unordered_map<std::string, std::pair<std::string, bool>> &extra_fields) const;
+
+	std::string format_string(sinsp_evt *evt, const std::string &format, const std::string &source) const;
 
 	std::map<std::string, std::string> get_field_values(sinsp_evt *evt, const std::string &source,
 					     const std::string &format) const ;

userspace/engine/rule_loader.h

@@ -20,6 +20,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 #include <optional>
+#include <unordered_map>
 #include <yaml-cpp/yaml.h>
 #include <nlohmann/json.hpp>
 #include "falco_source.h"
@@ -261,6 +262,25 @@ namespace rule_loader
 		nlohmann::json res_json;
 	};
 
+	struct extra_output_format_conf
+	{
+		std::string m_format;
+		std::string m_source;
+		std::string m_tag;
+		std::string m_rule;
+		bool m_replace_container_info;
+	};
+
+	struct extra_output_field_conf
+	{
+		std::string m_key;
+		std::string m_format;
+		std::string m_source;
+		std::string m_tag;
+		std::string m_rule;
+		bool m_raw;
+	};
+
 	/*!
 		\brief Contains the info required to load rule definitions
 	*/
@@ -278,8 +298,9 @@ namespace rule_loader
 		const std::string& content;
 		const indexed_vector<falco_source>& sources;
 		std::string name;
-		std::string output_extra;
-		bool replace_output_container_info = false;
+
+		std::vector<extra_output_format_conf> extra_output_format;
+		std::vector<extra_output_field_conf> extra_output_fields;
 
 		// outputs
 		std::unique_ptr<result> res;

userspace/engine/rule_loader_compiler.cpp

@@ -322,22 +322,6 @@ static std::shared_ptr<ast::expr> parse_condition(
 	}
 }
 
-static void apply_output_substitutions(
-	rule_loader::configuration& cfg,
-	std::string& out)
-{
-	if (out.find(s_container_info_fmt) != std::string::npos)
-	{
-		if (cfg.replace_output_container_info)
-		{
-			out = replace(out, s_container_info_fmt, cfg.output_extra);
-			return;
-		}
-		out = replace(out, s_container_info_fmt, s_default_extra_fmt);
-	}
-	out += cfg.output_extra.empty() ? "" : " " + cfg.output_extra;
-}
-
 void rule_loader::compiler::compile_list_infos(
 		configuration& cfg,
 		const collector& col,
@@ -510,13 +494,64 @@ void rule_loader::compiler::compile_rule_infos(
 		// build rule output message
 		rule.output = r.output;
 
-		// plugins sources do not have any container info and so we won't apply -pk, -pc, etc.
-		// on the other hand, when using plugins you might want to append custom output based on the plugin
-		// TODO: this is not flexible enough (esp. if you mix plugin with syscalls),
-		// it would be better to add configuration options to control the output.
-		if (!cfg.replace_output_container_info || r.source == falco_common::syscall_source)
+		for (auto& extra : cfg.extra_output_format)
 		{
-			apply_output_substitutions(cfg, rule.output);
+			if (extra.m_source != "" && r.source != extra.m_source)
+			{
+				continue;
+			}
+
+			if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0)
+			{
+				continue;
+			}
+
+			if (extra.m_rule != "" && r.name != extra.m_rule)
+			{
+				continue;
+			}
+
+			if (extra.m_replace_container_info)
+			{
+				if (rule.output.find(s_container_info_fmt) != std::string::npos)
+				{
+					rule.output = replace(rule.output, s_container_info_fmt, extra.m_format);
+				}
+				else
+				{
+					rule.output = rule.output + " " + extra.m_format;
+				}
+			} else
+			{
+				rule.output = rule.output + " " + extra.m_format;
+			}
+		}
+
+		if (rule.output.find(s_container_info_fmt) != std::string::npos)
+		{
+			rule.output = replace(rule.output, s_container_info_fmt, s_default_extra_fmt);
+		}
+
+		// build extra output fields if required
+
+		for (auto const& extra : cfg.extra_output_fields)
+		{
+			if (extra.m_source != "" && r.source != extra.m_source)
+			{
+				continue;
+			}
+
+			if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0)
+			{
+				continue;
+			}
+
+			if (extra.m_rule != "" && r.name != extra.m_rule)
+			{
+				continue;
+			}
+
+			rule.extra_output_fields[extra.m_key] = {extra.m_format, extra.m_raw};
 		}
 
 		// validate the rule's output
@@ -538,6 +573,18 @@ void rule_loader::compiler::compile_rule_infos(
 				r.output_ctx);
 		}
 
+		// validate the rule's extra fields if any
+		for (auto const& ef : rule.extra_output_fields)
+		{
+			if(!is_format_valid(*cfg.sources.at(r.source), ef.second.first, err))
+			{
+				throw rule_load_exception(
+					falco::load_result::load_result::LOAD_ERR_COMPILE_OUTPUT,
+					err,
+					r.output_ctx);
+			}
+		}
+
 		if (!compile_condition(cfg,
 				  macro_resolver,
 				  lists,