falco_formats only formats events now, no lua bindings

Modify falco_formats to only be responsible for resolving a rule's
output string or coming up with a map of field name->field values from
a given output string.

It relies on the changes in
https://github.com/falcosecurity/libs/pull/77 to use generic
formatters for a given source.

Remove lua bindings to create a formatter/free a formatter. Those were
unused as of the changes in
https://github.com/falcosecurity/falco/pull/1451, so finally remove
them now.

Signed-off-by: Mark Stemm <mark.stemm@gmail.com>

userspace/engine/formats.cpp

@@ -20,169 +20,49 @@ limitations under the License.
 #include "falco_engine.h"
 #include "banned.h" // This raises a compilation error when certain functions are used
 
-sinsp *falco_formats::s_inspector = NULL;
+falco_formats::falco_formats(falco_engine *engine,
-falco_engine *falco_formats::s_engine = NULL;
-bool falco_formats::s_json_output = false;
-bool falco_formats::s_json_include_output_property = true;
-bool falco_formats::s_json_include_tags_property = true;
-std::unique_ptr<sinsp_evt_formatter_cache> falco_formats::s_formatters = NULL;
-
-const static struct luaL_Reg ll_falco[] =
-	{
-		{"formatter", &falco_formats::lua_formatter},
-		{"free_formatter", &falco_formats::lua_free_formatter},
-		{NULL, NULL}};
-
-void falco_formats::init(sinsp *inspector,
-			 falco_engine *engine,
-			 lua_State *ls,
-			 bool json_output,
 			     bool json_include_output_property,
 			     bool json_include_tags_property)
+	: m_falco_engine(engine),
+	m_json_include_output_property(json_include_output_property),
+	m_json_include_tags_property(json_include_tags_property)
 {
-	s_inspector = inspector;
-	s_engine = engine;
-	s_json_output = json_output;
-	s_json_include_output_property = json_include_output_property;
-	s_json_include_tags_property = json_include_tags_property;
-
-	// todo(leogr): we should have used std::make_unique, but we cannot since it's not C++14
-	s_formatters = std::unique_ptr<sinsp_evt_formatter_cache>(new sinsp_evt_formatter_cache(s_inspector));
-
-	luaL_openlib(ls, "formats", ll_falco, 0);
 }
 
-int falco_formats::lua_formatter(lua_State *ls)
+falco_formats::~falco_formats()
 {
-	string source = luaL_checkstring(ls, -2);
-	string format = luaL_checkstring(ls, -1);
-
-	try
-	{
-		if(source == "syscall")
-		{
-			sinsp_evt_formatter *formatter;
-			formatter = new sinsp_evt_formatter(s_inspector, format);
-			lua_pushnil(ls);
-			lua_pushlightuserdata(ls, formatter);
-		}
-		else
-		{
-			json_event_formatter *formatter;
-			formatter = new json_event_formatter(s_engine->json_factory(), format);
-			lua_pushnil(ls);
-			lua_pushlightuserdata(ls, formatter);
-		}
-	}
-	catch(exception &e)
-	{
-		std::ostringstream os;
-
-		os << "Invalid output format '"
-		   << format
-		   << "': '"
-		   << e.what()
-		   << "'";
-
-		lua_pushstring(ls, os.str().c_str());
-		lua_pushnil(ls);
-	}
-
-	return 2;
 }
 
-int falco_formats::lua_free_formatter(lua_State *ls)
+string falco_formats::format_event(gen_event *evt, const std::string &rule, const std::string &source,
-{
-	if(!lua_islightuserdata(ls, -1) ||
-	   !lua_isstring(ls, -2))
-
-	{
-		luaL_error(ls, "Invalid argument passed to free_formatter");
-	}
-
-	string source = luaL_checkstring(ls, -2);
-
-	if(source == "syscall")
-	{
-		sinsp_evt_formatter *formatter = (sinsp_evt_formatter *)lua_topointer(ls, -1);
-		delete(formatter);
-	}
-	else
-	{
-		json_event_formatter *formatter = (json_event_formatter *)lua_topointer(ls, -1);
-		delete(formatter);
-	}
-
-	return 0;
-}
-
-string falco_formats::format_event(const gen_event *evt, const std::string &rule, const std::string &source,
 				   const std::string &level, const std::string &format, std::set<std::string> &tags)
 {
-
 	string line;
+
+	std::shared_ptr<gen_event_formatter> formatter;
+
+	formatter = m_falco_engine->create_formatter(source, format);
+
+	// Format the original output string, regardless of output format
+	formatter->tostring_withformat(evt, line, gen_event_formatter::OF_NORMAL);
+
+	if(formatter->get_output_format() == gen_event_formatter::OF_JSON)
+	{
 		string json_line;
-	string sformat = format;
 
-	if(strcmp(source.c_str(), "syscall") == 0)
+		// Format the event into a json object with all fields resolved
-	{
+		formatter->tostring(evt, json_line);
-		// This is "output"
-		s_formatters->tostring((sinsp_evt *)evt, sformat, &line);
-
-		if(s_json_output)
-		{
-			sinsp_evt::param_fmt cur_fmt = s_inspector->get_buffer_format();
-			switch(cur_fmt)
-			{
-			case sinsp_evt::PF_NORMAL:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSON);
-				break;
-			case sinsp_evt::PF_EOLS:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSONEOLS);
-				break;
-			case sinsp_evt::PF_HEX:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSONHEX);
-				break;
-			case sinsp_evt::PF_HEXASCII:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSONHEXASCII);
-				break;
-			case sinsp_evt::PF_BASE64:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSONBASE64);
-				break;
-			default:
-				// do nothing
-				break;
-			}
-			// This is output fields
-			s_formatters->tostring((sinsp_evt *)evt, sformat, &json_line);
 
 		// The formatted string might have a leading newline. If it does, remove it.
 		if(json_line[0] == '\n')
 		{
 			json_line.erase(0, 1);
 		}
-			s_inspector->set_buffer_format(cur_fmt);
-		}
-	}
-	else
-	{
-		json_event_formatter formatter(s_engine->json_factory(), sformat);
-
-		line = formatter.tostring((json_event *)evt);
-
-		if(s_json_output)
-		{
-			json_line = formatter.tojson((json_event *)evt);
-		}
-	}
 
 		// For JSON output, the formatter returned a json-as-text
 		// object containing all the fields in the original format
 		// message as well as the event time in ns. Use this to build
 		// a more detailed object containing the event time, rule,
 		// severity, full output, and fields.
-	if(s_json_output)
-	{
 		Json::Value event;
 		Json::Value rule_tags;
 		Json::FastWriter writer;
@@ -204,13 +84,13 @@ string falco_formats::format_event(const gen_event *evt, const std::string &rule
 		event["priority"] = level;
 		event["source"] = source;
 
-		if(s_json_include_output_property)
+		if(m_json_include_output_property)
 		{
 			// This is the filled-in output line.
 			event["output"] = line;
 		}
 
-		if(s_json_include_tags_property)
+		if(m_json_include_tags_property)
 		{
 			if (tags.size() == 0)
 			{
@@ -249,19 +129,19 @@ string falco_formats::format_event(const gen_event *evt, const std::string &rule
 	return line.c_str();
 }
 
-map<string, string> falco_formats::resolve_tokens(const gen_event *evt, const std::string &source, const std::string &format)
+map<string, string> falco_formats::get_field_values(gen_event *evt, const std::string &source,
+						    const std::string &format)
 {
-	string sformat = format;
+	std::shared_ptr<gen_event_formatter> formatter;
-	map<string, string> values;
+
-	if(source == "syscall")
+	formatter = m_falco_engine->create_formatter(source, format);
+
+	map<string, string> ret;
+
+	if (! formatter->get_field_values(evt, ret))
 	{
-		s_formatters->resolve_tokens((sinsp_evt *)evt, sformat, values);
+		throw falco_exception("Could not extract all field values from event");
 	}
-	// k8s_audit
+
-	else
+	return ret;
-	{
-		json_event_formatter json_formatter(s_engine->json_factory(), sformat);
-		values = json_formatter.tomap((json_event *)evt);
-	}
-	return values;
 }

userspace/engine/formats.h

@@ -16,7 +16,7 @@ limitations under the License.
 
 #pragma once
 
-#include "sinsp.h"
+#include <string>
 
 extern "C"
 {
@@ -25,37 +25,26 @@ extern "C"
 #include "lauxlib.h"
 }
 
-#include "json_evt.h"
+#include <gen_filter.h>
-#include "falco_engine.h"
 
-class sinsp_evt_formatter;
+#include "falco_engine.h"
 
 class falco_formats
 {
 public:
-	static void init(sinsp *inspector,
+	falco_formats(falco_engine *engine,
-			 falco_engine *engine,
-			 lua_State *ls,
-			 bool json_output,
 		      bool json_include_output_property,
 		      bool json_include_tags_property);
+	virtual ~falco_formats();
 
-	// formatter = falco.formatter(format_string)
+	std::string format_event(gen_event *evt, const std::string &rule, const std::string &source,
-	static int lua_formatter(lua_State *ls);
-
-	// falco.free_formatter(formatter)
-	static int lua_free_formatter(lua_State *ls);
-
-	static string format_event(const gen_event *evt, const std::string &rule, const std::string &source,
 				 const std::string &level, const std::string &format, std::set<std::string> &tags);
 
-	static map<string, string> resolve_tokens(const gen_event *evt, const std::string &source,
+	map<string, string> get_field_values(gen_event *evt, const std::string &source,
 					     const std::string &format);
 
-	static sinsp *s_inspector;
+protected:
-	static falco_engine *s_engine;
+	falco_engine *m_falco_engine;
-	static std::unique_ptr<sinsp_evt_formatter_cache> s_formatters;
+	bool m_json_include_output_property;
-	static bool s_json_output;
+	bool m_json_include_tags_property;
-	static bool s_json_include_output_property;
-	static bool s_json_include_tags_property;
 };

userspace/falco/falco_outputs.cpp

@@ -60,7 +60,8 @@ falco_outputs::~falco_outputs()
 	}
 }
 
-void falco_outputs::init(bool json_output,
+void falco_outputs::init(falco_engine *engine,
+			 bool json_output,
 			 bool json_include_output_property,
 			 bool json_include_tags_property,
 			 uint32_t timeout,
@@ -73,14 +74,9 @@ void falco_outputs::init(bool json_output,
 		throw falco_exception("falco_outputs already initialized");
 	}
 
-	m_json_output = json_output;
+	m_formats.reset(new falco_formats(engine, json_include_output_property, json_include_tags_property));
 
-	// Note that falco_formats is already initialized by the engine,
+	m_json_output = json_output;
-	// and the following json options are not used within the engine.
-	// So we can safely update them.
-	falco_formats::s_json_output = json_output;
-	falco_formats::s_json_include_output_property = json_include_output_property;
-	falco_formats::s_json_include_tags_property = json_include_tags_property;
 
 	m_timeout = std::chrono::milliseconds(timeout);
 
@@ -192,8 +188,8 @@ void falco_outputs::handle_event(gen_event *evt, string &rule, string &source,
 		sformat += " " + format;
 	}
 
-	cmsg.msg = falco_formats::format_event(evt, rule, source, falco_common::priority_names[priority], sformat, tags);
+	cmsg.msg = m_formats->format_event(evt, rule, source, falco_common::priority_names[priority], sformat, tags);
-	cmsg.fields = falco_formats::resolve_tokens(evt, source, sformat);
+	cmsg.fields = m_formats->get_field_values(evt, source, sformat);
 	cmsg.tags.insert(tags.begin(), tags.end());
 
 	cmsg.type = ctrl_msg_type::CTRL_MSG_OUTPUT;

userspace/falco/falco_outputs.h

@@ -25,6 +25,7 @@ limitations under the License.
 #include "token_bucket.h"
 #include "falco_engine.h"
 #include "outputs.h"
+#include "formats.h"
 #include "tbb/concurrent_queue.h"
 
 //
@@ -38,7 +39,8 @@ public:
 	falco_outputs();
 	virtual ~falco_outputs();
 
-	void init(bool json_output,
+	void init(falco_engine *engine,
+		  bool json_output,
 		  bool json_include_output_property,
 		  bool json_include_tags_property,
 		  uint32_t timeout,
@@ -63,6 +65,7 @@ public:
 	void reopen_outputs();
 
 private:
+	std::unique_ptr<falco_formats> m_formats;
 	bool m_initialized;
 
 	std::vector<falco::outputs::abstract_output *> m_outputs;