Validate rule outputs when loading rules.

Validate rule outputs when loading rules by attempting to create a
formatter based on the rule's output field. If there's an error, it will
propagate up through load_rules and cause falco to exit rather than
discover the problem only when trying to format the event and the rule's
output field.

This required moving formats.{cpp,h} into the falco engine directory
from the falco general directory. Note that these functions are loaded
twice in the two lua states used by falco (engine and outputs).

There's also a couple of minor cleanups:

 - falco_formats had a private instance variable that was unused, remove
   it.
 - rename the package for the falco_formats functions to formats instead
   of falco so it's more standalone.
 - don't throw a c++ exception in falco_formats::formatter. Instead
   generate a lua error, which is handled more cleanly.
 - free_formatter doesn't return any values, so set the return value of
   the function to 0.

userspace/engine/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp")
 include_directories("${PROJECT_BINARY_DIR}/userspace/engine")
 include_directories("${LUAJIT_INCLUDE}")
 
-add_library(falco_engine STATIC rules.cpp falco_common.cpp falco_engine.cpp)
+add_library(falco_engine STATIC rules.cpp falco_common.cpp falco_engine.cpp formats.cpp)
 
 target_include_directories(falco_engine PUBLIC
   "${LUAJIT_INCLUDE}")

userspace/engine/falco_engine.cpp

@@ -24,6 +24,8 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 #include "falco_engine.h"
 #include "config_falco_engine.h"
 
+#include "formats.h"
+
 extern "C" {
 #include "lpeg.h"
 #include "lyaml.h"
@@ -73,6 +75,15 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al
 	{
 		m_rules = new falco_rules(m_inspector, this, m_ls);
 	}
+
+	// Note that falco_formats is added to both the lua state used
+	// by the falco engine as well as the separate lua state used
+	// by falco outputs.  Within the engine, only
+	// formats.formatter is used, so we can unconditionally set
+	// json_output to false.
+	bool json_output = false;
+	falco_formats::init(m_inspector, m_ls, json_output);
+
 	m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info);
 }
 

userspace/engine/formats.cpp

@@ -0,0 +1,130 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <json/json.h>
+
+#include "formats.h"
+#include "logger.h"
+#include "falco_engine.h"
+
+
+sinsp* falco_formats::s_inspector = NULL;
+bool s_json_output = false;
+
+const static struct luaL_reg ll_falco [] =
+{
+	{"formatter", &falco_formats::formatter},
+	{"free_formatter", &falco_formats::free_formatter},
+	{"format_event", &falco_formats::format_event},
+	{NULL,NULL}
+};
+
+void falco_formats::init(sinsp* inspector, lua_State *ls, bool json_output)
+{
+	s_inspector = inspector;
+	s_json_output = json_output;
+
+	luaL_openlib(ls, "formats", ll_falco, 0);
+}
+
+int falco_formats::formatter(lua_State *ls)
+{
+	string format = luaL_checkstring(ls, 1);
+	sinsp_evt_formatter* formatter;
+	try
+	{
+		formatter = new sinsp_evt_formatter(s_inspector, format);
+	}
+	catch(sinsp_exception& e)
+	{
+		luaL_error(ls, "Invalid output format '%s': '%s'", format.c_str(), e.what());
+	}
+
+	lua_pushlightuserdata(ls, formatter);
+
+	return 1;
+}
+
+int falco_formats::free_formatter(lua_State *ls)
+{
+	if (!lua_islightuserdata(ls, -1))
+	{
+		luaL_error(ls, "Invalid argument passed to free_formatter");
+	}
+
+	sinsp_evt_formatter *formatter = (sinsp_evt_formatter *) lua_topointer(ls, 1);
+
+	delete(formatter);
+
+	return 0;
+}
+
+int falco_formats::format_event (lua_State *ls)
+{
+	string line;
+
+	if (!lua_islightuserdata(ls, -1) ||
+	    !lua_isstring(ls, -2) ||
+	    !lua_isstring(ls, -3) ||
+	    !lua_islightuserdata(ls, -4)) {
+		throw falco_exception("Invalid arguments passed to format_event()\n");
+	}
+	sinsp_evt* evt = (sinsp_evt*)lua_topointer(ls, 1);
+	const char *rule = (char *) lua_tostring(ls, 2);
+	const char *level = (char *) lua_tostring(ls, 3);
+	sinsp_evt_formatter* formatter = (sinsp_evt_formatter*)lua_topointer(ls, 4);
+
+	formatter->tostring(evt, &line);
+
+	// For JSON output, the formatter returned just the output
+	// string containing the format text and values. Use this to
+	// build a more detailed object containing the event time,
+	// rule, severity, full output, and fields.
+	if (s_json_output) {
+		Json::Value event;
+		Json::FastWriter writer;
+
+		// Convert the time-as-nanoseconds to a more json-friendly ISO8601.
+		time_t evttime = evt->get_ts()/1000000000;
+		char time_sec[20]; // sizeof "YYYY-MM-DDTHH:MM:SS"
+		char time_ns[12]; // sizeof ".sssssssssZ"
+		string iso8601evttime;
+
+		strftime(time_sec, sizeof(time_sec), "%FT%T", gmtime(&evttime));
+		snprintf(time_ns, sizeof(time_ns), ".%09luZ", evt->get_ts() % 1000000000);
+		iso8601evttime = time_sec;
+		iso8601evttime += time_ns;
+		event["time"] = iso8601evttime;
+		event["rule"] = rule;
+		event["priority"] = level;
+		event["output"] = line;
+
+		line = writer.write(event);
+
+		// Json::FastWriter may add a trailing newline. If it
+		// does, remove it.
+		if (line[line.length()-1] == '\n')
+		{
+			line.resize(line.length()-1);
+		}
+	}
+
+	lua_pushstring(ls, line.c_str());
+	return 1;
+}
+

userspace/engine/formats.h

@@ -0,0 +1,46 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "sinsp.h"
+
+extern "C" {
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+}
+
+class sinsp_evt_formatter;
+
+class falco_formats
+{
+ public:
+	static void init(sinsp* inspector, lua_State *ls, bool json_output);
+
+	// formatter = falco.formatter(format_string)
+	static int formatter(lua_State *ls);
+
+	// falco.free_formatter(formatter)
+	static int free_formatter(lua_State *ls);
+
+	// formatted_string = falco.format_event(evt, formatter)
+	static int format_event(lua_State *ls);
+
+	static sinsp* s_inspector;
+};

userspace/engine/lua/rule_loader.lua

@@ -281,6 +281,12 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
 		   v['output'] = v['output'].." "..extra
 		end
 	    end
+
+	    -- Ensure that the output field is properly formatted by
+	    -- creating a formatter from it. Any error will be thrown
+	    -- up to the top level.
+	    formatter = formats.formatter(v['output'])
+	    formats.free_formatter(formatter)
 	 else
 	    error ("Unexpected type in load_rule: "..filter_ast.type)
 	 end