Merge pull request #103 from draios/falco-engine

Falco engine

.gitignore

@@ -6,6 +6,8 @@ test/traces-negative
 test/traces-positive
 test/traces-info
 test/job-results
+test/.phoronix-test-suite
+test/results*.json.*
 
 userspace/falco/lua/re.lua
 userspace/falco/lua/lpeg.so

CMakeLists.txt

@@ -6,8 +6,8 @@ if(NOT DEFINED FALCO_VERSION)
 	set(FALCO_VERSION "0.1.1dev")
 endif()
 
-if(NOT DEFINED DIR_ETC)
-	set(DIR_ETC "/etc")
+if(NOT DEFINED FALCO_ETC_DIR)
+	set(FALCO_ETC_DIR "/etc")
 endif()
 
 if(NOT CMAKE_BUILD_TYPE)
@@ -39,6 +39,7 @@ set(PACKAGE_NAME "falco")
 set(PROBE_VERSION "${FALCO_VERSION}")
 set(PROBE_NAME "sysdig-probe")
 set(PROBE_DEVICE_NAME "sysdig")
+set(CMAKE_INSTALL_PREFIX /usr)
 
 set(CMD_MAKE make)
 
@@ -160,11 +161,12 @@ ExternalProject_Add(luajit
 		INSTALL_COMMAND "")
 
 set (LPEG_SRC "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg")
+set (LPEG_LIB "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg/build/lpeg.a")
 ExternalProject_Add(lpeg
                     DEPENDS luajit
 		    URL "http://s3.amazonaws.com/download.draios.com/dependencies/lpeg-1.0.0.tar.gz"
                     URL_MD5 "0aec64ccd13996202ad0c099e2877ece"
-                    BUILD_COMMAND LUA_INCLUDE=${LUAJIT_INCLUDE} "${PROJECT_SOURCE_DIR}/scripts/build-lpeg.sh"
+                    BUILD_COMMAND LUA_INCLUDE=${LUAJIT_INCLUDE} "${PROJECT_SOURCE_DIR}/scripts/build-lpeg.sh" "${LPEG_SRC}/build"
                     BUILD_IN_SOURCE 1
 		    CONFIGURE_COMMAND ""
                     INSTALL_COMMAND "")
@@ -188,17 +190,19 @@ ExternalProject_Add(lyaml
                     BUILD_COMMAND ${CMD_MAKE}
                     BUILD_IN_SOURCE 1
 		    CONFIGURE_COMMAND ./configure --enable-static LIBS=-L../../../libyaml-prefix/src/libyaml/src/.libs CFLAGS=-I../../../libyaml-prefix/src/libyaml/include CPPFLAGS=-I../../../libyaml-prefix/src/libyaml/include LUA_INCLUDE=-I../../../luajit-prefix/src/luajit/src LUA=../../../luajit-prefix/src/luajit/src/luajit
-                    INSTALL_COMMAND sh -c "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/falco/lua")
+                    INSTALL_COMMAND sh -c "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/engine/lua")
 
 install(FILES falco.yaml
-	DESTINATION "${DIR_ETC}")
+	DESTINATION "${FALCO_ETC_DIR}")
 
 add_subdirectory("${SYSDIG_DIR}/driver" "${PROJECT_BINARY_DIR}/driver")
 add_subdirectory("${SYSDIG_DIR}/userspace/libscap" "${PROJECT_BINARY_DIR}/userspace/libscap")
 add_subdirectory("${SYSDIG_DIR}/userspace/libsinsp" "${PROJECT_BINARY_DIR}/userspace/libsinsp")
 
-add_subdirectory(rules)
 add_subdirectory(scripts)
+set(FALCO_SINSP_LIBRARY sinsp)
+set(FALCO_SHARE_DIR share/falco)
+add_subdirectory(userspace/engine)
 add_subdirectory(userspace/falco)
 
 

rules/CMakeLists.txt

@@ -1,3 +1,13 @@
-install(FILES falco_rules.yaml
-	DESTINATION "${DIR_ETC}")
+if(NOT DEFINED FALCO_ETC_DIR)
+  set(FALCO_ETC_DIR "/etc")
+endif()
+
+if(DEFINED FALCO_COMPONENT)
+install(FILES falco_rules.yaml
+        COMPONENT "${FALCO_COMPONENT}"
+	DESTINATION "${FALCO_ETC_DIR}")
+else()
+install(FILES falco_rules.yaml
+	DESTINATION "${FALCO_ETC_DIR}")
+endif()
 

scripts/build-lpeg.sh

@@ -1,17 +1,29 @@
-#!/bin/sh
+#!/bin/bash
 
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcap.c -o lpcap.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcode.c -o lpcode.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpprint.c -o lpprint.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lptree.c -o lptree.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpvm.c -o lpvm.o
+set -ex
+
+PREFIX=$1
+
+if [ -z $PREFIX ]; then
+    PREFIX=.
+fi
+
+mkdir -p $PREFIX
+
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcap.c -o $PREFIX/lpcap.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcode.c -o $PREFIX/lpcode.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpprint.c -o $PREFIX/lpprint.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lptree.c -o $PREFIX/lptree.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpvm.c -o $PREFIX/lpvm.o
 
 
 # For building lpeg.so, which we don't need now that we're statically linking lpeg.a into falco
 #gcc -shared -o lpeg.so -L/usr/local/lib lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
 #gcc -shared -o lpeg.so -L/usr/local/lib lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
 
+pushd $PREFIX
 /usr/bin/ar cr lpeg.a lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
 /usr/bin/ranlib lpeg.a
+popd
 
 chmod ug+w re.lua

test/falco_test.py

@@ -26,8 +26,28 @@ class FalcoTest(Test):
         self.json_output = self.params.get('json_output', '*', default=False)
         self.rules_file = self.params.get('rules_file', '*', default=os.path.join(self.basedir, '../rules/falco_rules.yaml'))
 
-        if not os.path.isabs(self.rules_file):
-            self.rules_file = os.path.join(self.basedir, self.rules_file)
+        if not isinstance(self.rules_file, list):
+            self.rules_file = [self.rules_file]
+
+        self.rules_args = ""
+
+        for file in self.rules_file:
+            if not os.path.isabs(file):
+                file = os.path.join(self.basedir, file)
+            self.rules_args = self.rules_args + "-r " + file + " "
+
+        self.disabled_rules = self.params.get('disabled_rules', '*', default='')
+
+        if self.disabled_rules == '':
+            self.disabled_rules = []
+
+        if not isinstance(self.disabled_rules, list):
+            self.disabled_rules = [self.disabled_rules]
+
+        self.disabled_args = ""
+
+        for rule in self.disabled_rules:
+            self.disabled_args = self.disabled_args + "-D " + rule + " "
 
         self.rules_warning = self.params.get('rules_warning', '*', default=False)
         if self.rules_warning == False:
@@ -49,6 +69,9 @@ class FalcoTest(Test):
         if self.should_detect:
             self.detect_level = self.params.get('detect_level', '*')
 
+            if not isinstance(self.detect_level, list):
+                self.detect_level = [self.detect_level]
+
         # Doing this in 2 steps instead of simply using
         # module_is_loaded to avoid logging lsmod output to the log.
         lsmod_output = process.system_output("lsmod", verbose=False)
@@ -105,16 +128,17 @@ class FalcoTest(Test):
             if events_detected == 0:
                 self.fail("Detected {} events when should have detected > 0".format(events_detected))
 
-            level_line = '{}: (\d+)'.format(self.detect_level)
+            for level in self.detect_level:
+                level_line = '(?i){}: (\d+)'.format(level)
                 match = re.search(level_line, res.stdout)
 
                 if match is None:
-                self.fail("Could not find a line '{}: <count>' in falco output".format(self.detect_level))
+                    self.fail("Could not find a line '{}: <count>' in falco output".format(level))
 
                     events_detected = int(match.group(1))
 
                     if not events_detected > 0:
-                self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, self.detect_level))
+                        self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, level))
 
     def check_json_output(self, res):
         if self.json_output:
@@ -131,8 +155,8 @@ class FalcoTest(Test):
         self.log.info("Trace file %s", self.trace_file)
 
         # Run the provided trace file though falco
-        cmd = '{}/userspace/falco/falco -r {} -c {}/../falco.yaml -e {} -o json_output={} -v'.format(
-            self.falcodir, self.rules_file, self.falcodir, self.trace_file, self.json_output)
+        cmd = '{}/userspace/falco/falco {} {} -c {}/../falco.yaml -e {} -o json_output={} -v'.format(
+            self.falcodir, self.rules_args, self.disabled_args, self.falcodir, self.trace_file, self.json_output)
 
         self.falco_proc = process.SubProcess(cmd)
 

test/falco_tests.yaml.in

@@ -1,13 +1,13 @@
 trace_files: !mux
   builtin_rules_no_warnings:
     detect: False
-    trace_file: empty.scap
+    trace_file: trace_files/empty.scap
     rules_warning: False
 
   test_warnings:
     detect: False
-    trace_file: empty.scap
-    rules_file: falco_rules_warnings.yaml
+    trace_file: trace_files/empty.scap
+    rules_file: rules/falco_rules_warnings.yaml
     rules_warning:
       - no_evttype
       - evttype_not_equals
@@ -60,3 +60,48 @@ trace_files: !mux
       - repeated_evttypes_with_in: [open]
       - repeated_evttypes_with_separate_in: [open]
       - repeated_evttypes_with_mix: [open]
+
+  multiple_rules_first_empty:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules_last_empty:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/empty_rules.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules:
+    detect: True
+    detect_level:
+      - WARNING
+      - INFO
+      - ERROR
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/double_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  disabled_rules:
+    detect: False
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    disabled_rules:
+      - open_from_cat
+    trace_file: trace_files/cat_write.scap
+
+  disabled_rules_using_regex:
+    detect: False
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    disabled_rules:
+      - "open.*"
+    trace_file: trace_files/cat_write.scap

test/rules/double_rule.yaml

@@ -0,0 +1,13 @@
+# This ruleset depends on the is_cat macro defined in single_rule.yaml
+
+- rule: exec_from_cat
+  desc: A process named cat does execve
+  condition: evt.type=execve and is_cat
+  output: "An exec was seen (command=%proc.cmdline)"
+  priority: ERROR
+
+- rule: access_from_cat
+  desc: A process named cat does an access
+  condition: evt.type=access and is_cat
+  output: "An access was seen (command=%proc.cmdline)"
+  priority: INFO

test/rules/single_rule.yaml

@@ -0,0 +1,8 @@
+- macro: is_cat
+  condition: proc.name=cat
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/run_regression_tests.sh

@@ -38,12 +38,12 @@ EOF
 function prepare_multiplex_file() {
     cp $SCRIPTDIR/falco_tests.yaml.in $MULT_FILE
 
-    prepare_multiplex_fileset traces-positive True Warning False
-    prepare_multiplex_fileset traces-negative False Warning True
-    prepare_multiplex_fileset traces-info True Informational False
+    prepare_multiplex_fileset traces-positive True WARNING False
+    prepare_multiplex_fileset traces-negative False WARNING True
+    prepare_multiplex_fileset traces-info True INFO False
 
-    prepare_multiplex_fileset traces-positive True Warning True
-    prepare_multiplex_fileset traces-info True Informational True
+    prepare_multiplex_fileset traces-positive True WARNING True
+    prepare_multiplex_fileset traces-info True INFO True
 
     echo "Contents of $MULT_FILE:"
     cat $MULT_FILE

userspace/engine/CMakeLists.txt

@@ -0,0 +1,31 @@
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp/third-party/jsoncpp")
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap")
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp")
+include_directories("${PROJECT_BINARY_DIR}/userspace/engine")
+include_directories("${LUAJIT_INCLUDE}")
+
+add_library(falco_engine STATIC rules.cpp falco_common.cpp falco_engine.cpp)
+
+target_include_directories(falco_engine PUBLIC
+  "${LUAJIT_INCLUDE}")
+
+target_link_libraries(falco_engine
+  "${FALCO_SINSP_LIBRARY}"
+  "${LPEG_LIB}"
+  "${LYAML_LIB}"
+  "${LIBYAML_LIB}")
+
+configure_file(config_falco_engine.h.in config_falco_engine.h)
+
+if(DEFINED FALCO_COMPONENT)
+install(DIRECTORY lua
+	DESTINATION "${FALCO_SHARE_DIR}"
+	COMPONENT "${FALCO_COMPONENT}"
+	FILES_MATCHING PATTERN *.lua)
+else()
+install(DIRECTORY lua
+	DESTINATION "${FALCO_SHARE_DIR}"
+	FILES_MATCHING PATTERN *.lua)
+endif()
+
+add_subdirectory("${PROJECT_SOURCE_DIR}/../falco/rules" "${PROJECT_BINARY_DIR}/rules")

userspace/engine/config_falco_engine.h.in

@@ -0,0 +1,4 @@
+#pragma once
+
+#define FALCO_ENGINE_LUA_DIR "${CMAKE_INSTALL_PREFIX}/${FALCO_SHARE_DIR}/lua/"
+#define FALCO_ENGINE_SOURCE_LUA_DIR "${PROJECT_SOURCE_DIR}/../falco/userspace/engine/lua/"

userspace/engine/falco_common.cpp

@@ -0,0 +1,90 @@
+#include <fstream>
+
+#include "config_falco_engine.h"
+#include "falco_common.h"
+
+falco_common::falco_common()
+{
+	m_ls = lua_open();
+	luaL_openlibs(m_ls);
+}
+
+falco_common::~falco_common()
+{
+	if(m_ls)
+	{
+		lua_close(m_ls);
+	}
+}
+
+void falco_common::set_inspector(sinsp *inspector)
+{
+	m_inspector = inspector;
+}
+
+void falco_common::init(const char *lua_main_filename, const char *source_dir)
+{
+	ifstream is;
+	string lua_dir = FALCO_ENGINE_LUA_DIR;
+	string lua_main_path = lua_dir + lua_main_filename;
+
+	is.open(lua_main_path);
+	if (!is.is_open())
+	{
+		lua_dir = source_dir;
+		lua_main_path = lua_dir + lua_main_filename;
+
+		is.open(lua_main_path);
+		if (!is.is_open())
+		{
+			throw falco_exception("Could not find Falco Lua entrypoint (tried " +
+					      string(FALCO_ENGINE_LUA_DIR) + lua_main_filename + ", " +
+					      string(source_dir) + lua_main_filename + ")");
+		}
+	}
+
+	// Initialize Lua interpreter
+	add_lua_path(lua_dir);
+
+	// Load the main program, which defines all the available functions.
+	string scriptstr((istreambuf_iterator<char>(is)),
+			 istreambuf_iterator<char>());
+
+	if(luaL_loadstring(m_ls, scriptstr.c_str()) || lua_pcall(m_ls, 0, 0, 0))
+	{
+		throw falco_exception("Failed to load script " +
+			lua_main_path + ": " + lua_tostring(m_ls, -1));
+	}
+}
+
+void falco_common::add_lua_path(string &path)
+{
+	string cpath = string(path);
+	path += "?.lua";
+	cpath += "?.so";
+
+	lua_getglobal(m_ls, "package");
+
+	lua_getfield(m_ls, -1, "path");
+	string cur_path = lua_tostring(m_ls, -1 );
+	cur_path += ';';
+	lua_pop(m_ls, 1);
+
+	cur_path.append(path.c_str());
+
+	lua_pushstring(m_ls, cur_path.c_str());
+	lua_setfield(m_ls, -2, "path");
+
+	lua_getfield(m_ls, -1, "cpath");
+	string cur_cpath = lua_tostring(m_ls, -1 );
+	cur_cpath += ';';
+	lua_pop(m_ls, 1);
+
+	cur_cpath.append(cpath.c_str());
+
+	lua_pushstring(m_ls, cur_cpath.c_str());
+	lua_setfield(m_ls, -2, "cpath");
+
+	lua_pop(m_ls, 1);
+}
+

userspace/engine/falco_common.h

@@ -0,0 +1,69 @@
+#pragma once
+
+#include <string>
+#include <exception>
+
+extern "C" {
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+}
+
+#include <sinsp.h>
+
+//
+// Most falco_* classes can throw exceptions. Unless directly related
+// to low-level failures like inability to open file, etc, they will
+// be of this type.
+//
+
+struct falco_exception : std::exception
+{
+	falco_exception()
+	{
+	}
+
+	virtual ~falco_exception() throw()
+	{
+	}
+
+	falco_exception(std::string error_str)
+	{
+		m_error_str = error_str;
+	}
+
+	char const* what() const throw()
+	{
+		return m_error_str.c_str();
+	}
+
+	std::string m_error_str;
+};
+
+//
+// This is the base class of falco_engine/falco_output. It is
+// responsible for managing a lua state and associated inspector and
+// loading a single "main" lua file into that state.
+//
+
+class falco_common
+{
+public:
+	falco_common();
+	virtual ~falco_common();
+
+	void init(const char *lua_main_filename, const char *source_dir);
+
+	void set_inspector(sinsp *inspector);
+
+protected:
+	lua_State *m_ls;
+
+	sinsp *m_inspector;
+
+private:
+	void add_lua_path(std::string &path);
+};
+
+
+

userspace/engine/falco_engine.cpp

@@ -0,0 +1,182 @@
+#include <cstdlib>
+#include <unistd.h>
+#include <string>
+#include <fstream>
+
+#include "falco_engine.h"
+#include "config_falco_engine.h"
+
+extern "C" {
+#include "lpeg.h"
+#include "lyaml.h"
+}
+
+#include "utils.h"
+
+
+string lua_on_event = "on_event";
+string lua_print_stats = "print_stats";
+
+using namespace std;
+
+falco_engine::falco_engine(bool seed_rng)
+	: m_rules(NULL), m_sampling_ratio(1), m_sampling_multiplier(0)
+{
+	luaopen_lpeg(m_ls);
+	luaopen_yaml(m_ls);
+
+	falco_common::init(m_lua_main_filename.c_str(), FALCO_ENGINE_SOURCE_LUA_DIR);
+	falco_rules::init(m_ls);
+
+	if(seed_rng)
+	{
+		srandom((unsigned) getpid());
+	}
+}
+
+falco_engine::~falco_engine()
+{
+	if (m_rules)
+	{
+		delete m_rules;
+	}
+}
+
+void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events)
+{
+	// The engine must have been given an inspector by now.
+	if(! m_inspector)
+	{
+		throw falco_exception("No inspector provided");
+	}
+
+	if(!m_rules)
+	{
+		m_rules = new falco_rules(m_inspector, this, m_ls);
+	}
+	m_rules->load_rules(rules_content, verbose, all_events);
+}
+
+void falco_engine::load_rules_file(const string &rules_filename, bool verbose, bool all_events)
+{
+	ifstream is;
+
+	is.open(rules_filename);
+	if (!is.is_open())
+	{
+		throw falco_exception("Could not open rules filename " +
+				      rules_filename + " " +
+				      "for reading");
+	}
+
+	string rules_content((istreambuf_iterator<char>(is)),
+			     istreambuf_iterator<char>());
+
+	load_rules(rules_content, verbose, all_events);
+}
+
+void falco_engine::enable_rule(string &pattern, bool enabled)
+{
+	m_evttype_filter.enable(pattern, enabled);
+}
+
+falco_engine::rule_result *falco_engine::process_event(sinsp_evt *ev)
+{
+
+	if(should_drop_evt())
+	{
+		return NULL;
+	}
+
+	if(!m_evttype_filter.run(ev))
+	{
+		return NULL;
+	}
+
+	struct rule_result *res = new rule_result();
+
+	lua_getglobal(m_ls, lua_on_event.c_str());
+
+	if(lua_isfunction(m_ls, -1))
+	{
+		lua_pushlightuserdata(m_ls, ev);
+		lua_pushnumber(m_ls, ev->get_check_id());
+
+		if(lua_pcall(m_ls, 2, 3, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Error invoking function output: " + string(lerr);
+			throw falco_exception(err);
+		}
+		res->evt = ev;
+		const char *p =  lua_tostring(m_ls, -3);
+		res->rule = p;
+		res->priority = lua_tostring(m_ls, -2);
+		res->format = lua_tostring(m_ls, -1);
+	}
+	else
+	{
+		throw falco_exception("No function " + lua_on_event + " found in lua compiler module");
+	}
+
+	return res;
+}
+
+void falco_engine::describe_rule(string *rule)
+{
+	return m_rules->describe_rule(rule);
+}
+
+// Print statistics on the the rules that triggered
+void falco_engine::print_stats()
+{
+	lua_getglobal(m_ls, lua_print_stats.c_str());
+
+	if(lua_isfunction(m_ls, -1))
+	{
+		if(lua_pcall(m_ls, 0, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Error invoking function print_stats: " + string(lerr);
+			throw falco_exception(err);
+		}
+	}
+	else
+	{
+		throw falco_exception("No function " + lua_print_stats + " found in lua rule loader module");
+	}
+
+}
+
+void falco_engine::add_evttype_filter(string &rule,
+				      list<uint32_t> &evttypes,
+				      sinsp_filter* filter)
+{
+	m_evttype_filter.add(rule, evttypes, filter);
+}
+
+void falco_engine::set_sampling_ratio(uint32_t sampling_ratio)
+{
+	m_sampling_ratio = sampling_ratio;
+}
+
+void falco_engine::set_sampling_multiplier(double sampling_multiplier)
+{
+	m_sampling_multiplier = sampling_multiplier;
+}
+
+inline bool falco_engine::should_drop_evt()
+{
+	if(m_sampling_multiplier == 0)
+	{
+		return false;
+	}
+
+	if(m_sampling_ratio == 1)
+	{
+		return false;
+	}
+
+	double coin = (random() * (1.0/RAND_MAX));
+	return (coin >= (1.0/(m_sampling_multiplier * m_sampling_ratio)));
+}

userspace/engine/falco_engine.h

@@ -0,0 +1,118 @@
+#pragma once
+
+#include <string>
+
+#include "sinsp.h"
+#include "filter.h"
+
+#include "rules.h"
+
+#include "falco_common.h"
+
+//
+// This class acts as the primary interface between a program and the
+// falco rules engine. Falco outputs (writing to files/syslog/etc) are
+// handled in a separate class falco_outputs.
+//
+
+class falco_engine : public falco_common
+{
+public:
+	falco_engine(bool seed_rng=true);
+	virtual ~falco_engine();
+
+	//
+	// Load rules either directly or from a filename.
+	//
+	void load_rules_file(const std::string &rules_filename, bool verbose, bool all_events);
+	void load_rules(const std::string &rules_content, bool verbose, bool all_events);
+
+	//
+	// Enable/Disable any rules matching the provided pattern (regex).
+	//
+	void enable_rule(std::string &pattern, bool enabled);
+
+	struct rule_result {
+		sinsp_evt *evt;
+		std::string rule;
+		std::string priority;
+		std::string format;
+	};
+
+	//
+	// Given an event, check it against the set of rules in the
+	// engine and if a matching rule is found, return details on
+	// the rule that matched. If no rule matched, returns NULL.
+	//
+	// the reutrned rule_result is allocated and must be delete()d.
+	rule_result *process_event(sinsp_evt *ev);
+
+	//
+	// Print details on the given rule. If rule is NULL, print
+	// details on all rules.
+	//
+	void describe_rule(std::string *rule);
+
+	//
+	// Print statistics on how many events matched each rule.
+	//
+	void print_stats();
+
+	//
+	// Add a filter, which is related to the specified list of
+	// event types, to the engine.
+	//
+	void add_evttype_filter(std::string &rule,
+				list<uint32_t> &evttypes,
+				sinsp_filter* filter);
+
+	//
+	// Set the sampling ratio, which can affect which events are
+	// matched against the set of rules.
+	//
+	void set_sampling_ratio(uint32_t sampling_ratio);
+
+	//
+	// Set the sampling ratio multiplier, which can affect which
+	// events are matched against the set of rules.
+	//
+	void set_sampling_multiplier(double sampling_multiplier);
+
+private:
+
+	//
+	// Determine whether the given event should be matched at all
+	// against the set of rules, given the current sampling
+	// ratio/multiplier.
+	//
+	inline bool should_drop_evt();
+
+	falco_rules *m_rules;
+        sinsp_evttype_filter m_evttype_filter;
+
+	//
+	// Here's how the sampling ratio and multiplier influence
+	// whether or not an event is dropped in
+	// should_drop_evt(). The intent is that m_sampling_ratio is
+	// generally changing external to the engine e.g. in the main
+	// inspector class based on how busy the inspector is. A
+	// sampling ratio implies no dropping. Values > 1 imply
+	// increasing levels of dropping. External to the engine, the
+	// sampling ratio results in events being dropped at the
+	// kernel/inspector interface.
+	//
+	// The sampling multiplier is an amplification to the sampling
+	// factor in m_sampling_ratio. If 0, no additional events are
+	// dropped other than those that might be dropped by the
+	// kernel/inspector interface. If 1, events that make it past
+	// the kernel module are subject to an additional level of
+	// dropping at the falco engine, scaling with the sampling
+	// ratio in m_sampling_ratio.
+	//
+
+	uint32_t m_sampling_ratio;
+	double m_sampling_multiplier;
+
+	std::string m_lua_main_filename = "rule_loader.lua";
+};
+

userspace/engine/lua/rule_loader.lua

@@ -5,7 +5,6 @@
 
 --]]
 
-local output = require('output')
 local compiler = require "compiler"
 local yaml = require"lyaml"
 
@@ -101,31 +100,23 @@ function set_output(output_format, state)
    end
 end
 
-local function priority(s)
-   s = string.lower(s)
-   for i,v in ipairs(output.levels) do
-      if (string.find(string.lower(v), "^"..s)) then
-	 return i - 1 -- (syslog levels start at 0, lua indices start at 1)
-      end
-   end
-   error("Invalid severity level: "..s)
-end
-
 -- Note that the rules_by_name and rules_by_idx refer to the same rule
 -- object. The by_name index is used for things like describing rules,
 -- and the by_idx index is used to map the relational node index back
 -- to a rule.
 local state = {macros={}, lists={}, filter_ast=nil, rules_by_name={}, n_rules=0, rules_by_idx={}}
 
-function load_rules(filename, rules_mgr, verbose, all_events)
+function load_rules(rules_content, rules_mgr, verbose, all_events)
 
    compiler.set_verbose(verbose)
    compiler.set_all_events(all_events)
 
-   local f = assert(io.open(filename, "r"))
-   local s = f:read("*all")
-   f:close()
-   local rules = yaml.load(s)
+   local rules = yaml.load(rules_content)
+
+   if rules == nil then
+      -- An empty rules file is acceptable
+      return
+   end
 
    for i,v in ipairs(rules) do -- iterate over yaml list
 
@@ -168,8 +159,6 @@ function load_rules(filename, rules_mgr, verbose, all_events)
 	    end
 	 end
 
-	 -- Convert the priority as a string to a level now
-	 v['level'] = priority(v['priority'])
 	 state.rules_by_name[v['rule']] = v
 
 	 local filter_ast, evttypes = compiler.compile_filter(v['rule'], v['condition'],
@@ -190,7 +179,7 @@ function load_rules(filename, rules_mgr, verbose, all_events)
 	    install_filter(filter_ast.filter.value)
 
 	    -- Pass the filter and event types back up
-	    falco_rules.add_filter(rules_mgr, evttypes)
+	    falco_rules.add_filter(rules_mgr, v['rule'], evttypes)
 
 	    -- Rule ASTs are merged together into one big AST, with "OR" between each
 	    -- rule.
@@ -256,11 +245,7 @@ function describe_rule(name)
    end
 end
 
-local rule_output_counts = {total=0, by_level={}, by_name={}}
-
-for idx=0,table.getn(output.levels)-1,1 do
-   rule_output_counts.by_level[idx] = 0
-end
+local rule_output_counts = {total=0, by_priority={}, by_name={}}
 
 function on_event(evt_, rule_id)
 
@@ -271,10 +256,10 @@ function on_event(evt_, rule_id)
    rule_output_counts.total = rule_output_counts.total + 1
    local rule = state.rules_by_idx[rule_id]
 
-   if rule_output_counts.by_level[rule.level] == nil then
-      rule_output_counts.by_level[rule.level] = 1
+   if rule_output_counts.by_priority[rule.priority] == nil then
+      rule_output_counts.by_priority[rule.priority] = 1
    else
-      rule_output_counts.by_level[rule.level] = rule_output_counts.by_level[rule.level] + 1
+      rule_output_counts.by_priority[rule.priority] = rule_output_counts.by_priority[rule.priority] + 1
    end
 
    if rule_output_counts.by_name[rule.rule] == nil then
@@ -283,17 +268,14 @@ function on_event(evt_, rule_id)
       rule_output_counts.by_name[rule.rule] = rule_output_counts.by_name[rule.rule] + 1
    end
 
-   output.event(evt_, rule.rule, rule.level, rule.output)
+   return rule.rule, rule.priority, rule.output
 end
 
 function print_stats()
    print("Events detected: "..rule_output_counts.total)
    print("Rule counts by severity:")
-   for idx, level in ipairs(output.levels) do
-      -- To keep the output concise, we only print 0 counts for error, warning, and info levels
-      if rule_output_counts.by_level[idx-1] > 0 or level == "Error" or level == "Warning" or level == "Informational" then
-	 print ("   "..level..": "..rule_output_counts.by_level[idx-1])
-      end
+   for priority, count in pairs(rule_output_counts.by_priority) do
+      print ("   "..priority..": "..count)
    end
 
    print("Triggered rules by rule name:")

userspace/engine/rules.cpp

@@ -7,20 +7,17 @@ extern "C" {
 #include "lauxlib.h"
 }
 
+#include "falco_engine.h"
 const static struct luaL_reg ll_falco_rules [] =
 {
 	{"add_filter", &falco_rules::add_filter},
 	{NULL,NULL}
 };
 
-falco_rules::falco_rules(sinsp* inspector, lua_State *ls, string lua_main_filename)
+falco_rules::falco_rules(sinsp* inspector, falco_engine *engine, lua_State *ls)
+	: m_inspector(inspector), m_engine(engine), m_ls(ls)
 {
-        m_inspector = inspector;
-	m_ls = ls;
-
 	m_lua_parser = new lua_parser(inspector, m_ls);
-
-	load_compiler(lua_main_filename);
 }
 
 void falco_rules::init(lua_State *ls)
@@ -30,14 +27,15 @@ void falco_rules::init(lua_State *ls)
 
 int falco_rules::add_filter(lua_State *ls)
 {
-	if (! lua_islightuserdata(ls, -2) ||
+	if (! lua_islightuserdata(ls, -3) ||
+	    ! lua_isstring(ls, -2) ||
 	    ! lua_istable(ls, -1))
 	{
-		falco_logger::log(LOG_ERR, "Invalid arguments passed to add_filter()\n");
-		throw sinsp_exception("add_filter error");
+		throw falco_exception("Invalid arguments passed to add_filter()\n");
 	}
 
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -2);
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -3);
+	const char *rulec = lua_tostring(ls, -2);
 
 	list<uint32_t> evttypes;
 
@@ -51,44 +49,23 @@ int falco_rules::add_filter(lua_State *ls)
 		lua_pop(ls, 1);
 	}
 
-	rules->add_filter(evttypes);
+	std::string rule = rulec;
+	rules->add_filter(rule, evttypes);
 
 	return 0;
 }
 
-void falco_rules::add_filter(list<uint32_t> &evttypes)
+void falco_rules::add_filter(string &rule, list<uint32_t> &evttypes)
 {
 	// While the current rule was being parsed, a sinsp_filter
 	// object was being populated by lua_parser. Grab that filter
-	// and pass it to the inspector.
+	// and pass it to the engine.
 	sinsp_filter *filter = m_lua_parser->get_filter(true);
 
-	m_inspector->add_evttype_filter(evttypes, filter);
+	m_engine->add_evttype_filter(rule, evttypes, filter);
 }
 
-void falco_rules::load_compiler(string lua_main_filename)
-{
-	ifstream is;
-	is.open(lua_main_filename);
-	if(!is.is_open())
-	{
-		throw sinsp_exception("can't open file " + lua_main_filename);
-	}
-
-	string scriptstr((istreambuf_iterator<char>(is)),
-			 istreambuf_iterator<char>());
-
-	//
-	// Load the compiler script
-	//
-	if(luaL_loadstring(m_ls, scriptstr.c_str()) || lua_pcall(m_ls, 0, 0, 0))
-	{
-		throw sinsp_exception("Failed to load script " +
-			lua_main_filename + ": " + lua_tostring(m_ls, -1));
-	}
-}
-
-void falco_rules::load_rules(string rules_filename, bool verbose, bool all_events)
+void falco_rules::load_rules(const string &rules_content, bool verbose, bool all_events)
 {
 	lua_getglobal(m_ls, m_lua_load_rules.c_str());
 	if(lua_isfunction(m_ls, -1))
@@ -158,7 +135,7 @@ void falco_rules::load_rules(string rules_filename, bool verbose, bool all_event
 
 		lua_setglobal(m_ls, m_lua_ignored_syscalls.c_str());
 
-		lua_pushstring(m_ls, rules_filename.c_str());
+		lua_pushstring(m_ls, rules_content.c_str());
 		lua_pushlightuserdata(m_ls, this);
 		lua_pushboolean(m_ls, (verbose ? 1 : 0));
 		lua_pushboolean(m_ls, (all_events ? 1 : 0));
@@ -166,10 +143,10 @@ void falco_rules::load_rules(string rules_filename, bool verbose, bool all_event
 		{
 			const char* lerr = lua_tostring(m_ls, -1);
 			string err = "Error loading rules:" + string(lerr);
-			throw sinsp_exception(err);
+			throw falco_exception(err);
 		}
 	} else {
-		throw sinsp_exception("No function " + m_lua_load_rules + " found in lua rule module");
+		throw falco_exception("No function " + m_lua_load_rules + " found in lua rule module");
 	}
 }
 
@@ -189,19 +166,14 @@ void falco_rules::describe_rule(std::string *rule)
 		{
 			const char* lerr = lua_tostring(m_ls, -1);
 			string err = "Could not describe " + (rule == NULL ? "all rules" : "rule " + *rule) + ": " + string(lerr);
-			throw sinsp_exception(err);
+			throw falco_exception(err);
 		}
 	} else {
-		throw sinsp_exception("No function " + m_lua_describe_rule + " found in lua rule module");
+		throw falco_exception("No function " + m_lua_describe_rule + " found in lua rule module");
 	}
 }
 
 
-sinsp_filter* falco_rules::get_filter()
-{
-	return m_lua_parser->get_filter();
-}
-
 falco_rules::~falco_rules()
 {
 	delete m_lua_parser;

userspace/engine/rules.h

@@ -3,33 +3,33 @@
 #include <list>
 
 #include "sinsp.h"
+
 #include "lua_parser.h"
 
+class falco_engine;
+
 class falco_rules
 {
  public:
-	falco_rules(sinsp* inspector, lua_State *ls, string lua_main_filename);
+	falco_rules(sinsp* inspector, falco_engine *engine, lua_State *ls);
 	~falco_rules();
-	void load_rules(string rules_filename, bool verbose, bool all_events);
+	void load_rules(const string &rules_content, bool verbose, bool all_events);
 	void describe_rule(string *rule);
-	sinsp_filter* get_filter();
 
 	static void init(lua_State *ls);
 	static int add_filter(lua_State *ls);
 
  private:
-	void load_compiler(string lua_main_filename);
-
-	void add_filter(list<uint32_t> &evttypes);
+	void add_filter(string &rule, list<uint32_t> &evttypes);
 
 	lua_parser* m_lua_parser;
 	sinsp* m_inspector;
+	falco_engine *m_engine;
 	lua_State* m_ls;
 
 	string m_lua_load_rules = "load_rules";
 	string m_lua_ignored_syscalls = "ignored_syscalls";
 	string m_lua_ignored_events = "ignored_events";
 	string m_lua_events = "events";
-	string m_lua_on_event = "on_event";
 	string m_lua_describe_rule = "describe_rule";
 };

userspace/falco/CMakeLists.txt

@@ -3,22 +3,20 @@ include_directories("${LUAJIT_INCLUDE}")
 
 include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap")
 include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp")
+include_directories("${PROJECT_SOURCE_DIR}/userspace/engine")
 include_directories("${PROJECT_BINARY_DIR}/userspace/falco")
 include_directories("${CURL_INCLUDE_DIR}")
 include_directories("${YAMLCPP_INCLUDE_DIR}")
 include_directories("${DRAIOS_DEPENDENCIES_DIR}/yaml-${DRAIOS_YAML_VERSION}/target/include")
 
-add_executable(falco configuration.cpp formats.cpp fields.cpp rules.cpp logger.cpp falco.cpp)
+add_executable(falco configuration.cpp formats.cpp logger.cpp falco_outputs.cpp falco.cpp)
 
-target_link_libraries(falco sinsp)
+target_link_libraries(falco falco_engine sinsp)
 target_link_libraries(falco
-	"${LPEG_SRC}/lpeg.a"
-	"${LYAML_LIB}"
         "${LIBYAML_LIB}"
 	"${YAMLCPP_LIB}")
 
 
-set(FALCO_LUA_MAIN "rule_loader.lua")
 configure_file(config_falco.h.in config_falco.h)
 
 install(TARGETS falco DESTINATION bin)

userspace/falco/config_falco.h.in

@@ -2,13 +2,10 @@
 
 #define FALCO_VERSION "${FALCO_VERSION}"
 
-#define FALCO_LUA_DIR "/usr/share/falco/lua/"
+#define FALCO_LUA_DIR "${CMAKE_INSTALL_PREFIX}/${FALCO_SHARE_DIR}/lua/"
 #define FALCO_SOURCE_DIR "${PROJECT_SOURCE_DIR}"
 #define FALCO_SOURCE_CONF_FILE "${PROJECT_SOURCE_DIR}/falco.yaml"
 #define FALCO_INSTALL_CONF_FILE "/etc/falco.yaml"
 #define FALCO_SOURCE_LUA_DIR "${PROJECT_SOURCE_DIR}/userspace/falco/lua/"
 
-
-#define FALCO_LUA_MAIN "${FALCO_LUA_MAIN}"
-
 #define PROBE_NAME "${PROBE_NAME}"

userspace/falco/configuration.cpp

@@ -1,32 +1,42 @@
 #include "configuration.h"
-#include "config_falco.h"
-#include "sinsp.h"
 #include "logger.h"
 
 using namespace std;
 
+falco_configuration::falco_configuration()
+	: m_config(NULL)
+{
+}
+
+falco_configuration::~falco_configuration()
+{
+	if (m_config)
+	{
+		delete m_config;
+	}
+}
 
 // If we don't have a configuration file, we just use stdout output and all other defaults
-void falco_configuration::init(std::list<std::string> &cmdline_options)
+void falco_configuration::init(list<string> &cmdline_options)
 {
 	init_cmdline_options(cmdline_options);
 
-	output_config stdout_output;
+	falco_outputs::output_config stdout_output;
 	stdout_output.name = "stdout";
 	m_outputs.push_back(stdout_output);
 }
 
-void falco_configuration::init(string conf_filename, std::list<std::string> &cmdline_options)
+void falco_configuration::init(string conf_filename, list<string> &cmdline_options)
 {
 	string m_config_file = conf_filename;
 	m_config = new yaml_configuration(m_config_file);
 
 	init_cmdline_options(cmdline_options);
 
-	m_rules_filename = m_config->get_scalar<string>("rules_file", "/etc/falco_rules.yaml");
+	m_rules_filenames.push_back(m_config->get_scalar<string>("rules_file", "/etc/falco_rules.yaml"));
 	m_json_output = m_config->get_scalar<bool>("json_output", false);
 
-	output_config file_output;
+	falco_outputs::output_config file_output;
 	file_output.name = "file";
 	if (m_config->get_scalar<bool>("file_output", "enabled", false))
 	{
@@ -34,27 +44,27 @@ void falco_configuration::init(string conf_filename, std::list<std::string> &cmd
 		filename = m_config->get_scalar<string>("file_output", "filename", "");
 		if (filename == string(""))
 		{
-			throw sinsp_exception("Error reading config file (" + m_config_file + "): file output enabled but no filename in configuration block");
+			throw invalid_argument("Error reading config file (" + m_config_file + "): file output enabled but no filename in configuration block");
 		}
 		file_output.options["filename"] = filename;
 		m_outputs.push_back(file_output);
 	}
 
-	output_config stdout_output;
+	falco_outputs::output_config stdout_output;
 	stdout_output.name = "stdout";
 	if (m_config->get_scalar<bool>("stdout_output", "enabled", false))
 	{
 		m_outputs.push_back(stdout_output);
 	}
 
-	output_config syslog_output;
+	falco_outputs::output_config syslog_output;
 	syslog_output.name = "syslog";
 	if (m_config->get_scalar<bool>("syslog_output", "enabled", false))
 	{
 		m_outputs.push_back(syslog_output);
 	}
 
-	output_config program_output;
+	falco_outputs::output_config program_output;
 	program_output.name = "program";
 	if (m_config->get_scalar<bool>("program_output", "enabled", false))
 	{
@@ -70,7 +80,7 @@ void falco_configuration::init(string conf_filename, std::list<std::string> &cmd
 
 	if (m_outputs.size() == 0)
 	{
-		throw sinsp_exception("Error reading config file (" + m_config_file + "): No outputs configured. Please configure at least one output file output enabled but no filename in configuration block");
+		throw invalid_argument("Error reading config file (" + m_config_file + "): No outputs configured. Please configure at least one output file output enabled but no filename in configuration block");
 	}
 
 	falco_logger::log_stderr = m_config->get_scalar<bool>("log_stderr", false);
@@ -90,7 +100,7 @@ static bool split(const string &str, char delim, pair<string,string> &parts)
 	return true;
 }
 
-void falco_configuration::init_cmdline_options(std::list<std::string> &cmdline_options)
+void falco_configuration::init_cmdline_options(list<string> &cmdline_options)
 {
 	for(const string &option : cmdline_options)
 	{
@@ -98,13 +108,13 @@ void falco_configuration::init_cmdline_options(std::list<std::string> &cmdline_o
 	}
 }
 
-void falco_configuration::set_cmdline_option(const std::string &opt)
+void falco_configuration::set_cmdline_option(const string &opt)
 {
 	pair<string,string> keyval;
 	pair<string,string> subkey;
 
 	if (! split(opt, '=', keyval)) {
-		throw sinsp_exception("Error parsing config option \"" + opt + "\". Must be of the form key=val or key.subkey=val");
+		throw invalid_argument("Error parsing config option \"" + opt + "\". Must be of the form key=val or key.subkey=val");
 	}
 
 	if (split(keyval.first, '.', subkey)) {

userspace/falco/configuration.h

@@ -1,13 +1,12 @@
 #pragma once
 
 #include <yaml-cpp/yaml.h>
+#include <string>
+#include <vector>
+#include <list>
 #include <iostream>
 
-struct output_config
-{
-	std::string name;
-	std::map<std::string, std::string> options;
-};
+#include "falco_outputs.h"
 
 class yaml_configuration
 {
@@ -17,7 +16,7 @@ public:
 	{
 		m_path = path;
 		YAML::Node config;
-		std::vector<output_config> outputs;
+		std::vector<falco_outputs::output_config> outputs;
 		try
 		{
 			m_root = YAML::LoadFile(path);
@@ -118,12 +117,15 @@ private:
 class falco_configuration
 {
  public:
+	falco_configuration();
+	virtual ~falco_configuration();
+
 	void init(std::string conf_filename, std::list<std::string> &cmdline_options);
 	void init(std::list<std::string> &cmdline_options);
 
-	std::string m_rules_filename;
+	std::list<std::string> m_rules_filenames;
 	bool m_json_output;
-	std::vector<output_config> m_outputs;
+	std::vector<falco_outputs::output_config> m_outputs;
  private:
 	void init_cmdline_options(std::list<std::string> &cmdline_options);
 

userspace/falco/falco.cpp

@@ -1,32 +1,23 @@
 #define __STDC_FORMAT_MACROS
 
 #include <stdio.h>
-#include <iostream>
+#include <fstream>
+#include <set>
+#include <list>
+#include <string>
 #include <signal.h>
-#include <time.h>
 #include <fcntl.h>
 #include <sys/stat.h>
-#include <algorithm>
 #include <unistd.h>
 #include <getopt.h>
 
-extern "C" {
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
-#include "lpeg.h"
-#include "lyaml.h"
-}
-
 #include <sinsp.h>
-#include "config_falco.h"
-#include "configuration.h"
-#include "rules.h"
-#include "formats.h"
-#include "fields.h"
+
 #include "logger.h"
-#include "utils.h"
-#include <yaml-cpp/yaml.h>
+
+#include "configuration.h"
+#include "falco_engine.h"
+#include "config_falco.h"
 
 bool g_terminate = false;
 //
@@ -53,6 +44,8 @@ static void usage()
 	   " -p, --pidfile <pid_file>      When run as a daemon, write pid to specified file\n"
            " -e <events_file>              Read the events from <events_file> (in .scap format) instead of tapping into live.\n"
            " -r <rules_file>               Rules file (defaults to value set in configuration file, or /etc/falco_rules.yaml).\n"
+	   "                               Can be specified multiple times to read from multiple files.\n"
+	   " -D <pattern>                  Disable any rules matching the regex <pattern>. Can be specified multiple times.\n"
 	   " -L                            Show the name and description of all rules and exit.\n"
 	   " -l <rule>                     Show the name and description of the rule with name <rule> and exit.\n"
 	   " -v                            Verbose output.\n"
@@ -61,7 +54,7 @@ static void usage()
     );
 }
 
-static void display_fatal_err(const string &msg, bool daemon)
+static void display_fatal_err(const string &msg)
 {
 	falco_logger::log(LOG_ERR, msg);
 
@@ -75,23 +68,18 @@ static void display_fatal_err(const string &msg, bool daemon)
 	}
 }
 
-string lua_on_event = "on_event";
-string lua_add_output = "add_output";
-string lua_print_stats = "print_stats";
-
 // Splitting into key=value or key.subkey=value will be handled by configuration class.
 std::list<string> cmdline_options;
 
 //
 // Event processing loop
 //
-void do_inspect(sinsp* inspector,
-		falco_rules* rules,
-		lua_State* ls)
+void do_inspect(falco_engine *engine,
+		falco_outputs *outputs,
+		sinsp* inspector)
 {
 	int32_t res;
 	sinsp_evt* ev;
-	string line;
 
 	//
 	// Loop through the events
@@ -129,110 +117,18 @@ void do_inspect(sinsp* inspector,
 			continue;
 		}
 
-		lua_getglobal(ls, lua_on_event.c_str());
-
-		if(lua_isfunction(ls, -1))
+		// As the inspector has no filter at its level, all
+		// events are returned here. Pass them to the falco
+		// engine, which will match the event against the set
+		// of rules. If a match is found, pass the event to
+		// the outputs.
+		falco_engine::rule_result *res = engine->process_event(ev);
+		if(res)
 		{
-			lua_pushlightuserdata(ls, ev);
-			lua_pushnumber(ls, ev->get_check_id());
-
-			if(lua_pcall(ls, 2, 0, 0) != 0)
-			{
-				const char* lerr = lua_tostring(ls, -1);
-				string err = "Error invoking function output: " + string(lerr);
-				throw sinsp_exception(err);
+			outputs->handle_event(res->evt, res->rule, res->priority, res->format);
+			delete(res);
 		}
 	}
-		else
-		{
-			throw sinsp_exception("No function " + lua_on_event + " found in lua compiler module");
-		}
-	}
-}
-
-void add_lua_path(lua_State *ls, string path)
-{
-	string cpath = string(path);
-	path += "?.lua";
-	cpath += "?.so";
-
-	lua_getglobal(ls, "package");
-
-	lua_getfield(ls, -1, "path");
-	string cur_path = lua_tostring(ls, -1 );
-	cur_path += ';';
-	lua_pop(ls, 1);
-
-	cur_path.append(path.c_str());
-
-	lua_pushstring(ls, cur_path.c_str());
-	lua_setfield(ls, -2, "path");
-
-	lua_getfield(ls, -1, "cpath");
-	string cur_cpath = lua_tostring(ls, -1 );
-	cur_cpath += ';';
-	lua_pop(ls, 1);
-
-	cur_cpath.append(cpath.c_str());
-
-	lua_pushstring(ls, cur_cpath.c_str());
-	lua_setfield(ls, -2, "cpath");
-
-	lua_pop(ls, 1);
-}
-
-void add_output(lua_State *ls, output_config oc)
-{
-
-	uint8_t nargs = 1;
-	lua_getglobal(ls, lua_add_output.c_str());
-
-	if(!lua_isfunction(ls, -1))
-	{
-		throw sinsp_exception("No function " + lua_add_output + " found. ");
-	}
-	lua_pushstring(ls, oc.name.c_str());
-
-	// If we have options, build up a lua table containing them
-	if (oc.options.size())
-	{
-		nargs = 2;
-		lua_createtable(ls, 0, oc.options.size());
-
-		for (auto it = oc.options.cbegin(); it != oc.options.cend(); ++it)
-		{
-			lua_pushstring(ls, (*it).second.c_str());
-			lua_setfield(ls, -2, (*it).first.c_str());
-		}
-	}
-
-	if(lua_pcall(ls, nargs, 0, 0) != 0)
-	{
-		const char* lerr = lua_tostring(ls, -1);
-		throw sinsp_exception(string(lerr));
-	}
-
-}
-
-// Print statistics on the the rules that triggered
-void print_stats(lua_State *ls)
-{
-	lua_getglobal(ls, lua_print_stats.c_str());
-
-	if(lua_isfunction(ls, -1))
-	{
-		if(lua_pcall(ls, 0, 0, 0) != 0)
-		{
-			const char* lerr = lua_tostring(ls, -1);
-			string err = "Error invoking function print_stats: " + string(lerr);
-			throw sinsp_exception(err);
-		}
-	}
-	else
-	{
-		throw sinsp_exception("No function " + lua_print_stats + " found in lua rule loader module");
-	}
-
 }
 
 //
@@ -242,15 +138,13 @@ int falco_init(int argc, char **argv)
 {
 	int result = EXIT_SUCCESS;
 	sinsp* inspector = NULL;
-	falco_rules* rules = NULL;
+	falco_engine *engine = NULL;
+	falco_outputs *outputs = NULL;
 	int op;
 	int long_index = 0;
-	string lua_main_filename;
 	string scap_filename;
 	string conf_filename;
-	string rules_filename;
-	string lua_dir = FALCO_LUA_DIR;
-	lua_State* ls = NULL;
+	list<string> rules_filenames;
 	bool daemon = false;
 	string pidfilename = "/var/run/falco.pid";
 	bool describe_all_rules = false;
@@ -271,12 +165,20 @@ int falco_init(int argc, char **argv)
 	try
 	{
 		inspector = new sinsp();
+		engine = new falco_engine();
+		engine->set_inspector(inspector);
+
+		outputs = new falco_outputs();
+		outputs->set_inspector(inspector);
+
+		set<string> disabled_rule_patterns;
+		string pattern;
 
 		//
 		// Parse the args
 		//
 		while((op = getopt_long(argc, argv,
-                                        "c:ho:e:r:dp:Ll:vA",
+                                        "c:ho:e:r:D:dp:Ll:vA",
                                         long_options, &long_index)) != -1)
 		{
 			switch(op)
@@ -294,7 +196,11 @@ int falco_init(int argc, char **argv)
 				scap_filename = optarg;
 				break;
 			case 'r':
-				rules_filename = optarg;
+				rules_filenames.push_back(optarg);
+				break;
+			case 'D':
+				pattern = optarg;
+				disabled_rule_patterns.insert(pattern);
 				break;
 			case 'd':
 				daemon = true;
@@ -325,29 +231,29 @@ int falco_init(int argc, char **argv)
 
 		// Some combinations of arguments are not allowed.
 		if (daemon && pidfilename == "") {
-			throw sinsp_exception("If -d is provided, a pid file must also be provided");
+			throw std::invalid_argument("If -d is provided, a pid file must also be provided");
 		}
 
-		ifstream* conf_stream;
+		ifstream conf_stream;
 		if (conf_filename.size())
 		{
-			conf_stream = new ifstream(conf_filename);
-			if (!conf_stream->good())
+			conf_stream.open(conf_filename);
+			if (!conf_stream.is_open())
 			{
-				throw sinsp_exception("Could not find configuration file at " + conf_filename);
+				throw std::runtime_error("Could not find configuration file at " + conf_filename);
 			}
 		}
 		else
 		{
-			conf_stream = new ifstream(FALCO_SOURCE_CONF_FILE);
-			if (conf_stream->good())
+			conf_stream.open(FALCO_SOURCE_CONF_FILE);
+			if (!conf_stream.is_open())
 			{
 				conf_filename = FALCO_SOURCE_CONF_FILE;
 			}
 			else
 			{
-				conf_stream = new ifstream(FALCO_INSTALL_CONF_FILE);
-				if (conf_stream->good())
+				conf_stream.open(FALCO_INSTALL_CONF_FILE);
+				if (!conf_stream.is_open())
 				{
 					conf_filename = FALCO_INSTALL_CONF_FILE;
 				}
@@ -371,66 +277,47 @@ int falco_init(int argc, char **argv)
 			falco_logger::log(LOG_INFO, "Falco initialized. No configuration file found, proceeding with defaults\n");
 		}
 
-		if (rules_filename.size())
+		if (rules_filenames.size())
 		{
-			config.m_rules_filename = rules_filename;
+			config.m_rules_filenames = rules_filenames;
 		}
 
-		lua_main_filename = lua_dir + FALCO_LUA_MAIN;
-		if (!std::ifstream(lua_main_filename))
+		for (auto filename : config.m_rules_filenames)
 		{
-			lua_dir = FALCO_SOURCE_LUA_DIR;
-			lua_main_filename = lua_dir + FALCO_LUA_MAIN;
-			if (!std::ifstream(lua_main_filename))
-			{
-				falco_logger::log(LOG_ERR, "Could not find Falco Lua libraries (tried " +
-						     string(FALCO_LUA_DIR FALCO_LUA_MAIN) + ", " +
-						     lua_main_filename + "). Exiting.\n");
-				result = EXIT_FAILURE;
-				goto exit;
-			}
+			engine->load_rules_file(filename, verbose, all_events);
+			falco_logger::log(LOG_INFO, "Parsed rules from file " + filename + "\n");
 		}
 
-		// Initialize Lua interpreter
-		ls = lua_open();
-		luaL_openlibs(ls);
-		luaopen_lpeg(ls);
-		luaopen_yaml(ls);
-		add_lua_path(ls, lua_dir);
-
-		rules = new falco_rules(inspector, ls, lua_main_filename);
-
-		falco_formats::init(inspector, ls, config.m_json_output);
-		falco_fields::init(inspector, ls);
-
-		falco_logger::init(ls);
-		falco_rules::init(ls);
+		for (auto pattern : disabled_rule_patterns)
+		{
+			falco_logger::log(LOG_INFO, "Disabling rules matching pattern: " + pattern + "\n");
+			engine->enable_rule(pattern, false);
+		}
 
+		outputs->init(config.m_json_output);
 
 		if(!all_events)
 		{
 			inspector->set_drop_event_flags(EF_DROP_FALCO);
 		}
-		rules->load_rules(config.m_rules_filename, verbose, all_events);
-		falco_logger::log(LOG_INFO, "Parsed rules from file " + config.m_rules_filename + "\n");
 
 		if (describe_all_rules)
 		{
-			rules->describe_rule(NULL);
+			engine->describe_rule(NULL);
 			goto exit;
 		}
 
 		if (describe_rule != "")
 		{
-			rules->describe_rule(&describe_rule);
+			engine->describe_rule(&describe_rule);
 			goto exit;
 		}
 
 		inspector->set_hostname_and_port_resolution_mode(false);
 
-		for(std::vector<output_config>::iterator it = config.m_outputs.begin(); it != config.m_outputs.end(); ++it)
+		for(auto output : config.m_outputs)
 		{
-			add_output(ls, *it);
+			outputs->add_output(output);
 		}
 
 		if(signal(SIGINT, signal_callback) == SIG_ERR)
@@ -522,23 +409,17 @@ int falco_init(int argc, char **argv)
 			open("/dev/null", O_RDWR);
 		}
 
-		do_inspect(inspector,
-			   rules,
-			   ls);
+		do_inspect(engine,
+			   outputs,
+			   inspector);
 
 		inspector->close();
 
-		print_stats(ls);
+		engine->print_stats();
 	}
-	catch(sinsp_exception& e)
+	catch(exception &e)
 	{
-		display_fatal_err("Runtime error: " + string(e.what()) + ". Exiting.\n", daemon);
-
-		result = EXIT_FAILURE;
-	}
-	catch(...)
-	{
-		display_fatal_err("Unexpected error, Exiting\n", daemon);
+		display_fatal_err("Runtime error: " + string(e.what()) + ". Exiting.\n");
 
 		result = EXIT_FAILURE;
 	}
@@ -546,11 +427,9 @@ int falco_init(int argc, char **argv)
 exit:
 
 	delete inspector;
+	delete engine;
+	delete outputs;
 
-	if(ls)
-	{
-		lua_close(ls);
-	}
 	return result;
 }
 

userspace/falco/falco_outputs.cpp

@@ -0,0 +1,92 @@
+
+#include "falco_outputs.h"
+
+#include "config_falco.h"
+
+
+#include "formats.h"
+#include "logger.h"
+
+using namespace std;
+
+falco_outputs::falco_outputs()
+{
+
+}
+
+falco_outputs::~falco_outputs()
+{
+
+}
+
+void falco_outputs::init(bool json_output)
+{
+	// The engine must have been given an inspector by now.
+	if(! m_inspector)
+	{
+		throw falco_exception("No inspector provided");
+	}
+
+	falco_common::init(m_lua_main_filename.c_str(), FALCO_SOURCE_LUA_DIR);
+
+	falco_formats::init(m_inspector, m_ls, json_output);
+
+	falco_logger::init(m_ls);
+}
+
+void falco_outputs::add_output(output_config oc)
+{
+	uint8_t nargs = 1;
+	lua_getglobal(m_ls, m_lua_add_output.c_str());
+
+	if(!lua_isfunction(m_ls, -1))
+	{
+		throw falco_exception("No function " + m_lua_add_output + " found. ");
+	}
+	lua_pushstring(m_ls, oc.name.c_str());
+
+	// If we have options, build up a lua table containing them
+	if (oc.options.size())
+	{
+		nargs = 2;
+		lua_createtable(m_ls, 0, oc.options.size());
+
+		for (auto it = oc.options.cbegin(); it != oc.options.cend(); ++it)
+		{
+			lua_pushstring(m_ls, (*it).second.c_str());
+			lua_setfield(m_ls, -2, (*it).first.c_str());
+		}
+	}
+
+	if(lua_pcall(m_ls, nargs, 0, 0) != 0)
+	{
+		const char* lerr = lua_tostring(m_ls, -1);
+		throw falco_exception(string(lerr));
+	}
+
+}
+
+void falco_outputs::handle_event(sinsp_evt *ev, string &level, string &priority, string &format)
+{
+	lua_getglobal(m_ls, m_lua_output_event.c_str());
+
+	if(lua_isfunction(m_ls, -1))
+	{
+		lua_pushlightuserdata(m_ls, ev);
+		lua_pushstring(m_ls, level.c_str());
+		lua_pushstring(m_ls, priority.c_str());
+		lua_pushstring(m_ls, format.c_str());
+
+		if(lua_pcall(m_ls, 4, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Error invoking function output: " + string(lerr);
+			throw falco_exception(err);
+		}
+	}
+	else
+	{
+		throw falco_exception("No function " + m_lua_output_event + " found in lua compiler module");
+	}
+
+}

userspace/falco/falco_outputs.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "falco_common.h"
+
+//
+// This class acts as the primary interface between a program and the
+// falco output engine. The falco rules engine is implemented by a
+// separate class falco_engine.
+//
+
+class falco_outputs : public falco_common
+{
+public:
+	falco_outputs();
+	virtual ~falco_outputs();
+
+	// The way to refer to an output (file, syslog, stdout,
+	// etc). An output has a name and set of options.
+	struct output_config
+	{
+		std::string name;
+		std::map<std::string, std::string> options;
+	};
+
+	void init(bool json_output);
+
+	void add_output(output_config oc);
+
+	//
+	// ev is an event that has matched some rule. Pass the event
+	// to all configured outputs.
+	//
+	void handle_event(sinsp_evt *ev, std::string &level, std::string &priority, std::string &format);
+
+private:
+	std::string m_lua_add_output = "add_output";
+	std::string m_lua_output_event = "output_event";
+	std::string m_lua_main_filename = "output.lua";
+};

userspace/falco/fields.cpp

@@ -1,76 +0,0 @@
-#include "fields.h"
-#include "chisel_api.h"
-#include "filterchecks.h"
-
-
-extern sinsp_filter_check_list g_filterlist;
-
-const static struct luaL_reg ll_falco [] =
-{
-	{"field", &falco_fields::field},
-	{NULL,NULL}
-};
-
-sinsp* falco_fields::s_inspector = NULL;
-
-std::map<string, sinsp_filter_check*> falco_fields::s_fieldname_map;
-
-
-void falco_fields::init(sinsp* inspector, lua_State *ls)
-{
-	s_inspector = inspector;
-
-	luaL_openlib(ls, "falco", ll_falco, 0);
-}
-
-int falco_fields::field(lua_State *ls)
-{
-
-	sinsp_filter_check* chk=NULL;
-
-	if (!lua_islightuserdata(ls, 1))
-	{
-		string err = "invalid argument passed to falco.field()";
-		fprintf(stderr, "%s\n", err.c_str());
-		throw sinsp_exception("falco.field() error");
-	}
-	sinsp_evt* evt = (sinsp_evt*)lua_topointer(ls, 1);
-
-	string fieldname = luaL_checkstring(ls, 2);
-
-	if (s_fieldname_map.count(fieldname) == 0)
-	{
-
-		chk = g_filterlist.new_filter_check_from_fldname(fieldname,
-								 s_inspector,
-								 false);
-
-		if(chk == NULL)
-		{
-			string err = "nonexistent fieldname passed to falco.field(): " + string(fieldname);
-			fprintf(stderr, "%s\n", err.c_str());
-			throw sinsp_exception("falco.field() error");
-		}
-
-		chk->parse_field_name(fieldname.c_str(), true);
-		s_fieldname_map[fieldname] = chk;
-	}
-	else
-	{
-		chk = s_fieldname_map[fieldname];
-	}
-
-	uint32_t vlen;
-	uint8_t* rawval = chk->extract(evt, &vlen);
-
-	if(rawval != NULL)
-	{
-		return lua_cbacks::rawval_to_lua_stack(ls, rawval, chk->get_field_info(), vlen);
-	}
-	else
-	{
-		lua_pushnil(ls);
-		return 1;
-	}
-}
-

userspace/falco/fields.h

@@ -1,21 +0,0 @@
-#pragma once
-
-#include "sinsp.h"
-
-extern "C" {
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
-}
-
-class falco_fields
-{
- public:
-	static void init(sinsp* inspector, lua_State *ls);
-
-	// value = falco.field(evt, fieldname)
-	static int field(lua_State *ls);
-
-	static sinsp* s_inspector;
-	static std::map<string, sinsp_filter_check*> s_fieldname_map;
-};

userspace/falco/formats.cpp

@@ -2,6 +2,7 @@
 
 #include "formats.h"
 #include "logger.h"
+#include "falco_engine.h"
 
 
 sinsp* falco_formats::s_inspector = NULL;
@@ -10,6 +11,7 @@ bool s_json_output = false;
 const static struct luaL_reg ll_falco [] =
 {
 	{"formatter", &falco_formats::formatter},
+	{"free_formatter", &falco_formats::free_formatter},
 	{"format_event", &falco_formats::format_event},
 	{NULL,NULL}
 };
@@ -32,9 +34,7 @@ int falco_formats::formatter(lua_State *ls)
 	}
 	catch(sinsp_exception& e)
 	{
-		falco_logger::log(LOG_ERR, "Invalid output format '" + format + "'.\n");
-
-		throw sinsp_exception("set_formatter error");
+		throw falco_exception("Invalid output format '" + format + "'.\n");
 	}
 
 	lua_pushlightuserdata(ls, formatter);
@@ -42,6 +42,20 @@ int falco_formats::formatter(lua_State *ls)
 	return 1;
 }
 
+int falco_formats::free_formatter(lua_State *ls)
+{
+	if (!lua_islightuserdata(ls, -1))
+	{
+		throw falco_exception("Invalid argument passed to free_formatter");
+	}
+
+	sinsp_evt_formatter *formatter = (sinsp_evt_formatter *) lua_topointer(ls, 1);
+
+	delete(formatter);
+
+	return 1;
+}
+
 int falco_formats::format_event (lua_State *ls)
 {
 	string line;
@@ -50,8 +64,7 @@ int falco_formats::format_event (lua_State *ls)
 	    !lua_isstring(ls, -2) ||
 	    !lua_isstring(ls, -3) ||
 	    !lua_islightuserdata(ls, -4)) {
-		falco_logger::log(LOG_ERR, "Invalid arguments passed to format_event()\n");
-		throw sinsp_exception("format_event error");
+		throw falco_exception("Invalid arguments passed to format_event()\n");
 	}
 	sinsp_evt* evt = (sinsp_evt*)lua_topointer(ls, 1);
 	const char *rule = (char *) lua_tostring(ls, 2);

userspace/falco/formats.h

@@ -18,6 +18,9 @@ class falco_formats
 	// formatter = falco.formatter(format_string)
 	static int formatter(lua_State *ls);
 
+	// falco.free_formatter(formatter)
+	static int free_formatter(lua_State *ls);
+
 	// formatted_string = falco.format_event(evt, formatter)
 	static int format_event(lua_State *ls);
 

userspace/falco/logger.cpp

@@ -1,9 +1,6 @@
 #include <ctime>
 #include "logger.h"
 #include "chisel_api.h"
-#include "filterchecks.h"
-
-
 
 const static struct luaL_reg ll_falco [] =
 {

userspace/falco/lua/output.lua

@@ -6,10 +6,7 @@ mod.levels = levels
 
 local outputs = {}
 
-function mod.stdout(evt, rule, level, format)
-   format = "*%evt.time: "..levels[level+1].." "..format
-   formatter = falco.formatter(format)
-   msg = falco.format_event(evt, rule, levels[level+1], formatter)
+function mod.stdout(level, msg)
    print (msg)
 end
 
@@ -26,29 +23,17 @@ function mod.file_validate(options)
 
 end
 
-function mod.file(evt, rule, level, format, options)
-   format = "*%evt.time: "..levels[level+1].." "..format
-   formatter = falco.formatter(format)
-   msg = falco.format_event(evt, rule, levels[level+1], formatter)
-
+function mod.file(level, msg)
    file = io.open(options.filename, "a+")
    file:write(msg, "\n")
    file:close()
 end
 
-function mod.syslog(evt, rule, level, format)
-
-   formatter = falco.formatter(format)
-   msg = falco.format_event(evt, rule, levels[level+1], formatter)
+function mod.syslog(level, msg)
    falco.syslog(level, msg)
 end
 
-function mod.program(evt, rule, level, format, options)
-
-   format = "*%evt.time: "..levels[level+1].." "..format
-   formatter = falco.formatter(format)
-   msg = falco.format_event(evt, rule, levels[level+1], formatter)
-
+function mod.program(level, msg)
    -- XXX Ideally we'd check that the program ran
    -- successfully. However, the luajit we're using returns true even
    -- when the shell can't run the program.
@@ -59,10 +44,27 @@ function mod.program(evt, rule, level, format, options)
    file:close()
 end
 
-function mod.event(event, rule, level, format)
-   for index,o in ipairs(outputs) do
-      o.output(event, rule, level, format, o.config)
+local function level_of(s)
+   s = string.lower(s)
+   for i,v in ipairs(levels) do
+      if (string.find(string.lower(v), "^"..s)) then
+	 return i - 1 -- (syslog levels start at 0, lua indices start at 1)
       end
+   end
+   error("Invalid severity level: "..s)
+end
+
+function output_event(event, rule, priority, format)
+   local level = level_of(priority)
+   format = "*%evt.time: "..levels[level+1].." "..format
+   formatter = falco.formatter(format)
+   msg = falco.format_event(event, rule, levels[level+1], formatter)
+
+   for index,o in ipairs(outputs) do
+      o.output(level, msg)
+   end
+
+   falco.free_formatter(formatter)
 end
 
 function add_output(output_name, config)