Add support for tagging rules.

- in lua, look for a tags attribute to each rule. This is passed up in
  add_filter as a tags argument (as a lua table). If not present, an
  empty table is used. The tags table is iterated to populate a set
  of tags as strings, which is passed to add_filter().
- A new method falco_engine::enable_rule_by_tag is similar to
  enable_rule(), but is given a set of tag strings. Any rules containing
  one of the tags is enabled/disabled.
- The list of event types has been changed to a set to more accurately
  reflect its purpose.
- New argument to falco -T allows disabling all rules matching a given
  tag, via enable_rule_by_tag(). It can be provided multiple times.
- New argument to falco -t allows running those rules matching a given
  tag. If provided all rules are first disabled. It can be
  provided multiple times, but can not be combined with -T or
  -D (disable rules by name)
- falco_enging supports the notion of a ruleset. The idea is that you
  can choose a set of rules that are enabled/disabled by using
  enable_rule()/enable_rule_by_tag() in combination with a
  ruleset. Later, in process_event() you include that ruleset and the
  rules you had previously enabled will be run.
- rulsets are provided as strings in enable_rule()/enable_rule_by_tag()
  and as numbers in process_event()--this avoids the overhead of string
  lookups per-event. Ruleset ids are created on the fly as needed. A
  utility method find_ruleset_id() looks up the ruleset id for a given
  name. The default ruleset is NULL string/0 numeric if not provided.
- Although the ruleset is a useful falco engine feature, it isn't that
  important to the falco standalone program, so it's not
  documented. However, you can change the ruleset by providing
  FALCO_RULESET in the environment.

userspace/engine/falco_engine.cpp

@@ -40,7 +40,8 @@ string lua_print_stats = "print_stats";
 using namespace std;
 
 falco_engine::falco_engine(bool seed_rng)
-	: m_rules(NULL), m_sampling_ratio(1), m_sampling_multiplier(0),
+	: m_rules(NULL), m_next_ruleset_id(0),
+	  m_sampling_ratio(1), m_sampling_multiplier(0),
 	  m_replace_container_info(false)
 {
 	luaopen_lpeg(m_ls);
@@ -107,20 +108,51 @@ void falco_engine::load_rules_file(const string &rules_filename, bool verbose, b
 	load_rules(rules_content, verbose, all_events);
 }
 
-void falco_engine::enable_rule(string &pattern, bool enabled)
+void falco_engine::enable_rule(string &pattern, bool enabled, string *ruleset)
 {
-	m_evttype_filter->enable(pattern, enabled);
+	uint16_t ruleset_id = 0;
+
+	if(ruleset)
+	{
+		ruleset_id = find_ruleset_id(*ruleset);
+	}
+
+	m_evttype_filter->enable(pattern, enabled, ruleset_id);
 }
 
-unique_ptr<falco_engine::rule_result> falco_engine::process_event(sinsp_evt *ev)
+void falco_engine::enable_rule_by_tag(set<string> &tags, bool enabled, string *ruleset)
 {
+	uint16_t ruleset_id = 0;
 
+	if(ruleset)
+	{
+		ruleset_id = find_ruleset_id(*ruleset);
+	}
+
+	m_evttype_filter->enable_tags(tags, enabled, ruleset_id);
+}
+
+uint16_t falco_engine::find_ruleset_id(std::string &ruleset)
+{
+	auto it = m_known_rulesets.find(ruleset);
+
+	if(it == m_known_rulesets.end())
+	{
+		m_known_rulesets[ruleset] = ++m_next_ruleset_id;
+		it = m_known_rulesets.find(ruleset);
+	}
+
+	return it->second;
+}
+
+unique_ptr<falco_engine::rule_result> falco_engine::process_event(sinsp_evt *ev, uint16_t ruleset_id)
+{
 	if(should_drop_evt())
 	{
 		return unique_ptr<struct rule_result>();
 	}
 
-	if(!m_evttype_filter->run(ev))
+	if(!m_evttype_filter->run(ev, ruleset_id))
 	{
 		return unique_ptr<struct rule_result>();
 	}
@@ -182,10 +214,11 @@ void falco_engine::print_stats()
 }
 
 void falco_engine::add_evttype_filter(string &rule,
-				      list<uint32_t> &evttypes,
+				      set<uint32_t> &evttypes,
+				      set<string> &tags,
 				      sinsp_filter* filter)
 {
-	m_evttype_filter->add(rule, evttypes, filter);
+	m_evttype_filter->add(rule, evttypes, tags, filter);
 }
 
 void falco_engine::clear_filters()

userspace/engine/falco_engine.h

@@ -20,6 +20,7 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 
 #include <string>
 #include <memory>
+#include <set>
 
 #include "sinsp.h"
 #include "filter.h"
@@ -47,9 +48,18 @@ public:
 	void load_rules(const std::string &rules_content, bool verbose, bool all_events);
 
 	//
-	// Enable/Disable any rules matching the provided pattern (regex).
+	// Enable/Disable any rules matching the provided pattern
+	// (regex). If ruleset is non-NULL, enable/disable these
+	// rules in the context of the provided ruleset. The ruleset
+	// can later be passed as an argument to process_event(). This
+	// allows for different sets of rules being active at once.
 	//
-	void enable_rule(std::string &pattern, bool enabled);
+	void enable_rule(std::string &pattern, bool enabled, std::string *ruleset = NULL);
+
+	//
+	// Enable/Disable any rules with any of the provided tags (set, exact matches only)
+	//
+	void enable_rule_by_tag(std::set<std::string> &tags, bool enabled, std::string *ruleset = NULL);
 
 	struct rule_result {
 		sinsp_evt *evt;
@@ -58,13 +68,26 @@ public:
 		std::string format;
 	};
 
+	//
+	// Return the ruleset id corresponding to this ruleset name,
+	// creating a new one if necessary. If you provide any ruleset
+	// to enable_rule/enable_rule_by_tag(), you should look up the
+	// ruleset id and pass it to process_event().
+	//
+	uint16_t find_ruleset_id(std::string &ruleset);
+
 	//
 	// Given an event, check it against the set of rules in the
 	// engine and if a matching rule is found, return details on
 	// the rule that matched. If no rule matched, returns NULL.
 	//
-	// the reutrned rule_result is allocated and must be delete()d.
+	// If ruleset is non-NULL, use the enabled/disabled status
-	std::unique_ptr<rule_result> process_event(sinsp_evt *ev);
+	// associated with the provided ruleset. This is only useful
+	// when you have previously called enable_rule/enable_rule_by_tag
+	// with a non-NULL ruleset.
+	//
+	// the returned rule_result is allocated and must be delete()d.
+	std::unique_ptr<rule_result> process_event(sinsp_evt *ev, uint16_t ruleset_id = 0);
 
 	//
 	// Print details on the given rule. If rule is NULL, print
@@ -78,11 +101,12 @@ public:
 	void print_stats();
 
 	//
-	// Add a filter, which is related to the specified list of
+	// Add a filter, which is related to the specified set of
 	// event types, to the engine.
 	//
 	void add_evttype_filter(std::string &rule,
-				list<uint32_t> &evttypes,
+				std::set<uint32_t> &evttypes,
+				std::set<std::string> &tags,
 				sinsp_filter* filter);
 
 	// Clear all existing filters.
@@ -120,6 +144,8 @@ private:
 	inline bool should_drop_evt();
 
 	falco_rules *m_rules;
+	uint16_t m_next_ruleset_id;
+	std::map<string, uint16_t> m_known_rulesets;
 	std::unique_ptr<sinsp_evttype_filter> m_evttype_filter;
 
 	//

userspace/engine/lua/rule_loader.lua

@@ -308,8 +308,12 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
 
 	 install_filter(filter_ast.filter.value)
 
+	 if (v['tags'] == nil) then
+	    v['tags'] = {}
+	 end
+
 	 -- Pass the filter and event types back up
-	 falco_rules.add_filter(rules_mgr, v['rule'], evttypes)
+	 falco_rules.add_filter(rules_mgr, v['rule'], evttypes, v['tags'])
 
 	 -- Rule ASTs are merged together into one big AST, with "OR" between each
 	 -- rule.

userspace/engine/rules.cpp

@@ -65,42 +65,55 @@ void falco_rules::clear_filters()
 
 int falco_rules::add_filter(lua_State *ls)
 {
-	if (! lua_islightuserdata(ls, -3) ||
+	if (! lua_islightuserdata(ls, -4) ||
-	    ! lua_isstring(ls, -2) ||
+	    ! lua_isstring(ls, -3) ||
+	    ! lua_istable(ls, -2) ||
 	    ! lua_istable(ls, -1))
 	{
 		throw falco_exception("Invalid arguments passed to add_filter()\n");
 	}
 
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -3);
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -4);
-	const char *rulec = lua_tostring(ls, -2);
+	const char *rulec = lua_tostring(ls, -3);
 
-	list<uint32_t> evttypes;
+	set<uint32_t> evttypes;
+
+	lua_pushnil(ls);  /* first key */
+	while (lua_next(ls, -3) != 0) {
+                // key is at index -2, value is at index
+                // -1. We want the keys.
+		evttypes.insert(luaL_checknumber(ls, -2));
+
+		// Remove value, keep key for next iteration
+		lua_pop(ls, 1);
+	}
+
+	set<string> tags;
 
 	lua_pushnil(ls);  /* first key */
 	while (lua_next(ls, -2) != 0) {
                 // key is at index -2, value is at index
                 // -1. We want the keys.
-		evttypes.push_back(luaL_checknumber(ls, -2));
+		tags.insert(lua_tostring(ls, -1));
 
 		// Remove value, keep key for next iteration
 		lua_pop(ls, 1);
 	}
 
 	std::string rule = rulec;
-	rules->add_filter(rule, evttypes);
+	rules->add_filter(rule, evttypes, tags);
 
 	return 0;
 }
 
-void falco_rules::add_filter(string &rule, list<uint32_t> &evttypes)
+void falco_rules::add_filter(string &rule, set<uint32_t> &evttypes, set<string> &tags)
 {
 	// While the current rule was being parsed, a sinsp_filter
 	// object was being populated by lua_parser. Grab that filter
 	// and pass it to the engine.
 	sinsp_filter *filter = m_lua_parser->get_filter(true);
 
-	m_engine->add_evttype_filter(rule, evttypes, filter);
+	m_engine->add_evttype_filter(rule, evttypes, tags, filter);
 }
 
 int falco_rules::enable_rule(lua_State *ls)

userspace/engine/rules.h

@@ -18,7 +18,7 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 
 #pragma once
 
-#include <list>
+#include <set>
 
 #include "sinsp.h"
 
@@ -42,7 +42,7 @@ class falco_rules
 
  private:
 	void clear_filters();
-	void add_filter(string &rule, list<uint32_t> &evttypes);
+	void add_filter(string &rule, std::set<uint32_t> &evttypes, std::set<string> &tags);
 	void enable_rule(string &rule, bool enabled);
 
 	lua_parser* m_lua_parser;

userspace/falco/falco.cpp

@@ -60,6 +60,7 @@ static void usage()
 	   " -A                            Monitor all events, including those with EF_DROP_FALCO flag.\n"
 	   " -d, --daemon                  Run as a daemon\n"
 	   " -D <pattern>                  Disable any rules matching the regex <pattern>. Can be specified multiple times.\n"
+	   "                               Can not be specified with -t.\n"
            " -e <events_file>              Read the events from <events_file> (in .scap format) instead of tapping into live.\n"
 	   " -k <url>, --k8s-api=<url>\n"
 	   "                               Enable Kubernetes support by connecting to the API server\n"
@@ -100,6 +101,10 @@ static void usage()
 	   "                               Can be specified multiple times to read from multiple files.\n"
 	   " -s <stats_file>               If specified, write statistics related to falco's reading/processing of events\n"
 	   "                               to this file. (Only useful in live mode).\n"
+	   " -T <tag>                      Disable any rules with a tag=<tag>. Can be specified multiple times.\n"
+	   "                               Can not be specified with -t.\n"
+	   " -t <tag>                      Only run those rules with a tag=<tag>. Can be specified multiple times.\n"
+	   "                               Can not be specified with -T/-D.\n"
 	   " -v                            Verbose output.\n"
 	   "\n"
     );
@@ -128,7 +133,8 @@ std::list<string> cmdline_options;
 uint64_t do_inspect(falco_engine *engine,
 		    falco_outputs *outputs,
 		    sinsp* inspector,
-		    string &stats_filename)
+		    string &stats_filename,
+		    uint16_t ruleset_id)
 {
 	uint64_t num_evts = 0;
 	int32_t res;
@@ -188,7 +194,7 @@ uint64_t do_inspect(falco_engine *engine,
 		// engine, which will match the event against the set
 		// of rules. If a match is found, pass the event to
 		// the outputs.
-		unique_ptr<falco_engine::rule_result> res = engine->process_event(ev);
+		unique_ptr<falco_engine::rule_result> res = engine->process_event(ev, ruleset_id);
 		if(res)
 		{
 			outputs->handle_event(res->evt, res->rule, res->priority, res->format);
@@ -259,12 +265,15 @@ int falco_init(int argc, char **argv)
 	{
 		set<string> disabled_rule_patterns;
 		string pattern;
+		string all_rules = ".*";
+		set<string> disabled_rule_tags;
+		set<string> enabled_rule_tags;
 
 		//
 		// Parse the args
 		//
 		while((op = getopt_long(argc, argv,
-                                        "hc:AdD:e:k:K:Ll:m:o:P:p:r:s:vw:",
+                                        "hc:AdD:e:k:K:Ll:m:o:P:p:r:s:T:t:vw:",
                                         long_options, &long_index)) != -1)
 		{
 			switch(op)
@@ -339,6 +348,12 @@ int falco_init(int argc, char **argv)
 			case 's':
 				stats_filename = optarg;
 				break;
+			case 'T':
+				disabled_rule_tags.insert(optarg);
+				break;
+			case 't':
+				enabled_rule_tags.insert(optarg);
+				break;
 			case 'v':
 				verbose = true;
 				break;
@@ -358,6 +373,18 @@ int falco_init(int argc, char **argv)
 		engine = new falco_engine();
 		engine->set_inspector(inspector);
 		engine->set_extra(output_format, replace_container_info);
+		string *ruleset = NULL;
+		string ruleset_env;
+		uint16_t ruleset_id = 0;
+
+		// The ruleset feature is really falco
+		// engine-specific, so we don't advertise it. But it
+		// is possible to specify an alternate ruleset via the environment.
+		if (getenv("FALCO_RULESET") != NULL)
+		{
+			ruleset_env = getenv("FALCO_RULESET");
+			ruleset = &ruleset_env;
+		}
 
 		outputs = new falco_outputs();
 		outputs->set_inspector(inspector);
@@ -421,10 +448,44 @@ int falco_init(int argc, char **argv)
 			falco_logger::log(LOG_INFO, "Parsed rules from file " + filename + "\n");
 		}
 
+		// You can't both disable and enable rules
+		if((disabled_rule_patterns.size() + disabled_rule_tags.size() > 0) &&
+		    enabled_rule_tags.size() > 0) {
+			throw std::invalid_argument("You can not specify both disabled (-D/-T) and enabled (-t) rules");
+		}
+
+		// If a ruleset was provided, we must first explicitly enable all rules.
+		if(ruleset)
+		{
+			engine->enable_rule(all_rules, true, ruleset);
+		}
+
 		for (auto pattern : disabled_rule_patterns)
 		{
 			falco_logger::log(LOG_INFO, "Disabling rules matching pattern: " + pattern + "\n");
-			engine->enable_rule(pattern, false);
+			engine->enable_rule(pattern, false, ruleset);
+		}
+
+		if(disabled_rule_tags.size() > 0)
+		{
+			for(auto tag : disabled_rule_tags)
+			{
+				falco_logger::log(LOG_INFO, "Disabling rules with tag: " + tag + "\n");
+			}
+			engine->enable_rule_by_tag(disabled_rule_tags, false, ruleset);
+		}
+
+		if(enabled_rule_tags.size() > 0)
+		{
+
+			// Since we only want to enable specific
+			// rules, first disable all rules.
+			engine->enable_rule(all_rules, false, ruleset);
+			for(auto tag : enabled_rule_tags)
+			{
+				falco_logger::log(LOG_INFO, "Enabling rules with tag: " + tag + "\n");
+			}
+			engine->enable_rule_by_tag(enabled_rule_tags, true, ruleset);
 		}
 
 		outputs->init(config.m_json_output, config.m_notifications_rate, config.m_notifications_max_burst);
@@ -607,10 +668,15 @@ int falco_init(int argc, char **argv)
 		delete mesos_api;
 		mesos_api = 0;
 
+		if(ruleset)
+		{
+			ruleset_id = engine->find_ruleset_id(*ruleset);
+		}
 		num_evts = do_inspect(engine,
 				      outputs,
 				      inspector,
-				      stats_filename);
+				      stats_filename,
+				      ruleset_id);
 
 		duration = ((double)clock()) / CLOCKS_PER_SEC - duration;