Provide the entire compile output to ruleset vs individual add()s

In order to support external rules loaders that may extend the falco
rules format with new top level objects, move away from providing
individual filter objects to the filter_ruleset via calls to add().

Instead, pass the entire compile output returned by the compiler to
the ruleset using a new method add_compile_output(). Custom users can
then cast back the compile output to the appropriate derived class for
use in the ruleset.

Move the declaration of the compile output to a standalone class so it
can be used by rulesets without including the entire rules loader
header files, and add a new factory method new_compile_output() to the
compiler so it can create a derived class if necessary.

This change is
backwards-compatible with existing rulesets, as the default
implementation of add_compile_output() simply iterates over rules and
calls add() for each rule.

This change also speeds up rule loading. Previously, each rule
condition was compiled twice:

1. First, in the compiler, to see if it was valid.
2. Second, in the falco engine before providing each rule to the
ruleset.

Add the compiled filter to the falco_rule object instead of throwing
it away in the compiler.

Signed-off-by: Mark Stemm <mark.stemm@gmail.com>

userspace/engine/falco_engine.cpp

@@ -205,8 +205,13 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 		// clear the rules known by the engine and each ruleset
 		m_rules.clear();
 		for (auto &src : m_sources)
+
+		// add rules to each ruleset
 		{
 			src.ruleset = src.ruleset_factory->new_ruleset();
+			src.ruleset->add_compile_output(*(m_last_compile_output.get()),
+							m_min_priority,
+							src.name);
 		}
 
 		// add rules to the engine and the rulesets
@@ -225,15 +230,9 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 				throw falco_exception("can't find internal rule info at name: " + name);
 			}
 
-			// the rule is ok, we can add it to the engine and the rulesets
-			// note: the compiler should guarantee that the rule's condition
-			// is a valid sinsp filter
 			auto source = find_source(rule.source);
-			std::shared_ptr<gen_event_filter> filter(
-				sinsp_filter_compiler(source->filter_factory, rule.condition.get()).compile());
 			auto rule_id = m_rules.insert(rule, rule.name);
 			m_rules.at(rule_id)->id = rule_id;
-			source->ruleset->add(rule, filter, rule.condition);
 
 			// By default rules are enabled/disabled for the default ruleset
 			if(info->enabled)

userspace/engine/falco_engine.h

@@ -416,7 +416,7 @@ private:
 	std::map<std::string, uint16_t> m_known_rulesets;
 	falco_common::priority_type m_min_priority;
 
-	std::unique_ptr<rule_loader::compiler::compile_output> m_last_compile_output;
+	std::unique_ptr<rule_loader::compile_output> m_last_compile_output;
 
 	//
 	// Here's how the sampling ratio and multiplier influence

userspace/engine/falco_rule.h

@@ -83,4 +83,5 @@ struct falco_rule
 	std::set<std::string> exception_fields;
 	falco_common::priority_type priority;
 	std::shared_ptr<libsinsp::filter::ast::expr> condition;
+	std::shared_ptr<gen_event_filter> filter;
 };

userspace/engine/filter_ruleset.h

@@ -18,6 +18,7 @@ limitations under the License.
 #pragma once
 
 #include "falco_rule.h"
+#include "rule_loader_compile_output.h"
 #include <filter/ast.h>
 #include <filter.h>
 #include <event.h>
@@ -50,6 +51,35 @@ public:
 		std::shared_ptr<gen_event_filter> filter,
 		std::shared_ptr<libsinsp::filter::ast::expr> condition) = 0;
 
+	/*!
+		\brief Adds all rules contained in the provided
+		rule_loader::compile_output struct. Only
+		those rules with the provided source and those rules
+		with priority >= min_priority should be added. The
+		intent is that this replaces add(). However, we retain
+		add() for backwards compatibility. Any rules added via
+		add() are also added to this ruleset. The default
+		implementation iterates over rules and calls add(),
+		but can be overridden.
+		\param rule The compile output.
+		\param min_priority Only add rules with priority above this priority.
+		\param source Only add rules with source equal to this source.
+	*/
+	virtual void add_compile_output(
+		const rule_loader::compile_output& compile_output,
+		falco_common::priority_type min_priority,
+		const std::string& source)
+	{
+		for (const auto& rule : compile_output.rules)
+		{
+			if(rule.priority <= min_priority &&
+			   rule.source == source)
+			{
+				add(rule, rule.filter, rule.condition);
+			}
+		}
+	};
+
 	/*!
 		\brief Erases the internal state. All rules are disabled in each
 		ruleset, and all the rules defined with add() are removed.

userspace/engine/rule_loader_compile_output.h

@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+#include "indexed_vector.h"
+#include "falco_rule.h"
+
+namespace rule_loader
+{
+	struct compile_output
+	{
+		compile_output() = default;
+		virtual ~compile_output() = default;
+		compile_output(compile_output&&) = default;
+		compile_output& operator = (compile_output&&) = default;
+		compile_output(const compile_output&) = default;
+		compile_output& operator = (const compile_output&) = default;
+
+		indexed_vector<falco_list> lists;
+		indexed_vector<falco_macro> macros;
+		indexed_vector<falco_rule> rules;
+	};
+};
+

userspace/engine/rule_loader_compiler.cpp

@@ -529,6 +529,11 @@ void rule_loader::compiler::compile_rule_infos(
 	}
 }
 
+std::unique_ptr<rule_loader::compile_output> rule_loader::compiler::new_compile_output()
+{
+	return std::make_unique<compile_output>();
+}
+
 void rule_loader::compiler::compile(
 		configuration& cfg,
 		const collector& col,

userspace/engine/rule_loader_compiler.h

@@ -18,6 +18,7 @@ limitations under the License.
 #pragma once
 
 #include "rule_loader.h"
+#include "rule_loader_compile_output.h"
 #include "rule_loader_collector.h"
 #include "indexed_vector.h"
 #include "falco_rule.h"
@@ -31,23 +32,6 @@ namespace rule_loader
 class compiler
 {
 public:
-	/*!
-		\brief The output of a compilation.
-	*/
-	struct compile_output
-	{
-		compile_output() = default;
-		virtual ~compile_output() = default;
-		compile_output(compile_output&&) = default;
-		compile_output& operator = (compile_output&&) = default;
-		compile_output(const compile_output&) = default;
-		compile_output& operator = (const compile_output&) = default;
-
-		indexed_vector<falco_list> lists;
-		indexed_vector<falco_macro> macros;
-		indexed_vector<falco_rule> rules;
-	};
-
 	compiler() = default;
 	virtual ~compiler() = default;
 	compiler(compiler&&) = default;
@@ -55,6 +39,10 @@ public:
 	compiler(const compiler&) = default;
 	compiler& operator = (const compiler&) = default;
 
+	// Return a new result object, suitable for passing to
+	// compile().
+        virtual std::unique_ptr<compile_output> new_compile_output();
+
 	/*!
 		\brief Compiles a list of falco rules
 	*/