Use new load_rules() methods to load all rules at once

This speeds up rules loading a bit because rules are only compiled
once instead of for each rules file.

This doesn't change rules validation yet. Validation needs some
additional work to handle splitting the (single) load result back into
individual results for the json/text based output.

Signed-off-by: Mark Stemm <mark.stemm@gmail.com>

userspace/engine/falco_engine.cpp

@@ -189,26 +189,69 @@ void falco_engine::load_rules(const std::string &rules_content, bool verbose, bo
 
 std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_content, const std::string &name)
 {
-	rule_loader::configuration cfg(rules_content, m_sources, name);
+	std::vector<std::reference_wrapper<const std::string>> rules_contents;
-	cfg.min_priority = m_min_priority;
+	std::vector<std::reference_wrapper<const std::string>> names;
-	cfg.output_extra = m_extra;
-	cfg.replace_output_container_info = m_replace_container_info;
-	cfg.default_ruleset_id = m_default_ruleset_id;
 
-	rule_loader::reader reader;
+	rules_contents.emplace_back(rules_content);
-	if (reader.read(cfg, m_rule_collector))
+	names.emplace_back(name);
+
+	return load_rules_refs(rules_contents, names);
+}
+
+std::unique_ptr<load_result> falco_engine::load_rules(const std::vector<std::string> &rules_contents,
+							       const std::vector<std::string> &names)
+{
+	std::vector<std::reference_wrapper<const std::string>> rules_contents_refs(rules_contents.begin(), rules_contents.end());
+	std::vector<std::reference_wrapper<const std::string>> names_refs(names.begin(), names.end());
+
+	return load_rules_refs(rules_contents_refs, names_refs);
+}
+
+std::unique_ptr<load_result> falco_engine::load_rules_refs(const std::vector<std::reference_wrapper<const std::string>> &rules_contents,
+							   const std::vector<std::reference_wrapper<const std::string>> &names)
+{
+	if(rules_contents.size() != names.size() ||
+	   rules_contents.size() == 0)
 	{
-		for (auto &src : m_sources)
+		rule_loader::context ctx("Provided rules contents arrays");
-		{
-			src.ruleset = src.ruleset_factory->new_ruleset();
-		}
 
-		rule_loader::compiler compiler;
+		std::unique_ptr<rule_loader::result> res(new rule_loader::result("Provided rules contents arrays"));
-		m_rules.clear();
+
-		compiler.compile(cfg, m_rule_collector, m_rules);
+		res->add_error(load_result::LOAD_ERR_FILE_READ, "Lists of rules contents and names must have same non-zero length", ctx);
+
+// Old gcc versions (e.g. 4.8.3) won't allow move elision but newer versions
+// (e.g. 10.2.1) would complain about the redundant move.
+#if __GNUC__ > 4
+		return res;
+#else
+		return std::move(res);
+#endif
 	}
 
-	if (cfg.res->successful())
+	std::unique_ptr<rule_loader::configuration> cfg;
+	for(size_t idx = 0; idx < rules_contents.size(); idx++)
+	{
+		cfg = std::make_unique<rule_loader::configuration>(rules_contents[idx], m_sources, names[idx]);
+		cfg->min_priority = m_min_priority;
+		cfg->output_extra = m_extra;
+		cfg->replace_output_container_info = m_replace_container_info;
+		cfg->default_ruleset_id = m_default_ruleset_id;
+
+		rule_loader::reader reader;
+		if (reader.read(*(cfg.get()), m_rule_collector))
+		{
+			for (auto &src : m_sources)
+			{
+				src.ruleset = src.ruleset_factory->new_ruleset();
+			}
+		}
+	}
+
+	rule_loader::compiler compiler;
+	m_rules.clear();
+	compiler.compile(*(cfg.get()), m_rule_collector, m_rules);
+
+	if (cfg->res->successful())
 	{
 		m_rule_stats_manager.clear();
 		for (const auto &r : m_rules)
@@ -217,7 +260,7 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 		}
 	}
 
-	return std::move(cfg.res);
+	return std::move(cfg->res);
 }
 
 void falco_engine::load_rules_file(const std::string &rules_filename, bool verbose, bool all_events)
@@ -233,29 +276,44 @@ void falco_engine::load_rules_file(const std::string &rules_filename, bool verbo
 
 std::unique_ptr<load_result> falco_engine::load_rules_file(const std::string &rules_filename)
 {
-	std::string rules_content;
+	std::vector<std::string> rules_filenames;
 
-	try {
+	rules_filenames.emplace_back(rules_filename);
-		read_file(rules_filename, rules_content);
+
-	}
+	return load_rules_files(rules_filenames);
-	catch (falco_exception &e)
+}
+
+std::unique_ptr<load_result> falco_engine::load_rules_files(const std::vector<std::string> &rules_filenames)
+{
+	std::vector<std::string> rules_contents;
+
+	for(auto &filename : rules_filenames)
 	{
-		rule_loader::context ctx(rules_filename);
+		std::string rules_content;
 
-		std::unique_ptr<rule_loader::result> res(new rule_loader::result(rules_filename));
+		try {
+			read_file(filename, rules_content);
+			rules_contents.emplace_back(std::move(rules_content));
+		}
+		catch (falco_exception &e)
+		{
+			rule_loader::context ctx(filename);
 
-		res->add_error(load_result::LOAD_ERR_FILE_READ, e.what(), ctx);
+			std::unique_ptr<rule_loader::result> res(new rule_loader::result(filename));
+
+			res->add_error(load_result::LOAD_ERR_FILE_READ, e.what(), ctx);
 
 // Old gcc versions (e.g. 4.8.3) won't allow move elision but newer versions
 // (e.g. 10.2.1) would complain about the redundant move.
 #if __GNUC__ > 4
-		return res;
+			return res;
 #else
-		return std::move(res);
+			return std::move(res);
 #endif
+		}
 	}
 
-	return load_rules(rules_content, rules_filename);
+	return load_rules(rules_contents, rules_filenames);
 }
 
 void falco_engine::enable_rule(const std::string &substring, bool enabled, const std::string &ruleset)

userspace/engine/falco_engine.h

@@ -23,9 +23,11 @@ limitations under the License.
 #pragma once
 
 #include <atomic>
+#include <functional>
 #include <string>
 #include <memory>
 #include <set>
+#include <vector>
 
 #include <nlohmann/json.hpp>
 
@@ -73,6 +75,15 @@ public:
 	std::unique_ptr<falco::load_result> load_rules_file(const std::string &rules_filename);
 	std::unique_ptr<falco::load_result> load_rules(const std::string &rules_content, const std::string &name);
 
+	//
+	// Identical to above, but allows providing a vector of files
+	// instead of a single file at a time. (This speeds up loading
+	// a bit because rule compilation can be deferred until all
+	// the files are read).
+	std::unique_ptr<falco::load_result> load_rules_files(const std::vector<std::string> &rules_filenames);
+	std::unique_ptr<falco::load_result> load_rules(const std::vector<std::string> &rules_contents,
+						       const std::vector<std::string> &names);
+
 	//
 	// Enable/Disable any rules matching the provided substring.
 	// If the substring is "", all rules are enabled/disabled.
@@ -273,6 +284,11 @@ public:
 
 private:
 
+	// Used by all the load_rules_* variants above with
+	// reference_wrapper to avoid copies.
+	std::unique_ptr<falco::load_result> load_rules_refs(const std::vector<std::reference_wrapper<const std::string>> &rules_contents,
+							    const std::vector<std::reference_wrapper<const std::string>> &names);
+
 	// Throws falco_exception if the file can not be read
 	void read_file(const std::string& filename, std::string& contents);
 

userspace/falco/app/actions/load_rules_files.cpp

@@ -50,11 +50,25 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 	}
 
 	std::vector<std::string> rules_contents;
+	std::vector<std::string> rules_filenames;
 	falco::load_result::rules_contents_t rc;
+	std::string filenames;
+
+	for(auto &filename : s.config->m_loaded_rules_filenames)
+	{
+		if(!filenames.empty())
+		{
+			filenames += ", ";
+		}
+
+		filenames += filename;
+
+		rules_filenames.push_back(filename);
+	}
 
 	try {
-		read_files(s.config->m_loaded_rules_filenames.begin(),
+		read_files(rules_filenames.begin(),
-			   s.config->m_loaded_rules_filenames.end(),
+			   rules_filenames.end(),
 			   rules_contents,
 			   rc);
 	}
@@ -64,25 +78,22 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 	}
 
 	std::string err = "";
-	for(auto &filename : s.config->m_loaded_rules_filenames)
+
+	falco_logger::log(LOG_INFO, "Loading rules from file(s): " + filenames);
+	std::unique_ptr<falco::load_result> res;
+
+	res = s.engine->load_rules(rules_contents, rules_filenames);
+
+	if(!res->successful())
 	{
-		falco_logger::log(LOG_INFO, "Loading rules from file " + filename + "\n");
+		// Return the summary version as the error
-		std::unique_ptr<falco::load_result> res;
+		err = res->as_string(true, rc);
+	}
 
-		res = s.engine->load_rules(rc.at(filename), filename);
+	// If verbose is true, also print any warnings
-
+	if(s.options.verbose && res->has_warnings())
-		if(!res->successful())
+	{
-		{
+		fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
-			// Return the summary version as the error
-			err = res->as_string(true, rc);
-			break;
-		}
-
-		// If verbose is true, also print any warnings
-		if(s.options.verbose && res->has_warnings())
-		{
-			fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
-		}
 	}
 
 	// note: we have an egg-and-chicken problem here. We would like to check