Use new load_rules() methods to load all rules at once

This speeds up rules loading a bit because rules are only compiled
once instead of for each rules file.

This doesn't change rules validation yet. Validation needs some
additional work to handle splitting the (single) load result back into
individual results for the json/text based output.

Signed-off-by: Mark Stemm <mark.stemm@gmail.com>

.github/workflows/reusable_test_packages.yaml

@@ -44,14 +44,7 @@ jobs:
           cd falco-${{ inputs.version }}-${{ inputs.arch }}
           sudo cp -r * /
 
-      # aarch64 job run on amazon-linux-2
-      - name: Install dependencies for falco-driver-loader tests on aarch64
-        if: ${{ inputs.arch == 'aarch64' }}
-        run: |
-          sudo yum update -y
-          sudo yum install -y build-essential clang make llvm gcc dkms kernel-devel-$(uname -r)
-
-      # x86_64 job run on ubuntu-22.04
+      # x86_64 job run on ubuntu-22.04 and here we can install kernel-headers
       - name: Install dependencies for falco-driver-loader tests on x86
         if: ${{ inputs.arch == 'x86_64' }}
         run: |
@@ -70,6 +63,8 @@ jobs:
           go generate ./...
           popd
 
+      # Right now we are not able to install kernel-headers on our ARM64 self-hosted runner.
+      # For this reason, we disable the falco-driver-loader tests, which require kernel headers on the host.
       - name: Run regression tests
         run: |
           pushd submodules/falcosecurity-testing
@@ -77,7 +72,9 @@ jobs:
           if ${{ inputs.static && 'false' || 'true' }}; then
             ./build/falcoctl.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
             ./build/k8saudit.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
-            sudo ./build/falco-driver-loader.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
+            if ${{ inputs.arch == 'x86_64' && 'true' || 'false' }}; then
+                sudo ./build/falco-driver-loader.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
+            fi
           fi
           cat ./report.txt | go-junit-report -set-exit-code > report.xml
           popd

README.md

@@ -46,8 +46,7 @@ Considerations and guidance for Falco adopters:
 
 ## How to Contribute
 
-Please refer to the [contributing guide](https://github.com/falcosecurity/.github/blob/main/CONTRIBUTING.md) and the [code of conduct](https://github.com/falcosecurity/evolution/CODE_OF_CONDUCT.md) for more information on how to contribute.
-
+Please refer to the [contributing guide](https://github.com/falcosecurity/.github/blob/main/CONTRIBUTING.md) and the [code of conduct](https://github.com/falcosecurity/evolution/blob/main/CODE_OF_CONDUCT.md) for more information on how to contribute.
 
 ## Join the Community
 

cmake/modules/driver.cmake

@@ -33,8 +33,8 @@ else()
   # In case you want to test against another driver version (or branch, or commit) just pass the variable -
   # ie., `cmake -DDRIVER_VERSION=dev ..`
   if(NOT DRIVER_VERSION)
-    set(DRIVER_VERSION "6301c01b9279c3f4981df24b3e8e0d97c18f76e8")
-    set(DRIVER_CHECKSUM "SHA256=cc5c51b4a01cf83d36c3af0670a36b2c8b55f3baebc03736725dc6425898d018")
+    set(DRIVER_VERSION "6.0.0+driver")
+    set(DRIVER_CHECKSUM "SHA256=573cef7b9c69cfe1d5d8b873d2a20ad8235a2a96997df6bcebd120692dee7a91")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

cmake/modules/falcosecurity-libs.cmake

@@ -34,8 +34,8 @@ else()
   # In case you want to test against another falcosecurity/libs version (or branch, or commit) just pass the variable -
   # ie., `cmake -DFALCOSECURITY_LIBS_VERSION=dev ..` 
   if(NOT FALCOSECURITY_LIBS_VERSION)
-    set(FALCOSECURITY_LIBS_VERSION "6301c01b9279c3f4981df24b3e8e0d97c18f76e8")
-    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=cc5c51b4a01cf83d36c3af0670a36b2c8b55f3baebc03736725dc6425898d018")
+    set(FALCOSECURITY_LIBS_VERSION "0.13.0-rc1")
+    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=a75526b664bce2ba05912e056e48be39b0b1cb797b2055d107e55afbee2c8233")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

unit_tests/falco/test_configuration.cpp

@@ -88,16 +88,88 @@ TEST(Configuration, modify_yaml_fields)
 {
 	std::string key = "base_value.subvalue.subvalue2.boolean";
 	yaml_helper conf;
-	
+
     /* Get original value */
     conf.load_from_string(sample_yaml);
 	ASSERT_EQ(conf.get_scalar<bool>(key, false), true);
-	
+
     /* Modify the original value */
     conf.set_scalar<bool>(key, false);
 	ASSERT_EQ(conf.get_scalar<bool>(key, true), false);
-	
+
     /* Modify it again */
     conf.set_scalar<bool>(key, true);
 	ASSERT_EQ(conf.get_scalar<bool>(key, false), true);
 }
+
+TEST(Configuration, configuration_environment_variables)
+{
+    // Set an environment variable for testing purposes
+    std::string env_var_value = "envVarValue";
+    std::string env_var_name = "ENV_VAR";
+    std::string default_value = "default";
+    setenv(env_var_name.c_str(), env_var_value.c_str(), 1);
+    yaml_helper conf;
+
+    std::string sample_yaml =
+        "base_value:\n"
+        "    id: $ENV_VAR\n"
+        "    name: '${ENV_VAR}'\n"
+        "    string: my_string\n"
+        "    invalid: $${ENV_VAR}\n"
+		"    invalid_env: $$ENV_VAR\n"
+        "    escaped: \"${ENV_VAR}\"\n"
+        "    subvalue:\n"
+        "        subvalue2:\n"
+        "            boolean: ${UNSED_XX_X_X_VAR}\n"
+        "base_value_2:\n"
+        "    sample_list:\n"
+        "        - ${ENV_VAR}\n"
+        "        - ' ${ENV_VAR}'\n"
+        "        - $UNSED_XX_X_X_VAR\n";
+    conf.load_from_string(sample_yaml);
+
+    /* Check if the base values are defined */
+    ASSERT_TRUE(conf.is_defined("base_value"));
+    ASSERT_TRUE(conf.is_defined("base_value_2"));
+    ASSERT_FALSE(conf.is_defined("unknown_base_value"));
+
+    /* Test fetching of a regular string without any environment variable */
+    std::string base_value_string = conf.get_scalar<std::string>("base_value.string", default_value);
+    ASSERT_EQ(base_value_string, "my_string");
+
+    /* Test fetching of escaped environment variable format. Should return the string as-is after stripping the leading `$` */
+    std::string base_value_invalid = conf.get_scalar<std::string>("base_value.invalid", default_value);
+    ASSERT_EQ(base_value_invalid, "${ENV_VAR}");
+
+	/* Test fetching of invalid escaped environment variable format. Should return the string as-is */
+    std::string base_value_invalid_env = conf.get_scalar<std::string>("base_value.invalid_env", default_value);
+    ASSERT_EQ(base_value_invalid_env, "$$ENV_VAR");
+
+    /* Test fetching of strings that contain environment variables */
+    std::string base_value_id = conf.get_scalar<std::string>("base_value.id", default_value);
+    ASSERT_EQ(base_value_id, "$ENV_VAR"); // Does not follow the `${VAR}` format, so it should be treated as a regular string
+
+    std::string base_value_name = conf.get_scalar<std::string>("base_value.name", default_value);
+    ASSERT_EQ(base_value_name, env_var_value); // Proper environment variable format
+
+    std::string base_value_escaped = conf.get_scalar<std::string>("base_value.escaped", default_value);
+    ASSERT_EQ(base_value_escaped, env_var_value); // Environment variable within quotes
+
+    /* Test fetching of an undefined environment variable. Expected to return the default value.*/
+    std::string unknown_boolean = conf.get_scalar<std::string>("base_value.subvalue.subvalue2.boolean", default_value);
+    ASSERT_EQ(unknown_boolean, default_value);
+
+    /* Test fetching of environment variables from a list */
+    std::string base_value_2_list_0 = conf.get_scalar<std::string>("base_value_2.sample_list[0]", default_value);
+    ASSERT_EQ(base_value_2_list_0, env_var_value); // Proper environment variable format
+
+    std::string base_value_2_list_1 = conf.get_scalar<std::string>("base_value_2.sample_list[1]", default_value);
+    ASSERT_EQ(base_value_2_list_1, " ${ENV_VAR}"); // Environment variable preceded by a space, hence treated as a regular string
+
+    std::string base_value_2_list_2 = conf.get_scalar<std::string>("base_value_2.sample_list[2]", default_value);
+    ASSERT_EQ(base_value_2_list_2, "$UNSED_XX_X_X_VAR"); // Does not follow the `${VAR}` format, so should be treated as a regular string
+
+    /* Clear the set environment variable after testing */
+    unsetenv(env_var_name.c_str());
+}

userspace/engine/falco_engine.cpp

@@ -189,26 +189,69 @@ void falco_engine::load_rules(const std::string &rules_content, bool verbose, bo
 
 std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_content, const std::string &name)
 {
-	rule_loader::configuration cfg(rules_content, m_sources, name);
-	cfg.min_priority = m_min_priority;
-	cfg.output_extra = m_extra;
-	cfg.replace_output_container_info = m_replace_container_info;
-	cfg.default_ruleset_id = m_default_ruleset_id;
+	std::vector<std::reference_wrapper<const std::string>> rules_contents;
+	std::vector<std::reference_wrapper<const std::string>> names;
 
-	rule_loader::reader reader;
-	if (reader.read(cfg, m_rule_collector))
+	rules_contents.emplace_back(rules_content);
+	names.emplace_back(name);
+
+	return load_rules_refs(rules_contents, names);
+}
+
+std::unique_ptr<load_result> falco_engine::load_rules(const std::vector<std::string> &rules_contents,
+							       const std::vector<std::string> &names)
+{
+	std::vector<std::reference_wrapper<const std::string>> rules_contents_refs(rules_contents.begin(), rules_contents.end());
+	std::vector<std::reference_wrapper<const std::string>> names_refs(names.begin(), names.end());
+
+	return load_rules_refs(rules_contents_refs, names_refs);
+}
+
+std::unique_ptr<load_result> falco_engine::load_rules_refs(const std::vector<std::reference_wrapper<const std::string>> &rules_contents,
+							   const std::vector<std::reference_wrapper<const std::string>> &names)
+{
+	if(rules_contents.size() != names.size() ||
+	   rules_contents.size() == 0)
 	{
-		for (auto &src : m_sources)
-		{
-			src.ruleset = src.ruleset_factory->new_ruleset();
-		}
+		rule_loader::context ctx("Provided rules contents arrays");
 
-		rule_loader::compiler compiler;
-		m_rules.clear();
-		compiler.compile(cfg, m_rule_collector, m_rules);
+		std::unique_ptr<rule_loader::result> res(new rule_loader::result("Provided rules contents arrays"));
+
+		res->add_error(load_result::LOAD_ERR_FILE_READ, "Lists of rules contents and names must have same non-zero length", ctx);
+
+// Old gcc versions (e.g. 4.8.3) won't allow move elision but newer versions
+// (e.g. 10.2.1) would complain about the redundant move.
+#if __GNUC__ > 4
+		return res;
+#else
+		return std::move(res);
+#endif
 	}
 
-	if (cfg.res->successful())
+	std::unique_ptr<rule_loader::configuration> cfg;
+	for(size_t idx = 0; idx < rules_contents.size(); idx++)
+	{
+		cfg = std::make_unique<rule_loader::configuration>(rules_contents[idx], m_sources, names[idx]);
+		cfg->min_priority = m_min_priority;
+		cfg->output_extra = m_extra;
+		cfg->replace_output_container_info = m_replace_container_info;
+		cfg->default_ruleset_id = m_default_ruleset_id;
+
+		rule_loader::reader reader;
+		if (reader.read(*(cfg.get()), m_rule_collector))
+		{
+			for (auto &src : m_sources)
+			{
+				src.ruleset = src.ruleset_factory->new_ruleset();
+			}
+		}
+	}
+
+	rule_loader::compiler compiler;
+	m_rules.clear();
+	compiler.compile(*(cfg.get()), m_rule_collector, m_rules);
+
+	if (cfg->res->successful())
 	{
 		m_rule_stats_manager.clear();
 		for (const auto &r : m_rules)
@@ -217,7 +260,7 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 		}
 	}
 
-	return std::move(cfg.res);
+	return std::move(cfg->res);
 }
 
 void falco_engine::load_rules_file(const std::string &rules_filename, bool verbose, bool all_events)
@@ -233,29 +276,44 @@ void falco_engine::load_rules_file(const std::string &rules_filename, bool verbo
 
 std::unique_ptr<load_result> falco_engine::load_rules_file(const std::string &rules_filename)
 {
-	std::string rules_content;
+	std::vector<std::string> rules_filenames;
 
-	try {
-		read_file(rules_filename, rules_content);
-	}
-	catch (falco_exception &e)
+	rules_filenames.emplace_back(rules_filename);
+
+	return load_rules_files(rules_filenames);
+}
+
+std::unique_ptr<load_result> falco_engine::load_rules_files(const std::vector<std::string> &rules_filenames)
+{
+	std::vector<std::string> rules_contents;
+
+	for(auto &filename : rules_filenames)
 	{
-		rule_loader::context ctx(rules_filename);
+		std::string rules_content;
 
-		std::unique_ptr<rule_loader::result> res(new rule_loader::result(rules_filename));
+		try {
+			read_file(filename, rules_content);
+			rules_contents.emplace_back(std::move(rules_content));
+		}
+		catch (falco_exception &e)
+		{
+			rule_loader::context ctx(filename);
 
-		res->add_error(load_result::LOAD_ERR_FILE_READ, e.what(), ctx);
+			std::unique_ptr<rule_loader::result> res(new rule_loader::result(filename));
+
+			res->add_error(load_result::LOAD_ERR_FILE_READ, e.what(), ctx);
 
 // Old gcc versions (e.g. 4.8.3) won't allow move elision but newer versions
 // (e.g. 10.2.1) would complain about the redundant move.
 #if __GNUC__ > 4
-		return res;
+			return res;
 #else
-		return std::move(res);
+			return std::move(res);
 #endif
+		}
 	}
 
-	return load_rules(rules_content, rules_filename);
+	return load_rules(rules_contents, rules_filenames);
 }
 
 void falco_engine::enable_rule(const std::string &substring, bool enabled, const std::string &ruleset)

userspace/engine/falco_engine.h

@@ -23,9 +23,11 @@ limitations under the License.
 #pragma once
 
 #include <atomic>
+#include <functional>
 #include <string>
 #include <memory>
 #include <set>
+#include <vector>
 
 #include <nlohmann/json.hpp>
 
@@ -73,6 +75,15 @@ public:
 	std::unique_ptr<falco::load_result> load_rules_file(const std::string &rules_filename);
 	std::unique_ptr<falco::load_result> load_rules(const std::string &rules_content, const std::string &name);
 
+	//
+	// Identical to above, but allows providing a vector of files
+	// instead of a single file at a time. (This speeds up loading
+	// a bit because rule compilation can be deferred until all
+	// the files are read).
+	std::unique_ptr<falco::load_result> load_rules_files(const std::vector<std::string> &rules_filenames);
+	std::unique_ptr<falco::load_result> load_rules(const std::vector<std::string> &rules_contents,
+						       const std::vector<std::string> &names);
+
 	//
 	// Enable/Disable any rules matching the provided substring.
 	// If the substring is "", all rules are enabled/disabled.
@@ -273,6 +284,11 @@ public:
 
 private:
 
+	// Used by all the load_rules_* variants above with
+	// reference_wrapper to avoid copies.
+	std::unique_ptr<falco::load_result> load_rules_refs(const std::vector<std::reference_wrapper<const std::string>> &rules_contents,
+							    const std::vector<std::reference_wrapper<const std::string>> &names);
+
 	// Throws falco_exception if the file can not be read
 	void read_file(const std::string& filename, std::string& contents);
 

userspace/falco/app/actions/load_rules_files.cpp

@@ -50,11 +50,25 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 	}
 
 	std::vector<std::string> rules_contents;
+	std::vector<std::string> rules_filenames;
 	falco::load_result::rules_contents_t rc;
+	std::string filenames;
+
+	for(auto &filename : s.config->m_loaded_rules_filenames)
+	{
+		if(!filenames.empty())
+		{
+			filenames += ", ";
+		}
+
+		filenames += filename;
+
+		rules_filenames.push_back(filename);
+	}
 
 	try {
-		read_files(s.config->m_loaded_rules_filenames.begin(),
-			   s.config->m_loaded_rules_filenames.end(),
+		read_files(rules_filenames.begin(),
+			   rules_filenames.end(),
 			   rules_contents,
 			   rc);
 	}
@@ -64,25 +78,22 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 	}
 
 	std::string err = "";
-	for(auto &filename : s.config->m_loaded_rules_filenames)
+
+	falco_logger::log(LOG_INFO, "Loading rules from file(s): " + filenames);
+	std::unique_ptr<falco::load_result> res;
+
+	res = s.engine->load_rules(rules_contents, rules_filenames);
+
+	if(!res->successful())
 	{
-		falco_logger::log(LOG_INFO, "Loading rules from file " + filename + "\n");
-		std::unique_ptr<falco::load_result> res;
+		// Return the summary version as the error
+		err = res->as_string(true, rc);
+	}
 
-		res = s.engine->load_rules(rc.at(filename), filename);
-
-		if(!res->successful())
-		{
-			// Return the summary version as the error
-			err = res->as_string(true, rc);
-			break;
-		}
-
-		// If verbose is true, also print any warnings
-		if(s.options.verbose && res->has_warnings())
-		{
-			fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
-		}
+	// If verbose is true, also print any warnings
+	if(s.options.verbose && res->has_warnings())
+	{
+		fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
 	}
 
 	// note: we have an egg-and-chicken problem here. We would like to check

userspace/falco/yaml_helper.h

@@ -72,6 +72,41 @@ public:
 		get_node(node, key);
 		if(node.IsDefined())
 		{
+			std::string value = node.as<std::string>();
+
+			// Helper function to convert string to the desired type T
+			auto convert_str_to_t = [&default_value](const std::string& str) -> T {
+				std::stringstream ss(str);
+				T result;
+				if (ss >> result) return result;
+				return default_value;
+			};
+
+			// If the value starts with `$$`, check for a subsequent `{...}`
+			if (value.size() >= 3 && value[0] == '$' && value[1] == '$')
+			{
+				// If after stripping the first `$`, the string format is like `${VAR}`, treat it as a plain string and don't resolve.
+				if (value[2] == '{' && value[value.size() - 1] == '}')
+				{
+					value = value.substr(1);
+					return convert_str_to_t(value);
+				}
+				else return convert_str_to_t(value);
+			}
+
+			// Check if the value is an environment variable reference
+			if(value.size() >= 2 && value[0] == '$' && value[1] == '{' && value[value.size() - 1] == '}')
+			{
+				// Format: ${ENV_VAR_NAME}
+				std::string env_var = value.substr(2, value.size() - 3);
+
+				const char* env_value = std::getenv(env_var.c_str()); // Get the environment variable value
+				if(env_value) return convert_str_to_t(env_value);
+
+				return default_value;
+			}
+
+			// If it's not an environment variable reference, return the value as is
 			return node.as<T>();
 		}
 
@@ -118,10 +153,10 @@ private:
 	 * The provided key string can navigate the document in its
 	 * nested nodes, with arbitrary depth. The key string follows
 	 * this regular language:
-	 * 
+	 *
 	 * Key 		:= NodeKey ('.' NodeKey)*
 	 * NodeKey	:= (any)+ ('[' (integer)+ ']')*
-	 * 
+	 *
 	 * Some examples of accepted key strings:
 	 * - NodeName
 	 * - ListValue[3].subvalue
@@ -146,7 +181,7 @@ private:
 					if (i > 0 && nodeKey.empty() && key[i - 1] != '.')
 					{
 						throw std::runtime_error(
-							"Parsing error: expected '.' character at pos " 
+							"Parsing error: expected '.' character at pos "
 							+ std::to_string(i - 1));
 					}
 					nodeKey += c;
@@ -157,7 +192,7 @@ private:
 					if (nodeKey.empty())
 					{
 						throw std::runtime_error(
-							"Parsing error: unexpected character at pos " 
+							"Parsing error: unexpected character at pos "
 							+ std::to_string(i));
 					}
 					ret.reset(ret[nodeKey]);
@@ -181,7 +216,7 @@ private:
 			throw std::runtime_error("Config error at key \"" + key + "\": " + std::string(e.what()));
 		}
 	}
-	
+
 	template<typename T>
 	void get_sequence_from_node(T& ret, const YAML::Node& node) const
 	{
@@ -250,7 +285,7 @@ namespace YAML {
 				default:
 					break;
 			}
-			
+
 			return true;
 		}
 	};