new(userspace,unit_tests): introduce the possibility to split main config file into multiple config files.

The PR introduces a `includes` keyword in the config file,
that points to a list of strings (paths to other config files).

Signed-off-by: Federico Di Pierro <nierro92@gmail.com>

unit_tests/falco/test_configuration.cpp

@@ -109,6 +109,113 @@ TEST(Configuration, modify_yaml_fields)
 	ASSERT_EQ(conf.get_scalar<bool>(key, false), true);
 }
 
+TEST(Configuration, configuration_include_files)
+{
+	const std::string main_conf_yaml =
+		"includes:\n"
+		"  - conf_2.yaml\n"
+		"  - conf_3.yaml\n"
+		"foo: bar\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+	const std::string conf_yaml_2 =
+		"includes:\n"
+		"  - conf_4.yaml\n"
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo3: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	yaml_helper conf;
+
+	/* Test that a secondary config file is not able to include anything, triggering an exception. */
+	const std::string conf_yaml_4 =
+		"base_value_4:\n"
+		"    id: 4\n";
+
+	outfile.open("conf_4.yaml");
+	outfile << conf_yaml_4;
+	outfile.close();
+
+	ASSERT_ANY_THROW(conf.load_from_file("main.yaml"));
+
+	/* Test that every included config file was correctly parsed */
+	const std::string conf_yaml_2_ok =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2_ok;
+	outfile.close();
+
+	ASSERT_NO_THROW(conf.load_from_file("main.yaml"));
+
+	ASSERT_TRUE(conf.is_defined("foo"));
+	ASSERT_EQ(conf.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(conf.is_defined("base_value.id"));
+	ASSERT_EQ(conf.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(conf.is_defined("base_value.name"));
+	ASSERT_EQ(conf.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(conf.is_defined("foo2"));
+	ASSERT_EQ(conf.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(conf.is_defined("base_value_2.id"));
+	ASSERT_EQ(conf.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(conf.is_defined("foo3"));
+	ASSERT_EQ(conf.get_scalar<std::string>("foo3", ""), "bar3");
+	ASSERT_TRUE(conf.is_defined("base_value_3.id"));
+	ASSERT_EQ(conf.get_scalar<int>("base_value_3.id", 0), 3);
+	ASSERT_TRUE(conf.is_defined("base_value_3.name"));
+	ASSERT_EQ(conf.get_scalar<std::string>("base_value_3.name", ""), "foo3");
+
+	/* Test that included config files are not able to override configs from main file */
+	const std::string conf_yaml_3_override =
+		"base_value:\n"
+		"    id: 3\n";
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3_override;
+	outfile.close();
+
+	ASSERT_ANY_THROW(conf.load_from_file("main.yaml"));
+
+	/* Test that including an unexistent file triggers an exception */
+	const std::string main_conf_unexistent_yaml =
+		"includes:\n"
+		"  - conf_5.yaml\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+
+	outfile.open("main.yaml");
+	outfile << main_conf_unexistent_yaml;
+	outfile.close();
+
+	ASSERT_ANY_THROW(conf.load_from_file("main.yaml"));
+
+	// Cleanup everything
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+	std::filesystem::remove("conf_4.yaml");
+}
+
 TEST(Configuration, configuration_environment_variables)
 {
     // Set an environment variable for testing purposes

userspace/falco/yaml_helper.h

@@ -31,6 +31,7 @@ limitations under the License.
 #include <set>
 #include <iostream>
 #include <fstream>
+#include <filesystem>
 
 #include "config_falco.h"
 
@@ -83,7 +84,7 @@ public:
 	void load_from_string(const std::string& input)
 	{
 		m_root = YAML::Load(input);
-		pre_process_env_vars();
+		pre_process_env_vars(m_root);
 	}
 
 	/**
@@ -91,8 +92,52 @@ public:
 	*/
 	void load_from_file(const std::string& path)
 	{
-		m_root = YAML::LoadFile(path);
+		m_root = load_from_file_int(path);
-		pre_process_env_vars();
+
+		const auto ppath = std::filesystem::path(path);
+		const auto config_folder = ppath.std::filesystem::path::parent_path();
+		// Parse files to be included
+		std::vector<std::string> include_files;
+		get_sequence<std::vector<std::string>>(include_files, "includes");
+		for(const std::string& include_file : include_files)
+		{
+			// If user specifies a relative include file,
+			// make it relative to main config file folder,
+			// instead of cwd.
+			auto include_file_path = std::filesystem::path(include_file);
+			if (!include_file_path.is_absolute())
+			{
+				include_file_path = config_folder;
+				include_file_path += include_file;
+			}
+			auto loaded_nodes = load_from_file_int(include_file_path.string());
+			for(auto n : loaded_nodes)
+			{
+				/*
+		                 * To avoid recursion hell,
+				 * we don't support `includes` directives from included config files
+				 * (that use load_from_file_int recursively).
+				 */
+				const auto &key = n.first.Scalar();
+				if (key == "includes")
+				{
+					throw std::runtime_error("Config error: 'includes' directive in included config file " + include_file + ".");
+				}
+
+				YAML::Node node;
+				get_node(node, key);
+				if (!node.IsDefined())
+				{
+					// There was no such node in root config file; proceed.
+					node = n.second;
+				}
+				else
+				{
+					throw std::runtime_error("Config error: included config files cannot override root config nodes: "
+								 + include_file + " tried to override '" + key + "'.");
+				}
+			}
+		}
 	}
 
 	/**
@@ -153,13 +198,20 @@ public:
 private:
 	YAML::Node m_root;
 
+	YAML::Node load_from_file_int(const std::string& path)
+	{
+		auto root = YAML::LoadFile(path);
+		pre_process_env_vars(root);
+		return root;
+	}
+
 	/*
 	 * When loading a yaml file,
 	 * we immediately pre process all scalar values through a visitor private API,
 	 * and resolve any "${env_var}" to its value;
 	 * moreover, any "$${str}" is resolved to simply "${str}".
 	 */
-	void pre_process_env_vars()
+	void pre_process_env_vars(YAML::Node& root)
 	{
 		yaml_visitor([](YAML::Node &scalar) {
 				auto value = scalar.as<std::string>();
@@ -215,7 +267,7 @@ private:
 					start_pos = value.find("$", start_pos);
 				}
 				scalar = value;
-			})(m_root);
+			})(root);
 	}
 
 	/**