From b3ebf9f57e70e576ead67c34e8eafb8bcceaff0b Mon Sep 17 00:00:00 2001 From: Federico Di Pierro Date: Thu, 18 Jan 2024 15:14:24 +0100 Subject: [PATCH] new(userspace,unit_tests): introduce the possibility to split main config file into multiple config files. The PR introduces a `includes` keyword in the config file, that points to a list of strings (paths to other config files). Signed-off-by: Federico Di Pierro --- unit_tests/falco/test_configuration.cpp | 107 ++++++++++++++++++++++++ userspace/falco/yaml_helper.h | 62 ++++++++++++-- 2 files changed, 164 insertions(+), 5 deletions(-) diff --git a/unit_tests/falco/test_configuration.cpp b/unit_tests/falco/test_configuration.cpp index 5abb13e0..ea73088e 100644 --- a/unit_tests/falco/test_configuration.cpp +++ b/unit_tests/falco/test_configuration.cpp @@ -109,6 +109,113 @@ TEST(Configuration, modify_yaml_fields) ASSERT_EQ(conf.get_scalar(key, false), true); } +TEST(Configuration, configuration_include_files) +{ + const std::string main_conf_yaml = + "includes:\n" + " - conf_2.yaml\n" + " - conf_3.yaml\n" + "foo: bar\n" + "base_value:\n" + " id: 1\n" + " name: foo\n"; + const std::string conf_yaml_2 = + "includes:\n" + " - conf_4.yaml\n" + "foo2: bar2\n" + "base_value_2:\n" + " id: 2\n"; + const std::string conf_yaml_3 = + "foo3: bar3\n" + "base_value_3:\n" + " id: 3\n" + " name: foo3\n"; + + std::ofstream outfile("main.yaml"); + outfile << main_conf_yaml; + outfile.close(); + + outfile.open("conf_2.yaml"); + outfile << conf_yaml_2; + outfile.close(); + + outfile.open("conf_3.yaml"); + outfile << conf_yaml_3; + outfile.close(); + + yaml_helper conf; + + /* Test that a secondary config file is not able to include anything, triggering an exception. */ + const std::string conf_yaml_4 = + "base_value_4:\n" + " id: 4\n"; + + outfile.open("conf_4.yaml"); + outfile << conf_yaml_4; + outfile.close(); + + ASSERT_ANY_THROW(conf.load_from_file("main.yaml")); + + /* Test that every included config file was correctly parsed */ + const std::string conf_yaml_2_ok = + "foo2: bar2\n" + "base_value_2:\n" + " id: 2\n"; + + outfile.open("conf_2.yaml"); + outfile << conf_yaml_2_ok; + outfile.close(); + + ASSERT_NO_THROW(conf.load_from_file("main.yaml")); + + ASSERT_TRUE(conf.is_defined("foo")); + ASSERT_EQ(conf.get_scalar("foo", ""), "bar"); + ASSERT_TRUE(conf.is_defined("base_value.id")); + ASSERT_EQ(conf.get_scalar("base_value.id", 0), 1); + ASSERT_TRUE(conf.is_defined("base_value.name")); + ASSERT_EQ(conf.get_scalar("base_value.name", ""), "foo"); + ASSERT_TRUE(conf.is_defined("foo2")); + ASSERT_EQ(conf.get_scalar("foo2", ""), "bar2"); + ASSERT_TRUE(conf.is_defined("base_value_2.id")); + ASSERT_EQ(conf.get_scalar("base_value_2.id", 0), 2); + ASSERT_TRUE(conf.is_defined("foo3")); + ASSERT_EQ(conf.get_scalar("foo3", ""), "bar3"); + ASSERT_TRUE(conf.is_defined("base_value_3.id")); + ASSERT_EQ(conf.get_scalar("base_value_3.id", 0), 3); + ASSERT_TRUE(conf.is_defined("base_value_3.name")); + ASSERT_EQ(conf.get_scalar("base_value_3.name", ""), "foo3"); + + /* Test that included config files are not able to override configs from main file */ + const std::string conf_yaml_3_override = + "base_value:\n" + " id: 3\n"; + outfile.open("conf_3.yaml"); + outfile << conf_yaml_3_override; + outfile.close(); + + ASSERT_ANY_THROW(conf.load_from_file("main.yaml")); + + /* Test that including an unexistent file triggers an exception */ + const std::string main_conf_unexistent_yaml = + "includes:\n" + " - conf_5.yaml\n" + "base_value:\n" + " id: 1\n" + " name: foo\n"; + + outfile.open("main.yaml"); + outfile << main_conf_unexistent_yaml; + outfile.close(); + + ASSERT_ANY_THROW(conf.load_from_file("main.yaml")); + + // Cleanup everything + std::filesystem::remove("main.yaml"); + std::filesystem::remove("conf_2.yaml"); + std::filesystem::remove("conf_3.yaml"); + std::filesystem::remove("conf_4.yaml"); +} + TEST(Configuration, configuration_environment_variables) { // Set an environment variable for testing purposes diff --git a/userspace/falco/yaml_helper.h b/userspace/falco/yaml_helper.h index 7b0dcab0..9b0d9b99 100644 --- a/userspace/falco/yaml_helper.h +++ b/userspace/falco/yaml_helper.h @@ -31,6 +31,7 @@ limitations under the License. #include #include #include +#include #include "config_falco.h" @@ -83,7 +84,7 @@ public: void load_from_string(const std::string& input) { m_root = YAML::Load(input); - pre_process_env_vars(); + pre_process_env_vars(m_root); } /** @@ -91,8 +92,52 @@ public: */ void load_from_file(const std::string& path) { - m_root = YAML::LoadFile(path); - pre_process_env_vars(); + m_root = load_from_file_int(path); + + const auto ppath = std::filesystem::path(path); + const auto config_folder = ppath.std::filesystem::path::parent_path(); + // Parse files to be included + std::vector include_files; + get_sequence>(include_files, "includes"); + for(const std::string& include_file : include_files) + { + // If user specifies a relative include file, + // make it relative to main config file folder, + // instead of cwd. + auto include_file_path = std::filesystem::path(include_file); + if (!include_file_path.is_absolute()) + { + include_file_path = config_folder; + include_file_path += include_file; + } + auto loaded_nodes = load_from_file_int(include_file_path.string()); + for(auto n : loaded_nodes) + { + /* + * To avoid recursion hell, + * we don't support `includes` directives from included config files + * (that use load_from_file_int recursively). + */ + const auto &key = n.first.Scalar(); + if (key == "includes") + { + throw std::runtime_error("Config error: 'includes' directive in included config file " + include_file + "."); + } + + YAML::Node node; + get_node(node, key); + if (!node.IsDefined()) + { + // There was no such node in root config file; proceed. + node = n.second; + } + else + { + throw std::runtime_error("Config error: included config files cannot override root config nodes: " + + include_file + " tried to override '" + key + "'."); + } + } + } } /** @@ -153,13 +198,20 @@ public: private: YAML::Node m_root; + YAML::Node load_from_file_int(const std::string& path) + { + auto root = YAML::LoadFile(path); + pre_process_env_vars(root); + return root; + } + /* * When loading a yaml file, * we immediately pre process all scalar values through a visitor private API, * and resolve any "${env_var}" to its value; * moreover, any "$${str}" is resolved to simply "${str}". */ - void pre_process_env_vars() + void pre_process_env_vars(YAML::Node& root) { yaml_visitor([](YAML::Node &scalar) { auto value = scalar.as(); @@ -215,7 +267,7 @@ private: start_pos = value.find("$", start_pos); } scalar = value; - })(m_root); + })(root); } /**