new(userspace,unit_tests)!: add a way to specify merge-strategy for config_files.

By default we now use the `append` merge-strategy:
* existing sequence keys will be appended
* existing scalar keys will be overridden
* non-existing keys will be added

We also have an `override` merge-strategy:
* existing keys will be overridden
* non-existing keys will be added

Finally, there is an `add-only` merge-strategy:
* existing keys will be ignored
* non-existing keys will be added

Signed-off-by: Federico Di Pierro <nierro92@gmail.com>

falco.yaml

@@ -143,6 +143,21 @@
 # Also, nested include is not allowed, ie: included config files won't be able to include other config files.
 #
 # Like for 'rules_files', specifying a folder will load all the configs files present in it in a lexicographical order.
+#
+# 3 merge-strategies are available: `append`, `override` and `add-only`.
+# The default merge-strategy is `append`:
+#   * existing sequence keys will be appended
+#   * existing scalar keys will be overridden
+#   * non-existing keys will be added
+# To enable the `override` merge-strategy, `@` must be prepended to the entry, eg: "@/$HOME/override.yaml".
+#   * existing keys will be overridden
+#   * non-existing keys will be added
+#   Please note that since `@` is a reserved character by yaml spec, you need to quote the whole string.
+# To enable the `add-only` merge-strategy, `+` must be prepended to the entry, eg: "+/$HOME/append.yaml".
+#   * existing keys will be ignored
+#   * non-existing keys will be added
+#
+# When a merge-strategy is enabled for a folder entry, all of the included config files will use that merge-strategy.
 config_files:
   - /etc/falco/config.d
 

unit_tests/falco/test_configuration_config_files.cpp

@@ -245,6 +245,190 @@ TEST(Configuration, configuration_config_files_override) {
 	std::filesystem::remove("conf_3.yaml");
 }
 
+TEST(Configuration, configuration_config_files_sequence) {
+	/* Test that included config files are able to override configs from main file */
+	const std::string main_conf_yaml = yaml_helper::configs_key +
+	                                   ":\n"
+	                                   "  - conf_2.yaml\n"  // default merge-strategy: append
+	                                   "  - conf_3.yaml\n"
+	                                   "foo: [ bar ]\n"
+	                                   "base_value:\n"
+	                                   "    id: 1\n"
+	                                   "    name: foo\n";
+	const std::string conf_yaml_2 =
+	        "foo: [ bar2 ]\n"  // append to foo sequence
+	        "base_value_2:\n"
+	        "    id: 2\n";
+	const std::string conf_yaml_3 =
+	        "base_value:\n"  // override base_value
+	        "    id: 3\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(res.size(), 3);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	std::vector<std::string> foos;
+	auto expected_foos = std::vector<std::string>{"bar", "bar2"};
+	ASSERT_NO_THROW(falco_config.m_config.get_sequence<std::vector<std::string>>(foos, "foo"));
+	ASSERT_EQ(foos.size(), 2);  // 2 elements in `foo` sequence because we appended to it
+	for(size_t i = 0; i < foos.size(); ++i) {
+		EXPECT_EQ(foos[i], expected_foos[i])
+		        << "Vectors foo's and expected_foo's differ at index " << i;
+	}
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 3);  // overridden!
+	ASSERT_FALSE(falco_config.m_config.is_defined(
+	        "base_value.name"));  // no more present since entire `base_value` block was overridden
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_FALSE(falco_config.m_config.is_defined("base_value_3.id"));  // not defined
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+}
+
+TEST(Configuration, configuration_config_files_sequence_override) {
+	/* Test that included config files are able to override configs from main file */
+	const std::string main_conf_yaml = yaml_helper::configs_key +
+	                                   ":\n"
+	                                   "  - '@conf_2.yaml'\n"  // merge-strategy: override
+	                                   "  - conf_3.yaml\n"
+	                                   "foo: [ bar ]\n"
+	                                   "base_value:\n"
+	                                   "    id: 1\n"
+	                                   "    name: foo\n";
+	const std::string conf_yaml_2 =
+	        "foo: [ bar2 ]\n"  // override foo sequence
+	        "base_value_2:\n"
+	        "    id: 2\n";
+	const std::string conf_yaml_3 =
+	        "base_value:\n"  // override base_value
+	        "    id: 3\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(res.size(), 3);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	std::vector<std::string> foos;
+	auto expected_foos = std::vector<std::string>{"bar2"};
+	ASSERT_NO_THROW(falco_config.m_config.get_sequence<std::vector<std::string>>(foos, "foo"));
+	ASSERT_EQ(foos.size(), 1);  // one element in `foo` sequence because we overrode it
+	for(size_t i = 0; i < foos.size(); ++i) {
+		EXPECT_EQ(foos[i], expected_foos[i])
+		        << "Vectors foo's and expected_foo's differ at index " << i;
+	}
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 3);  // overridden!
+	ASSERT_FALSE(falco_config.m_config.is_defined(
+	        "base_value.name"));  // no more present since entire `base_value` block was overridden
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_FALSE(falco_config.m_config.is_defined("base_value_3.id"));  // not defined
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+}
+
+TEST(Configuration, configuration_config_files_sequence_addonly) {
+	/* Test that included config files are able to override configs from main file */
+	const std::string main_conf_yaml = yaml_helper::configs_key +
+	                                   ":\n"
+	                                   "  - +conf_2.yaml\n"  // merge-strategy: add-only
+	                                   "  - conf_3.yaml\n"
+	                                   "foo: [ bar ]\n"
+	                                   "base_value:\n"
+	                                   "    id: 1\n"
+	                                   "    name: foo\n";
+	const std::string conf_yaml_2 =
+	        "foo: [ bar2 ]\n"  // ignored: add-only strategy
+	        "base_value_2:\n"
+	        "    id: 2\n";
+	const std::string conf_yaml_3 =
+	        "base_value:\n"  // override base_value
+	        "    id: 3\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(res.size(), 3);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	std::vector<std::string> foos;
+	auto expected_foos =
+	        std::vector<std::string>{"bar"};  // bar2 is ignored because of merge-strategy: add-only
+	ASSERT_NO_THROW(falco_config.m_config.get_sequence<std::vector<std::string>>(foos, "foo"));
+	ASSERT_EQ(foos.size(), 1);  // one element in `foo` sequence because we overrode it
+	for(size_t i = 0; i < foos.size(); ++i) {
+		EXPECT_EQ(foos[i], expected_foos[i])
+		        << "Vectors foo's and expected_foo's differ at index " << i;
+	}
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 3);  // overridden!
+	ASSERT_FALSE(falco_config.m_config.is_defined(
+	        "base_value.name"));  // no more present since entire `base_value` block was overridden
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_FALSE(falco_config.m_config.is_defined("base_value_3.id"));  // not defined
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+}
+
 TEST(Configuration, configuration_config_files_unexistent) {
 	/* Test that including an unexistent file just skips it */
 	const std::string main_conf_yaml = yaml_helper::configs_key +

userspace/engine/yaml_helper.h

@@ -85,6 +85,27 @@ public:
 	inline static const std::string validation_failed = "failed";
 	inline static const std::string validation_none = "none";
 
+	enum include_files_strategy {
+		STRATEGY_APPEND,  // append to existing sequence keys, override scalar keys and add new ones
+		STRATEGY_OVERRIDE,  // override existing keys (sequences too) and add new ones
+		STRATEGY_ADDONLY    // only add new keys and ignore existing ones
+	};
+
+	static inline enum include_files_strategy get_include_file_strategy(
+	        std::string& include_file_name) {
+		if(include_file_name.length() > 0) {
+			if(include_file_name[0] == '+') {
+				include_file_name.erase(0, 1);
+				return STRATEGY_ADDONLY;
+			}
+			if(include_file_name[0] == '@') {
+				include_file_name.erase(0, 1);
+				return STRATEGY_OVERRIDE;
+			}
+		}
+		return STRATEGY_APPEND;
+	}
+
 	/**
 	 * Load all the YAML document represented by the input string.
 	 * Since this is used by rule loader, does not process env vars.
@@ -137,6 +158,7 @@ public:
 	}
 
 	void include_config_file(const std::string& include_file_path,
+	                         enum include_files_strategy strategy = STRATEGY_APPEND,
 	                         const nlohmann::json& schema = {},
 	                         std::vector<std::string>* schema_warnings = nullptr) {
 		auto loaded_nodes = load_from_file_int(include_file_path, schema, schema_warnings);
@@ -152,10 +174,24 @@ public:
 				                         "' directive in included config file " +
 				                         include_file_path + ".");
 			}
-			// We allow to override keys.
-			// We don't need to use `get_node()` here,
-			// since key is a top-level one.
-			m_root[key] = n.second;
+			switch(strategy) {
+			case STRATEGY_APPEND:
+				if(n.second.IsSequence()) {
+					for(const auto& item : n.second) {
+						m_root[key].push_back(item);
+					}
+					break;
+				}
+				// fallthrough
+			case STRATEGY_OVERRIDE:
+				m_root[key] = n.second;
+				break;
+			case STRATEGY_ADDONLY:
+				if(!m_root[key].IsDefined()) {
+					m_root[key] = n.second;
+				}
+				break;
+			}
 		}
 	}
 

userspace/falco/configuration.cpp

@@ -164,7 +164,10 @@ void falco_configuration::merge_config_files(const std::string &config_name,
 	// Parse files to be included
 	std::vector<std::string> include_files;
 	m_config.get_sequence<std::vector<std::string>>(include_files, yaml_helper::configs_key);
-	for(const std::string &include_file : include_files) {
+	for(auto include_file : include_files) {
+		// This can modify include_file by dropping the '+/@' prefixes.
+		const auto merge_strategy = yaml_helper::get_include_file_strategy(include_file);
+
 		auto include_file_path = std::filesystem::path(include_file);
 		if(include_file_path == ppath) {
 			throw std::logic_error("Config error: '" + yaml_helper::configs_key +
@@ -177,11 +180,12 @@ void falco_configuration::merge_config_files(const std::string &config_name,
 		}
 		if(std::filesystem::is_regular_file(include_file_path)) {
 			m_loaded_configs_filenames.push_back(include_file);
-			m_config.include_config_file(include_file_path.string(),
+			m_config.include_config_file(include_file,
+			                             merge_strategy,
 			                             m_config_schema,
 			                             &validation_status);
 			// Only report top most schema validation status
-			res[include_file_path.string()] = validation_status[0];
+			res[include_file] = validation_status[0];
 		} else if(std::filesystem::is_directory(include_file_path)) {
 			m_loaded_configs_folders.push_back(include_file);
 			std::vector<std::string> v;
@@ -195,7 +199,10 @@ void falco_configuration::merge_config_files(const std::string &config_name,
 			}
 			std::sort(v.begin(), v.end());
 			for(const auto &f : v) {
-				m_config.include_config_file(f, m_config_schema, &validation_status);
+				m_config.include_config_file(f,
+				                             merge_strategy,
+				                             m_config_schema,
+				                             &validation_status);
 				// Only report top most schema validation status
 				res[f] = validation_status[0];
 			}