Falco application changes to support multiple files in rules results

Application changes to support multiple files when stringifying rules
results:

- In both validate_rules_files and load_rules_files, instead of
  loading each file individually and then calling load_rules(), add a
  separate step that loads all the files at once. The actual rules
  content strings are held in a vector. The map from filename to
  content (reference) points to entries in that vector.

- Both actions do the same work for this step, so put the
  implementation in a shared application template method read_files
  that works on iterators. It uses itertors because the load filenames
  are a list and the validate filenames are a vector.

Signed-off-by: Mark Stemm <mark.stemm@gmail.com>

userspace/falco/app_actions/load_rules_files.cpp

@@ -93,25 +93,37 @@ application::run_result application::load_rules_files()
 		falco_configuration::read_rules_file_directory(path, m_state->config->m_loaded_rules_filenames, m_state->config->m_loaded_rules_folders);
 	}
 
-	for (const auto& filename : m_state->config->m_loaded_rules_filenames)
+	std::vector<std::string> rules_contents;
+	falco::load_result::rules_contents_t rc;
+
+	try {
+		read_files(m_state->config->m_loaded_rules_filenames.begin(),
+			   m_state->config->m_loaded_rules_filenames.end(),
+			   rules_contents,
+			   rc);
+	}
+	catch(falco_exception& e)
+	{
+		return run_result::fatal(e.what());
+	}
+
+	for(auto &filename : m_state->config->m_loaded_rules_filenames)
 	{
 		falco_logger::log(LOG_INFO, "Loading rules from file " + filename + "\n");
 		std::unique_ptr<falco::load_result> res;
 
-		res = m_state->engine->load_rules_file(filename);
+		res = m_state->engine->load_rules(rc.at(filename), filename);
-
-		if((!res->successful() || (m_options.verbose && res->has_warnings())))
-		{
-			printf("%s\n",
-			       (m_state->config->m_json_output ?
-				res->as_json().dump().c_str() :
-				res->as_string(true).c_str()));
-		}
 
 		if(!res->successful())
 		{
 			// Return the summary version as the error
-			return run_result::fatal(res->as_string(false));
+			return run_result::fatal(res->as_string(true, rc));
+		}
+
+		// If verbose is true, also print any warnings
+		if(m_options.verbose && res->has_warnings())
+		{
+			fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
 		}
 	}
 

userspace/falco/app_actions/validate_rules_files.cpp

@@ -23,7 +23,43 @@ application::run_result application::validate_rules_files()
 {
 	if(m_options.validate_rules_filenames.size() > 0)
 	{
+
+		std::vector<std::string> rules_contents;
+		falco::load_result::rules_contents_t rc;
+
+		try {
+			read_files(m_options.validate_rules_filenames.begin(),
+				   m_options.validate_rules_filenames.end(),
+				   rules_contents,
+				   rc);
+		}
+		catch(falco_exception& e)
+		{
+			return run_result::fatal(e.what());
+		}
+
 		bool successful = true;
+
+		// The validation result is *always* printed to
+		// stdout. When json_output is true, the output is in
+		// json format and contains all errors/warnings for
+		// all files.
+		//
+
+		// When json_output is false, it contains a summary of
+		// each file and whether it was valid or not, along
+		// with any errors.  To match older falco behavior,
+		// this *only* contains errors.
+		//
+		// So for each file stdout will contain:
+		//
+		// <filename>: Ok
+		// or
+		// <filename>: Invalid
+		// [All Validation Errors]
+		//
+		// Warnings are only printed to stderr, and only
+		// printed when verbose is true.
 		std::string summary;
 
 		falco_logger::log(LOG_INFO, "Validating rules file(s):\n");
@@ -36,29 +72,44 @@ application::run_result application::validate_rules_files()
 		// validation result is a single json object.
 		nlohmann::json results = nlohmann::json::array();
 
-		for(auto file : m_options.validate_rules_filenames)
+		for(auto &filename : m_options.validate_rules_filenames)
 		{
 			std::unique_ptr<falco::load_result> res;
 
-			res = m_state->engine->load_rules_file(file);
+			res = m_state->engine->load_rules(rc.at(filename), filename);
 
 			successful &= res->successful();
 
-			if(summary != "")
-			{
-				summary += "\n";
-			}
-			summary += file + ": " + (res->successful() ? "Ok" : "Invalid");
-
 			if(m_state->config->m_json_output)
 			{
-				results.push_back(res->as_json());
+				results.push_back(res->as_json(rc));
 			}
 			else
 			{
-				if(!res->successful() || (m_options.verbose && res->has_warnings()))
+				if(summary != "")
 				{
-					printf("%s\n", res->as_string(true).c_str());
+					summary += "\n";
+				}
+
+				// Add to the summary if not successful, or successful
+				// with no warnings.
+				if(!res->successful() ||
+				   (res->successful() && !res->has_warnings()))
+				{
+					summary += res->as_string(true, rc);
+				}
+				else
+				{
+					// If here, there must be only warnings.
+					// Add a line to the summary noting that the
+					// file was ok without printing the warnings.
+					summary += filename + ": Ok";
+
+					// If verbose is true, print the warnings now.
+					if(m_options.verbose)
+					{
+						fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
+					}
 				}
 			}
 		}
@@ -69,10 +120,13 @@ application::run_result application::validate_rules_files()
 			res["falco_load_results"] = results;
 			printf("%s\n", res.dump().c_str());
 		}
+		else
+		{
+			printf("%s\n", summary.c_str());
+		}
 
 		if(successful)
 		{
-			printf("%s\n", summary.c_str());
 			return run_result::exit();
 		}
 		else

userspace/falco/application.h

@@ -134,6 +134,49 @@ private:
 		bool proceed;
 	};
 
+	// Convenience method. Read a sequence of filenames and fill
+	// in a vector of rules contents.
+        // Also fill in the provided rules_contents_t with a mapping from
+        // filename (reference) to content (reference).
+	// falco_exception if any file could not be read.
+	template<class InputIterator>
+	void read_files(InputIterator begin, InputIterator end,
+			std::vector<std::string>& rules_contents,
+			falco::load_result::rules_contents_t& rc)
+	{
+		// Read the contents in a first pass
+		for(auto it = begin; it != end; it++)
+		{
+			std::string &filename = *it;
+			std::ifstream is;
+			is.open(filename);
+			if (!is.is_open())
+			{
+				throw falco_exception("Could not open file " + filename + " for reading");
+			}
+
+			std::string rules_content((istreambuf_iterator<char>(is)),
+						  istreambuf_iterator<char>());
+			rules_contents.emplace_back(std::move(rules_content));
+		}
+
+		// Populate the map in a second pass to avoid
+		// references becoming invalid.
+		auto it = begin;
+		auto rit = rules_contents.begin();
+		for(; it != end && rit != rules_contents.end(); it++, rit++)
+		{
+			rc.emplace(*it, *rit);
+		}
+
+		// Both it and rit must be at the end, otherwise
+		// there's a bug in the above
+		if(it != end || rit != rules_contents.end())
+		{
+			throw falco_exception("Unexpected mismatch in rules content name/rules content sets?");
+		}
+	}
+
 	// These methods comprise the code the application "runs". The
 	// order in which the methods run is in application.cpp.
 	run_result create_signal_handlers();