Skip output json format (#342)

* Add option to exclude output property in json fmt

New falco.yaml option json_include_output_property controls where the
formatted string "output" is included in the json object when json
output is enabled. By default the string is included.

* Add tests for new json output option

New test sets json_include_output_property to false and then verifies
that the json output does *not* contain the surrounding text "Warning an
open...".

falco.yaml

@@ -14,6 +14,11 @@ rules_file:
 # Whether to output events in json or text
 json_output: false
 
+# When using json output, whether or not to include the "output" property
+# itself (e.g. "File below a known binary directory opened for writing
+# (user=root ....") in the json output.
+json_include_output_property: true
+
 # Send information logs to stderr and/or syslog Note these are *not* security
 # notification logs! These are just Falco lifecycle (and possibly error) logs.
 log_stderr: true

test/falco_test.py

@@ -30,6 +30,7 @@ class FalcoTest(Test):
             self.trace_file = os.path.join(self.basedir, self.trace_file)
 
         self.json_output = self.params.get('json_output', '*', default=False)
+        self.json_include_output_property = self.params.get('json_include_output_property', '*', default=True)
         self.priority = self.params.get('priority', '*', default='debug')
         self.rules_file = self.params.get('rules_file', '*', default=os.path.join(self.basedir, '../rules/falco_rules.yaml'))
 
@@ -249,7 +250,11 @@ class FalcoTest(Test):
             for line in res.stdout.splitlines():
                 if line.startswith('{'):
                     obj = json.loads(line)
-                    for attr in ['time', 'rule', 'priority', 'output']:
+                    if self.json_include_output_property:
+                        attrs = ['time', 'rule', 'priority', 'output']
+                    else:
+                        attrs = ['time', 'rule', 'priority']
+                    for attr in attrs:
                         if not attr in obj:
                             self.fail("Falco JSON object {} does not contain property \"{}\"".format(line, attr))
 
@@ -348,8 +353,8 @@ class FalcoTest(Test):
             trace_arg = "-e {}".format(self.trace_file)
 
         # Run falco
-        cmd = '{} {} {} -c {} {} -o json_output={} -o priority={} -v'.format(
+        cmd = '{} {} {} -c {} {} -o json_output={} -o json_include_output_property={} -o priority={} -v'.format(
-            self.falco_binary_path, self.rules_args, self.disabled_args, self.conf_file, trace_arg, self.json_output, self.priority)
+            self.falco_binary_path, self.rules_args, self.disabled_args, self.conf_file, trace_arg, self.json_output, self.json_include_output_property, self.priority)
 
         for tag in self.disable_tags:
             cmd += ' -T {}'.format(tag)

test/falco_tests.yaml

@@ -655,3 +655,12 @@ trace_files: !mux
       - rules/rule_append_false.yaml
     trace_file: trace_files/cat_write.scap
 
+  json_output_no_output_property:
+    json_output: True
+    json_include_output_property: False
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/rule_append.yaml
+    trace_file: trace_files/cat_write.scap
+    stdout_contains: "^(?!.*Warning An open of /dev/null was seen.*)"

userspace/engine/falco_engine.cpp

@@ -88,7 +88,8 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al
 	// formats.formatter is used, so we can unconditionally set
 	// json_output to false.
 	bool json_output = false;
-	falco_formats::init(m_inspector, m_ls, json_output);
+	bool json_include_output_property = false;
+	falco_formats::init(m_inspector, m_ls, json_output, json_include_output_property);
 
 	m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info, m_min_priority);
 }

userspace/engine/formats.cpp

@@ -25,6 +25,7 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 
 sinsp* falco_formats::s_inspector = NULL;
 bool falco_formats::s_json_output = false;
+bool falco_formats::s_json_include_output_property = true;
 sinsp_evt_formatter_cache *falco_formats::s_formatters = NULL;
 
 const static struct luaL_reg ll_falco [] =
@@ -36,10 +37,11 @@ const static struct luaL_reg ll_falco [] =
 	{NULL,NULL}
 };
 
-void falco_formats::init(sinsp* inspector, lua_State *ls, bool json_output)
+void falco_formats::init(sinsp* inspector, lua_State *ls, bool json_output, bool json_include_output_property)
 {
 	s_inspector = inspector;
 	s_json_output = json_output;
+	s_json_include_output_property = json_include_output_property;
 	if(!s_formatters)
 	{
 		s_formatters = new sinsp_evt_formatter_cache(s_inspector);
@@ -155,8 +157,12 @@ int falco_formats::format_event (lua_State *ls)
 		event["time"] = iso8601evttime;
 		event["rule"] = rule;
 		event["priority"] = level;
+
+		if(s_json_include_output_property)
+		{
 			// This is the filled-in output line.
 			event["output"] = line;
+		}
 
 		full_line = writer.write(event);
 

userspace/engine/formats.h

@@ -31,7 +31,7 @@ class sinsp_evt_formatter;
 class falco_formats
 {
  public:
-	static void init(sinsp* inspector, lua_State *ls, bool json_output);
+	static void init(sinsp* inspector, lua_State *ls, bool json_output, bool json_include_output_property);
 
 	// formatter = falco.formatter(format_string)
 	static int formatter(lua_State *ls);
@@ -48,4 +48,5 @@ class falco_formats
 	static sinsp* s_inspector;
 	static sinsp_evt_formatter_cache *s_formatters;
 	static bool s_json_output;
+	static bool s_json_include_output_property;
 };

userspace/falco/configuration.cpp

@@ -67,6 +67,7 @@ void falco_configuration::init(string conf_filename, list<string> &cmdline_optio
 	}
 
 	m_json_output = m_config->get_scalar<bool>("json_output", false);
+	m_json_include_output_property = m_config->get_scalar<bool>("json_include_output_property", true);
 
 	falco_outputs::output_config file_output;
 	file_output.name = "file";

userspace/falco/configuration.h

@@ -167,6 +167,7 @@ class falco_configuration
 
 	std::list<std::string> m_rules_filenames;
 	bool m_json_output;
+	bool m_json_include_output_property;
 	std::vector<falco_outputs::output_config> m_outputs;
 	uint32_t m_notifications_rate;
 	uint32_t m_notifications_max_burst;

userspace/falco/falco.cpp

@@ -547,6 +547,7 @@ int falco_init(int argc, char **argv)
 		}
 
 		outputs->init(config.m_json_output,
+			      config.m_json_include_output_property,
 			      config.m_notifications_rate, config.m_notifications_max_burst,
 			      config.m_buffered_outputs);
 

userspace/falco/falco_outputs.cpp

@@ -52,7 +52,9 @@ falco_outputs::~falco_outputs()
 	}
 }
 
-void falco_outputs::init(bool json_output, uint32_t rate, uint32_t max_burst, bool buffered)
+void falco_outputs::init(bool json_output,
+			 bool json_include_output_property,
+			 uint32_t rate, uint32_t max_burst, bool buffered)
 {
 	// The engine must have been given an inspector by now.
 	if(! m_inspector)
@@ -65,7 +67,7 @@ void falco_outputs::init(bool json_output, uint32_t rate, uint32_t max_burst, bo
 	// Note that falco_formats is added to both the lua state used
 	// by the falco engine as well as the separate lua state used
 	// by falco outputs.
-	falco_formats::init(m_inspector, m_ls, json_output);
+	falco_formats::init(m_inspector, m_ls, json_output, json_include_output_property);
 
 	falco_logger::init(m_ls);
 

userspace/falco/falco_outputs.h

@@ -41,7 +41,9 @@ public:
 		std::map<std::string, std::string> options;
 	};
 
-	void init(bool json_output, uint32_t rate, uint32_t max_burst, bool buffered);
+	void init(bool json_output,
+		  bool json_include_output_property,
+		  uint32_t rate, uint32_t max_burst, bool buffered);
 
 	void add_output(output_config oc);