Merge pull request #89 from draios/update-json-output

Add more useful json output.

test/falco_test.py

@@ -2,6 +2,7 @@
 
 import os
 import re
+import json
 
 from avocado import Test
 from avocado.utils import process
@@ -17,6 +18,7 @@ class FalcoTest(Test):
 
         self.should_detect = self.params.get('detect', '*')
         self.trace_file = self.params.get('trace_file', '*')
+        self.json_output = self.params.get('json_output', '*')
 
         if self.should_detect:
             self.detect_level = self.params.get('detect_level', '*')
@@ -35,8 +37,8 @@ class FalcoTest(Test):
         self.log.info("Trace file %s", self.trace_file)
 
         # Run the provided trace file though falco
-        cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {}'.format(
-            self.falcodir, self.falcodir, self.falcodir, self.trace_file)
+        cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {} -o json_output={}'.format(
+            self.falcodir, self.falcodir, self.falcodir, self.trace_file, self.json_output)
 
         self.falco_proc = process.SubProcess(cmd)
 
@@ -71,6 +73,15 @@ class FalcoTest(Test):
             if not events_detected > 0:
                 self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, self.detect_level))
 
+        if self.json_output:
+            # Just verify that any lines starting with '{' are valid json objects.
+            # Doesn't do any deep inspection of the contents.
+            for line in res.stdout.splitlines():
+                if line.startswith('{'):
+                    obj = json.loads(line)
+                    for attr in ['time', 'rule', 'priority', 'output']:
+                        if not attr in obj:
+                            self.fail("Falco JSON object {} does not contain property \"{}\"".format(line, attr))
         pass
 
 

test/run_regression_tests.sh

@@ -13,40 +13,35 @@ function download_trace_files() {
     done
 }
 
+function prepare_multiplex_fileset() {
+
+    dir=$1
+    detect=$2
+    detect_level=$3
+    json_output=$4
+
+    for trace in $SCRIPTDIR/$dir/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+	cat << EOF >> $MULT_FILE
+  $NAME-detect-$detect-json-$json_output:
+    detect: $detect
+    detect_level: $detect_level
+    trace_file: $trace
+    json_output: $json_output
+EOF
+    done
+}
+
 function prepare_multiplex_file() {
     echo "trace_files: !mux" > $MULT_FILE
 
-    for trace in $SCRIPTDIR/traces-positive/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
-	cat << EOF >> $MULT_FILE
-  $NAME:
-    detect: True
-    detect_level: Warning
-    trace_file: $trace
-EOF
-    done
+    prepare_multiplex_fileset traces-positive True Warning False
+    prepare_multiplex_fileset traces-negative False Warning True
+    prepare_multiplex_fileset traces-info True Informational False
 
-    for trace in $SCRIPTDIR/traces-negative/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
-	cat << EOF >> $MULT_FILE
-  $NAME:
-    detect: False
-    trace_file: $trace
-EOF
-    done
-
-    for trace in $SCRIPTDIR/traces-info/*.scap ; do
-	[ -e "$trace" ] || continue
-	NAME=`basename $trace .scap`
-	cat << EOF >> $MULT_FILE
-  $NAME:
-    detect: True
-    detect_level: Informational
-    trace_file: $trace
-EOF
-    done
+    prepare_multiplex_fileset traces-positive True Warning True
+    prepare_multiplex_fileset traces-info True Informational True
 
     echo "Contents of $MULT_FILE:"
     cat $MULT_FILE

userspace/falco/falco.cpp

@@ -242,7 +242,6 @@ int falco_init(int argc, char **argv)
 	sinsp* inspector = NULL;
 	falco_rules* rules = NULL;
 	int op;
-	sinsp_evt::param_fmt event_buffer_format;
 	int long_index = 0;
 	string lua_main_filename;
 	string scap_filename;
@@ -391,7 +390,7 @@ int falco_init(int argc, char **argv)
 
 		rules = new falco_rules(inspector, ls, lua_main_filename);
 
-		falco_formats::init(inspector, ls);
+		falco_formats::init(inspector, ls, config.m_json_output);
 		falco_fields::init(inspector, ls);
 
 		falco_logger::init(ls);
@@ -416,16 +415,6 @@ int falco_init(int argc, char **argv)
 
 		inspector->set_hostname_and_port_resolution_mode(false);
 
-		if (config.m_json_output)
-		{
-			event_buffer_format = sinsp_evt::PF_JSON;
-		}
-		else
-		{
-			event_buffer_format = sinsp_evt::PF_NORMAL;
-		}
-		inspector->set_buffer_format(event_buffer_format);
-
 		for(std::vector<output_config>::iterator it = config.m_outputs.begin(); it != config.m_outputs.end(); ++it)
 		{
 			add_output(ls, *it);

userspace/falco/formats.cpp

@@ -1,8 +1,11 @@
+#include <json/json.h>
+
 #include "formats.h"
 #include "logger.h"
 
 
 sinsp* falco_formats::s_inspector = NULL;
+bool s_json_output = false;
 
 const static struct luaL_reg ll_falco [] =
 {
@@ -11,9 +14,10 @@ const static struct luaL_reg ll_falco [] =
 	{NULL,NULL}
 };
 
-void falco_formats::init(sinsp* inspector, lua_State *ls)
+void falco_formats::init(sinsp* inspector, lua_State *ls, bool json_output)
 {
 	s_inspector = inspector;
+	s_json_output = json_output;
 
 	luaL_openlib(ls, "falco", ll_falco, 0);
 }
@@ -42,15 +46,53 @@ int falco_formats::format_event (lua_State *ls)
 {
 	string line;
 
-	if (!lua_islightuserdata(ls, -1) || !lua_islightuserdata(ls, -2)) {
+	if (!lua_islightuserdata(ls, -1) ||
+	    !lua_isstring(ls, -2) ||
+	    !lua_isstring(ls, -3) ||
+	    !lua_islightuserdata(ls, -4)) {
 		falco_logger::log(LOG_ERR, "Invalid arguments passed to format_event()\n");
 		throw sinsp_exception("format_event error");
 	}
 	sinsp_evt* evt = (sinsp_evt*)lua_topointer(ls, 1);
-	sinsp_evt_formatter* formatter = (sinsp_evt_formatter*)lua_topointer(ls, 2);
+	const char *rule = (char *) lua_tostring(ls, 2);
+	const char *level = (char *) lua_tostring(ls, 3);
+	sinsp_evt_formatter* formatter = (sinsp_evt_formatter*)lua_topointer(ls, 4);
 
 	formatter->tostring(evt, &line);
 
+	// For JSON output, the formatter returned just the output
+	// string containing the format text and values. Use this to
+	// build a more detailed object containing the event time,
+	// rule, severity, full output, and fields.
+	if (s_json_output) {
+		Json::Value event;
+		Json::FastWriter writer;
+
+		// Convert the time-as-nanoseconds to a more json-friendly ISO8601.
+		time_t evttime = evt->get_ts()/1000000000;
+		char time_sec[20]; // sizeof "YYYY-MM-DDTHH:MM:SS"
+		char time_ns[12]; // sizeof ".sssssssssZ"
+		string iso8601evttime;
+
+		strftime(time_sec, sizeof(time_sec), "%FT%T", gmtime(&evttime));
+		snprintf(time_ns, sizeof(time_ns), ".%09luZ", evt->get_ts() % 1000000000);
+		iso8601evttime = time_sec;
+		iso8601evttime += time_ns;
+		event["time"] = iso8601evttime;
+		event["rule"] = rule;
+		event["priority"] = level;
+		event["output"] = line;
+
+		line = writer.write(event);
+
+		// Json::FastWriter may add a trailing newline. If it
+		// does, remove it.
+		if (line[line.length()-1] == '\n')
+		{
+			line.resize(line.length()-1);
+		}
+	}
+
 	lua_pushstring(ls, line.c_str());
 	return 1;
 }

userspace/falco/formats.h

@@ -13,7 +13,7 @@ class sinsp_evt_formatter;
 class falco_formats
 {
  public:
-	static void init(sinsp* inspector, lua_State *ls);
+	static void init(sinsp* inspector, lua_State *ls, bool json_output);
 
 	// formatter = falco.formatter(format_string)
 	static int formatter(lua_State *ls);

userspace/falco/lua/output.lua

@@ -6,10 +6,10 @@ mod.levels = levels
 
 local outputs = {}
 
-function mod.stdout(evt, level, format)
+function mod.stdout(evt, rule, level, format)
    format = "*%evt.time: "..levels[level+1].." "..format
    formatter = falco.formatter(format)
-   msg = falco.format_event(evt, formatter)
+   msg = falco.format_event(evt, rule, levels[level+1], formatter)
    print (msg)
 end
 
@@ -26,26 +26,26 @@ function mod.file_validate(options)
 
 end
 
-function mod.file(evt, level, format, options)
+function mod.file(evt, rule, level, format, options)
    format = "%evt.time: "..levels[level+1].." "..format
    formatter = falco.formatter(format)
-   msg = falco.format_event(evt, formatter)
+   msg = falco.format_event(evt, rule, levels[level+1], formatter)
 
    file = io.open(options.filename, "a+")
    file:write(msg, "\n")
    file:close()
 end
 
-function mod.syslog(evt, level, format)
+function mod.syslog(evt, rule, level, format)
 
    formatter = falco.formatter(format)
-   msg = falco.format_event(evt, formatter)
+   msg = falco.format_event(evt, rule, levels[level+1], formatter)
    falco.syslog(level, msg)
 end
 
-function mod.event(event, level, format)
+function mod.event(event, rule, level, format)
    for index,o in ipairs(outputs) do
-      o.output(event, level, format, o.config)
+      o.output(event, rule, level, format, o.config)
    end
 end
 

userspace/falco/lua/rule_loader.lua

@@ -256,7 +256,7 @@ function on_event(evt_, rule_id)
       rule_output_counts.by_name[rule.rule] = rule_output_counts.by_name[rule.rule] + 1
    end
 
-   output.event(evt_, rule.level, rule.output)
+   output.event(evt_, rule.rule, rule.level, rule.output)
 end
 
 function print_stats()