From 52a7c775960fb1752c47c49db2257eaee8f308da Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Mon, 6 Jun 2016 16:52:40 -0700 Subject: [PATCH 1/2] Add more useful json output. Instead of using sysdig's json output, which only contains the fields from the format string without any formatting text, use the string output to build a json object containing the format string, rule name, severity, and the event time (converted to a json-friendly ISO8601). This fixes https://github.com/draios/falco/issues/82. --- userspace/falco/falco.cpp | 13 +------- userspace/falco/formats.cpp | 48 +++++++++++++++++++++++++++-- userspace/falco/formats.h | 2 +- userspace/falco/lua/output.lua | 16 +++++----- userspace/falco/lua/rule_loader.lua | 2 +- 5 files changed, 56 insertions(+), 25 deletions(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index b73f33a3..d269a473 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -242,7 +242,6 @@ int falco_init(int argc, char **argv) sinsp* inspector = NULL; falco_rules* rules = NULL; int op; - sinsp_evt::param_fmt event_buffer_format; int long_index = 0; string lua_main_filename; string scap_filename; @@ -391,7 +390,7 @@ int falco_init(int argc, char **argv) rules = new falco_rules(inspector, ls, lua_main_filename); - falco_formats::init(inspector, ls); + falco_formats::init(inspector, ls, config.m_json_output); falco_fields::init(inspector, ls); falco_logger::init(ls); @@ -416,16 +415,6 @@ int falco_init(int argc, char **argv) inspector->set_hostname_and_port_resolution_mode(false); - if (config.m_json_output) - { - event_buffer_format = sinsp_evt::PF_JSON; - } - else - { - event_buffer_format = sinsp_evt::PF_NORMAL; - } - inspector->set_buffer_format(event_buffer_format); - for(std::vector::iterator it = config.m_outputs.begin(); it != config.m_outputs.end(); ++it) { add_output(ls, *it); diff --git a/userspace/falco/formats.cpp b/userspace/falco/formats.cpp index 0ff87068..142df600 100644 --- a/userspace/falco/formats.cpp +++ b/userspace/falco/formats.cpp @@ -1,8 +1,11 @@ +#include + #include "formats.h" #include "logger.h" sinsp* falco_formats::s_inspector = NULL; +bool s_json_output = false; const static struct luaL_reg ll_falco [] = { @@ -11,9 +14,10 @@ const static struct luaL_reg ll_falco [] = {NULL,NULL} }; -void falco_formats::init(sinsp* inspector, lua_State *ls) +void falco_formats::init(sinsp* inspector, lua_State *ls, bool json_output) { s_inspector = inspector; + s_json_output = json_output; luaL_openlib(ls, "falco", ll_falco, 0); } @@ -42,15 +46,53 @@ int falco_formats::format_event (lua_State *ls) { string line; - if (!lua_islightuserdata(ls, -1) || !lua_islightuserdata(ls, -2)) { + if (!lua_islightuserdata(ls, -1) || + !lua_isstring(ls, -2) || + !lua_isstring(ls, -3) || + !lua_islightuserdata(ls, -4)) { falco_logger::log(LOG_ERR, "Invalid arguments passed to format_event()\n"); throw sinsp_exception("format_event error"); } sinsp_evt* evt = (sinsp_evt*)lua_topointer(ls, 1); - sinsp_evt_formatter* formatter = (sinsp_evt_formatter*)lua_topointer(ls, 2); + const char *rule = (char *) lua_tostring(ls, 2); + const char *level = (char *) lua_tostring(ls, 3); + sinsp_evt_formatter* formatter = (sinsp_evt_formatter*)lua_topointer(ls, 4); formatter->tostring(evt, &line); + // For JSON output, the formatter returned just the output + // string containing the format text and values. Use this to + // build a more detailed object containing the event time, + // rule, severity, full output, and fields. + if (s_json_output) { + Json::Value event; + Json::FastWriter writer; + + // Convert the time-as-nanoseconds to a more json-friendly ISO8601. + time_t evttime = evt->get_ts()/1000000000; + char time_sec[20]; // sizeof "YYYY-MM-DDTHH:MM:SS" + char time_ns[12]; // sizeof ".sssssssssZ" + string iso8601evttime; + + strftime(time_sec, sizeof(time_sec), "%FT%T", gmtime(&evttime)); + snprintf(time_ns, sizeof(time_ns), ".%09luZ", evt->get_ts() % 1000000000); + iso8601evttime = time_sec; + iso8601evttime += time_ns; + event["time"] = iso8601evttime; + event["rule"] = rule; + event["priority"] = level; + event["output"] = line; + + line = writer.write(event); + + // Json::FastWriter may add a trailing newline. If it + // does, remove it. + if (line[line.length()-1] == '\n') + { + line.resize(line.length()-1); + } + } + lua_pushstring(ls, line.c_str()); return 1; } diff --git a/userspace/falco/formats.h b/userspace/falco/formats.h index 73f69b0d..6f369bf3 100644 --- a/userspace/falco/formats.h +++ b/userspace/falco/formats.h @@ -13,7 +13,7 @@ class sinsp_evt_formatter; class falco_formats { public: - static void init(sinsp* inspector, lua_State *ls); + static void init(sinsp* inspector, lua_State *ls, bool json_output); // formatter = falco.formatter(format_string) static int formatter(lua_State *ls); diff --git a/userspace/falco/lua/output.lua b/userspace/falco/lua/output.lua index 0bef1712..245f5cb4 100644 --- a/userspace/falco/lua/output.lua +++ b/userspace/falco/lua/output.lua @@ -6,10 +6,10 @@ mod.levels = levels local outputs = {} -function mod.stdout(evt, level, format) +function mod.stdout(evt, rule, level, format) format = "*%evt.time: "..levels[level+1].." "..format formatter = falco.formatter(format) - msg = falco.format_event(evt, formatter) + msg = falco.format_event(evt, rule, levels[level+1], formatter) print (msg) end @@ -26,26 +26,26 @@ function mod.file_validate(options) end -function mod.file(evt, level, format, options) +function mod.file(evt, rule, level, format, options) format = "%evt.time: "..levels[level+1].." "..format formatter = falco.formatter(format) - msg = falco.format_event(evt, formatter) + msg = falco.format_event(evt, rule, levels[level+1], formatter) file = io.open(options.filename, "a+") file:write(msg, "\n") file:close() end -function mod.syslog(evt, level, format) +function mod.syslog(evt, rule, level, format) formatter = falco.formatter(format) - msg = falco.format_event(evt, formatter) + msg = falco.format_event(evt, rule, levels[level+1], formatter) falco.syslog(level, msg) end -function mod.event(event, level, format) +function mod.event(event, rule, level, format) for index,o in ipairs(outputs) do - o.output(event, level, format, o.config) + o.output(event, rule, level, format, o.config) end end diff --git a/userspace/falco/lua/rule_loader.lua b/userspace/falco/lua/rule_loader.lua index 6f07e701..8bb55edf 100644 --- a/userspace/falco/lua/rule_loader.lua +++ b/userspace/falco/lua/rule_loader.lua @@ -256,7 +256,7 @@ function on_event(evt_, rule_id) rule_output_counts.by_name[rule.rule] = rule_output_counts.by_name[rule.rule] + 1 end - output.event(evt_, rule.level, rule.output) + output.event(evt_, rule.rule, rule.level, rule.output) end function print_stats() From 995e61210e1e6f4e36ee070d8a10aa1e52b3b2c9 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Tue, 7 Jun 2016 13:35:27 -0700 Subject: [PATCH 2/2] Add regression tests for json output. Modify falco_test.py to look for a boolean multiplex attribute 'json_output'. If true, examine the lines of the output and for any line that begins with '{', parse it as json and ensure it has the 4 attributes we expect. Modify run_regression_tests to have a utility function prepare_multiplex_fileset that does the work of looping over files in a directory, along with detect, level, and json output arguments. The appropriate multiplex attributes are added for each file. Use that utility function to test json output for the positive and informational directories along with non-json output. The negative directory is only tested once. --- test/falco_test.py | 15 ++++++++-- test/run_regression_tests.sh | 55 ++++++++++++++++-------------------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/test/falco_test.py b/test/falco_test.py index a2ff0847..adb35767 100644 --- a/test/falco_test.py +++ b/test/falco_test.py @@ -2,6 +2,7 @@ import os import re +import json from avocado import Test from avocado.utils import process @@ -17,6 +18,7 @@ class FalcoTest(Test): self.should_detect = self.params.get('detect', '*') self.trace_file = self.params.get('trace_file', '*') + self.json_output = self.params.get('json_output', '*') if self.should_detect: self.detect_level = self.params.get('detect_level', '*') @@ -35,8 +37,8 @@ class FalcoTest(Test): self.log.info("Trace file %s", self.trace_file) # Run the provided trace file though falco - cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {}'.format( - self.falcodir, self.falcodir, self.falcodir, self.trace_file) + cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {} -o json_output={}'.format( + self.falcodir, self.falcodir, self.falcodir, self.trace_file, self.json_output) self.falco_proc = process.SubProcess(cmd) @@ -71,6 +73,15 @@ class FalcoTest(Test): if not events_detected > 0: self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, self.detect_level)) + if self.json_output: + # Just verify that any lines starting with '{' are valid json objects. + # Doesn't do any deep inspection of the contents. + for line in res.stdout.splitlines(): + if line.startswith('{'): + obj = json.loads(line) + for attr in ['time', 'rule', 'priority', 'output']: + if not attr in obj: + self.fail("Falco JSON object {} does not contain property \"{}\"".format(line, attr)) pass diff --git a/test/run_regression_tests.sh b/test/run_regression_tests.sh index 1057ab61..b46646a1 100755 --- a/test/run_regression_tests.sh +++ b/test/run_regression_tests.sh @@ -13,40 +13,35 @@ function download_trace_files() { done } +function prepare_multiplex_fileset() { + + dir=$1 + detect=$2 + detect_level=$3 + json_output=$4 + + for trace in $SCRIPTDIR/$dir/*.scap ; do + [ -e "$trace" ] || continue + NAME=`basename $trace .scap` + cat << EOF >> $MULT_FILE + $NAME-detect-$detect-json-$json_output: + detect: $detect + detect_level: $detect_level + trace_file: $trace + json_output: $json_output +EOF + done +} + function prepare_multiplex_file() { echo "trace_files: !mux" > $MULT_FILE - for trace in $SCRIPTDIR/traces-positive/*.scap ; do - [ -e "$trace" ] || continue - NAME=`basename $trace .scap` - cat << EOF >> $MULT_FILE - $NAME: - detect: True - detect_level: Warning - trace_file: $trace -EOF - done + prepare_multiplex_fileset traces-positive True Warning False + prepare_multiplex_fileset traces-negative False Warning True + prepare_multiplex_fileset traces-info True Informational False - for trace in $SCRIPTDIR/traces-negative/*.scap ; do - [ -e "$trace" ] || continue - NAME=`basename $trace .scap` - cat << EOF >> $MULT_FILE - $NAME: - detect: False - trace_file: $trace -EOF - done - - for trace in $SCRIPTDIR/traces-info/*.scap ; do - [ -e "$trace" ] || continue - NAME=`basename $trace .scap` - cat << EOF >> $MULT_FILE - $NAME: - detect: True - detect_level: Informational - trace_file: $trace -EOF - done + prepare_multiplex_fileset traces-positive True Warning True + prepare_multiplex_fileset traces-info True Informational True echo "Contents of $MULT_FILE:" cat $MULT_FILE