diff --git a/test/falco_test.py b/test/falco_test.py index a2ff0847..adb35767 100644 --- a/test/falco_test.py +++ b/test/falco_test.py @@ -2,6 +2,7 @@ import os import re +import json from avocado import Test from avocado.utils import process @@ -17,6 +18,7 @@ class FalcoTest(Test): self.should_detect = self.params.get('detect', '*') self.trace_file = self.params.get('trace_file', '*') + self.json_output = self.params.get('json_output', '*') if self.should_detect: self.detect_level = self.params.get('detect_level', '*') @@ -35,8 +37,8 @@ class FalcoTest(Test): self.log.info("Trace file %s", self.trace_file) # Run the provided trace file though falco - cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {}'.format( - self.falcodir, self.falcodir, self.falcodir, self.trace_file) + cmd = '{}/userspace/falco/falco -r {}/../rules/falco_rules.yaml -c {}/../falco.yaml -e {} -o json_output={}'.format( + self.falcodir, self.falcodir, self.falcodir, self.trace_file, self.json_output) self.falco_proc = process.SubProcess(cmd) @@ -71,6 +73,15 @@ class FalcoTest(Test): if not events_detected > 0: self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, self.detect_level)) + if self.json_output: + # Just verify that any lines starting with '{' are valid json objects. + # Doesn't do any deep inspection of the contents. + for line in res.stdout.splitlines(): + if line.startswith('{'): + obj = json.loads(line) + for attr in ['time', 'rule', 'priority', 'output']: + if not attr in obj: + self.fail("Falco JSON object {} does not contain property \"{}\"".format(line, attr)) pass diff --git a/test/run_regression_tests.sh b/test/run_regression_tests.sh index 1057ab61..b46646a1 100755 --- a/test/run_regression_tests.sh +++ b/test/run_regression_tests.sh @@ -13,40 +13,35 @@ function download_trace_files() { done } +function prepare_multiplex_fileset() { + + dir=$1 + detect=$2 + detect_level=$3 + json_output=$4 + + for trace in $SCRIPTDIR/$dir/*.scap ; do + [ -e "$trace" ] || continue + NAME=`basename $trace .scap` + cat << EOF >> $MULT_FILE + $NAME-detect-$detect-json-$json_output: + detect: $detect + detect_level: $detect_level + trace_file: $trace + json_output: $json_output +EOF + done +} + function prepare_multiplex_file() { echo "trace_files: !mux" > $MULT_FILE - for trace in $SCRIPTDIR/traces-positive/*.scap ; do - [ -e "$trace" ] || continue - NAME=`basename $trace .scap` - cat << EOF >> $MULT_FILE - $NAME: - detect: True - detect_level: Warning - trace_file: $trace -EOF - done + prepare_multiplex_fileset traces-positive True Warning False + prepare_multiplex_fileset traces-negative False Warning True + prepare_multiplex_fileset traces-info True Informational False - for trace in $SCRIPTDIR/traces-negative/*.scap ; do - [ -e "$trace" ] || continue - NAME=`basename $trace .scap` - cat << EOF >> $MULT_FILE - $NAME: - detect: False - trace_file: $trace -EOF - done - - for trace in $SCRIPTDIR/traces-info/*.scap ; do - [ -e "$trace" ] || continue - NAME=`basename $trace .scap` - cat << EOF >> $MULT_FILE - $NAME: - detect: True - detect_level: Informational - trace_file: $trace -EOF - done + prepare_multiplex_fileset traces-positive True Warning True + prepare_multiplex_fileset traces-info True Informational True echo "Contents of $MULT_FILE:" cat $MULT_FILE diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index b73f33a3..d269a473 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -242,7 +242,6 @@ int falco_init(int argc, char **argv) sinsp* inspector = NULL; falco_rules* rules = NULL; int op; - sinsp_evt::param_fmt event_buffer_format; int long_index = 0; string lua_main_filename; string scap_filename; @@ -391,7 +390,7 @@ int falco_init(int argc, char **argv) rules = new falco_rules(inspector, ls, lua_main_filename); - falco_formats::init(inspector, ls); + falco_formats::init(inspector, ls, config.m_json_output); falco_fields::init(inspector, ls); falco_logger::init(ls); @@ -416,16 +415,6 @@ int falco_init(int argc, char **argv) inspector->set_hostname_and_port_resolution_mode(false); - if (config.m_json_output) - { - event_buffer_format = sinsp_evt::PF_JSON; - } - else - { - event_buffer_format = sinsp_evt::PF_NORMAL; - } - inspector->set_buffer_format(event_buffer_format); - for(std::vector::iterator it = config.m_outputs.begin(); it != config.m_outputs.end(); ++it) { add_output(ls, *it); diff --git a/userspace/falco/formats.cpp b/userspace/falco/formats.cpp index 0ff87068..142df600 100644 --- a/userspace/falco/formats.cpp +++ b/userspace/falco/formats.cpp @@ -1,8 +1,11 @@ +#include + #include "formats.h" #include "logger.h" sinsp* falco_formats::s_inspector = NULL; +bool s_json_output = false; const static struct luaL_reg ll_falco [] = { @@ -11,9 +14,10 @@ const static struct luaL_reg ll_falco [] = {NULL,NULL} }; -void falco_formats::init(sinsp* inspector, lua_State *ls) +void falco_formats::init(sinsp* inspector, lua_State *ls, bool json_output) { s_inspector = inspector; + s_json_output = json_output; luaL_openlib(ls, "falco", ll_falco, 0); } @@ -42,15 +46,53 @@ int falco_formats::format_event (lua_State *ls) { string line; - if (!lua_islightuserdata(ls, -1) || !lua_islightuserdata(ls, -2)) { + if (!lua_islightuserdata(ls, -1) || + !lua_isstring(ls, -2) || + !lua_isstring(ls, -3) || + !lua_islightuserdata(ls, -4)) { falco_logger::log(LOG_ERR, "Invalid arguments passed to format_event()\n"); throw sinsp_exception("format_event error"); } sinsp_evt* evt = (sinsp_evt*)lua_topointer(ls, 1); - sinsp_evt_formatter* formatter = (sinsp_evt_formatter*)lua_topointer(ls, 2); + const char *rule = (char *) lua_tostring(ls, 2); + const char *level = (char *) lua_tostring(ls, 3); + sinsp_evt_formatter* formatter = (sinsp_evt_formatter*)lua_topointer(ls, 4); formatter->tostring(evt, &line); + // For JSON output, the formatter returned just the output + // string containing the format text and values. Use this to + // build a more detailed object containing the event time, + // rule, severity, full output, and fields. + if (s_json_output) { + Json::Value event; + Json::FastWriter writer; + + // Convert the time-as-nanoseconds to a more json-friendly ISO8601. + time_t evttime = evt->get_ts()/1000000000; + char time_sec[20]; // sizeof "YYYY-MM-DDTHH:MM:SS" + char time_ns[12]; // sizeof ".sssssssssZ" + string iso8601evttime; + + strftime(time_sec, sizeof(time_sec), "%FT%T", gmtime(&evttime)); + snprintf(time_ns, sizeof(time_ns), ".%09luZ", evt->get_ts() % 1000000000); + iso8601evttime = time_sec; + iso8601evttime += time_ns; + event["time"] = iso8601evttime; + event["rule"] = rule; + event["priority"] = level; + event["output"] = line; + + line = writer.write(event); + + // Json::FastWriter may add a trailing newline. If it + // does, remove it. + if (line[line.length()-1] == '\n') + { + line.resize(line.length()-1); + } + } + lua_pushstring(ls, line.c_str()); return 1; } diff --git a/userspace/falco/formats.h b/userspace/falco/formats.h index 73f69b0d..6f369bf3 100644 --- a/userspace/falco/formats.h +++ b/userspace/falco/formats.h @@ -13,7 +13,7 @@ class sinsp_evt_formatter; class falco_formats { public: - static void init(sinsp* inspector, lua_State *ls); + static void init(sinsp* inspector, lua_State *ls, bool json_output); // formatter = falco.formatter(format_string) static int formatter(lua_State *ls); diff --git a/userspace/falco/lua/output.lua b/userspace/falco/lua/output.lua index 0bef1712..245f5cb4 100644 --- a/userspace/falco/lua/output.lua +++ b/userspace/falco/lua/output.lua @@ -6,10 +6,10 @@ mod.levels = levels local outputs = {} -function mod.stdout(evt, level, format) +function mod.stdout(evt, rule, level, format) format = "*%evt.time: "..levels[level+1].." "..format formatter = falco.formatter(format) - msg = falco.format_event(evt, formatter) + msg = falco.format_event(evt, rule, levels[level+1], formatter) print (msg) end @@ -26,26 +26,26 @@ function mod.file_validate(options) end -function mod.file(evt, level, format, options) +function mod.file(evt, rule, level, format, options) format = "%evt.time: "..levels[level+1].." "..format formatter = falco.formatter(format) - msg = falco.format_event(evt, formatter) + msg = falco.format_event(evt, rule, levels[level+1], formatter) file = io.open(options.filename, "a+") file:write(msg, "\n") file:close() end -function mod.syslog(evt, level, format) +function mod.syslog(evt, rule, level, format) formatter = falco.formatter(format) - msg = falco.format_event(evt, formatter) + msg = falco.format_event(evt, rule, levels[level+1], formatter) falco.syslog(level, msg) end -function mod.event(event, level, format) +function mod.event(event, rule, level, format) for index,o in ipairs(outputs) do - o.output(event, level, format, o.config) + o.output(event, rule, level, format, o.config) end end diff --git a/userspace/falco/lua/rule_loader.lua b/userspace/falco/lua/rule_loader.lua index 6f07e701..8bb55edf 100644 --- a/userspace/falco/lua/rule_loader.lua +++ b/userspace/falco/lua/rule_loader.lua @@ -256,7 +256,7 @@ function on_event(evt_, rule_id) rule_output_counts.by_name[rule.rule] = rule_output_counts.by_name[rule.rule] + 1 end - output.event(evt_, rule.level, rule.output) + output.event(evt_, rule.rule, rule.level, rule.output) end function print_stats()