Merge pull request #19 from draios/rework-outputs

Rework outputs

rules/base.txt

@@ -77,7 +77,7 @@ read and not proc.name in (sshd, sudo, su) and not_cron and sensitive_files | WA
 modify and (bin_dir_rename or bin_dir_mkdir) | WARNING Modify bin dir (%user.name %proc.name %evt.dir %evt.type %evt.args %fd.name)
 
 # Don't load shared objects coming from unexpected places
-read and fd.name contains .so and not (ubuntu_so_dirs or centos_so_dirs) | WARNING output.first_sequence(evt, "fd.filename", "shared_obj", "Loaded .so from unexpected dir (%user.name %proc.name %evt.dir %evt.type %evt.args %fd.name)")
+read and fd.name contains .so and not (ubuntu_so_dirs or centos_so_dirs) | WARNING Loaded .so from unexpected dir (%user.name %proc.name %evt.dir %evt.type %evt.args %fd.name)
 
 # Attempts to access things that shouldn't be
 evt.res = EACCES | INFO System call returned EACCESS (%user.name %proc.name %evt.dir %evt.type %evt.args %fd.name)

userspace/digwatch/digwatch.cpp

@@ -31,6 +31,8 @@ static void signal_callback(int signal)
 }
 
 
+std::vector<string> valid_output_names {"stdout", "syslog"};
+
 //
 // Program help
 //
@@ -44,11 +46,7 @@ static void usage()
 	   "                    Name of lua compiler main file\n"
 	   "                    (default: rules_loader.lua)\n"
 	   " -N                 Don't convert port numbers to names.\n"
-	   " -r <readfile>, --read=<readfile>\n"
-	   "                    Read the events from <readfile>.\n"
-	   " --unbuffered       Turn off output buffering. This causes every single line\n"
-	   "                    emitted by digwatch to be flushed, which generates higher CPU\n"
-	   "                    usage but is useful when piping digwatch's output into another\n"
+	   " -o                 Output type (options are 'stdout', 'syslog', default is 'stdout')\n"
 	   "                    process or into a script.\n"
 	   "\n"
     );
@@ -61,6 +59,7 @@ string lua_on_event = "on_event";
 //
 void do_inspect(sinsp* inspector,
 		digwatch_rules* rules,
+		string output_name,
 		lua_State* ls)
 {
 	int32_t res;
@@ -110,8 +109,9 @@ void do_inspect(sinsp* inspector,
 		{
 			lua_pushlightuserdata(ls, ev);
 			lua_pushnumber(ls, ev->get_check_id());
+			lua_pushstring(ls, output_name.c_str());
 
-			if(lua_pcall(ls, 2, 0, 0) != 0)
+			if(lua_pcall(ls, 3, 0, 0) != 0)
 			{
 				const char* lerr = lua_tostring(ls, -1);
 				string err = "Error invoking function output: " + string(lerr);
@@ -168,6 +168,7 @@ int digwatch_init(int argc, char **argv)
 	sinsp_evt::param_fmt event_buffer_format = sinsp_evt::PF_NORMAL;
 	int long_index = 0;
 	string lua_main_filename;
+	string output_name = "stdout";
 	string lua_dir = DIGWATCH_INSTALLATION_DIR;
 	lua_State* ls = NULL;
 
@@ -175,21 +176,19 @@ int digwatch_init(int argc, char **argv)
 	{
 		{"help", no_argument, 0, 'h' },
 		{"main-lua", required_argument, 0, 'u' },
-		{"readfile", required_argument, 0, 'r' },
-		{"unbuffered", no_argument, 0, 0 },
 		{0, 0, 0, 0}
 	};
 
 	try
 	{
 		inspector = new sinsp();
-
+		bool valid;
 
 		//
 		// Parse the args
 		//
 		while((op = getopt_long(argc, argv,
-                                        "hm:Nr:",
+                                        "hm:No:",
                                         long_options, &long_index)) != -1)
 		{
 			switch(op)
@@ -203,6 +202,14 @@ int digwatch_init(int argc, char **argv)
 			case 'N':
 				inspector->set_hostname_and_port_resolution_mode(false);
 				break;
+			case 'o':
+				valid = std::find(valid_output_names.begin(), valid_output_names.end(), optarg) != valid_output_names.end();
+				if (!valid)
+				{
+					throw sinsp_exception(string("Invalid output name ") + optarg);
+				}
+				output_name = optarg;
+				break;
 			case '?':
 				result = EXIT_FAILURE;
 				goto exit;
@@ -287,6 +294,7 @@ int digwatch_init(int argc, char **argv)
 
 		do_inspect(inspector,
 			   rules,
+			   output_name,
 			   ls);
 
 		inspector->close();

userspace/digwatch/lua/compiler.lua

@@ -160,7 +160,7 @@ local function normalize_level(level)
    level = string.lower(level)
    for i,v in ipairs(valid_levels) do
       if (string.find(v, "^"..level)) then
-	 return v
+	 return i - 1 -- (syslog levels start at 0, lua indices start at 1)
       end
    end
    error("Invalid severity level: "..level)
@@ -179,10 +179,6 @@ local function outputformat (level, format)
    return {type = "OutputFormat", level = normalize_level(level), value = format}
 end
 
-local function functioncall (level, str, mname, fname, args)
-   return {type = "FunctionCall", level = normalize_level(level), mname = mname, fname = fname, arguments = args, source = str}
-end
-
 local function rule(filter, output)
    if not output then
       output = outputformat(nil)
@@ -229,7 +225,7 @@ local G = {
   MacroDef = (C(V"Macro") * V"Skip" * V"Colon" * (V"Filter"));
 
   FuncArgs = symb("(") * list(V"Value", symb(",")) * symb(")");
-  Output = (C(V"Identifier") * V"Skip" * C(V"Name" * P(".") * V"Name" * V"FuncArgs") / functioncall) + (C(V"Identifier") * V"Skip" * C(P(1)^0) / outputformat);
+  Output = C(V"Identifier") * V"Skip" * C(P(1)^0) / outputformat;
 
   -- Terminals
   Value = terminal "Number" + terminal "String" + terminal "BareString";
@@ -473,11 +469,6 @@ function print_ast(ast, level)
    elseif t == "OutputFormat" then
       print(ast.value)
 
-   elseif t == "FunctionCall" then
-      print(ast.mname..ast.fname .. "(" )
-      print_ast(ast.arguments)
-      print(")")
-
    elseif t == "Filter" then
       print_ast(ast.value, level)
 

userspace/digwatch/lua/output.lua

@@ -1,38 +1,22 @@
 local mod = {}
 
-function mod.syslog(evt, level, format)
-   nixio = require("nixio")
-   format = "%evt.time: "..format
+levels = {"Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug"}
+
+function mod.stdout(evt, level, format)
+   format = "%evt.time: "..levels[level+1].." "..format
    formatter = digwatch.formatter(format)
    msg = digwatch.format_event(evt, formatter)
-   nixio.syslog(level, msg)
+   print (msg)
 end
 
+function mod.syslog(evt, level, format)
+   -- https://neopallium.github.io/nixio/modules/nixio.html#nixio.syslog
+   levels = {"emerg", "alert", "crit", "err", "warning", "notice", "info", "debug"}
 
-local first_sequence_state = {}
-
-function mod.first_sequence(evt, fieldname, key, format)
-   local field_value = digwatch.field(evt, fieldname)
-   local now = os.time()
-
-   format = "%evt.time: "..format
-
-   if first_sequence_state[key] == nil then
-      first_sequence_state[key] = {}
-   end
-
-   if first_sequence_state[key][field_value] == nil or
-   now - first_sequence_state[key][field_value] > 5 then
-      formatter = digwatch.formatter(format)
-      msg = digwatch.format_event(evt, formatter)
-      print (msg)
-   end
-   if field_value == nil then
-      formatter = digwatch.formatter(format)
-      s = digwatch.format_event(evt, formatter)
-      error("first_sequence: field '"..fieldname.."' is nil in event ("..s..")")
-   end
-   first_sequence_state[key][field_value] = now
+   nixio = require("nixio")
+   formatter = digwatch.formatter(format)
+   msg = digwatch.format_event(evt, formatter)
+   nixio.syslog(levels[level+1], msg)
 end
 
 return mod

userspace/digwatch/lua/rule_loader.lua

@@ -113,11 +113,8 @@ function set_output(output_ast)
 	 format = output_ast.value
       end
 
-      state.outputs[state.n_rules] = {type="format", formatter=digwatch.formatter("%evt.time: "..format)}
+      state.outputs[state.n_rules] = {format=format, level = output_ast.level}
 
-   elseif (output_ast.type == "FunctionCall") then
-      require(output_ast.mname)
-      state.outputs[state.n_rules] = {type="function", mname = output_ast.mname, source=output_ast.source}
    else
       error ("Unexpected type in set_output: ".. output_ast.type)
    end
@@ -162,18 +159,17 @@ function on_done()
    io.flush()
 end
 
-evt = nil
-function on_event(evt_, rule_id)
+local outputs = require('output')
+
+function on_event(evt_, rule_id, output_name)
+   if not (type(outputs[output_name]) == 'function') then
+      error("rule_loader.on_event(): invalid output_name: ", output_name)
+   end
+
    if state.outputs[rule_id] == nil then
       error ("rule_loader.on_event(): event with invalid rule_id: ", rule_id)
    end
 
-   if state.outputs[rule_id].type == "format" then
-      print(digwatch.format_event(evt_, state.outputs[rule_id].formatter))
-   elseif state.outputs[rule_id].type == "function" then
-      local reqmod =  "local "..state.outputs[rule_id].mname.." = require('" ..state.outputs[rule_id].mname .. "')";
-      evt = evt_
-      assert(loadstring(reqmod .. state.outputs[rule_id].source))()
-   end
+   outputs[output_name](evt_, state.outputs[rule_id].level, state.outputs[rule_id].format)
 end