Merge pull request #5 from draios/replumb-events

Support output formats

userspace/digwatch/CMakeLists.txt

@@ -10,7 +10,7 @@ include_directories(${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap)
 include_directories(${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp)
 include_directories("${PROJECT_BINARY_DIR}/userspace/digwatch")
 
-add_executable(digwatch rules.cpp digwatch.cpp)
+add_executable(digwatch formats.cpp rules.cpp digwatch.cpp)
 
 target_link_libraries(digwatch sinsp)
 

userspace/digwatch/digwatch.cpp

@@ -18,6 +18,7 @@ extern "C" {
 #include <sinsp.h>
 #include <config_digwatch.h>
 #include "rules.h"
+#include "formats.h"
 #include "digwatch.h"
 #include "utils.h"
 
@@ -28,7 +29,7 @@ extern "C" {
 static void usage()
 {
     printf(
-	   "Usage: digwatch [options] [-p <output_format>] rules_filename\n\n"
+	   "Usage: digwatch [options] rules_filename\n\n"
 	   "Options:\n"
 	   " -h, --help         Print this page\n"
 	   " -m <filename>, --main-lua <filename>\n"
@@ -55,13 +56,15 @@ static void usage()
 captureinfo do_inspect(sinsp* inspector,
 		       uint64_t cnt,
 		       int duration_to_tot,
-		       sinsp_evt_formatter* formatter)
+		       digwatch_rules* rules,
+		       digwatch_formats* formats)
 {
 	captureinfo retval;
 	int32_t res;
 	sinsp_evt* ev;
 	string line;
         int duration_start = 0;
+	sinsp_evt_formatter* formatter;
 
 	//
 	// Loop through the events
@@ -113,12 +116,21 @@ captureinfo do_inspect(sinsp* inspector,
 		{
 			continue;
 		}
-		if(formatter->tostring(ev, &line))
+
+		formatter = formats->lookup_formatter(ev->get_check_id());
+		if (!formatter)
 		{
-			cout << line;
+			throw sinsp_exception("Error: No formatter for event with id %d " + to_string(ev->get_check_id()));
-			cout << endl;
 		}
 
+		bool has_all = formatter->tostring(ev, &line);
+		if (!has_all) {
+			cout << "(missing fields) ";
+		}
+		cout << line;
+		cout << endl;
+
+
 	}
 
 	return retval;
@@ -132,15 +144,16 @@ int digwatch_init(int argc, char **argv)
 	int result;
 	sinsp* inspector = NULL;
 	digwatch_rules* rules = NULL;
+	digwatch_formats* formats = NULL;
 	int op;
 	uint64_t cnt = -1;
 	sinsp_evt::param_fmt event_buffer_format = sinsp_evt::PF_NORMAL;
 	int duration_to_tot = 0;
 	captureinfo cinfo;
-	string output_format;
 	int long_index = 0;
 	string lua_main_filename;
 	string lua_dir = DIGWATCH_INSTALLATION_DIR;
+	lua_State* ls;
 
 	static struct option long_options[] =
 	{
@@ -152,8 +165,6 @@ int digwatch_init(int argc, char **argv)
 		{0, 0, 0, 0}
 	};
 
-	output_format = "*%evt.num %evt.outputtime %evt.cpu %proc.name (%thread.tid) %evt.dir %evt.type %evt.info";
-
 	try
 	{
 		inspector = new sinsp();
@@ -240,10 +251,6 @@ int digwatch_init(int argc, char **argv)
 		}
 
 		//
-		// Create the event formatter
-		//
-		sinsp_evt_formatter formatter(inspector, output_format);
-
 		char* env_lua_dir = getenv("DIGWATCH_LUA_DIR");
 		if (env_lua_dir)
 		{
@@ -256,7 +263,12 @@ int digwatch_init(int argc, char **argv)
 			lua_main_filename = lua_dir + DIGWATCH_LUA_MAIN;
 		}
 
-		rules = new digwatch_rules(inspector, lua_main_filename, lua_dir);
+		// Initialize Lua interpreter
+		ls = lua_open();
+		luaL_openlibs(ls);
+
+		rules = new digwatch_rules(inspector, ls, lua_main_filename, lua_dir);
+		formats = new digwatch_formats(inspector, ls);
 
 		rules->load_rules(rules_file);
 		inspector->set_filter(rules->get_filter());
@@ -265,7 +277,8 @@ int digwatch_init(int argc, char **argv)
 		cinfo = do_inspect(inspector,
 				   cnt,
 				   duration_to_tot,
-				   &formatter);
+				   rules,
+				   formats);
 
 		inspector->close();
 	}
@@ -287,6 +300,7 @@ exit:
 		delete inspector;
 	}
 
+	lua_close(ls);
 	return result;
 }
 

userspace/digwatch/formats.cpp

@@ -0,0 +1,57 @@
+#include "formats.h"
+
+extern "C" {
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+}
+
+std::map<uint32_t, sinsp_evt_formatter*> g_format_map;
+sinsp* g_inspector;
+
+const static struct luaL_reg ll_digwatch [] =
+{
+	{"set_formatter", &digwatch_formats::set_formatter},
+	{NULL,NULL}
+};
+
+digwatch_formats::digwatch_formats(sinsp* inspector, lua_State *ls)
+{
+	g_inspector = inspector;
+
+	m_ls = ls;
+
+	luaL_openlib(m_ls, "digwatch", ll_digwatch, 0);
+}
+
+int digwatch_formats::set_formatter (lua_State *ls) {
+	uint32_t index = luaL_checkinteger(ls, 1);
+	string format = luaL_checkstring(ls, 2);
+
+	try
+	{
+		if(format == "" || format == "default")
+		{
+			g_format_map[index] = new sinsp_evt_formatter(g_inspector, DEFAULT_OUTPUT_STR);
+		}
+		else
+		{
+			g_format_map[index] = new sinsp_evt_formatter(g_inspector, format);
+		}
+	}
+	catch(sinsp_exception& e)
+	{
+		string err = "invalid output format " + format;
+		fprintf(stderr, "%s\n", err.c_str());
+		throw sinsp_exception("set_formatter error");
+	}
+
+	return 0;
+}
+
+sinsp_evt_formatter* digwatch_formats::lookup_formatter(uint32_t index)
+{
+	return g_format_map[index];
+}
+
+

userspace/digwatch/formats.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "sinsp.h"
+#include "lua_parser.h"
+
+class sinsp_evt_formatter;
+
+class digwatch_formats
+{
+ public:
+	digwatch_formats(sinsp* inspector, lua_State *ls);
+
+	// set_formatter(index, format_string)
+	static int set_formatter(lua_State *ls);
+	sinsp_evt_formatter* lookup_formatter(uint32_t index);
+
+ private:
+	lua_State* m_ls;
+};

userspace/digwatch/lua/compiler.lua

@@ -168,6 +168,9 @@ local function outputformat (format)
 end
 
 local function rule(filter, output)
+   if not output then
+      output = outputformat("")
+   end
    return {type = "Rule", filter = filter, output = output}
 end
 
@@ -336,14 +339,24 @@ end
 
 --]]
 function expand_macros(node, defs, changed)
-   if node.type == "Filter" then
+
+   function copy(obj)
+      if type(obj) ~= 'table' then return obj end
+      local res = {}
+      for k, v in pairs(obj) do res[copy(k)] = copy(v) end
+      return res
+   end
+
+   if (node.type == "Rule") then
+      macros = expand_macros(node.filter, defs, changed)
+   elseif node.type == "Filter" then
       if (node.value.type == "Macro") then
          if (defs[node.value.value] == nil) then
-            tostring = require 'ml'.tstring
             error("Undefined macro '".. node.value.value .. "' used in filter.")
          end
-         node.value = defs[node.value.value]
+         node.value = copy(defs[node.value.value])
          changed = true
+	 return changed
       end
       return expand_macros(node.value, defs, changed)
 
@@ -353,7 +366,7 @@ function expand_macros(node, defs, changed)
          if (defs[node.left.value] == nil) then
             error("Undefined macro '".. node.left.value .. "' used in filter.")
          end
-         node.left = defs[node.left.value]
+         node.left = copy(defs[node.left.value])
          changed = true
       end
 
@@ -361,7 +374,7 @@ function expand_macros(node, defs, changed)
          if (defs[node.right.value] == nil) then
             error("Undefined macro ".. node.right.value .. "used in filter.")
          end
-         node.right = defs[node.right.value]
+         node.right = copy(defs[node.right.value])
          changed = true
       end
 
@@ -374,7 +387,7 @@ function expand_macros(node, defs, changed)
          if (defs[node.argument.value] == nil) then
             error("Undefined macro ".. node.argument.value .. "used in filter.")
          end
-         node.argument = defs[node.argument.value]
+         node.argument = copy(defs[node.argument.value])
          changed = true
       end
       return expand_macros(node.argument, defs, changed)
@@ -442,7 +455,7 @@ function print_ast(node, level)
    elseif t == "MacroDef" then
       -- don't print for now
    else
-      error ("Unexpected type: "..t)
+      error ("Unexpected type in print_ast: "..t)
    end
 end
 compiler.parser.print_ast = print_ast
@@ -460,22 +473,9 @@ end
 
 
 --[[
-   Sets up compiler state and returns it.
+   Compiles a single line from a digwatch ruleset and updates the passed-in macros table. Returns the AST of the line.
-
-   This is an opaque blob that is passed into subsequent compiler calls and
-   should not be modified by the client.
-
-   It holds state such as macro definitions that must be kept across calls
-   to the line-oriented compiler.
 --]]
-function compiler.init()
+function compiler.compile_line(line, macro_defs)
-   return {macros={}, ast=nil}
-end
-
---[[
-   Compiles a single line from a digwatch ruleset and updates the passed-in state object. Returns the AST of the line.
---]]
-function compiler.compile_line(line, state)
    local ast, error_msg = compiler.parser.parse_line(line)
 
    if (error_msg) then
@@ -501,7 +501,7 @@ function compiler.compile_line(line, state)
    end
 
    for m, _ in pairs(macros) do
-      if state.macros[m] == nil then
+      if macros[m] == nil then
          error ("Undefined macro '"..m.."' used in '"..line.."'")
       end
    end
@@ -509,7 +509,7 @@ function compiler.compile_line(line, state)
    if (ast.type == "MacroDef") then
       -- Parsed line is a macro definition, so update our dictionary of macros and
       -- return
-      state.macros[ast.name] = ast.value
+      macro_defs[ast.name] = ast.value
       return ast
 
    elseif (ast.type == "Rule") then
@@ -519,14 +519,9 @@ function compiler.compile_line(line, state)
       expand_in(ast.filter)
 
       repeat
-         expanded  = expand_macros(ast, state.macros, false)
+	 expanded  = expand_macros(ast, macro_defs, false)
       until expanded == false
 
-      if (state.ast == nil) then
-	 state.ast = ast
-      else
-	 state.ast = { type = "BinaryBoolOp", operator = "or", left = state.ast, right = ast }
-      end
    else
       error("Unexpected top-level AST type: "..ast.type)
    end

userspace/digwatch/lua/parser-smoke.sh

@@ -38,6 +38,7 @@ good "not (not (a))"
 good "not (a.b=1)"
 good "not (a.a exists)"
 good "not a"
+good "a.b = 1 and not a"
 good "not not a"
 good "(not not a)"
 good "not a.b=1"

userspace/digwatch/lua/rule_loader.lua

@@ -7,20 +7,37 @@
 
 local compiler = require "compiler"
 
+local function mark_check_nodes(ast, index)
+   local t = ast.type
+
+   if t == "BinaryBoolOp" then
+      mark_check_nodes(ast.left, index)
+      mark_check_nodes(ast.right, index)
+
+   elseif t == "UnaryBoolOp" then
+      mark_check_nodes(ast.argument, index)
+
+   elseif t == "BinaryRelOp" then
+      ast.index = index
+
+   elseif t == "UnaryRelOp"  then
+      ast.index = index
+
+   else
+      error ("Unexpected type in install_filter: "..t)
+   end
+end
+
 local function install_filter(node)
    local t = node.type
 
-   if t == "Filter" then
+   if t == "BinaryBoolOp" then
-      install_filter(node.value)
-
-   elseif t == "BinaryBoolOp" then
       filter.nest() --io.write("(")
       install_filter(node.left)
       filter.bool_op(node.operator) --io.write(" "..node.operator.." ")
       install_filter(node.right)
       filter.unnest() --io.write(")")
 
-
    elseif t == "UnaryBoolOp" then
       filter.nest() --io.write("(")
       filter.bool_op(node.operator) -- io.write(" "..node.operator.." ")
@@ -28,15 +45,15 @@ local function install_filter(node)
       filter.unnest() -- io.write(")")
 
    elseif t == "BinaryRelOp" then
-      filter.rel_expr(node.left.value, node.operator, node.right.value)
+      filter.rel_expr(node.left.value, node.operator, node.right.value, node.index)
       -- io.write(node.left.value.." "..node.operator.." "..node.right.value)
 
    elseif t == "UnaryRelOp"  then
-      filter.rel_expr(node.argument.value, node.operator)
+      filter.rel_expr(node.argument.value, node.operator, node.index)
       --io.write(node.argument.value.." "..node.operator)
 
    else
-      error ("Unexpected type: "..t)
+      error ("Unexpected type in install_filter: "..t)
    end
 end
 
@@ -53,14 +70,43 @@ end
 
 local state
 
+--[[
+   Sets up compiler state and returns it.
+
+   It holds state such as macro definitions that must be kept across calls
+   to the line-oriented compiler.
+--]]
+local function init()
+   return {macros={}, filter_ast=nil, n_rules=0}
+end
+
 
 function load_rule(r)
    if (state == nil) then
-      state = compiler.init()
+      state = init()
+   end
+   local line_ast = compiler.compile_line(r, state.macros)
+
+   if (line_ast.type == nil) then -- blank line
+      return
+   elseif (line_ast.type == "MacroDef") then
+      return
+   elseif (not (line_ast.type == "Rule")) then
+      error ("Unexpected type in load_rule: "..line_ast.type)
+   end
+
+   digwatch.set_formatter(state.n_rules, line_ast.output.value)
+   mark_check_nodes(line_ast.filter.value, state.n_rules)
+
+   state.n_rules = state.n_rules + 1
+
+   if (state.filter_ast == nil) then
+      state.filter_ast = line_ast.filter.value
+   else
+      state.filter_ast = { type = "BinaryBoolOp", operator = "or", left = state.filter_ast, right = line_ast.filter.value }
    end
-   compiler.compile_line(r, state)
 end
 
 function on_done()
-   install_filter(state.ast)
+   install_filter(state.filter_ast)
 end

userspace/digwatch/lua/test.lua

@@ -5,10 +5,11 @@ if #arg ~= 1 then
     os.exit(1)
 end
 
-local state = compiler.init()
+local macros = {}
+local ast
 
 local function doit(line)
-   local ast = compiler.compile_line(line, state)
+   ast = compiler.compile_line(line, macros)
 
    if not ast then
       print("error", error_msg)
@@ -20,8 +21,8 @@ for str in string.gmatch(arg[1], "([^;]+)") do
    doit(str)
 end
 
-if not (state.ast == nil) then -- can be nil if only macros
+if not (ast) then
-   compiler.parser.print_ast(state.ast)
+   compiler.parser.print_ast(ast)
 end
 
 os.exit(0)

userspace/digwatch/rules.cpp

@@ -6,11 +6,10 @@ extern "C" {
 #include "lauxlib.h"
 }
 
-digwatch_rules::digwatch_rules(sinsp* inspector, string lua_main_filename, string lua_dir)
+
+digwatch_rules::digwatch_rules(sinsp* inspector, lua_State *ls, string lua_main_filename, string lua_dir)
 {
-	// Initialize Lua interpreter
+	m_ls = ls;
-	m_ls = lua_open();
-	luaL_openlibs(m_ls);
 
 	m_lua_parser = new lua_parser(inspector, m_ls);
 
@@ -84,7 +83,7 @@ void digwatch_rules::load_rules(string rules_filename)
 		if(lua_pcall(m_ls, 1, 0, 0) != 0)
 		{
 			const char* lerr = lua_tostring(m_ls, -1);
-			string err = "Error loading rule: " + string(lerr);
+			string err = "Error loading rule '" + line + "':" + string(lerr);
 			throw sinsp_exception(err);
 		}
 	}

userspace/digwatch/rules.h

@@ -6,7 +6,7 @@
 class digwatch_rules
 {
  public:
-	digwatch_rules(sinsp* inspector, string lua_main_filename, string lua_dir);
+	digwatch_rules(sinsp* inspector, lua_State *ls, string lua_main_filename, string lua_dir);
 	~digwatch_rules();
 	void load_rules(string rules_filename);
 	sinsp_filter* get_filter();
@@ -17,6 +17,8 @@ class digwatch_rules
 
 	lua_parser* m_lua_parser;
 	lua_State* m_ls;
+
 	string m_lua_load_rule = "load_rule";
 	string m_lua_on_done = "on_done";
+	string m_lua_on_event = "on_event";
 };