Merge pull request #2 from draios/digwatch-scaffolding

Digwatch initial implementation

CMakeLists.txt

@@ -0,0 +1,51 @@
+cmake_minimum_required(VERSION 2.6)
+
+project(digwatch)
+
+set(DRAIOS_DEBUG_FLAGS "-D_DEBUG")
+#set(DRAIOS_FEATURE_FLAGS "-DPPM_ENABLE_SENTINEL")
+
+set(CMAKE_C_FLAGS "-Wall -ggdb ${DRAIOS_FEATURE_FLAGS}")
+set(CMAKE_CXX_FLAGS "-Wall -ggdb --std=c++0x ${DRAIOS_FEATURE_FLAGS}")
+set(CMAKE_C_FLAGS_DEBUG "${DRAIOS_DEBUG_FLAGS}")
+set(CMAKE_CXX_FLAGS_DEBUG "${DRAIOS_DEBUG_FLAGS}")
+# Add "-fno-inline -fno-omit-frame-pointer" for perf
+set(CMAKE_C_FLAGS_RELEASE "-O3 -fno-strict-aliasing -DNDEBUG")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fno-strict-aliasing -DNDEBUG")
+
+add_definitions(-DPLATFORM_NAME="${CMAKE_SYSTEM_NAME}")
+add_definitions(-DHAS_CAPTURE)
+
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+	set(KBUILD_FLAGS "${DRAIOS_DEBUG_FLAGS} ${DRAIOS_FEATURE_FLAGS}")
+else()
+	set(KBUILD_FLAGS "${DRAIOS_FEATURE_FLAGS}")
+endif()
+
+set(PACKAGE_NAME "draios-digwatch")
+
+add_definitions(-DK8S_DISABLE_THREAD)
+
+set(SYSDIG_DIR ${PROJECT_SOURCE_DIR}/../sysdig)
+
+add_subdirectory(${SYSDIG_DIR} ${PROJECT_BINARY_DIR}/sysdig)
+
+set(CURL_BUNDLE_DIR "${sysdig_BINARY_DIR}/curl-prefix/src/curl")
+set(CURL_INCLUDE_DIR "${CURL_BUNDLE_DIR}/include/")
+set(CURL_LIBRARIES "${CURL_BUNDLE_DIR}/lib/.libs/libcurl.a")
+
+set(LUAJIT_INCLUDE_DIR "${sysdig_BINARY_DIR}/luajit-prefix/src/luajit/src")
+
+if(NOT USE_BUNDLED_OPENSSL)
+	set(CURL_SSL_OPTION "")
+else()
+	set(CURL_SSL_OPTION "--with-ssl=${OPENSSL_INSTALL_DIR}")
+endif()
+
+message(STATUS "Using bundled curl in '${CURL_BUNDLE_DIR}'")
+message(STATUS "Using SSL for curl in '${CURL_SSL_OPTION}'")
+
+
+
+add_subdirectory(userspace/digwatch)
+

lua/compiler.lua

@@ -1,15 +1,13 @@
 --[[
-   Sysdig grammar and parser.
+   Digwatch grammar and parser.
 
-   Much of the scaffolding and helpers was deriverd Andre Murbach Maidl's Lua parser (https://github.com/andremm/lua-parser).
+   Much of the scaffolding and helpers was derived from Andre Murbach Maidl's Lua parser (https://github.com/andremm/lua-parser).
 
    Parses regular filters following the existing sysdig filter syntax (*), as well as "macro" definitions. Macro definitions are written like:
 
    inbound: (syscall.type=listen and evt.dir='>') or (syscall.type=accept and evt.dir='<')
 
-   (*) There currently one known difference with the syntax implemented in libsinsp:
+   (*) There is currently one known difference with the syntax implemented in libsinsp: In libsinsp, field names cannot start with 'a', 'o', or 'n'. With this parser they can.
-
-   - In libsinsp, field names cannot start with 'a', 'o', or 'n'. With this parser they can
 
 --]]
 
@@ -228,13 +226,13 @@ local G = {
   OrOp = kw("or") / "or";
   AndOp = kw("and") / "and";
   Colon = kw(":");
-  RelOp = symb("=") / "eq" +
+  RelOp = symb("=") / "=" +
-          symb("==") / "eq" +
+          symb("==") / "==" +
-          symb("!=") / "ne" +
+          symb("!=") / "!=" +
-          symb("<=") / "le" +
+          symb("<=") / "<=" +
-          symb(">=") / "ge" +
+          symb(">=") / ">=" +
-          symb("<") / "lt" +
+          symb("<") / "<" +
-          symb(">") / "gt" +
+          symb(">") / ">" +
           symb("contains") / "contains" +
           symb("icontains") / "icontains";
   InOp = kw("in") / "in";
@@ -455,17 +453,17 @@ end
    to the line-oriented compiler.
 --]]
 function compiler.init()
-   return {macros={}}
+   return {macros={}, ast=nil}
 end
 
 --[[
-   Compiles a digwatch filter or macro
+   Compiles a single line from a digwatch ruleset and updates the passed-in state object. Returns the AST of the line.
 --]]
 function compiler.compile_line(line, state)
    local ast, error_msg = compiler.parser.parse_line(line)
 
    if (error_msg) then
-      return nil, error_msg
+      error(error_msg)
    end
 
    local macros = get_macros(ast.value, {})
@@ -476,20 +474,31 @@ function compiler.compile_line(line, state)
    end
 
    if (ast.type == "MacroDef") then
+      -- Parsed line is a macro definition, so update our dictionary of macros and
+      -- return
       state.macros[ast.name] = ast.value
-      return ast, error_msg
+      return ast
+
    elseif (ast.type == "Filter") then
+      -- Line is a filter, so expand in-clauses and macro references, then
+      -- stitch it into global ast
+
       expand_in(ast)
 
       repeat
          expanded  = expand_macros(ast, state.macros, false)
       until expanded == false
 
+      if (state.ast == nil) then
+	 state.ast = ast
+      else
+	 state.ast = { type = "BinaryBoolOp", operator = "or", left = state.ast, right = ast }
+      end
    else
       error("Unexpected top-level AST type: "..ast.type)
    end
 
-   return ast, error_msg
+   return ast
 end
 
 

lua/rule_loader.lua

@@ -0,0 +1,78 @@
+--[[
+   Compile and install digwatch rules.
+
+   This module exports functions that are called from digwatch c++-side to compile and install a set of rules.
+
+--]]
+
+local compiler = require "compiler"
+
+local function install_filter(node)
+   local t = node.type
+
+   if t == "Filter" then
+      install_filter(node.value)
+
+   elseif t == "BinaryBoolOp" then
+      filter.nest() --io.write("(")
+      install_filter(node.left)
+      filter.bool_op(node.operator) --io.write(" "..node.operator.." ")
+      install_filter(node.right)
+      filter.unnest() --io.write(")")
+
+
+   elseif t == "UnaryBoolOp" then
+      filter.nest() --io.write("(")
+      filter.bool_op(node.operator) -- io.write(" "..node.operator.." ")
+      install_filter(node.argument)
+      filter.unnest() -- io.write(")")
+
+   elseif t == "BinaryRelOp" then
+      filter.rel_expr(node.left.value, node.operator, node.right.value)
+      -- io.write(node.left.value.." "..node.operator.." "..node.right.value)
+
+      -- xxx still need to handle case where node.right.type=="list"
+
+   elseif t == "UnaryRelOp"  then
+      filter.rel_expr(node.argument.value, node.operator)
+      --io.write(node.argument.value.." "..node.operator)
+
+   elseif t == "List" then
+      error("Not handling List yet")
+
+   elseif t == "FieldName" or t == "Number" or t == "String" or t == "BareString" or t == "Macro" then
+      error ("Unexpected type: "..t)
+
+      io.write(t.." "..node.value)
+
+   elseif t == "MacroDef" then
+      error ("Unexpected type: "..t)
+   else
+      error ("Unexpected type: "..t)
+   end
+end
+
+
+-- filter.rel_expr("proc.name",  "=", "cat")
+-- filter.bool_op("and")
+-- filter.nest()
+-- filter.nest()
+-- filter.rel_expr("fd.num",  "=", "1")
+-- filter.bool_op("or")
+-- filter.rel_expr("fd.num",  "=", "2")
+-- filter.unnest()
+-- filter.unnest()
+
+local state
+
+
+function load_rule(r)
+   if (state == nil) then
+      state = compiler.init()
+   end
+   compiler.compile_line(r, state)
+end
+
+function on_done()
+   install_filter(state.ast)
+end

lua/test.lua

@@ -1,4 +1,4 @@
-local compiler = require "sysdig-parser"
+local compiler = require "compiler"
 
 if #arg ~= 1 then
     print("Usage: test.lua <string>")
@@ -8,18 +8,19 @@ end
 local state = compiler.init()
 
 local function doit(line)
-   local ast, error_msg = compiler.compile_line(line, state)
+   local ast = compiler.compile_line(line, state)
 
    if not ast then
       print("error", error_msg)
       os.exit(1)
    end
 
-   compiler.parser.print_ast(ast)
 end
 for str in string.gmatch(arg[1], "([^;]+)") do
    doit(str)
 end
 
+compiler.parser.print_ast(state.ast)
+
 os.exit(0)
 

userspace/digwatch/CMakeLists.txt

@@ -0,0 +1,19 @@
+include_directories(${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp/third-party/jsoncpp)
+include_directories("${LUAJIT_INCLUDE_DIR}")
+
+if(NOT APPLE)
+    include_directories("${CURL_INCLUDE_DIR}")
+endif()
+
+
+include_directories(${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap)
+include_directories(${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp)
+include_directories("${PROJECT_BINARY_DIR}/userspace/digwatch")
+
+add_executable(digwatch rules.cpp digwatch.cpp)
+
+target_link_libraries(digwatch sinsp)
+
+
+set(DIGWATCH_LUA_MAIN "rule_loader.lua")
+configure_file(config_digwatch.h.in config_digwatch.h)

userspace/digwatch/config_digwatch.h.in

@@ -0,0 +1,7 @@
+#pragma once
+
+#define DIGWATCH_VERSION "${DIGWATCH_VERSION}"
+
+#define DIGWATCH_INSTALLATION_DIR "${CMAKE_INSTALL_PREFIX}"
+
+#define DIGWATCH_LUA_MAIN "${DIGWATCH_LUA_MAIN}"

userspace/digwatch/digwatch.cpp

@@ -0,0 +1,299 @@
+#define __STDC_FORMAT_MACROS
+
+#include <stdio.h>
+#include <iostream>
+#include <time.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <algorithm>
+#include <unistd.h>
+#include <getopt.h>
+
+extern "C" {
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+}
+
+#include <sinsp.h>
+#include <config_digwatch.h>
+#include "rules.h"
+#include "digwatch.h"
+#include "utils.h"
+
+
+//
+// Program help
+//
+static void usage()
+{
+    printf(
+	   "Usage: digwatch [options] [-p <output_format>] rules_filename\n\n"
+	   "Options:\n"
+	   " -h, --help         Print this page\n"
+	   " -m <filename>, --main-lua <filename>\n"
+	   "                    Name of lua compiler main file\n"
+	   "                    (default: rules_loader.lua)\n"
+	   " -M <num_seconds>   Stop collecting after <num_seconds> reached.\n"
+	   " -N                 Don't convert port numbers to names.\n"
+	   " -n <num>, --numevents=<num>\n"
+	   "                    Stop capturing after <num> events\n"
+	   " -r <readfile>, --read=<readfile>\n"
+	   "                    Read the events from <readfile>.\n"
+	   " --unbuffered       Turn off output buffering. This causes every single line\n"
+	   "                    emitted by digwatch to be flushed, which generates higher CPU\n"
+	   "                    usage but is useful when piping digwatch's output into another\n"
+	   "                    process or into a script.\n"
+	   "\n"
+    );
+}
+
+
+//
+// Event processing loop
+//
+captureinfo do_inspect(sinsp* inspector,
+		       uint64_t cnt,
+		       int duration_to_tot,
+		       sinsp_evt_formatter* formatter)
+{
+	captureinfo retval;
+	int32_t res;
+	sinsp_evt* ev;
+	string line;
+        int duration_start = 0;
+
+	//
+	// Loop through the events
+	//
+	duration_start = ((double)clock()) / CLOCKS_PER_SEC;
+	while(1)
+	{
+		if(duration_to_tot > 0)
+		{
+			int duration_tot = ((double)clock()) / CLOCKS_PER_SEC - duration_start;
+			if(duration_tot >= duration_to_tot)
+			{
+				break;
+			}
+		}
+		if(retval.m_nevts == cnt)
+		{
+			//
+			// End of capture, either because the user stopped it, or because
+			// we reached the event count specified with -n.
+			//
+			break;
+		}
+
+		res = inspector->next(&ev);
+
+		if(res == SCAP_TIMEOUT)
+		{
+			continue;
+		}
+		else if(res == SCAP_EOF)
+		{
+			break;
+		}
+		else if(res != SCAP_SUCCESS)
+		{
+			//
+			// Event read error.
+			// Notify the chisels that we're exiting, and then die with an error.
+			//
+			cerr << "res = " << res << endl;
+			throw sinsp_exception(inspector->getlasterr().c_str());
+		}
+
+		retval.m_nevts++;
+
+		if(!inspector->is_debug_enabled() &&
+			ev->get_category() & EC_INTERNAL)
+		{
+			continue;
+		}
+		if(formatter->tostring(ev, &line))
+		{
+			cout << line;
+			cout << endl;
+		}
+
+	}
+
+	return retval;
+}
+
+//
+// ARGUMENT PARSING AND PROGRAM SETUP
+//
+int digwatch_init(int argc, char **argv)
+{
+	int result;
+	sinsp* inspector = NULL;
+	digwatch_rules* rules = NULL;
+	int op;
+	uint64_t cnt = -1;
+	sinsp_evt::param_fmt event_buffer_format = sinsp_evt::PF_NORMAL;
+	int duration_to_tot = 0;
+	captureinfo cinfo;
+	string output_format;
+	int long_index = 0;
+	string lua_main_filename;
+	string lua_dir = DIGWATCH_INSTALLATION_DIR;
+
+	static struct option long_options[] =
+	{
+		{"help", no_argument, 0, 'h' },
+		{"numevents", required_argument, 0, 'n' },
+		{"main-lua", required_argument, 0, 'u' },
+		{"readfile", required_argument, 0, 'r' },
+		{"unbuffered", no_argument, 0, 0 },
+		{0, 0, 0, 0}
+	};
+
+	output_format = "*%evt.num %evt.outputtime %evt.cpu %proc.name (%thread.tid) %evt.dir %evt.type %evt.info";
+
+	try
+	{
+		inspector = new sinsp();
+
+
+		//
+		// Parse the args
+		//
+		while((op = getopt_long(argc, argv,
+                                        "hm:M:Nn:r:",
+                                        long_options, &long_index)) != -1)
+		{
+			switch(op)
+			{
+			case 'h':
+				usage();
+				result = EXIT_SUCCESS;
+				goto exit;
+			case 'm':
+				lua_main_filename = optarg;
+				break;
+			case 'M':
+				duration_to_tot = atoi(optarg);
+				if(duration_to_tot <= 0)
+				{
+					throw sinsp_exception(string("invalid duration") + optarg);
+				}
+				break;
+			case 'N':
+				inspector->set_hostname_and_port_resolution_mode(false);
+				break;
+			case 'n':
+				try
+				{
+					cnt = sinsp_numparser::parseu64(optarg);
+				}
+				catch(...)
+				{
+					throw sinsp_exception("can't parse the -n argument, make sure it's a number");
+				}
+
+				if(cnt <= 0)
+				{
+					throw sinsp_exception(string("invalid event count ") + optarg);
+				}
+				break;
+			case '?':
+				result = EXIT_FAILURE;
+				goto exit;
+			default:
+				break;
+			}
+
+		}
+
+		inspector->set_buffer_format(event_buffer_format);
+
+		string rules_file;
+
+		if(optind < argc)
+		{
+#ifdef HAS_FILTERING
+			for(int32_t j = optind ; j < argc; j++)
+			{
+				rules_file += argv[j];
+				if(j < argc - 1)
+				{
+					rules_file += " ";
+				}
+			}
+
+#else
+			fprintf(stderr, "filtering not compiled.\n");
+			result = EXIT_FAILURE;
+			goto exit;
+#endif
+		}
+
+		if (rules_file.size() == 0) {
+			usage();
+			result = EXIT_FAILURE;
+			goto exit;
+
+		}
+
+		//
+		// Create the event formatter
+		//
+		sinsp_evt_formatter formatter(inspector, output_format);
+
+		char* env_lua_dir = getenv("DIGWATCH_LUA_DIR");
+		if (env_lua_dir)
+		{
+			lua_dir = string(env_lua_dir);
+		}
+
+		trim(lua_main_filename);
+		if (lua_main_filename.size() == 0)
+		{
+			lua_main_filename = lua_dir + DIGWATCH_LUA_MAIN;
+		}
+
+		rules = new digwatch_rules(inspector, lua_main_filename, lua_dir);
+
+		rules->load_rules(rules_file);
+		inspector->set_filter(rules->get_filter());
+		inspector->open("");
+
+		cinfo = do_inspect(inspector,
+				   cnt,
+				   duration_to_tot,
+				   &formatter);
+
+		inspector->close();
+	}
+	catch(sinsp_exception& e)
+	{
+		cerr << e.what() << endl;
+		result = EXIT_FAILURE;
+	}
+	catch(...)
+	{
+		printf("Exception\n");
+		result = EXIT_FAILURE;
+	}
+
+exit:
+
+	if(inspector)
+	{
+		delete inspector;
+	}
+
+	return result;
+}
+
+//
+// MAIN
+//
+int main(int argc, char **argv)
+{
+	return digwatch_init(argc, argv);
+}

userspace/digwatch/digwatch.h

@@ -0,0 +1,41 @@
+/*
+Copyright (C) 2013-2014 Draios inc.
+
+This file is part of sysdig.
+
+sysdig is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+sysdig is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with sysdig.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#ifdef HAS_CAPTURE
+#include "../../driver/driver_config.h"
+#endif // HAS_CAPTURE
+
+
+
+//
+// Capture results
+//
+class captureinfo
+{
+public:
+	captureinfo()
+	{
+		m_nevts = 0;
+		m_time = 0;
+	}
+
+	uint64_t m_nevts;
+	uint64_t m_time;
+};

userspace/digwatch/rules.cpp

@@ -0,0 +1,116 @@
+#include "rules.h"
+
+extern "C" {
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+}
+
+digwatch_rules::digwatch_rules(sinsp* inspector, string lua_main_filename, string lua_dir)
+{
+	// Initialize Lua interpreter
+	m_ls = lua_open();
+	luaL_openlibs(m_ls);
+
+	m_lua_parser = new lua_parser(inspector, m_ls);
+
+	add_lua_path(lua_dir);
+	load_compiler(lua_main_filename);
+}
+
+void digwatch_rules::add_lua_path(string path)
+{
+	path += "?.lua";
+
+	lua_getglobal(m_ls, "package");
+	lua_getfield(m_ls, -1, "path");
+
+	string cur_path = lua_tostring(m_ls, -1 );
+	cur_path += ';';
+
+	cur_path.append(path.c_str());
+	lua_pop(m_ls, 1);
+
+	lua_pushstring(m_ls, cur_path.c_str());
+	lua_setfield(m_ls, -2, "path");
+	lua_pop(m_ls, 1);
+}
+
+void digwatch_rules::load_compiler(string lua_main_filename)
+{
+	ifstream is;
+	is.open(lua_main_filename);
+	if(!is.is_open())
+	{
+		throw sinsp_exception("can't open file " + lua_main_filename);
+	}
+
+	string scriptstr((istreambuf_iterator<char>(is)),
+			 istreambuf_iterator<char>());
+
+	//
+	// Load the compiler script
+	//
+	if(luaL_loadstring(m_ls, scriptstr.c_str()) || lua_pcall(m_ls, 0, 0, 0))
+	{
+		throw sinsp_exception("Failed to load script " +
+			lua_main_filename + ": " + lua_tostring(m_ls, -1));
+	}
+}
+
+void digwatch_rules::load_rules(string rules_filename)
+{
+	ifstream is;
+	is.open(rules_filename);
+	if(!is.is_open())
+	{
+		throw sinsp_exception("can't open file " + rules_filename);
+	}
+
+	lua_getglobal(m_ls, m_lua_load_rule.c_str());
+	if(lua_isfunction(m_ls, -1))
+	{
+		lua_pop(m_ls, 1);
+	} else {
+		throw sinsp_exception("No function " + m_lua_load_rule + " found in lua compiler module");
+	}
+
+	std::string line;
+	while (std::getline(is, line))
+	{
+		lua_getglobal(m_ls, m_lua_load_rule.c_str());
+		lua_pushstring(m_ls, line.c_str());
+
+		if(lua_pcall(m_ls, 1, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Error loading rule: " + string(lerr);
+			throw sinsp_exception(err);
+		}
+	}
+
+	lua_getglobal(m_ls, m_lua_on_done.c_str());
+	if(lua_isfunction(m_ls, -1))
+	{
+		if(lua_pcall(m_ls, 0, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Error installing rules: " + string(lerr);
+			throw sinsp_exception(err);
+		}
+	} else {
+		throw sinsp_exception("No function " + m_lua_on_done + " found in lua compiler module");
+	}
+
+}
+
+sinsp_filter* digwatch_rules::get_filter()
+{
+	return m_lua_parser->get_filter();
+}
+
+digwatch_rules::~digwatch_rules()
+{
+	delete m_lua_parser;
+}
+

userspace/digwatch/rules.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "sinsp.h"
+#include "lua_parser.h"
+
+class digwatch_rules
+{
+ public:
+	digwatch_rules(sinsp* inspector, string lua_main_filename, string lua_dir);
+	~digwatch_rules();
+	void load_rules(string rules_filename);
+	sinsp_filter* get_filter();
+
+ private:
+	void add_lua_path(string path);
+	void load_compiler(string lua_main_filename);
+
+	lua_parser* m_lua_parser;
+	lua_State* m_ls;
+	string m_lua_load_rule = "load_rule";
+	string m_lua_on_done = "on_done";
+};