Updating docs for 0.5.0.

New changelog and readme updates.

CHANGELOG.md

@@ -2,6 +2,38 @@
 
 This file documents all notable changes to Falco. The release numbering uses [semantic versioning](http://semver.org).
 
+## v0.5.0
+
+Released 2016-12-22
+
+Starting with this release, we're adding a new section "Rule Changes" devoted to changes to the default ruleset `falco_rules.yaml`.
+
+### Major Changes
+
+* Cache event formatting objects so they are not re-created for every falco notification. This can result in significant speedups when the ruleset results in lots of notifications. [[#158](https://github.com/draios/falco/pull/158)]
+* Falco notifications are now throttled by a token bucket, preventing a flood of notifications when many events match a rule. Controlled by the `outputs, rate` and `outputs, max_burst` options. [[#161](https://github.com/draios/falco/pull/161)]
+
+### Minor Changes
+
+* When run from a container, you can provide the environment variable `SYSDIG_SKIP_LOAD` to skip the process of building/loading the kernel module. Thanks @carlsverre for the fix. [[#145](https://github.com/draios/falco/pull/145)]
+* Fully implement `USE_BUNDLED_DEPS` within CMakeFiles so you can build with external third-party libraries. [[#147](https://github.com/draios/falco/pull/147)]
+* Improve error messages that result when trying to load a rule with a malformed `output:` attribute [[#150](https://github.com/draios/falco/pull/150)] [[#151](https://github.com/draios/falco/pull/151)]
+* Add the ability to write event capture statistics to a file via the `-s <statsfile>` option. [[#155](https://github.com/draios/falco/pull/155)]
+* New configuration option `log_level` controls the verbosity of falco's logging. [[#160](https://github.com/draios/falco/pull/160)]
+
+
+### Bug Fixes
+
+* Improve compatibility with Sysdig Cloud Agent build [[#148](https://github.com/draios/falco/pull/148)]
+
+### Rule Changes
+
+* Add DNF as non-alerting for RPM and package management. Thanks @djcross for the fix. [[#153](https://github.com/draios/falco/pull/153)]
+* Make `google_containers/kube-proxy` a trusted image, affecting the File Open by Privileged Container/Sensitive Mount by Container rules. [[#159](https://github.com/draios/falco/pull/159)]
+* Add fail2ban-server as a program that can spawn shells. Thanks @jcoetzee for the fix. [[#168](https://github.com/draios/falco/pull/168)]
+* Add systemd as a program that can access sensitive files. Thanks @jcoetzee for the fix. [[#169](https://github.com/draios/falco/pull/169)]
+* Add apt/apt-get as programs that can spawn shells. Thanks @jcoetzee for the fix. [[#170](https://github.com/draios/falco/pull/170)]
+
 ## v0.4.0
 
 Released 2016-10-25

CMakeLists.txt

@@ -47,99 +47,193 @@ set(SYSDIG_DIR "${PROJECT_SOURCE_DIR}/../sysdig")
 
 include(ExternalProject)
 
-set(ZLIB_SRC "${PROJECT_BINARY_DIR}/zlib-prefix/src/zlib")
-message(STATUS "Using bundled zlib in '${ZLIB_SRC}'")
-set(ZLIB_INCLUDE "${ZLIB_SRC}")
-set(ZLIB_LIB "${ZLIB_SRC}/libz.a")
-ExternalProject_Add(zlib
+option(USE_BUNDLED_DEPS "Enable bundled dependencies instead of using the system ones" ON)
+
+#
+# zlib
+
+option(USE_BUNDLED_ZLIB "Enable building of the bundled zlib" ${USE_BUNDLED_DEPS})
+
+if(NOT USE_BUNDLED_ZLIB)
+	find_path(ZLIB_INCLUDE zlib.h PATH_SUFFIXES zlib)
+	find_library(ZLIB_LIB NAMES z)
+	if(ZLIB_INCLUDE AND ZLIB_LIB)
+		message(STATUS "Found zlib: include: ${ZLIB_INCLUDE}, lib: ${ZLIB_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system zlib")
+	endif()
+else()
+        set(ZLIB_SRC "${PROJECT_BINARY_DIR}/zlib-prefix/src/zlib")
+        message(STATUS "Using bundled zlib in '${ZLIB_SRC}'")
+        set(ZLIB_INCLUDE "${ZLIB_SRC}")
+        set(ZLIB_LIB "${ZLIB_SRC}/libz.a")
+        ExternalProject_Add(zlib
 		URL "http://download.draios.com/dependencies/zlib-1.2.8.tar.gz"
 		URL_MD5 "44d667c142d7cda120332623eab69f40"
 		CONFIGURE_COMMAND "./configure"
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
-set(JQ_SRC "${PROJECT_BINARY_DIR}/jq-prefix/src/jq")
-message(STATUS "Using bundled jq in '${JQ_SRC}'")
-set(JQ_INCLUDE "${JQ_SRC}")
-set(JQ_LIB "${JQ_SRC}/.libs/libjq.a")
-ExternalProject_Add(jq
+#
+# jq
+#
+option(USE_BUNDLED_JQ "Enable building of the bundled jq" ${USE_BUNDLED_DEPS})
+if(NOT USE_BUNDLED_JQ)
+	find_path(JQ_INCLUDE jq.h PATH_SUFFIXES jq)
+	find_library(JQ_LIB NAMES jq)
+	if(JQ_INCLUDE AND JQ_LIB)
+		message(STATUS "Found jq: include: ${JQ_INCLUDE}, lib: ${JQ_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system jq")
+	endif()
+else()
+        set(JQ_SRC "${PROJECT_BINARY_DIR}/jq-prefix/src/jq")
+        message(STATUS "Using bundled jq in '${JQ_SRC}'")
+        set(JQ_INCLUDE "${JQ_SRC}")
+        set(JQ_LIB "${JQ_SRC}/.libs/libjq.a")
+        ExternalProject_Add(jq
                 URL "http://download.draios.com/dependencies/jq-1.5.tar.gz"
                 URL_MD5 "0933532b086bd8b6a41c1b162b1731f9"
                 CONFIGURE_COMMAND ./configure --disable-maintainer-mode --enable-all-static --disable-dependency-tracking
                 BUILD_COMMAND ${CMD_MAKE} LDFLAGS=-all-static
                 BUILD_IN_SOURCE 1
                 INSTALL_COMMAND "")
+endif()
 
 set(JSONCPP_SRC "${SYSDIG_DIR}/userspace/libsinsp/third-party/jsoncpp")
 set(JSONCPP_INCLUDE "${JSONCPP_SRC}")
 set(JSONCPP_LIB_SRC "${JSONCPP_SRC}/jsoncpp.cpp")
 
+#
+# curses
+#
 # we pull this in because libsinsp won't build without it
-set(CURSES_BUNDLE_DIR "${PROJECT_BINARY_DIR}/ncurses-prefix/src/ncurses")
-set(CURSES_INCLUDE_DIR "${CURSES_BUNDLE_DIR}/include/")
-set(CURSES_LIBRARIES "${CURSES_BUNDLE_DIR}/lib/libncurses.a")
-message(STATUS "Using bundled ncurses in '${CURSES_BUNDLE_DIR}'")
-ExternalProject_Add(ncurses
+
+option(USE_BUNDLED_NCURSES "Enable building of the bundled ncurses" ${USE_BUNDLED_DEPS})
+
+if(NOT USE_BUNDLED_NCURSES)
+	set(CURSES_NEED_NCURSES TRUE)
+	find_package(Curses REQUIRED)
+	message(STATUS "Found ncurses: include: ${CURSES_INCLUDE_DIR}, lib: ${CURSES_LIBRARIES}")
+else()
+	set(CURSES_BUNDLE_DIR "${PROJECT_BINARY_DIR}/ncurses-prefix/src/ncurses")
+	set(CURSES_INCLUDE_DIR "${CURSES_BUNDLE_DIR}/include/")
+	set(CURSES_LIBRARIES "${CURSES_BUNDLE_DIR}/lib/libncurses.a")
+	message(STATUS "Using bundled ncurses in '${CURSES_BUNDLE_DIR}'")
+	ExternalProject_Add(ncurses
 		URL "http://download.draios.com/dependencies/ncurses-6.0-20150725.tgz"
 		URL_MD5 "32b8913312e738d707ae68da439ca1f4"
 		CONFIGURE_COMMAND ./configure --without-cxx --without-cxx-binding --without-ada --without-manpages --without-progs --without-tests --with-terminfo-dirs=/etc/terminfo:/lib/terminfo:/usr/share/terminfo
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
+#
+# libb64
+#
+option(USE_BUNDLED_B64 "Enable building of the bundled b64" ${USE_BUNDLED_DEPS})
 
-set(B64_SRC "${PROJECT_BINARY_DIR}/b64-prefix/src/b64")
-message(STATUS "Using bundled b64 in '${B64_SRC}'")
-set(B64_INCLUDE "${B64_SRC}/include")
-set(B64_LIB "${B64_SRC}/src/libb64.a")
-ExternalProject_Add(b64
+if(NOT USE_BUNDLED_B64)
+	find_path(B64_INCLUDE NAMES b64/encode.h)
+	find_library(B64_LIB NAMES b64)
+	if(B64_INCLUDE AND B64_LIB)
+		message(STATUS "Found b64: include: ${B64_INCLUDE}, lib: ${B64_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system b64")
+	endif()
+else()
+	set(B64_SRC "${PROJECT_BINARY_DIR}/b64-prefix/src/b64")
+	message(STATUS "Using bundled b64 in '${B64_SRC}'")
+	set(B64_INCLUDE "${B64_SRC}/include")
+	set(B64_LIB "${B64_SRC}/src/libb64.a")
+	ExternalProject_Add(b64
 		URL "http://download.draios.com/dependencies/libb64-1.2.src.zip"
 		URL_MD5 "a609809408327117e2c643bed91b76c5"
 		CONFIGURE_COMMAND ""
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
+#
+# yamlcpp
+#
+option(USE_BUNDLED_YAMLCPP "Enable building of the bundled yamlcpp" ${USE_BUNDLED_DEPS})
 
-set(YAMLCPP_SRC "${PROJECT_BINARY_DIR}/yamlcpp-prefix/src/yamlcpp")
-message(STATUS "Using bundled yaml-cpp in '${YAMLCPP_SRC}'")
-set(YAMLCPP_LIB "${YAMLCPP_SRC}/libyaml-cpp.a")
-set(YAMLCPP_INCLUDE_DIR "${YAMLCPP_SRC}/include")
-# Once the next version of yaml-cpp is released (first version not requiring
-# boost), we can switch to that and no longer pull from github.
-ExternalProject_Add(yamlcpp
+if(NOT USE_BUNDLED_YAMLCPP)
+	find_path(YAMLCPP_INCLUDE_DIR NAMES yaml-cpp/yaml.h)
+	find_library(YAMLCPP_LIB NAMES yaml-cpp)
+	if(YAMLCPP_INCLUDE_DIR AND YAMLCPP_LIB)
+		message(STATUS "Found yamlcpp: include: ${YAMLCPP_INCLUDE_DIR}, lib: ${YAMLCPP_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system yamlcpp")
+	endif()
+else()
+	set(YAMLCPP_SRC "${PROJECT_BINARY_DIR}/yamlcpp-prefix/src/yamlcpp")
+	message(STATUS "Using bundled yaml-cpp in '${YAMLCPP_SRC}'")
+	set(YAMLCPP_LIB "${YAMLCPP_SRC}/libyaml-cpp.a")
+	set(YAMLCPP_INCLUDE_DIR "${YAMLCPP_SRC}/include")
+	# Once the next version of yaml-cpp is released (first version not requiring
+	# boost), we can switch to that and no longer pull from github.
+	ExternalProject_Add(yamlcpp
 		GIT_REPOSITORY "https://github.com/jbeder/yaml-cpp.git"
 		GIT_TAG "7d2873ce9f2202ea21b6a8c5ecbc9fe38032c229"
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
-set(OPENSSL_BUNDLE_DIR "${PROJECT_BINARY_DIR}/openssl-prefix/src/openssl")
-set(OPENSSL_INSTALL_DIR "${OPENSSL_BUNDLE_DIR}/target")
-set(OPENSSL_INCLUDE_DIR "${PROJECT_BINARY_DIR}/openssl-prefix/src/openssl/include")
-set(OPENSSL_LIBRARY_SSL "${OPENSSL_INSTALL_DIR}/lib/libssl.a")
-set(OPENSSL_LIBRARY_CRYPTO "${OPENSSL_INSTALL_DIR}/lib/libcrypto.a")
+#
+# OpenSSL
+#
+option(USE_BUNDLED_OPENSSL "Enable building of the bundled OpenSSL" ${USE_BUNDLED_DEPS})
 
-message(STATUS "Using bundled openssl in '${OPENSSL_BUNDLE_DIR}'")
+if(NOT USE_BUNDLED_OPENSSL)
+	find_package(OpenSSL REQUIRED)
+	message(STATUS "Found OpenSSL: include: ${OPENSSL_INCLUDE_DIR}, lib: ${OPENSSL_LIBRARIES}")
+else()
 
-ExternalProject_Add(openssl
+	set(OPENSSL_BUNDLE_DIR "${PROJECT_BINARY_DIR}/openssl-prefix/src/openssl")
+	set(OPENSSL_INSTALL_DIR "${OPENSSL_BUNDLE_DIR}/target")
+	set(OPENSSL_INCLUDE_DIR "${PROJECT_BINARY_DIR}/openssl-prefix/src/openssl/include")
+	set(OPENSSL_LIBRARY_SSL "${OPENSSL_INSTALL_DIR}/lib/libssl.a")
+	set(OPENSSL_LIBRARY_CRYPTO "${OPENSSL_INSTALL_DIR}/lib/libcrypto.a")
+
+	message(STATUS "Using bundled openssl in '${OPENSSL_BUNDLE_DIR}'")
+
+	ExternalProject_Add(openssl
 		URL "http://download.draios.com/dependencies/openssl-1.0.2d.tar.gz"
 		URL_MD5 "38dd619b2e77cbac69b99f52a053d25a"
 		CONFIGURE_COMMAND ./config shared --prefix=${OPENSSL_INSTALL_DIR}
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND ${CMD_MAKE} install)
+endif()
 
-set(CURL_SSL_OPTION "--with-ssl=${OPENSSL_INSTALL_DIR}")
+#
+# libcurl
+#
+option(USE_BUNDLED_CURL "Enable building of the bundled curl" ${USE_BUNDLED_DEPS})
 
+if(NOT USE_BUNDLED_CURL)
+	find_package(CURL REQUIRED)
+	message(STATUS "Found CURL: include: ${CURL_INCLUDE_DIR}, lib: ${CURL_LIBRARIES}")
+else()
+	set(CURL_BUNDLE_DIR "${PROJECT_BINARY_DIR}/curl-prefix/src/curl")
+	set(CURL_INCLUDE_DIR "${CURL_BUNDLE_DIR}/include/")
+	set(CURL_LIBRARIES "${CURL_BUNDLE_DIR}/lib/.libs/libcurl.a")
 
-set(CURL_BUNDLE_DIR "${PROJECT_BINARY_DIR}/curl-prefix/src/curl")
-set(CURL_INCLUDE_DIR "${CURL_BUNDLE_DIR}/include/")
-set(CURL_LIBRARIES "${CURL_BUNDLE_DIR}/lib/.libs/libcurl.a")
-message(STATUS "Using bundled curl in '${CURL_BUNDLE_DIR}'")
-message(STATUS "Using SSL for curl in '${CURL_SSL_OPTION}'")
+	if(NOT USE_BUNDLED_OPENSSL)
+		set(CURL_SSL_OPTION "--with-ssl")
+	else()
+		set(CURL_SSL_OPTION "--with-ssl=${OPENSSL_INSTALL_DIR}")
+                message(STATUS "Using bundled curl in '${CURL_BUNDLE_DIR}'")
+                message(STATUS "Using SSL for curl in '${CURL_SSL_OPTION}'")
+	endif()
 
-ExternalProject_Add(curl
+	ExternalProject_Add(curl
 		DEPENDS openssl
 		URL "http://download.draios.com/dependencies/curl-7.45.0.tar.bz2"
 		URL_MD5 "62c1a352b28558f25ba6209214beadc8"
@@ -147,50 +241,120 @@ ExternalProject_Add(curl
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
-set(LUAJIT_SRC "${PROJECT_BINARY_DIR}/luajit-prefix/src/luajit/src")
-message(STATUS "Using bundled LuaJIT in '${LUAJIT_SRC}'")
-set(LUAJIT_INCLUDE "${LUAJIT_SRC}")
-set(LUAJIT_LIB "${LUAJIT_SRC}/libluajit.a")
-ExternalProject_Add(luajit
+#
+# LuaJIT
+#
+option(USE_BUNDLED_LUAJIT "Enable building of the bundled LuaJIT" ${USE_BUNDLED_DEPS})
+
+if(NOT USE_BUNDLED_LUAJIT)
+	find_path(LUAJIT_INCLUDE luajit.h PATH_SUFFIXES luajit-2.0 luajit)
+	find_library(LUAJIT_LIB NAMES luajit luajit-5.1)
+	if(LUAJIT_INCLUDE AND LUAJIT_LIB)
+		message(STATUS "Found LuaJIT: include: ${LUAJIT_INCLUDE}, lib: ${LUAJIT_LIB}")
+	else()
+		# alternatively try stock Lua
+		find_package(Lua51)
+		set(LUAJIT_LIB ${LUA_LIBRARY})
+		set(LUAJIT_INCLUDE ${LUA_INCLUDE_DIR})
+
+		if(NOT ${LUA51_FOUND})
+			message(FATAL_ERROR "Couldn't find system LuaJIT or Lua")
+		endif()
+	endif()
+else()
+	set(LUAJIT_SRC "${PROJECT_BINARY_DIR}/luajit-prefix/src/luajit/src")
+	message(STATUS "Using bundled LuaJIT in '${LUAJIT_SRC}'")
+	set(LUAJIT_INCLUDE "${LUAJIT_SRC}")
+	set(LUAJIT_LIB "${LUAJIT_SRC}/libluajit.a")
+	ExternalProject_Add(luajit
 		URL "http://download.draios.com/dependencies/LuaJIT-2.0.3.tar.gz"
 		URL_MD5 "f14e9104be513913810cd59c8c658dc0"
 		CONFIGURE_COMMAND ""
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
-set (LPEG_SRC "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg")
-set (LPEG_LIB "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg/build/lpeg.a")
-ExternalProject_Add(lpeg
-                    DEPENDS luajit
-		    URL "http://s3.amazonaws.com/download.draios.com/dependencies/lpeg-1.0.0.tar.gz"
-                    URL_MD5 "0aec64ccd13996202ad0c099e2877ece"
-                    BUILD_COMMAND LUA_INCLUDE=${LUAJIT_INCLUDE} "${PROJECT_SOURCE_DIR}/scripts/build-lpeg.sh" "${LPEG_SRC}/build"
-                    BUILD_IN_SOURCE 1
-		    CONFIGURE_COMMAND ""
-                    INSTALL_COMMAND "")
+#
+# Lpeg
+#
+option(USE_BUNDLED_LPEG "Enable building of the bundled lpeg" ${USE_BUNDLED_DEPS})
 
+if(NOT USE_BUNDLED_LPEG)
+	find_library(LPEG_LIB NAMES lpeg.a)
+	if(LPEG_LIB)
+		message(STATUS "Found lpeg: lib: ${LPEG_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system lpeg")
+	endif()
+else()
+	set(LPEG_SRC "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg")
+	set(LPEG_LIB "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg/build/lpeg.a")
+	ExternalProject_Add(lpeg
+                DEPENDS luajit
+		URL "http://s3.amazonaws.com/download.draios.com/dependencies/lpeg-1.0.0.tar.gz"
+                URL_MD5 "0aec64ccd13996202ad0c099e2877ece"
+                BUILD_COMMAND LUA_INCLUDE=${LUAJIT_INCLUDE} "${PROJECT_SOURCE_DIR}/scripts/build-lpeg.sh" "${LPEG_SRC}/build"
+                BUILD_IN_SOURCE 1
+		CONFIGURE_COMMAND ""
+                INSTALL_COMMAND "")
+endif()
 
-set (LIBYAML_SRC "${PROJECT_BINARY_DIR}/libyaml-prefix/src/libyaml/src")
-set(LIBYAML_LIB "${LIBYAML_SRC}/.libs/libyaml.a")
-ExternalProject_Add(libyaml
-		    URL "http://download.draios.com/dependencies/libyaml-0.1.4.tar.gz"
-                    URL_MD5 "4a4bced818da0b9ae7fc8ebc690792a7"
-                    BUILD_COMMAND ${CMD_MAKE}
-                    BUILD_IN_SOURCE 1
-		    CONFIGURE_COMMAND ./bootstrap && ./configure
-                    INSTALL_COMMAND "")
+#
+# Libyaml
+#
+option(USE_BUNDLED_LIBYAML "Enable building of the bundled libyaml" ${USE_BUNDLED_DEPS})
 
-set (LYAML_SRC "${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/ext/yaml")
-set(LYAML_LIB "${LYAML_SRC}/.libs/yaml.a")
-ExternalProject_Add(lyaml
-		    URL "http://download.draios.com/dependencies/lyaml-release-v6.0.tar.gz"
-                    URL_MD5 "dc3494689a0dce7cf44e7a99c72b1f30"
-                    BUILD_COMMAND ${CMD_MAKE}
-                    BUILD_IN_SOURCE 1
-		    CONFIGURE_COMMAND ./configure --enable-static LIBS=-L../../../libyaml-prefix/src/libyaml/src/.libs CFLAGS=-I../../../libyaml-prefix/src/libyaml/include CPPFLAGS=-I../../../libyaml-prefix/src/libyaml/include LUA_INCLUDE=-I../../../luajit-prefix/src/luajit/src LUA=../../../luajit-prefix/src/luajit/src/luajit
-                    INSTALL_COMMAND sh -c "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/engine/lua")
+if(NOT USE_BUNDLED_LIBYAML)
+	# Note: to distinguish libyaml.a and yaml.a we specify a full
+	# file name here, so you'll have to arrange for static
+	# libraries being available.
+	find_library(LIBYAML_LIB NAMES libyaml.a)
+	if(LIBYAML_LIB)
+		message(STATUS "Found libyaml: lib: ${LIBYAML_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system libyaml")
+	endif()
+else()
+	set(LIBYAML_SRC "${PROJECT_BINARY_DIR}/libyaml-prefix/src/libyaml/src")
+	set(LIBYAML_LIB "${LIBYAML_SRC}/.libs/libyaml.a")
+	ExternalProject_Add(libyaml
+		URL "http://download.draios.com/dependencies/libyaml-0.1.4.tar.gz"
+                URL_MD5 "4a4bced818da0b9ae7fc8ebc690792a7"
+                BUILD_COMMAND ${CMD_MAKE}
+                BUILD_IN_SOURCE 1
+		CONFIGURE_COMMAND ./bootstrap && ./configure
+                INSTALL_COMMAND "")
+endif()
+
+#
+# lyaml
+#
+option(USE_BUNDLED_LYAML "Enable building of the bundled lyaml" ${USE_BUNDLED_DEPS})
+
+if(NOT USE_BUNDLED_LYAML)
+	# Note: to distinguish libyaml.a and yaml.a we specify a full
+	# file name here, so you'll have to arrange for static
+	# libraries being available.
+	find_library(LYAML_LIB NAMES yaml.a)
+	if(LYAML_LIB)
+		message(STATUS "Found lyaml: lib: ${LYAML_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system lyaml")
+	endif()
+else()
+	set(LYAML_SRC "${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/ext/yaml")
+	set(LYAML_LIB "${LYAML_SRC}/.libs/yaml.a")
+	ExternalProject_Add(lyaml
+		URL "http://download.draios.com/dependencies/lyaml-release-v6.0.tar.gz"
+                URL_MD5 "dc3494689a0dce7cf44e7a99c72b1f30"
+                BUILD_COMMAND ${CMD_MAKE}
+                BUILD_IN_SOURCE 1
+		CONFIGURE_COMMAND ./configure --enable-static LIBS=-L../../../libyaml-prefix/src/libyaml/src/.libs CFLAGS=-I../../../libyaml-prefix/src/libyaml/include CPPFLAGS=-I../../../libyaml-prefix/src/libyaml/include LUA_INCLUDE=-I../../../luajit-prefix/src/luajit/src LUA=../../../luajit-prefix/src/luajit/src/luajit
+                INSTALL_COMMAND sh -c "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/engine/lua")
+endif()
 
 install(FILES falco.yaml
 	DESTINATION "${FALCO_ETC_DIR}")
@@ -201,7 +365,7 @@ add_subdirectory("${SYSDIG_DIR}/userspace/libsinsp" "${PROJECT_BINARY_DIR}/users
 
 add_subdirectory(scripts)
 set(FALCO_SINSP_LIBRARY sinsp)
-set(FALCO_SHARE_DIR share/falco)
+set(FALCO_SHARE_DIR ${CMAKE_INSTALL_PREFIX}/share/falco)
 add_subdirectory(userspace/engine)
 add_subdirectory(userspace/falco)
 

README.md

@@ -2,7 +2,7 @@
 
 ####Latest release
 
-**v0.4.0**
+**v0.5.0**
 Read the [change log](https://github.com/draios/falco/blob/dev/CHANGELOG.md)
 
 Dev Branch: [![Build Status](https://travis-ci.org/draios/falco.svg?branch=dev)](https://travis-ci.org/draios/falco)<br />
@@ -22,6 +22,11 @@ Falco can detect and alert on any behavior that involves making Linux system cal
 - A non-device file is written to `/dev`
 - A standard system binary (like `ls`) makes an outbound network connection
 
+#### How Falco Compares to Other Security Tools like SELinux, Auditd, etc.
+
+One of the questions we often get when we talk about Sysdig Falco is “How does it compare to other tools like SELinux, AppArmor, Auditd, etc. that also have security policies?”. We wrote a [blog post](https://sysdig.com/blog/selinux-seccomp-falco-technical-discussion/) comparing Falco to other tools.
+
+
 Documentation
 ---
 [Visit the wiki] (https://github.com/draios/falco/wiki) for full documentation on falco.

docker/dev/docker-entrypoint.sh

@@ -1,13 +1,17 @@
 #!/bin/bash
 #set -e
 
-echo "* Setting up /usr/src links from host"
+# Set the SYSDIG_SKIP_LOAD variable to skip loading the sysdig kernel module
 
-for i in $(ls $SYSDIG_HOST_ROOT/usr/src)
-do 
-	ln -s $SYSDIG_HOST_ROOT/usr/src/$i /usr/src/$i
-done
+if [[ -z "${SYSDIG_SKIP_LOAD}" ]]; then
+    echo "* Setting up /usr/src links from host"
 
-/usr/bin/sysdig-probe-loader
+    for i in $(ls $SYSDIG_HOST_ROOT/usr/src)
+    do
+        ln -s $SYSDIG_HOST_ROOT/usr/src/$i /usr/src/$i
+    done
+
+    /usr/bin/sysdig-probe-loader
+fi
 
 exec "$@"

docker/stable/docker-entrypoint.sh

@@ -1,13 +1,17 @@
 #!/bin/bash
 #set -e
 
-echo "* Setting up /usr/src links from host"
+# Set the SYSDIG_SKIP_LOAD variable to skip loading the sysdig kernel module
 
-for i in $(ls $SYSDIG_HOST_ROOT/usr/src)
-do 
-	ln -s $SYSDIG_HOST_ROOT/usr/src/$i /usr/src/$i
-done
+if [[ -z "${SYSDIG_SKIP_LOAD}" ]]; then
+    echo "* Setting up /usr/src links from host"
 
-/usr/bin/sysdig-probe-loader
+    for i in $(ls $SYSDIG_HOST_ROOT/usr/src)
+    do
+        ln -s $SYSDIG_HOST_ROOT/usr/src/$i /usr/src/$i
+    done
+
+    /usr/bin/sysdig-probe-loader
+fi
 
 exec "$@"

falco.yaml

@@ -9,6 +9,27 @@ json_output: false
 log_stderr: true
 log_syslog: true
 
+# Minimum log level to include in logs. Note: these levels are
+# separate from the priority field of rules. This refers only to the
+# log level of falco's internal logging. Can be one of "emergency",
+# "alert", "critical", "error", "warning", "notice", "info", "debug".
+log_level: info
+
+# A throttling mechanism implemented as a token bucket limits the
+# rate of falco notifications. This throttling is controlled by the following configuration
+# options:
+#  - rate: the number of tokens (i.e. right to send a notification)
+#    gained per second. Defaults to 1.
+#  - max_burst: the maximum number of tokens outstanding. Defaults to 1000.
+#
+# With these defaults, falco could send up to 1000 notifications after
+# an initial quiet period, and then up to 1 notification per second
+# afterward. It would gain the full burst back after 1000 seconds of
+# no activity.
+
+outputs:
+  rate: 1
+  max_burst: 1000
 
 # Where security notifications should go.
 # Multiple outputs can be enabled.

rules/falco_rules.yaml

@@ -76,7 +76,7 @@
 
 # dpkg -L login | grep bin | xargs ls -ld | grep -v '^d' | awk '{print $9}' | xargs -L 1 basename | tr "\\n" ","
 - list: login_binaries
-  items: [login, systemd-logind, su, nologin, faillog, lastlog, newgrp, sg]
+  items: [login, systemd, systemd-logind, su, nologin, faillog, lastlog, newgrp, sg]
 
 # dpkg -L passwd | grep bin | xargs ls -ld | grep -v '^d' | awk '{print $9}' | xargs -L 1 basename | tr "\\n" ","
 - list: passwd_binaries
@@ -116,7 +116,7 @@
 # The truncated dpkg-preconfigu is intentional, process names are
 # truncated at the sysdig level.
 - list: package_mgmt_binaries
-  items: [dpkg, dpkg-preconfigu, rpm, rpmkey, yum, frontend]
+  items: [dpkg, dpkg-preconfigu, dnf, rpm, rpmkey, yum, frontend]
 
 - macro: package_mgmt_procs
   condition: proc.name in (package_mgmt_binaries)
@@ -236,7 +236,7 @@
 # Only let rpm-related programs write to the rpm database
 - rule: Write below rpm database
   desc: an attempt to write to the rpm database by any non-rpm related program
-  condition: fd.name startswith /var/lib/rpm and open_write and not proc.name in (rpm,rpmkey,yum)
+  condition: fd.name startswith /var/lib/rpm and open_write and not proc.name in (dnf,rpm,rpmkey,yum)
   output: "Rpm database opened for writing by a non-rpm program (command=%proc.cmdline file=%fd.name)"
   priority: WARNING
 
@@ -282,17 +282,17 @@
 
 - rule: Run shell untrusted
   desc: an attempt to spawn a shell by a non-shell program. Exceptions are made for trusted binaries.
-  condition: spawned_process and not container and shell_procs and proc.pname exists and not proc.pname in (cron_binaries, shell_binaries, sshd, sudo, docker_binaries, k8s_binaries, su, tmux, screen, emacs, systemd, login, flock, fbash, nginx, monit, supervisord, dragent, aws, initdb, docker-compose, make, configure, awk, falco)
+  condition: spawned_process and not container and shell_procs and proc.pname exists and not proc.pname in (cron_binaries, shell_binaries, sshd, sudo, docker_binaries, k8s_binaries, su, tmux, screen, emacs, systemd, login, flock, fbash, nginx, monit, supervisord, dragent, aws, initdb, docker-compose, make, configure, awk, falco, fail2ban-server, apt-get, apt)
   output: "Shell spawned by untrusted binary (user=%user.name shell=%proc.name parent=%proc.pname cmdline=%proc.cmdline)"
   priority: WARNING
 
 - macro: trusted_containers
-  condition: (container.image startswith sysdig/agent or container.image startswith sysdig/falco or container.image startswith sysdig/sysdig or container.image startswith gcr.io/google_containers/hyperkube)
+  condition: (container.image startswith sysdig/agent or container.image startswith sysdig/falco or container.image startswith sysdig/sysdig or container.image startswith gcr.io/google_containers/hyperkube or container.image startswith gcr.io/google_containers/kube-proxy)
 
 - rule: File Open by Privileged Container
   desc: Any open by a privileged container. Exceptions are made for known trusted images.
   condition: (open_read or open_write) and container and container.privileged=true and not trusted_containers
-  output: File opened for read/write by non-privileged container (user=%user.name command=%proc.cmdline %container.info file=%fd.name)
+  output: File opened for read/write by privileged container (user=%user.name command=%proc.cmdline %container.info file=%fd.name)
   priority: WARNING
 
 - macro: sensitive_mount

test/cpu_monitor.sh

@@ -6,4 +6,4 @@ VARIANT=$3
 RESULTS_FILE=$4
 CPU_INTERVAL=$5
 
-top -d $CPU_INTERVAL -b -p $SUBJ_PID | grep -E '(falco|sysdig|dragent)' --line-buffered | awk -v benchmark=$BENCHMARK -v variant=$VARIANT '{printf("{\"time\": \"%s\", \"sample\": %d, \"benchmark\": \"%s\", \"variant\": \"%s\", \"cpu_usage\": %s},\n", strftime("%Y-%m-%d %H:%M:%S", systime(), 1), NR, benchmark, variant, $9); fflush();}' >> $RESULTS_FILE
+top -d $CPU_INTERVAL -b -p $SUBJ_PID | grep -E '(falco|sysdig|dragent|test_mm)' --line-buffered | awk -v benchmark=$BENCHMARK -v variant=$VARIANT '{printf("{\"time\": \"%s\", \"sample\": %d, \"benchmark\": \"%s\", \"variant\": \"%s\", \"cpu_usage\": %s},\n", strftime("%Y-%m-%d %H:%M:%S", systime(), 1), NR, benchmark, variant, $9); fflush();}' >> $RESULTS_FILE

test/falco_test.py

@@ -17,6 +17,8 @@ class FalcoTest(Test):
         """
         self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build'))
 
+        self.stderr_contains = self.params.get('stderr_contains', '*', default='')
+        self.exit_status = self.params.get('exit_status', '*', default=0)
         self.should_detect = self.params.get('detect', '*', default=False)
         self.trace_file = self.params.get('trace_file', '*')
 
@@ -197,9 +199,18 @@ class FalcoTest(Test):
 
         res = self.falco_proc.run(timeout=180, sig=9)
 
+        if self.stderr_contains != '':
+            match = re.search(self.stderr_contains, res.stderr)
+            if match is None:
+                self.fail("Stderr of falco process did not contain content matching {}".format(self.stderr_contains))
+
+        if res.exit_status != self.exit_status:
+            self.error("Falco command \"{}\" exited with unexpected return value {} (!= {})".format(
+                cmd, res.exit_status, self.exit_status))
+
+        # No need to check any outputs if the falco process exited abnormally.
         if res.exit_status != 0:
-            self.error("Falco command \"{}\" exited with non-zero return value {}".format(
-                cmd, res.exit_status))
+            return
 
         self.check_rules_warnings(res)
         if len(self.rules_events) > 0:

test/falco_tests.yaml.in

@@ -95,6 +95,13 @@ trace_files: !mux
       - rules/double_rule.yaml
     trace_file: trace_files/cat_write.scap
 
+  invalid_rule_output:
+    exit_status: 1
+    stderr_contains: "Runtime error: Error loading rules:.* Invalid output format 'An open was seen %not_a_real_field': 'invalid formatting token not_a_real_field'. Exiting."
+    rules_file:
+      - rules/invalid_rule_output.yaml
+    trace_file: trace_files/cat_write.scap
+
   disabled_rules:
     detect: False
     rules_file:

test/plot-live.r

@@ -13,23 +13,35 @@ if (substr(script.basename, 1, 1) != '/') {
 
 results = paste(script.basename, "results.json", sep='/')
 output = "./output.png"
+metric = "cpu"
 
 GetoptLong(
     "results=s", "Path to results file",
     "benchmark=s", "Benchmark from results file to graph",
     "variant=s@", "Variant(s) to include in graph. Can be specified multiple times",
-    "output=s", "Output graph file"
+    "output=s", "Output graph file",
+    "metric=s", "Metric to graph. Can be one of (cpu|drops)"
 )
 
+if (metric == "cpu") {
+    data_metric="cpu_usage"
+    yaxis_label="CPU Usage (%)"
+    title="Falco/Sysdig/Multimatch CPU Usage: %s"
+} else if (metric == "drops") {
+    data_metric="drop_pct"
+    yaxis_label="Event Drops (%)"
+    title="Falco/Sysdig/Multimatch Event Drops: %s"
+}
+
 res <- fromJSON(results, flatten=TRUE)
 
 res2 = res[res$benchmark == benchmark & res$variant %in% variant,]
 
-plot <- ggplot(data=res2, aes(x=sample, y=cpu_usage, group=variant, colour=variant)) +
+plot <- ggplot(data=res2, aes(x=sample, y=get(data_metric), group=variant, colour=variant)) +
     geom_line() +
-    ylab("CPU Usage (%)") +
+    ylab(yaxis_label) +
     xlab("Time") +
-    ggtitle(sprintf("Falco/Sysdig CPU Usage: %s", benchmark))
+    ggtitle(sprintf(title, benchmark))
     theme(legend.position=c(.2, .88));
 
 print(paste("Writing graph to", output, sep=" "))

test/rules/invalid_rule_output.yaml

@@ -0,0 +1,5 @@
+- rule: rule_with_invalid_output
+  desc: A rule with an invalid output field
+  condition: evt.type=open
+  output: "An open was seen %not_a_real_field"
+  priority: WARNING

test/run_performance_tests.sh

@@ -28,7 +28,11 @@ function time_cmd() {
 function run_falco_on() {
     file="$1"
 
-    cmd="$ROOT/userspace/falco/falco -c $ROOT/../falco.yaml -r $ROOT/../rules/falco_rules.yaml --option=stdout_output.enabled=false -e $file"
+    if [ -z $RULES_FILE ]; then
+	RULES_FILE=$SOURCE/rules/falco_rules.yaml
+    fi
+
+    cmd="$ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -r $SOURCE/rules/falco_rules.yaml --option=stdout_output.enabled=false -e $file -A"
 
     time_cmd "$cmd" "$file"
 }
@@ -131,15 +135,26 @@ function start_monitor_cpu_usage() {
 
 function start_subject_prog() {
 
-    echo "   starting falco/sysdig/agent program"
     # Do a blocking sudo command now just to ensure we have a password
     sudo bash -c ""
 
-    if [[ $ROOT == *"falco"* ]]; then
-	sudo $ROOT/userspace/falco/falco -c $ROOT/../falco.yaml -r $ROOT/../rules/falco_rules.yaml --option=stdout_output.enabled=false > ./prog-output.txt 2>&1 &
+    if [[ $ROOT == *"multimatch"* ]]; then
+	echo "   starting test_mm..."
+	if [ -z $RULES_FILE ]; then
+	    RULES_FILE=$SOURCE/../output/rules.yaml
+	fi
+	sudo FALCO_STATS_EXTRA_variant=$VARIANT FALCO_STATS_EXTRA_benchmark=$live_test $ROOT/test_mm -S $SOURCE/search_order.yaml -s $STATS_FILE -r $RULES_FILE > ./prog-output.txt 2>&1 &
+    elif [[ $ROOT == *"falco"* ]]; then
+	echo "   starting falco..."
+	if [ -z $RULES_FILE ]; then
+	    RULES_FILE=$SOURCE/rules/falco_rules.yaml
+	fi
+	sudo FALCO_STATS_EXTRA_variant=$VARIANT FALCO_STATS_EXTRA_benchmark=$live_test $ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -s $STATS_FILE -r $RULES_FILE --option=stdout_output.enabled=false > ./prog-output.txt -A 2>&1 &
     elif [[ $ROOT == *"sysdig"* ]]; then
+	echo "   starting sysdig..."
 	sudo $ROOT/userspace/sysdig/sysdig -N -z evt.type=none &
     else
+	echo "   starting agent..."
 	write_agent_config
 	pushd $ROOT/userspace/dragent
 	sudo ./dragent > ./prog-output.txt 2>&1 &
@@ -180,8 +195,8 @@ function run_juttle_examples() {
 
 function run_kubernetes_demo() {
     pushd $SCRIPTDIR/../../infrastructure/test-infrastructures/kubernetes-demo
-    bash run-local.sh
-    bash init.sh
+    sudo bash run-local.sh
+    sudo bash init.sh
     sleep 600
     docker stop $(docker ps -qa)
     docker rm -fv $(docker ps -qa)
@@ -306,8 +321,11 @@ usage() {
     echo "   -h/--help: show this help"
     echo "   -v/--variant: a variant name to attach to this set of test results"
     echo "   -r/--root: root directory containing falco/sysdig binaries (i.e. where you ran 'cmake')"
+    echo "   -s/--source: root directory containing falco/sysdig source code"
     echo "   -R/--results: append test results to this file"
+    echo "   -S/--stats: append capture statistics to this file (only works for falco/test_mm)"
     echo "   -o/--output: append program output to this file"
+    echo "   -U/--rules: path to rules file (only applicable for falco/test_mm)"
     echo "   -t/--test: test to run. Argument has the following format:"
     echo "       trace:<trace>: read the specified trace file."
     echo "            trace:all means run all traces"
@@ -325,7 +343,7 @@ usage() {
     echo "   -F/--falco-agent: When running an agent, whether or not to enable falco"
 }
 
-OPTS=`getopt -o hv:r:R:o:t:T: --long help,variant:,root:,results:,output:,test:,tracedir:,agent-autodrop:,falco-agent: -n $0 -- "$@"`
+OPTS=`getopt -o hv:r:s:R:S:o:U:t:T: --long help,variant:,root:,source:,results:,stats:,output:,rules:,test:,tracedir:,agent-autodrop:,falco-agent: -n $0 -- "$@"`
 
 if [ $? != 0 ]; then
     echo "Exiting" >&2
@@ -336,9 +354,12 @@ eval set -- "$OPTS"
 
 VARIANT="falco"
 ROOT=`dirname $0`/../build
+SOURCE=$ROOT
 SCRIPTDIR=`dirname $0`
 RESULTS_FILE=`dirname $0`/results.json
+STATS_FILE=`dirname $0`/capture_stats.json
 OUTPUT_FILE=`dirname $0`/program-output.txt
+RULES_FILE=
 TEST=trace:all
 TRACEDIR=/tmp/falco-perf-traces.$USER
 CPU_INTERVAL=10
@@ -350,8 +371,11 @@ while true; do
 	-h | --help ) usage; exit 1;;
 	-v | --variant ) VARIANT="$2"; shift 2;;
 	-r | --root ) ROOT="$2"; shift 2;;
+	-s | --source ) SOURCE="$2"; shift 2;;
 	-R | --results ) RESULTS_FILE="$2"; shift 2;;
+	-S | --stats ) STATS_FILE="$2"; shift 2;;
 	-o | --output ) OUTPUT_FILE="$2"; shift 2;;
+	-U | --rules ) RULES_FILE="$2"; shift 2;;
 	-t | --test ) TEST="$2"; shift 2;;
 	-T | --tracedir ) TRACEDIR="$2"; shift 2;;
 	-A | --agent-autodrop ) AGENT_AUTODROP="$2"; shift 2;;
@@ -372,12 +396,23 @@ fi
 
 ROOT=`realpath $ROOT`
 
+if [ -z $SOURCE ]; then
+    echo "A source directory containing falco/sysdig source code. Not continuing."
+    exit 1
+fi
+
+SOURCE=`realpath $SOURCE`
 
 if [ -z $RESULTS_FILE ]; then
     echo "An output file for test results must be provided. Not continuing."
     exit 1
 fi
 
+if [ -z $STATS_FILE ]; then
+    echo "An output file for capture statistics must be provided. Not continuing."
+    exit 1
+fi
+
 if [ -z $OUTPUT_FILE ]; then
     echo "An file for program output must be provided. Not continuing."
     exit 1

userspace/engine/CMakeLists.txt

@@ -4,7 +4,7 @@ include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp")
 include_directories("${PROJECT_BINARY_DIR}/userspace/engine")
 include_directories("${LUAJIT_INCLUDE}")
 
-add_library(falco_engine STATIC rules.cpp falco_common.cpp falco_engine.cpp)
+add_library(falco_engine STATIC rules.cpp falco_common.cpp falco_engine.cpp token_bucket.cpp formats.cpp)
 
 target_include_directories(falco_engine PUBLIC
   "${LUAJIT_INCLUDE}")

userspace/engine/config_falco_engine.h.in

@@ -18,5 +18,5 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 
 #pragma once
 
-#define FALCO_ENGINE_LUA_DIR "${CMAKE_INSTALL_PREFIX}/${FALCO_SHARE_DIR}/lua/"
+#define FALCO_ENGINE_LUA_DIR "${FALCO_SHARE_DIR}/lua/"
 #define FALCO_ENGINE_SOURCE_LUA_DIR "${PROJECT_SOURCE_DIR}/../falco/userspace/engine/lua/"

userspace/engine/falco_engine.cpp

@@ -24,6 +24,8 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 #include "falco_engine.h"
 #include "config_falco_engine.h"
 
+#include "formats.h"
+
 extern "C" {
 #include "lpeg.h"
 #include "lyaml.h"
@@ -38,7 +40,8 @@ string lua_print_stats = "print_stats";
 using namespace std;
 
 falco_engine::falco_engine(bool seed_rng)
-	: m_rules(NULL), m_sampling_ratio(1), m_sampling_multiplier(0)
+	: m_rules(NULL), m_sampling_ratio(1), m_sampling_multiplier(0),
+	  m_replace_container_info(false)
 {
 	luaopen_lpeg(m_ls);
 	luaopen_yaml(m_ls);
@@ -72,7 +75,16 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al
 	{
 		m_rules = new falco_rules(m_inspector, this, m_ls);
 	}
-	m_rules->load_rules(rules_content, verbose, all_events);
+
+	// Note that falco_formats is added to both the lua state used
+	// by the falco engine as well as the separate lua state used
+	// by falco outputs.  Within the engine, only
+	// formats.formatter is used, so we can unconditionally set
+	// json_output to false.
+	bool json_output = false;
+	falco_formats::init(m_inspector, m_ls, json_output);
+
+	m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info);
 }
 
 void falco_engine::load_rules_file(const string &rules_filename, bool verbose, bool all_events)
@@ -98,20 +110,20 @@ void falco_engine::enable_rule(string &pattern, bool enabled)
 	m_evttype_filter.enable(pattern, enabled);
 }
 
-falco_engine::rule_result *falco_engine::process_event(sinsp_evt *ev)
+unique_ptr<falco_engine::rule_result> falco_engine::process_event(sinsp_evt *ev)
 {
 
 	if(should_drop_evt())
 	{
-		return NULL;
+		return unique_ptr<struct rule_result>();
 	}
 
 	if(!m_evttype_filter.run(ev))
 	{
-		return NULL;
+		return unique_ptr<struct rule_result>();
 	}
 
-	struct rule_result *res = new rule_result();
+	unique_ptr<struct rule_result> res(new rule_result());
 
 	lua_getglobal(m_ls, lua_on_event.c_str());
 
@@ -184,6 +196,12 @@ void falco_engine::set_sampling_multiplier(double sampling_multiplier)
 	m_sampling_multiplier = sampling_multiplier;
 }
 
+void falco_engine::set_extra(string &extra, bool replace_container_info)
+{
+	m_extra = extra;
+	m_replace_container_info = replace_container_info;
+}
+
 inline bool falco_engine::should_drop_evt()
 {
 	if(m_sampling_multiplier == 0)

userspace/engine/falco_engine.h

@@ -63,7 +63,7 @@ public:
 	// the rule that matched. If no rule matched, returns NULL.
 	//
 	// the reutrned rule_result is allocated and must be delete()d.
-	rule_result *process_event(sinsp_evt *ev);
+	std::unique_ptr<rule_result> process_event(sinsp_evt *ev);
 
 	//
 	// Print details on the given rule. If rule is NULL, print
@@ -96,6 +96,16 @@ public:
 	//
 	void set_sampling_multiplier(double sampling_multiplier);
 
+	//
+	// You can optionally add "extra" formatting fields to the end
+	// of all output expressions. You can also choose to replace
+	// %container.info with the extra information or add it to the
+	// end of the expression. This is used in open source falco to
+	// add k8s/mesos/container information to outputs when
+	// available.
+	//
+	void set_extra(string &extra, bool replace_container_info);
+
 private:
 
 	//
@@ -132,5 +142,8 @@ private:
 	double m_sampling_multiplier;
 
 	std::string m_lua_main_filename = "rule_loader.lua";
+
+	std::string m_extra;
+	bool m_replace_container_info;
 };
 

userspace/engine/formats.cpp

@@ -39,7 +39,7 @@ void falco_formats::init(sinsp* inspector, lua_State *ls, bool json_output)
 	s_inspector = inspector;
 	s_json_output = json_output;
 
-	luaL_openlib(ls, "falco", ll_falco, 0);
+	luaL_openlib(ls, "formats", ll_falco, 0);
 }
 
 int falco_formats::formatter(lua_State *ls)
@@ -49,14 +49,13 @@ int falco_formats::formatter(lua_State *ls)
 	try
 	{
 		formatter = new sinsp_evt_formatter(s_inspector, format);
+		lua_pushlightuserdata(ls, formatter);
 	}
 	catch(sinsp_exception& e)
 	{
-		throw falco_exception("Invalid output format '" + format + "'.\n");
+		luaL_error(ls, "Invalid output format '%s': '%s'", format.c_str(), e.what());
 	}
 
-	lua_pushlightuserdata(ls, formatter);
-
 	return 1;
 }
 
@@ -64,14 +63,14 @@ int falco_formats::free_formatter(lua_State *ls)
 {
 	if (!lua_islightuserdata(ls, -1))
 	{
-		throw falco_exception("Invalid argument passed to free_formatter");
+		luaL_error(ls, "Invalid argument passed to free_formatter");
 	}
 
 	sinsp_evt_formatter *formatter = (sinsp_evt_formatter *) lua_topointer(ls, 1);
 
 	delete(formatter);
 
-	return 1;
+	return 0;
 }
 
 int falco_formats::format_event (lua_State *ls)

userspace/engine/formats.h

@@ -43,7 +43,4 @@ class falco_formats
 	static int format_event(lua_State *ls);
 
 	static sinsp* s_inspector;
-
- private:
-	lua_State* m_ls;
 };

userspace/engine/lua/compiler.lua

@@ -290,8 +290,8 @@ function compiler.compile_macro(line, list_defs)
    local ast, error_msg = parser.parse_filter(line)
 
    if (error_msg) then
-      print ("Compilation error when compiling \""..line.."\": ", error_msg)
-      error(error_msg)
+      msg = "Compilation error when compiling \""..line.."\": ".. error_msg
+      error(msg)
    end
 
    -- Traverse the ast looking for events/syscalls in the ignored
@@ -315,8 +315,8 @@ function compiler.compile_filter(name, source, macro_defs, list_defs)
    local ast, error_msg = parser.parse_filter(source)
 
    if (error_msg) then
-      print ("Compilation error when compiling \""..source.."\": ", error_msg)
-      error(error_msg)
+      msg = "Compilation error when compiling \""..source.."\": "..error_msg
+      error(msg)
    end
 
    -- Traverse the ast looking for events/syscalls in the ignored

userspace/engine/lua/rule_loader.lua

@@ -123,7 +123,46 @@ end
 -- to a rule.
 local state = {macros={}, lists={}, filter_ast=nil, rules_by_name={}, n_rules=0, rules_by_idx={}}
 
-function load_rules(rules_content, rules_mgr, verbose, all_events)
+-- From http://lua-users.org/wiki/TableUtils
+--
+function table.val_to_str ( v )
+  if "string" == type( v ) then
+    v = string.gsub( v, "\n", "\\n" )
+    if string.match( string.gsub(v,"[^'\"]",""), '^"+$' ) then
+      return "'" .. v .. "'"
+    end
+    return '"' .. string.gsub(v,'"', '\\"' ) .. '"'
+  else
+    return "table" == type( v ) and table.tostring( v ) or
+      tostring( v )
+  end
+end
+
+function table.key_to_str ( k )
+  if "string" == type( k ) and string.match( k, "^[_%a][_%a%d]*$" ) then
+    return k
+  else
+    return "[" .. table.val_to_str( k ) .. "]"
+  end
+end
+
+function table.tostring( tbl )
+  local result, done = {}, {}
+  for k, v in ipairs( tbl ) do
+    table.insert( result, table.val_to_str( v ) )
+    done[ k ] = true
+  end
+  for k, v in pairs( tbl ) do
+    if not done[ k ] then
+      table.insert( result,
+        table.key_to_str( k ) .. "=" .. table.val_to_str( v ) )
+    end
+  end
+  return "{" .. table.concat( result, "," ) .. "}"
+end
+
+
+function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replace_container_info)
 
    compiler.set_verbose(verbose)
    compiler.set_all_events(all_events)
@@ -135,6 +174,10 @@ function load_rules(rules_content, rules_mgr, verbose, all_events)
       return
    end
 
+   if type(rules) ~= "table" then
+      error("Rules content \""..rules_content.."\" is not yaml")
+   end
+
    for i,v in ipairs(rules) do -- iterate over yaml list
 
       if (not (type(v) == "table")) then
@@ -164,9 +207,9 @@ function load_rules(rules_content, rules_mgr, verbose, all_events)
 
 	 state.lists[v['list']] = items
 
-      else -- rule
+      elseif (v['rule']) then
 
-	 if (v['rule'] == nil) then
+	 if (v['rule'] == nil or type(v['rule']) == "table") then
 	    error ("Missing name in rule")
 	 end
 
@@ -214,9 +257,41 @@ function load_rules(rules_content, rules_mgr, verbose, all_events)
 	    if (v['enabled'] == false) then
 	       falco_rules.enable_rule(rules_mgr, v['rule'], 0)
 	    end
+
+	    -- If the format string contains %container.info, replace it
+	    -- with extra. Otherwise, add extra onto the end of the format
+	    -- string.
+	    if string.find(v['output'], "%container.info", nil, true) ~= nil then
+
+		-- There may not be any extra, or we're not supposed
+		-- to replace it, in which case we use the generic
+		-- "%container.name (id=%container.id)"
+	       if replace_container_info == false then
+		  v['output'] = string.gsub(v['output'], "%%container.info", "%%container.name (id=%%container.id)")
+		  if extra ~= "" then
+		     v['output'] = v['output'].." "..extra
+		  end
+	       else
+		  safe_extra = string.gsub(extra, "%%", "%%%%")
+		  v['output'] = string.gsub(v['output'], "%%container.info", safe_extra)
+	       end
+	    else
+	       -- Just add the extra to the end
+		if extra ~= "" then
+		   v['output'] = v['output'].." "..extra
+		end
+	    end
+
+	    -- Ensure that the output field is properly formatted by
+	    -- creating a formatter from it. Any error will be thrown
+	    -- up to the top level.
+	    formatter = formats.formatter(v['output'])
+	    formats.free_formatter(formatter)
 	 else
 	    error ("Unexpected type in load_rule: "..filter_ast.type)
 	 end
+      else
+	 error ("Unknown rule object: "..table.tostring(v))
       end
    end
 

userspace/engine/rules.cpp

@@ -108,7 +108,9 @@ void falco_rules::enable_rule(string &rule, bool enabled)
 	m_engine->enable_rule(rule, enabled);
 }
 
-void falco_rules::load_rules(const string &rules_content, bool verbose, bool all_events)
+void falco_rules::load_rules(const string &rules_content,
+			     bool verbose, bool all_events,
+			     string &extra, bool replace_container_info)
 {
 	lua_getglobal(m_ls, m_lua_load_rules.c_str());
 	if(lua_isfunction(m_ls, -1))
@@ -182,7 +184,9 @@ void falco_rules::load_rules(const string &rules_content, bool verbose, bool all
 		lua_pushlightuserdata(m_ls, this);
 		lua_pushboolean(m_ls, (verbose ? 1 : 0));
 		lua_pushboolean(m_ls, (all_events ? 1 : 0));
-		if(lua_pcall(m_ls, 4, 0, 0) != 0)
+		lua_pushstring(m_ls, extra.c_str());
+		lua_pushboolean(m_ls, (replace_container_info ? 1 : 0));
+		if(lua_pcall(m_ls, 6, 0, 0) != 0)
 		{
 			const char* lerr = lua_tostring(m_ls, -1);
 			string err = "Error loading rules:" + string(lerr);

userspace/engine/rules.h

@@ -31,7 +31,8 @@ class falco_rules
  public:
 	falco_rules(sinsp* inspector, falco_engine *engine, lua_State *ls);
 	~falco_rules();
-	void load_rules(const string &rules_content, bool verbose, bool all_events);
+	void load_rules(const string &rules_content, bool verbose, bool all_events,
+			std::string &extra, bool replace_container_info);
 	void describe_rule(string *rule);
 
 	static void init(lua_State *ls);

userspace/engine/token_bucket.cpp

@@ -0,0 +1,71 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <cstddef>
+#include <sys/time.h>
+
+#include "utils.h"
+#include "token_bucket.h"
+
+token_bucket::token_bucket()
+{
+	init(1, 1);
+}
+
+token_bucket::~token_bucket()
+{
+}
+
+void token_bucket::init(uint32_t rate, uint32_t max_tokens)
+{
+	m_rate = rate;
+	m_max_tokens = max_tokens;
+	m_tokens = max_tokens;
+	m_last_seen = sinsp_utils::get_current_time_ns();
+}
+
+bool token_bucket::claim()
+{
+	// Determine the number of tokens gained. Delta between
+	// last_seen and now, divided by the rate.
+	uint64_t now = sinsp_utils::get_current_time_ns();
+	uint64_t tokens_gained = (now - m_last_seen) / (m_rate * 1000000000);
+	m_last_seen = now;
+
+	m_tokens += tokens_gained;
+
+	//
+	// Cap at max_tokens
+	//
+	if(m_tokens > m_max_tokens)
+	{
+		m_tokens = m_max_tokens;
+	}
+
+	//
+	// If tokens is < 1, can't claim.
+	//
+	if(m_tokens < 1)
+	{
+		return false;
+	}
+
+	m_tokens--;
+
+	return true;
+}

userspace/engine/token_bucket.h

@@ -0,0 +1,65 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <cstdint>
+
+// A simple token bucket that accumulates tokens at a fixed rate and allows
+// for limited bursting in the form of "banked" tokens.
+class token_bucket
+{
+public:
+	token_bucket();
+	virtual ~token_bucket();
+
+	//
+	// Initialize the token bucket and start accumulating tokens
+	//
+	void init(uint32_t rate, uint32_t max_tokens);
+
+	//
+	// Returns true if a token can be claimed. Also updates
+	// internal metrics.
+	//
+	bool claim();
+private:
+
+	//
+	// The number of tokens generated per second.
+	//
+	uint64_t m_rate;
+
+	//
+	// The maximum number of tokens that can be banked for future
+	// claim()s.
+	//
+	uint64_t m_max_tokens;
+
+	//
+	// The current number of tokens
+	//
+	uint64_t m_tokens;
+
+	//
+	// The last time claim() was called (or the object was created).
+	// Nanoseconds since the epoch.
+	//
+	uint64_t m_last_seen;
+};
+

userspace/falco/CMakeLists.txt

@@ -9,7 +9,7 @@ include_directories("${CURL_INCLUDE_DIR}")
 include_directories("${YAMLCPP_INCLUDE_DIR}")
 include_directories("${DRAIOS_DEPENDENCIES_DIR}/yaml-${DRAIOS_YAML_VERSION}/target/include")
 
-add_executable(falco configuration.cpp formats.cpp logger.cpp falco_outputs.cpp falco.cpp)
+add_executable(falco configuration.cpp logger.cpp falco_outputs.cpp statsfilewriter.cpp falco.cpp)
 
 target_link_libraries(falco falco_engine sinsp)
 target_link_libraries(falco

userspace/falco/configuration.cpp

@@ -101,6 +101,13 @@ void falco_configuration::init(string conf_filename, list<string> &cmdline_optio
 		throw invalid_argument("Error reading config file (" + m_config_file + "): No outputs configured. Please configure at least one output file output enabled but no filename in configuration block");
 	}
 
+	string log_level = m_config->get_scalar<string>("log_level", "info");
+
+	falco_logger::set_level(log_level);
+
+	m_notifications_rate = m_config->get_scalar<uint32_t>("outputs", "rate", 1);
+	m_notifications_max_burst = m_config->get_scalar<uint32_t>("outputs", "max_burst", 1000);
+
 	falco_logger::log_stderr = m_config->get_scalar<bool>("log_stderr", false);
 	falco_logger::log_syslog = m_config->get_scalar<bool>("log_syslog", true);
 }

userspace/falco/configuration.h

@@ -144,6 +144,8 @@ class falco_configuration
 	std::list<std::string> m_rules_filenames;
 	bool m_json_output;
 	std::vector<falco_outputs::output_config> m_outputs;
+	uint32_t m_notifications_rate;
+	uint32_t m_notifications_max_burst;
  private:
 	void init_cmdline_options(std::list<std::string> &cmdline_options);
 

userspace/falco/falco.cpp

@@ -36,6 +36,7 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 #include "configuration.h"
 #include "falco_engine.h"
 #include "config_falco.h"
+#include "statsfilewriter.h"
 
 bool g_terminate = false;
 //
@@ -97,6 +98,8 @@ static void usage()
 	   " -P, --pidfile <pid_file>      When run as a daemon, write pid to specified file\n"
            " -r <rules_file>               Rules file (defaults to value set in configuration file, or /etc/falco_rules.yaml).\n"
 	   "                               Can be specified multiple times to read from multiple files.\n"
+	   " -s <stats_file>               If specified, write statistics related to falco's reading/processing of events\n"
+	   "                               to this file. (Only useful in live mode).\n"
 	   " -v                            Verbose output.\n"
 	   "\n"
     );
@@ -124,11 +127,23 @@ std::list<string> cmdline_options;
 //
 uint64_t do_inspect(falco_engine *engine,
 		    falco_outputs *outputs,
-		    sinsp* inspector)
+		    sinsp* inspector,
+		    string &stats_filename)
 {
 	uint64_t num_evts = 0;
 	int32_t res;
 	sinsp_evt* ev;
+	StatsFileWriter writer;
+
+	if (stats_filename != "")
+	{
+		string errstr;
+
+		if (!writer.init(inspector, stats_filename, 5, errstr))
+		{
+			throw falco_exception(errstr);
+		}
+	}
 
 	//
 	// Loop through the events
@@ -138,6 +153,8 @@ uint64_t do_inspect(falco_engine *engine,
 
 		res = inspector->next(&ev);
 
+		writer.handle();
+
 		if (g_terminate)
 		{
 			break;
@@ -171,11 +188,10 @@ uint64_t do_inspect(falco_engine *engine,
 		// engine, which will match the event against the set
 		// of rules. If a match is found, pass the event to
 		// the outputs.
-		falco_engine::rule_result *res = engine->process_event(ev);
+		unique_ptr<falco_engine::rule_result> res = engine->process_event(ev);
 		if(res)
 		{
 			outputs->handle_event(res->evt, res->rule, res->priority, res->format);
-			delete(res);
 		}
 
 		num_evts++;
@@ -203,6 +219,7 @@ int falco_init(int argc, char **argv)
 	string pidfilename = "/var/run/falco.pid";
 	bool describe_all_rules = false;
 	string describe_rule = "";
+	string stats_filename = "";
 	bool verbose = false;
 	bool all_events = false;
 	string* k8s_api = 0;
@@ -247,7 +264,7 @@ int falco_init(int argc, char **argv)
 		// Parse the args
 		//
 		while((op = getopt_long(argc, argv,
-                                        "hc:AdD:e:k:K:Ll:m:o:P:p:r:vw:",
+                                        "hc:AdD:e:k:K:Ll:m:o:P:p:r:s:vw:",
                                         long_options, &long_index)) != -1)
 		{
 			switch(op)
@@ -319,6 +336,9 @@ int falco_init(int argc, char **argv)
 			case 'r':
 				rules_filenames.push_back(optarg);
 				break;
+			case 's':
+				stats_filename = optarg;
+				break;
 			case 'v':
 				verbose = true;
 				break;
@@ -337,10 +357,10 @@ int falco_init(int argc, char **argv)
 		inspector = new sinsp();
 		engine = new falco_engine();
 		engine->set_inspector(inspector);
+		engine->set_extra(output_format, replace_container_info);
 
 		outputs = new falco_outputs();
 		outputs->set_inspector(inspector);
-		outputs->set_extra(output_format, replace_container_info);
 
 		// Some combinations of arguments are not allowed.
 		if (daemon && pidfilename == "") {
@@ -407,7 +427,7 @@ int falco_init(int argc, char **argv)
 			engine->enable_rule(pattern, false);
 		}
 
-		outputs->init(config.m_json_output);
+		outputs->init(config.m_json_output, config.m_notifications_rate, config.m_notifications_max_burst);
 
 		if(!all_events)
 		{
@@ -589,7 +609,8 @@ int falco_init(int argc, char **argv)
 
 		num_evts = do_inspect(engine,
 				      outputs,
-				      inspector);
+				      inspector,
+				      stats_filename);
 
 		duration = ((double)clock()) / CLOCKS_PER_SEC - duration;
 

userspace/falco/falco_outputs.cpp

@@ -27,17 +27,31 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 using namespace std;
 
 falco_outputs::falco_outputs()
-	: m_replace_container_info(false)
+	: m_initialized(false)
 {
 
 }
 
 falco_outputs::~falco_outputs()
 {
+	if(m_initialized)
+	{
+		lua_getglobal(m_ls, m_lua_output_cleanup.c_str());
 
+		if(!lua_isfunction(m_ls, -1))
+		{
+			throw falco_exception("No function " + m_lua_output_cleanup + " found. ");
+		}
+
+		if(lua_pcall(m_ls, 0, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			throw falco_exception(string(lerr));
+		}
+	}
 }
 
-void falco_outputs::init(bool json_output)
+void falco_outputs::init(bool json_output, uint32_t rate, uint32_t max_burst)
 {
 	// The engine must have been given an inspector by now.
 	if(! m_inspector)
@@ -47,15 +61,16 @@ void falco_outputs::init(bool json_output)
 
 	falco_common::init(m_lua_main_filename.c_str(), FALCO_SOURCE_LUA_DIR);
 
+	// Note that falco_formats is added to both the lua state used
+	// by the falco engine as well as the separate lua state used
+	// by falco outputs.
 	falco_formats::init(m_inspector, m_ls, json_output);
 
 	falco_logger::init(m_ls);
-}
 
-void falco_outputs::set_extra(string &extra, bool replace_container_info)
-{
-	m_extra = extra;
-	m_replace_container_info = replace_container_info;
+	m_notifications_tb.init(rate, max_burst);
+
+	m_initialized = true;
 }
 
 void falco_outputs::add_output(output_config oc)
@@ -90,46 +105,22 @@ void falco_outputs::add_output(output_config oc)
 
 }
 
-void falco_outputs::handle_event(sinsp_evt *ev, string &level, string &priority, string &format)
+void falco_outputs::handle_event(sinsp_evt *ev, string &rule, string &priority, string &format)
 {
+	if(!m_notifications_tb.claim())
+	{
+		falco_logger::log(LOG_DEBUG, "Skipping rate-limited notification for rule " + rule + "\n");
+		return;
+	}
+
 	lua_getglobal(m_ls, m_lua_output_event.c_str());
 
-	// If the format string contains %container.info, replace it
-	// with extra. Otherwise, add extra onto the end of the format
-	// string.
-	string format_w_extra = format;
-	size_t pos;
-
-	if((pos = format_w_extra.find("%container.info")) != string::npos)
-	{
-		// There may not be any extra, or we're not supposed
-		// to replace it, in which case we use the generic
-		// "%container.name (id=%container.id)"
-		if(m_extra == "" || ! m_replace_container_info)
-		{
-			// 15 == strlen(%container.info)
-			format_w_extra.replace(pos, 15, "%container.name (id=%container.id)");
-		}
-		else
-		{
-			format_w_extra.replace(pos, 15, m_extra);
-		}
-	}
-	else
-	{
-		// Just add the extra to the end
-		if (m_extra != "")
-		{
-			format_w_extra += " " + m_extra;
-		}
-	}
-
 	if(lua_isfunction(m_ls, -1))
 	{
 		lua_pushlightuserdata(m_ls, ev);
-		lua_pushstring(m_ls, level.c_str());
+		lua_pushstring(m_ls, rule.c_str());
 		lua_pushstring(m_ls, priority.c_str());
-		lua_pushstring(m_ls, format_w_extra.c_str());
+		lua_pushstring(m_ls, format.c_str());
 
 		if(lua_pcall(m_ls, 4, 0, 0) != 0)
 		{

userspace/falco/falco_outputs.h

@@ -19,6 +19,7 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 #pragma once
 
 #include "falco_common.h"
+#include "token_bucket.h"
 
 //
 // This class acts as the primary interface between a program and the
@@ -40,22 +41,24 @@ public:
 		std::map<std::string, std::string> options;
 	};
 
-	void init(bool json_output);
+	void init(bool json_output, uint32_t rate, uint32_t max_burst);
 
 	void add_output(output_config oc);
 
-	void set_extra(string &extra, bool replace_container_info);
-
 	//
 	// ev is an event that has matched some rule. Pass the event
 	// to all configured outputs.
 	//
-	void handle_event(sinsp_evt *ev, std::string &level, std::string &priority, std::string &format);
+	void handle_event(sinsp_evt *ev, std::string &rule, std::string &priority, std::string &format);
 
 private:
+	bool m_initialized;
+
+	// Rate limits notifications
+	token_bucket m_notifications_tb;
+
 	std::string m_lua_add_output = "add_output";
 	std::string m_lua_output_event = "output_event";
+	std::string m_lua_output_cleanup = "output_cleanup";
 	std::string m_lua_main_filename = "output.lua";
-	std::string m_extra;
-	bool m_replace_container_info;
 };

userspace/falco/logger.cpp

@@ -20,18 +20,62 @@ along with falco.  If not, see <http://www.gnu.org/licenses/>.
 #include "logger.h"
 #include "chisel_api.h"
 
+#include "falco_common.h"
+
 const static struct luaL_reg ll_falco [] =
 {
 	{"syslog", &falco_logger::syslog},
 	{NULL,NULL}
 };
 
+int falco_logger::level = LOG_INFO;
 
 void falco_logger::init(lua_State *ls)
 {
 	luaL_openlib(ls, "falco", ll_falco, 0);
 }
 
+void falco_logger::set_level(string &level)
+{
+	if(level == "emergency")
+	{
+		falco_logger::level = LOG_EMERG;
+	}
+	else if(level == "alert")
+	{
+		falco_logger::level = LOG_ALERT;
+	}
+	else if(level == "critical")
+	{
+		falco_logger::level = LOG_CRIT;
+	}
+	else if(level == "error")
+	{
+		falco_logger::level = LOG_ERR;
+	}
+	else if(level == "warning")
+	{
+		falco_logger::level = LOG_WARNING;
+	}
+	else if(level == "notice")
+	{
+		falco_logger::level = LOG_NOTICE;
+	}
+	else if(level == "info")
+	{
+		falco_logger::level = LOG_INFO;
+	}
+	else if(level == "debug")
+	{
+		falco_logger::level = LOG_DEBUG;
+	}
+	else
+	{
+		throw falco_exception("Unknown log level " + level);
+	}
+}
+
+
 int falco_logger::syslog(lua_State *ls) {
 	int priority = luaL_checknumber(ls, 1);
 
@@ -49,6 +93,12 @@ bool falco_logger::log_stderr = true;
 bool falco_logger::log_syslog = true;
 
 void falco_logger::log(int priority, const string msg) {
+
+	if(priority > falco_logger::level)
+	{
+		return;
+	}
+
 	if (falco_logger::log_syslog) {
 		::syslog(priority, "%s", msg.c_str());
 	}

userspace/falco/logger.h

@@ -32,11 +32,15 @@ class falco_logger
  public:
 	static void init(lua_State *ls);
 
+	// Will throw exception if level is unknown.
+	static void set_level(string &level);
+
 	// value = falco.syslog(level, message)
 	static int syslog(lua_State *ls);
 
 	static void log(int priority, const string msg);
 
+	static int level;
 	static bool log_stderr;
 	static bool log_syslog;
 };

userspace/falco/lua/output.lua

@@ -24,6 +24,8 @@ mod.levels = levels
 
 local outputs = {}
 
+local formatters = {}
+
 function mod.stdout(level, msg)
    print (msg)
 end
@@ -75,14 +77,26 @@ end
 function output_event(event, rule, priority, format)
    local level = level_of(priority)
    format = "*%evt.time: "..levels[level+1].." "..format
-   formatter = falco.formatter(format)
-   msg = falco.format_event(event, rule, levels[level+1], formatter)
+   if formatters[rule] == nil then
+      formatter = formats.formatter(format)
+      formatters[rule] = formatter
+   else
+      formatter = formatters[rule]
+   end
+
+   msg = formats.format_event(event, rule, levels[level+1], formatter)
 
    for index,o in ipairs(outputs) do
       o.output(level, msg, o.config)
    end
+end
 
-   falco.free_formatter(formatter)
+function output_cleanup()
+   for rule, formatter in pairs(formatters) do
+      formats.free_formatter(formatter)
+   end
+
+   formatters = {}
 end
 
 function add_output(output_name, config)

userspace/falco/statsfilewriter.cpp

@@ -0,0 +1,120 @@
+#include <sys/time.h>
+#include <signal.h>
+
+#include "statsfilewriter.h"
+
+using namespace std;
+
+static bool g_save_stats = false;
+static void timer_handler (int signum)
+{
+	g_save_stats = true;
+}
+
+extern char **environ;
+
+StatsFileWriter::StatsFileWriter()
+	: m_num_stats(0), m_inspector(NULL)
+{
+}
+
+StatsFileWriter::~StatsFileWriter()
+{
+	m_output.close();
+}
+
+bool StatsFileWriter::init(sinsp *inspector, string &filename, uint32_t interval_sec, string &errstr)
+{
+	struct itimerval timer;
+	struct sigaction handler;
+
+	m_inspector = inspector;
+
+	m_output.exceptions ( ofstream::failbit | ofstream::badbit );
+	m_output.open(filename, ios_base::app);
+
+	memset (&handler, 0, sizeof (handler));
+	handler.sa_handler = &timer_handler;
+	if (sigaction(SIGALRM, &handler, NULL) == -1)
+	{
+		errstr = string("Could not set up signal handler for periodic timer: ") + strerror(errno);
+		return false;
+	}
+
+	timer.it_value.tv_sec = interval_sec;
+	timer.it_value.tv_usec = 0;
+	timer.it_interval = timer.it_value;
+	if (setitimer(ITIMER_REAL, &timer, NULL) == -1)
+	{
+		errstr = string("Could not set up periodic timer: ") + strerror(errno);
+		return false;
+	}
+
+	// (Undocumented) feature. Take any environment keys prefixed
+	// with FALCO_STATS_EXTRA_XXX and add them to the output. Used by
+	// run_performance_tests.sh.
+	for(uint32_t i=0; environ[i]; i++)
+	{
+		char *p = strstr(environ[i], "=");
+		if(!p)
+		{
+			errstr = string("Could not find environment separator in ") + string(environ[i]);
+			return false;
+		}
+		string key(environ[i], p-environ[i]);
+		string val(p+1, strlen(environ[i])-(p-environ[i])-1);
+		if(key.compare(0, 18, "FALCO_STATS_EXTRA_") == 0)
+		{
+			string sub = key.substr(18);
+			if (m_extra != "")
+			{
+				m_extra += ", ";
+			}
+			m_extra += "\"" + sub + "\": " + "\"" + val + "\"";
+		}
+	}
+
+	return true;
+}
+
+void StatsFileWriter::handle()
+{
+	if (g_save_stats)
+	{
+		scap_stats cstats;
+		scap_stats delta;
+
+		g_save_stats = false;
+		m_num_stats++;
+		m_inspector->get_capture_stats(&cstats);
+
+		if(m_num_stats == 1)
+		{
+			delta = cstats;
+		}
+		else
+		{
+			delta.n_evts = cstats.n_evts - m_last_stats.n_evts;
+			delta.n_drops = cstats.n_drops - m_last_stats.n_drops;
+			delta.n_preemptions = cstats.n_preemptions - m_last_stats.n_preemptions;
+		}
+
+		m_output << "{\"sample\": " << m_num_stats;
+		if(m_extra != "")
+		{
+			m_output << ", " << m_extra;
+		}
+		m_output << ", \"cur\": {" <<
+			"\"events\": " << cstats.n_evts <<
+			", \"drops\": " << cstats.n_drops <<
+			", \"preemptions\": " << cstats.n_preemptions <<
+			"}, \"delta\": {" <<
+			"\"events\": " << delta.n_evts <<
+			", \"drops\": " << delta.n_drops <<
+			", \"preemptions\": " << delta.n_preemptions <<
+			"}, \"drop_pct\": " << (delta.n_evts == 0 ? 0 : (100.0*delta.n_drops/delta.n_evts)) <<
+			"}," << endl;
+
+		m_last_stats = cstats;
+	}
+}

userspace/falco/statsfilewriter.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <fstream>
+#include <string>
+#include <map>
+
+#include <sinsp.h>
+
+// Periodically collects scap stats files and writes them to a file as
+// json.
+
+class StatsFileWriter {
+public:
+	StatsFileWriter();
+	virtual ~StatsFileWriter();
+
+	// Returns success as bool. On false fills in errstr.
+	bool init(sinsp *inspector, std::string &filename,
+		  uint32_t interval_sec,
+		  string &errstr);
+
+	// Should be called often (like for each event in a sinsp
+	// loop).
+	void handle();
+
+protected:
+	uint32_t m_num_stats;
+	sinsp *m_inspector;
+	std::ofstream m_output;
+	std::string m_extra;
+	scap_stats m_last_stats;
+};