Escape double-quotes in aws cloudtrail rule

The rule Delete Bucket Public Access Block has a predicate
`json.value[/requestParameters/publicAccessBlock]=""` to match
an event snippet like this:

```
			"requestParameters": {
				"publicAccessBlock": "",
```

The cloudtrail plugin properly returns `""` for this field, but the
yaml representation was a literal back-to-back quote, which gets
interpreted by the yaml parser to be an empty string.

Escaping the back-to-back quote fixes the ambiguity.

Signed-off-by: Mark Stemm <mark.stemm@gmail.com>

.circleci/config.yml

@@ -292,6 +292,7 @@ jobs:
           BUILD_DIR: "/build-static"
           BUILD_TYPE: "release"
           SKIP_PACKAGES_TESTS: "true"
+          SKIP_PLUGINS_TESTS: "true"
     steps:
       - setup_remote_docker
       - attach_workspace:
@@ -303,7 +304,7 @@ jobs:
           path: /build-static/release/integration-tests-xunit
   "tests/driver-loader/integration":
     machine:
-      image: ubuntu-1604:202004-01
+      image: ubuntu-2004:202107-02
     steps:
       - attach_workspace:
           at: /tmp/ws
@@ -390,9 +391,15 @@ jobs:
             /source/falco/scripts/publish-rpm -f /build/release/falco-${FALCO_VERSION}-x86_64.rpm -r rpm-dev
       - run:
           name: Publish bin-dev
+          command: |
+            FALCO_VERSION=$(cat /build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
+            /source/falco/scripts/publish-bin -f /build/release/falco-${FALCO_VERSION}-x86_64.tar.gz -r bin-dev -a x86_64
+      - run:
+          name: Publish bin-static-dev
           command: |
             FALCO_VERSION=$(cat /build-static/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
-            /source/falco/scripts/publish-bin -f /build-static/release/falco-${FALCO_VERSION}-x86_64.tar.gz -r bin-dev -a x86_64
+            cp -f /build-static/release/falco-${FALCO_VERSION}-x86_64.tar.gz /build-static/release/falco-${FALCO_VERSION}-static-x86_64.tar.gz
+            /source/falco/scripts/publish-bin -f /build-static/release/falco-${FALCO_VERSION}-static-x86_64.tar.gz -r bin-dev -a x86_64
   "publish/packages-deb-dev":
     docker:
       - image: docker.io/debian:stable
@@ -504,9 +511,15 @@ jobs:
             /source/falco/scripts/publish-rpm -f /build/release/falco-${FALCO_VERSION}-x86_64.rpm -r rpm
       - run:
           name: Publish bin
+          command: |
+            FALCO_VERSION=$(cat /build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
+            /source/falco/scripts/publish-bin -f /build/release/falco-${FALCO_VERSION}-x86_64.tar.gz -r bin -a x86_64
+      - run:
+          name: Publish bin-static
           command: |
             FALCO_VERSION=$(cat /build-static/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
-            /source/falco/scripts/publish-bin -f /build-static/release/falco-${FALCO_VERSION}-x86_64.tar.gz -r bin -a x86_64
+            cp -f /build-static/release/falco-${FALCO_VERSION}-x86_64.tar.gz /build-static/release/falco-${FALCO_VERSION}-static-x86_64.tar.gz
+            /source/falco/scripts/publish-bin -f /build/release/falco-${FALCO_VERSION}-static-x86_64.tar.gz -r bin -a x86_64
   "publish/packages-deb":
     docker:
       - image: docker.io/debian:stable
@@ -710,7 +723,6 @@ workflows:
             - falco
             - test-infra
           requires:
-            - "build/musl"
             - "rpm/sign"
           filters:
             tags:

.gitignore

@@ -10,11 +10,8 @@ test/.phoronix-test-suite
 test/results*.json.*
 test/build
 
-userspace/engine/lua/lyaml
-userspace/engine/lua/lyaml.lua
-
 .vscode/*
 
 .luacheckcache
 
-*.idea*
+*.idea*

ADOPTERS.md

@@ -5,7 +5,7 @@ Known end users with notable contributions to the project include:
 * IBM
 * Red Hat
 
-Falco is being used by numerous other companies, both large and small, to build higher layer products and services. The list includes but is not limited to: 
+Falco is being used by numerous other companies, both large and small, to build higher layer products and services. The list includes but is not limited to:
 * Equinix Metal
 * IEEE
 * Lowes
@@ -26,6 +26,8 @@ This is a list of production adopters of Falco (in alphabetical order):
 
 * [Frame.io](https://frame.io/) - Frame.io is a cloud-based (SaaS) video review and collaboration platform that enables users to securely upload source media, work-in-progress edits, dailies, and more into private workspaces where they can invite their team and clients to collaborate on projects. Understanding what is running on production servers, and the context around why things are running is even more tricky now that we have further abstractions like Docker and Kubernetes. To get this needed visibility into our system, we rely on Falco. Falco's ability to collect raw system calls such as open, connect, exec, along with their arguments offer key insights on what is happening on the production system and became the foundation of our intrusion detection and alerting system.
 
+* [Giant Swarm](https://www.giantswarm.io/) - Giant Swarm manages Kubernetes clusters and infrastructure for enterprises across multiple cloud providers as well as several flavors of on-premises data centers. Our platform provisions and monitors pure "vanilla" Kubernetes clusters which can be augmented with managed solutions to many common Kubernetes challenges, including security. We use Falco for anomaly detection as part of our collection of entirely open-source tools for securing our own clusters, and offer the same capabilities to our customers as part of our [managed security offering](https://docs.giantswarm.io/app-platform/apps/security/).
+
 * [GitLab](https://about.gitlab.com/direction/defend/container_host_security/) - GitLab is a complete DevOps platform, delivered as a single application, fundamentally changing the way Development, Security, and Ops teams collaborate. GitLab Ultimate provides the single tool teams need to find, triage, and fix vulnerabilities in applications, services, and cloud-native environments enabling them to manage their risk. This provides them with repeatable, defensible processes that automate security and compliance policies. GitLab includes a tight integration with Falco, allowing users to defend their containerized applications from attacks while running in production.
 
 * [League](https://league.com/ca/) - League provides health benefits management services to help employees understand and get the most from their benefits, and employers to provide effective, efficient plans. Falco is used to monitor our deployed services on Kubernetes, protecting against malicious access to containers which could lead to leaks of PHI or other sensitive data. The Falco alerts are logged in Stackdriver for grouping and further analysis. In the future, we're hoping for integrations with Prometheus and AlertManager as well.
@@ -42,9 +44,11 @@ This is a list of production adopters of Falco (in alphabetical order):
 
 * [Qonto](https://qonto.com) - Qonto is a modern banking for SMEs and freelancers. Qonto provides a fully featured business account with a simplified accounting flow. Falco is used by our SecOps team to detect suspicous behaviors in our clusters.
 
+* [Raft](https://goraft.tech) - Raft is a government contractor that offers cloud-native solutions across many different agencies including DoD (Department of Defense), HHS (Health and Human Services), as well as within CFPB (Consumer Finance Protection Bureau). Raft leverages Falco to detect threats in our client's Kubernetes clusters and as a Host Intrusion Detection System. Raft proudly recommends Falco across all our different projects.
+
 * [Replicated](https://www.replicated.com/) - Replicated is the modern way to ship on-prem software. Replicated gives software vendors a container-based platform for easily deploying cloud native applications inside customers'​ environments to provide greater security and control. Replicated uses Falco as runtime security to detect threats in the Kubernetes clusters which host our critical SaaS services.
 
-* [Secureworks](https://www.secureworks.com/) - Secureworks is a leading worldwide cybersecurity company with a cloud-native security product that combines the power of human intellect with security analytics to unify detection and response across cloud, network, and endpoint environments for improved security operations and outcomes. Our Taegis XDR platform and detection system processes petabytes of security relevant data to expose active threats amongst the billions of daily events from our customers. We are proud to protect our platform’s Kubernetes deployments, as well as help our customers protect their own Linux and container environments, using Falco. 
+* [Secureworks](https://www.secureworks.com/) - Secureworks is a leading worldwide cybersecurity company with a cloud-native security product that combines the power of human intellect with security analytics to unify detection and response across cloud, network, and endpoint environments for improved security operations and outcomes. Our Taegis XDR platform and detection system processes petabytes of security relevant data to expose active threats amongst the billions of daily events from our customers. We are proud to protect our platform’s Kubernetes deployments, as well as help our customers protect their own Linux and container environments, using Falco.
 
 * [Shopify](https://www.shopify.com) - Shopify is the leading multi-channel commerce platform. Merchants use Shopify to design, set up, and manage their stores across multiple sales channels, including mobile, web, social media, marketplaces, brick-and-mortar locations, and pop-up shops. The platform also provides merchants with a powerful back-office and a single view of their business, from payments to shipping. The Shopify platform was engineered for reliability and scale, making enterprise-level technology available to businesses of all sizes. Shopify uses Falco to complement its Host and Network Intrusion Detection Systems.
 
@@ -56,12 +60,18 @@ This is a list of production adopters of Falco (in alphabetical order):
 
 * [Swissblock Technologies](https://swissblock.net/) At Swissblock we connect the dots by combining cutting-edge algorithmic trading strategies with in-depth market analysis. We route all Falco events to our control systems, both monitoring and logging. Being able to deeply analyse alerts, we can understand what is running on our Kubernetes clusters and check against security policies, specifically defined for each workload. A set of alarms notifies us in case of critical events, letting us react fast. In the near future we plan to build a little application to route Kubernetes internal events directly to Falco, fully leveraging Falco PodSecurityPolicies analyses.
 
-* [Shapesecurity/F5] (https://www.shapesecurity.com/) Shapesecurity defends against application fraud attacks like Account Take Over, Credential Stuffing, Fake Accounts, etc. Required by FedRamp certification, we needed to find a FIM solution to help monitor and protect our Kubernetes clusters. Traditional FIM solutions were not scalable and not working for our environment, but with Falco we found the solution we needed. Falco's detection capabilities have helped us identify anomalous behaviour within our clusters. We leverage Sidekick (https://github.com/falcosecurity/charts/tree/master/falcosidekick) to send Falco alerts to a PubSub which in turn publishes those alerts to our SIEM (SumoLogic)
+* [Shapesecurity/F5](https://www.shapesecurity.com/) Shapesecurity defends against application fraud attacks like Account Take Over, Credential Stuffing, Fake Accounts, etc. Required by FedRamp certification, we needed to find a FIM solution to help monitor and protect our Kubernetes clusters. Traditional FIM solutions were not scalable and not working for our environment, but with Falco we found the solution we needed. Falco's detection capabilities have helped us identify anomalous behaviour within our clusters. We leverage Sidekick (https://github.com/falcosecurity/charts/tree/master/falcosidekick) to send Falco alerts to a PubSub which in turn publishes those alerts to our SIEM (SumoLogic)
 
 * [Yahoo! JAPAN](https://www.yahoo.co.jp/) Yahoo! JAPAN is a leading company of internet in Japan. We build an AI Platform in our private cloud and provide it to scientists in our company. AI Platform is a multi-tenant Kubernetes environment and more flexible, faster, more efficient Machine Learning environment. Falco is used to detect unauthorized commands and malicious access and our AI Platform is monitored and alerted by Falco.
 
 * [Sysdig](https://www.sysdig.com/) Sysdig originally created Falco in 2016 to detect unexpected or suspicious activity using a rules engine on top of the data that comes from the sysdig kernel system call probe. Sysdig provides tooling to help with vulnerability management, compliance, detection, incident response and forensics in Cloud-native environments. Sysdig Secure has extended Falco to include: a rule library, the ability to update macros, lists & rules via the user interface and API, automated tuning of rules, and rule creation based on profiling known system behavior. On top of the basic Falco rules, Sysdig Secure implements the concept of a "Security policy" that can comprise several rules which are evaluated for a user-defined infrastructure scope like Kubernetes namespaces, OpenShift clusters, deployment workload, cloud regions etc.
 
+## Projects that use Falco libs
+
+* [R6/Phoenix](https://r6security.com/) is an attack surface protection company that uses moving target defense to provide fully automated, proactive and devops friendly security to its customers. There are a set of policies you can add to enable the moving target defense capabilities. Some of them are triggered by a combination of Falco's findings. You can kill, restart and rename pods according to the ever changing policies.
+
+* [SysFlow](https://sysflow.io) SysFlow is a cloud-native system telemetry framework that focuses on data abstraction, behavioral analytics, and noise reduction. At its core, SysFlow exposes a compact open telemetry format that records workload behaviors by connecting event and flow representations of process control flows, file interactions, and network communications. The resulting abstraction encodes a graph structure that enables provenance reasoning on host and container environments, and fast retrieval of security-relevant information.
+
 ## Adding a name
 
-If you would like to add your name to this file, submit a pull request with your change. 
+If you would like to add your name to this file, submit a pull request with your change.

CMakeLists.txt

@@ -67,6 +67,7 @@ endif()
 
 if(MUSL_OPTIMIZED_BUILD)
   set(MUSL_FLAGS "-static -Os -fPIE -pie")
+  add_definitions(-DMUSL_OPTIMIZED)
 endif()
 
 # explicitly set hardening flags
@@ -110,12 +111,12 @@ set(CMD_MAKE make)
 
 include(ExternalProject)
 
-# LuaJIT
-include(luajit)
-
 # libs
 include(falcosecurity-libs)
 
+# LuaJit provided by libs
+include(luajit)
+
 # jq
 include(jq)
 
@@ -143,6 +144,9 @@ if(NOT MINIMAL_BUILD)
 
   # libcurl
   include(curl)
+
+  # civetweb
+  include(civetweb)
 endif()
 
 # Lpeg
@@ -157,24 +161,6 @@ include(lyaml)
 # One TBB
 include(tbb)
 
-if(NOT MINIMAL_BUILD)
-  # civetweb
-  set(CIVETWEB_SRC "${PROJECT_BINARY_DIR}/civetweb-prefix/src/civetweb/")
-  set(CIVETWEB_LIB "${CIVETWEB_SRC}/install/lib/libcivetweb.a")
-  set(CIVETWEB_INCLUDE_DIR "${CIVETWEB_SRC}/install/include")
-  message(STATUS "Using bundled civetweb in '${CIVETWEB_SRC}'")
-  ExternalProject_Add(
-    civetweb
-    URL "https://github.com/civetweb/civetweb/archive/v1.11.tar.gz"
-    URL_HASH "SHA256=de7d5e7a2d9551d325898c71e41d437d5f7b51e754b242af897f7be96e713a42"
-    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E make_directory ${CIVETWEB_SRC}/install/lib
-    COMMAND ${CMAKE_COMMAND} -E make_directory ${CIVETWEB_SRC}/install/include
-    BUILD_IN_SOURCE 1
-    BUILD_COMMAND ${CMD_MAKE} COPT="-DNO_FILES" WITH_CPP=1
-    BUILD_BYPRODUCTS ${CIVETWEB_LIB}
-    INSTALL_COMMAND ${CMD_MAKE} COPT="-DNO_FILES" install-lib install-headers PREFIX=${CIVETWEB_SRC}/install "WITH_CPP=1")
-endif()
-
 #string-view-lite
 include(DownloadStringViewLite)
 
@@ -212,6 +198,7 @@ include(static-analysis)
 # Shared build variables
 set(FALCO_SINSP_LIBRARY sinsp)
 set(FALCO_SHARE_DIR share/falco)
+set(FALCO_PLUGINS_DIR ${FALCO_SHARE_DIR}/plugins)
 set(FALCO_ABSOLUTE_SHARE_DIR "${CMAKE_INSTALL_PREFIX}/${FALCO_SHARE_DIR}")
 set(FALCO_BIN_DIR bin)
 
@@ -220,5 +207,9 @@ add_subdirectory(userspace/engine)
 add_subdirectory(userspace/falco)
 add_subdirectory(tests)
 
+if(NOT MUSL_OPTIMIZED_BUILD)
+  include(plugins)
+endif()
+
 # Packages configuration
 include(CPackConfig)

OWNERS

@@ -4,6 +4,7 @@ approvers:
   - leodido
   - mstemm
   - leogr
+  - jasondellaluce
 reviewers:
   - fntlnz
   - kaizhe
@@ -12,3 +13,4 @@ reviewers:
   - mfdii
   - mstemm
   - leogr
+  - jasondellaluce

cmake/cpack/CMakeCPackOptions.cmake

@@ -10,5 +10,4 @@ endif()
 
 if(CPACK_GENERATOR MATCHES "TGZ")
 	set(CPACK_SET_DESTDIR "ON")
-	set(CPACK_STRIP_FILES "OFF")
 endif()

cmake/modules/DownloadFakeIt.cmake

@@ -14,8 +14,8 @@ include(ExternalProject)
 
 set(FAKEIT_INCLUDE ${CMAKE_BINARY_DIR}/fakeit-prefix/include)
 
-set(FAKEIT_EXTERNAL_URL URL https://github.com/eranpeer/fakeit/archive/2.0.5.tar.gz URL_HASH
-                        SHA256=298539c773baca6ecbc28914306bba19d1008e098f8adc3ad3bb00e993ecdf15)
+set(FAKEIT_EXTERNAL_URL URL https://github.com/eranpeer/fakeit/archive/2.0.9.tar.gz URL_HASH
+                        SHA256=dc4ee7b17a84c959019b92c20fce6dc9426e9e170b6edf84db6cb2e188520cd7)
 
 ExternalProject_Add(
   fakeit-external

cmake/modules/GetFalcoVersion.cmake

@@ -21,7 +21,7 @@ if(NOT FALCO_VERSION)
   git_get_exact_tag(FALCO_TAG)
   if(NOT FALCO_TAG)
     # Obtain the closest tag
-    git_describe(FALCO_VERSION "--always" "--tags")
+    git_describe(FALCO_VERSION "--always" "--tags" "--abbrev=7")
     # Fallback version
     if(FALCO_VERSION MATCHES "NOTFOUND$")
       set(FALCO_VERSION "0.0.0")
@@ -31,29 +31,33 @@ if(NOT FALCO_VERSION)
   else()
     # A tag has been found: use it as the Falco version
     set(FALCO_VERSION "${FALCO_TAG}")
-    # Remove the starting "v" in case there is one
-    string(REGEX REPLACE "^v(.*)" "\\1" FALCO_VERSION "${FALCO_TAG}")
-  endif()
-  # TODO(leodido) > ensure Falco version is semver before extracting parts Populate partial version variables
-  string(REGEX MATCH "^(0|[1-9][0-9]*)" FALCO_VERSION_MAJOR "${FALCO_VERSION}")
-  string(REGEX REPLACE "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\..*" "\\2" FALCO_VERSION_MINOR "${FALCO_VERSION}")
-  string(REGEX REPLACE "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*).*" "\\3" FALCO_VERSION_PATCH
-                       "${FALCO_VERSION}")
-  string(
-    REGEX
-    REPLACE
-      "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*).*"
-      "\\5"
-      FALCO_VERSION_PRERELEASE
-      "${FALCO_VERSION}")
-  if(FALCO_VERSION_PRERELEASE STREQUAL "${FALCO_VERSION}")
-    set(FALCO_VERSION_PRERELEASE "")
-  endif()
-  if(NOT FALCO_VERSION_BUILD)
-    string(REGEX REPLACE ".*\\+([0-9a-zA-Z-]+(\\.[0-9a-zA-Z-]+)*)" "\\1" FALCO_VERSION_BUILD "${FALCO_VERSION}")
-  endif()
-  if(FALCO_VERSION_BUILD STREQUAL "${FALCO_VERSION}")
-    set(FALCO_VERSION_BUILD "")
   endif()
 endif()
+
+# Remove the starting "v" in case there is one
+string(REGEX REPLACE "^v(.*)" "\\1" FALCO_VERSION "${FALCO_VERSION}")
+
+# TODO(leodido) > ensure Falco version is semver before extracting parts Populate partial version variables
+string(REGEX MATCH "^(0|[1-9][0-9]*)" FALCO_VERSION_MAJOR "${FALCO_VERSION}")
+string(REGEX REPLACE "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\..*" "\\2" FALCO_VERSION_MINOR "${FALCO_VERSION}")
+string(REGEX REPLACE "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*).*" "\\3" FALCO_VERSION_PATCH
+                     "${FALCO_VERSION}")
+string(
+  REGEX
+  REPLACE
+    "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*).*"
+    "\\5"
+    FALCO_VERSION_PRERELEASE
+    "${FALCO_VERSION}")
+
+if(FALCO_VERSION_PRERELEASE STREQUAL "${FALCO_VERSION}")
+  set(FALCO_VERSION_PRERELEASE "")
+endif()
+if(NOT FALCO_VERSION_BUILD)
+  string(REGEX REPLACE ".*\\+([0-9a-zA-Z-]+(\\.[0-9a-zA-Z-]+)*)" "\\1" FALCO_VERSION_BUILD "${FALCO_VERSION}")
+endif()
+if(FALCO_VERSION_BUILD STREQUAL "${FALCO_VERSION}")
+  set(FALCO_VERSION_BUILD "")
+endif()
+
 message(STATUS "Falco version: ${FALCO_VERSION}")

cmake/modules/civetweb.cmake

@@ -0,0 +1,52 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+
+set(CIVETWEB_SRC "${PROJECT_BINARY_DIR}/civetweb-prefix/src/civetweb/")
+set(CIVETWEB_LIB "${CIVETWEB_SRC}/install/${CMAKE_INSTALL_LIBDIR}/libcivetweb.a")
+SET(CIVETWEB_CPP_LIB "${CIVETWEB_SRC}/install/${CMAKE_INSTALL_LIBDIR}/libcivetweb-cpp.a")
+set(CIVETWEB_INCLUDE_DIR "${CIVETWEB_SRC}/install/include")
+message(STATUS "Using bundled civetweb in '${CIVETWEB_SRC}'")
+if (USE_BUNDLED_OPENSSL)
+    ExternalProject_Add(
+            civetweb
+            DEPENDS openssl
+            URL "https://github.com/civetweb/civetweb/archive/v1.15.tar.gz"
+            URL_HASH "SHA256=90a533422944ab327a4fbb9969f0845d0dba05354f9cacce3a5005fa59f593b9"
+            INSTALL_DIR ${CIVETWEB_SRC}/install
+            CMAKE_ARGS
+            -DBUILD_TESTING=off
+            -DCIVETWEB_BUILD_TESTING=off
+            -DCIVETWEB_ENABLE_CXX=on
+            -DCIVETWEB_ENABLE_SERVER_EXECUTABLE=off
+            -DCIVETWEB_ENABLE_SSL_DYNAMIC_LOADING=off
+            -DCIVETWEB_SERVE_NO_FILES=on
+            -DCMAKE_INSTALL_PREFIX=${CIVETWEB_SRC}/install
+            -DOPENSSL_ROOT_DIR:PATH=${OPENSSL_INSTALL_DIR}
+            -DOPENSSL_USE_STATIC_LIBS:BOOL=TRUE
+            BUILD_BYPRODUCTS ${CIVETWEB_LIB} ${CIVETWEB_CPP_LIB})
+else()
+    ExternalProject_Add(
+            civetweb
+            URL "https://github.com/civetweb/civetweb/archive/v1.15.tar.gz"
+            URL_HASH "SHA256=90a533422944ab327a4fbb9969f0845d0dba05354f9cacce3a5005fa59f593b9"
+            INSTALL_DIR ${CIVETWEB_SRC}/install
+            CMAKE_ARGS
+            -DBUILD_TESTING=off
+            -DCIVETWEB_BUILD_TESTING=off
+            -DCIVETWEB_ENABLE_CXX=on
+            -DCIVETWEB_ENABLE_SERVER_EXECUTABLE=off
+            -DCIVETWEB_ENABLE_SSL_DYNAMIC_LOADING=off
+            -DCIVETWEB_SERVE_NO_FILES=on
+            -DCMAKE_INSTALL_PREFIX=${CIVETWEB_SRC}/install
+            BUILD_BYPRODUCTS ${CIVETWEB_LIB} ${CIVETWEB_CPP_LIB})
+endif()

cmake/modules/falcosecurity-libs-repo/CMakeLists.txt

@@ -24,5 +24,4 @@ ExternalProject_Add(
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ""
   INSTALL_COMMAND ""
-  TEST_COMMAND ""
-  PATCH_COMMAND patch -p1 -i ${CMAKE_CURRENT_SOURCE_DIR}/patch/libscap.patch && patch -p1 -i ${CMAKE_CURRENT_SOURCE_DIR}/patch/luajit.patch)
+  TEST_COMMAND "")

cmake/modules/falcosecurity-libs-repo/patch/libscap.patch

@@ -1,49 +0,0 @@
-diff --git a/userspace/libscap/scap.c b/userspace/libscap/scap.c
-index 6f51588e..5f9ea84e 100644
---- a/userspace/libscap/scap.c
-+++ b/userspace/libscap/scap.c
-@@ -55,7 +55,7 @@ limitations under the License.
- //#define NDEBUG
- #include <assert.h>
- 
--static const char *SYSDIG_BPF_PROBE_ENV = "SYSDIG_BPF_PROBE";
-+static const char *SYSDIG_BPF_PROBE_ENV = "FALCO_BPF_PROBE";
- 
- //
- // Probe version string size
-@@ -114,7 +114,7 @@ scap_t* scap_open_udig_int(char *error, int32_t *rc,
- static uint32_t get_max_consumers()
- {
- 	uint32_t max;
--	FILE *pfile = fopen("/sys/module/" PROBE_DEVICE_NAME "_probe/parameters/max_consumers", "r");
-+	FILE *pfile = fopen("/sys/module/" PROBE_DEVICE_NAME "/parameters/max_consumers", "r");
- 	if(pfile != NULL)
- 	{
- 		int w = fscanf(pfile, "%"PRIu32, &max);
-@@ -186,7 +186,7 @@ scap_t* scap_open_live_int(char *error, int32_t *rc,
- 				return NULL;
- 			}
- 
--			snprintf(buf, sizeof(buf), "%s/.sysdig/%s-bpf.o", home, PROBE_NAME);
-+			snprintf(buf, sizeof(buf), "%s/.falco/%s-bpf.o", home, PROBE_NAME);
- 			bpf_probe = buf;
- 		}
- 	}
-@@ -344,7 +344,7 @@ scap_t* scap_open_live_int(char *error, int32_t *rc,
- 				else if(errno == EBUSY)
- 				{
- 					uint32_t curr_max_consumers = get_max_consumers();
--					snprintf(error, SCAP_LASTERR_SIZE, "Too many sysdig instances attached to device %s. Current value for /sys/module/" PROBE_DEVICE_NAME "_probe/parameters/max_consumers is '%"PRIu32"'.", filename, curr_max_consumers);
-+					snprintf(error, SCAP_LASTERR_SIZE, "Too many Falco instances attached to device %s. Current value for /sys/module/" PROBE_DEVICE_NAME "/parameters/max_consumers is '%"PRIu32"'.", filename, curr_max_consumers);
- 				}
- 				else
- 				{
-@@ -2175,7 +2175,7 @@ int32_t scap_disable_dynamic_snaplen(scap_t* handle)
- 
- const char* scap_get_host_root()
- {
--	char* p = getenv("SYSDIG_HOST_ROOT");
-+	char* p = getenv("HOST_ROOT");
- 	static char env_str[SCAP_MAX_PATH_SIZE + 1];
- 	static bool inited = false;
- 	if (! inited) {

cmake/modules/falcosecurity-libs-repo/patch/luajit.patch

@@ -1,57 +0,0 @@
-diff --git a/userspace/chisel/chisel.cpp b/userspace/chisel/chisel.cpp
-index 0a6e3cf8..0c2e255a 100644
---- a/userspace/chisel/chisel.cpp
-+++ b/userspace/chisel/chisel.cpp
-@@ -98,7 +98,7 @@ void lua_stackdump(lua_State *L)
- // Lua callbacks
- ///////////////////////////////////////////////////////////////////////////////
- #ifdef HAS_LUA_CHISELS
--const static struct luaL_reg ll_sysdig [] =
-+const static struct luaL_Reg ll_sysdig [] =
- {
- 	{"set_filter", &lua_cbacks::set_global_filter},
- 	{"set_snaplen", &lua_cbacks::set_snaplen},
-@@ -134,7 +134,7 @@ const static struct luaL_reg ll_sysdig [] =
- 	{NULL,NULL}
- };
- 
--const static struct luaL_reg ll_chisel [] =
-+const static struct luaL_Reg ll_chisel [] =
- {
- 	{"request_field", &lua_cbacks::request_field},
- 	{"set_filter", &lua_cbacks::set_filter},
-@@ -146,7 +146,7 @@ const static struct luaL_reg ll_chisel [] =
- 	{NULL,NULL}
- };
- 
--const static struct luaL_reg ll_evt [] =
-+const static struct luaL_Reg ll_evt [] =
- {
- 	{"field", &lua_cbacks::field},
- 	{"get_num", &lua_cbacks::get_num},
-diff --git a/userspace/chisel/lua_parser.cpp b/userspace/chisel/lua_parser.cpp
-index 0e26617d..78810d96 100644
---- a/userspace/chisel/lua_parser.cpp
-+++ b/userspace/chisel/lua_parser.cpp
-@@ -32,7 +32,7 @@ extern "C" {
- #include "lauxlib.h"
- }
- 
--const static struct luaL_reg ll_filter [] =
-+const static struct luaL_Reg ll_filter [] =
- {
- 	{"rel_expr", &lua_parser_cbacks::rel_expr},
- 	{"bool_op", &lua_parser_cbacks::bool_op},
-diff --git a/userspace/chisel/lua_parser_api.cpp b/userspace/chisel/lua_parser_api.cpp
-index c89e9126..c3d8008a 100644
---- a/userspace/chisel/lua_parser_api.cpp
-+++ b/userspace/chisel/lua_parser_api.cpp
-@@ -266,7 +266,7 @@ int lua_parser_cbacks::rel_expr(lua_State *ls)
- 					string err = "Got non-table as in-expression operand\n";
- 					throw sinsp_exception("parser API error");
- 				}
--				int n = luaL_getn(ls, 4);  /* get size of table */
-+				int n = lua_objlen (ls, 4);  /* get size of table */
- 				for (i=1; i<=n; i++)
- 				{
- 					lua_rawgeti(ls, 4, i);

cmake/modules/falcosecurity-libs.cmake

@@ -16,26 +16,31 @@ set(FALCOSECURITY_LIBS_CMAKE_WORKING_DIR "${CMAKE_BINARY_DIR}/falcosecurity-libs
 
 file(MAKE_DIRECTORY ${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR})
 
-# The falcosecurity/libs git reference (branch name, commit hash, or tag) To update falcosecurity/libs version for the next release, change the
-# default below In case you want to test against another falcosecurity/libs version just pass the variable - ie., `cmake
-# -DFALCOSECURITY_LIBS_VERSION=dev ..`
-if(NOT FALCOSECURITY_LIBS_VERSION)
-  set(FALCOSECURITY_LIBS_VERSION "3aa7a83bf7b9e6229a3824e3fd1f4452d1e95cb4")
-  set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=1edb535b3778fcfb46bbeeda891f176a1bd591bebd7b89c27f04837e55a52beb")
+if(FALCOSECURITY_LIBS_SOURCE_DIR)
+  set(FALCOSECURITY_LIBS_VERSION "local")
+  message(STATUS "Using local falcosecurity/libs in '${FALCOSECURITY_LIBS_SOURCE_DIR}'")
+else()
+  # The falcosecurity/libs git reference (branch name, commit hash, or tag) To update falcosecurity/libs version for the next release, change the
+  # default below In case you want to test against another falcosecurity/libs version just pass the variable - ie., `cmake
+  # -DFALCOSECURITY_LIBS_VERSION=dev ..`
+  if(NOT FALCOSECURITY_LIBS_VERSION)
+    set(FALCOSECURITY_LIBS_VERSION "319368f1ad778691164d33d59945e00c5752cd27")
+    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=2cf44f06a282e8cee7aa1f775a08ea94c06e275faaf0636b21eb06af28cf4b3f")
+  endif()
+
+  # cd /path/to/build && cmake /path/to/source
+  execute_process(COMMAND "${CMAKE_COMMAND}" -DFALCOSECURITY_LIBS_VERSION=${FALCOSECURITY_LIBS_VERSION} -DFALCOSECURITY_LIBS_CHECKSUM=${FALCOSECURITY_LIBS_CHECKSUM}
+                          ${FALCOSECURITY_LIBS_CMAKE_SOURCE_DIR} WORKING_DIRECTORY ${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR})
+
+  # todo(leodido, fntlnz) > use the following one when CMake version will be >= 3.13
+
+  # execute_process(COMMAND "${CMAKE_COMMAND}" -B ${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR} WORKING_DIRECTORY
+  # "${FALCOSECURITY_LIBS_CMAKE_SOURCE_DIR}")
+
+  execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR}")
+  set(FALCOSECURITY_LIBS_SOURCE_DIR "${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR}/falcosecurity-libs-prefix/src/falcosecurity-libs")
 endif()
 
-# cd /path/to/build && cmake /path/to/source
-execute_process(COMMAND "${CMAKE_COMMAND}" -DFALCOSECURITY_LIBS_VERSION=${FALCOSECURITY_LIBS_VERSION} -DFALCOSECURITY_LIBS_CHECKSUM=${FALCOSECURITY_LIBS_CHECKSUM}
-                        ${FALCOSECURITY_LIBS_CMAKE_SOURCE_DIR} WORKING_DIRECTORY ${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR})
-
-# todo(leodido, fntlnz) > use the following one when CMake version will be >= 3.13
-
-# execute_process(COMMAND "${CMAKE_COMMAND}" -B ${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR} WORKING_DIRECTORY
-# "${FALCOSECURITY_LIBS_CMAKE_SOURCE_DIR}")
-
-execute_process(COMMAND "${CMAKE_COMMAND}" --build . WORKING_DIRECTORY "${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR}")
-set(FALCOSECURITY_LIBS_SOURCE_DIR "${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR}/falcosecurity-libs-prefix/src/falcosecurity-libs")
-
 add_definitions(-D_GNU_SOURCE)
 add_definitions(-DHAS_CAPTURE)
 if(MUSL_OPTIMIZED_BUILD)
@@ -43,6 +48,10 @@ if(MUSL_OPTIMIZED_BUILD)
 endif()
 
 set(PROBE_VERSION "${FALCOSECURITY_LIBS_VERSION}")
+set(PROBE_NAME "falco")
+set(DRIVER_PACKAGE_NAME "falco")
+set(SCAP_BPF_PROBE_ENV_VAR_NAME "FALCO_BPF_PROBE")
+set(SCAP_HOST_ROOT_ENV_VAR_NAME "HOST_ROOT")
 
 if(NOT LIBSCAP_DIR)
   set(LIBSCAP_DIR "${FALCOSECURITY_LIBS_SOURCE_DIR}")
@@ -60,8 +69,19 @@ set(WITH_CHISEL ON CACHE BOOL "")
 set(USE_BUNDLED_TBB ON CACHE BOOL "")
 set(USE_BUNDLED_B64 ON CACHE BOOL "")
 set(USE_BUNDLED_JSONCPP ON CACHE BOOL "")
+set(USE_BUNDLED_LUAJIT ON CACHE BOOL "")
 
 list(APPEND CMAKE_MODULE_PATH "${FALCOSECURITY_LIBS_SOURCE_DIR}/cmake/modules")
 
+include(CheckSymbolExists)
+check_symbol_exists(strlcpy "string.h" HAVE_STRLCPY)
+if(HAVE_STRLCPY)
+	message(STATUS "Existing strlcpy found, will *not* use local definition by setting -DHAVE_STRLCPY.")
+	add_definitions(-DHAVE_STRLCPY)
+else()
+	message(STATUS "No strlcpy found, will use local definition")
+endif()
+
 include(libscap)
 include(libsinsp)
+

cmake/modules/luajit.cmake

@@ -1,30 +0,0 @@
-#
-# Copyright (C) 2020 The Falco Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-#
-
-if(NOT LUAJIT_INCLUDE)
-  set(LUAJIT_SRC "${PROJECT_BINARY_DIR}/luajit-prefix/src/luajit/src")
-  message(STATUS "Using bundled LuaJIT in '${LUAJIT_SRC}'")
-  set(LUAJIT_INCLUDE "${LUAJIT_SRC}")
-  set(LUAJIT_LIB "${LUAJIT_SRC}/libluajit.a")
-  externalproject_add(
-    luajit
-    GIT_REPOSITORY "https://github.com/LuaJIT/LuaJIT"
-    GIT_TAG "1d8b747c161db457e032a023ebbff511f5de5ec2"
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND ${CMD_MAKE}
-    BUILD_IN_SOURCE 1
-    BUILD_BYPRODUCTS ${LUAJIT_LIB}
-    INSTALL_COMMAND ""
-  )
-endif()
-include_directories("${LUAJIT_INCLUDE}")

cmake/modules/lyaml.cmake

@@ -11,9 +11,10 @@
 # specific language governing permissions and limitations under the License.
 #
 
-set(LYAML_SRC "${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/ext/yaml")
-set(LYAML_LIB "${LYAML_SRC}/.libs/yaml.a")
-message(STATUS "Using bundled lyaml in '${LYAML_SRC}'")
+set(LYAML_ROOT "${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml")
+set(LYAML_LIB "${LYAML_ROOT}/ext/yaml/.libs/yaml.a")
+set(LYAML_LUA_DIR "${LYAML_ROOT}/lib")
+message(STATUS "Using bundled lyaml in '${LYAML_ROOT}'")
 externalproject_add(
   lyaml
   DEPENDS luajit libyaml
@@ -22,7 +23,6 @@ externalproject_add(
   BUILD_COMMAND ${CMD_MAKE}
   BUILD_IN_SOURCE 1
   BUILD_BYPRODUCTS ${LYAML_LIB}
+  INSTALL_COMMAND ""
   CONFIGURE_COMMAND ./configure --enable-static CFLAGS=-I${LIBYAML_INSTALL_DIR}/include CPPFLAGS=-I${LIBYAML_INSTALL_DIR}/include LDFLAGS=-L${LIBYAML_INSTALL_DIR}/lib LIBS=-lyaml LUA=${LUAJIT_SRC}/luajit LUA_INCLUDE=-I${LUAJIT_INCLUDE}
-  INSTALL_COMMAND sh -c
-  "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/engine/lua"
 )

cmake/modules/plugins.cmake

@@ -0,0 +1,36 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+
+include(ExternalProject)
+
+string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} PLUGINS_SYSTEM_NAME)
+
+ExternalProject_Add(
+  cloudtrail-plugin
+  URL "https://download.falco.org/plugins/stable/cloudtrail-0.2.2-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
+  URL_HASH "SHA256=1628717e48b2ba1b9c78c9081e2ec23e4d88bb1a7b68b12cf8dff7f247b5b9b1"
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND ""
+  INSTALL_COMMAND "")
+
+install(FILES "${PROJECT_BINARY_DIR}/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so" DESTINATION "${FALCO_PLUGINS_DIR}")
+
+ExternalProject_Add(
+  json-plugin
+  URL "https://download.falco.org/plugins/stable/json-0.2.1-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
+  URL_HASH "SHA256=14d1cf4c3c651af0daec7a45162ef91172d6f0baba787f0eff0227b3cf2ca39c"
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND ""
+  INSTALL_COMMAND "")
+
+install(FILES "${PROJECT_BINARY_DIR}/json-plugin-prefix/src/json-plugin/libjson.so" DESTINATION "${FALCO_PLUGINS_DIR}")

docker/builder/root/usr/bin/usage

@@ -18,7 +18,7 @@ How to use.
     * docker run -ti falcosecurity/falco-builder bash
 
     To build Falco it needs:
-    - a bind-mount on the source directory (ie., the directory containing Falco and sysdig source as siblings)
+    - a bind-mount on the source directory (ie., the directory containing the Falco source as sibling)
 
     Optionally, you can also bind-mount the build directory.
     So, you can execute it from the Falco root directory as follows.

docker/no-driver/Dockerfile

@@ -20,7 +20,7 @@ RUN curl -L -o falco.tar.gz \
 RUN sed -e 's/time_format_iso_8601: false/time_format_iso_8601: true/' < /falco/etc/falco/falco.yaml > /falco/etc/falco/falco.yaml.new \
     && mv /falco/etc/falco/falco.yaml.new /falco/etc/falco/falco.yaml
 
-FROM scratch
+FROM debian:11-slim
 
 LABEL maintainer="cncf-falco-dev@lists.cncf.io"
 

falco.yaml

@@ -33,6 +33,32 @@ rules_file:
   - /etc/falco/k8s_audit_rules.yaml
   - /etc/falco/rules.d
 
+
+#
+# Plugins that are available for use. These plugins are not loaded by
+# default, as they require explicit configuration to point to
+# cloudtrail log files.
+#
+
+# To learn more about the supported formats for
+# init_config/open_params for the cloudtrail plugin, see the README at
+# https://github.com/falcosecurity/plugins/blob/master/plugins/cloudtrail/README.md.
+plugins:
+  - name: cloudtrail
+    library_path: libcloudtrail.so
+    init_config: ""
+    open_params: ""
+  - name: json
+    library_path: libjson.so
+    init_config: ""
+
+# Setting this list to empty ensures that the above plugins are *not*
+# loaded and enabled by default. If you want to use the above plugins,
+# set a meaningful init_config/open_params for the cloudtrail plugin
+# and then change this to:
+# load_plugins: [cloudtrail, json]
+load_plugins: []
+
 # If true, the times displayed in log messages and output messages
 # will be in ISO 8601. By default, times are displayed in the local
 # time zone, as governed by /etc/localtime.
@@ -66,7 +92,7 @@ log_level: info
 # Minimum rule priority level to load and run. All rules having a
 # priority more severe than this level will be loaded/run.  Can be one
 # of "emergency", "alert", "critical", "error", "warning", "notice",
-# "info", "debug".
+# "informational", "debug".
 priority: debug
 
 # Whether or not output to any of the output channels below is
@@ -220,6 +246,7 @@ program_output:
 http_output:
   enabled: false
   url: http://some.url
+  user_agent: "falcosecurity/falco"
 
 # Falco supports running a gRPC server with two main binding types
 # 1. Over the network with mandatory mutual TLS authentication (mTLS)

rules/CMakeLists.txt

@@ -23,6 +23,7 @@ if(NOT DEFINED FALCO_RULES_DEST_FILENAME)
   set(FALCO_LOCAL_RULES_DEST_FILENAME "falco_rules.local.yaml")
   set(FALCO_APP_RULES_DEST_FILENAME "application_rules.yaml")
   set(FALCO_K8S_AUDIT_RULES_DEST_FILENAME "k8s_audit_rules.yaml")
+  set(FALCO_AWS_CLOUDTRAIL_RULES_DEST_FILENAME "aws_cloudtrail_rules.yaml")
 endif()
 
 if(DEFINED FALCO_COMPONENT)
@@ -59,5 +60,10 @@ else()
     DESTINATION "${FALCO_ETC_DIR}/rules.available"
     RENAME "${FALCO_APP_RULES_DEST_FILENAME}")
 
+  install(
+    FILES aws_cloudtrail_rules.yaml
+    DESTINATION "${FALCO_ETC_DIR}"
+    RENAME "${FALCO_AWS_CLOUDTRAIL_RULES_DEST_FILENAME}")
+
   install(DIRECTORY DESTINATION "${FALCO_ETC_DIR}/rules.d")
 endif()

rules/aws_cloudtrail_rules.yaml

@@ -0,0 +1,442 @@
+#
+# Copyright (C) 2022 The Falco Authors.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# All rules files related to plugins should require engine version 10
+- required_engine_version: 10
+
+# These rules can be read by cloudtrail plugin version 0.1.0, or
+# anything semver-compatible.
+- required_plugin_versions:
+  - name: cloudtrail
+    version: 0.2.2
+  - name: json
+    version: 0.2.1
+
+# Note that this rule is disabled by default. It's useful only to
+# verify that the cloudtrail plugin is sending events properly.  The
+# very broad condition evt.num > 0 only works because the rule source
+# is limited to aws_cloudtrail. This ensures that the only events that
+# are matched against the rule are from the cloudtrail plugin (or
+# a different plugin with the same source).
+- rule: All Cloudtrail Events
+  desc: Match all cloudtrail events.
+  condition:
+    evt.num > 0
+  output: Some Cloudtrail Event (evtnum=%evt.num info=%evt.plugininfo ts=%evt.time.iso8601 id=%ct.id error=%ct.error)
+  priority: DEBUG
+  tags:
+  - cloud
+  - aws
+  source: aws_cloudtrail
+  enabled: false
+
+- rule: Console Login Through Assume Role
+  desc: Detect a console login through Assume Role.
+  condition:
+    ct.name="ConsoleLogin" and not ct.error exists
+    and ct.user.identitytype="AssumedRole"
+    and json.value[/responseElements/ConsoleLogin]="Success"
+  output:
+    Detected a console login through Assume Role
+    (principal=%ct.user.principalid,
+    assumedRole=%ct.user.arn,
+    requesting IP=%ct.srcip,
+    AWS region=%ct.region)
+  priority: WARNING
+  tags:
+  - cloud
+  - aws
+  - aws_console
+  - aws_iam
+  source: aws_cloudtrail
+
+- rule: Console Login Without MFA
+  desc: Detect a console login without MFA.
+  condition:
+    ct.name="ConsoleLogin" and not ct.error exists
+    and ct.user.identitytype!="AssumedRole"
+    and json.value[/responseElements/ConsoleLogin]="Success"
+    and json.value[/additionalEventData/MFAUsed]="No"
+  output:
+    Detected a console login without MFA
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region)
+  priority: CRITICAL
+  tags:
+    - cloud
+    - aws
+    - aws_console
+    - aws_iam
+  source: aws_cloudtrail
+
+- rule: Console Root Login Without MFA
+  desc: Detect root console login without MFA.
+  condition:
+    ct.name="ConsoleLogin" and not ct.error exists
+    and json.value[/additionalEventData/MFAUsed]="No"
+    and ct.user.identitytype!="AssumedRole"
+    and json.value[/responseElements/ConsoleLogin]="Success"
+    and ct.user.identitytype="Root"
+  output:
+    Detected a root console login without MFA.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region)
+  priority: CRITICAL
+  tags:
+    - cloud
+    - aws
+    - aws_console
+    - aws_iam
+  source: aws_cloudtrail
+
+- rule: Deactivate MFA for Root User
+  desc: Detect deactivating MFA configuration for root.
+  condition:
+    ct.name="DeactivateMFADevice" and not ct.error exists
+    and ct.user.identitytype="Root"
+    and ct.request.username="AWS ROOT USER"
+  output:
+    Multi Factor Authentication configuration has been disabled for root
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     MFA serial number=%ct.request.serialnumber)
+  priority: CRITICAL
+  tags:
+    - cloud
+    - aws
+    - aws_iam
+  source: aws_cloudtrail
+
+- rule: Create AWS user
+  desc: Detect creation of a new AWS user.
+  condition:
+    ct.name="CreateUser" and not ct.error exists
+  output:
+    A new AWS user has been created
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     new user created=%ct.request.username)
+  priority: INFO
+  tags:
+    - cloud
+    - aws
+    - aws_iam
+  source: aws_cloudtrail
+
+- rule: Create Group
+  desc: Detect creation of a new user group.
+  condition:
+    ct.name="CreateGroup" and not ct.error exists
+  output:
+    A new user group has been created.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     group name=%ct.request.groupname)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_iam
+  source: aws_cloudtrail
+
+- rule: Delete Group
+  desc: Detect deletion of a user group.
+  condition:
+    ct.name="DeleteGroup" and not ct.error exists
+  output:
+    A user group has been deleted.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     group name=%ct.request.groupname)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_iam
+  source: aws_cloudtrail
+
+- rule: ECS Service Created
+  desc: Detect a new service is created in ECS.
+  condition:
+    ct.src="ecs.amazonaws.com" and
+    ct.name="CreateService" and
+    not ct.error exists
+  output:
+    A new service has been created in ECS
+    (requesting user=%ct.user,
+    requesting IP=%ct.srcip,
+    AWS region=%ct.region,
+    cluster=%ct.request.cluster,
+    service name=%ct.request.servicename,
+    task definition=%ct.request.taskdefinition)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_ecs
+    - aws_fargate
+  source: aws_cloudtrail
+
+- rule: ECS Task Run or Started
+  desc: Detect a new task is started in ECS.
+  condition:
+    ct.src="ecs.amazonaws.com" and
+    (ct.name="RunTask" or ct.name="StartTask") and
+    not ct.error exists
+  output:
+    A new task has been started in ECS
+    (requesting user=%ct.user,
+    requesting IP=%ct.srcip,
+    AWS region=%ct.region,
+    cluster=%ct.request.cluster,
+    task definition=%ct.request.taskdefinition)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_ecs
+    - aws_fargate
+  source: aws_cloudtrail
+
+- rule: Create Lambda Function
+  desc: Detect creation of a Lambda function.
+  condition:
+    ct.name="CreateFunction20150331" and not ct.error exists
+  output:
+    Lambda function has been created.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     lambda function=%ct.request.functionname)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_lambda
+  source: aws_cloudtrail
+
+- rule: Update Lambda Function Code
+  desc: Detect updates to a Lambda function code.
+  condition:
+    ct.name="UpdateFunctionCode20150331v2" and not ct.error exists
+  output:
+    The code of a Lambda function has been updated.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     lambda function=%ct.request.functionname)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_lambda
+  source: aws_cloudtrail
+
+- rule: Update Lambda Function Configuration
+  desc: Detect updates to a Lambda function configuration.
+  condition:
+    ct.name="UpdateFunctionConfiguration20150331v2" and not ct.error exists
+  output:
+    The configuration of a Lambda function has been updated.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     lambda function=%ct.request.functionname)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_lambda
+  source: aws_cloudtrail
+
+- rule: Run Instances
+  desc: Detect launching of a specified number of instances.
+  condition:
+    ct.name="RunInstances" and not ct.error exists
+  output:
+    A number of instances have been launched.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     availability zone=%ct.request.availabilityzone,
+     subnet id=%ct.response.subnetid,
+     reservation id=%ct.response.reservationid)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_ec2
+  source: aws_cloudtrail
+
+# Only instances launched on regions in this list are approved.
+- list: approved_regions
+  items:
+    - us-east-0
+
+- rule: Run Instances in Non-approved Region
+  desc: Detect launching of a specified number of instances in a non-approved region.
+  condition:
+    ct.name="RunInstances" and not ct.error exists and
+    not ct.region in (approved_regions)
+  output:
+    A number of instances have been launched in a non-approved region.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     availability zone=%ct.request.availabilityzone,
+     subnet id=%ct.response.subnetid,
+     reservation id=%ct.response.reservationid,
+     image id=%json.value[/responseElements/instancesSet/items/0/instanceId])
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_ec2
+  source: aws_cloudtrail
+
+- rule: Delete Bucket Encryption
+  desc: Detect deleting configuration to use encryption for bucket storage.
+  condition:
+    ct.name="DeleteBucketEncryption" and not ct.error exists
+  output:
+    A encryption configuration for a bucket has been deleted
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     bucket=%s3.bucket)
+  priority: CRITICAL
+  tags:
+    - cloud
+    - aws
+    - aws_s3
+  source: aws_cloudtrail
+
+- rule: Delete Bucket Public Access Block
+  desc: Detect deleting blocking public access to bucket.
+  condition:
+    ct.name="PutBucketPublicAccessBlock" and not ct.error exists and
+    json.value[/requestParameters/publicAccessBlock]='""' and
+      (json.value[/requestParameters/PublicAccessBlockConfiguration/RestrictPublicBuckets]=false or
+      json.value[/requestParameters/PublicAccessBlockConfiguration/BlockPublicPolicy]=false or
+      json.value[/requestParameters/PublicAccessBlockConfiguration/BlockPublicAcls]=false or
+      json.value[/requestParameters/PublicAccessBlockConfiguration/IgnorePublicAcls]=false)
+  output:
+    A pulic access block for a bucket has been deleted
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     bucket=%s3.bucket)
+  priority: CRITICAL
+  tags:
+    - cloud
+    - aws
+    - aws_s3
+  source: aws_cloudtrail
+
+- rule: List Buckets
+  desc: Detect listing of all S3 buckets.
+  condition:
+    ct.name="ListBuckets" and not ct.error exists
+  output:
+    A list of all S3 buckets has been requested.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     host=%ct.request.host)
+  priority: WARNING
+  enabled: false
+  tags:
+    - cloud
+    - aws
+    - aws_s3
+  source: aws_cloudtrail
+
+- rule: Put Bucket ACL
+  desc: Detect setting the permissions on an existing bucket using access control lists.
+  condition:
+    ct.name="PutBucketAcl" and not ct.error exists
+  output:
+    The permissions on an existing bucket have been set using access control lists.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     bucket name=%s3.bucket)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_s3
+  source: aws_cloudtrail
+
+- rule: Put Bucket Policy
+  desc: Detect applying an Amazon S3 bucket policy to an Amazon S3 bucket.
+  condition:
+    ct.name="PutBucketPolicy" and not ct.error exists
+  output:
+    An Amazon S3 bucket policy has been applied to an Amazon S3 bucket.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     bucket name=%s3.bucket,
+     policy=%ct.request.policy)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_s3
+  source: aws_cloudtrail
+
+- rule: CloudTrail Trail Created
+  desc: Detect creation of a new trail.
+  condition:
+    ct.name="CreateTrail" and not ct.error exists
+  output:
+    A new trail has been created.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     trail name=%ct.request.name)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_cloudtrail
+  source: aws_cloudtrail
+
+- rule: CloudTrail Logging Disabled
+  desc: The CloudTrail logging has been disabled, this could be potentially malicious.
+  condition:
+    ct.name="StopLogging" and not ct.error exists
+  output:
+    The CloudTrail logging has been disabled.
+    (requesting user=%ct.user,
+     requesting IP=%ct.srcip,
+     AWS region=%ct.region,
+     resource name=%ct.request.name)
+  priority: WARNING
+  tags:
+    - cloud
+    - aws
+    - aws_cloudtrail
+  source: aws_cloudtrail
+

rules/falco_rules.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2020 The Falco Authors.
+# Copyright (C) 2022 The Falco Authors.
 #
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -29,13 +29,13 @@
 #   condition: (syscall.type=read and evt.dir=> and fd.type in (file, directory))
 
 - macro: open_write
-  condition: (evt.type=open or evt.type=openat) and evt.is_open_write=true and fd.typechar='f' and fd.num>=0
+  condition: evt.type in (open,openat,openat2) and evt.is_open_write=true and fd.typechar='f' and fd.num>=0
 
 - macro: open_read
-  condition: (evt.type=open or evt.type=openat) and evt.is_open_read=true and fd.typechar='f' and fd.num>=0
+  condition: evt.type in (open,openat,openat2) and evt.is_open_read=true and fd.typechar='f' and fd.num>=0
 
 - macro: open_directory
-  condition: (evt.type=open or evt.type=openat) and evt.is_open_read=true and fd.typechar='d' and fd.num>=0
+  condition: evt.type in (open,openat,openat2) and evt.is_open_read=true and fd.typechar='d' and fd.num>=0
 
 - macro: never_true
   condition: (evt.num=0)
@@ -63,11 +63,14 @@
   condition: rename or remove
 
 - macro: spawned_process
-  condition: evt.type = execve and evt.dir=<
+  condition: evt.type in (execve, execveat) and evt.dir=<
 
 - macro: create_symlink
   condition: evt.type in (symlink, symlinkat) and evt.dir=<
 
+- macro: create_hardlink
+  condition: evt.type in (link, linkat) and evt.dir=<
+
 - macro: chmod
   condition: (evt.type in (chmod, fchmod, fchmodat) and evt.dir=<)
 
@@ -216,11 +219,11 @@
 - list: deb_binaries
   items: [dpkg, dpkg-preconfigu, dpkg-reconfigur, dpkg-divert, apt, apt-get, aptitude,
     frontend, preinst, add-apt-reposit, apt-auto-remova, apt-key,
-    apt-listchanges, unattended-upgr, apt-add-reposit, apt-config, apt-cache, apt.systemd.dai
+    apt-listchanges, unattended-upgr, apt-add-reposit, apt-cache, apt.systemd.dai
     ]
 
 # The truncated dpkg-preconfigu is intentional, process names are
-# truncated at the sysdig level.
+# truncated at the falcosecurity-libs level.
 - list: package_mgmt_binaries
   items: [rpm_binaries, deb_binaries, update-alternat, gem, pip, pip3, sane-utils.post, alternatives, chef-client, apk, snapd]
 
@@ -333,7 +336,7 @@
 # for efficiency.
 - macro: inbound_outbound
   condition: >
-    ((((evt.type in (accept,listen,connect) and evt.dir=<)) or
+    ((((evt.type in (accept,listen,connect) and evt.dir=<)) and
      (fd.typechar = 4 or fd.typechar = 6)) and
      (fd.ip != "0.0.0.0" and fd.net != "127.0.0.0/8") and
      (evt.rawres >= 0 or evt.res = EINPROGRESS))
@@ -1826,6 +1829,7 @@
     k8s.gcr.io/ip-masq-agent-amd64,
     k8s.gcr.io/kube-proxy,
     k8s.gcr.io/prometheus-to-sd,
+    public.ecr.aws/falcosecurity/falco,
     quay.io/calico/node,
     sysdig/sysdig,
     sematext_images
@@ -1854,7 +1858,7 @@
 - list: falco_sensitive_mount_images
   items: [
     docker.io/sysdig/sysdig, sysdig/sysdig,
-    docker.io/falcosecurity/falco, falcosecurity/falco,
+    docker.io/falcosecurity/falco, falcosecurity/falco, public.ecr.aws/falcosecurity/falco,
     gcr.io/google_containers/hyperkube,
     gcr.io/google_containers/kube-proxy, docker.io/calico/node,
     docker.io/rook/toolbox, docker.io/cloudnativelabs/kube-router, docker.io/consul,
@@ -1870,19 +1874,6 @@
               container.image.repository in (falco_sensitive_mount_images) or
               container.image.repository startswith quay.io/sysdig/)
 
-# These container images are allowed to run with hostnetwork=true
-- list: falco_hostnetwork_images
-  items: [
-    gcr.io/google-containers/prometheus-to-sd,
-    gcr.io/projectcalico-org/typha,
-    gcr.io/projectcalico-org/node,
-    gke.gcr.io/gke-metadata-server,
-    gke.gcr.io/kube-proxy,
-    gke.gcr.io/netd-amd64,
-    k8s.gcr.io/ip-masq-agent-amd64
-    k8s.gcr.io/prometheus-to-sd,
-    ]
-
 # Add conditions to this macro (probably in a separate file,
 # overwriting this macro) to specify additional containers that are
 # allowed to perform sensitive mounts.
@@ -1905,12 +1896,13 @@
 
 # For now, only considering a full mount of /etc as
 # sensitive. Ideally, this would also consider all subdirectories
-# below /etc as well, but the globbing mechanism used by sysdig
+# below /etc as well, but the globbing mechanism
 # doesn't allow exclusions of a full pattern, only single characters.
 - macro: sensitive_mount
   condition: (container.mount.dest[/proc*] != "N/A" or
               container.mount.dest[/var/run/docker.sock] != "N/A" or
               container.mount.dest[/var/run/crio/crio.sock] != "N/A" or
+              container.mount.dest[/run/containerd/containerd.sock] != "N/A" or
               container.mount.dest[/var/lib/kubelet] != "N/A" or
               container.mount.dest[/var/lib/kubelet/pki] != "N/A" or
               container.mount.dest[/] != "N/A" or
@@ -2295,7 +2287,7 @@
   desc: creating any files below /dev other than known programs that manage devices. Some rootkits hide files in /dev.
   condition: >
     fd.directory = /dev and
-    (evt.type = creat or ((evt.type = open or evt.type = openat) and evt.arg.flags contains O_CREAT))
+    (evt.type = creat or (evt.type in (open,openat,openat2) and evt.arg.flags contains O_CREAT))
     and not proc.name in (dev_creation_binaries)
     and not fd.name in (allowed_dev_files)
     and not fd.name startswith /dev/tty
@@ -2363,7 +2355,8 @@
      docker.io/sysdig/sysdig, docker.io/falcosecurity/falco,
      sysdig/sysdig, falcosecurity/falco,
      fluent/fluentd-kubernetes-daemonset, prom/prometheus,
-     ibm_cloud_containers)
+     ibm_cloud_containers,
+     public.ecr.aws/falcosecurity/falco)
      or (k8s.ns.name = "kube-system"))
 
 - macro: k8s_api_server
@@ -2705,8 +2698,18 @@
     create_symlink and
     (evt.arg.target in (sensitive_file_names) or evt.arg.target in (sensitive_directory_names))
   output: >
-    Symlinks created over senstivie files (user=%user.name user_loginuid=%user.loginuid command=%proc.cmdline target=%evt.arg.target linkpath=%evt.arg.linkpath parent_process=%proc.pname)
-  priority: NOTICE
+    Symlinks created over sensitive files (user=%user.name user_loginuid=%user.loginuid command=%proc.cmdline target=%evt.arg.target linkpath=%evt.arg.linkpath parent_process=%proc.pname)
+  priority: WARNING
+  tags: [file, mitre_exfiltration]
+
+- rule: Create Hardlink Over Sensitive Files
+  desc: Detect hardlink created over sensitive files
+  condition: >
+    create_hardlink and
+    (evt.arg.oldpath in (sensitive_file_names))
+  output: >
+    Hardlinks created over sensitive files (user=%user.name user_loginuid=%user.loginuid command=%proc.cmdline target=%evt.arg.oldpath linkpath=%evt.arg.newpath parent_process=%proc.pname)
+  priority: WARNING
   tags: [file, mitre_exfiltration]
 
 - list: miner_ports
@@ -2802,7 +2805,7 @@
   condition: (evt.type in (sendto, sendmsg) and evt.dir=< and (fd.net != "127.0.0.0/8" and not fd.snet in (rfc_1918_addresses)) and ((minerpool_http) or (minerpool_https) or (minerpool_other)))
 
 - macro: trusted_images_query_miner_domain_dns
-  condition: (container.image.repository in (docker.io/falcosecurity/falco, falcosecurity/falco))
+  condition: (container.image.repository in (docker.io/falcosecurity/falco, falcosecurity/falco, public.ecr.aws/falcosecurity/falco))
   append: false
 
 # The rule is disabled by default.
@@ -2817,7 +2820,7 @@
 
 - rule: Detect crypto miners using the Stratum protocol
   desc: Miners typically specify the mining pool to connect to with a URI that begins with 'stratum+tcp'
-  condition: spawned_process and proc.cmdline contains "stratum+tcp"
+  condition: spawned_process and (proc.cmdline contains "stratum+tcp" or proc.cmdline contains "stratum2+tcp" or proc.cmdline contains "stratum+ssl" or proc.cmdline contains "stratum2+ssl")
   output: Possible miner running (command=%proc.cmdline container=%container.info image=%container.image.repository)
   priority: CRITICAL
   tags: [process, mitre_execution]
@@ -2953,7 +2956,7 @@
 
 # The two Container Drift rules below will fire when a new executable is created in a container.
 # There are two ways to create executables - file is created with execution permissions or permissions change of existing file.
-# We will use a new sysdig filter, is_open_exec, to find all files creations with execution permission, and will trace all chmods in a container.
+# We will use a new filter, is_open_exec, to find all files creations with execution permission, and will trace all chmods in a container.
 # The use case we are targeting here is an attempt to execute code that was not shipped as part of a container (drift) -
 # an activity that might be malicious or non-compliant.
 # Two things to pay attention to:
@@ -2986,7 +2989,7 @@
 - rule: Container Drift Detected (open+create)
   desc: New executable created in a container due to open+create
   condition: >
-    evt.type in (open,openat,creat) and
+    evt.type in (open,openat,openat2,creat) and
     evt.is_open_exec=true and
     container and
     not runc_writing_exec_fifo and
@@ -3036,7 +3039,7 @@
 # A privilege escalation to root through heap-based buffer overflow
 - rule: Sudo Potential Privilege Escalation
   desc: Privilege escalation vulnerability affecting sudo (<= 1.9.5p2). Executing sudo using sudoedit -s or sudoedit -i command with command-line argument that ends with a single backslash character from an unprivileged user it's possible to elevate the user privileges to root.
-  condition: spawned_process and user.uid != 0 and proc.name=sudoedit and (proc.args contains -s or proc.args contains -i) and (proc.args contains "\ " or proc.args endswith \)
+  condition: spawned_process and user.uid != 0 and (proc.name=sudoedit or proc.name = sudo) and (proc.args contains -s or proc.args contains -i or proc.args contains --login) and (proc.args contains "\ " or proc.args endswith \)
   output: "Detect Sudo Privilege Escalation Exploit (CVE-2021-3156) (user=%user.name parent=%proc.pname cmdline=%proc.cmdline %container.info)"
   priority: CRITICAL
   tags: [filesystem, mitre_privilege_escalation]
@@ -3083,6 +3086,40 @@
   priority: CRITICAL
   tags: [syscall, mitre_defense_evasion]
 
+- list: ingress_remote_file_copy_binaries
+  items: [wget]
+
+- macro: ingress_remote_file_copy_procs
+  condition: (proc.name in (ingress_remote_file_copy_binaries))
+
+# Users should overwrite this macro to specify conditions under which a
+# Custom condition for use of ingress remote file copy tool in container
+- macro: user_known_ingress_remote_file_copy_activities
+  condition: (never_true)
+
+-  macro: curl_download
+   condition: proc.name = curl and
+              (proc.cmdline contains " > " or
+              proc.cmdline contains " >> " or
+              proc.cmdline contains " | " or
+              proc.cmdline contains " -o " or
+              proc.cmdline contains " --output " or
+              proc.cmdline contains " -O " or
+              proc.cmdline contains " --remote-name ")
+
+- rule: Launch Ingress Remote File Copy Tools in Container
+  desc: Detect ingress remote file copy tools launched in container
+  condition: >
+    spawned_process and
+    container and
+    (ingress_remote_file_copy_procs or curl_download) and
+    not user_known_ingress_remote_file_copy_activities
+  output: >
+    Ingress remote file copy tool launched in container (user=%user.name user_loginuid=%user.loginuid command=%proc.cmdline parent_process=%proc.pname
+    container_id=%container.id container_name=%container.name image=%container.image.repository:%container.image.tag)
+  priority: NOTICE
+  tags: [network, process, mitre_command_and_control]
+
 # Application rules have moved to application_rules.yaml. Please look
 # there if you want to enable them by adding to
 # falco_rules.local.yaml.

rules/k8s_audit_rules.yaml

@@ -152,6 +152,19 @@
   source: k8s_audit
   tags: [k8s]
 
+# These container images are allowed to run with hostnetwork=true
+- list: falco_hostnetwork_images
+  items: [
+    gcr.io/google-containers/prometheus-to-sd,
+    gcr.io/projectcalico-org/typha,
+    gcr.io/projectcalico-org/node,
+    gke.gcr.io/gke-metadata-server,
+    gke.gcr.io/kube-proxy,
+    gke.gcr.io/netd-amd64,
+    k8s.gcr.io/ip-masq-agent-amd64
+    k8s.gcr.io/prometheus-to-sd,
+    ]
+
 # Corresponds to K8s CIS Benchmark 1.7.4
 - rule: Create HostNetwork Pod
   desc: Detect an attempt to start a pod using the host network.
@@ -302,9 +315,31 @@
   items: []
 
 - list: known_sa_list
-  items: ["pod-garbage-collector","resourcequota-controller","cronjob-controller","generic-garbage-collector",
-          "daemon-set-controller","endpointslice-controller","deployment-controller", "replicaset-controller",
-          "endpoint-controller", "namespace-controller", "statefulset-controller", "disruption-controller"]
+  items: [
+    coredns,
+    coredns-autoscaler,
+    cronjob-controller,
+    daemon-set-controller,
+    deployment-controller,
+    disruption-controller,
+    endpoint-controller,
+    endpointslice-controller,
+    endpointslicemirroring-controller,
+    generic-garbage-collector,
+    horizontal-pod-autoscaler,
+    job-controller,
+    namespace-controller,
+    node-controller,
+    persistent-volume-binder,
+    pod-garbage-collector,
+    pv-protection-controller,
+    pvc-protection-controller,
+    replicaset-controller,
+    resourcequota-controller,
+    root-ca-cert-publisher,
+    service-account-controller,
+    statefulset-controller
+    ]
 
 - macro: trusted_sa
   condition: (ka.target.name in (known_sa_list, user_known_sa_list))

scripts/falco-driver-loader

@@ -160,13 +160,26 @@ load_kernel_module_compile() {
 		echo "make CC=${CURRENT_GCC} \$@" >> /tmp/falco-dkms-make
 		chmod +x /tmp/falco-dkms-make
 		if dkms install --directive="MAKE='/tmp/falco-dkms-make'" -m "${DRIVER_NAME}" -v "${DRIVER_VERSION}" -k "${KERNEL_RELEASE}" 2>/dev/null; then
-			echo "* ${DRIVER_NAME} module installed in dkms, trying to insmod"
-			if insmod "/var/lib/dkms/${DRIVER_NAME}/${DRIVER_VERSION}/${KERNEL_RELEASE}/${ARCH}/module/${DRIVER_NAME}.ko" > /dev/null 2>&1; then
+			echo "* ${DRIVER_NAME} module installed in dkms"
+			KO_FILE="/var/lib/dkms/${DRIVER_NAME}/${DRIVER_VERSION}/${KERNEL_RELEASE}/${ARCH}/module/${DRIVER_NAME}"
+			if [ -f "$KO_FILE.ko" ]; then
+				KO_FILE="$KO_FILE.ko"
+			elif [ -f "$KO_FILE.ko.gz" ]; then
+				KO_FILE="$KO_FILE.ko.gz"
+			elif [ -f "$KO_FILE.ko.xz" ]; then
+				KO_FILE="$KO_FILE.ko.xz"
+			elif [ -f "$KO_FILE.ko.zst" ]; then
+				KO_FILE="$KO_FILE.ko.zst"
+			else
+				>&2 echo "${DRIVER_NAME} module file not found"
+				return
+			fi
+			echo "* ${DRIVER_NAME} module found: ${KO_FILE}"
+			echo "* Trying insmod"
+			chcon -t modules_object_t "$KO_FILE" > /dev/null 2>&1 || true
+			if insmod "$KO_FILE" > /dev/null 2>&1; then
 				echo "* Success: ${DRIVER_NAME} module found and loaded in dkms"
 				exit 0
-			elif insmod "/var/lib/dkms/${DRIVER_NAME}/${DRIVER_VERSION}/${KERNEL_RELEASE}/${ARCH}/module/${DRIVER_NAME}.ko.xz" > /dev/null 2>&1; then
-				echo "* Success: ${DRIVER_NAME} module found and loaded in dkms (xz)"
-				exit 0
 			else
 				echo "* Unable to insmod ${DRIVER_NAME} module"
 			fi
@@ -193,8 +206,13 @@ load_kernel_module_download() {
 	echo "* Trying to download a prebuilt ${DRIVER_NAME} module from ${URL}"
 	if curl -L --create-dirs "${FALCO_DRIVER_CURL_OPTIONS}" -o "${HOME}/.falco/${FALCO_KERNEL_MODULE_FILENAME}" "${URL}"; then
 		echo "* Download succeeded"
-		insmod "${HOME}/.falco/${FALCO_KERNEL_MODULE_FILENAME}" && echo "* Success: ${DRIVER_NAME} module found and inserted"
-		exit $?
+		chcon -t modules_object_t "${HOME}/.falco/${FALCO_KERNEL_MODULE_FILENAME}" > /dev/null 2>&1 || true
+		if insmod "${HOME}/.falco/${FALCO_KERNEL_MODULE_FILENAME}"; then
+			echo "* Success: ${DRIVER_NAME} module found and inserted"
+			exit 0
+		else
+			>&2 echo "Unable to insmod the prebuilt ${DRIVER_NAME} module"
+		fi	
 	else
 		>&2 echo "Unable to find a prebuilt ${DRIVER_NAME} module"
 		return
@@ -238,11 +256,6 @@ load_kernel_module() {
 		exit 0
 	fi
 
-	echo "* Trying to load a system ${DRIVER_NAME} module, if present"
-	if modprobe "${DRIVER_NAME}" > /dev/null 2>&1; then
-		echo "* Success: ${DRIVER_NAME} module found and loaded with modprobe"
-		exit 0
-	fi
 
 	echo "* Looking for a ${DRIVER_NAME} module locally (kernel ${KERNEL_RELEASE})"
 
@@ -252,6 +265,7 @@ load_kernel_module() {
 
 	if [ -f "${HOME}/.falco/${FALCO_KERNEL_MODULE_FILENAME}" ]; then
 		echo "* Found a prebuilt ${DRIVER_NAME} module at ${HOME}/.falco/${FALCO_KERNEL_MODULE_FILENAME}, loading it"
+		chcon -t modules_object_t "${HOME}/.falco/${FALCO_KERNEL_MODULE_FILENAME}" > /dev/null 2>&1 || true
 		insmod "${HOME}/.falco/${FALCO_KERNEL_MODULE_FILENAME}" && echo "* Success: ${DRIVER_NAME} module found and inserted"
 		exit $?
 	fi
@@ -264,6 +278,13 @@ load_kernel_module() {
 		load_kernel_module_compile
 	fi
 
+	# Last try (might load a previous driver version)
+	echo "* Trying to load a system ${DRIVER_NAME} module, if present"
+	if modprobe "${DRIVER_NAME}" > /dev/null 2>&1; then
+		echo "* Success: ${DRIVER_NAME} module found and loaded with modprobe"
+		exit 0
+	fi
+
 	# Not able to download a prebuilt module nor to compile one on-the-fly
 	>&2 echo "Consider compiling your own ${DRIVER_NAME} driver and loading it or getting in touch with the Falco community"
 	exit 1
@@ -296,16 +317,16 @@ clean_kernel_module() {
 		return
 	fi
 
-	DRIVER_VERSIONS=$(dkms status -m "${DRIVER_NAME}" | cut -d',' -f2 | sed -e 's/^[[:space:]]*//')
+	DRIVER_VERSIONS=$(dkms status -m "${DRIVER_NAME}" | cut -d',' -f1 | sed -e 's/^[[:space:]]*//')
 	if [ -z "${DRIVER_VERSIONS}" ]; then
 		echo "* There is no ${DRIVER_NAME} module in dkms"
 		return
 	fi
 	for CURRENT_VER in ${DRIVER_VERSIONS}; do
-		if dkms remove -m "${DRIVER_NAME}" -v "${CURRENT_VER}" --all 2>/dev/null; then
-			echo "* Removing ${DRIVER_NAME}/${CURRENT_VER} succeeded"
+		if dkms remove "${CURRENT_VER}" --all 2>/dev/null; then
+			echo "* Removing ${CURRENT_VER} succeeded"
 		else
-			echo "* Removing ${DRIVER_NAME}/${CURRENT_VER} failed"
+			echo "* Removing ${CURRENT_VER} failed"
 			exit 1
 		fi
 	done
@@ -634,4 +655,4 @@ if [ -z "$source_only" ]; then
 				;;
 		esac
 	fi
-fi
+fi

scripts/publish-deb

@@ -23,7 +23,7 @@ add_deb() {
     cp -f $3 $1/$2
     pushd $1/$2 > /dev/null
     rm -f $(basename -- $3).asc
-    gpg --detach-sign --armor $(basename -- $3)
+    gpg --detach-sign --digest-algo SHA256 --armor $(basename -- $3)
     popd > /dev/null
 }
 
@@ -63,7 +63,7 @@ update_repo() {
       ${release_dir} > ${release_dir}/Release
 
     # release signature
-    gpg --detach-sign --armor ${release_dir}/Release
+    gpg --detach-sign --digest-algo SHA256 --armor ${release_dir}/Release
     rm -f ${release_dir}/Release.gpg
     mv ${release_dir}/Release.asc ${release_dir}/Release.gpg
 
@@ -129,4 +129,4 @@ aws s3 sync ${tmp_repo_path}/dists ${s3_bucket_repo}/dists --delete --acl public
 
 aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/${debSuite}/${package}
 aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/${debSuite}/${package}.asc
-aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/dists/*
+aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/dists/*

scripts/publish-rpm

@@ -22,7 +22,7 @@ add_rpm() {
     cp -f $2 $1
     pushd $1 > /dev/null
     rm -f $(basename -- $2).asc
-    gpg --detach-sign --armor $(basename -- $2)
+    gpg --detach-sign --digest-algo SHA256 --armor $(basename -- $2)
     popd > /dev/null
 }
 
@@ -33,7 +33,7 @@ update_repo() {
     pushd $1 > /dev/null
     createrepo --update --no-database .
     rm -f repodata/repomd.xml.asc
-    gpg --detach-sign --armor repodata/repomd.xml
+    gpg --detach-sign --digest-algo SHA256 --armor repodata/repomd.xml
     popd > /dev/null
 }
 
@@ -93,4 +93,4 @@ aws s3 sync ${tmp_repo_path}/repodata ${s3_bucket_repo}/repodata --delete --acl
 
 aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/${package}
 aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/${package}.asc
-aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/repodata/*
+aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/repodata/*

test/CMakeLists.txt

@@ -1,4 +1,7 @@
 add_subdirectory(trace_files)
 
 add_custom_target(test-trace-files ALL)
-add_dependencies(test-trace-files trace-files-base-scap trace-files-psp trace-files-k8s-audit)
+add_dependencies(test-trace-files trace-files-base-scap trace-files-psp trace-files-k8s-audit trace-files-plugins)
+
+add_subdirectory(plugins)
+add_subdirectory(confs/plugins)

test/confs/plugins/CMakeLists.txt

@@ -0,0 +1,16 @@
+# This list is populated at cmake time, not build time
+file(GLOB test_conf_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/*.yaml")
+
+foreach(conf_file_path ${test_conf_files})
+	get_filename_component(conf_file ${conf_file_path} NAME)
+	add_custom_target(test-conf-${conf_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${conf_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${conf_file}
+		COMMAND sed -e s!BUILD_DIR!${CMAKE_BINARY_DIR}! < ${CMAKE_CURRENT_SOURCE_DIR}/${conf_file} > ${CMAKE_CURRENT_BINARY_DIR}/${conf_file}
+		DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${conf_file})
+	list(APPEND PLUGINS_CONF_FILES_TARGETS test-conf-${conf_file})
+endforeach()
+
+add_custom_target(conf-files-plugins ALL)
+add_dependencies(conf-files-plugins ${PLUGINS_CONF_FILES_TARGETS})

test/confs/plugins/cloudtrail_json_create_instances.yaml

@@ -0,0 +1,14 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: cloudtrail
+    library_path: BUILD_DIR/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so
+    init_config: ""
+    open_params: "BUILD_DIR/test/trace_files/plugins/alice_start_instances.json"
+  - name: json
+    library_path: BUILD_DIR/json-plugin-prefix/src/json-plugin/libjson.so
+    init_config: ""
+
+# Optional
+load_plugins: [cloudtrail, json]

test/confs/plugins/cloudtrail_json_create_instances_bigevent.yaml

@@ -0,0 +1,14 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: cloudtrail
+    library_path: BUILD_DIR/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so
+    init_config: ""
+    open_params: "BUILD_DIR/test/trace_files/plugins/alice_start_instances_bigevent.json"
+  - name: json
+    library_path: BUILD_DIR/json-plugin-prefix/src/json-plugin/libjson.so
+    init_config: ""
+
+# Optional
+load_plugins: [cloudtrail, json]

test/confs/plugins/incompatible_extract_sources.yaml

@@ -0,0 +1,14 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: cloudtrail
+    library_path: BUILD_DIR/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so
+    init_config: ""
+    open_params: ""
+  - name: test_extract_p1
+    library_path: BUILD_DIR/test/plugins/libtest_extract_p1.so
+    init_config: ""
+
+# Optional
+load_plugins: [cloudtrail, test_extract_p1]

test/confs/plugins/incompatible_plugin_api.yaml

@@ -0,0 +1,10 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: incompatible_plugin_api
+    library_path: BUILD_DIR/test/plugins/libtest_incompat_api.so
+    init_config: ""
+
+# Optional
+load_plugins: [incompatible_plugin_api]

test/confs/plugins/multiple_source_plugins.yaml

@@ -0,0 +1,15 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: cloudtrail
+    library_path: BUILD_DIR/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so
+    init_config: ""
+    open_params: "BUILD_DIR/test/trace_files/plugins/alice_start_instances.json"
+  - name: test_source
+    library_path: BUILD_DIR/test/plugins/libtest_source.so
+    init_config: ""
+    open_params: ""
+
+# Optional
+load_plugins: [cloudtrail, test_source]

test/confs/plugins/overlap_extract_sources.yaml

@@ -0,0 +1,17 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: test_source
+    library_path: BUILD_DIR/test/plugins/libtest_source.so
+    init_config: ""
+    open_params: ""
+  - name: test_extract_p1
+    library_path: BUILD_DIR/test/plugins/libtest_extract_p1.so
+    init_config: ""
+  - name: test_extract_p2
+    library_path: BUILD_DIR/test/plugins/libtest_extract_p2.so
+    init_config: ""
+
+# Optional
+load_plugins: [test_source, test_extract_p1, test_extract_p2]

test/confs/plugins/wrong_plugin_path.yaml

@@ -0,0 +1,10 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: wrong_plugin_path
+    library_path: BUILD_DIR/test/plugins/wrong_plugin_path.so
+    init_config: ""
+
+# Optional
+load_plugins: [wrong_plugin_path]

test/falco_k8s_audit_tests.yaml

@@ -49,6 +49,7 @@ trace_files: !mux
     detect: False
     rules_file:
       - ../rules/falco_rules.yaml
+      - ../rules/k8s_audit_rules.yaml
       - ./rules/k8s_audit/engine_v4_k8s_audit_rules.yaml
       - ./rules/k8s_audit/trust_nginx_container.yaml
     trace_file: trace_files/k8s_audit/create_nginx_pod_privileged.json
@@ -74,6 +75,7 @@ trace_files: !mux
     detect: False
     rules_file:
       - ../rules/falco_rules.yaml
+      - ../rules/k8s_audit_rules.yaml
       - ./rules/k8s_audit/engine_v4_k8s_audit_rules.yaml
       - ./rules/k8s_audit/trust_nginx_container.yaml
     trace_file: trace_files/k8s_audit/create_nginx_pod_hostnetwork.json
@@ -632,3 +634,12 @@ trace_files: !mux
       - ../rules/k8s_audit_rules.yaml
     trace_file: trace_files/k8s_audit/fal_01_003.json
     stderr_contains: 'Could not read k8s audit event line #1, "{"kind": 0}": Data not recognized as a k8s audit event, stopping'
+
+  json_pointer_correct_parse:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - ./rules/k8s_audit/single_rule_with_json_pointer.yaml
+    detect_counts:
+      - json_pointer_example: 1
+    trace_file: trace_files/k8s_audit/create_nginx_pod_unprivileged.json

test/falco_test.py

@@ -82,6 +82,7 @@ class FalcoTest(Test):
 
         self.exit_status = self.params.get('exit_status', '*', default=0)
         self.should_detect = self.params.get('detect', '*', default=False)
+        self.check_detection_counts = self.params.get('check_detection_counts', '*', default=True)
         self.trace_file = self.params.get('trace_file', '*', default='')
 
         if self.trace_file and not os.path.isabs(self.trace_file):
@@ -94,6 +95,7 @@ class FalcoTest(Test):
             'json_include_tags_property', '*', default=True)
         self.all_events = self.params.get('all_events', '*', default=False)
         self.priority = self.params.get('priority', '*', default='debug')
+        self.addl_cmdline_opts = self.params.get('addl_cmdline_opts', '*', default='')
         self.rules_file = self.params.get(
             'rules_file', '*', default=os.path.join(self.basedir, '../rules/falco_rules.yaml'))
 
@@ -130,6 +132,7 @@ class FalcoTest(Test):
 
         self.conf_file = self.params.get(
             'conf_file', '*', default=os.path.join(self.basedir, '../falco.yaml'))
+        self.conf_file = self.conf_file.replace("BUILD_DIR", build_dir)
         if not os.path.isabs(self.conf_file):
             self.conf_file = os.path.join(self.basedir, self.conf_file)
 
@@ -617,9 +620,9 @@ class FalcoTest(Test):
                 self.log.debug("Converted Rules: {}".format(psp_rules))
 
         # Run falco
-        cmd = '{} {} {} -c {} {} -o json_output={} -o json_include_output_property={} -o json_include_tags_property={} -o priority={} -v'.format(
+        cmd = '{} {} {} -c {} {} -o json_output={} -o json_include_output_property={} -o json_include_tags_property={} -o priority={} -v {}'.format(
             self.falco_binary_path, self.rules_args, self.disabled_args, self.conf_file, trace_arg, self.json_output,
-            self.json_include_output_property, self.json_include_tags_property, self.priority)
+            self.json_include_output_property, self.json_include_tags_property, self.priority, self.addl_cmdline_opts)
 
         for tag in self.disable_tags:
             cmd += ' -T {}'.format(tag)
@@ -650,13 +653,13 @@ class FalcoTest(Test):
                 self.fail("Stdout was not exactly {}".format(self.stderr_is))
 
         for pattern in self.stderr_contains:
-            match = re.search(pattern, res.stderr.decode("utf-8"))
+            match = re.search(pattern, res.stderr.decode("utf-8"), re.DOTALL)
             if match is None:
                 self.fail(
                     "Stderr of falco process did not contain content matching {}".format(pattern))
 
         for pattern in self.stdout_contains:
-            match = re.search(pattern, res.stdout.decode("utf-8"))
+            match = re.search(pattern, res.stdout.decode("utf-8"), re.DOTALL)
             if match is None:
                 self.fail("Stdout of falco process '{}' did not contain content matching {}".format(
                     res.stdout.decode("utf-8"), pattern))
@@ -684,7 +687,7 @@ class FalcoTest(Test):
         self.check_rules_warnings(res)
         if len(self.rules_events) > 0:
             self.check_rules_events(res)
-        if len(self.validate_rules_file) == 0:
+        if len(self.validate_rules_file) == 0 and self.check_detection_counts:
             self.check_detections(res)
         if len(self.detect_counts) > 0:
             self.check_detections_by_rule(res)

test/falco_tests.yaml

@@ -32,20 +32,10 @@ trace_files: !mux
       - leading_not
       - not_equals_at_end
       - not_at_end
-      - not_before_trailing_evttype
-      - not_equals_before_trailing_evttype
       - not_equals_and_not
-      - not_equals_before_in
-      - not_before_in
-      - not_in_before_in
-      - leading_in_not_equals_before_evttype
       - leading_in_not_equals_at_evttype
       - not_with_evttypes
       - not_with_evttypes_addl
-      - not_equals_before_evttype
-      - not_equals_before_in_evttype
-      - not_before_evttype
-      - not_before_evttype_using_in
     rules_events:
       - no_warnings: [execve]
       - no_evttype: [all]
@@ -364,7 +354,7 @@ trace_files: !mux
     exit_status: 1
     stdout_is: |+
       1 errors:
-      Rule must have exceptions or condition property
+      Appended rule must have exceptions or condition property
       ---
       - rule: no condition rule
         append: true
@@ -644,6 +634,20 @@ trace_files: !mux
     rules_file:
       - rules/single_rule_enabled_flag.yaml
     trace_file: trace_files/cat_write.scap
+  
+  disabled_rule_using_false_enabled_flag_only:
+    detect: False
+    rules_file:
+      - rules/disabled_rule_using_enabled_flag_only.yaml
+    trace_file: trace_files/cat_write.scap
+  
+  enabled_rule_using_false_enabled_flag_only:
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/enabled_rule_using_enabled_flag_only.yaml
+    trace_file: trace_files/cat_write.scap
+    stdout_contains: "Warning An open was seen"
 
   disabled_and_enabled_rules_1:
     exit_status: 1
@@ -1142,6 +1146,8 @@ trace_files: !mux
     detect_level: INFO
     rules_file:
       - rules/syscalls.yaml
+    rules_warning:
+      - detect_madvise
     detect_counts:
       - detect_madvise: 2
       - detect_open: 2
@@ -1160,7 +1166,8 @@ trace_files: !mux
 
   skip_unknown_noevt:
     detect: False
-    stdout_contains: Skipping rule "Contains Unknown Event And Skipping". contains unknown filter proc.nobody
+    rules_warning:
+      - Contains Unknown Event And Skipping
     rules_file:
       - rules/skip_unknown_evt.yaml
     trace_file: trace_files/cat_write.scap
@@ -1175,7 +1182,7 @@ trace_files: !mux
     exit_status: 1
     stderr_contains: |+
       Could not load rules file.*skip_unknown_error.yaml: 1 errors:
-      rule "Contains Unknown Event And Not Skipping". contains unknown filter proc.nobody
+      Rule Contains Unknown Event And Not Skipping: error filter_check called with nonexistent field proc.nobody
       ---
       - rule: Contains Unknown Event And Not Skipping
         desc: Contains an unknown event
@@ -1192,7 +1199,7 @@ trace_files: !mux
     exit_status: 1
     stderr_contains: |+
       Could not load rules file .*skip_unknown_unspec.yaml: 1 errors:
-      rule "Contains Unknown Event And Unspecified". contains unknown filter proc.nobody
+      Rule Contains Unknown Event And Unspecified: error filter_check called with nonexistent field proc.nobody
       ---
       - rule: Contains Unknown Event And Unspecified
         desc: Contains an unknown event

test/falco_tests_exceptions.yaml

@@ -177,10 +177,18 @@ trace_files: !mux
     trace_file: trace_files/cat_write.scap
 
   rule_exception_append_item_not_in_rule:
-    exit_status: 0
-    stderr_contains: |+
-      1 warnings:
-      Rule My Rule with append=true: no set of fields matching name ex2
+    exit_status: 1
+    stdout_is: |+
+      1 errors:
+      Rule exception new item ex2: must have fields property with a list of fields
+      ---
+      - rule: My Rule
+        exceptions:
+          - name: ex2
+            values:
+              - [apache, /tmp]
+        append: true
+      ---
     validate_rules_file:
       - rules/exceptions/append_item_not_in_rule.yaml
     trace_file: trace_files/cat_write.scap
@@ -311,4 +319,35 @@ trace_files: !mux
       - rules/exceptions/rule_exception_single_field_append.yaml
     trace_file: trace_files/cat_write.scap
 
+  rule_exception_new_single_field_append:
+    detect: False
+    detect_level: WARNING
+    rules_file:
+      - rules/exceptions/rule_exception_new_single_field_append.yaml
+    trace_file: trace_files/cat_write.scap
+  
+  rule_exception_new_second_field_append:
+    detect: False
+    detect_level: WARNING
+    rules_file:
+      - rules/exceptions/rule_exception_new_second_field_append.yaml
+    trace_file: trace_files/cat_write.scap
+
+  rule_exception_new_append_no_field:
+    exit_status: 1
+    stdout_is: |+
+      1 errors:
+      Rule exception new item proc_cmdline: must have fields property with a list of fields
+      ---
+      - rule: Open From Cat
+        exceptions:
+          - name: proc_cmdline
+            comps: in
+            values:
+              - "cat /dev/null"
+        append: true
+      ---
+    validate_rules_file:
+      - rules/exceptions/rule_exception_new_no_field_append.yaml
+    trace_file: trace_files/cat_write.scap
 

test/falco_tests_plugins.yaml

@@ -0,0 +1,106 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+# This file is part of Falco.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+trace_files: !mux
+
+  list_plugins:
+    check_detection_counts: False
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances.yaml
+    addl_cmdline_opts: --list-plugins
+    stdout_contains: "2 Plugins Loaded.*Name: cloudtrail.*Type: source plugin.*Name: json.*Type: extractor plugin"
+
+  list_plugin_fields:
+    check_detection_counts: False
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances.yaml
+    addl_cmdline_opts: --list
+    stdout_contains: "ct.id"
+
+  detect_create_instance:
+    detect: True
+    detect_level: INFO
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    detect_counts:
+      - 'Cloudtrail Create Instance': 1
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances.yaml
+
+  detect_create_instance_bigevent:
+    detect: True
+    detect_level: INFO
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    detect_counts:
+      - 'Cloudtrail Create Instance': 1
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances_bigevent.yaml
+
+  multiple_source_plugins:
+    exit_status: 1
+    stderr_contains: "Runtime error: Can not load multiple source plugins. cloudtrail already loaded. Exiting."
+    conf_file: BUILD_DIR/test/confs/plugins/multiple_source_plugins.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+
+  incompatible_extract_sources:
+    exit_status: 1
+    stderr_contains: "Runtime error: Extractor plugin not compatible with event source aws_cloudtrail. Exiting."
+    conf_file: BUILD_DIR/test/confs/plugins/incompatible_extract_sources.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+
+  overlap_extract_sources:
+    exit_status: 1
+    stderr_contains: "Runtime error: Extractor plugins have overlapping compatible event source test_source. Exiting."
+    conf_file: BUILD_DIR/test/confs/plugins/overlap_extract_sources.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+
+  incompat_plugin_api:
+    exit_status: 1
+    stderr_contains: "Unsupported plugin required api version 10000000.0.0"
+    conf_file: BUILD_DIR/test/confs/plugins/incompatible_plugin_api.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+
+  incompat_plugin_rules_version:
+    exit_status: 1
+    stderr_contains: "Runtime error: Plugin cloudtrail version .* not compatible with required plugin version 100000.0.0. Exiting."
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_incompat_plugin_version.yaml
+
+  wrong_plugin_path:
+    exit_status: 1
+    stderr_contains: "error loading plugin.*No such file or directory. Exiting"
+    conf_file: BUILD_DIR/test/confs/plugins/wrong_plugin_path.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_incompat_plugin_version.yaml
+
+  no_plugins_unknown_source:
+    detect: False
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    trace_file: trace_files/empty.scap
+    rules_warning:
+      - Cloudtrail Create Instance
+    stderr_contains: "Rule Cloudtrail Create Instance: warning .unknown-source.: unknown source aws_cloudtrail, skipping"
+
+

test/falco_traces.yaml.in

@@ -111,12 +111,17 @@ traces: !mux
     detect_counts:
       - "Read sensitive file untrusted": 1
 
+  # This should *not* generate any falco alerts as of the changes in
+  # https://github.com/falcosecurity/libs/pull/94--the execve event in
+  # this trace file is PPME_SYSCALL_EXECVE_18, which was deprecated by
+  # PPME_SYSCALL_EXECVE_19 in 2018.
+  #
+  # This activity in this trace file overlaps with the activity in
+  # falco-event-generator.scap so the rule is still being tested.
   run-shell-untrusted:
     trace_file: traces-positive/run-shell-untrusted.scap
-    detect: True
+    detect: False
     detect_level: DEBUG
-    detect_counts:
-      - "Run shell untrusted": 1
 
   system-binaries-network-activity:
     trace_file: traces-positive/system-binaries-network-activity.scap

test/plot-live.r

@@ -1,69 +0,0 @@
-#
-# Copyright (C) 2019 The Falco Authors.
-#
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require(jsonlite)
-library(ggplot2)
-library(GetoptLong)
-
-initial.options <- commandArgs(trailingOnly = FALSE)
-file.arg.name <- "--file="
-script.name <- sub(file.arg.name, "", initial.options[grep(file.arg.name, initial.options)])
-script.basename <- dirname(script.name)
-
-if (substr(script.basename, 1, 1) != '/') {
-    script.basename = paste(getwd(), script.basename, sep='/')
-}
-
-results = paste(script.basename, "results.json", sep='/')
-output = "./output.png"
-metric = "cpu"
-
-GetoptLong(
-    "results=s", "Path to results file",
-    "benchmark=s", "Benchmark from results file to graph",
-    "variant=s@", "Variant(s) to include in graph. Can be specified multiple times",
-    "output=s", "Output graph file",
-    "metric=s", "Metric to graph. Can be one of (cpu|drops)"
-)
-
-if (metric == "cpu") {
-    data_metric="cpu_usage"
-    yaxis_label="CPU Usage (%)"
-    title="Falco/Sysdig/Multimatch CPU Usage: %s"
-} else if (metric == "drops") {
-    data_metric="drop_pct"
-    yaxis_label="Event Drops (%)"
-    title="Falco/Sysdig/Multimatch Event Drops: %s"
-}
-
-res <- fromJSON(results, flatten=TRUE)
-
-res2 = res[res$benchmark == benchmark & res$variant %in% variant,]
-
-plot <- ggplot(data=res2, aes(x=sample, y=get(data_metric), group=variant, colour=variant)) +
-    geom_line() +
-    ylab(yaxis_label) +
-    xlab("Time") +
-    ggtitle(sprintf(title, benchmark))
-    theme(legend.position=c(.2, .88));
-
-print(paste("Writing graph to", output, sep=" "))
-ggsave(file=output)
-
-
-
-

test/plot-traces.r

@@ -1,52 +0,0 @@
-#
-# Copyright (C) 2019 The Falco Authors.
-#
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require(jsonlite)
-library(ggplot2)
-library(reshape)
-
-res <- fromJSON("/home/mstemm/results.txt", flatten=TRUE)
-
-plot <- ggplot(data=res, aes(x=config, y=elapsed.real)) +
-    geom_bar(stat = "summary", fun.y = "mean") +
-    coord_flip() +
-    facet_grid(shortfile ~ .) +
-    ylab("Wall Clock Time (sec)") +
-    xlab("Trace File/Program")
-
-
-ggsave(file="/mnt/sf_mstemm/res-real.png")
-
-plot <- ggplot(data=res, aes(x=config, y=elapsed.user)) +
-    geom_bar(stat = "summary", fun.y = "mean") +
-    coord_flip() +
-    facet_grid(shortfile ~ .) +
-    ylab("User Time (sec)") +
-    xlab("Trace File/Program")
-
-
-ggsave(file="/mnt/sf_mstemm/res-user.png")
-
-res2 <- melt(res, id.vars = c("config", "shortfile"), measure.vars = c("elapsed.sys", "elapsed.user"))
-plot <- ggplot(data=res2, aes(x=config, y=value, fill=variable, order=variable)) +
-     geom_bar(stat = "summary", fun.y = "mean") +
-     coord_flip() +
-     facet_grid(shortfile ~ .) +
-     ylab("User/System Time (sec)") +
-     xlab("Trace File/Program")
-
-ggsave(file="/mnt/sf_mstemm/res-sys-user.png")

test/plugins/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_library(test_extract_p1 SHARED test_extract.cpp)
+add_library(test_extract_p2 SHARED test_extract.cpp)
+add_library(test_source SHARED test_source.cpp)
+add_library(test_incompat_api SHARED incompat_api.cpp)
+
+target_include_directories(
+	test_extract_p1 PUBLIC "${LIBSCAP_INCLUDE_DIRS}")
+
+target_include_directories(
+	test_extract_p2 PUBLIC "${LIBSCAP_INCLUDE_DIRS}")
+
+target_include_directories(
+	test_source PUBLIC "${LIBSCAP_INCLUDE_DIRS}")

test/plugins/incompat_api.cpp

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019 The Falco Authors.
+Copyright (C) 2021 The Falco Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#pragma once
+#include <string.h>
+#include <stdlib.h>
 
-#define FALCO_ENGINE_LUA_DIR "${FALCO_ABSOLUTE_SHARE_DIR}/lua/"
-#define FALCO_ENGINE_SOURCE_LUA_DIR "${PROJECT_SOURCE_DIR}/userspace/engine/lua/"
+// Don't need any function other than plugin_get_required_api_version,
+// plugin load will fail after that.
+static const char *pl_required_api_version = "10000000.0.0";
+
+extern "C"
+const char* plugin_get_required_api_version()
+{
+	return pl_required_api_version;
+}

test/plugins/test_extract.cpp

@@ -0,0 +1,116 @@
+/*
+Copyright (C) 2021 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <plugin_info.h>
+
+static const char *pl_required_api_version = "0.1.0";
+static uint32_t    pl_type                 = TYPE_EXTRACTOR_PLUGIN;
+static const char *pl_name_base            = "test_extract";
+static char pl_name[1024];
+static const char *pl_desc                 = "Test Plugin For Regression Tests";
+static const char *pl_contact              = "github.com/falcosecurity/falco";
+static const char *pl_version              = "0.1.0";
+static const char *pl_extract_sources      = "[\"test_source\"]";
+static const char *pl_fields               = "[]";
+
+// This struct represents the state of a plugin. Just has a placeholder string value.
+typedef struct plugin_state
+{
+} plugin_state;
+
+extern "C"
+const char* plugin_get_required_api_version()
+{
+	return pl_required_api_version;
+}
+
+extern "C"
+uint32_t plugin_get_type()
+{
+	return pl_type;
+}
+
+extern "C"
+const char* plugin_get_name()
+{
+	// Add a random-ish suffix to the end, as some tests load
+	// multiple copies of this plugin
+	snprintf(pl_name, sizeof(pl_name)-1, "%s%ld\n", pl_name_base, random());
+	return pl_name;
+}
+
+extern "C"
+const char* plugin_get_description()
+{
+	return pl_desc;
+}
+
+extern "C"
+const char* plugin_get_contact()
+{
+	return pl_contact;
+}
+
+extern "C"
+const char* plugin_get_version()
+{
+	return pl_version;
+}
+
+extern "C"
+const char* plugin_get_extract_event_sources()
+{
+	return pl_extract_sources;
+}
+
+extern "C"
+const char* plugin_get_fields()
+{
+	return pl_fields;
+}
+
+extern "C"
+const char* plugin_get_last_error(ss_plugin_t* s)
+{
+	return NULL;
+}
+
+extern "C"
+ss_plugin_t* plugin_init(const char* config, int32_t* rc)
+{
+	// Note: Using new/delete is okay, as long as the plugin
+	// framework is not deleting the memory.
+	plugin_state *ret = new plugin_state();
+	*rc = SS_PLUGIN_SUCCESS;
+	return ret;
+}
+
+extern "C"
+void plugin_destroy(ss_plugin_t* s)
+{
+	plugin_state *ps = (plugin_state *) s;
+
+	delete(ps);
+}
+
+extern "C"
+int32_t plugin_extract_fields(ss_plugin_t *s, const ss_plugin_event *evt, uint32_t num_fields, ss_plugin_extract_field *fields)
+{
+	return SS_PLUGIN_SUCCESS;
+}

test/plugins/test_source.cpp

@@ -0,0 +1,161 @@
+/*
+Copyright (C) 2021 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <plugin_info.h>
+
+static const char *pl_required_api_version = "0.1.0";
+static uint32_t    pl_type                 = TYPE_SOURCE_PLUGIN;
+static uint32_t    pl_id                   = 999;
+static const char *pl_name                 = "test_source";
+static const char *pl_desc                 = "Test Plugin For Regression Tests";
+static const char *pl_contact              = "github.com/falcosecurity/falco";
+static const char *pl_version              = "0.1.0";
+static const char *pl_event_source         = "test_source";
+static const char *pl_fields               = "[]";
+
+// This struct represents the state of a plugin. Just has a placeholder string value.
+typedef struct plugin_state
+{
+} plugin_state;
+
+typedef struct instance_state
+{
+} instance_state;
+
+extern "C"
+const char* plugin_get_required_api_version()
+{
+	return pl_required_api_version;
+}
+
+extern "C"
+uint32_t plugin_get_type()
+{
+	return pl_type;
+}
+
+extern "C"
+uint32_t plugin_get_id()
+{
+	return pl_id;
+}
+
+extern "C"
+const char* plugin_get_name()
+{
+	return pl_name;
+}
+
+extern "C"
+const char* plugin_get_description()
+{
+	return pl_desc;
+}
+
+extern "C"
+const char* plugin_get_contact()
+{
+	return pl_contact;
+}
+
+extern "C"
+const char* plugin_get_version()
+{
+	return pl_version;
+}
+
+extern "C"
+const char* plugin_get_event_source()
+{
+	return pl_event_source;
+}
+
+extern "C"
+const char* plugin_get_fields()
+{
+	return pl_fields;
+}
+
+extern "C"
+const char* plugin_get_last_error(ss_plugin_t* s)
+{
+	return NULL;
+}
+
+extern "C"
+ss_plugin_t* plugin_init(const char* config, int32_t* rc)
+{
+	// Note: Using new/delete is okay, as long as the plugin
+	// framework is not deleting the memory.
+	plugin_state *ret = new plugin_state();
+
+	*rc = SS_PLUGIN_SUCCESS;
+
+	return ret;
+}
+
+extern "C"
+void plugin_destroy(ss_plugin_t* s)
+{
+	plugin_state *ps = (plugin_state *) s;
+
+	delete(ps);
+}
+
+extern "C"
+ss_instance_t* plugin_open(ss_plugin_t* s, const char* params, int32_t* rc)
+{
+	// Note: Using new/delete is okay, as long as the plugin
+	// framework is not deleting the memory.
+	instance_state *ret = new instance_state();
+	*rc = SS_PLUGIN_SUCCESS;
+
+	return ret;
+}
+
+extern "C"
+void plugin_close(ss_plugin_t* s, ss_instance_t* i)
+{
+	instance_state *istate = (instance_state *) i;
+
+	delete(istate);
+}
+
+extern "C"
+int32_t plugin_next_batch(ss_plugin_t* s, ss_instance_t* i, uint32_t *nevts, ss_plugin_event **evts)
+{
+	return SS_PLUGIN_EOF;
+}
+
+// This plugin does not implement plugin_next_batch, due to the lower
+// overhead of calling C functions from the plugin framework compared
+// to calling Go functions.
+
+extern "C"
+const char *plugin_event_to_string(ss_plugin_t *s, const uint8_t *data, uint32_t datalen)
+{
+	return "";
+}
+
+extern "C"
+int32_t plugin_extract_fields(ss_plugin_t *s, const ss_plugin_event *evt, uint32_t num_fields, ss_plugin_extract_field *fields)
+{
+	return SS_PLUGIN_SUCCESS;
+}

test/rules/disabled_rule_using_enabled_flag_only.yaml

@@ -0,0 +1,24 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and proc.name=cat
+  output: "An open was seen"
+  priority: WARNING
+
+- rule: open_from_cat
+  enabled: false

test/rules/enabled_rule_using_enabled_flag_only.yaml

@@ -0,0 +1,25 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+- rule: open_from_cat
+  desc: A process named cat does an open
+  condition: evt.type=open and proc.name=cat
+  output: "An open was seen"
+  priority: WARNING
+  enabled: false
+
+- rule: open_from_cat
+  enabled: true

test/rules/exceptions/rule_exception_new_no_field_append.yaml

@@ -1,6 +1,5 @@
-#!/usr/bin/env bash
 #
-# Copyright (C) 2019 The Falco Authors.
+# Copyright (C) 2021 The Falco Authors.
 #
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,10 +15,17 @@
 # limitations under the License.
 #
 
-SUBJ_PID=$1
-BENCHMARK=$2
-VARIANT=$3
-RESULTS_FILE=$4
-CPU_INTERVAL=$5
+- rule: Open From Cat
+  desc: A process named cat does an open
+  condition: evt.type=open and proc.name=cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING
+
+- rule: Open From Cat
+  exceptions:
+    - name: proc_cmdline
+      comps: in
+      values:
+        - "cat /dev/null"
+  append: true
 
-top -d $CPU_INTERVAL -b -p $SUBJ_PID | grep -E '(falco|sysdig|dragent|test_mm)' --line-buffered | awk -v benchmark=$BENCHMARK -v variant=$VARIANT '{printf("{\"time\": \"%s\", \"sample\": %d, \"benchmark\": \"%s\", \"variant\": \"%s\", \"cpu_usage\": %s},\n", strftime("%Y-%m-%d %H:%M:%S", systime(), 1), NR, benchmark, variant, $9); fflush();}' >> $RESULTS_FILE

test/rules/exceptions/rule_exception_new_second_field_append.yaml

@@ -0,0 +1,39 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+- rule: Open From Cat
+  desc: A process named cat does an open
+  condition: evt.type=open and proc.name=cat
+  output: "An open was seen (command=%proc.cmdline)"
+  exceptions:
+    - name: proc_cmdline
+      fields: proc.cmdline
+      comps: in
+      values:
+        - cat /dev/zero
+  priority: WARNING
+
+- rule: Open From Cat
+  exceptions:
+    - name: proc_cmdline_2
+      fields: proc.cmdline
+      comps: in
+      values:
+        - "cat /dev/null"
+  append: true
+
+

test/rules/exceptions/rule_exception_new_single_field_append.yaml

@@ -0,0 +1,33 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+- rule: Open From Cat
+  desc: A process named cat does an open
+  condition: evt.type=open and proc.name=cat
+  output: "An open was seen (command=%proc.cmdline)"
+  priority: WARNING
+
+- rule: Open From Cat
+  exceptions:
+    - name: proc_cmdline
+      fields: proc.cmdline
+      comps: in
+      values:
+        - "cat /dev/null"
+  append: true
+
+

test/rules/invalid_append_rule_without_condition.yaml

@@ -1,2 +1,24 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+- rule: no condition rule
+  desc: simpe rule 
+  condition: evt.type=open
+  output: simple output
+  priority: WARNING
+
 - rule: no condition rule
   append: true

test/rules/k8s_audit/single_rule_with_json_pointer.yaml

@@ -0,0 +1,23 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+- rule: json_pointer_example
+  desc: A rule example using JSON Pointer (RFC 6901)
+  condition: jevt.value[/annotations/authorization.k8s.io~1decision] exists and not (jevt.value[/a~1~0b] exists)
+  output: "JSON Pointer Test Alert"
+  priority: WARNING
+  source: k8s_audit

test/rules/plugins/cloudtrail_create_instances.yaml

@@ -0,0 +1,6 @@
+- rule: Cloudtrail Create Instance
+  desc: Detect Creating an EC2 Instance
+  condition: evt.num > 0 and ct.name="StartInstances"
+  output: EC2 Instance Created (evtnum=%evt.num info=%evt.plugininfo id=%ct.id user name=%json.value[/userIdentity/userName])
+  priority: INFO
+  source: aws_cloudtrail

test/rules/plugins/cloudtrail_incompat_plugin_version.yaml

@@ -0,0 +1,10 @@
+- required_plugin_versions:
+    - name: cloudtrail
+      version: 100000.0.0
+
+- rule: Cloudtrail Create Instance
+  desc: Detect Creating an EC2 Instance
+  condition: evt.num > 0 and ct.name="StartInstances"
+  output: EC2 Instance Created (evtnum=%evt.num info=%evt.plugininfo id=%ct.id user name=%json.value[/userIdentity/userName])
+  priority: INFO
+  source: aws_cloudtrail

test/rules/rule_append.yaml

@@ -16,10 +16,10 @@
 #
 - rule: my_rule
   desc: A process named cat does an open
-  condition: evt.type=open and fd.name=not-a-real-file
+  condition: (evt.type=open and fd.name=not-a-real-file)
   output: "An open of /dev/null was seen (command=%proc.cmdline)"
   priority: WARNING
 
 - rule: my_rule
   append: true
-  condition: or fd.name=/dev/null
+  condition: or (evt.type=open and fd.name=/dev/null)

test/rules/tagged_rules.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 - macro: open_read
-  condition: (evt.type=open or evt.type=openat) and evt.is_open_read=true and fd.typechar='f'
+  condition: evt.type in (open,openat,openat2) and evt.is_open_read=true and fd.typechar='f'
 
 - rule: open_1
   desc: open one

test/run_performance_tests.sh

@@ -1,444 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (C) 2019 The Falco Authors.
-#
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#set -x
-
-trap "cleanup; exit" SIGHUP SIGINT SIGTERM
-
-TRACE_FILES_BASE_URL=${TRACE_FILES_BASE_URL:-"https://download.falco.org/fixtures/trace-files/"}
-
-function download_trace_files() {
-
-    (mkdir -p $TRACEDIR && rm -rf $TRACEDIR/traces-perf && curl -fo $TRACEDIR/traces-perf.zip "${TRACE_FILES_BASE_URL}traces-perf.zip" && unzip -d $TRACEDIR $TRACEDIR/traces-perf.zip && rm -f $TRACEDIR/traces-perf.zip) || exit 1
-
-}
-
-function time_cmd() {
-    cmd="$1"
-    file="$2"
-
-    benchmark=`basename $file .scap`
-
-    echo -n "$benchmark: "
-    for i in `seq 1 5`; do
-	echo -n "$i "
-	time=`date --iso-8601=sec`
-	/usr/bin/time -a -o $RESULTS_FILE --format "{\"time\": \"$time\", \"benchmark\": \"$benchmark\", \"file\": \"$file\", \"variant\": \"$VARIANT\", \"elapsed\": {\"real\": %e, \"user\": %U, \"sys\": %S}}," $cmd >> $OUTPUT_FILE 2>&1
-    done
-    echo ""
-}
-
-function run_falco_on() {
-    file="$1"
-
-    if [ -z $RULES_FILE ]; then
-	RULES_FILE=$SOURCE/rules/falco_rules.yaml
-    fi
-
-    cmd="$ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -r $SOURCE/rules/falco_rules.yaml --option=stdout_output.enabled=false -e $file -A"
-
-    time_cmd "$cmd" "$file"
-}
-
-function run_sysdig_on() {
-    file="$1"
-
-    cmd="$ROOT/userspace/sysdig/sysdig -N -z -r $file evt.type=none"
-
-    time_cmd "$cmd" "$file"
-}
-
-function write_agent_config() {
-    cat > $ROOT/userspace/dragent/dragent.yaml <<EOF
-customerid: XXX
-app_checks_enabled: false
-log:
-  file_priority: info
-  console_priority: info
-  event_priority: info
-jmx:
-  enabled: false
-statsd:
-  enabled: false
-collector: collector-staging.sysdigcloud.com
-EOF
-
-    if [ $FALCO_AGENT == 1 ]; then
-	cat >> $ROOT/userspace/dragent/dragent.yaml <<EOF
-falco_engine:
-  enabled: true
-  rules_filename: /etc/falco_rules.yaml
-  sampling_multiplier: 0
-EOF
-    else
-	cat >> $ROOT/userspace/dragent/dragent.yaml <<EOF
-falco_engine:
-  enabled: false
-EOF
-    fi
-
-    if [ $AGENT_AUTODROP == 1 ]; then
-	cat >> $ROOT/userspace/dragent/dragent.yaml <<EOF
-autodrop:
-  enabled: true
-EOF
-    else
-	cat >> $ROOT/userspace/dragent/dragent.yaml <<EOF
-autodrop:
-  enabled: false
-EOF
-    fi
-
-    cat $ROOT/userspace/dragent/dragent.yaml
-}
-
-function run_agent_on() {
-
-    file="$1"
-
-    write_agent_config
-
-    cmd="$ROOT/userspace/dragent/dragent -r $file"
-
-    time_cmd "$cmd" "$file"
-}
-
-function run_trace() {
-
-    if [ ! -e $TRACEDIR ]; then
-	download_trace_files
-    fi
-
-    trace_file="$1"
-
-    if [ $trace_file == "all" ]; then
-	files=($TRACEDIR/traces-perf/*.scap)
-    else
-	files=($TRACEDIR/traces-perf/$trace_file.scap)
-    fi
-
-    for file in "${files[@]}"; do
-	if [[ $ROOT == *"falco"* ]]; then
-	    run_falco_on "$file"
-	elif [[ $ROOT == *"sysdig"* ]]; then
-	    run_sysdig_on "$file"
-	else
-	    run_agent_on "$file"
-	fi
-    done
-}
-
-function start_monitor_cpu_usage() {
-    echo "   monitoring cpu usage for falcosecurity/falco program"
-
-    setsid bash `dirname $0`/cpu_monitor.sh $SUBJ_PID $live_test $VARIANT $RESULTS_FILE $CPU_INTERVAL &
-    CPU_PID=$!
-    sleep 5
-}
-
-function start_subject_prog() {
-
-    # Do a blocking sudo command now just to ensure we have a password
-    sudo bash -c ""
-
-    if [[ $ROOT == *"multimatch"* ]]; then
-	echo "   starting test_mm..."
-	if [ -z $RULES_FILE ]; then
-	    RULES_FILE=$SOURCE/../output/rules.yaml
-	fi
-	sudo FALCO_STATS_EXTRA_variant=$VARIANT FALCO_STATS_EXTRA_benchmark=$live_test $ROOT/test_mm -S $SOURCE/search_order.yaml -s $STATS_FILE -r $RULES_FILE > ./prog-output.txt 2>&1 &
-    elif [[ $ROOT == *"falco"* ]]; then
-	echo "   starting falco..."
-	if [ -z $RULES_FILE ]; then
-	    RULES_FILE=$SOURCE/rules/falco_rules.yaml
-	fi
-	sudo FALCO_STATS_EXTRA_variant=$VARIANT FALCO_STATS_EXTRA_benchmark=$live_test $ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -s $STATS_FILE -r $RULES_FILE --option=stdout_output.enabled=false > ./prog-output.txt -A 2>&1 &
-    elif [[ $ROOT == *"sysdig"* ]]; then
-	echo "   starting sysdig..."
-	sudo $ROOT/userspace/sysdig/sysdig -N -z evt.type=none &
-    else
-	echo "   starting agent..."
-	write_agent_config
-	pushd $ROOT/userspace/dragent
-	sudo ./dragent > ./prog-output.txt 2>&1 &
-	popd
-    fi
-
-    SUDO_PID=$!
-    sleep 5
-    if [[ $ROOT == *"agent"* ]]; then
-	# The agent spawns several processes all below a main monitor
-	# process. We want the child with the lowest pid.
-	MON_PID=`ps -h -o pid --ppid $SUDO_PID`
-	SUBJ_PID=`ps -h -o pid --ppid $MON_PID | head -1`
-    else
-	SUBJ_PID=`ps -h -o pid --ppid $SUDO_PID`
-    fi
-
-    if [ -z $SUBJ_PID ]; then
-	echo "Could not find pid of subject program--did it start successfully? Not continuing."
-	exit 1
-    fi
-}
-
-function run_htop() {
-    screen -S htop-screen -d -m /usr/bin/htop -d2
-    sleep 90
-    screen -X -S htop-screen quit
-}
-
-function run_juttle_examples() {
-    pushd $SCRIPTDIR/../../juttle-engine/examples
-    docker-compose -f dc-juttle-engine.yml -f aws-cloudwatch/dc-aws-cloudwatch.yml -f elastic-newstracker/dc-elastic.yml -f github-tutorial/dc-elastic.yml -f nginx_logs/dc-nginx-logs.yml -f postgres-diskstats/dc-postgres.yml -f cadvisor-influx/dc-cadvisor-influx.yml up -d
-    sleep 120
-    docker-compose -f dc-juttle-engine.yml -f aws-cloudwatch/dc-aws-cloudwatch.yml -f elastic-newstracker/dc-elastic.yml -f github-tutorial/dc-elastic.yml -f nginx_logs/dc-nginx-logs.yml -f postgres-diskstats/dc-postgres.yml -f cadvisor-influx/dc-cadvisor-influx.yml stop
-    docker-compose -f dc-juttle-engine.yml -f aws-cloudwatch/dc-aws-cloudwatch.yml -f elastic-newstracker/dc-elastic.yml -f github-tutorial/dc-elastic.yml -f nginx_logs/dc-nginx-logs.yml -f postgres-diskstats/dc-postgres.yml -f cadvisor-influx/dc-cadvisor-influx.yml rm -fv
-    popd
-}
-
-function run_kubernetes_demo() {
-    pushd $SCRIPTDIR/../../infrastructure/test-infrastructures/kubernetes-demo
-    sudo bash run-local.sh
-    sudo bash init.sh
-    sleep 600
-    docker stop $(docker ps -qa)
-    docker rm -fv $(docker ps -qa)
-    popd
-}
-
-function run_live_test() {
-
-    live_test="$1"
-
-    echo "Running live test $live_test"
-
-    case "$live_test" in
-	htop ) CPU_INTERVAL=2;;
-	* ) CPU_INTERVAL=10;;
-    esac
-
-    start_subject_prog
-    start_monitor_cpu_usage
-
-    echo "   starting live program and waiting for it to finish"
-    case "$live_test" in
-	htop ) run_htop ;;
-	juttle-examples ) run_juttle_examples ;;
-	kube-demo ) run_kubernetes_demo ;;
-	* ) usage; cleanup; exit 1 ;;
-    esac
-
-    cleanup
-
-}
-
-function cleanup() {
-
-    if [ -n "$SUBJ_PID" ] ; then
-	echo "   stopping falco/sysdig program $SUBJ_PID"
-	sudo kill $SUBJ_PID
-    fi
-
-    if [ -n "$CPU_PID" ] ; then
-	echo "   stopping cpu monitor program $CPU_PID"
-	kill -- -$CPU_PID
-    fi
-}
-
-run_live_tests() {
-    test="$1"
-
-    if [ $test == "all" ]; then
-	tests="htop juttle-examples kube-demo"
-    else
-	tests=$test
-    fi
-
-    for test in $tests; do
-	run_live_test $test
-    done
-}
-
-function run_phoronix_test() {
-
-    live_test="$1"
-
-    case "$live_test" in
-	pts/aio-stress | pts/fs-mark | pts/iozone | pts/network-loopback | pts/nginx | pts/pybench | pts/redis | pts/sqlite | pts/unpack-linux ) CPU_INTERVAL=2;;
-	* ) CPU_INTERVAL=10;;
-    esac
-
-    echo "Running phoronix test $live_test"
-
-    start_subject_prog
-    start_monitor_cpu_usage
-
-    echo "   starting phoronix test and waiting for it to finish"
-
-    TEST_RESULTS_NAME=$VARIANT FORCE_TIMES_TO_RUN=1 phoronix-test-suite default-run $live_test
-
-    cleanup
-
-}
-
-# To install and configure phoronix:
-#  (redhat instructions, adapt as necessary for ubuntu or other distros)
-#   - install phoronix: yum install phoronix-test-suite.noarch
-#   - install dependencies not handled by phoronix: yum install libaio-devel pcre-devel popt-devel glibc-static zlib-devel nc bc
-#   - fix trivial bugs in tests:
-#      - edit ~/.phoronix-test-suite/installed-tests/pts/network-loopback-1.0.1/network-loopback line "nc -d -l 9999 > /dev/null &" to "nc -d -l 9999 > /dev/null &"
-#      - edit ~/.phoronix-test-suite/test-profiles/pts/nginx-1.1.0/test-definition.xml line "<Arguments>-n 500000 -c 100 http://localhost:8088/test.html</Arguments>" to "<Arguments>-n 500000 -c 100 http://127.0.0.1:8088/test.html</Arguments>"
-#   - phoronix batch-install <test list below>
-
-function run_phoronix_tests() {
-
-    test="$1"
-
-    if [ $test == "all" ]; then
-	tests="pts/aio-stress pts/apache pts/blogbench pts/compilebench pts/dbench pts/fio pts/fs-mark pts/iozone pts/network-loopback pts/nginx pts/pgbench pts/phpbench pts/postmark pts/pybench pts/redis pts/sqlite pts/unpack-linux"
-    else
-	tests=$test
-    fi
-
-    for test in $tests; do
-	run_phoronix_test $test
-    done
-}
-
-run_tests() {
-
-    IFS=':' read -ra PARTS <<< "$TEST"
-
-    case "${PARTS[0]}" in
-	trace ) run_trace "${PARTS[1]}" ;;
-	live ) run_live_tests "${PARTS[1]}" ;;
-	phoronix ) run_phoronix_tests "${PARTS[1]}" ;;
-	* ) usage; exit 1 ;;
-    esac
-}
-
-usage() {
-    echo "Usage: $0 [options]"
-    echo ""
-    echo "Options:"
-    echo "   -h/--help: show this help"
-    echo "   -v/--variant: a variant name to attach to this set of test results"
-    echo "   -r/--root: root directory containing falco/sysdig binaries (i.e. where you ran 'cmake')"
-    echo "   -s/--source: root directory containing falco/sysdig source code"
-    echo "   -R/--results: append test results to this file"
-    echo "   -S/--stats: append capture statistics to this file (only works for falco/test_mm)"
-    echo "   -o/--output: append program output to this file"
-    echo "   -U/--rules: path to rules file (only applicable for falco/test_mm)"
-    echo "   -t/--test: test to run. Argument has the following format:"
-    echo "       trace:<trace>: read the specified trace file."
-    echo "            trace:all means run all traces"
-    echo "       live:<live test>: run the specified live test."
-    echo "            live:all means run all live tests."
-    echo "            possible live tests:"
-    echo "                live:htop: run htop -d2"
-    echo "                live:kube-demo: run kubernetes demo from infrastructure repo"
-    echo "                live:juttle-examples: run a juttle demo environment based on docker-compose"
-    echo "       phoronix:<test>: run the specified phoronix test."
-    echo "            if <test> is not 'all', it is passed directly to the command line of \"phoronix-test-suite run <test>\""
-    echo "            if <test> is 'all', a built-in set of phoronix tests will be chosen and run"
-    echo "   -T/--tracedir: Look for trace files in this directory. If doesn't exist, will download trace files from s3"
-    echo "   -A/--agent-autodrop: When running an agent, whether or not to enable autodrop"
-    echo "   -F/--falco-agent: When running an agent, whether or not to enable falco"
-}
-
-OPTS=`getopt -o hv:r:s:R:S:o:U:t:T: --long help,variant:,root:,source:,results:,stats:,output:,rules:,test:,tracedir:,agent-autodrop:,falco-agent: -n $0 -- "$@"`
-
-if [ $? != 0 ]; then
-    echo "Exiting" >&2
-    exit 1
-fi
-
-eval set -- "$OPTS"
-
-VARIANT="falco"
-ROOT=`dirname $0`/../build
-SOURCE=$ROOT
-SCRIPTDIR=`dirname $0`
-RESULTS_FILE=`dirname $0`/results.json
-STATS_FILE=`dirname $0`/capture_stats.json
-OUTPUT_FILE=`dirname $0`/program-output.txt
-RULES_FILE=
-TEST=trace:all
-TRACEDIR=/tmp/falco-perf-traces.$USER
-CPU_INTERVAL=10
-AGENT_AUTODROP=1
-FALCO_AGENT=1
-
-while true; do
-    case "$1" in
-	-h | --help ) usage; exit 1;;
-	-v | --variant ) VARIANT="$2"; shift 2;;
-	-r | --root ) ROOT="$2"; shift 2;;
-	-s | --source ) SOURCE="$2"; shift 2;;
-	-R | --results ) RESULTS_FILE="$2"; shift 2;;
-	-S | --stats ) STATS_FILE="$2"; shift 2;;
-	-o | --output ) OUTPUT_FILE="$2"; shift 2;;
-	-U | --rules ) RULES_FILE="$2"; shift 2;;
-	-t | --test ) TEST="$2"; shift 2;;
-	-T | --tracedir ) TRACEDIR="$2"; shift 2;;
-	-A | --agent-autodrop ) AGENT_AUTODROP="$2"; shift 2;;
-	-F | --falco-agent ) FALCO_AGENT="$2"; shift 2;;
-	* ) break;;
-    esac
-done
-
-if [ -z $VARIANT ]; then
-    echo "A test variant name must be provided. Not continuing."
-    exit 1
-fi
-
-if [ -z $ROOT ]; then
-    echo "A root directory containing a falco/sysdig binary must be provided. Not continuing."
-    exit 1
-fi
-
-ROOT=`realpath $ROOT`
-
-if [ -z $SOURCE ]; then
-    echo "A source directory containing falco/sysdig source code. Not continuing."
-    exit 1
-fi
-
-SOURCE=`realpath $SOURCE`
-
-if [ -z $RESULTS_FILE ]; then
-    echo "An output file for test results must be provided. Not continuing."
-    exit 1
-fi
-
-if [ -z $STATS_FILE ]; then
-    echo "An output file for capture statistics must be provided. Not continuing."
-    exit 1
-fi
-
-if [ -z $OUTPUT_FILE ]; then
-    echo "An file for program output must be provided. Not continuing."
-    exit 1
-fi
-
-if [ -z $TEST ]; then
-    echo "A test must be provided. Not continuing."
-    exit 1
-fi
-
-run_tests

test/run_regression_tests.sh

@@ -20,6 +20,7 @@ set -euo pipefail
 SCRIPT=$(readlink -f $0)
 SCRIPTDIR=$(dirname "$SCRIPT")
 SKIP_PACKAGES_TESTS=${SKIP_PACKAGES_TESTS:-false}
+SKIP_PLUGINS_TESTS=${SKIP_PLUGINS_TESTS:-false}
 TRACE_FILES_BASE_URL=${TRACE_FILES_BASE_URL:-"https://download.falco.org/fixtures/trace-files/"}
 
 # Trace file tarballs are now versioned. Any time a substantial change
@@ -104,6 +105,10 @@ function run_tests() {
     if [ "$SKIP_PACKAGES_TESTS" = false ] ; then
         suites+=($SCRIPTDIR/falco_tests_package.yaml)
     fi
+
+    if [ "$SKIP_PLUGINS_TESTS" = false ] ; then
+        suites+=($SCRIPTDIR/falco_tests_plugins.yaml)
+    fi
     
     XUNIT_DIR="${OPT_BUILD_DIR}/integration-tests-xunit"
     mkdir -p "${XUNIT_DIR}"

test/trace_files/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_subdirectory(k8s_audit)
 add_subdirectory(psp)
+add_subdirectory(plugins)
 
 # Note: list of traces is created at cmake time, not build time
 file(GLOB test_trace_files
@@ -16,4 +17,4 @@ foreach(trace_file_path ${test_trace_files})
 endforeach()
 
 add_custom_target(trace-files-base-scap ALL)
-add_dependencies(trace-files-base-scap ${BASE_SCAP_TRACE_FILES_TARGETS})
+add_dependencies(trace-files-base-scap ${BASE_SCAP_TRACE_FILES_TARGETS})

test/trace_files/plugins/CMakeLists.txt

@@ -0,0 +1,16 @@
+# Note: list of traces is created at cmake time, not build time
+file(GLOB test_trace_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/*.json")
+
+foreach(trace_file_path ${test_trace_files})
+	get_filename_component(trace_file ${trace_file_path} NAME)
+	add_custom_target(test-trace-${trace_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${trace_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		DEPENDS ${trace_file_path})
+	list(APPEND PLUGINS_TRACE_FILES_TARGETS test-trace-${trace_file})
+endforeach()
+
+add_custom_target(trace-files-plugins ALL)
+add_dependencies(trace-files-plugins ${PLUGINS_TRACE_FILES_TARGETS})

test/trace_files/plugins/alice_start_instances.json

@@ -0,0 +1,31 @@
+{"Records": [{
+    "eventVersion": "1.0",
+    "userIdentity": {
+	"type": "IAMUser",
+	"principalId": "EX_PRINCIPAL_ID",
+	"arn": "arn:aws:iam::123456789012:user/Alice",
+	"accessKeyId": "EXAMPLE_KEY_ID",
+	"accountId": "123456789012",
+	"userName": "Alice"
+    },
+    "eventTime": "2014-03-06T21:22:54Z",
+    "eventSource": "ec2.amazonaws.com",
+    "eventType": "AwsApiCall",
+    "eventName": "StartInstances",
+    "awsRegion": "us-east-2",
+    "sourceIPAddress": "205.251.233.176",
+    "userAgent": "ec2-api-tools 1.6.12.2",
+    "requestParameters": {"instancesSet": {"items": [{"instanceId": "i-ebeaf9e2"}]}},
+    "responseElements": {"instancesSet": {"items": [{
+	"instanceId": "i-ebeaf9e2",
+	"currentState": {
+	    "code": 0,
+	    "name": "pending"
+	},
+	"previousState": {
+	    "code": 80,
+	    "name": "stopped"
+	}
+    }]}}
+}]}
+

tests/CMakeLists.txt

@@ -15,12 +15,25 @@
 # the License.
 #
 if(MINIMAL_BUILD)
-  set(FALCO_TESTS_SOURCES test_base.cpp engine/test_token_bucket.cpp engine/test_rulesets.cpp engine/test_falco_utils.cpp)
+  set(
+    FALCO_TESTS_SOURCES
+    test_base.cpp
+    engine/test_rulesets.cpp
+    engine/test_falco_utils.cpp
+    falco/test_configuration.cpp
+  )
 else()
-  set(FALCO_TESTS_SOURCES test_base.cpp engine/test_token_bucket.cpp engine/test_rulesets.cpp engine/test_falco_utils.cpp falco/test_webserver.cpp)
+  set(
+    FALCO_TESTS_SOURCES
+    test_base.cpp
+    engine/test_rulesets.cpp
+    engine/test_falco_utils.cpp
+    falco/test_configuration.cpp
+    falco/test_webserver.cpp
+  )
 endif()
 
-set(FALCO_TESTED_LIBRARIES falco_engine)
+set(FALCO_TESTED_LIBRARIES falco_engine ${YAMLCPP_LIB})
 
 SET(FALCO_TESTS_ARGUMENTS "" CACHE STRING "Test arguments to pass to the Falco test suite")
 
@@ -46,6 +59,7 @@ if(FALCO_BUILD_TESTS)
       PUBLIC "${CATCH2_INCLUDE}"
             "${FAKEIT_INCLUDE}"
             "${PROJECT_SOURCE_DIR}/userspace/engine"
+            "${PROJECT_BINARY_DIR}/userspace/falco"
             "${YAMLCPP_INCLUDE_DIR}"
             "${PROJECT_SOURCE_DIR}/userspace/falco")
   else()
@@ -54,6 +68,7 @@ if(FALCO_BUILD_TESTS)
       PUBLIC "${CATCH2_INCLUDE}"
             "${FAKEIT_INCLUDE}"
             "${PROJECT_SOURCE_DIR}/userspace/engine"
+            "${PROJECT_BINARY_DIR}/userspace/falco"
             "${YAMLCPP_INCLUDE_DIR}"
             "${CIVETWEB_INCLUDE_DIR}"
             "${PROJECT_SOURCE_DIR}/userspace/falco")

tests/engine/test_rulesets.cpp

@@ -25,15 +25,25 @@ static uint16_t default_ruleset = 0;
 static uint16_t non_default_ruleset = 3;
 static uint16_t other_non_default_ruleset = 2;
 static std::set<std::string> tags = {"some_tag", "some_other_tag"};
-static std::set<uint32_t> event_tags = {1};
+
+static std::shared_ptr<gen_event_filter> create_filter()
+{
+	// The actual contents of the filters don't matter here.
+	sinsp_filter_compiler compiler(NULL, "evt.type=open");
+	sinsp_filter *f = compiler.compile();
+
+	std::shared_ptr<gen_event_filter> ret(f);
+
+	return ret;
+}
 
 TEST_CASE("Should enable/disable for exact match w/ default ruleset", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable("one_rule", exact_match, enabled);
 	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
@@ -45,10 +55,10 @@ TEST_CASE("Should enable/disable for exact match w/ default ruleset", "[rulesets
 TEST_CASE("Should enable/disable for exact match w/ specific ruleset", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable("one_rule", exact_match, enabled, non_default_ruleset);
 	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1);
@@ -64,10 +74,10 @@ TEST_CASE("Should enable/disable for exact match w/ specific ruleset", "[ruleset
 TEST_CASE("Should not enable for exact match different rule name", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable("some_other_rule", exact_match, enabled);
 	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
@@ -76,10 +86,10 @@ TEST_CASE("Should not enable for exact match different rule name", "[rulesets]")
 TEST_CASE("Should enable/disable for exact match w/ substring and default ruleset", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable("one_rule", substring_match, enabled);
 	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
@@ -91,10 +101,10 @@ TEST_CASE("Should enable/disable for exact match w/ substring and default rulese
 TEST_CASE("Should not enable for substring w/ exact_match", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable("one_", exact_match, enabled);
 	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
@@ -103,10 +113,10 @@ TEST_CASE("Should not enable for substring w/ exact_match", "[rulesets]")
 TEST_CASE("Should enable/disable for prefix match w/ default ruleset", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable("one_", substring_match, enabled);
 	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
@@ -118,10 +128,10 @@ TEST_CASE("Should enable/disable for prefix match w/ default ruleset", "[ruleset
 TEST_CASE("Should enable/disable for suffix match w/ default ruleset", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable("_rule", substring_match, enabled);
 	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
@@ -133,10 +143,10 @@ TEST_CASE("Should enable/disable for suffix match w/ default ruleset", "[ruleset
 TEST_CASE("Should enable/disable for substring match w/ default ruleset", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable("ne_ru", substring_match, enabled);
 	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
@@ -148,10 +158,10 @@ TEST_CASE("Should enable/disable for substring match w/ default ruleset", "[rule
 TEST_CASE("Should enable/disable for substring match w/ specific ruleset", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable("ne_ru", substring_match, enabled, non_default_ruleset);
 	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1);
@@ -167,11 +177,11 @@ TEST_CASE("Should enable/disable for substring match w/ specific ruleset", "[rul
 TEST_CASE("Should enable/disable for tags w/ default ruleset", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 	std::set<std::string> want_tags = {"some_tag"};
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable_tags(want_tags, enabled);
 	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
@@ -183,11 +193,11 @@ TEST_CASE("Should enable/disable for tags w/ default ruleset", "[rulesets]")
 TEST_CASE("Should enable/disable for tags w/ specific ruleset", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 	std::set<std::string> want_tags = {"some_tag"};
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable_tags(want_tags, enabled, non_default_ruleset);
 	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1);
@@ -203,11 +213,11 @@ TEST_CASE("Should enable/disable for tags w/ specific ruleset", "[rulesets]")
 TEST_CASE("Should not enable for different tags", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 	std::set<std::string> want_tags = {"some_different_tag"};
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable_tags(want_tags, enabled);
 	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0);
@@ -216,11 +226,11 @@ TEST_CASE("Should not enable for different tags", "[rulesets]")
 TEST_CASE("Should enable/disable for overlapping tags", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 	std::set<std::string> want_tags = {"some_tag", "some_different_tag"};
 
-	r.add(rule_name, tags, event_tags, filter);
+	r.add(rule_name, tags, filter);
 
 	r.enable_tags(want_tags, enabled);
 	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
@@ -232,15 +242,15 @@ TEST_CASE("Should enable/disable for overlapping tags", "[rulesets]")
 TEST_CASE("Should enable/disable for incremental adding tags", "[rulesets]")
 {
 	falco_ruleset r;
-	gen_event_filter *rule1_filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> rule1_filter = create_filter();
 	string rule1_name = "one_rule";
 	std::set<std::string> rule1_tags = {"rule1_tag"};
-	r.add(rule1_name, rule1_tags, event_tags, rule1_filter);
+	r.add(rule1_name, rule1_tags, rule1_filter);
 
-	gen_event_filter *rule2_filter = new gen_event_filter();
+	std::shared_ptr<gen_event_filter> rule2_filter = create_filter();
 	string rule2_name = "two_rule";
 	std::set<std::string> rule2_tags = {"rule2_tag"};
-	r.add(rule2_name, rule2_tags, event_tags, rule2_filter);
+	r.add(rule2_name, rule2_tags, rule2_filter);
 
 	std::set<std::string> want_tags;
 

tests/engine/test_token_bucket.cpp

@@ -1,83 +0,0 @@
-/*
-Copyright (C) 2019 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include <memory>
-
-#include "token_bucket.h"
-#include <catch.hpp>
-
-using namespace Catch::literals;
-
-TEST_CASE("token bucket default ctor", "[token_bucket]")
-{
-	auto tb = std::make_shared<token_bucket>();
-
-	REQUIRE(tb->get_tokens() == 1);
-
-	SECTION("initialising with specific time, rate 2 tokens/sec")
-	{
-		auto max = 2.0;
-		uint64_t now = 1;
-		tb->init(1.0, max, now);
-		REQUIRE(tb->get_last_seen() == now);
-		REQUIRE(tb->get_tokens() == max);
-	}
-}
-
-TEST_CASE("token bucket ctor with custom timer", "[token_bucket]")
-{
-	auto t = []() -> uint64_t { return 22; };
-	auto tb = std::make_shared<token_bucket>(t);
-
-	REQUIRE(tb->get_tokens() == 1);
-	REQUIRE(tb->get_last_seen() == 22);
-}
-
-TEST_CASE("token bucket with 2 tokens/sec rate, max 10 tokens", "[token_bucket]")
-{
-	auto tb = std::make_shared<token_bucket>();
-	tb->init(2.0, 10, 1);
-
-	SECTION("claiming 5 tokens")
-	{
-		bool claimed = tb->claim(5, 1000000001);
-		REQUIRE(tb->get_last_seen() == 1000000001);
-		REQUIRE(tb->get_tokens() == 5.0_a);
-		REQUIRE(claimed);
-
-		SECTION("claiming all the 7 remaining tokens")
-		{
-			bool claimed = tb->claim(7, 2000000001);
-			REQUIRE(tb->get_last_seen() == 2000000001);
-			REQUIRE(tb->get_tokens() == 0.0_a);
-			REQUIRE(claimed);
-
-			SECTION("claiming 1 token more than the 2 available fails")
-			{
-				bool claimed = tb->claim(3, 3000000001);
-				REQUIRE(tb->get_last_seen() == 3000000001);
-				REQUIRE(tb->get_tokens() == 2.0_a);
-				REQUIRE_FALSE(claimed);
-			}
-		}
-	}
-}
-
-TEST_CASE("token bucket default initialization", "[token_bucket]")
-{
-	token_bucket tb;
-	REQUIRE(tb.get_tokens() == 1);
-}

tests/falco/test_configuration.cpp

@@ -0,0 +1,106 @@
+/*
+Copyright (C) 2021 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+#include "configuration.h"
+#include <catch.hpp>
+
+string sample_yaml = 
+    "base_value:\n"
+    "    id: 1\n"
+    "    name: 'sample_name'\n"
+    "    subvalue:\n"
+    "      subvalue2:\n"
+    "        boolean: true\n"
+    "base_value_2:\n"
+    "  sample_list:\n"
+    "    - elem1\n"
+    "    - elem2\n"
+    "    - elem3\n"
+;
+
+TEST_CASE("configuration must load YAML data", "[configuration]")
+{
+    yaml_configuration conf;
+
+    SECTION("broken YAML")
+    {
+        string sample_broken_yaml = sample_yaml + " /  bad_symbol";
+        REQUIRE_THROWS(conf.load_from_string(sample_broken_yaml));
+    }
+
+    SECTION("valid YAML")
+    {    
+        REQUIRE_NOTHROW(conf.load_from_string(sample_yaml));
+    }
+
+    SECTION("clearing and reloading")
+    {   
+        conf.load_from_string(sample_yaml);
+        REQUIRE(conf.is_defined("base_value") == true);
+        conf.clear();
+        REQUIRE(conf.is_defined("base_value") == false);
+        conf.load_from_string(sample_yaml);
+        REQUIRE(conf.is_defined("base_value") == true);
+    }
+}
+
+TEST_CASE("configuration must read YAML fields", "[configuration]")
+{
+	yaml_configuration conf;
+    conf.load_from_string(sample_yaml);
+
+    SECTION("base level")
+    {
+        REQUIRE(conf.is_defined("base_value") == true);
+        REQUIRE(conf.is_defined("base_value_2") == true);
+        REQUIRE(conf.is_defined("unknown_base_value") == false);
+    }
+
+    SECTION("arbitrary depth nesting")
+    {
+        REQUIRE(conf.get_scalar<int>("base_value.id", -1) == 1);
+        REQUIRE(conf.get_scalar<string>("base_value.name", "none") == "sample_name");
+        REQUIRE(conf.get_scalar<bool>("base_value.subvalue.subvalue2.boolean", false) == true);
+    }
+    
+    SECTION("list field elements")
+    {
+        REQUIRE(conf.get_scalar<string>("base_value_2.sample_list[0]", "none") == "elem1");
+        REQUIRE(conf.get_scalar<string>("base_value_2.sample_list[1]", "none") == "elem2");
+        REQUIRE(conf.get_scalar<string>("base_value_2.sample_list[2]", "none") == "elem3");
+    }
+
+    SECTION("sequence")
+    {
+        vector<string> seq;
+        conf.get_sequence(seq, "base_value_2.sample_list");
+        REQUIRE(seq.size() == 3);
+        REQUIRE(seq[0] == "elem1");
+        REQUIRE(seq[1] == "elem2");
+        REQUIRE(seq[2] == "elem3");
+    }
+}
+
+TEST_CASE("configuration must modify YAML fields", "[configuration]")
+{
+    string key = "base_value.subvalue.subvalue2.boolean";
+	yaml_configuration conf;
+    conf.load_from_string(sample_yaml);
+    REQUIRE(conf.get_scalar<bool>(key, false) == true);
+    conf.set_scalar<bool>(key, false);
+    REQUIRE(conf.get_scalar<bool>(key, true) == false);
+    conf.set_scalar<bool>(key, true);
+    REQUIRE(conf.get_scalar<bool>(key, false) == true);
+}

userspace/engine/CMakeLists.txt

@@ -10,6 +10,8 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
+add_subdirectory(lua)
+
 set(FALCO_ENGINE_SOURCE_FILES
     rules.cpp
     falco_common.cpp
@@ -17,7 +19,6 @@ set(FALCO_ENGINE_SOURCE_FILES
     falco_utils.cpp
     json_evt.cpp
     ruleset.cpp
-    token_bucket.cpp
     formats.cpp)
 
 add_library(falco_engine STATIC ${FALCO_ENGINE_SOURCE_FILES})
@@ -37,7 +38,8 @@ if(MINIMAL_BUILD)
       "${STRING_VIEW_LITE_INCLUDE}"
       "${LIBSCAP_INCLUDE_DIRS}"
       "${LIBSINSP_INCLUDE_DIRS}"
-      "${PROJECT_BINARY_DIR}/userspace/engine")
+      "${PROJECT_BINARY_DIR}/userspace/engine"
+      "${PROJECT_BINARY_DIR}/userspace/engine/lua")
 else()
   target_include_directories(
     falco_engine
@@ -49,24 +51,8 @@ else()
       "${STRING_VIEW_LITE_INCLUDE}"
       "${LIBSCAP_INCLUDE_DIRS}"
       "${LIBSINSP_INCLUDE_DIRS}"
-      "${PROJECT_BINARY_DIR}/userspace/engine")
+      "${PROJECT_BINARY_DIR}/userspace/engine"
+      "${PROJECT_BINARY_DIR}/userspace/engine/lua")
 endif()
 
-target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${LPEG_LIB}" "${LYAML_LIB}" "${LIBYAML_LIB}")
-
-configure_file(config_falco_engine.h.in config_falco_engine.h)
-
-if(DEFINED FALCO_COMPONENT)
-  install(
-    DIRECTORY lua
-    DESTINATION "${FALCO_SHARE_DIR}"
-    COMPONENT "${FALCO_COMPONENT}"
-    FILES_MATCHING
-    PATTERN *.lua)
-else()
-  install(
-    DIRECTORY lua
-    DESTINATION "${FALCO_SHARE_DIR}"
-    FILES_MATCHING
-    PATTERN *.lua)
-endif()
+target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${LPEG_LIB}" "${LYAML_LIB}" "${LIBYAML_LIB}" luafiles)

userspace/engine/falco_common.cpp

@@ -16,9 +16,9 @@ limitations under the License.
 
 #include <fstream>
 
-#include "config_falco_engine.h"
 #include "falco_common.h"
 #include "banned.h" // This raises a compilation error when certain functions are used
+#include "falco_engine_lua_files.hh"
 
 std::vector<std::string> falco_common::priority_names = {
 	"Emergency",
@@ -48,73 +48,33 @@ falco_common::~falco_common()
 	}
 }
 
-void falco_common::set_inspector(sinsp *inspector)
+void falco_common::init()
 {
-	m_inspector = inspector;
-}
-
-void falco_common::init(const char *lua_main_filename, const char *alternate_lua_dir)
-{
-	ifstream is;
-	string lua_dir = FALCO_ENGINE_LUA_DIR;
-	string lua_main_path = lua_dir + lua_main_filename;
-
-	is.open(lua_main_path);
-	if (!is.is_open())
+	// Strings in the list lua_module_strings need to be loaded as
+	// lua modules, which also involves adding them to the
+	// package.module table.
+	for(const auto &pair : lua_module_strings)
 	{
-		lua_dir = alternate_lua_dir;
-		lua_main_path = lua_dir + lua_main_filename;
+		lua_getglobal(m_ls, "package");
+		lua_getfield(m_ls, -1, "preload");
 
-		is.open(lua_main_path);
-		if (!is.is_open())
+		if(luaL_loadstring(m_ls, pair.first))
 		{
-			throw falco_exception("Could not find Falco Lua entrypoint (tried " +
-					      string(FALCO_ENGINE_LUA_DIR) + lua_main_filename + ", " +
-					      string(alternate_lua_dir) + lua_main_filename + ")");
+			throw falco_exception("Failed to load embedded lua code " +
+					      string(pair.second) + ": " + lua_tostring(m_ls, -1));
+		}
+
+		lua_setfield(m_ls, -2, pair.second);
+	}
+
+	// Strings in the list lua_code_strings need to be loaded and
+	// evaluated so any public functions can be directly called.
+	for(const auto &str : lua_code_strings)
+	{
+		if(luaL_loadstring(m_ls, str) || lua_pcall(m_ls, 0, 0, 0))
+		{
+			throw falco_exception("Failed to load + evaluate embedded lua code " +
+					      string(str) + ": " + lua_tostring(m_ls, -1));
 		}
 	}
-
-	// Initialize Lua interpreter
-	add_lua_path(lua_dir);
-
-	// Load the main program, which defines all the available functions.
-	string scriptstr((istreambuf_iterator<char>(is)),
-			 istreambuf_iterator<char>());
-
-	if(luaL_loadstring(m_ls, scriptstr.c_str()) || lua_pcall(m_ls, 0, 0, 0))
-	{
-		throw falco_exception("Failed to load script " +
-			lua_main_path + ": " + lua_tostring(m_ls, -1));
-	}
-}
-
-void falco_common::add_lua_path(string &path)
-{
-	string cpath = string(path);
-	path += "?.lua";
-	cpath += "?.so";
-
-	lua_getglobal(m_ls, "package");
-
-	lua_getfield(m_ls, -1, "path");
-	string cur_path = lua_tostring(m_ls, -1 );
-	cur_path += ';';
-	lua_pop(m_ls, 1);
-
-	cur_path.append(path.c_str());
-
-	lua_pushstring(m_ls, cur_path.c_str());
-	lua_setfield(m_ls, -2, "path");
-
-	lua_getfield(m_ls, -1, "cpath");
-	string cur_cpath = lua_tostring(m_ls, -1 );
-	cur_cpath += ';';
-	lua_pop(m_ls, 1);
-
-	cur_cpath.append(cpath.c_str());
-
-	lua_pushstring(m_ls, cur_cpath.c_str());
-	lua_setfield(m_ls, -2, "cpath");
-
-	lua_pop(m_ls, 1);
 }

userspace/engine/falco_common.h

@@ -69,9 +69,7 @@ public:
 	falco_common();
 	virtual ~falco_common();
 
-	void init(const char *lua_main_filename, const char *alternate_lua_dir);
-
-	void set_inspector(sinsp *inspector);
+	void init();
 
     // Priority levels, as a vector of strings
 	static std::vector<std::string> priority_names;
@@ -93,9 +91,4 @@ protected:
 	lua_State *m_ls;
 
 	std::mutex m_ls_semaphore;
-
-	sinsp *m_inspector;
-
-private:
-	void add_lua_path(std::string &path);
-};
+};

userspace/engine/falco_engine.cpp

@@ -19,10 +19,12 @@ limitations under the License.
 #include <string>
 #include <fstream>
 
+#include <sinsp.h>
+#include <plugin.h>
+
 #include "falco_engine.h"
 #include "falco_utils.h"
 #include "falco_engine_version.h"
-#include "config_falco_engine.h"
 
 #include "formats.h"
 
@@ -37,13 +39,12 @@ extern "C" {
 
 string lua_on_event = "on_event";
 string lua_print_stats = "print_stats";
+const std::string falco_engine::s_default_ruleset = "falco-default-ruleset";
 
 using namespace std;
 
-nlohmann::json::json_pointer falco_engine::k8s_audit_time = "/stageTimestamp"_json_pointer;
-
-falco_engine::falco_engine(bool seed_rng, const std::string& alternate_lua_dir)
-	: m_rules(NULL), m_next_ruleset_id(0),
+falco_engine::falco_engine(bool seed_rng)
+	: m_next_ruleset_id(0),
 	  m_min_priority(falco_common::PRIORITY_DEBUG),
 	  m_sampling_ratio(1), m_sampling_multiplier(0),
 	  m_replace_container_info(false)
@@ -51,29 +52,21 @@ falco_engine::falco_engine(bool seed_rng, const std::string& alternate_lua_dir)
 	luaopen_lpeg(m_ls);
 	luaopen_yaml(m_ls);
 
-	falco_common::init(m_lua_main_filename.c_str(), alternate_lua_dir.c_str());
+	falco_common::init();
 	falco_rules::init(m_ls);
 
-	m_sinsp_rules.reset(new falco_sinsp_ruleset());
-	m_k8s_audit_rules.reset(new falco_ruleset());
+	m_required_plugin_versions.clear();
 
 	if(seed_rng)
 	{
 		srandom((unsigned) getpid());
 	}
 
-	m_default_ruleset_id = find_ruleset_id(m_default_ruleset);
-
-	// Create this now so we can potentially list filters and exit
-	m_json_factory = make_shared<json_event_filter_factory>();
+	m_default_ruleset_id = find_ruleset_id(s_default_ruleset);
 }
 
 falco_engine::~falco_engine()
 {
-	if (m_rules)
-	{
-		delete m_rules;
-	}
 }
 
 uint32_t falco_engine::engine_version()
@@ -81,65 +74,76 @@ uint32_t falco_engine::engine_version()
 	return (uint32_t) FALCO_ENGINE_VERSION;
 }
 
-#define DESCRIPTION_TEXT_START 16
-
-#define CONSOLE_LINE_LEN 79
-
-void falco_engine::list_fields(bool names_only)
+// Return a key that uniquely represents a field class.
+// For now, we assume name + shortdesc is unique.
+static std::string fieldclass_key(const gen_event_filter_factory::filter_fieldclass_info &fld_info)
 {
-	for(auto &chk_field : json_factory().get_fields())
+	return fld_info.name + fld_info.shortdesc;
+}
+
+void falco_engine::list_fields(std::string &source, bool verbose, bool names_only)
+{
+	// Maps from field class name + short desc to list of event
+	// sources for which this field class can be used.
+	std::map<std::string,std::set<std::string>> fieldclass_event_sources;
+
+	// Do a first pass to group together classes that are
+	// applicable to multiple event sources.
+	for(auto &it : m_filter_factories)
 	{
-		if(!names_only)
+		if(source != "" && source != it.first)
 		{
-			printf("\n----------------------\n");
-			printf("Field Class: %s (%s)\n\n", chk_field.m_name.c_str(), chk_field.m_desc.c_str());
-			if(chk_field.m_class_info != "")
-			{
-				std::string str = falco::utils::wrap_text(chk_field.m_class_info, 0, 0, CONSOLE_LINE_LEN);
-				printf("%s\n", str.c_str());
-			}
+			continue;
 		}
 
-		for(auto &field : chk_field.m_fields)
+		for(auto &fld_class : it.second->get_fields())
 		{
-			printf("%s", field.m_name.c_str());
+			fieldclass_event_sources[fieldclass_key(fld_class)].insert(it.first);
+		}
+	}
 
-			if(names_only)
+	// The set of field classes already printed. Used to avoid
+	// printing field classes multiple times for different sources
+	std::set<std::string> seen_fieldclasses;
+
+	// In the second pass, actually print info, skipping duplicate
+	// field classes and also printing info on supported sources.
+	for(auto &it : m_filter_factories)
+	{
+		if(source != "" && source != it.first)
+		{
+			continue;
+		}
+
+		for(auto &fld_class : it.second->get_fields())
+		{
+			std::string key = fieldclass_key(fld_class);
+
+			if(seen_fieldclasses.find(key) != seen_fieldclasses.end())
 			{
-				printf("\n");
 				continue;
 			}
-			uint32_t namelen = field.m_name.size();
 
-			if(namelen >= DESCRIPTION_TEXT_START)
+			seen_fieldclasses.insert(key);
+
+			if(!names_only)
 			{
-				printf("\n");
-				namelen = 0;
+				printf("%s\n", fld_class.as_string(verbose,
+								   fieldclass_event_sources[fieldclass_key(fld_class)]).c_str());
 			}
-
-			for(uint32_t l = 0; l < DESCRIPTION_TEXT_START - namelen; l++)
+			else
 			{
-				printf(" ");
+				for(auto &field : fld_class.fields)
+				{
+					// Skip fields with the EPF_TABLE_ONLY flag.
+					if(field.tags.find("EPF_TABLE_ONLY") != field.tags.end())
+					{
+						continue;
+					}
+
+					printf("%s\n", field.name.c_str());
+				}
 			}
-
-			std::string desc = field.m_desc;
-			switch(field.m_idx_mode)
-			{
-			case json_event_filter_check::IDX_REQUIRED:
-			case json_event_filter_check::IDX_ALLOWED:
-				desc += " (";
-				desc += json_event_filter_check::s_index_mode_strs[field.m_idx_mode];
-				desc += ", ";
-				desc += json_event_filter_check::s_index_type_strs[field.m_idx_type];
-				desc += ")";
-				break;
-			case json_event_filter_check::IDX_NONE:
-			default:
-				break;
-			};
-
-			std::string str = falco::utils::wrap_text(desc, namelen, DESCRIPTION_TEXT_START, CONSOLE_LINE_LEN);
-			printf("%s\n", str.c_str());
 		}
 	}
 }
@@ -153,34 +157,18 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al
 
 void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version)
 {
-	// The engine must have been given an inspector by now.
-	if(! m_inspector)
-	{
-		throw falco_exception("No inspector provided");
-	}
-
-	if(!m_sinsp_factory)
-	{
-		m_sinsp_factory = make_shared<sinsp_filter_factory>(m_inspector);
-	}
-
 	if(!m_rules)
 	{
-		m_rules = new falco_rules(m_inspector,
-					  this,
-					  m_ls);
+		m_rules.reset(new falco_rules(this,
+					      m_ls));
+
+		for(auto const &it : m_filter_factories)
+		{
+			m_rules->add_filter_factory(it.first, it.second);
+		}
 	}
 
-	// Note that falco_formats is added to the lua state used
-	// by the falco engine only. Within the engine, only
-	// formats.formatter is used, so we can unconditionally set
-	// json_output to false.
-	bool json_output = false;
-	bool json_include_output_property = false;
-	bool json_include_tags_property = false;
-	falco_formats::init(m_inspector, this, m_ls, json_output, json_include_output_property, json_include_tags_property);
-
-	m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info, m_min_priority, required_engine_version);
+	m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info, m_min_priority, required_engine_version, m_required_plugin_versions);
 }
 
 void falco_engine::load_rules_file(const string &rules_filename, bool verbose, bool all_events)
@@ -213,13 +201,10 @@ void falco_engine::enable_rule(const string &substring, bool enabled, const stri
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
 	bool match_exact = false;
 
-	m_sinsp_rules->enable(substring, match_exact, enabled, ruleset_id);
-	m_k8s_audit_rules->enable(substring, match_exact, enabled, ruleset_id);
-}
-
-void falco_engine::enable_rule(const string &substring, bool enabled)
-{
-	enable_rule(substring, enabled, m_default_ruleset);
+	for(auto &it : m_rulesets)
+	{
+		it.second->enable(substring, match_exact, enabled, ruleset_id);
+	}
 }
 
 void falco_engine::enable_rule_exact(const string &rule_name, bool enabled, const string &ruleset)
@@ -227,26 +212,20 @@ void falco_engine::enable_rule_exact(const string &rule_name, bool enabled, cons
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
 	bool match_exact = true;
 
-	m_sinsp_rules->enable(rule_name, match_exact, enabled, ruleset_id);
-	m_k8s_audit_rules->enable(rule_name, match_exact, enabled, ruleset_id);
-}
-
-void falco_engine::enable_rule_exact(const string &rule_name, bool enabled)
-{
-	enable_rule_exact(rule_name, enabled, m_default_ruleset);
+	for(auto &it : m_rulesets)
+	{
+		it.second->enable(rule_name, match_exact, enabled, ruleset_id);
+	}
 }
 
 void falco_engine::enable_rule_by_tag(const set<string> &tags, bool enabled, const string &ruleset)
 {
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
 
-	m_sinsp_rules->enable_tags(tags, enabled, ruleset_id);
-	m_k8s_audit_rules->enable_tags(tags, enabled, ruleset_id);
-}
-
-void falco_engine::enable_rule_by_tag(const set<string> &tags, bool enabled)
-{
-	enable_rule_by_tag(tags, enabled, m_default_ruleset);
+	for(auto &it : m_rulesets)
+	{
+		it.second->enable_tags(tags, enabled, ruleset_id);
+	}
 }
 
 void falco_engine::set_min_priority(falco_common::priority_type priority)
@@ -272,68 +251,85 @@ uint64_t falco_engine::num_rules_for_ruleset(const std::string &ruleset)
 {
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
 
-	return m_sinsp_rules->num_rules_for_ruleset(ruleset_id) +
-		m_k8s_audit_rules->num_rules_for_ruleset(ruleset_id);
+	uint64_t ret = 0;
+	for(auto &it : m_rulesets)
+	{
+		ret += it.second->num_rules_for_ruleset(ruleset_id);
+	}
+
+	return ret;
 }
 
-void falco_engine::evttypes_for_ruleset(std::vector<bool> &evttypes, const std::string &ruleset)
+void falco_engine::evttypes_for_ruleset(std::string &source, std::set<uint16_t> &evttypes, const std::string &ruleset)
 {
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
 
-	return m_sinsp_rules->evttypes_for_ruleset(evttypes, ruleset_id);
+	auto it = m_rulesets.find(source);
+	if(it == m_rulesets.end())
+	{
+		string err = "Unknown event source " + source;
+		throw falco_exception(err);
+	}
+
+	it->second->evttypes_for_ruleset(evttypes, ruleset_id);
+
 }
 
-void falco_engine::syscalls_for_ruleset(std::vector<bool> &syscalls, const std::string &ruleset)
+std::shared_ptr<gen_event_formatter> falco_engine::create_formatter(const std::string &source,
+								    const std::string &output)
 {
-	uint16_t ruleset_id = find_ruleset_id(ruleset);
+	auto it = m_format_factories.find(source);
 
-	return m_sinsp_rules->syscalls_for_ruleset(syscalls, ruleset_id);
+	if(it == m_format_factories.end())
+	{
+		string err = "Unknown event source " + source;
+		throw falco_exception(err);
+	}
+
+	return it->second->create_formatter(output);
 }
 
-unique_ptr<falco_engine::rule_result> falco_engine::process_sinsp_event(sinsp_evt *ev, uint16_t ruleset_id)
+unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::string &source, gen_event *ev, uint16_t ruleset_id)
 {
 	if(should_drop_evt())
 	{
 		return unique_ptr<struct rule_result>();
 	}
 
-	if(!m_sinsp_rules->run(ev, ruleset_id))
+	auto it = m_rulesets.find(source);
+	if(it == m_rulesets.end())
+	{
+		string err = "Unknown event source " + source;
+		throw falco_exception(err);
+	}
+
+	if (!it->second->run(ev, ruleset_id))
 	{
 		return unique_ptr<struct rule_result>();
 	}
 
 	unique_ptr<struct rule_result> res(new rule_result());
-	res->source = "syscall";
+	res->source = source;
 
 	populate_rule_result(res, ev);
 
 	return res;
 }
 
-unique_ptr<falco_engine::rule_result> falco_engine::process_sinsp_event(sinsp_evt *ev)
+unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::string &source, gen_event *ev)
 {
-	return process_sinsp_event(ev, m_default_ruleset_id);
+	return process_event(source, ev, m_default_ruleset_id);
 }
 
-unique_ptr<falco_engine::rule_result> falco_engine::process_k8s_audit_event(json_event *ev, uint16_t ruleset_id)
+void falco_engine::add_source(const std::string &source,
+			      std::shared_ptr<gen_event_filter_factory> filter_factory,
+			      std::shared_ptr<gen_event_formatter_factory> formatter_factory)
 {
-	if(should_drop_evt())
-	{
-		return unique_ptr<struct rule_result>();
-	}
+	m_filter_factories[source] = filter_factory;
+	m_format_factories[source] = formatter_factory;
 
-	// All k8s audit events have the single tag "1".
-	if(!m_k8s_audit_rules->run((gen_event *) ev, 1, ruleset_id))
-	{
-		return unique_ptr<struct rule_result>();
-	}
-
-	unique_ptr<struct rule_result> res(new rule_result());
-	res->source = "k8s_audit";
-
-	populate_rule_result(res, ev);
-
-	return res;
+	std::shared_ptr<falco_ruleset> ruleset(new falco_ruleset());
+	m_rulesets[source] = ruleset;
 }
 
 void falco_engine::populate_rule_result(unique_ptr<struct rule_result> &res, gen_event *ev)
@@ -386,84 +382,6 @@ void falco_engine::populate_rule_result(unique_ptr<struct rule_result> &res, gen
 	}
 }
 
-bool falco_engine::parse_k8s_audit_json(nlohmann::json &j, std::list<json_event> &evts, bool top)
-{
-	// Note that nlohmann::basic_json::value can throw  nlohmann::basic_json::type_error (302, 306)
-	try
-	{
-		// If the object is an array, call parse_k8s_audit_json again for each item.
-		if(j.is_array())
-		{
-			if(top)
-			{
-				for(auto &item : j)
-				{
-					// Note we only handle a single top level array, to
-					// avoid excessive recursion.
-					if(! parse_k8s_audit_json(item, evts, false))
-					{
-						return false;
-					}
-				}
-
-				return true;
-			}
-			else
-			{
-				return false;
-			}
-		}
-
-		// If the kind is EventList, split it into individual events
-		if(j.value("kind", "<NA>") == "EventList")
-		{
-			for(auto &je : j["items"])
-			{
-				evts.emplace_back();
-				je["kind"] = "Event";
-
-				uint64_t ns = 0;
-				if(!sinsp_utils::parse_iso_8601_utc_string(je.value(k8s_audit_time, "<NA>"), ns))
-				{
-					return false;
-				}
-
-				std::string tmp;
-				sinsp_utils::ts_to_string(ns, &tmp, false, true);
-
-				evts.back().set_jevt(je, ns);
-			}
-
-			return true;
-		}
-		else if(j.value("kind", "<NA>") == "Event")
-		{
-			evts.emplace_back();
-			uint64_t ns = 0;
-			if(!sinsp_utils::parse_iso_8601_utc_string(j.value(k8s_audit_time, "<NA>"), ns))
-			{
-				return false;
-			}
-
-			evts.back().set_jevt(j, ns);
-			return true;
-		}
-		else
-		{
-			return false;
-		}
-	}
-	catch(exception &e)
-	{
-		return false;
-	}
-}
-
-unique_ptr<falco_engine::rule_result> falco_engine::process_k8s_audit_event(json_event *ev)
-{
-	return process_k8s_audit_event(ev, m_default_ruleset_id);
-}
-
 void falco_engine::describe_rule(string *rule)
 {
 	return m_rules->describe_rule(rule);
@@ -490,29 +408,68 @@ void falco_engine::print_stats()
 
 }
 
-void falco_engine::add_sinsp_filter(string &rule,
-				    set<uint32_t> &evttypes,
-				    set<uint32_t> &syscalls,
-				    set<string> &tags,
-				    sinsp_filter* filter)
+void falco_engine::add_filter(std::shared_ptr<gen_event_filter> filter,
+			      std::string &rule,
+			      std::string &source,
+			      std::set<std::string> &tags)
 {
-	m_sinsp_rules->add(rule, evttypes, syscalls, tags, filter);
+	auto it = m_rulesets.find(source);
+	if(it == m_rulesets.end())
+	{
+		string err = "Unknown event source " + source;
+		throw falco_exception(err);
+	}
+
+	it->second->add(rule, tags, filter);
 }
 
-void falco_engine::add_k8s_audit_filter(string &rule,
-					set<string> &tags,
-					json_event_filter* filter)
+bool falco_engine::is_source_valid(const std::string &source)
 {
-	// All k8s audit events have a single tag "1".
-	std::set<uint32_t> event_tags = {1};
+	return (m_rulesets.find(source) != m_rulesets.end());
+}
 
-	m_k8s_audit_rules->add(rule, tags, event_tags, filter);
+bool falco_engine::is_plugin_compatible(const std::string &name,
+					const std::string &version,
+					std::string &required_version)
+{
+	sinsp_plugin::version plugin_version(version.c_str());
+
+	if(!plugin_version.m_valid)
+	{
+		throw falco_exception(string("Plugin version string ") + version + " not valid");
+	}
+
+	if(m_required_plugin_versions.find(name) == m_required_plugin_versions.end())
+	{
+		// No required engine versions, so no restrictions. Compatible.
+		return true;
+	}
+
+	for(auto &rversion : m_required_plugin_versions[name])
+	{
+		sinsp_plugin::version req_version(rversion.c_str());
+		if(req_version.m_version_major > plugin_version.m_version_major)
+		{
+			required_version = rversion;
+			return false;
+		}
+
+	}
+
+	return true;
 }
 
 void falco_engine::clear_filters()
 {
-	m_sinsp_rules.reset(new falco_sinsp_ruleset());
-	m_k8s_audit_rules.reset(new falco_ruleset());
+	m_rulesets.clear();
+
+	for(auto &it : m_filter_factories)
+	{
+		std::shared_ptr<falco_ruleset> ruleset(new falco_ruleset());
+		m_rulesets[it.first] = ruleset;
+	}
+
+	m_required_plugin_versions.clear();
 }
 
 void falco_engine::set_sampling_ratio(uint32_t sampling_ratio)
@@ -546,23 +503,3 @@ inline bool falco_engine::should_drop_evt()
 	double coin = (random() * (1.0/RAND_MAX));
 	return (coin >= (1.0/(m_sampling_multiplier * m_sampling_ratio)));
 }
-
-sinsp_filter_factory &falco_engine::sinsp_factory()
-{
-	if(!m_sinsp_factory)
-	{
-		throw falco_exception("No sinsp factory created yet");
-	}
-
-	return *(m_sinsp_factory.get());
-}
-
-json_event_filter_factory &falco_engine::json_factory()
-{
-	if(!m_json_factory)
-	{
-		throw falco_exception("No json factory created yet");
-	}
-
-	return *(m_json_factory.get());
-}

userspace/engine/falco_engine.h

@@ -28,14 +28,10 @@ limitations under the License.
 
 #include <nlohmann/json.hpp>
 
-#include "sinsp.h"
-#include "filter.h"
-
-#include "json_evt.h"
+#include "gen_filter.h"
 #include "rules.h"
 #include "ruleset.h"
 
-#include "config_falco_engine.h"
 #include "falco_common.h"
 
 //
@@ -47,7 +43,7 @@ limitations under the License.
 class falco_engine : public falco_common
 {
 public:
-	falco_engine(bool seed_rng=true, const std::string& alternate_lua_dir=FALCO_ENGINE_SOURCE_LUA_DIR);
+	falco_engine(bool seed_rng=true);
 	virtual ~falco_engine();
 
 	// A given engine has a version which identifies the fields
@@ -57,7 +53,8 @@ public:
 	static uint32_t engine_version();
 
 	// Print to stdout (using printf) a description of each field supported by this engine.
-	void list_fields(bool names_only=false);
+	// If source is non-empty, only fields for the provided source are printed.
+	void list_fields(std::string &source, bool verbose, bool names_only);
 
 	//
 	// Load rules either directly or from a filename.
@@ -80,25 +77,16 @@ public:
 	// be passed as an argument to process_event(). This allows
 	// for different sets of rules being active at once.
 	//
-	void enable_rule(const std::string &substring, bool enabled, const std::string &ruleset);
-
-	// Wrapper that assumes the default ruleset
-	void enable_rule(const std::string &substring, bool enabled);
+	void enable_rule(const std::string &substring, bool enabled, const std::string &ruleset = s_default_ruleset);
 
 
 	// Like enable_rule, but the rule name must be an exact match.
-	void enable_rule_exact(const std::string &rule_name, bool enabled, const std::string &ruleset);
-
-	// Wrapper that assumes the default ruleset
-	void enable_rule_exact(const std::string &rule_name, bool enabled);
+	void enable_rule_exact(const std::string &rule_name, bool enabled, const std::string &ruleset = s_default_ruleset);
 
 	//
 	// Enable/Disable any rules with any of the provided tags (set, exact matches only)
 	//
-	void enable_rule_by_tag(const std::set<std::string> &tags, bool enabled, const std::string &ruleset);
-
-	// Wrapper that assumes the default ruleset
-	void enable_rule_by_tag(const std::set<std::string> &tags, bool enabled);
+	void enable_rule_by_tag(const std::set<std::string> &tags, bool enabled, const std::string &ruleset = s_default_ruleset);
 
 	// Only load rules having this priority or more severe.
 	void set_min_priority(falco_common::priority_type priority);
@@ -152,8 +140,8 @@ public:
 	//
 	void set_extra(string &extra, bool replace_container_info);
 
-	// **Methods Related to k8s audit log events, which are
-	// **represented as json objects.
+	// Represents the result of matching an event against a set of
+	// rules.
 	struct rule_result {
 		gen_event *evt;
 		std::string rule;
@@ -164,16 +152,6 @@ public:
 		std::set<std::string> tags;
 	};
 
-	//
-	// Given a raw json object, return a list of k8s audit event
-	// objects that represent the object. This method handles
-	// things such as EventList splitting.
-	//
-	// Returns true if the json object was recognized as a k8s
-	// audit event(s), false otherwise.
-	//
-	bool parse_k8s_audit_json(nlohmann::json &j, std::list<json_event> &evts, bool top=true);
-
 	//
 	// Given an event, check it against the set of rules in the
 	// engine and if a matching rule is found, return details on
@@ -185,84 +163,81 @@ public:
 	// with a ruleset string.
 	//
 	// the returned rule_result is allocated and must be delete()d.
-	std::unique_ptr<rule_result> process_k8s_audit_event(json_event *ev, uint16_t ruleset_id);
+	std::unique_ptr<rule_result> process_event(std::string &source, gen_event *ev, uint16_t ruleset_id);
 
 	//
 	// Wrapper assuming the default ruleset
 	//
-	std::unique_ptr<rule_result> process_k8s_audit_event(json_event *ev);
+	std::unique_ptr<rule_result> process_event(std::string &source, gen_event *ev);
 
 	//
-	// Add a k8s_audit filter to the engine
+	// Configure the engine to support events with the provided
+	// source, with the provided filter factory and formatter factory.
 	//
-	void add_k8s_audit_filter(std::string &rule,
-				  std::set<std::string> &tags,
-				  json_event_filter* filter);
+	void add_source(const std::string &source,
+			std::shared_ptr<gen_event_filter_factory> filter_factory,
+			std::shared_ptr<gen_event_formatter_factory> formatter_factory);
 
-	// **Methods Related to Sinsp Events e.g system calls
-	//
-	// Given a ruleset, fill in a bitset containing the event
-	// types for which this ruleset can run.
-	//
-	void evttypes_for_ruleset(std::vector<bool> &evttypes, const std::string &ruleset);
+	// Return whether or not there is a valid filter/formatter
+	// factory for this source.
+	bool is_source_valid(const std::string &source);
 
 	//
-	// Given a ruleset, fill in a bitset containing the syscalls
-	// for which this ruleset can run.
+	// Add a filter for the provided event source to the engine
 	//
-	void syscalls_for_ruleset(std::vector<bool> &syscalls, const std::string &ruleset);
+	void add_filter(std::shared_ptr<gen_event_filter> filter,
+			std::string &rule,
+			std::string &source,
+			std::set<std::string> &tags);
 
 	//
-	// Given an event, check it against the set of rules in the
-	// engine and if a matching rule is found, return details on
-	// the rule that matched. If no rule matched, returns NULL.
+	// Given an event source and ruleset, fill in a bitset
+	// containing the event types for which this ruleset can run.
 	//
-	// When ruleset_id is provided, use the enabled/disabled status
-	// associated with the provided ruleset. This is only useful
-	// when you have previously called enable_rule/enable_rule_by_tag
-	// with a ruleset string.
-	//
-	// the returned rule_result is allocated and must be delete()d.
-	std::unique_ptr<rule_result> process_sinsp_event(sinsp_evt *ev, uint16_t ruleset_id);
+	void evttypes_for_ruleset(std::string &source,
+				  std::set<uint16_t> &evttypes,
+				  const std::string &ruleset = s_default_ruleset);
 
 	//
-	// Wrapper assuming the default ruleset
+	// Given a source and output string, return an
+	// gen_event_formatter that can format output strings for an
+	// event.
 	//
-	std::unique_ptr<rule_result> process_sinsp_event(sinsp_evt *ev);
+	std::shared_ptr<gen_event_formatter> create_formatter(const std::string &source,
+							      const std::string &output);
 
-	//
-	// Add a filter, which is related to the specified set of
-	// event types/syscalls, to the engine.
-	//
-	void add_sinsp_filter(std::string &rule,
-			      std::set<uint32_t> &evttypes,
-			      std::set<uint32_t> &syscalls,
-			      std::set<std::string> &tags,
-			      sinsp_filter* filter);
-
-	sinsp_filter_factory &sinsp_factory();
-	json_event_filter_factory &json_factory();
+	// Return whether the provided plugin name + version is
+	// compatible with the current set of loaded rules files.
+	// required_version will be filled in with the required
+	// version when the method returns false.
+	bool is_plugin_compatible(const std::string &name, const std::string &version, std::string &required_version);
 
 private:
 
-	static nlohmann::json::json_pointer k8s_audit_time;
-
 	//
 	// Determine whether the given event should be matched at all
 	// against the set of rules, given the current sampling
 	// ratio/multiplier.
 	//
 	inline bool should_drop_evt();
-	shared_ptr<sinsp_filter_factory> m_sinsp_factory;
-	shared_ptr<json_event_filter_factory> m_json_factory;
 
-	falco_rules *m_rules;
+	// Maps from event source to object that can generate filters from rules
+	std::map<std::string, std::shared_ptr<gen_event_filter_factory>> m_filter_factories;
+
+	// Maps from event source to object that can format output strings in rules
+	std::map<std::string, std::shared_ptr<gen_event_formatter_factory>> m_format_factories;
+
+	// Maps from event source to the set of rules for that event source
+	std::map<std::string, std::shared_ptr<falco_ruleset>> m_rulesets;
+
+	std::unique_ptr<falco_rules> m_rules;
 	uint16_t m_next_ruleset_id;
 	std::map<string, uint16_t> m_known_rulesets;
 	falco_common::priority_type m_min_priority;
 
-	std::unique_ptr<falco_sinsp_ruleset> m_sinsp_rules;
-	std::unique_ptr<falco_ruleset> m_k8s_audit_rules;
+	// Maps from plugin to a list of required plugin versions
+	// found in any loaded rules files.
+	std::map<std::string, std::list<std::string>> m_required_plugin_versions;
 
 	void populate_rule_result(unique_ptr<struct rule_result> &res, gen_event *ev);
 
@@ -289,8 +264,7 @@ private:
 	uint32_t m_sampling_ratio;
 	double m_sampling_multiplier;
 
-	std::string m_lua_main_filename = "rule_loader.lua";
-	std::string m_default_ruleset = "falco-default-ruleset";
+	static const std::string s_default_ruleset;
 	uint32_t m_default_ruleset_id;
 
 	std::string m_extra;

userspace/engine/falco_engine_version.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2020 The Falco Authors.
+Copyright (C) 2021 The Falco Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,11 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// The version of rules/filter fields/etc supported by this falco
+// The version of rules/filter fields/etc supported by this Falco
 // engine.
-#define FALCO_ENGINE_VERSION (9)
+#define FALCO_ENGINE_VERSION (11)
 
 // This is the result of running "falco --list -N | sha256sum" and
-// represents the fields supported by this version of falco. It's used
+// represents the fields supported by this version of Falco. It's used
 // at build time to detect a changed set of fields.
-#define FALCO_FIELDS_CHECKSUM "2f324e2e66d4b423f53600e7e0fcf2f0ff72e4a87755c490f2ae8f310aba9386"
+#define FALCO_FIELDS_CHECKSUM "4de812495f8529ac20bda2b9774462b15911a51df293d59fe9ccb6b922fdeb9d"

userspace/engine/falco_utils.cpp

@@ -27,31 +27,6 @@ namespace falco
 namespace utils
 {
 
-std::string wrap_text(const std::string& str, uint32_t initial_pos, uint32_t indent, uint32_t line_len)
-{
-	std::string ret;
-
-	size_t len = str.size();
-
-	for(uint32_t l = 0; l < len; l++)
-	{
-		if(l % (line_len - indent) == 0 && l != 0)
-		{
-			ret += "\n";
-
-			for(uint32_t m = 0; m < indent; m++)
-			{
-				ret += " ";
-			}
-		}
-		ret += str.at(l);
-	}
-
-	ret += "\n";
-
-	return ret;
-}
-
 uint32_t hardware_concurrency()
 {
 	auto hc = std::thread::hardware_concurrency();

userspace/engine/formats.cpp

@@ -20,169 +20,49 @@ limitations under the License.
 #include "falco_engine.h"
 #include "banned.h" // This raises a compilation error when certain functions are used
 
-sinsp *falco_formats::s_inspector = NULL;
-falco_engine *falco_formats::s_engine = NULL;
-bool falco_formats::s_json_output = false;
-bool falco_formats::s_json_include_output_property = true;
-bool falco_formats::s_json_include_tags_property = true;
-std::unique_ptr<sinsp_evt_formatter_cache> falco_formats::s_formatters = NULL;
-
-const static struct luaL_Reg ll_falco[] =
-	{
-		{"formatter", &falco_formats::lua_formatter},
-		{"free_formatter", &falco_formats::lua_free_formatter},
-		{NULL, NULL}};
-
-void falco_formats::init(sinsp *inspector,
-			 falco_engine *engine,
-			 lua_State *ls,
-			 bool json_output,
-			 bool json_include_output_property,
-			 bool json_include_tags_property)
+falco_formats::falco_formats(falco_engine *engine,
+			     bool json_include_output_property,
+			     bool json_include_tags_property)
+	: m_falco_engine(engine),
+	m_json_include_output_property(json_include_output_property),
+	m_json_include_tags_property(json_include_tags_property)
 {
-	s_inspector = inspector;
-	s_engine = engine;
-	s_json_output = json_output;
-	s_json_include_output_property = json_include_output_property;
-	s_json_include_tags_property = json_include_tags_property;
-
-	// todo(leogr): we should have used std::make_unique, but we cannot since it's not C++14
-	s_formatters = std::unique_ptr<sinsp_evt_formatter_cache>(new sinsp_evt_formatter_cache(s_inspector));
-
-	luaL_openlib(ls, "formats", ll_falco, 0);
 }
 
-int falco_formats::lua_formatter(lua_State *ls)
+falco_formats::~falco_formats()
 {
-	string source = luaL_checkstring(ls, -2);
-	string format = luaL_checkstring(ls, -1);
-
-	try
-	{
-		if(source == "syscall")
-		{
-			sinsp_evt_formatter *formatter;
-			formatter = new sinsp_evt_formatter(s_inspector, format);
-			lua_pushnil(ls);
-			lua_pushlightuserdata(ls, formatter);
-		}
-		else
-		{
-			json_event_formatter *formatter;
-			formatter = new json_event_formatter(s_engine->json_factory(), format);
-			lua_pushnil(ls);
-			lua_pushlightuserdata(ls, formatter);
-		}
-	}
-	catch(exception &e)
-	{
-		std::ostringstream os;
-
-		os << "Invalid output format '"
-		   << format
-		   << "': '"
-		   << e.what()
-		   << "'";
-
-		lua_pushstring(ls, os.str().c_str());
-		lua_pushnil(ls);
-	}
-
-	return 2;
 }
 
-int falco_formats::lua_free_formatter(lua_State *ls)
-{
-	if(!lua_islightuserdata(ls, -1) ||
-	   !lua_isstring(ls, -2))
-
-	{
-		luaL_error(ls, "Invalid argument passed to free_formatter");
-	}
-
-	string source = luaL_checkstring(ls, -2);
-
-	if(source == "syscall")
-	{
-		sinsp_evt_formatter *formatter = (sinsp_evt_formatter *)lua_topointer(ls, -1);
-		delete(formatter);
-	}
-	else
-	{
-		json_event_formatter *formatter = (json_event_formatter *)lua_topointer(ls, -1);
-		delete(formatter);
-	}
-
-	return 0;
-}
-
-string falco_formats::format_event(const gen_event *evt, const std::string &rule, const std::string &source,
+string falco_formats::format_event(gen_event *evt, const std::string &rule, const std::string &source,
 				   const std::string &level, const std::string &format, std::set<std::string> &tags)
 {
-
 	string line;
-	string json_line;
-	string sformat = format;
 
-	if(strcmp(source.c_str(), "syscall") == 0)
+	std::shared_ptr<gen_event_formatter> formatter;
+
+	formatter = m_falco_engine->create_formatter(source, format);
+
+	// Format the original output string, regardless of output format
+	formatter->tostring_withformat(evt, line, gen_event_formatter::OF_NORMAL);
+
+	if(formatter->get_output_format() == gen_event_formatter::OF_JSON)
 	{
-		// This is "output"
-		s_formatters->tostring((sinsp_evt *)evt, sformat, &line);
+		string json_line;
 
-		if(s_json_output)
+		// Format the event into a json object with all fields resolved
+		formatter->tostring(evt, json_line);
+
+		// The formatted string might have a leading newline. If it does, remove it.
+		if(json_line[0] == '\n')
 		{
-			sinsp_evt::param_fmt cur_fmt = s_inspector->get_buffer_format();
-			switch(cur_fmt)
-			{
-			case sinsp_evt::PF_NORMAL:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSON);
-				break;
-			case sinsp_evt::PF_EOLS:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSONEOLS);
-				break;
-			case sinsp_evt::PF_HEX:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSONHEX);
-				break;
-			case sinsp_evt::PF_HEXASCII:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSONHEXASCII);
-				break;
-			case sinsp_evt::PF_BASE64:
-				s_inspector->set_buffer_format(sinsp_evt::PF_JSONBASE64);
-				break;
-			default:
-				// do nothing
-				break;
-			}
-			// This is output fields
-			s_formatters->tostring((sinsp_evt *)evt, sformat, &json_line);
-
-			// The formatted string might have a leading newline. If it does, remove it.
-			if(json_line[0] == '\n')
-			{
-				json_line.erase(0, 1);
-			}
-			s_inspector->set_buffer_format(cur_fmt);
+			json_line.erase(0, 1);
 		}
-	}
-	else
-	{
-		json_event_formatter formatter(s_engine->json_factory(), sformat);
 
-		line = formatter.tostring((json_event *)evt);
-
-		if(s_json_output)
-		{
-			json_line = formatter.tojson((json_event *)evt);
-		}
-	}
-
-	// For JSON output, the formatter returned a json-as-text
-	// object containing all the fields in the original format
-	// message as well as the event time in ns. Use this to build
-	// a more detailed object containing the event time, rule,
-	// severity, full output, and fields.
-	if(s_json_output)
-	{
+		// For JSON output, the formatter returned a json-as-text
+		// object containing all the fields in the original format
+		// message as well as the event time in ns. Use this to build
+		// a more detailed object containing the event time, rule,
+		// severity, full output, and fields.
 		Json::Value event;
 		Json::Value rule_tags;
 		Json::FastWriter writer;
@@ -204,15 +84,15 @@ string falco_formats::format_event(const gen_event *evt, const std::string &rule
 		event["priority"] = level;
 		event["source"] = source;
 
-		if(s_json_include_output_property)
+		if(m_json_include_output_property)
 		{
 			// This is the filled-in output line.
 			event["output"] = line;
 		}
-		
-		if(s_json_include_tags_property)
+
+		if(m_json_include_tags_property)
 		{
-			if (tags.size() == 0) 
+			if (tags.size() == 0)
 			{
 				// This sets an empty array
 				rule_tags = Json::arrayValue;
@@ -249,19 +129,19 @@ string falco_formats::format_event(const gen_event *evt, const std::string &rule
 	return line.c_str();
 }
 
-map<string, string> falco_formats::resolve_tokens(const gen_event *evt, const std::string &source, const std::string &format)
+map<string, string> falco_formats::get_field_values(gen_event *evt, const std::string &source,
+						    const std::string &format)
 {
-	string sformat = format;
-	map<string, string> values;
-	if(source == "syscall")
+	std::shared_ptr<gen_event_formatter> formatter;
+
+	formatter = m_falco_engine->create_formatter(source, format);
+
+	map<string, string> ret;
+
+	if (! formatter->get_field_values(evt, ret))
 	{
-		s_formatters->resolve_tokens((sinsp_evt *)evt, sformat, values);
+		throw falco_exception("Could not extract all field values from event");
 	}
-	// k8s_audit
-	else
-	{
-		json_event_formatter json_formatter(s_engine->json_factory(), sformat);
-		values = json_formatter.tomap((json_event *)evt);
-	}
-	return values;
+
+	return ret;
 }

userspace/engine/formats.h

@@ -16,7 +16,8 @@ limitations under the License.
 
 #pragma once
 
-#include "sinsp.h"
+#include <string>
+#include <map>
 
 extern "C"
 {
@@ -25,37 +26,26 @@ extern "C"
 #include "lauxlib.h"
 }
 
-#include "json_evt.h"
-#include "falco_engine.h"
+#include <gen_filter.h>
 
-class sinsp_evt_formatter;
+#include "falco_engine.h"
 
 class falco_formats
 {
 public:
-	static void init(sinsp *inspector,
-			 falco_engine *engine,
-			 lua_State *ls,
-			 bool json_output,
-			 bool json_include_output_property,
-			 bool json_include_tags_property);
+	falco_formats(falco_engine *engine,
+		      bool json_include_output_property,
+		      bool json_include_tags_property);
+	virtual ~falco_formats();
 
-	// formatter = falco.formatter(format_string)
-	static int lua_formatter(lua_State *ls);
+	std::string format_event(gen_event *evt, const std::string &rule, const std::string &source,
+				 const std::string &level, const std::string &format, std::set<std::string> &tags);
 
-	// falco.free_formatter(formatter)
-	static int lua_free_formatter(lua_State *ls);
+	map<string, string> get_field_values(gen_event *evt, const std::string &source,
+					     const std::string &format);
 
-	static string format_event(const gen_event *evt, const std::string &rule, const std::string &source,
-				   const std::string &level, const std::string &format, std::set<std::string> &tags);
-
-	static map<string, string> resolve_tokens(const gen_event *evt, const std::string &source,
-						  const std::string &format);
-
-	static sinsp *s_inspector;
-	static falco_engine *s_engine;
-	static std::unique_ptr<sinsp_evt_formatter_cache> s_formatters;
-	static bool s_json_output;
-	static bool s_json_include_output_property;
-	static bool s_json_include_tags_property;
+protected:
+	falco_engine *m_falco_engine;
+	bool m_json_include_output_property;
+	bool m_json_include_tags_property;
 };

userspace/engine/json_evt.cpp

@@ -50,6 +50,81 @@ uint64_t json_event::get_ts() const
 	return m_event_ts;
 }
 
+static nlohmann::json::json_pointer k8s_audit_time = "/stageTimestamp"_json_pointer;
+
+bool falco_k8s_audit::parse_k8s_audit_json(nlohmann::json &j, std::list<json_event> &evts, bool top)
+{
+	// Note that nlohmann::basic_json::value can throw  nlohmann::basic_json::type_error (302, 306)
+	try
+	{
+		// If the object is an array, call parse_k8s_audit_json again for each item.
+		if(j.is_array())
+		{
+			if(top)
+			{
+				for(auto &item : j)
+				{
+					// Note we only handle a single top level array, to
+					// avoid excessive recursion.
+					if(! parse_k8s_audit_json(item, evts, false))
+					{
+						return false;
+					}
+				}
+
+				return true;
+			}
+			else
+			{
+				return false;
+			}
+		}
+
+		// If the kind is EventList, split it into individual events
+		if(j.value("kind", "<NA>") == "EventList")
+		{
+			for(auto &je : j["items"])
+			{
+				evts.emplace_back();
+				je["kind"] = "Event";
+
+				uint64_t ns = 0;
+				if(!sinsp_utils::parse_iso_8601_utc_string(je.value(k8s_audit_time, "<NA>"), ns))
+				{
+					return false;
+				}
+
+				std::string tmp;
+				sinsp_utils::ts_to_string(ns, &tmp, false, true);
+
+				evts.back().set_jevt(je, ns);
+			}
+
+			return true;
+		}
+		else if(j.value("kind", "<NA>") == "Event")
+		{
+			evts.emplace_back();
+			uint64_t ns = 0;
+			if(!sinsp_utils::parse_iso_8601_utc_string(j.value(k8s_audit_time, "<NA>"), ns))
+			{
+				return false;
+			}
+
+			evts.back().set_jevt(j, ns);
+			return true;
+		}
+		else
+		{
+			return false;
+		}
+	}
+	catch(exception &e)
+	{
+		return false;
+	}
+}
+
 json_event_value::json_event_value()
 {
 }
@@ -615,7 +690,7 @@ size_t json_event_filter_check::parsed_size()
 	}
 }
 
-json_event_filter_check::check_info &json_event_filter_check::get_fields()
+json_event_filter_check::check_info &json_event_filter_check::get_info()
 {
 	return m_info;
 }
@@ -1357,7 +1432,7 @@ json_event_filter_factory::json_event_filter_factory()
 
 	for(auto &chk : m_defined_checks)
 	{
-		m_info.push_back(chk->get_fields());
+		m_info.push_back(chk->get_info());
 	}
 }
 
@@ -1389,36 +1464,96 @@ gen_event_filter_check *json_event_filter_factory::new_filtercheck(const char *f
 	return NULL;
 }
 
-std::list<json_event_filter_check::check_info> &json_event_filter_factory::get_fields()
+std::list<gen_event_filter_factory::filter_fieldclass_info> json_event_filter_factory::get_fields()
 {
-	return m_info;
+	std::list<gen_event_filter_factory::filter_fieldclass_info> ret;
+
+	// It's not quite copy to the public information, as m_info
+	// has addl info about indexing. That info is added to the
+	// description.
+
+	for(auto &chk: m_defined_checks)
+	{
+		json_event_filter_check::check_info &info = chk->get_info();
+		gen_event_filter_factory::filter_fieldclass_info cinfo;
+
+		cinfo.name = info.m_name;
+		cinfo.desc = info.m_desc;
+		cinfo.shortdesc = info.m_shortdesc;
+
+		for(auto &field : info.m_fields)
+		{
+			gen_event_filter_factory::filter_field_info info;
+			info.name = field.m_name;
+			info.desc = field.m_desc;
+
+			// All json fields work on strings
+			info.data_type = "CHARBUF";
+
+			switch(field.m_idx_mode)
+			{
+			case json_event_filter_check::IDX_REQUIRED:
+			case json_event_filter_check::IDX_ALLOWED:
+				info.tags.insert(json_event_filter_check::s_index_mode_strs[field.m_idx_mode]);
+				break;
+			case json_event_filter_check::IDX_NONE:
+			default:
+				break;
+			};
+
+			cinfo.fields.emplace_back(std::move(info));
+		}
+
+		ret.emplace_back(std::move(cinfo));
+	}
+
+	return ret;
 }
 
-json_event_formatter::json_event_formatter(json_event_filter_factory &json_factory, std::string &format):
-	m_format(format),
-	m_json_factory(json_factory)
+json_event_formatter::json_event_formatter(std::shared_ptr<gen_event_filter_factory> json_factory)
+	: m_output_format(OF_NORMAL), m_json_factory(json_factory)
 {
-	parse_format();
 }
 
 json_event_formatter::~json_event_formatter()
 {
 }
 
-std::string json_event_formatter::tostring(json_event *ev)
+void json_event_formatter::set_format(output_format of, const std::string &format)
 {
-	std::string ret;
+	m_output_format = of;
+	m_format = format;
+	parse_format();
+}
 
-	std::list<std::pair<std::string, std::string>> resolved;
-
-	resolve_tokens(ev, resolved);
-
-	for(auto &res : resolved)
+bool json_event_formatter::tostring_withformat(gen_event *gevt, std::string &output, gen_event_formatter::output_format of)
+{
+	json_event *ev = static_cast<json_event *>(gevt);
+	if(of == OF_JSON)
 	{
-		ret += res.second;
+		output = tojson(ev);
+		return true;
 	}
+	else
+	{
+		std::list<std::pair<std::string, std::string>> resolved;
 
-	return ret;
+		resolve_format(ev, resolved);
+
+		output = "";
+
+		for(auto &res : resolved)
+		{
+			output += res.second;
+		}
+
+		return true;
+	}
+}
+
+bool json_event_formatter::tostring(gen_event *gevt, std::string &output)
+{
+	return tostring_withformat(gevt, output, m_output_format);
 }
 
 std::string json_event_formatter::tojson(json_event *ev)
@@ -1428,7 +1563,7 @@ std::string json_event_formatter::tojson(json_event *ev)
 
 	std::list<std::pair<std::string, std::string>> resolved;
 
-	resolve_tokens(ev, resolved);
+	resolve_format(ev, resolved);
 
 	for(auto &res : resolved)
 	{
@@ -1443,12 +1578,13 @@ std::string json_event_formatter::tojson(json_event *ev)
 	return ret.dump();
 }
 
-std::map<std::string, std::string> json_event_formatter::tomap(json_event *ev)
+bool json_event_formatter::get_field_values(gen_event *gevt, std::map<std::string, std::string> &fields)
 {
-	std::map<std::string, std::string> ret;
+	json_event *ev = static_cast<json_event *>(gevt);
+
 	std::list<std::pair<std::string, std::string>> res;
 
-	resolve_tokens(ev, res);
+	resolve_format(ev, res);
 
 	for(auto &r : res)
 	{
@@ -1459,11 +1595,11 @@ std::map<std::string, std::string> json_event_formatter::tomap(json_event *ev)
 			{
 				r.second = "<NA>";
 			}
-			ret.insert(r);
+			fields.insert(r);
 		}
 	}
 
-	return ret;
+	return true;
 }
 
 void json_event_formatter::parse_format()
@@ -1485,7 +1621,7 @@ void json_event_formatter::parse_format()
 		{
 			// Skip the %
 			tformat.erase(0, 1);
-			json_event_filter_check *chk = (json_event_filter_check *)m_json_factory.new_filtercheck(tformat.c_str());
+			json_event_filter_check *chk = (json_event_filter_check *)m_json_factory->new_filtercheck(tformat.c_str());
 
 			if(!chk)
 			{
@@ -1521,7 +1657,7 @@ void json_event_formatter::parse_format()
 	}
 }
 
-void json_event_formatter::resolve_tokens(json_event *ev, std::list<std::pair<std::string, std::string>> &resolved)
+void json_event_formatter::resolve_format(json_event *ev, std::list<std::pair<std::string, std::string>> &resolved)
 {
 	for(auto tok : m_tokens)
 	{
@@ -1561,3 +1697,43 @@ void json_event_formatter::resolve_tokens(json_event *ev, std::list<std::pair<st
 		}
 	}
 }
+
+gen_event_formatter::output_format json_event_formatter::get_output_format()
+{
+	return m_output_format;
+}
+
+json_event_formatter_factory::json_event_formatter_factory(std::shared_ptr<gen_event_filter_factory> json_factory)
+	: m_output_format(gen_event_formatter::OF_NORMAL),  m_json_factory(json_factory)
+{
+}
+
+json_event_formatter_factory::~json_event_formatter_factory()
+{
+}
+
+void json_event_formatter_factory::set_output_format(gen_event_formatter::output_format of)
+{
+	m_formatters.clear();
+
+	m_output_format = of;
+}
+
+std::shared_ptr<gen_event_formatter> json_event_formatter_factory::create_formatter(const std::string &format)
+{
+	auto it = m_formatters.find(format);
+
+	if (it != m_formatters.end())
+	{
+		return it->second;
+	}
+
+	std::shared_ptr<gen_event_formatter> ret;
+
+	ret.reset(new json_event_formatter(m_json_factory));
+
+	ret->set_format(m_output_format, format);
+	m_formatters[format] = ret;
+
+	return ret;
+}

userspace/engine/json_evt.h

@@ -57,6 +57,19 @@ protected:
 	uint64_t m_event_ts;
 };
 
+namespace falco_k8s_audit {
+
+	//
+	// Given a raw json object, return a list of k8s audit event
+	// objects that represent the object. This method handles
+	// things such as EventList splitting.
+	//
+	// Returns true if the json object was recognized as a k8s
+	// audit event(s), false otherwise.
+	//
+	bool parse_k8s_audit_json(nlohmann::json &j, std::list<json_event> &evts, bool top=true);
+};
+
 // A class representing an extracted value or a value on the rhs of a
 // filter_check. This intentionally doesn't use the same types as
 // ppm_events_public.h to take advantage of actual classes instead of
@@ -153,8 +166,8 @@ public:
 	struct check_info
 	{
 		std::string m_name;
+		std::string m_shortdesc;
 		std::string m_desc;
-		std::string m_class_info;
 
 		std::list<field_info> m_fields;
 	};
@@ -176,7 +189,7 @@ public:
 	// brackets (e.g. ka.image[foo])
 	size_t parsed_size();
 
-	check_info &get_fields();
+	check_info &get_info();
 
 	//
 	// Allocate a new check of the same type. Must be overridden.
@@ -386,29 +399,41 @@ public:
 	gen_event_filter_check *new_filtercheck(const char *fldname);
 
 	// All defined field names
-	std::list<json_event_filter_check::check_info> &get_fields();
+	std::list<gen_event_filter_factory::filter_fieldclass_info> get_fields();
 
 private:
 	std::list<std::shared_ptr<json_event_filter_check>> m_defined_checks;
 	std::list<json_event_filter_check::check_info> m_info;
 };
 
-// Unlike the other classes, this does not inherit from a shared class
-// that's used both by json events and sinsp events. It might be
-// worthwhile, but it would require a lot of additional work to pull
-// up functionality into the generic filtercheck class.
-
-class json_event_formatter
+class json_event_formatter : public gen_event_formatter
 {
 public:
-	json_event_formatter(json_event_filter_factory &factory, std::string &format);
+	json_event_formatter(std::shared_ptr<gen_event_filter_factory> factory);
 	virtual ~json_event_formatter();
 
-	std::string tostring(json_event *ev);
-	std::string tojson(json_event *ev);
-	std::map<std::string, std::string> tomap(json_event *ev);
+	void set_format(output_format of, const std::string &format) override;
+	bool tostring(gen_event *evt, std::string &output) override;
+	bool tostring_withformat(gen_event *evt, std::string &output, gen_event_formatter::output_format of) override;
+	bool get_field_values(gen_event *evt, std::map<std::string, std::string> &fields) override;
+	output_format get_output_format() override;
 
-	void resolve_tokens(json_event *ev, std::list<std::pair<std::string, std::string>> &resolved);
+	std::string tojson(json_event *ev);
+
+	// Split the format string into a list of tuples, broken at
+	// output fields, where each tuple is either a block of text
+	// from the original format string, or a field value/pair from
+	// the original format string.
+	//
+	// For example, given a format string "some output
+	// (%proc.name)", this will fill in resolved with 3 tuples:
+	// - ["", "some output ("]
+	// - ["proc.name", "nginx"]
+	// - ["", ")"]
+	//
+	// This can be used either to return a resolved output string
+	// or a map of field name/value pairs.
+        void resolve_format(json_event *ev, std::list<std::pair<std::string, std::string>> &resolved);
 
 private:
 	void parse_format();
@@ -428,6 +453,8 @@ private:
 		std::shared_ptr<json_event_filter_check> check;
 	};
 
+	gen_event_formatter::output_format m_output_format;
+
 	// The original format string
 	std::string m_format;
 
@@ -436,5 +463,25 @@ private:
 	std::list<fmt_token> m_tokens;
 
 	// All the filterchecks required to resolve tokens in the format string
-	json_event_filter_factory &m_json_factory;
+	std::shared_ptr<gen_event_filter_factory> m_json_factory;
+};
+
+class json_event_formatter_factory : public gen_event_formatter_factory
+{
+public:
+	json_event_formatter_factory(std::shared_ptr<gen_event_filter_factory> factory);
+	virtual ~json_event_formatter_factory();
+
+	void set_output_format(gen_event_formatter::output_format of) override;
+
+	std::shared_ptr<gen_event_formatter> create_formatter(const std::string &format) override;
+
+protected:
+	// Maps from output string to formatter
+	std::map<std::string, std::shared_ptr<gen_event_formatter>> m_formatters;
+
+	gen_event_formatter::output_format m_output_format;
+
+	// All the filterchecks required to resolve tokens in the format string
+	std::shared_ptr<gen_event_filter_factory> m_json_factory;
 };

userspace/engine/lua/CMakeLists.txt

@@ -0,0 +1,21 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+file(GLOB_RECURSE lua_files ${CMAKE_CURRENT_SOURCE_DIR} *.lua)
+
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/falco_engine_lua_files.cpp
+	COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/lua-to-cpp.sh ${CMAKE_CURRENT_SOURCE_DIR} ${LYAML_LUA_DIR} ${CMAKE_CURRENT_BINARY_DIR}
+	DEPENDS ${lua_files} ${CMAKE_CURRENT_SOURCE_DIR}/lua-to-cpp.sh lyaml)
+
+add_library(luafiles falco_engine_lua_files.cpp)
+
+target_include_directories(luafiles PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

userspace/engine/lua/lua-to-cpp.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -euo pipefail
+
+LUA_FILE_DIR=$1
+LYAML_LUA_DIR=$2
+OUTPUT_DIR=$3
+
+MODULE_SYMS=()
+CODE_SYMS=()
+
+function add_lua_file {
+    filename=$1
+    is_module=$2
+
+    # Take the basename of the file
+    BASE_NAME=$(basename ${file} .lua)
+    SYMBOL_NAME="${BASE_NAME}_lua_file_contents"
+    FILE_CONTENTS=$(<${file})
+
+    # Add a symbol to the .cc file containing the contents of the file
+    echo "const char *${SYMBOL_NAME}=R\"LUAFILE(${FILE_CONTENTS})LUAFILE\";" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp
+
+    # Add an extern reference to the .hh file
+    echo "extern const char *${SYMBOL_NAME};" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh
+
+    if [[ "${is_module}" == "true" ]]; then
+	# Determine the module name for the file
+	if [[ "${file}" == *"/"* ]]; then
+	    MODULE_NAME=$(echo ${file} | tr / . | sed -e 's/.lua//')
+	else
+	    MODULE_NAME=$(basename ${file} .lua)
+	fi
+
+	# Add the pair (string contents, module name) to MODULE_SYMS
+	PAIR=$(echo "{${SYMBOL_NAME},\"${MODULE_NAME}\"}")
+	MODULE_SYMS+=(${PAIR})
+    else
+	# Add the string to CODE_SYMS
+	CODE_SYMS+=(${SYMBOL_NAME})
+    fi
+}
+
+cat <<EOF > ${OUTPUT_DIR}/falco_engine_lua_files.cpp
+// Automatically generated. Do not edit
+#include "falco_engine_lua_files.hh"
+EOF
+
+cat <<EOF > ${OUTPUT_DIR}/falco_engine_lua_files.hh
+#pragma once
+// Automatically generated. Do not edit
+#include <list>
+#include <utility>
+EOF
+
+# lyaml and any files in the "modules" subdirectory are treated as lua
+# modules.
+pushd ${LYAML_LUA_DIR}
+for file in *.lua */*.lua; do
+    add_lua_file $file "true"
+done
+popd
+
+pushd ${LUA_FILE_DIR}/modules
+for file in *.lua; do
+    add_lua_file $file "true"
+done
+popd
+
+# Any .lua files in this directory are treated as code with functions
+# to execute.
+pushd ${LUA_FILE_DIR}
+for file in ${LUA_FILE_DIR}/*.lua; do
+    add_lua_file $file "false"
+done
+popd
+
+# Create a list of lua module (string, module name) pairs from MODULE_SYMS
+echo "extern std::list<std::pair<const char *,const char *>> lua_module_strings;" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh
+echo "std::list<std::pair<const char *,const char *>> lua_module_strings = {$(IFS=, ; echo "${MODULE_SYMS[*]}")};" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp
+
+# Create a list of lua code strings from CODE_SYMS
+echo "extern std::list<const char *> lua_code_strings;" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh
+echo "std::list<const char *> lua_code_strings = {$(IFS=, ; echo "${CODE_SYMS[*]}")};" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp

userspace/engine/lua/modules/parser.lua

@@ -18,7 +18,7 @@
 
    Much of the scaffolding and helpers was derived from Andre Murbach Maidl's Lua parser (https://github.com/andremm/lua-parser).
 
-   While this is based on the sysdig filtering syntax (*), the Falco syntax is extended to support "macro" terms, which are just identifiers.
+   While this is based on the falcosecurity-libs filtering syntax (*), the Falco syntax is extended to support "macro" terms, which are just identifiers.
 
    (*) There is currently one known difference with the syntax implemented in libsinsp: In libsinsp, field names cannot start with 'a', 'o', or 'n'. With this parser they can.
 
@@ -196,9 +196,9 @@ local G = {
    Identifier = V "idStart" * V "idRest" ^ 0,
    Macro = V "idStart" * V "idRest" ^ 0 * -P ".",
    Int = digit ^ 1,
-   PathString = (alnum + S ",.-_/*?") ^ 1,
+   ArgString = (alnum + S ",.-_/*?~") ^ 1,
    PortRangeString = (V "Int" + S ":,") ^ 1,
-   Index = V "PortRangeString" + V "Int" + V "PathString",
+   Index = V "PortRangeString" + V "Int" + V "ArgString",
    FieldName = V "Identifier" * (P "." + V "Identifier") ^ 1 * (P "[" * V "Index" * P "]") ^ -1,
    Name = C(V "Identifier") * -V "idRest",
    Hex = (P("0x") + P("0X")) * xdigit ^ 1,

userspace/engine/lua/rule_loader.lua

@@ -20,7 +20,6 @@
 
 --]]
 
-local sinsp_rule_utils = require "sinsp_rule_utils"
 local compiler = require "compiler"
 local yaml = require"lyaml"
 
@@ -69,7 +68,7 @@ priorities = {
 --[[
    Take a filter AST and set it up in the libsinsp runtime, using the filter API.
 --]]
-local function install_filter(node, filter_api_lib, lua_parser, parent_bool_op)
+local function create_filter_obj(node, lua_parser, parent_bool_op)
    local t = node.type
 
    if t == "BinaryBoolOp" then
@@ -78,41 +77,84 @@ local function install_filter(node, filter_api_lib, lua_parser, parent_bool_op)
       -- never necessary when we have identical successive operators. so we
       -- avoid it as a runtime performance optimization.
       if (not(node.operator == parent_bool_op)) then
-	 filter_api_lib.nest(lua_parser) -- io.write("(")
+	 err = filter.nest(lua_parser) -- io.write("(")
+	 if err ~= nil then
+	    return err
+	 end
       end
 
-      install_filter(node.left, filter_api_lib, lua_parser, node.operator)
-      filter_api_lib.bool_op(lua_parser, node.operator) -- io.write(" "..node.operator.." ")
-      install_filter(node.right, filter_api_lib, lua_parser, node.operator)
+      err = create_filter_obj(node.left, lua_parser, node.operator)
+      if err ~= nil then
+	 return err
+      end
+
+      err = filter.bool_op(lua_parser, node.operator) -- io.write(" "..node.operator.." ")
+      if err ~= nil then
+	 return err
+      end
+
+      err = create_filter_obj(node.right, lua_parser, node.operator)
+      if err ~= nil then
+	 return err
+      end
 
       if (not (node.operator == parent_bool_op)) then
-	 filter_api_lib.unnest(lua_parser) -- io.write(")")
+	 err = filter.unnest(lua_parser) -- io.write(")")
+	 if err ~= nil then
+	    return err
+	 end
       end
 
    elseif t == "UnaryBoolOp" then
-      filter_api_lib.nest(lua_parser) --io.write("(")
-      filter_api_lib.bool_op(lua_parser, node.operator) -- io.write(" "..node.operator.." ")
-      install_filter(node.argument, filter_api_lib, lua_parser)
-      filter_api_lib.unnest(lua_parser) -- io.write(")")
+      err = filter.nest(lua_parser) --io.write("(")
+      if err ~= nil then
+	 return err
+      end
+
+      err = filter.bool_op(lua_parser, node.operator) -- io.write(" "..node.operator.." ")
+      if err ~= nil then
+	 return err
+      end
+
+      err = create_filter_obj(node.argument, lua_parser)
+      if err ~= nil then
+	 return err
+      end
+
+      err = filter.unnest(lua_parser) -- io.write(")")
+      if err ~= nil then
+	 return err
+      end
 
    elseif t == "BinaryRelOp" then
       if (node.operator == "in" or
           node.operator == "intersects" or
 	  node.operator == "pmatch") then
 	 elements = map(function (el) return el.value end, node.right.elements)
-	 filter_api_lib.rel_expr(lua_parser, node.left.value, node.operator, elements, node.index)
+	 err = filter.rel_expr(lua_parser, node.left.value, node.operator, elements, node.index)
+	 if err ~= nil then
+	    return err
+	 end
       else
-	 filter_api_lib.rel_expr(lua_parser, node.left.value, node.operator, node.right.value, node.index)
+	 err = filter.rel_expr(lua_parser, node.left.value, node.operator, node.right.value, node.index)
+	 if err ~= nil then
+	    return err
+	 end
       end
       -- io.write(node.left.value.." "..node.operator.." "..node.right.value)
 
    elseif t == "UnaryRelOp"  then
-      filter_api_lib.rel_expr(lua_parser, node.argument.value, node.operator, node.index)
+      err = filter.rel_expr(lua_parser, node.argument.value, node.operator, node.index)
+      if err ~= nil then
+	 return err
+      end
       --io.write(node.argument.value.." "..node.operator)
 
    else
-      error ("Unexpected type in install_filter: "..t)
+      return "Unexpected type in create_filter_obj: "..t
    end
+
+   return nil
 end
 
 function set_output(output_format, state)
@@ -251,19 +293,20 @@ function split_lines(rules_content)
 end
 
 function get_orig_yaml_obj(rules_lines, row)
-   local ret = ""
+    idx = row
+    local t = {}
+    while (idx <= #rules_lines) do
+        t[#t + 1] = rules_lines[idx]
+        idx = idx + 1
 
-   idx = row
-   while (idx <= #rules_lines) do
-      ret = ret..rules_lines[idx].."\n"
-      idx = idx + 1
-
-      if idx > #rules_lines or rules_lines[idx] == "" or string.sub(rules_lines[idx], 1, 1) == '-' then
-	 break
-      end
-   end
-
-   return ret
+        if idx > #rules_lines or rules_lines[idx] == "" or string.sub(rules_lines[idx], 1, 1) == '-' then
+            break
+        end
+    end
+    t[#t + 1] = ""
+    local ret = ""
+    ret = table.concat(t, "\n")
+    return ret
 end
 
 function get_lines(rules_lines, row, num_lines)
@@ -310,7 +353,7 @@ function build_error_with_context(ctx, err)
    return {ret}
 end
 
-function validate_exception_item_multi_fields(eitem, context)
+function validate_exception_item_multi_fields(rules_mgr, source, eitem, context)
 
    local name = eitem['name']
    local fields = eitem['fields']
@@ -329,7 +372,7 @@ function validate_exception_item_multi_fields(eitem, context)
       end
    end
    for k, fname in ipairs(fields) do
-      if not is_defined_filter(fname) then
+      if not falco_rules.is_defined_field(rules_mgr, source, fname) then
 	 return false, build_error_with_context(context, "Rule exception item "..name..": field name "..fname.." is not a supported filter field"), warnings
       end
    end
@@ -340,7 +383,7 @@ function validate_exception_item_multi_fields(eitem, context)
    end
 end
 
-function validate_exception_item_single_field(eitem, context)
+function validate_exception_item_single_field(rules_mgr, source, eitem, context)
 
    local name = eitem['name']
    local fields = eitem['fields']
@@ -355,7 +398,7 @@ function validate_exception_item_single_field(eitem, context)
 	 return false, build_error_with_context(context, "Rule exception item "..name..": fields and comps must both be strings"), warnings
       end
    end
-   if not is_defined_filter(fields) then
+   if not falco_rules.is_defined_field(rules_mgr, source, fields) then
       return false, build_error_with_context(context, "Rule exception item "..name..": field name "..fields.." is not a supported filter field"), warnings
    end
    if defined_comp_operators[comps] == nil then
@@ -363,37 +406,6 @@ function validate_exception_item_single_field(eitem, context)
    end
 end
 
-function is_defined_filter(filter)
-   if defined_noarg_filters[filter] ~= nil then
-      return true
-   else
-      bracket_idx = string.find(filter, "[", 1, true)
-
-      if bracket_idx ~= nil then
-	 subfilter = string.sub(filter, 1, bracket_idx-1)
-
-	 if defined_arg_filters[subfilter] ~= nil then
-	    return true
-	 end
-      end
-
-      dot_idx = string.find(filter, ".", 1, true)
-
-      while dot_idx ~= nil do
-	 subfilter = string.sub(filter, 1, dot_idx-1)
-
-	 if defined_arg_filters[subfilter] ~= nil then
-	    return true
-	 end
-
-	 dot_idx = string.find(filter, ".", dot_idx+1, true)
-      end
-   end
-
-   return false
-end
-
-
 function load_rules_doc(rules_mgr, doc, load_state)
 
    local warnings = {}
@@ -426,6 +438,29 @@ function load_rules_doc(rules_mgr, doc, load_state)
 	    return false, build_error_with_context(v['context'], "Rules require engine version "..v['required_engine_version']..", but engine version is "..falco_rules.engine_version(rules_mgr)), warnings
 	 end
 
+      elseif (v['required_plugin_versions']) then
+
+	 for _, vobj in ipairs(v['required_plugin_versions']) do
+	    if vobj['name'] == nil then
+	       return false, build_error_with_context(v['context'], "required_plugin_versions item must have name property"), warnings
+	    end
+
+	    if vobj['version'] == nil then
+	       return false, build_error_with_context(v['context'], "required_plugin_versions item must have version property"), warnings
+	    end
+
+	    -- In the rules yaml, it's a name + version. But it's
+	    -- possible, although unlikely, that a single yaml blob
+	    -- contains multiple docs, withe each doc having its own
+	    -- required_engine_version entry. So populate a map plugin
+	    -- name -> list of required plugin versions.
+	    if load_state.required_plugin_versions[vobj['name']] == nil then
+	       load_state.required_plugin_versions[vobj['name']] = {}
+	    end
+
+	    table.insert(load_state.required_plugin_versions[vobj['name']], vobj['version'])
+	 end
+
       elseif (v['macro']) then
 
 	 if (v['macro'] == nil or type(v['macro']) == "table") then
@@ -508,10 +543,6 @@ function load_rules_doc(rules_mgr, doc, load_state)
 	    return false, build_error_with_context(v['context'], "Rule name is empty"), warnings
 	 end
 
-	 if (v['condition'] == nil and v['exceptions'] == nil) then
-	    return false, build_error_with_context(v['context'], "Rule must have exceptions or condition property"), warnings
-	 end
-
 	 -- By default, if a rule's condition refers to an unknown
 	 -- filter like evt.type, etc the loader throws an error.
 	 if v['skip-if-unknown-filter'] == nil then
@@ -558,9 +589,9 @@ function load_rules_doc(rules_mgr, doc, load_state)
 		  -- Different handling if the fields property is a single item vs a list
 		  local valid, err
 		  if type(eitem['fields']) == "table" then
-		     valid, err = validate_exception_item_multi_fields(eitem, v['context'])
+		     valid, err = validate_exception_item_multi_fields(rules_mgr, v['source'], eitem, v['context'])
 		  else
-		     valid, err = validate_exception_item_single_field(eitem, v['context'])
+		     valid, err = validate_exception_item_single_field(rules_mgr, v['source'], eitem, v['context'])
 		  end
 
 		  if valid == false then
@@ -577,45 +608,82 @@ function load_rules_doc(rules_mgr, doc, load_state)
 		  return false, build_error_with_context(v['context'], "Rule " ..v['rule'].. " has 'append' key but no rule by that name already exists"), warnings
 	       end
 	    else
+          if (v['condition'] == nil and next(v['exceptions']) == nil) then
+             return false, build_error_with_context(v['context'], "Appended rule must have exceptions or condition property"), warnings
+          end
 
 	       if next(v['exceptions']) ~= nil then
 
 		  for _, eitem in ipairs(v['exceptions']) do
-		     local name = eitem['name']
-		     local fields = eitem['fields']
-		     local comps = eitem['comps']
 
-		     if name == nil then
+		     if eitem['name'] == nil then
 			return false, build_error_with_context(v['context'], "Rule exception item must have name property"), warnings
 		     end
 
-		     -- You can't append exception fields or comps to a rule
-		     if fields ~= nil then
-			return false, build_error_with_context(v['context'], "Can not append exception fields to existing rule, only values"), warnings
+		     -- Seperate case when a exception name is not found
+		     -- This means that a new exception is being appended
+
+		     local new_exception = true
+		     for _, rex_item in ipairs(state.rules_by_name[v['rule']]['exceptions']) do
+			if rex_item['name'] == eitem['name'] then
+			   new_exception = false
+			   break
+			end
 		     end
 
-		     if comps ~= nil then
-			return false, build_error_with_context(v['context'], "Can not append exception comps to existing rule, only values"), warnings
-		     end
+		     if new_exception then
+			local exceptions = state.rules_by_name[v['rule']]['exceptions']
+			
+			if eitem['fields'] == nil then
+			   return false, build_error_with_context(v['context'], "Rule exception new item "..eitem['name']..": must have fields property with a list of fields"), warnings
+			end
+			if eitem['values'] == nil then
+			   return false, build_error_with_context(v['context'], "Rule exception new item "..eitem['name']..": must have values property with a list of values"), warnings
+			end
+			
+			local valid, err
+			if type(eitem['fields']) == "table" then
+			   valid, err = validate_exception_item_multi_fields(rules_mgr, v['source'], eitem, v['context'])
+			else
+			   valid, err = validate_exception_item_single_field(rules_mgr, v['source'], eitem, v['context'])
+			end
+			
+			if valid == false then
+			   return valid, err, warnings
+			end
 
-		     -- You can append values. They are added to the
-		     -- corresponding name, if it exists. If no
-		     -- exception with that name exists, add a
-		     -- warning.
-		     if eitem['values'] ~= nil then
-			local found=false
-			for _, reitem in ipairs(state.rules_by_name[v['rule']]['exceptions']) do
-			   if reitem['name'] == eitem['name'] then
-			      found=true
-			      for _, values in ipairs(eitem['values']) do
-				 reitem['values'][#reitem['values'] + 1] = values
+			-- Insert the complete exception object
+			exceptions[#exceptions+1] = eitem
+		     else
+			-- Appends to existing exception here
+		   	-- You can't append exception fields or comps to an existing rule exception
+                        if eitem['fields'] ~= nil then
+			   return false, build_error_with_context(v['context'], "Can not append exception fields to existing rule, only values"), warnings
+                        end
+
+                        if eitem['comps'] ~= nil then
+                           return false, build_error_with_context(v['context'], "Can not append exception comps to existing rule, only values"), warnings
+                        end
+
+		     	-- You can append values. They are added to the
+		     	-- corresponding name, if it exists. If no
+		     	-- exception with that name exists, add a
+		     	-- warning.
+		     	if eitem['values'] ~= nil then
+			   local found=false
+			   for _, reitem in ipairs(state.rules_by_name[v['rule']]['exceptions']) do
+			      if reitem['name'] == eitem['name'] then
+			         found=true
+			         for _, values in ipairs(eitem['values']) do
+				    reitem['values'][#reitem['values'] + 1] = values
+			         end
 			      end
 			   end
-			end
 
-			if found == false then
-			   warnings[#warnings + 1] = "Rule "..v['rule'].." with append=true: no set of fields matching name "..eitem['name']
-			end
+			   if found == false then
+			      warnings[#warnings + 1] = "Rule "..v['rule'].." with append=true: no set of fields matching name "..eitem['name']
+			   end
+		        end
 		     end
 		  end
 	       end
@@ -629,14 +697,25 @@ function load_rules_doc(rules_mgr, doc, load_state)
 	    end
 
 	 else
-
+       local err = nil
 	    for j, field in ipairs({'condition', 'output', 'desc', 'priority'}) do
-	       if (v[field] == nil) then
-		  return false, build_error_with_context(v['context'], "Rule must have property "..field), warnings
+	       if (err == nil and v[field] == nil) then
+		       err = build_error_with_context(v['context'], "Rule must have property "..field)
 	       end
 	    end
 
-	    -- Convert the priority-as-string to a priority-as-number now
+       -- Handle spacial case where "enabled" flag is defined only
+       if (err ~= nil) then
+          if (v['enabled'] == nil) then
+            return false, err, warnings
+          else 
+             if state.rules_by_name[v['rule']] == nil then
+                return false, build_error_with_context(v['context'], "Rule " ..v['rule'].. " has 'enabled' key only, but no rule by that name already exists"), warnings
+             end
+             state.rules_by_name[v['rule']]['enabled'] = v['enabled']
+          end
+       else
+       -- Convert the priority-as-string to a priority-as-number now
 	    v['priority_num'] = priorities[v['priority']]
 
 	    if v['priority_num'] == nil then
@@ -659,6 +738,7 @@ function load_rules_doc(rules_mgr, doc, load_state)
 	    else
 	       state.skipped_rules_by_name[v['rule']] = v
 	    end
+       end
 	 end
       else
 	 local context = v['context']
@@ -675,65 +755,69 @@ end
 -- Populates exfields with all fields used
 function build_exception_condition_string_multi_fields(eitem, exfields)
 
-   local fields = eitem['fields']
-   local comps = eitem['comps']
+    local fields = eitem['fields']
+    local comps = eitem['comps']
 
-   local icond = "("
+    local icond = {}
 
-   for i, values in ipairs(eitem['values']) do
+    icond[#icond + 1] = "("
 
-      if #fields ~= #values then
-	 return nil, "Exception item "..eitem['name']..": fields and values lists must have equal length"
-      end
+    local lcount = 0
+    for i, values in ipairs(eitem['values']) do
+        if #fields ~= #values then
+            return nil, "Exception item " .. eitem['name'] .. ": fields and values lists must have equal length"
+        end
 
-      if icond ~= "(" then
-	 icond=icond.." or "
-      end
+        if lcount ~= 0 then
+            icond[#icond + 1] = " or "
+        end
+        lcount = lcount + 1
 
-      icond=icond.."("
+        icond[#icond + 1] = "("
 
-      for k=1,#fields do
-	 if k > 1 then
-	    icond=icond.." and "
-	 end
-	 local ival = values[k]
-	 local istr = ""
+        for k = 1, #fields do
+            if k > 1 then
+                icond[#icond + 1] = " and "
+            end
+            local ival = values[k]
+            local istr = ""
 
-	 -- If ival is a table, express it as (titem1, titem2, etc)
-	 if type(ival) == "table" then
-	    istr = "("
-	    for _, item in ipairs(ival) do
-	       if istr ~= "(" then
-		  istr = istr..", "
-	       end
-	       istr = istr..quote_item(item)
-	    end
-	    istr = istr..")"
-	 else
-	    -- If the corresponding operator is one that works on lists, possibly add surrounding parentheses.
-	    if defined_list_comp_operators[comps[k]] then
-	       istr = paren_item(ival)
-	    else
-	       -- Quote the value if not already quoted
-	       istr = quote_item(ival)
-	    end
-	 end
+            -- If ival is a table, express it as (titem1, titem2, etc)
+            if type(ival) == "table" then
+                istr = "("
+                for _, item in ipairs(ival) do
+                    if istr ~= "(" then
+                        istr = istr .. ", "
+                    end
+                    istr = istr .. quote_item(item)
+                end
+                istr = istr .. ")"
+            else
+                -- If the corresponding operator is one that works on lists, possibly add surrounding parentheses.
+                if defined_list_comp_operators[comps[k]] then
+                    istr = paren_item(ival)
+                else
+                    -- Quote the value if not already quoted
+                    istr = quote_item(ival)
+                end
+            end
 
-	 icond = icond..fields[k].." "..comps[k].." "..istr
-	 exfields[fields[k]] = true
-      end
+            icond[#icond + 1] = fields[k] .. " " .. comps[k] .. " " .. istr
+            exfields[fields[k]] = true
+        end
 
-      icond=icond..")"
-   end
+        icond[#icond + 1] = ")"
+    end
 
-   icond = icond..")"
+    icond[#icond + 1] = ")"
 
-   -- Don't return a trivially empty condition string
-   if icond == "()" then
-      icond = ""
-   end
+    -- Don't return a trivially empty condition string
+    local ret = table.concat(icond)
+    if ret == "()" then
+        return "", nil
+    end
 
-   return icond, nil
+    return ret, nil
 
 end
 
@@ -769,11 +853,10 @@ end
 -- Returns:
 -- - Load Result: bool
 -- - required engine version. will be nil when load result is false
+-- - required_plugin_versions. will be nil when load_result is false
 -- - List of Errors
 -- - List of Warnings
-function load_rules(sinsp_lua_parser,
-		    json_lua_parser,
-		    rules_content,
+function load_rules(rules_content,
 		    rules_mgr,
 		    verbose,
 		    all_events,
@@ -783,7 +866,7 @@ function load_rules(sinsp_lua_parser,
 
    local warnings = {}
 
-   local load_state = {lines={}, indices={}, cur_item_idx=0, min_priority=min_priority, required_engine_version=0}
+   local load_state = {lines={}, indices={}, cur_item_idx=0, min_priority=min_priority, required_engine_version=0, required_plugin_versions={}}
 
    load_state.lines, load_state.indices = split_lines(rules_content)
 
@@ -804,29 +887,29 @@ function load_rules(sinsp_lua_parser,
       row = tonumber(row)
       col = tonumber(col)
 
-      return false, nil, build_error(load_state.lines, row, 3, docs), warnings
+      return false, nil, nil, build_error(load_state.lines, row, 3, docs), warnings
    end
 
    if docs == nil then
       -- An empty rules file is acceptable
-      return true, load_state.required_engine_version, {}, warnings
+      return true, load_state.required_engine_version, {}, {}, warnings
    end
 
    if type(docs) ~= "table" then
-      return false, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml"), warnings
+      return false, nil, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml"), warnings
    end
 
    for docidx, doc in ipairs(docs) do
 
       if type(doc) ~= "table" then
-	 return false, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml"), warnings
+	 return false, nil, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml"), warnings
       end
 
       -- Look for non-numeric indices--implies that document is not array
       -- of objects.
       for key, val in pairs(doc) do
 	 if type(key) ~= "number" then
-	    return false, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml array of objects"), warnings
+	    return false, nil, nil, build_error(load_state.lines, 1, 1, "Rules content is not yaml array of objects"), warnings
 	 end
       end
 
@@ -839,7 +922,7 @@ function load_rules(sinsp_lua_parser,
       end
 
       if not res then
-	 return res, nil, errors, warnings
+	 return res, nil, nil, errors, warnings
       end
    end
 
@@ -880,13 +963,7 @@ function load_rules(sinsp_lua_parser,
       local status, ast = compiler.compile_macro(v['condition'], state.macros, state.lists)
 
       if status == false then
-	 return false, nil, build_error_with_context(v['context'], ast), warnings
-      end
-
-      if v['source'] == "syscall" then
-	 if not all_events then
-	    sinsp_rule_utils.check_for_ignored_syscalls_events(ast, 'macro', v['condition'])
-	 end
+	 return false, nil, nil, build_error_with_context(v['context'], ast), warnings
       end
 
       state.macros[v['macro']] = {["ast"] = ast.filter.value, ["used"] = false}
@@ -912,7 +989,7 @@ function load_rules(sinsp_lua_parser,
 	 end
 
 	 if err ~= nil then
-	    return false, nil, build_error_with_context(v['context'], err), warnings
+	    return false, nil, nil, build_error_with_context(v['context'], err), warnings
 	 end
 
 	 if icond ~= "" then
@@ -933,42 +1010,23 @@ function load_rules(sinsp_lua_parser,
 	 warn_evttypes = v['warn_evttypes']
       end
 
-      local status, filter_ast, filters = compiler.compile_filter(v['rule'], v['compile_condition'],
-								  state.macros, state.lists)
+      local status, filter_ast = compiler.compile_filter(v['rule'], v['compile_condition'],
+							 state.macros, state.lists)
 
       if status == false then
-	 return false, nil, build_error_with_context(v['context'], filter_ast), warnings
-      end
-
-      local evtttypes = {}
-      local syscallnums = {}
-
-      if v['source'] == "syscall" then
-	 if not all_events then
-	    sinsp_rule_utils.check_for_ignored_syscalls_events(filter_ast, 'rule', v['rule'])
-	 end
-
-	 evttypes, syscallnums = sinsp_rule_utils.get_evttypes_syscalls(name, filter_ast, v['compile_condition'], warn_evttypes, verbose)
-      end
-
-      -- If a filter in the rule doesn't exist, either skip the rule
-      -- or raise an error, depending on the value of
-      -- skip-if-unknown-filter.
-      for filter, _ in pairs(filters) do
-	 if not is_defined_filter(filter) then
-	    msg = "rule \""..v['rule'].."\": contains unknown filter "..filter
-	    warnings[#warnings + 1] = msg
-
-	    if not v['skip-if-unknown-filter'] then
-	       return false, nil, build_error_with_context(v['context'], msg), warnings
-	    else
-	       print("Skipping "..msg)
-	       goto next_rule
-	    end
-	 end
+	 return false, nil, nil, build_error_with_context(v['context'], filter_ast), warnings
       end
 
       if (filter_ast.type == "Rule") then
+
+	 valid = falco_rules.is_source_valid(rules_mgr, v['source'])
+
+	 if valid == false then
+	    msg = "Rule "..v['rule']..": warning (unknown-source): unknown source "..v['source']..", skipping"
+	    warnings[#warnings + 1] = msg
+	    goto next_rule
+	 end
+
 	 state.n_rules = state.n_rules + 1
 
 	 state.rules_by_idx[state.n_rules] = v
@@ -983,15 +1041,30 @@ function load_rules(sinsp_lua_parser,
 	 if (v['tags'] == nil) then
 	    v['tags'] = {}
 	 end
-	 if v['source'] == "syscall" then
-	    install_filter(filter_ast.filter.value, filter, sinsp_lua_parser)
-	    -- Pass the filter and event types back up
-	    falco_rules.add_filter(rules_mgr, v['rule'], evttypes, syscallnums, v['tags'])
 
-	 elseif v['source'] == "k8s_audit" then
-	    install_filter(filter_ast.filter.value, k8s_audit_filter, json_lua_parser)
+	 lua_parser = falco_rules.create_lua_parser(rules_mgr, v['source'])
+	 local err = create_filter_obj(filter_ast.filter.value, lua_parser)
+	 if err ~= nil then
 
-	    falco_rules.add_k8s_audit_filter(rules_mgr, v['rule'], v['tags'])
+	    -- If a rule has a property skip-if-unknown-filter: true,
+	    -- and the error is about an undefined field, print a
+	    -- message but continue.
+	    if v['skip-if-unknown-filter'] == true and string.find(err, "filter_check called with nonexistent field") ~= nil then
+	       msg = "Rule "..v['rule']..": warning (unknown-field):"
+	       warnings[#warnings + 1] = msg
+	    else
+	       msg = "Rule "..v['rule']..": error "..err
+	       return false, nil, nil, build_error_with_context(v['context'], msg), warnings
+	    end
+
+	 else
+	    num_evttypes = falco_rules.add_filter(rules_mgr, lua_parser, v['rule'], v['source'], v['tags'])
+	    if v['source'] == "syscall" and (num_evttypes == 0 or num_evttypes > 100) then
+	       if warn_evttypes == true then
+		  msg = "Rule "..v['rule']..": warning (no-evttype):\n".."         matches too many evt.type values.\n".."         This has a significant performance penalty."
+		  warnings[#warnings + 1] = msg
+	       end
+	    end
 	 end
 
 	 -- Rule ASTs are merged together into one big AST, with "OR" between each
@@ -1042,14 +1115,12 @@ function load_rules(sinsp_lua_parser,
 	 -- Ensure that the output field is properly formatted by
 	 -- creating a formatter from it. Any error will be thrown
 	 -- up to the top level.
-	 local err, formatter = formats.formatter(v['source'], v['output'])
-	 if err == nil then
-	    formats.free_formatter(v['source'], formatter)
-	 else
-	    return false, nil, build_error_with_context(v['context'], err), warnings
+	 local err = falco_rules.is_format_valid(rules_mgr, v['source'], v['output'])
+	 if err ~= nil then
+	    return false, nil, nil, build_error_with_context(v['context'], err), warnings
 	 end
       else
-	 return false, nil, build_error_with_context(v['context'], "Unexpected type in load_rule: "..filter_ast.type), warnings
+	 return false, nil, nil, build_error_with_context(v['context'], "Unexpected type in load_rule: "..filter_ast.type), warnings
       end
 
       ::next_rule::
@@ -1072,7 +1143,7 @@ function load_rules(sinsp_lua_parser,
 
    io.flush()
 
-   return true, load_state.required_engine_version, {}, warnings
+   return true, load_state.required_engine_version, load_state.required_plugin_versions, {}, warnings
 end
 
 local rule_fmt = "%-50s %s"

userspace/engine/lua/sinsp_rule_utils.lua

@@ -1,199 +0,0 @@
--- Copyright (C) 2019 The Falco Authors.
---
--- Licensed under the Apache License, Version 2.0 (the "License");
--- you may not use this file except in compliance with the License.
--- You may obtain a copy of the License at
---
---     http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-local parser = require("parser")
-local sinsp_rule_utils = {}
-
-function sinsp_rule_utils.check_for_ignored_syscalls_events(ast, filter_type, source)
-
-   function check_syscall(val)
-      if ignored_syscalls[val] then
-	 error("Ignored syscall \""..val.."\" in "..filter_type..": "..source)
-      end
-
-   end
-
-   function check_event(val)
-      if ignored_events[val] then
-	 error("Ignored event \""..val.."\" in "..filter_type..": "..source)
-      end
-   end
-
-   function cb(node)
-      if node.left.type == "FieldName" and
-	 (node.left.value == "evt.type" or
-	  node.left.value == "syscall.type") then
-
-	    if (node.operator == "in" or
-		node.operator == "intersects" or
-	        node.operator == "pmatch") then
-	       for i, v in ipairs(node.right.elements) do
-		  if v.type == "BareString" then
-		     if node.left.value == "evt.type" then
-			check_event(v.value)
-		     else
-			check_syscall(v.value)
-		     end
-		  end
-	       end
-	    else
-	       if node.right.type == "BareString" then
-		  if node.left.value == "evt.type" then
-		     check_event(node.right.value)
-		  else
-		     check_syscall(node.right.value)
-		  end
-	       end
-	    end
-      end
-   end
-
-   parser.traverse_ast(ast, {BinaryRelOp=1}, cb)
-end
-
--- Examine the ast and find the event types/syscalls for which the
--- rule should run. All evt.type references are added as event types
--- up until the first "!=" binary operator or unary not operator. If
--- no event type checks are found afterward in the rule, the rule is
--- considered optimized and is associated with the event type(s).
---
--- Otherwise, the rule is associated with a 'catchall' category and is
--- run for all event types/syscalls. (Also, a warning is printed).
---
-
-function sinsp_rule_utils.get_evttypes_syscalls(name, ast, source, warn_evttypes, verbose)
-
-   local evttypes = {}
-   local syscallnums = {}
-   local evtnames = {}
-   local found_event = false
-   local found_not = false
-   local found_event_after_not = false
-
-   function cb(node)
-     if node.type == "UnaryBoolOp" then
-	if node.operator == "not" then
-	   found_not = true
-	end
-     else
-	 if node.operator == "!=" then
-	    found_not = true
-	 end
-	 if node.left.type == "FieldName" and node.left.value == "evt.type" then
-	    found_event = true
-	    if found_not then
-	       found_event_after_not = true
-	    end
-	    if (node.operator == "in" or
-                node.operator == "intersects" or
-                node.operator == "pmatch") then
-	       for i, v in ipairs(node.right.elements) do
-		  if v.type == "BareString" then
-
-                     -- The event must be a known event
-                     if events[v.value] == nil and syscalls[v.value] == nil then
-                        error("Unknown event/syscall \""..v.value.."\" in filter: "..source)
-                     end
-
-		     evtnames[v.value] = 1
-		     if events[v.value] ~= nil then
-			for id in string.gmatch(events[v.value], "%S+") do
-			   evttypes[id] = 1
-			end
-		     end
-
-		     if syscalls[v.value] ~= nil then
-			for id in string.gmatch(syscalls[v.value], "%S+") do
-			   syscallnums[id] = 1
-			end
-		     end
-		  end
-	       end
-	    else
-	       if node.right.type == "BareString" then
-
-		  -- The event must be a known event
-		  if events[node.right.value] == nil and syscalls[node.right.value] == nil then
-		     error("Unknown event/syscall \""..node.right.value.."\" in filter: "..source)
-		  end
-
-		  evtnames[node.right.value] = 1
-		  if events[node.right.value] ~= nil then
-		     for id in string.gmatch(events[node.right.value], "%S+") do
-			evttypes[id] = 1
-		     end
-		  end
-
-		  if syscalls[node.right.value] ~= nil then
-		     for id in string.gmatch(syscalls[node.right.value], "%S+") do
-			syscallnums[id] = 1
-		     end
-		  end
-	       end
-	    end
-	 end
-      end
-   end
-
-   parser.traverse_ast(ast.filter.value, {BinaryRelOp=1, UnaryBoolOp=1} , cb)
-
-   if not found_event then
-      if warn_evttypes == true then
-	 io.stderr:write("Rule "..name..": warning (no-evttype):\n")
-	 io.stderr:write(source.."\n")
-	 io.stderr:write("         did not contain any evt.type restriction, meaning it will run for all event types.\n")
-	 io.stderr:write("         This has a significant performance penalty. Consider adding an evt.type restriction if possible.\n")
-      end
-      evttypes = {}
-      syscallnums = {}
-      evtnames = {}
-   end
-
-   if found_event_after_not then
-      if warn_evttypes == true then
-	 io.stderr:write("Rule "..name..": warning (trailing-evttype):\n")
-	 io.stderr:write(source.."\n")
-	 io.stderr:write("         does not have all evt.type restrictions at the beginning of the condition,\n")
-	 io.stderr:write("         or uses a negative match (i.e. \"not\"/\"!=\") for some evt.type restriction.\n")
-	 io.stderr:write("         This has a performance penalty, as the rule can not be limited to specific event types.\n")
-	 io.stderr:write("         Consider moving all evt.type restrictions to the beginning of the rule and/or\n")
-	 io.stderr:write("         replacing negative matches with positive matches if possible.\n")
-      end
-      evttypes = {}
-      syscallnums = {}
-      evtnames = {}
-   end
-
-   evtnames_only = {}
-   local num_evtnames = 0
-   for name, dummy in pairs(evtnames) do
-      table.insert(evtnames_only, name)
-      num_evtnames = num_evtnames + 1
-   end
-
-   if num_evtnames == 0 then
-      table.insert(evtnames_only, "all")
-   end
-
-   table.sort(evtnames_only)
-
-   if verbose then
-      io.stderr:write("Event types/Syscalls for rule "..name..": "..table.concat(evtnames_only, ",").."\n")
-   end
-
-   return evttypes, syscallnums
-end
-
-return sinsp_rule_utils

userspace/engine/rules.cpp

@@ -30,26 +30,32 @@ extern "C" {
 const static struct luaL_Reg ll_falco_rules[] =
 	{
 		{"clear_filters", &falco_rules::clear_filters},
+		{"create_lua_parser", &falco_rules::create_lua_parser},
 		{"add_filter", &falco_rules::add_filter},
-		{"add_k8s_audit_filter", &falco_rules::add_k8s_audit_filter},
 		{"enable_rule", &falco_rules::enable_rule},
 		{"engine_version", &falco_rules::engine_version},
+		{"is_source_valid", &falco_rules::is_source_valid},
+		{"is_format_valid", &falco_rules::is_format_valid},
+		{"is_defined_field", &falco_rules::is_defined_field},
 		{NULL, NULL}};
 
-falco_rules::falco_rules(sinsp* inspector,
-			 falco_engine *engine,
+falco_rules::falco_rules(falco_engine *engine,
 			 lua_State *ls)
-	: m_inspector(inspector),
-	  m_engine(engine),
+	: m_engine(engine),
 	  m_ls(ls)
 {
-	m_sinsp_lua_parser = new lua_parser(engine->sinsp_factory(), m_ls, "filter");
-	m_json_lua_parser = new lua_parser(engine->json_factory(), m_ls, "k8s_audit_filter");
+}
+
+void falco_rules::add_filter_factory(const std::string &source,
+				     std::shared_ptr<gen_event_filter_factory> factory)
+{
+	m_filter_factories[source] = factory;
 }
 
 void falco_rules::init(lua_State *ls)
 {
 	luaL_openlib(ls, "falco_rules", ll_falco_rules, 0);
+	lua_parser::register_callbacks(ls, "filter");
 }
 
 int falco_rules::clear_filters(lua_State *ls)
@@ -71,12 +77,51 @@ void falco_rules::clear_filters()
 	m_engine->clear_filters();
 }
 
+int falco_rules::create_lua_parser(lua_State *ls)
+{
+	if (! lua_islightuserdata(ls, -2) ||
+	    ! lua_isstring(ls, -1))
+	{
+		lua_pushstring(ls, "Invalid argument passed to create_lua_parser()");
+		lua_error(ls);
+	}
+
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -2);
+	std::string source = lua_tostring(ls, -1);
+
+	std::string errstr;
+	lua_parser *lp = rules->create_lua_parser(source, errstr);
+
+	if(lp == NULL) {
+		lua_pushstring(ls, errstr.c_str());
+		lua_error(ls);
+	}
+
+	lua_pushlightuserdata(ls, lp);
+	return 1;
+}
+
+lua_parser *falco_rules::create_lua_parser(std::string &source, std::string &errstr)
+{
+	auto it = m_filter_factories.find(source);
+
+	if(it == m_filter_factories.end())
+	{
+		errstr = string("Unknown event source ") + source;
+		return NULL;
+	}
+
+	lua_parser *lp = new lua_parser(it->second);
+
+	return lp;
+}
+
 int falco_rules::add_filter(lua_State *ls)
 {
 	if (! lua_islightuserdata(ls, -5) ||
-	    ! lua_isstring(ls, -4) ||
-	    ! lua_istable(ls, -3) ||
-	    ! lua_istable(ls, -2) ||
+	    ! lua_islightuserdata(ls, -4) ||
+	    ! lua_isstring(ls, -3) ||
+	    ! lua_isstring(ls, -2) ||
 	    ! lua_istable(ls, -1))
 	{
 		lua_pushstring(ls, "Invalid arguments passed to add_filter()");
@@ -84,31 +129,9 @@ int falco_rules::add_filter(lua_State *ls)
 	}
 
 	falco_rules *rules = (falco_rules *) lua_topointer(ls, -5);
-	const char *rulec = lua_tostring(ls, -4);
-
-	set<uint32_t> evttypes;
-
-	lua_pushnil(ls);  /* first key */
-	while (lua_next(ls, -4) != 0) {
-                // key is at index -2, value is at index
-                // -1. We want the keys.
-		evttypes.insert(luaL_checknumber(ls, -2));
-
-		// Remove value, keep key for next iteration
-		lua_pop(ls, 1);
-	}
-
-	set<uint32_t> syscalls;
-
-	lua_pushnil(ls);  /* first key */
-	while (lua_next(ls, -3) != 0) {
-                // key is at index -2, value is at index
-                // -1. We want the keys.
-		syscalls.insert(luaL_checknumber(ls, -2));
-
-		// Remove value, keep key for next iteration
-		lua_pop(ls, 1);
-	}
+	lua_parser *lp = (lua_parser *) lua_topointer(ls, -4);
+	std::string rule = lua_tostring(ls, -3);
+	std::string source = lua_tostring(ls, -2);
 
 	set<string> tags;
 
@@ -122,61 +145,25 @@ int falco_rules::add_filter(lua_State *ls)
 		lua_pop(ls, 1);
 	}
 
-	std::string rule = rulec;
-	rules->add_filter(rule, evttypes, syscalls, tags);
+	size_t num_evttypes = lp->filter()->evttypes().size();
 
-	return 0;
-}
-
-int falco_rules::add_k8s_audit_filter(lua_State *ls)
-{
-	if (! lua_islightuserdata(ls, -3) ||
-	    ! lua_isstring(ls, -2) ||
-	    ! lua_istable(ls, -1))
-	{
-		lua_pushstring(ls, "Invalid arguments passed to add_k8s_audit_filter()");
+	try {
+		rules->add_filter(lp->filter(), rule, source, tags);
+	} catch (exception &e) {
+		std::string errstr = string("Could not add rule to falco engine: ") + e.what();
+		lua_pushstring(ls, errstr.c_str());
 		lua_error(ls);
 	}
 
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -3);
-	const char *rulec = lua_tostring(ls, -2);
+	delete lp;
 
-	set<string> tags;
-
-	lua_pushnil(ls);  /* first key */
-	while (lua_next(ls, -2) != 0) {
-                // key is at index -2, value is at index
-                // -1. We want the values.
-		tags.insert(lua_tostring(ls, -1));
-
-		// Remove value, keep key for next iteration
-		lua_pop(ls, 1);
-	}
-
-	std::string rule = rulec;
-	rules->add_k8s_audit_filter(rule, tags);
-
-	return 0;
+	lua_pushnumber(ls, num_evttypes);
+	return 1;
 }
 
-void falco_rules::add_filter(string &rule, set<uint32_t> &evttypes, set<uint32_t> &syscalls, set<string> &tags)
+void falco_rules::add_filter(std::shared_ptr<gen_event_filter> filter, string &rule, string &source, set<string> &tags)
 {
-	// While the current rule was being parsed, a sinsp_filter
-	// object was being populated by lua_parser. Grab that filter
-	// and pass it to the engine.
-	sinsp_filter *filter = (sinsp_filter *) m_sinsp_lua_parser->get_filter(true);
-
-	m_engine->add_sinsp_filter(rule, evttypes, syscalls, tags, filter);
-}
-
-void falco_rules::add_k8s_audit_filter(string &rule, set<string> &tags)
-{
-	// While the current rule was being parsed, a sinsp_filter
-	// object was being populated by lua_parser. Grab that filter
-	// and pass it to the engine.
-	json_event_filter *filter = (json_event_filter *) m_json_lua_parser->get_filter(true);
-
-	m_engine->add_k8s_audit_filter(rule, tags, filter);
+	m_engine->add_filter(filter, rule, source, tags);
 }
 
 int falco_rules::enable_rule(lua_State *ls)
@@ -219,6 +206,128 @@ int falco_rules::engine_version(lua_State *ls)
 	return 1;
 }
 
+bool falco_rules::is_source_valid(const std::string &source)
+{
+	return m_engine->is_source_valid(source);
+}
+
+int falco_rules::is_source_valid(lua_State *ls)
+{
+	if (! lua_islightuserdata(ls, -2) ||
+	    ! lua_isstring(ls, -1))
+	{
+		lua_pushstring(ls, "Invalid arguments passed to is_source_valid");
+		lua_error(ls);
+	}
+
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -2);
+	string source = luaL_checkstring(ls, -1);
+
+	bool ret = rules->is_source_valid(source);
+
+	lua_pushboolean(ls, (ret ? 1 : 0));
+
+	return 1;
+}
+
+int falco_rules::is_format_valid(lua_State *ls)
+{
+	if (! lua_islightuserdata(ls, -3) ||
+	    ! lua_isstring(ls, -2) ||
+	    ! lua_isstring(ls, -1))
+	{
+		lua_pushstring(ls, "Invalid arguments passed to is_format_valid");
+		lua_error(ls);
+	}
+
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -3);
+	string source = luaL_checkstring(ls, -2);
+	string format = luaL_checkstring(ls, -1);
+	string errstr;
+
+	bool ret = rules->is_format_valid(source, format, errstr);
+
+	if (!ret)
+	{
+		lua_pushstring(ls, errstr.c_str());
+	}
+	else
+	{
+		lua_pushnil(ls);
+	}
+
+	return 1;
+}
+
+bool falco_rules::is_format_valid(const std::string &source, const std::string &format, std::string &errstr)
+{
+	bool ret = true;
+
+	try
+	{
+		std::shared_ptr<gen_event_formatter> formatter;
+
+		formatter = m_engine->create_formatter(source, format);
+	}
+	catch(exception &e)
+	{
+		std::ostringstream os;
+
+		os << "Invalid output format '"
+		   << format
+		   << "': '"
+		   << e.what()
+		   << "'";
+
+		errstr = os.str();
+		ret = false;
+	}
+
+	return ret;
+}
+
+int falco_rules::is_defined_field(lua_State *ls)
+{
+	if (! lua_islightuserdata(ls, -3) ||
+	    ! lua_isstring(ls, -2) ||
+	    ! lua_isstring(ls, -1))
+	{
+		lua_pushstring(ls, "Invalid arguments passed to is_defined_field");
+		lua_error(ls);
+	}
+
+	falco_rules *rules = (falco_rules *) lua_topointer(ls, -3);
+	string source = luaL_checkstring(ls, -2);
+	string fldname = luaL_checkstring(ls, -1);
+
+	bool ret = rules->is_defined_field(source, fldname);
+
+	lua_pushboolean(ls, (ret ? 1 : 0));
+
+	return 1;
+}
+
+bool falco_rules::is_defined_field(const std::string &source, const std::string &fldname)
+{
+	auto it = m_filter_factories.find(source);
+
+	if(it == m_filter_factories.end())
+	{
+		return false;
+	}
+
+	auto *chk = it->second->new_filtercheck(fldname.c_str());
+
+	if (chk == NULL)
+	{
+		return false;
+	}
+
+	delete(chk);
+
+	return true;
+}
+
 static std::list<std::string> get_lua_table_values(lua_State *ls, int idx)
 {
 	std::list<std::string> ret;
@@ -244,204 +353,52 @@ static std::list<std::string> get_lua_table_values(lua_State *ls, int idx)
 	return ret;
 }
 
+static void get_lua_table_list_values(lua_State *ls,
+				      int idx,
+				      std::map<std::string, std::list<std::string>> &required_plugin_versions)
+{
+	if (lua_isnil(ls, idx)) {
+		return;
+	}
+
+	lua_pushnil(ls);  /* first key */
+	while (lua_next(ls, idx-1) != 0) {
+                // key is at index -2, table of values is at index -1.
+		if (! lua_isstring(ls, -2)) {
+			std::string err = "Non-string key in table of strings";
+			throw falco_exception(err);
+		}
+
+		std::string key = string(lua_tostring(ls, -2));
+		std::list<std::string> vals = get_lua_table_values(ls, -1);
+
+		if (required_plugin_versions.find(key) == required_plugin_versions.end())
+		{
+			required_plugin_versions[key] = vals;
+		}
+		else
+		{
+			required_plugin_versions[key].insert(required_plugin_versions[key].end(),
+							     vals.begin(),
+							     vals.end());
+		}
+
+		// Remove value, keep key for next iteration
+		lua_pop(ls, 1);
+	}
+}
+
+
 void falco_rules::load_rules(const string &rules_content,
 			     bool verbose, bool all_events,
 			     string &extra, bool replace_container_info,
 			     falco_common::priority_type min_priority,
-			     uint64_t &required_engine_version)
+			     uint64_t &required_engine_version,
+			     std::map<std::string, std::list<std::string>> &required_plugin_versions)
 {
 	lua_getglobal(m_ls, m_lua_load_rules.c_str());
 	if(lua_isfunction(m_ls, -1))
 	{
-		// Create a table containing all events, so they can
-		// be mapped to event ids.
-		sinsp_evttables* einfo = m_inspector->get_event_info_tables();
-		const struct ppm_event_info* etable = einfo->m_event_info;
-		const struct ppm_syscall_desc* stable = einfo->m_syscall_info_table;
-
-		map<string,string> events_by_name;
-		for(uint32_t j = 0; j < PPM_EVENT_MAX; j++)
-		{
-			auto it = events_by_name.find(etable[j].name);
-
-			if (it == events_by_name.end()) {
-				events_by_name[etable[j].name] = to_string(j);
-			} else {
-				string cur = it->second;
-				cur += " ";
-				cur += to_string(j);
-				events_by_name[etable[j].name] = cur;
-			}
-		}
-
-		lua_newtable(m_ls);
-
-		for( auto kv : events_by_name)
-		{
-			lua_pushstring(m_ls, kv.first.c_str());
-			lua_pushstring(m_ls, kv.second.c_str());
-			lua_settable(m_ls, -3);
-		}
-
-		lua_setglobal(m_ls, m_lua_events.c_str());
-
-		map<string,string> syscalls_by_name;
-		for(uint32_t j = 0; j < PPM_SC_MAX; j++)
-		{
-			auto it = syscalls_by_name.find(stable[j].name);
-
-			if (it == syscalls_by_name.end())
-			{
-				syscalls_by_name[stable[j].name] = to_string(j);
-			}
-			else
-			{
-				string cur = it->second;
-				cur += " ";
-				cur += to_string(j);
-				syscalls_by_name[stable[j].name] = cur;
-			}
-		}
-
-		lua_newtable(m_ls);
-
-		for( auto kv : syscalls_by_name)
-		{
-			lua_pushstring(m_ls, kv.first.c_str());
-			lua_pushstring(m_ls, kv.second.c_str());
-			lua_settable(m_ls, -3);
-		}
-
-		lua_setglobal(m_ls, m_lua_syscalls.c_str());
-
-		// Create a table containing the syscalls/events that
-		// are ignored by the kernel module. load_rules will
-		// return an error if any rule references one of these
-		// syscalls/events.
-
-		lua_newtable(m_ls);
-
-		for(uint32_t j = 0; j < PPM_EVENT_MAX; j++)
-		{
-			if(etable[j].flags & EF_DROP_SIMPLE_CONS)
-			{
-				lua_pushstring(m_ls, etable[j].name);
-				lua_pushnumber(m_ls, 1);
-				lua_settable(m_ls, -3);
-			}
-		}
-
-		lua_setglobal(m_ls, m_lua_ignored_events.c_str());
-
-		lua_newtable(m_ls);
-
-		for(uint32_t j = 0; j < PPM_SC_MAX; j++)
-		{
-			if(stable[j].flags & EF_DROP_SIMPLE_CONS)
-			{
-				lua_pushstring(m_ls, stable[j].name);
-				lua_pushnumber(m_ls, 1);
-				lua_settable(m_ls, -3);
-			}
-		}
-
-		lua_setglobal(m_ls, m_lua_ignored_syscalls.c_str());
-
-		vector<const filter_check_info*> fc_plugins;
-		sinsp::get_filtercheck_fields_info(&fc_plugins);
-
-		set<string> no_argument_filters;
-		set<string> argument_filters;
-
-		for(uint32_t j = 0; j < fc_plugins.size(); j++)
-		{
-			const filter_check_info* fci = fc_plugins[j];
-
-			if(fci->m_flags & filter_check_info::FL_HIDDEN)
-			{
-				continue;
-			}
-
-			for(int32_t k = 0; k < fci->m_nfields; k++)
-			{
-				const filtercheck_field_info* fld = &fci->m_fields[k];
-
-				if(fld->m_flags & EPF_TABLE_ONLY ||
-				   fld->m_flags & EPF_PRINT_ONLY)
-				{
-					continue;
-				}
-
-				// Some filters can work with or without an argument
-				std::set<string> flexible_filters = {
-					"proc.aname",
-					"proc.apid"
-				};
-
-				if(fld->m_flags & EPF_REQUIRES_ARGUMENT ||
-				   flexible_filters.find(fld->m_name) != flexible_filters.end())
-				{
-					argument_filters.insert(fld->m_name);
-				}
-
-				if(!(fld->m_flags & EPF_REQUIRES_ARGUMENT) ||
-				   flexible_filters.find(fld->m_name) != flexible_filters.end())
-				{
-					no_argument_filters.insert(fld->m_name);
-				}
-			}
-		}
-
-		for(auto &chk_field : m_engine->json_factory().get_fields())
-		{
-			for(auto &field : chk_field.m_fields)
-			{
-				switch(field.m_idx_mode)
-				{
-				case json_event_filter_check::IDX_REQUIRED:
-					argument_filters.insert(field.m_name);
-					break;
-				case json_event_filter_check::IDX_ALLOWED:
-					argument_filters.insert(field.m_name);
-					no_argument_filters.insert(field.m_name);
-					break;
-				case json_event_filter_check::IDX_NONE:
-					no_argument_filters.insert(field.m_name);
-					break;
-				default:
-					break;
-				}
-			}
-		}
-
-		// Create tables containing all filtercheck
-		// names. They are split into names that require
-		// arguments and ones that do not.
-
-		lua_newtable(m_ls);
-
-		for(auto &field : argument_filters)
-		{
-			lua_pushstring(m_ls, field.c_str());
-			lua_pushnumber(m_ls, 1);
-			lua_settable(m_ls, -3);
-		}
-
-		lua_setglobal(m_ls, m_lua_defined_arg_filters.c_str());
-
-		lua_newtable(m_ls);
-
-		for(auto &field : no_argument_filters)
-		{
-			lua_pushstring(m_ls, field.c_str());
-			lua_pushnumber(m_ls, 1);
-			lua_settable(m_ls, -3);
-		}
-
-		lua_setglobal(m_ls, m_lua_defined_noarg_filters.c_str());
-
-		lua_pushlightuserdata(m_ls, m_sinsp_lua_parser);
-		lua_pushlightuserdata(m_ls, m_json_lua_parser);
 		lua_pushstring(m_ls, rules_content.c_str());
 		lua_pushlightuserdata(m_ls, this);
 		lua_pushboolean(m_ls, (verbose ? 1 : 0));
@@ -449,7 +406,7 @@ void falco_rules::load_rules(const string &rules_content,
 		lua_pushstring(m_ls, extra.c_str());
 		lua_pushboolean(m_ls, (replace_container_info ? 1 : 0));
 		lua_pushnumber(m_ls, min_priority);
-		if(lua_pcall(m_ls, 9, 4, 0) != 0)
+		if(lua_pcall(m_ls, 7, 5, 0) != 0)
 		{
 			const char* lerr = lua_tostring(m_ls, -1);
 
@@ -461,10 +418,12 @@ void falco_rules::load_rules(const string &rules_content,
 		// Returns:
 		// Load result: bool
 		// required engine version: will be nil when load result is false
+		// required_plugin_versions: will be nil when load result is false
 		// array of errors
 		// array of warnings
-		bool successful = lua_toboolean(m_ls, -4);
-		required_engine_version = lua_tonumber(m_ls, -3);
+		bool successful = lua_toboolean(m_ls, -5);
+		required_engine_version = lua_tonumber(m_ls, -4);
+		get_lua_table_list_values(m_ls, -3, required_plugin_versions);
 		std::list<std::string> errors = get_lua_table_values(m_ls, -2);
 		std::list<std::string> warnings = get_lua_table_values(m_ls, -1);
 
@@ -533,6 +492,4 @@ void falco_rules::describe_rule(std::string *rule)
 
 falco_rules::~falco_rules()
 {
-	delete m_sinsp_lua_parser;
-	delete m_json_lua_parser;
 }

userspace/engine/rules.h

@@ -32,41 +32,55 @@ class falco_engine;
 class falco_rules
 {
  public:
-	falco_rules(sinsp* inspector,
-		    falco_engine *engine,
+	falco_rules(falco_engine *engine,
 		    lua_State *ls);
 	~falco_rules();
+
+	void add_filter_factory(const std::string &source,
+				std::shared_ptr<gen_event_filter_factory> factory);
+
 	void load_rules(const string &rules_content, bool verbose, bool all_events,
 			std::string &extra, bool replace_container_info,
 			falco_common::priority_type min_priority,
-			uint64_t &required_engine_version);
+			uint64_t &required_engine_version,
+			std::map<std::string, std::list<std::string>> &required_plugin_versions);
 	void describe_rule(string *rule);
 
+	bool is_source_valid(const std::string &source);
+
+	bool is_format_valid(const std::string &source, const std::string &format, std::string &errstr);
+
+	bool is_defined_field(const std::string &source, const std::string &field);
+
 	static void init(lua_State *ls);
 	static int clear_filters(lua_State *ls);
+	static int create_lua_parser(lua_State *ls);
 	static int add_filter(lua_State *ls);
-	static int add_k8s_audit_filter(lua_State *ls);
 	static int enable_rule(lua_State *ls);
 	static int engine_version(lua_State *ls);
 
+	static int is_source_valid(lua_State *ls);
+
+	// err = falco_rules.is_format_valid(source, format_string)
+	static int is_format_valid(lua_State *ls);
+
+	// err = falco_rules.is_defined_field(source, field)
+	static int is_defined_field(lua_State *ls);
+
  private:
 	void clear_filters();
-	void add_filter(string &rule, std::set<uint32_t> &evttypes, std::set<uint32_t> &syscalls, std::set<string> &tags);
-	void add_k8s_audit_filter(string &rule, std::set<string> &tags);
+	// XXX/mstemm can I make this a shared_ptr?
+	lua_parser * create_lua_parser(std::string &source, std::string &errstr);
+	void add_filter(std::shared_ptr<gen_event_filter> filter, string &rule, string &source, std::set<string> &tags);
 	void enable_rule(string &rule, bool enabled);
 
-	lua_parser* m_sinsp_lua_parser;
-	lua_parser* m_json_lua_parser;
-	sinsp* m_inspector;
 	falco_engine *m_engine;
 	lua_State* m_ls;
 
+	// Maps from event source to an object that can create rules
+	// for that event source.
+	std::map<std::string, std::shared_ptr<gen_event_filter_factory>> m_filter_factories;
+
 	string m_lua_load_rules = "load_rules";
-	string m_lua_ignored_syscalls = "ignored_syscalls";
-	string m_lua_ignored_events = "ignored_events";
-	string m_lua_defined_arg_filters = "defined_arg_filters";
-	string m_lua_defined_noarg_filters = "defined_noarg_filters";
-	string m_lua_events = "events";
-	string m_lua_syscalls = "syscalls";
 	string m_lua_describe_rule = "describe_rule";
 };

userspace/engine/ruleset.cpp

@@ -17,6 +17,8 @@ limitations under the License.
 #include "ruleset.h"
 #include "banned.h" // This raises a compilation error when certain functions are used
 
+#include <algorithm>
+
 using namespace std;
 
 falco_ruleset::falco_ruleset()
@@ -25,127 +27,115 @@ falco_ruleset::falco_ruleset()
 
 falco_ruleset::~falco_ruleset()
 {
-	for(const auto &val : m_filters)
-	{
-		delete val.second->filter;
-		delete val.second;
-	}
-
-	for(auto &ruleset : m_rulesets)
-	{
-		delete ruleset;
-	}
-	m_filters.clear();
 }
 
-falco_ruleset::ruleset_filters::ruleset_filters():
-	m_num_filters(0)
+falco_ruleset::ruleset_filters::ruleset_filters()
 {
 }
 
 falco_ruleset::ruleset_filters::~ruleset_filters()
 {
-	for(uint32_t i = 0; i < m_filter_by_event_tag.size(); i++)
+}
+
+void falco_ruleset::ruleset_filters::add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
+{
+	// This is O(n) but it's also uncommon
+	// (when loading rules only).
+	auto pos = std::find(wrappers.begin(),
+			     wrappers.end(),
+			     wrap);
+
+	if(pos == wrappers.end())
 	{
-		if(m_filter_by_event_tag[i])
-		{
-			delete m_filter_by_event_tag[i];
-			m_filter_by_event_tag[i] = NULL;
-		}
+		wrappers.push_back(wrap);
 	}
 }
 
-void falco_ruleset::ruleset_filters::add_filter(filter_wrapper *wrap)
+void falco_ruleset::ruleset_filters::remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
 {
-
-	bool added = false;
-
-	for(uint32_t etag = 0; etag < wrap->event_tags.size(); etag++)
+	// This is O(n) but it's also uncommon
+	// (when loading rules only).
+	auto pos = std::find(wrappers.begin(),
+			     wrappers.end(),
+			     wrap);
+	if(pos != wrappers.end())
 	{
-		if(wrap->event_tags[etag])
-		{
-			added = true;
-			if(m_filter_by_event_tag.size() <= etag)
-			{
-				m_filter_by_event_tag.resize(etag + 1);
-			}
-
-			if(!m_filter_by_event_tag[etag])
-			{
-				m_filter_by_event_tag[etag] = new list<filter_wrapper *>();
-			}
-
-			m_filter_by_event_tag[etag]->push_back(wrap);
-		}
-	}
-
-	if(added)
-	{
-		m_num_filters++;
+		wrappers.erase(pos);
 	}
 }
 
-void falco_ruleset::ruleset_filters::remove_filter(filter_wrapper *wrap)
+void falco_ruleset::ruleset_filters::add_filter(std::shared_ptr<filter_wrapper> wrap)
 {
-	bool removed = false;
+	std::set<uint16_t> fevttypes = wrap->filter->evttypes();
 
-	for(uint32_t etag = 0; etag < wrap->event_tags.size(); etag++)
+	// TODO: who fills this one for rules without evt.type specified?
+	// Can this be actually empty?
+	// Is m_filter_all_event_types useful?
+	if(fevttypes.empty())
 	{
-		if(wrap->event_tags[etag])
+		// Should run for all event types
+		add_wrapper_to_list(m_filter_all_event_types, wrap);
+	}
+	else
+	{
+		for(auto &etype : fevttypes)
 		{
-			if(etag < m_filter_by_event_tag.size())
+			if(m_filter_by_event_type.size() <= etype)
 			{
-				list<filter_wrapper *> *l = m_filter_by_event_tag[etag];
-				if(l)
-				{
-					auto it = remove(l->begin(),
-							 l->end(),
-							 wrap);
+				m_filter_by_event_type.resize(etype + 1);
+			}
 
-					if(it != l->end())
-					{
-						removed = true;
+			add_wrapper_to_list(m_filter_by_event_type[etype], wrap);
+		}
+	}
 
-						l->erase(it,
-							 l->end());
+	m_filters.insert(wrap);
+}
 
-						if(l->size() == 0)
-						{
-							delete l;
-							m_filter_by_event_tag[etag] = NULL;
-						}
-					}
-				}
+void falco_ruleset::ruleset_filters::remove_filter(std::shared_ptr<filter_wrapper> wrap)
+{
+	std::set<uint16_t> fevttypes = wrap->filter->evttypes();
+
+	if(fevttypes.empty())
+	{
+		remove_wrapper_from_list(m_filter_all_event_types, wrap);
+	}
+	else
+	{
+		for(auto &etype : fevttypes)
+		{
+			if( etype < m_filter_by_event_type.size() )
+			{
+				remove_wrapper_from_list(m_filter_by_event_type[etype], wrap);
 			}
 		}
 	}
 
-	if(removed)
-	{
-		m_num_filters--;
-	}
+	m_filters.erase(wrap);
 }
 
 uint64_t falco_ruleset::ruleset_filters::num_filters()
 {
-	return m_num_filters;
+	return m_filters.size();
 }
 
-bool falco_ruleset::ruleset_filters::run(gen_event *evt, uint32_t etag)
+bool falco_ruleset::ruleset_filters::run(gen_event *evt)
 {
-	if(etag >= m_filter_by_event_tag.size())
+	if(evt->get_type() >= m_filter_by_event_type.size())
 	{
 		return false;
 	}
 
-	list<filter_wrapper *> *filters = m_filter_by_event_tag[etag];
-
-	if(!filters)
+	for(auto &wrap : m_filter_by_event_type[evt->get_type()])
 	{
-		return false;
+		if(wrap->filter->run(evt))
+		{
+			return true;
+		}
 	}
 
-	for(auto &wrap : *filters)
+	// Finally, try filters that are not specific to an event type.
+	for(auto &wrap : m_filter_all_event_types)
 	{
 		if(wrap->filter->run(evt))
 		{
@@ -156,83 +146,61 @@ bool falco_ruleset::ruleset_filters::run(gen_event *evt, uint32_t etag)
 	return false;
 }
 
-void falco_ruleset::ruleset_filters::event_tags_for_ruleset(vector<bool> &event_tags)
+void falco_ruleset::ruleset_filters::evttypes_for_ruleset(std::set<uint16_t> &evttypes)
 {
-	event_tags.assign(m_filter_by_event_tag.size(), false);
+	evttypes.clear();
 
-	for(uint32_t etag = 0; etag < m_filter_by_event_tag.size(); etag++)
+	for(auto &wrap : m_filters)
 	{
-		list<filter_wrapper *> *filters = m_filter_by_event_tag[etag];
-		if(filters)
-		{
-			event_tags[etag] = true;
-		}
+		auto fevttypes = wrap->filter->evttypes();
+		evttypes.insert(fevttypes.begin(), fevttypes.end());
 	}
 }
 
 void falco_ruleset::add(string &name,
 			set<string> &tags,
-			set<uint32_t> &event_tags,
-			gen_event_filter *filter)
+			std::shared_ptr<gen_event_filter> filter)
 {
-	filter_wrapper *wrap = new filter_wrapper();
+	std::shared_ptr<filter_wrapper> wrap(new filter_wrapper());
+	wrap->name = name;
+	wrap->tags = tags;
 	wrap->filter = filter;
 
-	for(auto &etag : event_tags)
-	{
-		wrap->event_tags.resize(etag + 1);
-		wrap->event_tags[etag] = true;
-	}
-
-	m_filters.insert(pair<string, filter_wrapper *>(name, wrap));
-
-	for(const auto &tag : tags)
-	{
-		auto it = m_filter_by_event_tag.lower_bound(tag);
-
-		if(it == m_filter_by_event_tag.end() ||
-		   it->first != tag)
-		{
-			it = m_filter_by_event_tag.emplace_hint(it,
-								make_pair(tag, list<filter_wrapper *>()));
-		}
-
-		it->second.push_back(wrap);
-	}
+	m_filters.insert(wrap);
 }
 
 void falco_ruleset::enable(const string &substring, bool match_exact, bool enabled, uint16_t ruleset)
 {
 	while(m_rulesets.size() < (size_t)ruleset + 1)
 	{
-		m_rulesets.push_back(new ruleset_filters());
+		m_rulesets.emplace_back(new ruleset_filters());
 	}
 
-	for(const auto &val : m_filters)
+	for(const auto &wrap : m_filters)
 	{
 		bool matches;
 
 		if(match_exact)
 		{
-			size_t pos = val.first.find(substring);
+			size_t pos = wrap->name.find(substring);
 
 			matches = (substring == "" || (pos == 0 &&
-						       substring.size() == val.first.size()));
+						       substring.size() == wrap->name.size()));
 		}
 		else
 		{
-			matches = (substring == "" || (val.first.find(substring) != string::npos));
+			matches = (substring == "" || (wrap->name.find(substring) != string::npos));
 		}
 
 		if(matches)
 		{
 			if(enabled)
 			{
-				m_rulesets[ruleset]->add_filter(val.second);
+				m_rulesets[ruleset]->add_filter(wrap);
 			}
 			else
 			{
-				m_rulesets[ruleset]->remove_filter(val.second);
+				m_rulesets[ruleset]->remove_filter(wrap);
 			}
 		}
 	}
@@ -242,12 +210,18 @@ void falco_ruleset::enable_tags(const set<string> &tags, bool enabled, uint16_t
 {
 	while(m_rulesets.size() < (size_t)ruleset + 1)
 	{
-		m_rulesets.push_back(new ruleset_filters());
+		m_rulesets.emplace_back(new ruleset_filters());
 	}
 
-	for(const auto &tag : tags)
+	for(const auto &wrap : m_filters)
 	{
-		for(const auto &wrap : m_filter_by_event_tag[tag])
+		std::set<string> intersect;
+
+		set_intersection(tags.begin(), tags.end(),
+				 wrap->tags.begin(), wrap->tags.end(),
+				 inserter(intersect, intersect.begin()));
+
+		if(!intersect.empty())
 		{
 			if(enabled)
 			{
@@ -265,141 +239,28 @@ uint64_t falco_ruleset::num_rules_for_ruleset(uint16_t ruleset)
 {
 	while(m_rulesets.size() < (size_t)ruleset + 1)
 	{
-		m_rulesets.push_back(new ruleset_filters());
+		m_rulesets.emplace_back(new ruleset_filters());
 	}
 
 	return m_rulesets[ruleset]->num_filters();
 }
 
-bool falco_ruleset::run(gen_event *evt, uint32_t etag, uint16_t ruleset)
+bool falco_ruleset::run(gen_event *evt, uint16_t ruleset)
 {
 	if(m_rulesets.size() < (size_t)ruleset + 1)
 	{
 		return false;
 	}
 
-	return m_rulesets[ruleset]->run(evt, etag);
+	return m_rulesets[ruleset]->run(evt);
 }
 
-void falco_ruleset::event_tags_for_ruleset(vector<bool> &evttypes, uint16_t ruleset)
+void falco_ruleset::evttypes_for_ruleset(set<uint16_t> &evttypes, uint16_t ruleset)
 {
 	if(m_rulesets.size() < (size_t)ruleset + 1)
 	{
 		return;
 	}
 
-	return m_rulesets[ruleset]->event_tags_for_ruleset(evttypes);
-}
-
-falco_sinsp_ruleset::falco_sinsp_ruleset()
-{
-}
-
-falco_sinsp_ruleset::~falco_sinsp_ruleset()
-{
-}
-
-void falco_sinsp_ruleset::add(string &name,
-			      set<uint32_t> &evttypes,
-			      set<uint32_t> &syscalls,
-			      set<string> &tags,
-			      sinsp_filter *filter)
-{
-	set<uint32_t> event_tags;
-
-	if(evttypes.size() + syscalls.size() == 0)
-	{
-		// If no evttypes or syscalls are specified, the filter is
-		// enabled for all evttypes/syscalls.
-		for(uint32_t i = 0; i < PPM_EVENT_MAX; i++)
-		{
-			evttypes.insert(i);
-		}
-
-		for(uint32_t i = 0; i < PPM_SC_MAX; i++)
-		{
-			syscalls.insert(i);
-		}
-	}
-
-	for(auto evttype : evttypes)
-	{
-		event_tags.insert(evttype_to_event_tag(evttype));
-	}
-
-	for(auto syscallid : syscalls)
-	{
-		event_tags.insert(syscall_to_event_tag(syscallid));
-	}
-
-	falco_ruleset::add(name, tags, event_tags, (gen_event_filter *)filter);
-}
-
-bool falco_sinsp_ruleset::run(sinsp_evt *evt, uint16_t ruleset)
-{
-	uint32_t etag;
-
-	uint16_t etype = evt->get_type();
-
-	if(etype == PPME_GENERIC_E || etype == PPME_GENERIC_X)
-	{
-		sinsp_evt_param *parinfo = evt->get_param(0);
-		uint16_t syscallid = *(uint16_t *)parinfo->m_val;
-
-		etag = syscall_to_event_tag(syscallid);
-	}
-	else
-	{
-		etag = evttype_to_event_tag(etype);
-	}
-
-	return falco_ruleset::run((gen_event *)evt, etag, ruleset);
-}
-
-void falco_sinsp_ruleset::evttypes_for_ruleset(vector<bool> &evttypes, uint16_t ruleset)
-{
-	vector<bool> event_tags;
-
-	event_tags_for_ruleset(event_tags, ruleset);
-
-	evttypes.assign(PPM_EVENT_MAX + 1, false);
-
-	for(uint32_t etype = 0; etype < PPM_EVENT_MAX; etype++)
-	{
-		uint32_t etag = evttype_to_event_tag(etype);
-
-		if(etag < event_tags.size() && event_tags[etag])
-		{
-			evttypes[etype] = true;
-		}
-	}
-}
-
-void falco_sinsp_ruleset::syscalls_for_ruleset(vector<bool> &syscalls, uint16_t ruleset)
-{
-	vector<bool> event_tags;
-
-	event_tags_for_ruleset(event_tags, ruleset);
-
-	syscalls.assign(PPM_EVENT_MAX + 1, false);
-
-	for(uint32_t syscallid = 0; syscallid < PPM_SC_MAX; syscallid++)
-	{
-		uint32_t etag = evttype_to_event_tag(syscallid);
-
-		if(etag < event_tags.size() && event_tags[etag])
-		{
-			syscalls[syscallid] = true;
-		}
-	}
-}
-
-uint32_t falco_sinsp_ruleset::evttype_to_event_tag(uint32_t evttype)
-{
-	return evttype;
-}
-
-uint32_t falco_sinsp_ruleset::syscall_to_event_tag(uint32_t syscallid)
-{
-	return PPM_EVENT_MAX + 1 + syscallid;
+	return m_rulesets[ruleset]->evttypes_for_ruleset(evttypes);
 }

userspace/engine/ruleset.h

@@ -36,8 +36,7 @@ public:
 
 	void add(std::string &name,
 		 std::set<std::string> &tags,
-		 std::set<uint32_t> &event_tags,
-		 gen_event_filter* filter);
+		 std::shared_ptr<gen_event_filter> filter);
 
 	// rulesets are arbitrary numbers and should be managed by the caller.
         // Note that rulesets are used to index into a std::vector so
@@ -65,23 +64,22 @@ public:
 	uint64_t num_rules_for_ruleset(uint16_t ruleset = 0);
 
 	// Match all filters against the provided event.
-	bool run(gen_event *evt, uint32_t etag, uint16_t ruleset = 0);
+	bool run(gen_event *evt, uint16_t ruleset = 0);
 
-	// Populate the provided vector, indexed by event tag, of the
-	// event tags associated with the given ruleset id. For
-	// example, event_tags[10] = true would mean that this ruleset
-	// relates to event tag 10.
-	void event_tags_for_ruleset(std::vector<bool> &event_tags, uint16_t ruleset);
+	// Populate the provided set of event types used by this ruleset.
+	void evttypes_for_ruleset(std::set<uint16_t> &evttypes, uint16_t ruleset);
 
 private:
 
-	struct filter_wrapper {
-		gen_event_filter *filter;
-
-		// Indexes from event tag to enabled/disabled.
-		std::vector<bool> event_tags;
+	class filter_wrapper {
+	public:
+		std::string name;
+		std::set<std::string> tags;
+		std::shared_ptr<gen_event_filter> filter;
 	};
 
+	typedef std::list<std::shared_ptr<filter_wrapper>> filter_wrapper_list;
+
 	// A group of filters all having the same ruleset
 	class ruleset_filters {
 	public:
@@ -89,63 +87,33 @@ private:
 
 		virtual ~ruleset_filters();
 
-		void add_filter(filter_wrapper *wrap);
-		void remove_filter(filter_wrapper *wrap);
+		void add_filter(std::shared_ptr<filter_wrapper> wrap);
+		void remove_filter(std::shared_ptr<filter_wrapper> wrap);
 
 		uint64_t num_filters();
 
-		bool run(gen_event *evt, uint32_t etag);
+		bool run(gen_event *evt);
 
-		void event_tags_for_ruleset(std::vector<bool> &event_tags);
+		void evttypes_for_ruleset(std::set<uint16_t> &evttypes);
 
 	private:
-		uint64_t m_num_filters;
+		void add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap);
+		void remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap);
 
-		// Maps from event tag to a list of filters. There can
-		// be multiple filters for a given event tag.
-		std::vector<std::list<filter_wrapper *> *> m_filter_by_event_tag;
+		// Vector indexes from event type to a set of filters. There can
+		// be multiple filters for a given event type.
+		// NOTE: This is used only when the event sub-type is 0.
+		std::vector<filter_wrapper_list> m_filter_by_event_type;
 
+		filter_wrapper_list m_filter_all_event_types;
+
+		// All filters added. Used to make num_filters() fast.
+		std::set<std::shared_ptr<filter_wrapper>> m_filters;
 	};
 
-	std::vector<ruleset_filters *> m_rulesets;
+	// Vector indexes from ruleset id to set of rules.
+	std::vector<std::shared_ptr<ruleset_filters>> m_rulesets;
 
-	// Maps from tag to list of filters having that tag.
-	std::map<std::string, std::list<filter_wrapper *>> m_filter_by_event_tag;
-
-	// This holds all the filters passed to add(), so they can
-	// be cleaned up.
-	std::map<std::string,filter_wrapper *> m_filters;
-};
-
-// falco_sinsp_ruleset is a specialization of falco_ruleset that
-// maps sinsp evttypes/syscalls to event tags.
-class falco_sinsp_ruleset : public falco_ruleset
-{
-public:
-	falco_sinsp_ruleset();
-	virtual ~falco_sinsp_ruleset();
-
-	void add(std::string &name,
-		 std::set<uint32_t> &evttypes,
-		 std::set<uint32_t> &syscalls,
-		 std::set<std::string> &tags,
-		 sinsp_filter* filter);
-
-	bool run(sinsp_evt *evt, uint16_t ruleset = 0);
-
-	// Populate the provided vector, indexed by event type, of the
-	// event types associated with the given ruleset id. For
-	// example, evttypes[10] = true would mean that this ruleset
-	// relates to event type 10.
-	void evttypes_for_ruleset(std::vector<bool> &evttypes, uint16_t ruleset);
-
-	// Populate the provided vector, indexed by syscall code, of the
-	// syscall codes associated with the given ruleset id. For
-	// example, syscalls[10] = true would mean that this ruleset
-	// relates to syscall code 10.
-	void syscalls_for_ruleset(std::vector<bool> &syscalls, uint16_t ruleset);
-
-private:
-	uint32_t evttype_to_event_tag(uint32_t evttype);
-	uint32_t syscall_to_event_tag(uint32_t syscallid);
+	// All filters added. The set of enabled filters is held in m_rulesets
+	std::set<std::shared_ptr<filter_wrapper>> m_filters;
 };

userspace/engine/token_bucket.cpp

@@ -1,89 +0,0 @@
-/*
-Copyright (C) 2019 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include <cstddef>
-#include <functional>
-#include <sys/time.h>
-
-#include "token_bucket.h"
-#include "utils.h"
-#include "banned.h" // This raises a compilation error when certain functions are used
-
-token_bucket::token_bucket():
-	token_bucket(sinsp_utils::get_current_time_ns)
-{
-}
-
-token_bucket::token_bucket(std::function<uint64_t()> timer)
-{
-	m_timer = timer;
-	init(1, 1);
-}
-
-token_bucket::~token_bucket()
-{
-}
-
-void token_bucket::init(double rate, double max_tokens, uint64_t now)
-{
-	m_rate = rate;
-	m_max_tokens = max_tokens;
-	m_tokens = max_tokens;
-	m_last_seen = now == 0 ? m_timer() : now;
-}
-
-bool token_bucket::claim()
-{
-	return claim(1, m_timer());
-}
-
-bool token_bucket::claim(double tokens, uint64_t now)
-{
-	double tokens_gained = m_rate * ((now - m_last_seen) / (1000000000.0));
-	m_last_seen = now;
-
-	m_tokens += tokens_gained;
-
-	//
-	// Cap at max_tokens
-	//
-	if(m_tokens > m_max_tokens)
-	{
-		m_tokens = m_max_tokens;
-	}
-
-	//
-	// If m_tokens is < tokens, can't claim.
-	//
-	if(m_tokens < tokens)
-	{
-		return false;
-	}
-
-	m_tokens -= tokens;
-
-	return true;
-}
-
-double token_bucket::get_tokens()
-{
-	return m_tokens;
-}
-
-uint64_t token_bucket::get_last_seen()
-{
-	return m_last_seen;
-}

userspace/engine/token_bucket.h

@@ -1,77 +0,0 @@
-/*
-Copyright (C) 2019 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#pragma once
-
-#include <cstdint>
-#include <functional>
-
-// A simple token bucket that accumulates tokens at a fixed rate and allows
-// for limited bursting in the form of "banked" tokens.
-class token_bucket
-{
-public:
-	token_bucket();
-	token_bucket(std::function<uint64_t()> timer);
-	virtual ~token_bucket();
-
-	//
-	// Initialize the token bucket and start accumulating tokens
-	//
-	void init(double rate, double max_tokens, uint64_t now = 0);
-
-	//
-	// Try to claim tokens tokens from the token bucket, using a
-	// timestamp of now. Returns true if the tokens could be
-	// claimed. Also updates internal metrics.
-	//
-	bool claim(double tokens, uint64_t now);
-
-	// Simpler version of claim that claims a single token and
-	// uses the current time for now
-	bool claim();
-
-	// Return the current number of tokens available
-	double get_tokens();
-
-	// Return the last time someone tried to claim a token.
-	uint64_t get_last_seen();
-
-private:
-	std::function<uint64_t()> m_timer;
-
-	//
-	// The number of tokens generated per second.
-	//
-	double m_rate;
-
-	//
-	// The maximum number of tokens that can be banked for future
-	// claim()s.
-	//
-	double m_max_tokens;
-
-	//
-	// The current number of tokens
-	//
-	double m_tokens;
-
-	//
-	// The last time claim() was called (or the object was created).
-	// Nanoseconds since the epoch.
-	//
-	uint64_t m_last_seen;
-};

userspace/falco/CMakeLists.txt

@@ -103,11 +103,11 @@ if(NOT MINIMAL_BUILD)
     "${GPR_LIB}"
     "${PROTOBUF_LIB}"
     "${CARES_LIB}"
-    "${OPENSSL_LIBRARY_SSL}"
-    "${OPENSSL_LIBRARY_CRYPTO}"
+    "${OPENSSL_LIBRARIES}"
     "${LIBYAML_LIB}"
     "${YAMLCPP_LIB}"
     "${CIVETWEB_LIB}"
+    "${CIVETWEB_CPP_LIB}"
   )
 endif()
 

userspace/falco/config_falco.h.in

@@ -28,6 +28,7 @@ limitations under the License.
 #define FALCO_SOURCE_DIR "${PROJECT_SOURCE_DIR}"
 #define FALCO_SOURCE_CONF_FILE "${PROJECT_SOURCE_DIR}/falco.yaml"
 #define FALCO_INSTALL_CONF_FILE "/etc/falco/falco.yaml"
+#define FALCO_ENGINE_PLUGINS_DIR "${FALCO_ABSOLUTE_SHARE_DIR}/plugins/"
 
 #define PROBE_NAME "@PROBE_NAME@"
 #define DRIVER_VERSION "@PROBE_VERSION@"

userspace/falco/configuration.cpp

@@ -16,6 +16,9 @@ limitations under the License.
 
 #include <algorithm>
 
+#include <list>
+#include <set>
+
 #include <dirent.h>
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -51,7 +54,16 @@ falco_configuration::~falco_configuration()
 void falco_configuration::init(string conf_filename, list<string> &cmdline_options)
 {
 	string m_config_file = conf_filename;
-	m_config = new yaml_configuration(m_config_file);
+	m_config = new yaml_configuration();
+	try
+	{
+		m_config->load_from_file(m_config_file);
+	}
+	catch(const std::exception& e)
+	{
+		std::cerr << "Cannot read config file (" + m_config_file + "): " + e.what() + "\n";
+		throw e;
+	}
 
 	init_cmdline_options(cmdline_options);
 
@@ -75,17 +87,17 @@ void falco_configuration::init(string conf_filename, list<string> &cmdline_optio
 
 	falco::outputs::config file_output;
 	file_output.name = "file";
-	if(m_config->get_scalar<bool>("file_output", "enabled", false))
+	if(m_config->get_scalar<bool>("file_output.enabled", false))
 	{
 		string filename, keep_alive;
-		filename = m_config->get_scalar<string>("file_output", "filename", "");
+		filename = m_config->get_scalar<string>("file_output.filename", "");
 		if(filename == string(""))
 		{
 			throw logic_error("Error reading config file (" + m_config_file + "): file output enabled but no filename in configuration block");
 		}
 		file_output.options["filename"] = filename;
 
-		keep_alive = m_config->get_scalar<string>("file_output", "keep_alive", "");
+		keep_alive = m_config->get_scalar<string>("file_output.keep_alive", "");
 		file_output.options["keep_alive"] = keep_alive;
 
 		m_outputs.push_back(file_output);
@@ -93,31 +105,31 @@ void falco_configuration::init(string conf_filename, list<string> &cmdline_optio
 
 	falco::outputs::config stdout_output;
 	stdout_output.name = "stdout";
-	if(m_config->get_scalar<bool>("stdout_output", "enabled", false))
+	if(m_config->get_scalar<bool>("stdout_output.enabled", false))
 	{
 		m_outputs.push_back(stdout_output);
 	}
 
 	falco::outputs::config syslog_output;
 	syslog_output.name = "syslog";
-	if(m_config->get_scalar<bool>("syslog_output", "enabled", false))
+	if(m_config->get_scalar<bool>("syslog_output.enabled", false))
 	{
 		m_outputs.push_back(syslog_output);
 	}
 
 	falco::outputs::config program_output;
 	program_output.name = "program";
-	if(m_config->get_scalar<bool>("program_output", "enabled", false))
+	if(m_config->get_scalar<bool>("program_output.enabled", false))
 	{
 		string program, keep_alive;
-		program = m_config->get_scalar<string>("program_output", "program", "");
+		program = m_config->get_scalar<string>("program_output.program", "");
 		if(program == string(""))
 		{
 			throw logic_error("Error reading config file (" + m_config_file + "): program output enabled but no program in configuration block");
 		}
 		program_output.options["program"] = program;
 
-		keep_alive = m_config->get_scalar<string>("program_output", "keep_alive", "");
+		keep_alive = m_config->get_scalar<string>("program_output.keep_alive", "");
 		program_output.options["keep_alive"] = keep_alive;
 
 		m_outputs.push_back(program_output);
@@ -125,10 +137,10 @@ void falco_configuration::init(string conf_filename, list<string> &cmdline_optio
 
 	falco::outputs::config http_output;
 	http_output.name = "http";
-	if(m_config->get_scalar<bool>("http_output", "enabled", false))
+	if(m_config->get_scalar<bool>("http_output.enabled", false))
 	{
 		string url;
-		url = m_config->get_scalar<string>("http_output", "url", "");
+		url = m_config->get_scalar<string>("http_output.url", "");
 
 		if(url == string(""))
 		{
@@ -136,25 +148,29 @@ void falco_configuration::init(string conf_filename, list<string> &cmdline_optio
 		}
 		http_output.options["url"] = url;
 
+		string user_agent;
+		user_agent = m_config->get_scalar<string>("http_output.user_agent","falcosecurity/falco");
+		http_output.options["user_agent"] = user_agent;
+
 		m_outputs.push_back(http_output);
 	}
 
-	m_grpc_enabled = m_config->get_scalar<bool>("grpc", "enabled", false);
-	m_grpc_bind_address = m_config->get_scalar<string>("grpc", "bind_address", "0.0.0.0:5060");
-	m_grpc_threadiness = m_config->get_scalar<uint32_t>("grpc", "threadiness", 0);
+	m_grpc_enabled = m_config->get_scalar<bool>("grpc.enabled", false);
+	m_grpc_bind_address = m_config->get_scalar<string>("grpc.bind_address", "0.0.0.0:5060");
+	m_grpc_threadiness = m_config->get_scalar<uint32_t>("grpc.threadiness", 0);
 	if(m_grpc_threadiness == 0)
 	{
 		m_grpc_threadiness = falco::utils::hardware_concurrency();
 	}
 	// todo > else limit threadiness to avoid oversubscription?
-	m_grpc_private_key = m_config->get_scalar<string>("grpc", "private_key", "/etc/falco/certs/server.key");
-	m_grpc_cert_chain = m_config->get_scalar<string>("grpc", "cert_chain", "/etc/falco/certs/server.crt");
-	m_grpc_root_certs = m_config->get_scalar<string>("grpc", "root_certs", "/etc/falco/certs/ca.crt");
+	m_grpc_private_key = m_config->get_scalar<string>("grpc.private_key", "/etc/falco/certs/server.key");
+	m_grpc_cert_chain = m_config->get_scalar<string>("grpc.cert_chain", "/etc/falco/certs/server.crt");
+	m_grpc_root_certs = m_config->get_scalar<string>("grpc.root_certs", "/etc/falco/certs/ca.crt");
 
 	falco::outputs::config grpc_output;
 	grpc_output.name = "grpc";
 	// gRPC output is enabled only if gRPC server is enabled too
-	if(m_config->get_scalar<bool>("grpc_output", "enabled", true) && m_grpc_enabled)
+	if(m_config->get_scalar<bool>("grpc_output.enabled", true) && m_grpc_enabled)
 	{
 		m_outputs.push_back(grpc_output);
 	}
@@ -170,8 +186,8 @@ void falco_configuration::init(string conf_filename, list<string> &cmdline_optio
 
 	m_output_timeout = m_config->get_scalar<uint32_t>("output_timeout", 2000);
 
-	m_notifications_rate = m_config->get_scalar<uint32_t>("outputs", "rate", 1);
-	m_notifications_max_burst = m_config->get_scalar<uint32_t>("outputs", "max_burst", 1000);
+	m_notifications_rate = m_config->get_scalar<uint32_t>("outputs.rate", 1);
+	m_notifications_max_burst = m_config->get_scalar<uint32_t>("outputs.max_burst", 1000);
 
 	string priority = m_config->get_scalar<string>("priority", "debug");
 	vector<string>::iterator it;
@@ -192,15 +208,15 @@ void falco_configuration::init(string conf_filename, list<string> &cmdline_optio
 	falco_logger::log_stderr = m_config->get_scalar<bool>("log_stderr", false);
 	falco_logger::log_syslog = m_config->get_scalar<bool>("log_syslog", true);
 
-	m_webserver_enabled = m_config->get_scalar<bool>("webserver", "enabled", false);
-	m_webserver_listen_port = m_config->get_scalar<uint32_t>("webserver", "listen_port", 8765);
-	m_webserver_k8s_audit_endpoint = m_config->get_scalar<string>("webserver", "k8s_audit_endpoint", "/k8s-audit");
-	m_webserver_k8s_healthz_endpoint = m_config->get_scalar<string>("webserver", "k8s_healthz_endpoint", "/healthz");
-	m_webserver_ssl_enabled = m_config->get_scalar<bool>("webserver", "ssl_enabled", false);
-	m_webserver_ssl_certificate = m_config->get_scalar<string>("webserver", "ssl_certificate", "/etc/falco/falco.pem");
+	m_webserver_enabled = m_config->get_scalar<bool>("webserver.enabled", false);
+	m_webserver_listen_port = m_config->get_scalar<uint32_t>("webserver.listen_port", 8765);
+	m_webserver_k8s_audit_endpoint = m_config->get_scalar<string>("webserver.k8s_audit_endpoint", "/k8s-audit");
+	m_webserver_k8s_healthz_endpoint = m_config->get_scalar<string>("webserver.k8s_healthz_endpoint", "/healthz");
+	m_webserver_ssl_enabled = m_config->get_scalar<bool>("webserver.ssl_enabled", false);
+	m_webserver_ssl_certificate = m_config->get_scalar<string>("webserver.ssl_certificate", "/etc/falco/falco.pem");
 
 	std::list<string> syscall_event_drop_acts;
-	m_config->get_sequence(syscall_event_drop_acts, "syscall_event_drops", "actions");
+	m_config->get_sequence(syscall_event_drop_acts, "syscall_event_drops.actions");
 
 	for(std::string &act : syscall_event_drop_acts)
 	{
@@ -239,33 +255,61 @@ void falco_configuration::init(string conf_filename, list<string> &cmdline_optio
 		m_syscall_evt_drop_actions.insert(syscall_evt_drop_action::IGNORE);
 	}
 
-	m_syscall_evt_drop_threshold = m_config->get_scalar<double>("syscall_event_drops", "threshold", .1);
+	m_syscall_evt_drop_threshold = m_config->get_scalar<double>("syscall_event_drops.threshold", .1);
 	if(m_syscall_evt_drop_threshold < 0 || m_syscall_evt_drop_threshold > 1)
 	{
 		throw logic_error("Error reading config file (" + m_config_file + "): syscall event drops threshold must be a double in the range [0, 1]");
 	}
-	m_syscall_evt_drop_rate = m_config->get_scalar<double>("syscall_event_drops", "rate", .03333);
-	m_syscall_evt_drop_max_burst = m_config->get_scalar<double>("syscall_event_drops", "max_burst", 1);
-	m_syscall_evt_simulate_drops = m_config->get_scalar<bool>("syscall_event_drops", "simulate_drops", false);
+	m_syscall_evt_drop_rate = m_config->get_scalar<double>("syscall_event_drops.rate", .03333);
+	m_syscall_evt_drop_max_burst = m_config->get_scalar<double>("syscall_event_drops.max_burst", 1);
+	m_syscall_evt_simulate_drops = m_config->get_scalar<bool>("syscall_event_drops.simulate_drops", false);
 
-	m_syscall_evt_timeout_max_consecutives = m_config->get_scalar<uint32_t>("syscall_event_timeouts", "max_consecutives", 1000);
+	m_syscall_evt_timeout_max_consecutives = m_config->get_scalar<uint32_t>("syscall_event_timeouts.max_consecutives", 1000);
 	if(m_syscall_evt_timeout_max_consecutives == 0)
 	{
 		throw logic_error("Error reading config file(" + m_config_file + "): the maximum consecutive timeouts without an event must be an unsigned integer > 0");
 	}
 
-	m_metadata_download_max_mb = m_config->get_scalar<uint32_t>("metadata_download", "max_mb", 100);
+	m_metadata_download_max_mb = m_config->get_scalar<uint32_t>("metadata_download.max_mb", 100);
 	if(m_metadata_download_max_mb > 1024)
 	{
 		throw logic_error("Error reading config file(" + m_config_file + "): metadata download maximum size should be < 1024 Mb");
 	}
-	m_metadata_download_chunk_wait_us = m_config->get_scalar<uint32_t>("metadata_download", "chunk_wait_us", 1000);
-	m_metadata_download_watch_freq_sec = m_config->get_scalar<uint32_t>("metadata_download", "watch_freq_sec", 1);
+	m_metadata_download_chunk_wait_us = m_config->get_scalar<uint32_t>("metadata_download.chunk_wait_us", 1000);
+	m_metadata_download_watch_freq_sec = m_config->get_scalar<uint32_t>("metadata_download.watch_freq_sec", 1);
 	if(m_metadata_download_watch_freq_sec == 0)
 	{
 		throw logic_error("Error reading config file(" + m_config_file + "): metadata download watch frequency seconds must be an unsigned integer > 0");
 	}
-	
+
+	std::set<std::string> load_plugins;
+
+	bool load_plugins_node_defined = m_config->is_defined("load_plugins");
+
+	m_config->get_sequence<set<string>>(load_plugins, "load_plugins");
+
+	std::list<falco_configuration::plugin_config> plugins;
+	try
+	{
+		m_config->get_sequence<std::list<falco_configuration::plugin_config>>(plugins, string("plugins"));
+	}
+	catch (exception &e)
+	{
+		// Might be thrown due to not being able to open files
+		throw logic_error("Error reading config file(" + m_config_file + "): could not load plugins config: " + e.what());
+	}
+
+	// If load_plugins was specified, only save plugins matching those in values
+	for (auto &p : plugins)
+	{
+		// If load_plugins was not specified at all, every
+		// plugin is added. Otherwise, the plugin must be in
+		// the load_plugins list.
+		if(!load_plugins_node_defined || load_plugins.find(p.m_name) != load_plugins.end())
+		{
+			m_plugins.push_back(p);
+		}
+	}
 }
 
 void falco_configuration::read_rules_file_directory(const string &path, list<string> &rules_filenames)
@@ -357,19 +401,11 @@ void falco_configuration::init_cmdline_options(list<string> &cmdline_options)
 void falco_configuration::set_cmdline_option(const string &opt)
 {
 	pair<string, string> keyval;
-	pair<string, string> subkey;
 
 	if(!split(opt, '=', keyval))
 	{
 		throw logic_error("Error parsing config option \"" + opt + "\". Must be of the form key=val or key.subkey=val");
 	}
 
-	if(split(keyval.first, '.', subkey))
-	{
-		m_config->set_scalar(subkey.first, subkey.second, keyval.second);
-	}
-	else
-	{
-		m_config->set_scalar(keyval.first, keyval.second);
-	}
+	m_config->set_scalar(keyval.first, keyval.second);
 }

userspace/falco/configuration.h

@@ -25,6 +25,9 @@ limitations under the License.
 #include <list>
 #include <set>
 #include <iostream>
+#include <fstream>
+
+#include "config_falco.h"
 
 #include "event_drops.h"
 #include "falco_outputs.h"
@@ -32,106 +35,152 @@ limitations under the License.
 class yaml_configuration
 {
 public:
-	std::string m_path;
-	yaml_configuration(const std::string& path)
+	/**
+	* Load the YAML document represented by the input string.
+	*/
+	void load_from_string(const std::string& input)
 	{
-		m_path = path;
-		YAML::Node config;
-		std::vector<falco::outputs::config> outputs;
-		try
-		{
-			m_root = YAML::LoadFile(path);
-		}
-		catch(const YAML::BadFile& ex)
-		{
-			std::cerr << "Error reading config file (" + path + "): " + ex.what() + "\n";
-			throw;
-		}
-		catch(const YAML::ParserException& ex)
-		{
-			std::cerr << "Cannot read config file (" + path + "): " + ex.what() + "\n";
-			throw;
-		}
+		m_root = YAML::Load(input);
 	}
 
 	/**
-	* Get a scalar value defined at the top level of the config
+	* Load the YAML document from the given file path.
+	*/
+	void load_from_file(const std::string& path)
+	{
+		m_root = YAML::LoadFile(path);
+	}
+
+	/**
+	 * Clears the internal loaded document.
+	 */
+	void clear()
+	{
+		m_root = YAML::Node();
+	}
+
+	/**
+	* Get a scalar value from the node identified by key.
 	*/
 	template<typename T>
 	const T get_scalar(const std::string& key, const T& default_value)
 	{
-		try
+		YAML::Node node;
+		get_node(node, key);
+		if(node.IsDefined())
 		{
-			auto node = m_root[key];
-			if(node.IsDefined())
-			{
-				return node.as<T>();
-			}
-		}
-		catch(const YAML::BadConversion& ex)
-		{
-			std::cerr << "Cannot read config file (" + m_path + "): wrong type at key " + key + "\n";
-			throw;
+			return node.as<T>();
 		}
 
 		return default_value;
 	}
 
 	/**
-	 * Set the top-level node identified by key to value
+	 * Set the node identified by key to value.
 	 */
 	template<typename T>
 	void set_scalar(const std::string& key, const T& value)
 	{
-		auto node = m_root;
-		if(node.IsDefined())
-		{
-			node[key] = value;
-		}
+		YAML::Node node;
+		get_node(node, key);
+		node = value;
 	}
 
 	/**
-	* Get a scalar value defined inside a 2 level nested structure like:
-	* file_output:
-	*   enabled: true
-	*   filename: output_file.txt
-	*
-	* get_scalar<bool>("file_output", "enabled", false)
+	* Get the sequence value from the node identified by key.
 	*/
 	template<typename T>
-	const T get_scalar(const std::string& key, const std::string& subkey, const T& default_value)
+	void get_sequence(T& ret, const std::string& key)
+	{
+		YAML::Node node;
+		get_node(node, key);
+		return get_sequence_from_node<T>(ret, node);
+	}
+
+	/**
+	* Return true if the node identified by key is defined.
+	*/
+	bool is_defined(const std::string& key)
+	{
+		YAML::Node node;
+		get_node(node, key);
+		return node.IsDefined();
+	}
+
+private:
+	YAML::Node m_root;
+	std::string m_input;
+	bool m_is_from_file;
+
+	/**
+	 * Key is a string representing a node in the YAML document.
+	 * The provided key string can navigate the document in its
+	 * nested nodes, with arbitrary depth. The key string follows
+	 * this regular language:
+	 * 
+	 * Key 		:= NodeKey ('.' NodeKey)*
+	 * NodeKey	:= (any)+ ('[' (integer)+ ']')*
+	 * 
+	 * Some examples of accepted key strings:
+	 * - NodeName
+	 * - ListValue[3].subvalue
+	 * - MatrixValue[1][3]
+	 * - value1.subvalue2.subvalue3
+	 */
+	void get_node(YAML::Node &ret, const std::string &key)
 	{
 		try
 		{
-			auto node = m_root[key][subkey];
-			if(node.IsDefined())
+			char c;
+			bool should_shift;
+			std::string nodeKey;
+			ret.reset(m_root);
+			for(std::string::size_type i = 0; i < key.size(); ++i)
 			{
-				return node.as<T>();
+				c = key[i];
+				should_shift = c == '.' || c == '[' || i == key.size() - 1;
+
+				if (c != '.' && c != '[')
+				{
+					if (i > 0 && nodeKey.empty() && key[i - 1] != '.')
+					{
+						throw runtime_error(
+							"Parsing error: expected '.' character at pos " 
+							+ to_string(i - 1));
+					}
+					nodeKey += c;
+				}
+
+				if (should_shift)
+				{
+					if (nodeKey.empty())
+					{
+						throw runtime_error(
+							"Parsing error: unexpected character at pos " 
+							+ to_string(i));
+					}
+					ret.reset(ret[nodeKey]);
+					nodeKey.clear();
+				}
+				if (c == '[')
+				{
+					auto close_param_idx = key.find(']', i);
+					int nodeIdx = std::stoi(key.substr(i + 1, close_param_idx - i - 1));
+					ret.reset(ret[nodeIdx]);
+					i = close_param_idx;
+					if (i < key.size() - 1 && key[i + 1] == '.')
+					{
+						i++;
+					}
+				}
 			}
 		}
-		catch(const YAML::BadConversion& ex)
+		catch(const std::exception& e)
 		{
-			std::cerr << "Cannot read config file (" + m_path + "): wrong type at key " + key + "." + subkey + "\n";
-			throw;
-		}
-
-		return default_value;
-	}
-
-	/**
-	 * Set the second-level node identified by key[key][subkey] to value.
-	 */
-	template<typename T>
-	void set_scalar(const std::string& key, const std::string& subkey, const T& value)
-	{
-		auto node = m_root;
-		if(node.IsDefined())
-		{
-			node[key][subkey] = value;
+			throw runtime_error("Config error at key \"" + key + "\": " + string(e.what()));
 		}
 	}
-
-	// called with the last variadic arg (where the sequence is expected to be found)
+	
 	template<typename T>
 	void get_sequence_from_node(T& ret, const YAML::Node& node)
 	{
@@ -150,40 +199,20 @@ public:
 			}
 		}
 	}
-
-	// called with the last variadic arg (where the sequence is expected to be found)
-	template<typename T>
-	void get_sequence(T& ret, const std::string& name)
-	{
-		return get_sequence_from_node<T>(ret, m_root[name]);
-	}
-
-	// called with the last variadic arg (where the sequence is expected to be found)
-	template<typename T>
-	void get_sequence(T& ret, const std::string& key, const std::string& subkey)
-	{
-		try
-		{
-			auto node = m_root[key];
-			if(node.IsDefined())
-			{
-				return get_sequence_from_node<T>(ret, node[subkey]);
-			}
-		}
-		catch(const YAML::BadConversion& ex)
-		{
-			std::cerr << "Cannot read config file (" + m_path + "): wrong type at key " + key + "." + subkey +"\n";
-			throw;
-		}
-	}
-
-private:
-	YAML::Node m_root;
 };
 
 class falco_configuration
 {
 public:
+
+	typedef struct {
+	public:
+		std::string m_name;
+		std::string m_library_path;
+		std::string m_init_config;
+		std::string m_open_params;
+	} plugin_config;
+
 	falco_configuration();
 	virtual ~falco_configuration();
 
@@ -234,6 +263,8 @@ public:
 	uint32_t m_metadata_download_chunk_wait_us;
 	uint32_t m_metadata_download_watch_freq_sec;
 
+	std::vector<plugin_config> m_plugins;
+
 private:
 	void init_cmdline_options(std::list<std::string>& cmdline_options);
 
@@ -247,3 +278,123 @@ private:
 
 	yaml_configuration* m_config;
 };
+
+namespace YAML {
+	template<>
+	struct convert<nlohmann::json> {
+		static bool decode(const Node& node, nlohmann::json& res)
+		{
+			int int_val;
+			double double_val;
+			bool bool_val;
+			std::string str_val;
+			nlohmann::json sub{};
+
+			switch (node.Type()) {
+				case YAML::NodeType::Map:
+					for (auto &&it: node)
+					{
+						YAML::convert<nlohmann::json>::decode(it.second, sub);
+						res[it.first.as<std::string>()] = sub;
+					}
+					break;
+				case YAML::NodeType::Sequence:
+					for (auto &&it : node)
+					{
+						YAML::convert<nlohmann::json>::decode(it, sub);
+						res.emplace_back(sub);
+					}
+					break;
+				case YAML::NodeType::Scalar:
+					if (YAML::convert<int>::decode(node, int_val))
+					{
+						res = int_val;
+					}
+					else if (YAML::convert<double>::decode(node, double_val))
+					{
+						res = double_val;
+					}
+					else if (YAML::convert<bool>::decode(node, bool_val))
+					{
+						res = bool_val;
+					}
+					else if (YAML::convert<std::string>::decode(node, str_val))
+					{
+						res = str_val;
+					}
+				default:
+					break;
+			}
+			
+			return true;
+		}
+	};
+
+	template<>
+	struct convert<falco_configuration::plugin_config> {
+
+		// Note that this loses the distinction between init configs
+		// defined as YAML maps or as opaque strings.
+		static Node encode(const falco_configuration::plugin_config & rhs) {
+			Node node;
+			node["name"] = rhs.m_name;
+			node["library_path"] = rhs.m_library_path;
+			node["init_config"] = rhs.m_init_config;
+			node["open_params"] = rhs.m_open_params;
+			return node;
+		}
+
+		static bool decode(const Node& node, falco_configuration::plugin_config & rhs) {
+			if(!node.IsMap())
+			{
+				return false;
+			}
+
+			if(!node["name"])
+			{
+				return false;
+			}
+			rhs.m_name = node["name"].as<std::string>();
+
+			if(!node["library_path"])
+			{
+				return false;
+			}
+			rhs.m_library_path = node["library_path"].as<std::string>();
+			if(rhs.m_library_path.at(0) != '/')
+			{
+				// prepend share dir if path is not absolute
+				rhs.m_library_path = string(FALCO_ENGINE_PLUGINS_DIR) + rhs.m_library_path;
+			}
+
+			if(!node["init_config"])
+			{
+				return false;
+			}
+			// By convention, if the init config is a YAML map we convert it
+			// in a JSON object string. This is useful for plugins implementing
+			// the `get_init_schema` API symbol, which right now support the
+			// JSON Schema specific. If we ever support other schema/data types,
+			// we may want to bundle the conversion logic in an ad-hoc class.
+			// The benefit of this is being able of parsing/editing the config as
+			// a YAML map instead of having an opaque string.
+			if (node["init_config"].IsMap())
+			{
+				nlohmann::json json;
+				YAML::convert<nlohmann::json>::decode(node["init_config"], json);
+				rhs.m_init_config = json.dump();
+			}
+			else
+			{
+				rhs.m_init_config = node["init_config"].as<std::string>();
+			}
+
+			if(node["open_params"])
+			{
+				rhs.m_open_params = node["open_params"].as<std::string>();
+			}
+
+			return true;
+		}
+	};
+}

userspace/falco/falco.cpp

@@ -33,6 +33,9 @@ limitations under the License.
 #include <getopt.h>
 
 #include <sinsp.h>
+#include <filter.h>
+#include <eventformatter.h>
+#include <plugin.h>
 
 #include "logger.h"
 #include "utils.h"
@@ -58,6 +61,9 @@ bool g_reopen_outputs = false;
 bool g_restart = false;
 bool g_daemonized = false;
 
+static std::string syscall_source = "syscall";
+static std::string k8s_audit_source = "k8s_audit";
+
 //
 // Helper functions
 //
@@ -88,7 +94,6 @@ static void usage()
 	   " -h, --help                    Print this page\n"
 	   " -c                            Configuration file (default " FALCO_SOURCE_CONF_FILE ", " FALCO_INSTALL_CONF_FILE ")\n"
 	   " -A                            Monitor all events, including those with EF_DROP_SIMPLE_CONS flag.\n"
-	   " --alternate-lua-dir <path>    Specify an alternate path for loading Falco lua files\n"
 	   " -b, --print-base64            Print data buffers in base64.\n"
 	   "                               This is useful for encoding binary data that needs to be used over media designed to.\n"
 	   " --cri <path>                  Path to CRI socket for container metadata.\n"
@@ -131,6 +136,11 @@ static void usage()
 	   " -l <rule>                     Show the name and description of the rule with name <rule> and exit.\n"
 	   " --list [<source>]             List all defined fields. If <source> is provided, only list those fields for\n"
 	   "                               the source <source>. Current values for <source> are \"syscall\", \"k8s_audit\"\n"
+	   " --list-fields-markdown [<source>]\n"
+	   "                               List fields in md\n"
+#ifndef MUSL_OPTIMIZED
+	   " --list-plugins                Print info on all loaded plugins and exit.\n"
+#endif
 #ifndef MINIMAL_BUILD
 	   " -m <url[,marathon_url]>, --mesos-api <url[,marathon_url]>\n"
 	   "                               Enable Mesos support by connecting to the API server\n"
@@ -243,6 +253,7 @@ static std::string read_file(std::string filename)
 uint64_t do_inspect(falco_engine *engine,
 			falco_outputs *outputs,
 			sinsp* inspector,
+		        std::string &event_source,
 			falco_configuration &config,
 			syscall_evt_drop_mgr &sdropmgr,
 			uint64_t duration_to_tot_ns,
@@ -307,7 +318,8 @@ uint64_t do_inspect(falco_engine *engine,
 			if(unlikely(ev == nullptr))
 			{
 				timeouts_since_last_success_or_msg++;
-				if(timeouts_since_last_success_or_msg > config.m_syscall_evt_timeout_max_consecutives)
+				if(event_source == syscall_source &&
+				   (timeouts_since_last_success_or_msg > config.m_syscall_evt_timeout_max_consecutives))
 				{
 					std::string rule = "Falco internal: timeouts notification";
 					std::string msg = rule + ". " + std::to_string(config.m_syscall_evt_timeout_max_consecutives) + " consecutive timeouts without event.";
@@ -371,7 +383,7 @@ uint64_t do_inspect(falco_engine *engine,
 		// engine, which will match the event against the set
 		// of rules. If a match is found, pass the event to
 		// the outputs.
-		unique_ptr<falco_engine::rule_result> res = engine->process_sinsp_event(ev);
+		unique_ptr<falco_engine::rule_result> res = engine->process_event(event_source, ev);
 		if(res)
 		{
 			outputs->handle_event(res->evt, res->rule, res->source, res->priority_num, res->format, res->tags);
@@ -424,21 +436,62 @@ static void print_all_ignored_events(sinsp *inspector)
 	printf("\n");
 }
 
+static void check_for_ignored_events(sinsp &inspector, falco_engine &engine)
+{
+	std::set<uint16_t> evttypes;
+	sinsp_evttables* einfo = inspector.get_event_info_tables();
+	const struct ppm_event_info* etable = einfo->m_event_info;
+
+	engine.evttypes_for_ruleset(syscall_source, evttypes);
+
+	// Save event names so we don't warn for both the enter and exit event.
+	std::set<std::string> warn_event_names;
+
+	for(auto evtnum : evttypes)
+	{
+		if(evtnum == PPME_GENERIC_E || evtnum == PPME_GENERIC_X)
+		{
+			continue;
+		}
+
+		if(!sinsp::simple_consumer_consider_evtnum(evtnum))
+		{
+			std::string name = etable[evtnum].name;
+			if(warn_event_names.find(name) == warn_event_names.end())
+			{
+				warn_event_names.insert(name);
+			}
+		}
+	}
+
+	// Print a single warning with the list of ignored events
+	if (!warn_event_names.empty())
+	{
+		std::string skipped_events;
+		bool first = true;
+		for (const auto& evtname : warn_event_names)
+		{
+			if (first)
+			{
+				skipped_events += evtname;
+				first = false;
+			} else
+			{
+				skipped_events += "," + evtname;
+			}
+		}
+		fprintf(stderr,"Rules match ignored syscall: warning (ignored-evttype):\n         loaded rules match the following events: %s;\n         but these events are not returned unless running falco with -A\n", skipped_events.c_str());
+	}
+}
+
 static void list_source_fields(falco_engine *engine, bool verbose, bool names_only, std::string &source)
 {
 	if(source.size() > 0 &&
-	   !(source == "syscall" || source == "k8s_audit"))
+	   !engine->is_source_valid(source))
 	{
-		throw std::invalid_argument("Value for --list must be \"syscall\" or \"k8s_audit\"");
-	}
-	if(source == "" || source == "syscall")
-	{
-		list_fields(verbose, false, names_only);
-	}
-	if(source == "" || source == "k8s_audit")
-	{
-		engine->list_fields(names_only);
+		throw std::invalid_argument("Value for --list must be a valid source type");
 	}
+	engine->list_fields(source, verbose, names_only);
 }
 
 //
@@ -482,6 +535,7 @@ int falco_init(int argc, char **argv)
 	bool print_ignored_events = false;
 	bool list_flds = false;
 	string list_flds_source = "";
+	bool list_plugins = false;
 	bool print_support = false;
 	string cri_socket_path;
 	bool cri_async = true;
@@ -512,7 +566,6 @@ int falco_init(int argc, char **argv)
 
 	static struct option long_options[] =
 		{
-			{"alternate-lua-dir", required_argument, 0},
 			{"cri", required_argument, 0},
 			{"daemon", no_argument, 0, 'd'},
 			{"disable-cri-async", no_argument, 0, 0},
@@ -523,6 +576,7 @@ int falco_init(int argc, char **argv)
 			{"k8s-api", required_argument, 0, 'k'},
 			{"k8s-node", required_argument, 0},
 			{"list", optional_argument, 0},
+			{"list-plugins", no_argument, 0},
 			{"mesos-api", required_argument, 0, 'm'},
 			{"option", required_argument, 0, 'o'},
 			{"pidfile", required_argument, 0, 'P'},
@@ -542,8 +596,7 @@ int falco_init(int argc, char **argv)
 	{
 		set<string> disabled_rule_substrings;
 		string substring;
-		string all_rules = "";
-		string alternate_lua_dir = FALCO_ENGINE_SOURCE_LUA_DIR;
+		string all_rules;
 		set<string> disabled_rule_tags;
 		set<string> enabled_rule_tags;
 
@@ -715,6 +768,12 @@ int falco_init(int argc, char **argv)
 						list_flds_source = optarg;
 					}
 				}
+#ifndef MUSL_OPTIMIZED
+				else if (string(long_options[long_index].name) == "list-plugins")
+				{
+					list_plugins = true;
+				}
+#endif
 				else if (string(long_options[long_index].name) == "stats-interval")
 				{
 					stats_interval = atoi(optarg);
@@ -730,16 +789,6 @@ int falco_init(int argc, char **argv)
 						disable_sources.insert(optarg);
 					}
 				}
-				else if (string(long_options[long_index].name)== "alternate-lua-dir")
-				{
-					if(optarg != NULL)
-					{
-						alternate_lua_dir = optarg;
-						if (alternate_lua_dir.back() != '/') {
-							alternate_lua_dir += '/';
-						}
-					}
-				}
 				break;
 
 			default:
@@ -775,30 +824,34 @@ int falco_init(int argc, char **argv)
 			return EXIT_SUCCESS;
 		}
 
-		engine = new falco_engine(true, alternate_lua_dir);
-		engine->set_inspector(inspector);
+		engine = new falco_engine(true);
 		engine->set_extra(output_format, replace_container_info);
 
-		if(list_flds)
-		{
-			list_source_fields(engine, verbose, names_only, list_flds_source);
-			return EXIT_SUCCESS;
-		}
+		// Create "factories" that can create filters/formatters for
+		// syscalls and k8s audit events.
+		std::shared_ptr<gen_event_filter_factory> syscall_filter_factory(new sinsp_filter_factory(inspector));
+		std::shared_ptr<gen_event_filter_factory> k8s_audit_filter_factory(new json_event_filter_factory());
+
+		std::shared_ptr<gen_event_formatter_factory> syscall_formatter_factory(new sinsp_evt_formatter_factory(inspector));
+		std::shared_ptr<gen_event_formatter_factory> k8s_audit_formatter_factory(new json_event_formatter_factory(k8s_audit_filter_factory));
+
+		engine->add_source(syscall_source, syscall_filter_factory, syscall_formatter_factory);
+		engine->add_source(k8s_audit_source, k8s_audit_filter_factory, k8s_audit_formatter_factory);
 
 		if(disable_sources.size() > 0)
 		{
 			auto it = disable_sources.begin();
 			while(it != disable_sources.end())
 			{
-				if(*it != "syscall" && *it != "k8s_audit")
+				if(*it != syscall_source && *it != k8s_audit_source)
 				{
 					it = disable_sources.erase(it);
 					continue;
 				}
 				++it;
 			}
-			disable_syscall = disable_sources.count("syscall") > 0;
-			disable_k8s_audit = disable_sources.count("k8s_audit") > 0;
+			disable_syscall = disable_sources.count(syscall_source) > 0;
+			disable_k8s_audit = disable_sources.count(k8s_audit_source) > 0;
 			if (disable_syscall && disable_k8s_audit) {
 				throw std::invalid_argument("The event source \"syscall\" and \"k8s_audit\" can not be disabled together");
 			}
@@ -879,6 +932,135 @@ int falco_init(int argc, char **argv)
 			throw std::runtime_error("Could not find configuration file at " + conf_filename);
 		}
 
+		// The event source is syscall by default. If an input
+		// plugin was found, the source is the source of that
+		// plugin.
+		std::string event_source = syscall_source;
+
+		// All filterchecks created by plugins go in this
+		// list. If we ever support multiple event sources at
+		// the same time, this (and the below factories) will
+		// have to be a map from event source to filtercheck
+		// list.
+		filter_check_list plugin_filter_checks;
+
+		// Factories that can create filters/formatters for
+		// the (single) source supported by the (single) input plugin.
+		std::shared_ptr<gen_event_filter_factory> plugin_filter_factory(new sinsp_filter_factory(inspector, plugin_filter_checks));
+		std::shared_ptr<gen_event_formatter_factory> plugin_formatter_factory(new sinsp_evt_formatter_factory(inspector, plugin_filter_checks));
+
+		std::shared_ptr<sinsp_plugin> input_plugin;
+		std::list<std::shared_ptr<sinsp_plugin>> extractor_plugins;
+		for(auto &p : config.m_plugins)
+		{
+			std::shared_ptr<sinsp_plugin> plugin;
+#ifdef MUSL_OPTIMIZED
+			throw std::invalid_argument(string("Can not load/use plugins with musl optimized build"));
+#else
+			falco_logger::log(LOG_INFO, "Loading plugin (" + p.m_name + ") from file " + p.m_library_path + "\n");
+
+			plugin = sinsp_plugin::register_plugin(inspector,
+							       p.m_library_path,
+							       (p.m_init_config.empty() ? NULL : (char *)p.m_init_config.c_str()),
+							       plugin_filter_checks);
+#endif
+
+			if(plugin->type() == TYPE_SOURCE_PLUGIN)
+			{
+				sinsp_source_plugin *splugin = static_cast<sinsp_source_plugin *>(plugin.get());
+
+				if(input_plugin)
+				{
+					throw std::invalid_argument(string("Can not load multiple source plugins. ") + input_plugin->name() + " already loaded");
+				}
+
+				input_plugin = plugin;
+				event_source = splugin->event_source();
+
+				inspector->set_input_plugin(p.m_name);
+				if(!p.m_open_params.empty())
+				{
+					inspector->set_input_plugin_open_params(p.m_open_params.c_str());
+				}
+
+				engine->add_source(event_source, plugin_filter_factory, plugin_formatter_factory);
+
+			} else {
+				extractor_plugins.push_back(plugin);
+			}
+		}
+
+		// Ensure that extractor plugins are compatible with the event source.
+		// Also, ensure that extractor plugins don't have overlapping compatible event sources.
+		std::set<std::string> compat_sources_seen;
+		for(auto plugin : extractor_plugins)
+		{
+			// If the extractor plugin names compatible sources,
+			// ensure that the input plugin's source is in the list
+			// of compatible sources.
+			sinsp_extractor_plugin *eplugin = static_cast<sinsp_extractor_plugin *>(plugin.get());
+			const std::set<std::string> &compat_sources = eplugin->extract_event_sources();
+			if(input_plugin &&
+			   !compat_sources.empty())
+			{
+				if (compat_sources.find(event_source) == compat_sources.end())
+				{
+					throw std::invalid_argument(string("Extractor plugin not compatible with event source ") + event_source);
+				}
+
+				for(const auto &compat_source : compat_sources)
+				{
+					if(compat_sources_seen.find(compat_source) != compat_sources_seen.end())
+					{
+						throw std::invalid_argument(string("Extractor plugins have overlapping compatible event source ") + compat_source);
+					}
+					compat_sources_seen.insert(compat_source);
+				}
+			}
+		}
+
+		if(config.m_json_output)
+		{
+			syscall_formatter_factory->set_output_format(gen_event_formatter::OF_JSON);
+			k8s_audit_formatter_factory->set_output_format(gen_event_formatter::OF_JSON);
+			plugin_formatter_factory->set_output_format(gen_event_formatter::OF_JSON);
+		}
+
+		std::list<sinsp_plugin::info> infos = sinsp_plugin::plugin_infos(inspector);
+
+		if(list_plugins)
+		{
+			std::ostringstream os;
+
+			for(auto &info : infos)
+			{
+				os << "Name: " << info.name << std::endl;
+				os << "Description: " << info.description << std::endl;
+				os << "Contact: " << info.contact << std::endl;
+				os << "Version: " << info.plugin_version.as_string() << std::endl;
+
+				if(info.type == TYPE_SOURCE_PLUGIN)
+				{
+					os << "Type: source plugin" << std::endl;
+					os << "ID: " << info.id << std::endl;
+				}
+				else
+				{
+					os << "Type: extractor plugin" << std::endl;
+				}
+				os << std::endl;
+			}
+
+			printf("%lu Plugins Loaded:\n\n%s\n", infos.size(), os.str().c_str());
+			return EXIT_SUCCESS;
+		}
+
+		if(list_flds)
+		{
+			list_source_fields(engine, verbose, names_only, list_flds_source);
+			return EXIT_SUCCESS;
+		}
+
 		if (rules_filenames.size())
 		{
 			config.m_rules_filenames = rules_filenames;
@@ -919,6 +1101,17 @@ int falco_init(int argc, char **argv)
 			required_engine_versions[filename] = required_engine_version;
 		}
 
+		// Ensure that all plugins are compatible with the loaded set of rules
+		for(auto &info : infos)
+		{
+			std::string required_version;
+
+			if(!engine->is_plugin_compatible(info.name, info.plugin_version.as_string(), required_version))
+			{
+				throw std::invalid_argument(std::string("Plugin ") + info.name + " version " + info.plugin_version.as_string() + " not compatible with required plugin version " + required_version);
+			}
+		}
+
 		// You can't both disable and enable rules
 		if((disabled_rule_substrings.size() + disabled_rule_tags.size() > 0) &&
 		    enabled_rule_tags.size() > 0) {
@@ -1016,6 +1209,15 @@ int falco_init(int argc, char **argv)
 
 		if(!all_events)
 		{
+			// For syscalls, see if any event types used by the
+			// loaded rules are ones with the EF_DROP_SIMPLE_CONS
+			// label.
+			check_for_ignored_events(*inspector, *engine);
+			// Drop EF_DROP_SIMPLE_CONS kernel side
+			inspector->set_simple_consumer();
+			// Eventually, drop any EF_DROP_SIMPLE_CONS event
+			// that reached userspace (there are some events that are not syscall-based
+			// like signaldeliver, that have the EF_DROP_SIMPLE_CONS flag)
 			inspector->set_drop_event_flags(EF_DROP_SIMPLE_CONS);
 		}
 
@@ -1120,14 +1322,15 @@ int falco_init(int argc, char **argv)
 
 		outputs = new falco_outputs();
 
-		outputs->init(config.m_json_output,
-			config.m_json_include_output_property,
-			config.m_json_include_tags_property,
-			config.m_output_timeout,
-			config.m_notifications_rate, config.m_notifications_max_burst,
-			config.m_buffered_outputs,
-			config.m_time_format_iso_8601,
-			hostname);
+		outputs->init(engine,
+			      config.m_json_output,
+			      config.m_json_include_output_property,
+			      config.m_json_include_tags_property,
+			      config.m_output_timeout,
+			      config.m_notifications_rate, config.m_notifications_max_burst,
+			      config.m_buffered_outputs,
+			      config.m_time_format_iso_8601,
+			      hostname);
 
 		for(auto output : config.m_outputs)
 		{
@@ -1136,7 +1339,7 @@ int falco_init(int argc, char **argv)
 
 		if(trace_filename.size())
 		{
-			// Try to open the trace file as a sysdig
+			// Try to open the trace file as a
 			// capture file first.
 			try {
 				inspector->open(trace_filename);
@@ -1313,7 +1516,7 @@ int falco_init(int argc, char **argv)
 		falco_logger::log(LOG_DEBUG, "Setting metadata download chunk wait time to " + to_string(config.m_metadata_download_chunk_wait_us) + " μs\n");
 		falco_logger::log(LOG_DEBUG, "Setting metadata download watch frequency to " + to_string(config.m_metadata_download_watch_freq_sec) + " seconds\n");
 		inspector->set_metadata_download_params(config.m_metadata_download_max_mb * 1024 * 1024, config.m_metadata_download_chunk_wait_us, config.m_metadata_download_watch_freq_sec);
-		
+
 		if(trace_filename.empty() && config.m_webserver_enabled && !disable_k8s_audit)
 		{
 			std::string ssl_option = (config.m_webserver_ssl_enabled ? " (SSL)" : "");
@@ -1357,6 +1560,7 @@ int falco_init(int argc, char **argv)
 			num_evts = do_inspect(engine,
 					      outputs,
 					      inspector,
+					      event_source,
 					      config,
 					      sdropmgr,
 					      uint64_t(duration_to_tot*ONE_SECOND_IN_NS),

userspace/falco/falco_outputs.cpp

@@ -60,12 +60,13 @@ falco_outputs::~falco_outputs()
 	}
 }
 
-void falco_outputs::init(bool json_output,
-		  bool json_include_output_property,
-		  bool json_include_tags_property,
-		  uint32_t timeout,
-		  uint32_t rate, uint32_t max_burst, bool buffered,
-		  bool time_format_iso_8601, std::string hostname)
+void falco_outputs::init(falco_engine *engine,
+			 bool json_output,
+			 bool json_include_output_property,
+			 bool json_include_tags_property,
+			 uint32_t timeout,
+			 uint32_t rate, uint32_t max_burst, bool buffered,
+			 bool time_format_iso_8601, std::string hostname)
 {
 	// Cannot be initialized more than one time.
 	if(m_initialized)
@@ -73,14 +74,9 @@ void falco_outputs::init(bool json_output,
 		throw falco_exception("falco_outputs already initialized");
 	}
 
-	m_json_output = json_output;
+	m_formats.reset(new falco_formats(engine, json_include_output_property, json_include_tags_property));
 
-	// Note that falco_formats is already initialized by the engine,
-	// and the following json options are not used within the engine.
-	// So we can safely update them.
-	falco_formats::s_json_output = json_output;
-	falco_formats::s_json_include_output_property = json_include_output_property;
-	falco_formats::s_json_include_tags_property = json_include_tags_property;
+	m_json_output = json_output;
 
 	m_timeout = std::chrono::milliseconds(timeout);
 
@@ -139,7 +135,7 @@ void falco_outputs::add_output(falco::outputs::config oc)
 		throw falco_exception("Output not supported: " + oc.name);
 	}
 
-	oo->init(oc, m_buffered, m_hostname);
+	oo->init(oc, m_buffered, m_hostname, m_json_output);
 	m_outputs.push_back(oo);
 }
 
@@ -159,7 +155,7 @@ void falco_outputs::handle_event(gen_event *evt, string &rule, string &source,
 	cmsg.rule = rule;
 
 	string sformat;
-	if(source == "syscall")
+	if(source != "k8s_audit")
 	{
 		if(m_time_format_iso_8601)
 		{
@@ -192,8 +188,8 @@ void falco_outputs::handle_event(gen_event *evt, string &rule, string &source,
 		sformat += " " + format;
 	}
 
-	cmsg.msg = falco_formats::format_event(evt, rule, source, falco_common::priority_names[priority], sformat, tags);
-	cmsg.fields = falco_formats::resolve_tokens(evt, source, sformat);
+	cmsg.msg = m_formats->format_event(evt, rule, source, falco_common::priority_names[priority], sformat, tags);
+	cmsg.fields = m_formats->get_field_values(evt, source, sformat);
 	cmsg.tags.insert(tags.begin(), tags.end());
 
 	cmsg.type = ctrl_msg_type::CTRL_MSG_OUTPUT;
@@ -332,7 +328,7 @@ void falco_outputs::worker() noexcept
 						o->reopen();
 						break;
 					default:
-						falco_logger::log(LOG_DEBUG, "Outputs worker received an unknown message type\n");	
+						falco_logger::log(LOG_DEBUG, "Outputs worker received an unknown message type\n");
 				}
 			}
 			catch(const exception &e)

userspace/falco/falco_outputs.h

@@ -25,6 +25,7 @@ limitations under the License.
 #include "token_bucket.h"
 #include "falco_engine.h"
 #include "outputs.h"
+#include "formats.h"
 #include "tbb/concurrent_queue.h"
 
 //
@@ -38,7 +39,8 @@ public:
 	falco_outputs();
 	virtual ~falco_outputs();
 
-	void init(bool json_output,
+	void init(falco_engine *engine,
+		  bool json_output,
 		  bool json_include_output_property,
 		  bool json_include_tags_property,
 		  uint32_t timeout,
@@ -63,6 +65,7 @@ public:
 	void reopen_outputs();
 
 private:
+	std::unique_ptr<falco_formats> m_formats;
 	bool m_initialized;
 
 	std::vector<falco::outputs::abstract_output *> m_outputs;

userspace/falco/outputs.h

@@ -63,11 +63,12 @@ class abstract_output
 public:
 	virtual ~abstract_output() {}
 
-	void init(config oc, bool buffered, std::string hostname)
+	void init(config oc, bool buffered, std::string hostname, bool json_output)
 	{
 		m_oc = oc;
 		m_buffered = buffered;
 		m_hostname = hostname;
+		m_json_output = json_output;
 	}
 
 	// Return the output's name as per its configuration.
@@ -89,6 +90,7 @@ protected:
 	config m_oc;
 	bool m_buffered;
 	std::string m_hostname;
+	bool m_json_output;
 };
 
 } // namespace outputs

userspace/falco/outputs.proto

@@ -45,10 +45,11 @@ message request {
 message response {
   google.protobuf.Timestamp time = 1;
   falco.schema.priority priority = 2;
-  falco.schema.source source = 3;
+  falco.schema.source source_deprecated = 3 [deprecated=true];
   string rule = 4;
   string output = 5;
   map<string, string> output_fields = 6;
   string hostname = 7;
   repeated string tags = 8;
+  string source = 9;
 }