build: correct conffiles for DEB package

Signed-off-by: Leonardo Grasso <me@leonardograsso.com>

.codespellignore

@@ -0,0 +1,3 @@
+aks
+creat
+chage

.github/workflows/codespell.yml

@@ -0,0 +1,14 @@
+name: Codespell
+on:
+  pull_request:
+jobs:
+  codespell:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: codespell-project/actions-codespell@master
+      with:
+        skip: .git
+        ignore_words_file: .codespellignore
+        check_filenames: true
+        check_hidden: true

.gitignore

@@ -12,6 +12,4 @@ test/build
 
 .vscode/*
 
-.luacheckcache
-
 *.idea*

.luacheckrc

@@ -1,8 +0,0 @@
-std = "min"
-cache = true
-include_files = {
-    "userspace/engine/lua/*.lua",
-    "userspace/engine/lua/lyaml/*.lua",
-    "*.luacheckrc"
-}
-exclude_files = {"build"}

ADOPTERS.md

@@ -32,6 +32,8 @@ This is a list of production adopters of Falco (in alphabetical order):
 
 * [GitLab](https://about.gitlab.com/direction/defend/container_host_security/) - GitLab is a complete DevOps platform, delivered as a single application, fundamentally changing the way Development, Security, and Ops teams collaborate. GitLab Ultimate provides the single tool teams need to find, triage, and fix vulnerabilities in applications, services, and cloud-native environments enabling them to manage their risk. This provides them with repeatable, defensible processes that automate security and compliance policies. GitLab includes a tight integration with Falco, allowing users to defend their containerized applications from attacks while running in production.
 
+* [gVisor](https://gvisor.dev/) - gVisor secures Kubernetes, containers, and workloads via an alternate execution environment that handles system calls in user space, blocking security issues before they reach the underlying host. gVisor provides defense-in-depth, protection against untrusted code execution, and a secure-by-default Kubernetes experience where containers are a security boundary. Falco can be used with gVisor to detect unusual or suspicious activity using its threat detection engine on top of gVisor runtime execution information.
+
 * [League](https://league.com/ca/) - League provides health benefits management services to help employees understand and get the most from their benefits, and employers to provide effective, efficient plans. Falco is used to monitor our deployed services on Kubernetes, protecting against malicious access to containers which could lead to leaks of PHI or other sensitive data. The Falco alerts are logged in Stackdriver for grouping and further analysis. In the future, we're hoping for integrations with Prometheus and AlertManager as well.
 
 * [Logz.io](https://logz.io/) - Logz.io is a cloud observability platform for modern engineering teams. The Logz.io platform consists of three products — Log Management, Infrastructure Monitoring, and Cloud SIEM — that work together to unify the jobs of monitoring, troubleshooting, and security. We empower engineers to deliver better software by offering the world's most popular open source observability tools — the ELK Stack, Grafana, and Jaeger — in a single, easy to use, and powerful platform purpose-built for monitoring distributed cloud environments. Cloud SIEM supports data from multiple sources, including Falco's alerts, and offers useful rules and dashboards content to visualize and manage incidents across your systems in a unified UI.

CHANGELOG.md

@@ -116,7 +116,7 @@ Released on 2022-01-31
 * rule(Create Hardlink Over Sensitive Files): new rule to detect hard links created over sensitive files [[#1810](https://github.com/falcosecurity/falco/pull/1810)] - [@sberkovich](https://github.com/sberkovich)
 * rule(Detect crypto miners using the Stratum protocol): add `stratum2+tcp` and `stratum+ssl` protocols detection [[#1810](https://github.com/falcosecurity/falco/pull/1810)] - [@sberkovich](https://github.com/sberkovich)
 * rule(Sudo Potential Privilege Escalation): correct special case for the CVE-2021-3156 exploit [[#1810](https://github.com/falcosecurity/falco/pull/1810)] - [@sberkovich](https://github.com/sberkovich)
-* rule(list falco_hostnetwork_images): moved to k8s_audit_rules.yaml to avoid a warning when usng falco_rules.yaml only [[#1681](https://github.com/falcosecurity/falco/pull/1681)] - [@leodido](https://github.com/leodido)
+* rule(list falco_hostnetwork_images): moved to k8s_audit_rules.yaml to avoid a warning when using falco_rules.yaml only [[#1681](https://github.com/falcosecurity/falco/pull/1681)] - [@leodido](https://github.com/leodido)
 * rule(list deb_binaries): remove `apt-config` [[#1860](https://github.com/falcosecurity/falco/pull/1860)] - [@Andreagit97](https://github.com/Andreagit97)
 * rule(Launch Remote File Copy Tools in Container): add additional binaries: curl and wget. [[#1771](https://github.com/falcosecurity/falco/pull/1771)] - [@ec4n6](https://github.com/ec4n6)
 * rule(list known_sa_list): add coredns, coredns-autoscaler, endpointslicemirroring-controller, horizontal-pod-autoscaler, job-controller, node-controller (nodelifecycle), persistent-volume-binder, pv-protection-controller, pvc-protection-controller, root-ca-cert-publisher and service-account-controller as allowed service accounts in the kube-system namespace [[#1760](https://github.com/falcosecurity/falco/pull/1760)] - [@sboschman](https://github.com/sboschman)
@@ -812,7 +812,7 @@ Released on 2020-02-24
 * rule(write below etc): add "dsc_host" as a ms oms program [[#1028](https://github.com/falcosecurity/falco/pull/1028)]
 * rule(write below etc): let mcafee write to /etc/cma.d  [[#1028](https://github.com/falcosecurity/falco/pull/1028)]
 * rule(write below etc): let avinetworks supervisor write some ssh cfg [[#1028](https://github.com/falcosecurity/falco/pull/1028)]
-* rule(write below etc): alow writes to /etc/pki from openshift secrets dir [[#1028](https://github.com/falcosecurity/falco/pull/1028)]
+* rule(write below etc): allow writes to /etc/pki from openshift secrets dir [[#1028](https://github.com/falcosecurity/falco/pull/1028)]
 * rule(write below root): let runc write to /exec.fifo [[#1028](https://github.com/falcosecurity/falco/pull/1028)]
 * rule(change thread namespace): let cilium-cni change namespaces [[#1028](https://github.com/falcosecurity/falco/pull/1028)]
 * rule(run shell untrusted): let puma reactor spawn shells [[#1028](https://github.com/falcosecurity/falco/pull/1028)]

CMakeLists.txt

@@ -115,9 +115,6 @@ include(ExternalProject)
 # libs
 include(falcosecurity-libs)
 
-# LuaJit provided by libs
-include(luajit)
-
 # jq
 include(jq)
 
@@ -152,15 +149,6 @@ endif()
 
 include(cxxopts)
 
-# Lpeg
-include(lpeg)
-
-# libyaml
-include(libyaml)
-
-# lyaml
-include(lyaml)
-
 # One TBB
 include(tbb)
 

brand/README.md

@@ -5,7 +5,7 @@
 
 This document describes The Falco Project's branding guidelines, language, and message.
 
-Content in this document can be used to publically share about Falco.
+Content in this document can be used to publicly share about Falco.
 
 
 

cmake/cpack/debian/conffiles

@@ -1,4 +1,2 @@
 /etc/falco/falco.yaml
-/etc/falco/falco_rules.yaml
-/etc/falco/rules.available/application_rules.yaml
 /etc/falco/falco_rules.local.yaml

cmake/modules/DownloadCatch.cmake

@@ -14,8 +14,8 @@ include(ExternalProject)
 
 set(CATCH2_INCLUDE ${CMAKE_BINARY_DIR}/catch2-prefix/include)
 
-set(CATCH_EXTERNAL_URL URL https://github.com/catchorg/catch2/archive/v2.12.1.tar.gz URL_HASH
-                       SHA256=e5635c082282ea518a8dd7ee89796c8026af8ea9068cd7402fb1615deacd91c3)
+set(CATCH_EXTERNAL_URL URL https://github.com/catchorg/catch2/archive/v2.13.9.tar.gz URL_HASH
+                       SHA256=06dbc7620e3b96c2b69d57bf337028bf245a211b3cddb843835bfe258f427a52)
 
 ExternalProject_Add(
   catch2

cmake/modules/falcosecurity-libs.cmake

@@ -24,8 +24,8 @@ else()
   # default below In case you want to test against another falcosecurity/libs version just pass the variable - ie., `cmake
   # -DFALCOSECURITY_LIBS_VERSION=dev ..`
   if(NOT FALCOSECURITY_LIBS_VERSION)
-    set(FALCOSECURITY_LIBS_VERSION "b7eb0dd65226a8dc254d228c8d950d07bf3521d2")
-    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=0f6dcdc3b94243c91294698ee343806539af81c5b33c60c6acf83fc1aa455e85")
+    set(FALCOSECURITY_LIBS_VERSION "b19f87e8aee663e4987a3db54570725e071ed105")
+    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=fd5888588796bf52848cf75434784770e61d9a79f344bf571fe495b61e92ddd3")
   endif()
 
   # cd /path/to/build && cmake /path/to/source
@@ -62,14 +62,9 @@ set(LIBSINSP_DIR "${FALCOSECURITY_LIBS_SOURCE_DIR}")
 set(CREATE_TEST_TARGETS OFF CACHE BOOL "")
 set(BUILD_LIBSCAP_EXAMPLES OFF CACHE BOOL "")
 
-# todo(leogr): although Falco does not actually depend on chisels, we need this for the lua_parser. 
-# Hopefully, we can switch off this in the future
-set(WITH_CHISEL ON CACHE BOOL "")
-
 set(USE_BUNDLED_TBB ON CACHE BOOL "")
 set(USE_BUNDLED_B64 ON CACHE BOOL "")
 set(USE_BUNDLED_JSONCPP ON CACHE BOOL "")
-set(USE_BUNDLED_LUAJIT ON CACHE BOOL "")
 
 list(APPEND CMAKE_MODULE_PATH "${FALCOSECURITY_LIBS_SOURCE_DIR}/cmake/modules")
 

cmake/modules/lpeg.cmake

@@ -1,28 +0,0 @@
-#
-# Copyright (C) 2020 The Falco Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-#
-
-set(LPEG_SRC "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg")
-set(LPEG_LIB "${PROJECT_BINARY_DIR}/lpeg-prefix/src/lpeg/build/lpeg.a")
-message(STATUS "Using bundled lpeg in '${LPEG_SRC}'")
-set(LPEG_DEPENDENCIES "")
-list(APPEND LPEG_DEPENDENCIES "luajit")
-ExternalProject_Add(
-  lpeg
-  DEPENDS ${LPEG_DEPENDENCIES}
-  URL "http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.0.2.tar.gz"
-  URL_HASH "SHA256=48d66576051b6c78388faad09b70493093264588fcd0f258ddaab1cdd4a15ffe"
-  BUILD_COMMAND LUA_INCLUDE=${LUAJIT_INCLUDE} "${PROJECT_SOURCE_DIR}/scripts/build-lpeg.sh" "${LPEG_SRC}/build"
-  BUILD_IN_SOURCE 1
-  BUILD_BYPRODUCTS ${LPEG_LIB}
-  CONFIGURE_COMMAND ""
-  INSTALL_COMMAND "")

cmake/modules/lyaml.cmake

@@ -1,28 +0,0 @@
-#
-# Copyright (C) 2020 The Falco Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-#
-
-set(LYAML_ROOT "${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml")
-set(LYAML_LIB "${LYAML_ROOT}/ext/yaml/.libs/yaml.a")
-set(LYAML_LUA_DIR "${LYAML_ROOT}/lib")
-message(STATUS "Using bundled lyaml in '${LYAML_ROOT}'")
-externalproject_add(
-  lyaml
-  DEPENDS luajit libyaml
-  URL "https://github.com/gvvaughan/lyaml/archive/release-v6.0.tar.gz"
-  URL_HASH "SHA256=9d7cf74d776999ff6f758c569d5202ff5da1f303c6f4229d3b41f71cd3a3e7a7"
-  BUILD_COMMAND ${CMD_MAKE}
-  BUILD_IN_SOURCE 1
-  BUILD_BYPRODUCTS ${LYAML_LIB}
-  INSTALL_COMMAND ""
-  CONFIGURE_COMMAND ./configure --enable-static CFLAGS=-I${LIBYAML_INSTALL_DIR}/include CPPFLAGS=-I${LIBYAML_INSTALL_DIR}/include LDFLAGS=-L${LIBYAML_INSTALL_DIR}/lib LIBS=-lyaml LUA=${LUAJIT_SRC}/luajit LUA_INCLUDE=-I${LUAJIT_INCLUDE}
-)

cmake/modules/plugins.cmake

@@ -15,20 +15,24 @@ include(ExternalProject)
 
 string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} PLUGINS_SYSTEM_NAME)
 
+# todo(jasondellaluce): switch this to a stable version once this plugin gets
+# released with a 1.0.0 required plugin api version
 ExternalProject_Add(
   cloudtrail-plugin
-  URL "https://download.falco.org/plugins/stable/cloudtrail-0.2.3-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
-  URL_HASH "SHA256=3dfce36f37a4f834b6078c6b78776414472a6ee775e8f262535313cc4031d0b7"
+  URL "https://download.falco.org/plugins/dev/cloudtrail-0.2.5-0.2.5-8%2B2c1bb25-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
+  URL_HASH "SHA256=eeefbeb639e41e37cd864042d8b4854e0af451e6b8b34a14c39332771e94ee5b"
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ""
   INSTALL_COMMAND "")
 
 install(FILES "${PROJECT_BINARY_DIR}/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so" DESTINATION "${FALCO_PLUGINS_DIR}")
 
+# todo(jasondellaluce): switch this to a stable version once this plugin gets
+# released with a 1.0.0 required plugin api version
 ExternalProject_Add(
   json-plugin
-  URL "https://download.falco.org/plugins/stable/json-0.2.2-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
-  URL_HASH "SHA256=83eb411c9f2125695875b229c6e7974e6a4cc7f028be146b79d26db30372af5e"
+  URL "https://download.falco.org/plugins/dev/json-0.2.2-0.2.2-24%2B2c1bb25-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
+  URL_HASH "SHA256=181ac01d11defee24ad5947fcd836e256e13c61ca9475253fb82b60297164748"
   CONFIGURE_COMMAND ""
   BUILD_COMMAND ""
   INSTALL_COMMAND "")

docker/README.md

@@ -7,10 +7,11 @@ This directory contains various ways to package Falco as a container and related
 | Name | Directory | Description |
 |---|---|---|
 | [falcosecurity/falco:latest](https://hub.docker.com/repository/docker/falcosecurity/falco), [falcosecurity/falco:_tag_](https://hub.docker.com/repository/docker/falcosecurity/falco), [falcosecurity/falco:master](https://hub.docker.com/repository/docker/falcosecurity/falco) | docker/falco | Falco (DEB built from git tag or from the master) with all the building toolchain. |
+| _not yet published (experimental)_ | docker/ubi | Falco (built from RedHat's UBI base image) with the building toolchain. |
 | [falcosecurity/falco-driver-loader:latest](https://hub.docker.com/repository/docker/falcosecurity/falco-driver-loader), [falcosecurity/falco-driver-loader:_tag_](https://hub.docker.com/repository/docker/falcosecurity/falco-driver-loader), [falcosecurity/falco-driver-loader:master](https://hub.docker.com/repository/docker/falcosecurity/falco-driver-loader) | docker/driver-loader | `falco-driver-loader` as entrypoint with the building toolchain. |
 | [falcosecurity/falco-no-driver:latest](https://hub.docker.com/repository/docker/falcosecurity/falco-no-driver), [falcosecurity/falco-no-driver:_tag_](https://hub.docker.com/repository/docker/falcosecurity/falco-no-driver),[falcosecurity/falco-no-driver:master](https://hub.docker.com/repository/docker/falcosecurity/falco-no-driver) | docker/no-driver | Falco (TGZ built from git tag or from the master) without the building toolchain. |
 | [falcosecurity/falco-builder:latest](https://hub.docker.com/repository/docker/falcosecurity/falco-builder) | docker/builder | The complete build tool chain for compiling Falco from source. See [the documentation](https://falco.org/docs/getting-started/source/) for more details on building from source. Used to build Falco (CI). |
 | [falcosecurity/falco-tester:latest](https://hub.docker.com/repository/docker/falcosecurity/falco-tester) | docker/tester | Container image for running the Falco test suite. Used to run Falco integration tests (CI). |
-| _to not be published_ | docker/local | Built on-the-fly and used by falco-tester. |
+| _not to be published_ | docker/local | Built on-the-fly and used by falco-tester. |
 
 > Note: `falco-builder`, `falco-tester` (and the `docker/local` image that it's built on the fly) are not integrated into the release process because they are development and CI tools that need to be manually pushed only when updated.

docker/builder/root/usr/bin/scl_enable

@@ -1,5 +1,5 @@
 # IMPORTANT: Do not add more content to this file unless you know what you are doing.
-#            This file is sourced everytime the shell session is opened.
+#            This file is sourced every time the shell session is opened.
 #
 # This will make scl collection binaries work out of box.
 unset BASH_ENV PROMPT_COMMAND ENV

docker/ubi/Dockerfile

@@ -0,0 +1,45 @@
+ARG UBI_VERSION=latest
+FROM registry.access.redhat.com/ubi8/ubi:${UBI_VERSION}
+
+ARG FALCO_VERSION
+RUN test -n "$FALCO_VERSION" || (echo "FALCO_VERSION  not set" && false)
+ENV FALCO_VERSION=${FALCO_VERSION}
+
+LABEL "name"="Falco Runtime Security"
+LABEL "vendor"="Falco"
+LABEL "version"="${FALCO_VERSION}"
+LABEL "release"="${FALCO_VERSION}"
+LABEL "ubi-version"="${UBI_VERSION}"
+LABEL "summary"="Falco is a security policy engine that monitors system calls and cloud events, and fires alerts when security policies are violated."
+LABEL "description"="Falco is a security policy engine that monitors system calls and cloud events, and fires alerts when security policies are violated."
+LABEL "io.k8s.display-name"="Falco"
+LABEL "io.k8s.description"="Falco is a security policy engine that monitors system calls and cloud events, and fires alerts when security policies are violated."
+LABEL maintainer="cncf-falco-dev@lists.cncf.io"
+LABEL usage="docker run -i -t --privileged -v /var/run/docker.sock:/host/var/run/docker.sock -v /dev:/host/dev -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro -v /etc:/host/etc --name NAME IMAGE"
+
+
+ENV HOST_ROOT /host
+ENV HOME /root
+
+RUN  dnf -y update && \
+     dnf -y install \
+          curl \
+          make \
+          cmake \
+          gcc \
+          llvm-toolset \
+          clang \
+          kmod \
+     && dnf -y clean all ; rm -rf /var/cache/{dnf,yum}
+
+RUN mkdir /build && cd /build/ && curl --remote-name-all -L https://github.com/dell/dkms/archive/refs/tags/v3.0.3.tar.gz && \
+     tar xvf v3.0.3.tar.gz && cd dkms-3.0.3 && make install-redhat && rm -rf /build
+
+RUN mkdir /deploy && cd /deploy/ && curl --remote-name-all -L https://download.falco.org/packages/bin/x86_64/falco-${FALCO_VERSION}-x86_64.tar.gz && \
+     cd / && tar --strip-components=1 -xvf /deploy/falco-${FALCO_VERSION}-x86_64.tar.gz && \
+     rm -rf /deploy
+
+COPY ./docker-entrypoint.sh /
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["/usr/bin/falco"]

docker/ubi/docker-entrypoint.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Copyright (C) 2022 The Falco Authors.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set the SKIP_DRIVER_LOADER variable to skip loading the driver
+
+if [[ -z "${SKIP_DRIVER_LOADER}" ]]; then
+
+    # Required by dkms to find the required dependencies on RedHat UBI
+    rm -fr /usr/src/kernels/ && rm -fr /usr/src/debug/
+    rm -fr /lib/modules && ln -s $HOST_ROOT/lib/modules /lib/modules
+    rm -fr /boot && ln -s $HOST_ROOT/boot /boot
+
+    echo "* Setting up /usr/src links from host"
+
+    for i in "$HOST_ROOT/usr/src"/*
+    do
+        base=$(basename "$i")
+        ln -s "$i" "/usr/src/$base"
+    done
+
+    /usr/bin/falco-driver-loader
+fi
+
+exec "$@"

proposals/20190826-grpc-outputs.md

@@ -47,7 +47,7 @@ The motivation behind this proposal is to design a new output implementation tha
 ### Non-Goals
 
 - To substitute existing outputs (stdout, syslog, etc.)
-- To support different queing systems than the default (round-robin) one
+- To support different queuing systems than the default (round-robin) one
 - To support queuing mechanisms for message retransmission
   - Users can have a local gRPC relay server along with Falco that multiplexes connections and handles retires and backoff
 - To change the output format

proposals/20191030-api.md

@@ -6,7 +6,7 @@ This is a proposal to better structure the Falco API.
 
 The Falco API is a set of contracts describing how users can interacts with Falco.
 
-By defining a set of interfaces the Falco Authors intend to decouple Falco from other softwares and data (eg., from the input sources) and, at the same time, make it more extensible.
+By defining a set of interfaces the Falco Authors intend to decouple Falco from other software and data (eg., from the input sources) and, at the same time, make it more extensible.
 
 Thus, this document intent is to propose a list of services that constitute the Falco API (targeting the first stable version of Falco, v1.0.0).
 

proposals/20210501-plugin-system.md

@@ -270,7 +270,7 @@ typedef struct
 	// Arguments:
 	// - s: the plugin state returned by init()
 	// - params: the open parameters, as a string. The format is defined by the plugin
-	//   itsef
+	//   itself
 	// - rc: pointer to an integer that will contain the open result, as a SCAP_* value
 	//   (e.g. SCAP_SUCCESS=0, SCAP_FAILURE=1)
 	// Return value: a pointer to the open context that will be passed to next(),

rules/falco_rules.yaml

@@ -1833,6 +1833,7 @@
     gke.gcr.io/kube-proxy,
     gke.gcr.io/gke-metadata-server,
     gke.gcr.io/netd-amd64,
+    gke.gcr.io/watcher-daemonset,
     gcr.io/google-containers/prometheus-to-sd,
     k8s.gcr.io/ip-masq-agent-amd64,
     k8s.gcr.io/kube-proxy,
@@ -2814,7 +2815,6 @@
 
 - macro: trusted_images_query_miner_domain_dns
   condition: (container.image.repository in (docker.io/falcosecurity/falco, falcosecurity/falco, public.ecr.aws/falcosecurity/falco))
-  append: false
 
 # The rule is disabled by default.
 # Note: falco will send DNS request to resolve miner pool domain which may trigger alerts in your environment.
@@ -3111,10 +3111,7 @@
 
 -  macro: curl_download
    condition: proc.name = curl and
-              (proc.cmdline contains " > " or
-              proc.cmdline contains " >> " or
-              proc.cmdline contains " | " or
-              proc.cmdline contains " -o " or
+              (proc.cmdline contains " -o " or
               proc.cmdline contains " --output " or
               proc.cmdline contains " -O " or
               proc.cmdline contains " --remote-name ")

rules/k8s_audit_rules.yaml

@@ -51,13 +51,24 @@
     cluster-autoscaler,
     "system:addon-manager",
     "cloud-controller-manager",
-    "eks:node-manager",
     "system:kube-controller-manager"
     ]
 
+- list: eks_allowed_k8s_users
+  items: [
+    "eks:node-manager",
+    "eks:certificate-controller",
+    "eks:fargate-scheduler",
+    "eks:k8s-metrics",
+    "eks:authenticator",
+    "eks:cluster-event-watcher",
+    "eks:nodewatcher",
+    "eks:pod-identity-mutating-webhook"
+    ]
+-
 - rule: Disallowed K8s User
   desc: Detect any k8s operation by users outside of an allowed set of users.
-  condition: kevt and non_system_user and not ka.user.name in (allowed_k8s_users)
+  condition: kevt and non_system_user and not ka.user.name in (allowed_k8s_users) and not ka.user.name in (eks_allowed_k8s_users)
   output: K8s Operation performed by user not in allowed list of users (user=%ka.user.name target=%ka.target.name/%ka.target.resource verb=%ka.verb uri=%ka.uri resp=%ka.response.code)
   priority: WARNING
   source: k8s_audit
@@ -174,6 +185,28 @@
   source: k8s_audit
   tags: [k8s]
 
+- list: falco_hostpid_images
+  items: []
+
+- rule: Create HostPid Pod
+  desc: Detect an attempt to start a pod using the host pid namespace.
+  condition: kevt and pod and kcreate and ka.req.pod.host_pid intersects (true) and not ka.req.pod.containers.image.repository in (falco_hostpid_images)
+  output: Pod started using host pid namespace (user=%ka.user.name pod=%ka.resp.name ns=%ka.target.namespace images=%ka.req.pod.containers.image)
+  priority: WARNING
+  source: k8s_audit
+  tags: [k8s]
+
+- list: falco_hostipc_images
+  items: []
+
+- rule: Create HostIPC Pod
+  desc: Detect an attempt to start a pod using the host ipc namespace.
+  condition: kevt and pod and kcreate and ka.req.pod.host_ipc intersects (true) and not ka.req.pod.containers.image.repository in (falco_hostipc_images)
+  output: Pod started using host ipc namespace (user=%ka.user.name pod=%ka.resp.name ns=%ka.target.namespace images=%ka.req.pod.containers.image)
+  priority: WARNING
+  source: k8s_audit
+  tags: [k8s]
+
 - macro: user_known_node_port_service
   condition: (k8s_audit_never_true)
 
@@ -288,6 +321,7 @@
     k8s.gcr.io/kube-apiserver,
     gke.gcr.io/kube-proxy,
     gke.gcr.io/netd-amd64,
+    gke.gcr.io/watcher-daemonset,
     k8s.gcr.io/addon-resizer
     k8s.gcr.io/prometheus-to-sd,
     k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64,
@@ -479,7 +513,7 @@
 - rule: K8s Serviceaccount Created
   desc: Detect any attempt to create a service account
   condition: (kactivity and kcreate and serviceaccount and response_successful)
-  output: K8s Serviceaccount Created (user=%ka.user.name user=%ka.target.name ns=%ka.target.namespace resp=%ka.response.code decision=%ka.auth.decision reason=%ka.auth.reason)
+  output: K8s Serviceaccount Created (user=%ka.user.name serviceaccount=%ka.target.name ns=%ka.target.namespace resp=%ka.response.code decision=%ka.auth.decision reason=%ka.auth.reason)
   priority: INFO
   source: k8s_audit
   tags: [k8s]
@@ -487,7 +521,7 @@
 - rule: K8s Serviceaccount Deleted
   desc: Detect any attempt to delete a service account
   condition: (kactivity and kdelete and serviceaccount and response_successful)
-  output: K8s Serviceaccount Deleted (user=%ka.user.name user=%ka.target.name ns=%ka.target.namespace resp=%ka.response.code decision=%ka.auth.decision reason=%ka.auth.reason)
+  output: K8s Serviceaccount Deleted (user=%ka.user.name serviceaccount=%ka.target.name ns=%ka.target.namespace resp=%ka.response.code decision=%ka.auth.decision reason=%ka.auth.reason)
   priority: INFO
   source: k8s_audit
   tags: [k8s]

rules/okta_rules.yaml

@@ -0,0 +1,177 @@
+#Example Rule on login in to OKTA. Disabled by default since it might be noisy
+#- rule: User logged in to OKTA
+#  desc: Detect the user login in to OKTA
+#  condition: okta.evt.type = "user.session.start" 
+#  output: "A user has logged in toOKTA (user=%okta.actor.name, ip=%okta.client.ip)"
+#       priority: NOTICE
+#  source: okta
+#  tags: [okta]
+
+- rule: User Changing password in to OKTA
+  desc: Detect a user change password in OKTA
+  condition: okta.evt.type = "user.account.update_password"
+  output: "A user has changed password from OKTA (user=%okta.actor.name, ip=%okta.client.ip)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+  enabled: false
+
+- rule: Creating a new OKTA user account
+  desc: Detect a new OKTA user account created in the OKTA environment
+  condition: okta.evt.type = "user.lifecycle.create"
+  output: "A new OKTA user account created (user=%okta.actor.name, target user=%okta.target.user.name)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+  enabled: false
+
+- rule: User accessing app via single sign on OKTA
+  desc: Detect a user accessing an app via OKTA
+  condition: okta.evt.type = "user.authentication.sso"
+  output: "A user has accessed an app using OKTA (user=%okta.actor.name, app=%okta.app)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+  enabled: false
+
+- rule: User has been locked out in OKTA
+  desc: Detect a user who has been locked out in OKTA
+  condition: okta.evt.type = "user.account.lock"
+  output: "A user has been locked out in OKTA (user=%okta.actor.name, ip=%okta.client.ip)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+- rule: User has been moved from suspended status in OKTA.
+  desc: Detect a user who has been moved from suspended status in OKTA
+  condition: okta.evt.type = "user.lifecycle.unsuspend"
+  output: "A user has been moved from suspended status in OKTA (user=%okta.actor.name, ip=%okta.client.ip, target user=%okta.target.user.name)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+  enabled: false
+
+- rule: User has been activated in OKTA
+  desc: Detect a user who has been activated in OKTA
+  condition: okta.evt.type = "user.lifecycle.activate"
+  output: "A user has been activated in OKTA (user=%okta.actor.name, ip=%okta.client.ip, target user=%okta.target.user.name)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+  enabled: false
+
+- rule: User has been deactivated in OKTA
+  desc: Detect a user who has been deactivated in OKTA
+  condition: okta.evt.type = "user.lifecycle.deactivate"
+  output: "A user has been deactivated in OKTA (user=%okta.actor.name, ip=%okta.client.ip, target user=%okta.target.user.name)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+  enabled: false
+
+- rule: User has been suspended in OKTA
+  desc: Detect a user who has been suspended in OKTA
+  condition: okta.evt.type = "user.lifecycle.suspended"
+  output: "A user has been suspended in OKTA (user=%okta.actor.name, ip=%okta.client.ip, target user=%okta.target.user.name)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+- rule: Admin permission has been assigned to a user in OKTA
+  desc: Detect an admin permission assigned to a user in OKTA
+  condition: okta.evt.type = "user.account.privilege.grant"
+  output: "A user has been locked out in OKTA (user=%okta.actor.name, ip=%okta.client.ip, target user=%okta.target.user.name)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+- rule: Creating a new OKTA API token
+  desc: Detect a new OKTA API token created in the OKTA environment
+  condition: okta.evt.type = "system.api_token.create"
+  output: "A new OKTA API token has been created in OKTA (user=%okta.actor.name, ip=%okta.client.ip)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+- rule: User accessing OKTA admin section
+  desc: Detect a user accessing OKTA admin section of your OKTA instance
+  condition: okta.evt.type = "user.session.access_admin_app"
+  output: "A user accessed the OKTA admin section of your OKTA instance (user=%okta.actor.name, ip=%okta.client.ip)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+- rule: Adding user in OKTA group
+  desc: Detect a new user added to an OKTA group
+  condition: okta.evt.type = "group.user_membership.add"
+  output: "A user has been added in an OKTA group (user=%okta.actor.name, target group=%okta.target.group.name, target user=%okta.target.user.name)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+  enabled: false
+
+- rule: removing MFA factor from user in OKTA
+  desc: Detect a removing MFA activity on a user in OKTA
+  condition: okta.evt.type = "user.mfa.factor.deactivate"
+  output: "A user has removed MFA factor in the OKTA account (user=%okta.actor.name, ip=%okta.client.ip)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+- rule: removing all MFA factor from user in OKTA
+  desc: Detect a removing MFA activity on a user in OKTA
+  condition: okta.evt.type = "user.mfa.factor.reset_all"
+  output: "A user has removed all MFA factor in the OKTA account (user=%okta.actor.name, ip=%okta.client.ip)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+- rule: User password reset by OKTA admin
+  desc: Detect a password reset on a user done by OKTA Admin Account
+  condition: okta.evt.type = "user.account.reset_password"
+  output: "A user password has been reset by an OKTA Admin account (user=%okta.actor.name, ip=%okta.client.ip, target user=%okta.target.user.name)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+- rule: User hitting the rate limit on requests in OKTA
+  desc: Detect a user who hit the rate limit on requests in OKTA
+  condition: okta.evt.type = "system.org.rate_limit.violation"
+  output: "A user has hitted the rate limit on requests in OKTA (user=%okta.actor.name, ip=%okta.client.ip)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+- rule: Adding user to application membership in OKTA
+  desc: Detect a user who has been added o application membership in OKTA
+  condition: okta.evt.type = "application.user_membership.add"
+  output: "A user has been added to an application membership in OKTA (user=%okta.actor.name, ip=%okta.client.ip, target user=%okta.target.user.name, app=%okta.app)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+  enabled: false
+
+- rule: User initiating impersonation session in OKTA
+  desc: Detect a user who initiate an impersonation session in OKTA
+  condition: okta.evt.type = "user.session.impersonation.initiate"
+  output: "A user has initiated an impersonation session in OKTA (user=%okta.actor.name, ip=%okta.client.ip, target user=%okta.target.user.name)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+
+# This list allows easily whitelisting countries that are
+# expected to see OKTA logins from.
+- list: allowed_countries_list
+  items: []
+
+- macro: user_known_countries
+  condition: (okta.client.geo.country in (allowed_countries_list))
+
+- rule: Detecting unknown logins using geolocation
+  desc: Detect a logins event based on user geolocation
+  condition: okta.evt.type = "user.session.start" and not user_known_countries
+  output: "A user logged in OKTA from a suspicious country (user=%okta.actor.name, ip=%okta.client.ip, country=%okta.client.geo.country)"
+  priority: NOTICE
+  source: okta
+  tags: [okta]
+  enabled: false

scripts/build-lpeg.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (C) 2019 The Falco Authors.
-#
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -ex
-
-PREFIX=$1
-
-if [ -z "$PREFIX" ]; then
-    PREFIX=.
-fi
-
-mkdir -p $PREFIX
-
-gcc -O2 -fPIC -I"$LUA_INCLUDE" -c lpcap.c -o $PREFIX/lpcap.o
-gcc -O2 -fPIC -I"$LUA_INCLUDE" -c lpcode.c -o $PREFIX/lpcode.o
-gcc -O2 -fPIC -I"$LUA_INCLUDE" -c lpprint.c -o $PREFIX/lpprint.o
-gcc -O2 -fPIC -I"$LUA_INCLUDE" -c lptree.c -o $PREFIX/lptree.o
-gcc -O2 -fPIC -I"$LUA_INCLUDE" -c lpvm.c -o $PREFIX/lpvm.o
-
-
-# For building lpeg.so, which we don't need now that we're statically linking lpeg.a into falco
-#gcc -shared -o lpeg.so -L/usr/local/lib lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
-#gcc -shared -o lpeg.so -L/usr/local/lib lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
-
-pushd $PREFIX
-/usr/bin/ar cr lpeg.a lpcap.o lpcode.o lpprint.o lptree.o lpvm.o
-/usr/bin/ranlib lpeg.a
-popd
-
-chmod ug+w re.lua

test/.gitignore

@@ -1 +1,2 @@
-falco_traces.yaml
+falco_traces.yaml
+venv/*

test/falco_tests.yaml

@@ -254,9 +254,6 @@ trace_files: !mux
     stdout_is: |+
       1 errors:
       Rules content is not yaml
-      ---
-      This is not yaml
-      ---
     validate_rules_file:
       - rules/invalid_not_yaml.yaml
     trace_file: trace_files/cat_write.scap
@@ -266,9 +263,6 @@ trace_files: !mux
     stdout_is: |+
       1 errors:
       Rules content is not yaml array of objects
-      ---
-      foo: bar
-      ---
     validate_rules_file:
       - rules/invalid_not_array.yaml
     trace_file: trace_files/cat_write.scap
@@ -277,7 +271,7 @@ trace_files: !mux
     exit_status: 1
     stdout_is: |+
       1 errors:
-      Unexpected element of type string. Each element should be a yaml associative array.
+      Unexpected element type. Each element should be a yaml associative array.
       ---
       - foo
       ---
@@ -299,12 +293,6 @@ trace_files: !mux
 
   invalid_yaml_parse_error:
     exit_status: 1
-    stdout_is: |+
-      1 errors:
-      mapping values are not allowed in this context
-      ---
-      this : is : not : yaml
-      ---
     validate_rules_file:
       - rules/invalid_yaml_parse_error.yaml
     trace_file: trace_files/cat_write.scap
@@ -339,7 +327,7 @@ trace_files: !mux
     exit_status: 1
     stdout_is: |+
       1 errors:
-      Rule must have property output
+      Rule must have properties 'condition', 'output', 'desc', and 'priority'
       ---
       - rule: no output rule
         desc: some desc
@@ -426,7 +414,7 @@ trace_files: !mux
     stdout_contains: |+
       .*invalid_base_macro.yaml: Ok
       .*invalid_append_macro.yaml: 1 errors:
-      Compilation error when compiling "evt.type=execve foo": 17: syntax error, unexpected 'foo', expecting 'or', 'and'
+      Compilation error when compiling "evt.type=execve foo": 17: unexpected token after 'execve', expecting 'or', 'and'
       ---
       - macro: some macro
         condition: evt.type=execve
@@ -458,7 +446,7 @@ trace_files: !mux
     exit_status: 1
     stdout_is: |+
       1 errors:
-      Compilation error when compiling "evt.type=execve foo": 17: syntax error, unexpected 'foo', expecting 'or', 'and'
+      Compilation error when compiling "evt.type=execve foo": 17: unexpected token after 'execve', expecting 'or', 'and'
       ---
       - macro: some macro
         condition: evt.type=execve
@@ -495,7 +483,7 @@ trace_files: !mux
     stdout_contains: |+
       .*invalid_base_rule.yaml: Ok
       .*invalid_append_rule.yaml: 1 errors:
-      Compilation error when compiling "evt.type=open bar": 15: syntax error, unexpected 'bar', expecting 'or', 'and'
+      Compilation error when compiling "evt.type=open bar": 15: unexpected token after 'open', expecting 'or', 'and'
       ---
       - rule: some rule
         desc: some desc
@@ -535,7 +523,7 @@ trace_files: !mux
   invalid_append_rule_multiple_docs:
     exit_status: 1
     stdout_contains: |+
-      Compilation error when compiling "evt.type=open bar": 15: syntax error, unexpected 'bar', expecting 'or', 'and'
+      Compilation error when compiling "evt.type=open bar": 15: unexpected token after 'open', expecting 'or', 'and'
       ---
       - rule: some rule
         desc: some desc

test/plugins/test_extract.cpp

@@ -19,7 +19,7 @@ limitations under the License.
 #include <string.h>
 #include <plugin_info.h>
 
-static const char *pl_required_api_version = "0.1.0";
+static const char *pl_required_api_version = "1.0.0";
 static uint32_t    pl_type                 = TYPE_EXTRACTOR_PLUGIN;
 static const char *pl_name_base            = "test_extract";
 static char pl_name[1024];

test/plugins/test_source.cpp

@@ -20,7 +20,7 @@ limitations under the License.
 
 #include <plugin_info.h>
 
-static const char *pl_required_api_version = "0.1.0";
+static const char *pl_required_api_version = "1.0.0";
 static uint32_t    pl_type                 = TYPE_SOURCE_PLUGIN;
 static uint32_t    pl_id                   = 999;
 static const char *pl_name                 = "test_source";

tests/CMakeLists.txt

@@ -20,6 +20,8 @@ if(MINIMAL_BUILD)
     test_base.cpp
     engine/test_rulesets.cpp
     engine/test_falco_utils.cpp
+    engine/test_filter_macro_resolver.cpp
+    engine/test_filter_evttype_resolver.cpp
     falco/test_configuration.cpp
   )
 else()
@@ -28,6 +30,8 @@ else()
     test_base.cpp
     engine/test_rulesets.cpp
     engine/test_falco_utils.cpp
+    engine/test_filter_macro_resolver.cpp
+    engine/test_filter_evttype_resolver.cpp
     falco/test_configuration.cpp
     falco/test_webserver.cpp
   )

tests/engine/test_filter_evttype_resolver.cpp

@@ -0,0 +1,237 @@
+/*
+Copyright (C) 2021 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+*/
+
+#include "filter_evttype_resolver.h"
+#include <catch.hpp>
+#include <sinsp.h>
+#include <filter/parser.h>
+
+using namespace std;
+using namespace libsinsp::filter;
+
+string to_string(set<uint16_t> s)
+{
+	string out = "[";
+	for(auto &val : s)
+	{
+        out += out.size() == 1 ? "" : ", ";
+		out += to_string(val);
+	}
+	out += "]";
+	return out;
+}
+
+void compare_evttypes(ast::expr* f, set<uint16_t> &expected)
+{
+    set<uint16_t> actual;
+    filter_evttype_resolver resolver;
+    resolver.evttypes(f, actual);
+    for(auto &etype : expected)
+    {
+        REQUIRE(actual.find(etype) != actual.end());
+    }
+    for(auto &etype : actual)
+    {
+        REQUIRE(expected.find(etype) != expected.end());
+    }
+}
+
+ast::expr* compile(const string &fltstr)
+{
+    libsinsp::filter::parser p(fltstr);
+    return p.parse();
+}
+
+TEST_CASE("Should find event types from filter", "[rule_loader]")
+{
+    filter_evttype_resolver resolver;
+    set<uint16_t> openat_only{
+		PPME_SYSCALL_OPENAT_2_E, PPME_SYSCALL_OPENAT_2_X };
+
+	set<uint16_t> close_only{
+		PPME_SYSCALL_CLOSE_E, PPME_SYSCALL_CLOSE_X };
+
+	set<uint16_t> openat_close{
+        PPME_SYSCALL_OPENAT_2_E, PPME_SYSCALL_OPENAT_2_X,
+        PPME_SYSCALL_CLOSE_E, PPME_SYSCALL_CLOSE_X };
+
+	set<uint16_t> not_openat;
+	set<uint16_t> not_openat_close;
+	set<uint16_t> not_close;
+	set<uint16_t> all_events;
+	set<uint16_t> no_events;
+    for(uint32_t i = 2; i < PPM_EVENT_MAX; i++)
+    {
+        // Skip "old" event versions that have been replaced
+        // by newer event versions, or events that are unused.
+        if(g_infotables.m_event_info[i].flags & (EF_OLD_VERSION | EF_UNUSED))
+        {
+            continue;
+        }
+        all_events.insert(i);
+        if(openat_only.find(i) == openat_only.end())
+        {
+            not_openat.insert(i);
+        }
+        if(openat_close.find(i) == openat_close.end())
+        {
+            not_openat_close.insert(i);
+        }
+        if (close_only.find(i) == close_only.end())
+        {
+            not_close.insert(i);
+        }
+    }
+
+    SECTION("evt_type_eq")
+    {
+        auto f = compile("evt.type=openat");
+        compare_evttypes(f, openat_only);
+    }
+
+    SECTION("evt_type_in")
+    {
+        auto f = compile("evt.type in (openat, close)");
+        compare_evttypes(f, openat_close);
+    }
+
+    SECTION("evt_type_ne")
+    {
+        auto f = compile("evt.type!=openat");
+        compare_evttypes(f, not_openat);
+    }
+
+    SECTION("not_evt_type_eq")
+    {
+        auto f = compile("not evt.type=openat");
+        compare_evttypes(f, not_openat);
+    }
+
+    SECTION("not_evt_type_in")
+    {
+        auto f = compile("not evt.type in (openat, close)");
+        compare_evttypes(f, not_openat_close);
+    }
+
+    SECTION("not_evt_type_ne")
+    {
+        auto f = compile("not evt.type != openat");
+        compare_evttypes(f, openat_only);
+    }
+
+    SECTION("evt_type_or")
+    {
+        auto f = compile("evt.type=openat or evt.type=close");
+        compare_evttypes(f, openat_close);
+    }
+
+    SECTION("not_evt_type_or")
+    {
+        auto f = compile("evt.type!=openat or evt.type!=close");
+        compare_evttypes(f, all_events);
+    }
+
+    SECTION("evt_type_or_ne")
+    {
+        auto f = compile("evt.type=close or evt.type!=openat");
+        compare_evttypes(f, not_openat);
+    }
+
+    SECTION("evt_type_and")
+    {
+        auto f = compile("evt.type=close and evt.type=openat");
+        compare_evttypes(f, no_events);
+    }
+
+    SECTION("evt_type_and_non_evt_type")
+    {
+        auto f = compile("evt.type=openat and proc.name=nginx");
+        compare_evttypes(f, openat_only);
+    }
+
+    SECTION("evt_type_and_non_evt_type_not")
+    {
+        auto f = compile("evt.type=openat and not proc.name=nginx");
+        compare_evttypes(f, openat_only);
+    }
+
+    SECTION("evt_type_and_nested")
+    {
+        auto f = compile("evt.type=openat and (proc.name=nginx)");
+        compare_evttypes(f, openat_only);
+    }
+
+    SECTION("evt_type_and_nested_multi")
+    {
+        auto f = compile("evt.type=openat and (evt.type=close and proc.name=nginx)");
+        compare_evttypes(f, no_events);
+    }
+
+    SECTION("non_evt_type")
+    {
+        auto f = compile("proc.name=nginx");
+        compare_evttypes(f, all_events);
+    }
+
+    SECTION("non_evt_type_or")
+    {
+        auto f = compile("evt.type=openat or proc.name=nginx");
+        compare_evttypes(f, all_events);
+    }
+
+    SECTION("non_evt_type_or_nested_first")
+    {
+        auto f = compile("(evt.type=openat) or proc.name=nginx");
+        compare_evttypes(f, all_events);
+    }
+
+    SECTION("non_evt_type_or_nested_second")
+    {
+        auto f = compile("evt.type=openat or (proc.name=nginx)");
+        compare_evttypes(f, all_events);
+    }
+
+    SECTION("non_evt_type_or_nested_multi")
+    {
+        auto f = compile("evt.type=openat or (evt.type=close and proc.name=nginx)");
+        compare_evttypes(f, openat_close);
+    }
+
+    SECTION("non_evt_type_or_nested_multi_not")
+    {
+        auto f = compile("evt.type=openat or not (evt.type=close and proc.name=nginx)");
+        compare_evttypes(f, not_close);
+    }
+
+    SECTION("non_evt_type_and_nested_multi_not")
+    {
+        auto f = compile("evt.type=openat and not (evt.type=close and proc.name=nginx)");
+        compare_evttypes(f, openat_only);
+    }
+
+    SECTION("ne_and_and")
+    {
+        auto f = compile("evt.type!=openat and evt.type!=close");
+        compare_evttypes(f, not_openat_close);
+    }
+
+    SECTION("not_not")
+    {
+        auto f = compile("not (not evt.type=openat)");
+        compare_evttypes(f, openat_only);
+    }
+}

tests/engine/test_filter_macro_resolver.cpp

@@ -0,0 +1,275 @@
+/*
+Copyright (C) 2020 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "filter_macro_resolver.h"
+#include <catch.hpp>
+
+using namespace std;
+using namespace libsinsp::filter::ast;
+
+TEST_CASE("Should resolve macros on a filter AST", "[rule_loader]")
+{
+	string macro_name = "test_macro";
+
+	SECTION("in the general case")
+	{
+		shared_ptr<expr> macro(
+			new unary_check_expr("test.field", "", "exists"));
+
+		expr* filter = new and_expr({
+			new unary_check_expr("evt.name", "", "exists"), 
+			new not_expr(
+				new value_expr(macro_name)
+			),
+		});
+		expr* expected_filter = new and_expr({
+			new unary_check_expr("evt.name", "", "exists"), 
+			new not_expr(clone(macro.get())),
+		});
+
+		filter_macro_resolver resolver;
+		resolver.set_macro(macro_name, macro);
+
+		// first run
+		REQUIRE(resolver.run(filter) == true);
+		REQUIRE(resolver.get_resolved_macros().size() == 1);
+		REQUIRE(*resolver.get_resolved_macros().begin() == macro_name);
+		REQUIRE(resolver.get_unknown_macros().empty());
+		REQUIRE(filter->is_equal(expected_filter));
+
+		// second run
+		REQUIRE(resolver.run(filter) == false);
+		REQUIRE(resolver.get_resolved_macros().empty());
+		REQUIRE(resolver.get_unknown_macros().empty());
+		REQUIRE(filter->is_equal(expected_filter));
+
+		delete filter;
+		delete expected_filter;
+	}
+
+	SECTION("with a single node")
+	{
+		shared_ptr<expr> macro(
+			new unary_check_expr("test.field", "", "exists"));
+
+		expr* filter = new value_expr(macro_name);
+
+		filter_macro_resolver resolver;
+		resolver.set_macro(macro_name, macro);
+
+		// first run
+		expr* old_filter_ptr = filter;
+		REQUIRE(resolver.run(filter) == true);
+		REQUIRE(filter != old_filter_ptr);
+		REQUIRE(resolver.get_resolved_macros().size() == 1);
+		REQUIRE(*resolver.get_resolved_macros().begin() == macro_name);
+		REQUIRE(resolver.get_unknown_macros().empty());
+		REQUIRE(filter->is_equal(macro.get()));
+
+		// second run
+		old_filter_ptr = filter;
+		REQUIRE(resolver.run(filter) == false);
+		REQUIRE(filter == old_filter_ptr);
+		REQUIRE(resolver.get_resolved_macros().empty());
+		REQUIRE(resolver.get_unknown_macros().empty());
+		REQUIRE(filter->is_equal(macro.get()));
+
+		delete filter;
+	}
+
+	SECTION("with multiple macros")
+	{
+		string a_macro_name = macro_name + "_1";
+		string b_macro_name = macro_name + "_2";
+
+		shared_ptr<expr> a_macro(
+			new unary_check_expr("one.field", "", "exists"));
+		shared_ptr<expr> b_macro(
+			new unary_check_expr("another.field", "", "exists"));
+
+		expr* filter = new or_expr({
+			new value_expr(a_macro_name),
+			new value_expr(b_macro_name),
+		});
+		expr* expected_filter = new or_expr({
+			clone(a_macro.get()),
+			clone(b_macro.get()),
+		});
+
+		filter_macro_resolver resolver;
+		resolver.set_macro(a_macro_name, a_macro);
+		resolver.set_macro(b_macro_name, b_macro);
+
+		// first run
+		REQUIRE(resolver.run(filter) == true);
+		REQUIRE(resolver.get_resolved_macros().size() == 2);
+		REQUIRE(resolver.get_resolved_macros().find(a_macro_name)
+				!= resolver.get_resolved_macros().end());
+		REQUIRE(resolver.get_resolved_macros().find(b_macro_name)
+				!= resolver.get_resolved_macros().end());
+		REQUIRE(resolver.get_unknown_macros().empty());
+		REQUIRE(filter->is_equal(expected_filter));
+
+		// second run
+		REQUIRE(resolver.run(filter) == false);
+		REQUIRE(resolver.get_resolved_macros().empty());
+		REQUIRE(resolver.get_unknown_macros().empty());
+		REQUIRE(filter->is_equal(expected_filter));
+
+		delete filter;
+		delete expected_filter;
+	}
+
+	SECTION("with nested macros")
+	{
+		string a_macro_name = macro_name + "_1";
+		string b_macro_name = macro_name + "_2";
+
+		shared_ptr<expr> a_macro(new and_expr({
+			new unary_check_expr("one.field", "", "exists"),
+			new value_expr(b_macro_name),
+		}));
+		shared_ptr<expr> b_macro(
+			new unary_check_expr("another.field", "", "exists"));
+
+		expr* filter = new value_expr(a_macro_name);
+		expr* expected_filter = new and_expr({
+			new unary_check_expr("one.field", "", "exists"),
+			new unary_check_expr("another.field", "", "exists"),
+		});
+
+		filter_macro_resolver resolver;
+		resolver.set_macro(a_macro_name, a_macro);
+		resolver.set_macro(b_macro_name, b_macro);
+
+		// first run
+		REQUIRE(resolver.run(filter) == true);
+		REQUIRE(resolver.get_resolved_macros().size() == 2);
+		REQUIRE(resolver.get_resolved_macros().find(a_macro_name)
+				!= resolver.get_resolved_macros().end());
+		REQUIRE(resolver.get_resolved_macros().find(b_macro_name)
+				!= resolver.get_resolved_macros().end());
+		REQUIRE(resolver.get_unknown_macros().empty());
+		REQUIRE(filter->is_equal(expected_filter));
+
+		// second run
+		REQUIRE(resolver.run(filter) == false);
+		REQUIRE(resolver.get_resolved_macros().empty());
+		REQUIRE(resolver.get_unknown_macros().empty());
+		REQUIRE(filter->is_equal(expected_filter));
+
+		delete filter;
+		delete expected_filter;
+	}
+}
+
+TEST_CASE("Should find unknown macros", "[rule_loader]")
+{
+	string macro_name = "test_macro";
+
+	SECTION("in the general case")
+	{
+		expr* filter = new and_expr({
+			new unary_check_expr("evt.name", "", "exists"), 
+			new not_expr(
+				new value_expr(macro_name)
+			),
+		});
+
+		filter_macro_resolver resolver;
+		REQUIRE(resolver.run(filter) == false);
+		REQUIRE(resolver.get_unknown_macros().size() == 1);
+		REQUIRE(*resolver.get_unknown_macros().begin() == macro_name);
+		REQUIRE(resolver.get_resolved_macros().empty());
+
+		delete filter;
+	}
+
+	SECTION("with nested macros")
+	{
+		string a_macro_name = macro_name + "_1";
+		string b_macro_name = macro_name + "_2";
+
+		shared_ptr<expr> a_macro(new and_expr({
+			new unary_check_expr("one.field", "", "exists"),
+			new value_expr(b_macro_name),
+		}));
+
+		expr* filter = new value_expr(a_macro_name);
+		expr* expected_filter = clone(a_macro.get());
+
+		filter_macro_resolver resolver;
+		resolver.set_macro(a_macro_name, a_macro);
+
+		// first run
+		REQUIRE(resolver.run(filter) == true);
+		REQUIRE(resolver.get_resolved_macros().size() == 1);
+		REQUIRE(*resolver.get_resolved_macros().begin() == a_macro_name);
+		REQUIRE(resolver.get_unknown_macros().size() == 1);
+		REQUIRE(*resolver.get_unknown_macros().begin() == b_macro_name);
+		REQUIRE(filter->is_equal(expected_filter));
+
+		delete filter;
+		delete expected_filter;
+	}
+}
+
+TEST_CASE("Should undefine macro", "[rule_loader]")
+{
+	string macro_name = "test_macro";
+	shared_ptr<expr> macro(new unary_check_expr("test.field", "", "exists"));
+	expr* a_filter = new value_expr(macro_name);
+	expr* b_filter = new value_expr(macro_name);
+	filter_macro_resolver resolver;
+
+	resolver.set_macro(macro_name, macro);
+	REQUIRE(resolver.run(a_filter) == true);
+	REQUIRE(resolver.get_resolved_macros().size() == 1);
+	REQUIRE(*resolver.get_resolved_macros().begin() == macro_name);
+	REQUIRE(resolver.get_unknown_macros().empty());
+	REQUIRE(a_filter->is_equal(macro.get()));
+
+	resolver.set_macro(macro_name, NULL);
+	REQUIRE(resolver.run(b_filter) == false);
+	REQUIRE(resolver.get_resolved_macros().empty());
+	REQUIRE(resolver.get_unknown_macros().size() == 1);
+	REQUIRE(*resolver.get_unknown_macros().begin() == macro_name);
+
+	delete a_filter;
+	delete b_filter;
+}
+
+// checks that the macro AST is cloned and not shared across resolved filters
+TEST_CASE("Should clone macro AST", "[rule_loader]")
+{
+	string macro_name = "test_macro";
+	shared_ptr<unary_check_expr> macro(
+		new unary_check_expr("test.field", "", "exists"));
+	expr* filter = new value_expr(macro_name);
+	filter_macro_resolver resolver;
+	
+	resolver.set_macro(macro_name, macro);
+	REQUIRE(resolver.run(filter) == true);
+	REQUIRE(resolver.get_resolved_macros().size() == 1);
+	REQUIRE(*resolver.get_resolved_macros().begin() == macro_name);
+	REQUIRE(resolver.get_unknown_macros().empty());
+	REQUIRE(filter->is_equal(macro.get()));
+
+	macro.get()->field = "another.field";
+	REQUIRE(!filter->is_equal(macro.get()));
+
+	delete filter;
+}

tests/engine/test_rulesets.cpp

@@ -25,6 +25,7 @@ static uint16_t default_ruleset = 0;
 static uint16_t non_default_ruleset = 3;
 static uint16_t other_non_default_ruleset = 2;
 static std::set<std::string> tags = {"some_tag", "some_other_tag"};
+static std::set<uint16_t> evttypes = { ppm_event_type::PPME_GENERIC_E };
 
 static std::shared_ptr<gen_event_filter> create_filter()
 {
@@ -37,234 +38,159 @@ static std::shared_ptr<gen_event_filter> create_filter()
 	return ret;
 }
 
-TEST_CASE("Should enable/disable for exact match w/ default ruleset", "[rulesets]")
+TEST_CASE("Should enable/disable on ruleset", "[rulesets]")
 {
 	falco_ruleset r;
 	std::shared_ptr<gen_event_filter> filter = create_filter();
 	string rule_name = "one_rule";
 	string source = "syscall";
 
-	r.add(source, rule_name, tags, filter);
+	r.add(source, rule_name, tags, evttypes, filter);
 
-	r.enable("one_rule", exact_match, enabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
+	SECTION("Should enable/disable for exact match w/ default ruleset")
+	{
+		r.enable("one_rule", exact_match, enabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
+
+		r.enable("one_rule", exact_match, disabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+	}
+
+	SECTION("Should enable/disable for exact match w/ specific ruleset")
+	{
+		r.enable("one_rule", exact_match, enabled, non_default_ruleset);
+		REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+		REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
+
+		r.enable("one_rule", exact_match, disabled, non_default_ruleset);
+		REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+		REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
+	}
+
+	SECTION("Should not enable for exact match different rule name")
+	{
+		r.enable("some_other_rule", exact_match, enabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+	}
+
+	SECTION("Should enable/disable for exact match w/ substring and default ruleset")
+	{
+		r.enable("one_rule", substring_match, enabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
+
+		r.enable("one_rule", substring_match, disabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+	}
+
+	SECTION("Should not enable for substring w/ exact_match")
+	{
+		r.enable("one_", exact_match, enabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+	}
+
+	SECTION("Should enable/disable for prefix match w/ default ruleset")
+	{
+		r.enable("one_", substring_match, enabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
+
+		r.enable("one_", substring_match, disabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+	}
+
+	SECTION("Should enable/disable for suffix match w/ default ruleset")
+	{
+		r.enable("_rule", substring_match, enabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
+
+		r.enable("_rule", substring_match, disabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+	}
+
+	SECTION("Should enable/disable for substring match w/ default ruleset")
+	{
+		r.enable("ne_ru", substring_match, enabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
+
+		r.enable("ne_ru", substring_match, disabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+	}
+
+	SECTION("Should enable/disable for substring match w/ specific ruleset")
+	{
+		r.enable("ne_ru", substring_match, enabled, non_default_ruleset);
+		REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+		REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
+
+		r.enable("ne_ru", substring_match, disabled, non_default_ruleset);
+		REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+		REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
+	}
+
+	SECTION("Should enable/disable for tags w/ default ruleset")
+	{
+		std::set<std::string> want_tags = {"some_tag"};
+
+		r.enable_tags(want_tags, enabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
+
+		r.enable_tags(want_tags, disabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+	}
+
+	SECTION("Should enable/disable for tags w/ specific ruleset")
+	{
+		std::set<std::string> want_tags = {"some_tag"};
+
+		r.enable_tags(want_tags, enabled, non_default_ruleset);
+		REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+		REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
+
+		r.enable_tags(want_tags, disabled, non_default_ruleset);
+		REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+		REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
+	}
+
+	SECTION("Should not enable for different tags")
+	{
+		std::set<std::string> want_tags = {"some_different_tag"};
+
+		r.enable_tags(want_tags, enabled);
+		REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0);
+	}
+
+	SECTION("Should enable/disable for overlapping tags")
+	{
+		std::set<std::string> want_tags = {"some_tag", "some_different_tag"};
+
+		r.enable_tags(want_tags, enabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
+
+		r.enable_tags(want_tags, disabled);
+		REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
+	}
 
-	r.enable("one_rule", exact_match, disabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
 }
 
-TEST_CASE("Should enable/disable for exact match w/ specific ruleset", "[rulesets]")
+TEST_CASE("Should enable/disable on ruleset for incremental adding tags", "[rulesets]")
 {
 	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
 	string source = "syscall";
 
-	r.add(source, rule_name, tags, filter);
-
-	r.enable("one_rule", exact_match, enabled, non_default_ruleset);
-	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-	REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
-
-	r.enable("one_rule", exact_match, disabled, non_default_ruleset);
-	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-	REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
-}
-
-TEST_CASE("Should not enable for exact match different rule name", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable("some_other_rule", exact_match, enabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-}
-
-TEST_CASE("Should enable/disable for exact match w/ substring and default ruleset", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable("one_rule", substring_match, enabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
-
-	r.enable("one_rule", substring_match, disabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-}
-
-TEST_CASE("Should not enable for substring w/ exact_match", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable("one_", exact_match, enabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-}
-
-TEST_CASE("Should enable/disable for prefix match w/ default ruleset", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable("one_", substring_match, enabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
-
-	r.enable("one_", substring_match, disabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-}
-
-TEST_CASE("Should enable/disable for suffix match w/ default ruleset", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable("_rule", substring_match, enabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
-
-	r.enable("_rule", substring_match, disabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-}
-
-TEST_CASE("Should enable/disable for substring match w/ default ruleset", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable("ne_ru", substring_match, enabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
-
-	r.enable("ne_ru", substring_match, disabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-}
-
-TEST_CASE("Should enable/disable for substring match w/ specific ruleset", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable("ne_ru", substring_match, enabled, non_default_ruleset);
-	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-	REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
-
-	r.enable("ne_ru", substring_match, disabled, non_default_ruleset);
-	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-	REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
-}
-
-TEST_CASE("Should enable/disable for tags w/ default ruleset", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-	std::set<std::string> want_tags = {"some_tag"};
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable_tags(want_tags, enabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
-
-	r.enable_tags(want_tags, disabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-}
-
-TEST_CASE("Should enable/disable for tags w/ specific ruleset", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-	std::set<std::string> want_tags = {"some_tag"};
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable_tags(want_tags, enabled, non_default_ruleset);
-	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-	REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
-
-	r.enable_tags(want_tags, disabled, non_default_ruleset);
-	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-	REQUIRE(r.num_rules_for_ruleset(other_non_default_ruleset) == 0);
-}
-
-TEST_CASE("Should not enable for different tags", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-	std::set<std::string> want_tags = {"some_different_tag"};
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable_tags(want_tags, enabled);
-	REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0);
-}
-
-TEST_CASE("Should enable/disable for overlapping tags", "[rulesets]")
-{
-	falco_ruleset r;
-	std::shared_ptr<gen_event_filter> filter = create_filter();
-	string rule_name = "one_rule";
-	string source = "syscall";
-	std::set<std::string> want_tags = {"some_tag", "some_different_tag"};
-
-	r.add(source, rule_name, tags, filter);
-
-	r.enable_tags(want_tags, enabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1);
-
-	r.enable_tags(want_tags, disabled);
-	REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0);
-}
-
-TEST_CASE("Should enable/disable for incremental adding tags", "[rulesets]")
-{
-	string source = "syscall";
-	falco_ruleset r;
 	std::shared_ptr<gen_event_filter> rule1_filter = create_filter();
 	string rule1_name = "one_rule";
 	std::set<std::string> rule1_tags = {"rule1_tag"};
-	r.add(source, rule1_name, rule1_tags, rule1_filter);
+	r.add(source, rule1_name, rule1_tags, evttypes, rule1_filter);
 
 	std::shared_ptr<gen_event_filter> rule2_filter = create_filter();
 	string rule2_name = "two_rule";
 	std::set<std::string> rule2_tags = {"rule2_tag"};
-	r.add(source, rule2_name, rule2_tags, rule2_filter);
+	r.add(source, rule2_name, rule2_tags, evttypes, rule2_filter);
 
 	std::set<std::string> want_tags;
 

userspace/engine/CMakeLists.txt

@@ -10,49 +10,49 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
-add_subdirectory(lua)
-
 set(FALCO_ENGINE_SOURCE_FILES
-    rules.cpp
     falco_common.cpp
     falco_engine.cpp
     falco_utils.cpp
     json_evt.cpp
     ruleset.cpp
-    formats.cpp)
+    formats.cpp
+    filter_macro_resolver.cpp
+    filter_evttype_resolver.cpp
+    rule_loader.cpp
+    rule_reader.cpp
+    stats_manager.cpp)
 
 add_library(falco_engine STATIC ${FALCO_ENGINE_SOURCE_FILES})
-add_dependencies(falco_engine njson lyaml lpeg string-view-lite)
+add_dependencies(falco_engine njson string-view-lite)
 
 if(USE_BUNDLED_DEPS)
-  add_dependencies(falco_engine libyaml)
+  add_dependencies(falco_engine yamlcpp)
 endif()
 
 if(MINIMAL_BUILD)
   target_include_directories(
     falco_engine
     PUBLIC
-      "${LUAJIT_INCLUDE}"
       "${NJSON_INCLUDE}"
       "${TBB_INCLUDE_DIR}"
       "${STRING_VIEW_LITE_INCLUDE}"
       "${LIBSCAP_INCLUDE_DIRS}"
       "${LIBSINSP_INCLUDE_DIRS}"
-      "${PROJECT_BINARY_DIR}/userspace/engine"
-      "${PROJECT_BINARY_DIR}/userspace/engine/lua")
+      "${YAMLCPP_INCLUDE_DIR}"
+      "${PROJECT_BINARY_DIR}/userspace/engine")
 else()
   target_include_directories(
     falco_engine
     PUBLIC
-      "${LUAJIT_INCLUDE}"
       "${NJSON_INCLUDE}"
       "${CURL_INCLUDE_DIR}"
       "${TBB_INCLUDE_DIR}"
       "${STRING_VIEW_LITE_INCLUDE}"
       "${LIBSCAP_INCLUDE_DIRS}"
       "${LIBSINSP_INCLUDE_DIRS}"
-      "${PROJECT_BINARY_DIR}/userspace/engine"
-      "${PROJECT_BINARY_DIR}/userspace/engine/lua")
+      "${YAMLCPP_INCLUDE_DIR}"
+      "${PROJECT_BINARY_DIR}/userspace/engine")
 endif()
 
-target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${LPEG_LIB}" "${LYAML_LIB}" "${LIBYAML_LIB}" luafiles)
+target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${YAMLCPP_LIB}")

userspace/engine/falco_common.cpp

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019 The Falco Authors.
+Copyright (C) 2022 The Falco Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,13 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include <fstream>
-
 #include "falco_common.h"
-#include "banned.h" // This raises a compilation error when certain functions are used
-#include "falco_engine_lua_files.hh"
 
-std::vector<std::string> falco_common::priority_names = {
+static vector<string> priority_names = {
 	"Emergency",
 	"Alert",
 	"Critical",
@@ -28,53 +24,59 @@ std::vector<std::string> falco_common::priority_names = {
 	"Warning",
 	"Notice",
 	"Informational",
-	"Debug"};
+	"Debug"
+};
 
-falco_common::falco_common()
+bool falco_common::parse_priority(string v, priority_type& out)
 {
-	m_ls = lua_open();
-	if(!m_ls)
+	for (size_t i = 0; i < priority_names.size(); i++)
 	{
-		throw falco_exception("Cannot open lua");
-	}
-	luaL_openlibs(m_ls);
-}
-
-falco_common::~falco_common()
-{
-	if(m_ls)
-	{
-		lua_close(m_ls);
-	}
-}
-
-void falco_common::init()
-{
-	// Strings in the list lua_module_strings need to be loaded as
-	// lua modules, which also involves adding them to the
-	// package.module table.
-	for(const auto &pair : lua_module_strings)
-	{
-		lua_getglobal(m_ls, "package");
-		lua_getfield(m_ls, -1, "preload");
-
-		if(luaL_loadstring(m_ls, pair.first))
+		// note: for legacy reasons, "Info" and "Informational" has been used
+		// interchangeably and ambiguously, so this is the only edge case for
+		// which we can't apply strict equality check
+		if (!strcasecmp(v.c_str(), priority_names[i].c_str())
+			|| (i == PRIORITY_INFORMATIONAL && !strcasecmp(v.c_str(), "info")))
 		{
-			throw falco_exception("Failed to load embedded lua code " +
-					      string(pair.second) + ": " + lua_tostring(m_ls, -1));
-		}
-
-		lua_setfield(m_ls, -2, pair.second);
-	}
-
-	// Strings in the list lua_code_strings need to be loaded and
-	// evaluated so any public functions can be directly called.
-	for(const auto &str : lua_code_strings)
-	{
-		if(luaL_loadstring(m_ls, str) || lua_pcall(m_ls, 0, 0, 0))
-		{
-			throw falco_exception("Failed to load + evaluate embedded lua code " +
-					      string(str) + ": " + lua_tostring(m_ls, -1));
+			out = (priority_type) i;
+			return true;
 		}
 	}
+	return false;
 }
+
+falco_common::priority_type falco_common::parse_priority(string v)
+{
+	falco_common::priority_type out;
+	if (!parse_priority(v, out))
+	{
+		throw falco_exception("Unknown priority value: " + v);
+	}
+	return out;
+}
+
+bool falco_common::format_priority(priority_type v, string& out, bool shortfmt)
+{
+	if ((size_t) v < priority_names.size())
+	{
+		if (v == PRIORITY_INFORMATIONAL && shortfmt)
+		{
+			out = "Info";
+		}
+		else
+		{
+			out = priority_names[(size_t) v];
+		}
+		return true;
+	}
+	return false;
+}
+
+string falco_common::format_priority(priority_type v, bool shortfmt)
+{
+	string out;
+	if(!format_priority(v, out, shortfmt))
+	{
+		throw falco_exception("Unknown priority enum value: " + to_string(v));
+	}
+	return out;
+}

userspace/engine/falco_common.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019 The Falco Authors.
+Copyright (C) 2022 The Falco Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,13 +19,6 @@ limitations under the License.
 #include <string>
 #include <exception>
 #include <mutex>
-
-extern "C" {
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
-}
-
 #include <sinsp.h>
 
 //
@@ -57,22 +50,9 @@ struct falco_exception : std::exception
 	std::string m_error_str;
 };
 
-//
-// This is the base class of falco_engine/falco_output. It is
-// responsible for managing a lua state and associated inspector and
-// loading a single "main" lua file into that state.
-//
-
-class falco_common
+namespace falco_common
 {
-public:
-	falco_common();
-	virtual ~falco_common();
-
-	void init();
-
-    // Priority levels, as a vector of strings
-	static std::vector<std::string> priority_names;
+	const string syscall_source = "syscall";
 
 	// Same as numbers/indices into the above vector
 	enum priority_type
@@ -86,9 +66,9 @@ public:
 		PRIORITY_INFORMATIONAL = 6,
 		PRIORITY_DEBUG = 7
 	};
-
-protected:
-	lua_State *m_ls;
-
-	std::mutex m_ls_semaphore;
+	
+	bool parse_priority(std::string v, priority_type& out);
+	priority_type parse_priority(std::string v);
+	bool format_priority(priority_type v, std::string& out, bool shortfmt=false);
+	std::string format_priority(priority_type v, bool shortfmt=false);
 };

userspace/engine/falco_engine.cpp

@@ -25,20 +25,13 @@ limitations under the License.
 #include "falco_engine.h"
 #include "falco_utils.h"
 #include "falco_engine_version.h"
+#include "rule_reader.h"
 
 #include "formats.h"
 
-extern "C" {
-#include "lpeg.h"
-#include "lyaml.h"
-}
-
 #include "utils.h"
 #include "banned.h" // This raises a compilation error when certain functions are used
 
-
-string lua_on_event = "on_event";
-string lua_print_stats = "print_stats";
 const std::string falco_engine::s_default_ruleset = "falco-default-ruleset";
 
 using namespace std;
@@ -49,14 +42,6 @@ falco_engine::falco_engine(bool seed_rng)
 	  m_sampling_ratio(1), m_sampling_multiplier(0),
 	  m_replace_container_info(false)
 {
-	luaopen_lpeg(m_ls);
-	luaopen_yaml(m_ls);
-
-	falco_common::init();
-	falco_rules::init(m_ls);
-
-	m_required_plugin_versions.clear();
-
 	if(seed_rng)
 	{
 		srandom((unsigned) getpid());
@@ -67,6 +52,9 @@ falco_engine::falco_engine(bool seed_rng)
 
 falco_engine::~falco_engine()
 {
+	m_rules.clear();
+	m_rule_loader.clear();
+	m_rule_stats_manager.clear();
 }
 
 uint32_t falco_engine::engine_version()
@@ -162,18 +150,45 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al
 
 void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version)
 {
-	if(!m_rules)
-	{
-		m_rules.reset(new falco_rules(this,
-					      m_ls));
+	rule_loader::configuration cfg(rules_content);
+	cfg.engine = this;
+	cfg.min_priority = m_min_priority;
+	cfg.output_extra = m_extra;
+	cfg.replace_output_container_info = m_replace_container_info;
 
-		for(auto const &it : m_filter_factories)
+	std::ostringstream os;
+	rule_reader reader;
+	bool success = reader.load(cfg, m_rule_loader);
+	if (success)
+	{
+		clear_filters();
+		m_rules.clear();
+		success = m_rule_loader.compile(cfg, m_rules);
+	}
+	if (!cfg.errors.empty())
+	{
+		os << cfg.errors.size() << " errors:" << std::endl;
+		for(auto &err : cfg.errors)
 		{
-			m_rules->add_filter_factory(it.first, it.second);
+			os << err << std::endl;
 		}
 	}
-
-	m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info, m_min_priority, required_engine_version, m_required_plugin_versions);
+	if (!cfg.warnings.empty())
+	{
+		os << cfg.warnings.size() << " warnings:" << std::endl;
+		for(auto &warn : cfg.warnings)
+		{
+			os << warn << std::endl;
+		}
+	}
+	if(!success)
+	{
+		throw falco_exception(os.str());
+	}
+	if (verbose && os.str() != "") {
+		// todo(jasondellaluce): introduce a logging callback in Falco
+		fprintf(stderr, "When reading rules content: %s", os.str().c_str());
+	}
 }
 
 void falco_engine::load_rules_file(const string &rules_filename, bool verbose, bool all_events)
@@ -208,7 +223,7 @@ void falco_engine::enable_rule(const string &substring, bool enabled, const stri
 
 	for(auto &it : m_rulesets)
 	{
-		it.second->enable(substring, match_exact, enabled, ruleset_id);
+		it.ruleset->enable(substring, match_exact, enabled, ruleset_id);
 	}
 }
 
@@ -219,7 +234,7 @@ void falco_engine::enable_rule_exact(const string &rule_name, bool enabled, cons
 
 	for(auto &it : m_rulesets)
 	{
-		it.second->enable(rule_name, match_exact, enabled, ruleset_id);
+		it.ruleset->enable(rule_name, match_exact, enabled, ruleset_id);
 	}
 }
 
@@ -229,7 +244,7 @@ void falco_engine::enable_rule_by_tag(const set<string> &tags, bool enabled, con
 
 	for(auto &it : m_rulesets)
 	{
-		it.second->enable_tags(tags, enabled, ruleset_id);
+		it.ruleset->enable_tags(tags, enabled, ruleset_id);
 	}
 }
 
@@ -259,7 +274,7 @@ uint64_t falco_engine::num_rules_for_ruleset(const std::string &ruleset)
 	uint64_t ret = 0;
 	for(auto &it : m_rulesets)
 	{
-		ret += it.second->num_rules_for_ruleset(ruleset_id);
+		ret += it.ruleset->num_rules_for_ruleset(ruleset_id);
 	}
 
 	return ret;
@@ -269,14 +284,14 @@ void falco_engine::evttypes_for_ruleset(std::string &source, std::set<uint16_t>
 {
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
 
-	auto it = m_rulesets.find(source);
+	auto it = find_ruleset(source);
 	if(it == m_rulesets.end())
 	{
 		string err = "Unknown event source " + source;
 		throw falco_exception(err);
 	}
 
-	it->second->evttypes_for_ruleset(evttypes, ruleset_id);
+	it->ruleset->evttypes_for_ruleset(evttypes, ruleset_id);
 
 }
 
@@ -294,186 +309,145 @@ std::shared_ptr<gen_event_formatter> falco_engine::create_formatter(const std::s
 	return it->second->create_formatter(output);
 }
 
-unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::string &source, gen_event *ev, uint16_t ruleset_id)
+unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::size_t source_idx, gen_event *ev, uint16_t ruleset_id)
 {
 	if(should_drop_evt())
 	{
 		return unique_ptr<struct rule_result>();
 	}
 
-	auto it = m_rulesets.find(source);
-	if(it == m_rulesets.end())
+	try
 	{
-		string err = "Unknown event source " + source;
+		auto &r = m_rulesets.at(source_idx);
+		if(!r.ruleset->run(ev, ruleset_id))
+		{
+			return unique_ptr<struct rule_result>();
+		}
+
+		unique_ptr<struct rule_result> res(new rule_result());
+		// note: indexes are 0-based, whereas check_ids are not
+		auto rule_idx = ev->get_check_id() - 1;
+		auto rule = m_rules.at(rule_idx);
+		if (!rule)
+		{
+			throw falco_exception("populate_rule_result error: unknown rule id "
+					+ to_string(rule_idx));
+		}
+		res->evt = ev;
+		res->rule = rule->name;
+		res->source = rule->source;
+		res->format = rule->output;
+		res->priority_num = rule->priority;
+		res->tags = rule->tags;
+		res->exception_fields = rule->exception_fields;
+		m_rule_stats_manager.on_event(m_rules, rule_idx);
+		return res;
+	}
+	catch(std::out_of_range const &exc)
+	{
+		std::string err = "Unknown event source index " + std::to_string(source_idx);
 		throw falco_exception(err);
 	}
-
-	if (!it->second->run(ev, ruleset_id))
-	{
-		return unique_ptr<struct rule_result>();
-	}
-
-	unique_ptr<struct rule_result> res(new rule_result());
-	res->source = source;
-
-	populate_rule_result(res, ev);
-
-	return res;
 }
 
-unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::string &source, gen_event *ev)
+unique_ptr<falco_engine::rule_result> falco_engine::process_event(std::size_t source_idx, gen_event *ev)
 {
-	return process_event(source, ev, m_default_ruleset_id);
+	return process_event(source_idx, ev, m_default_ruleset_id);
 }
 
-void falco_engine::add_source(const std::string &source,
-			      std::shared_ptr<gen_event_filter_factory> filter_factory,
-			      std::shared_ptr<gen_event_formatter_factory> formatter_factory)
+std::size_t falco_engine::add_source(const std::string &source,
+				     std::shared_ptr<gen_event_filter_factory> filter_factory,
+				     std::shared_ptr<gen_event_formatter_factory> formatter_factory)
 {
 	m_filter_factories[source] = filter_factory;
 	m_format_factories[source] = formatter_factory;
 
-	std::shared_ptr<falco_ruleset> ruleset(new falco_ruleset());
-	m_rulesets[source] = ruleset;
+	auto idx = m_rulesets.size();
+	m_rulesets.emplace_back(source, new falco_ruleset);
+	// here we just trust the caller they won't add the same source more than once
+	return idx;
 }
 
-void falco_engine::populate_rule_result(unique_ptr<struct rule_result> &res, gen_event *ev)
+std::shared_ptr<gen_event_filter_factory> falco_engine::get_filter_factory(
+	const std::string &source)
 {
-	std::lock_guard<std::mutex> guard(m_ls_semaphore);
-	lua_getglobal(m_ls, lua_on_event.c_str());
-	if(lua_isfunction(m_ls, -1))
+	auto it = m_filter_factories.find(source);
+	if(it == m_filter_factories.end())
 	{
-		lua_pushnumber(m_ls, ev->get_check_id());
-		if(lua_pcall(m_ls, 1, 5, 0) != 0)
-		{
-			const char* lerr = lua_tostring(m_ls, -1);
-			string err = "Error invoking function output: " + string(lerr);
-			throw falco_exception(err);
-		}
-		const char *p =  lua_tostring(m_ls, -5);
-		res->rule = p;
-		res->evt = ev;
-		res->priority_num = (falco_common::priority_type) lua_tonumber(m_ls, -4);
-		res->format = lua_tostring(m_ls, -3);
-
-		// Tags are passed back as a table, and is on the top of the stack
-		lua_pushnil(m_ls);  /* first key */
-		while (lua_next(m_ls, -2) != 0) {
-			// key is at index -2, value is at index
-			// -1. We want the value.
-			res->tags.insert(luaL_checkstring(m_ls, -1));
-
-			// Remove value, keep key for next iteration
-			lua_pop(m_ls, 1);
-		}
-		lua_pop(m_ls, 1); // Clean table leftover
-
-		// Exception fields are passed back as a table
-		lua_pushnil(m_ls);  /* first key */
-		while (lua_next(m_ls, -2) != 0) {
-			// key is at index -2, value is at index
-			// -1. We want the keys.
-			res->exception_fields.insert(luaL_checkstring(m_ls, -2));
-
-			// Remove value, keep key for next iteration
-			lua_pop(m_ls, 1);
-		}
-
-		lua_pop(m_ls, 4);
-	}
-	else
-	{
-		throw falco_exception("No function " + lua_on_event + " found in lua compiler module");
+		throw falco_exception(string("unknown event source: ") + source);
 	}
+	return it->second;
 }
 
 void falco_engine::describe_rule(string *rule)
 {
-	return m_rules->describe_rule(rule);
-}
-
-// Print statistics on the rules that triggered
-void falco_engine::print_stats()
-{
-	lua_getglobal(m_ls, lua_print_stats.c_str());
-
-	if(lua_isfunction(m_ls, -1))
+	static const char* rule_fmt = "%-50s %s\n";
+	fprintf(stdout, rule_fmt, "Rule", "Description");
+	fprintf(stdout, rule_fmt, "----",  "-----------");
+	if (!rule)
 	{
-		if(lua_pcall(m_ls, 0, 0, 0) != 0)
+		for (auto &r : m_rules)
 		{
-			const char* lerr = lua_tostring(m_ls, -1);
-			string err = "Error invoking function print_stats: " + string(lerr);
-			throw falco_exception(err);
+			auto str = falco::utils::wrap_text(r.description, 51, 110) + "\n";
+			fprintf(stdout, rule_fmt, r.name.c_str(), str.c_str());
 		}
 	}
 	else
 	{
-		throw falco_exception("No function " + lua_print_stats + " found in lua rule loader module");
+		auto r = m_rules.at(*rule);
+		auto str = falco::utils::wrap_text(r->description, 51, 110) + "\n";
+		fprintf(stdout, rule_fmt, r->name.c_str(), str.c_str());
 	}
 
 }
 
+void falco_engine::print_stats()
+{
+	string out;
+	m_rule_stats_manager.format(m_rules, out);
+	// todo(jasondellaluce): introduce a logging callback in Falco
+	fprintf(stdout, "%s", out.c_str());
+}
+
 void falco_engine::add_filter(std::shared_ptr<gen_event_filter> filter,
 			      std::string &rule,
 			      std::string &source,
+			      std::set<uint16_t> &evttypes,
 			      std::set<std::string> &tags)
 {
-	auto it = m_rulesets.find(source);
+	auto it = find_ruleset(source);
 	if(it == m_rulesets.end())
 	{
 		string err = "Unknown event source " + source;
 		throw falco_exception(err);
 	}
 
-	it->second->add(source, rule, tags, filter);
+	it->ruleset->add(source, rule, tags, evttypes, filter);
 }
 
 bool falco_engine::is_source_valid(const std::string &source)
 {
-	return (m_rulesets.find(source) != m_rulesets.end());
+	return (find_ruleset(source) != m_rulesets.end());
 }
 
 bool falco_engine::is_plugin_compatible(const std::string &name,
 					const std::string &version,
 					std::string &required_version)
 {
-	sinsp_plugin::version plugin_version(version);
-
-	if(!plugin_version.m_valid)
-	{
-		throw falco_exception(string("Plugin version string ") + version + " not valid");
-	}
-
-	if(m_required_plugin_versions.find(name) == m_required_plugin_versions.end())
-	{
-		// No required engine versions, so no restrictions. Compatible.
-		return true;
-	}
-
-	for(auto &rversion : m_required_plugin_versions[name])
-	{
-		sinsp_plugin::version req_version(rversion);
-		if (!plugin_version.check(req_version))
-		{
-			required_version = rversion;
-			return false;
-		}
-	}
-
-	return true;
+	return m_rule_loader.is_plugin_compatible(name, version, required_version);
 }
 
 void falco_engine::clear_filters()
 {
-	m_rulesets.clear();
-
-	for(auto &it : m_filter_factories)
+	for(auto &it : m_rulesets)
 	{
-		std::shared_ptr<falco_ruleset> ruleset(new falco_ruleset());
-		m_rulesets[it.first] = ruleset;
+		it.ruleset.reset(new falco_ruleset);
 	}
+}
 
-	m_required_plugin_versions.clear();
+void falco_engine::clear_loader()
+{
+	m_rule_loader.clear();
 }
 
 void falco_engine::set_sampling_ratio(uint32_t sampling_ratio)
@@ -507,3 +481,17 @@ inline bool falco_engine::should_drop_evt()
 	double coin = (random() * (1.0/RAND_MAX));
 	return (coin >= (1.0/(m_sampling_multiplier * m_sampling_ratio)));
 }
+
+inline std::vector<falco_engine::ruleset_node>::iterator falco_engine::find_ruleset(const std::string &source)
+{
+	return std::find_if(
+		m_rulesets.begin(), m_rulesets.end(),
+		[&source](const ruleset_node &r) { return r.source == source; });
+}
+
+inline std::vector<falco_engine::ruleset_node>::const_iterator falco_engine::find_ruleset(const std::string &source) const
+{
+	return std::find_if(
+		m_rulesets.cbegin(), m_rulesets.cend(),
+		[&source](const ruleset_node &r) { return r.source == source; });
+}

userspace/engine/falco_engine.h

@@ -29,9 +29,9 @@ limitations under the License.
 #include <nlohmann/json.hpp>
 
 #include "gen_filter.h"
-#include "rules.h"
 #include "ruleset.h"
-
+#include "rule_loader.h"
+#include "stats_manager.h"
 #include "falco_common.h"
 
 //
@@ -40,7 +40,7 @@ limitations under the License.
 // handled in a separate class falco_outputs.
 //
 
-class falco_engine : public falco_common
+class falco_engine
 {
 public:
 	falco_engine(bool seed_rng=true);
@@ -118,6 +118,15 @@ public:
 	// Clear all existing filters.
 	void clear_filters();
 
+	//
+	// Clear all the definitions of the internal rule loader (e.g. defined
+	// rules, macros, lists, engine/plugin version requirements). This is meant
+	// to be used to free-up memory at runtime when the definitions are not
+	// used anymore. Calling this between successive invocations of load_rules
+	// or load_rules_file can cause failures of features like appending.
+	//
+	void clear_loader();
+
 	//
 	// Set the sampling ratio, which can affect which events are
 	// matched against the set of rules.
@@ -163,20 +172,26 @@ public:
 	// with a ruleset string.
 	//
 	// the returned rule_result is allocated and must be delete()d.
-	std::unique_ptr<rule_result> process_event(std::string &source, gen_event *ev, uint16_t ruleset_id);
+	std::unique_ptr<rule_result> process_event(std::size_t source_idx, gen_event *ev, uint16_t ruleset_id);
 
 	//
 	// Wrapper assuming the default ruleset
 	//
-	std::unique_ptr<rule_result> process_event(std::string &source, gen_event *ev);
+	std::unique_ptr<rule_result> process_event(std::size_t source_idx, gen_event *ev);
 
 	//
 	// Configure the engine to support events with the provided
 	// source, with the provided filter factory and formatter factory.
+	// Return source index for fast lookup.
 	//
-	void add_source(const std::string &source,
-			std::shared_ptr<gen_event_filter_factory> filter_factory,
-			std::shared_ptr<gen_event_formatter_factory> formatter_factory);
+	std::size_t add_source(const std::string &source,
+			       std::shared_ptr<gen_event_filter_factory> filter_factory,
+			       std::shared_ptr<gen_event_formatter_factory> formatter_factory);
+
+	// todo(jasondellaluce): this is here for internal use, and
+	// will possibly be removed in the future
+	std::shared_ptr<gen_event_filter_factory> get_filter_factory(
+		const std::string &source);
 
 	// Return whether or not there is a valid filter/formatter
 	// factory for this source.
@@ -188,6 +203,7 @@ public:
 	void add_filter(std::shared_ptr<gen_event_filter> filter,
 			std::string &rule,
 			std::string &source,
+			std::set<uint16_t> &evttypes,
 			std::set<std::string> &tags);
 
 	//
@@ -213,6 +229,14 @@ public:
 	bool is_plugin_compatible(const std::string &name, const std::string &version, std::string &required_version);
 
 private:
+	struct ruleset_node
+	{
+		ruleset_node(const std::string &n, falco_ruleset *p):
+			source(n), ruleset(p) {}
+
+		std::string source;
+		mutable std::shared_ptr<falco_ruleset> ruleset;
+	};
 
 	//
 	// Determine whether the given event should be matched at all
@@ -221,6 +245,9 @@ private:
 	//
 	inline bool should_drop_evt();
 
+	inline std::vector<ruleset_node>::iterator find_ruleset(const std::string &source);
+	inline std::vector<ruleset_node>::const_iterator find_ruleset(const std::string &source) const;
+
 	// Maps from event source to object that can generate filters from rules
 	std::map<std::string, std::shared_ptr<gen_event_filter_factory>> m_filter_factories;
 
@@ -228,18 +255,16 @@ private:
 	std::map<std::string, std::shared_ptr<gen_event_formatter_factory>> m_format_factories;
 
 	// Maps from event source to the set of rules for that event source
-	std::map<std::string, std::shared_ptr<falco_ruleset>> m_rulesets;
+	std::vector<ruleset_node> m_rulesets;
+
+	rule_loader m_rule_loader;
+	indexed_vector<falco_rule> m_rules;
+	stats_manager m_rule_stats_manager;
 
-	std::unique_ptr<falco_rules> m_rules;
 	uint16_t m_next_ruleset_id;
 	std::map<string, uint16_t> m_known_rulesets;
 	falco_common::priority_type m_min_priority;
 
-	// Maps from plugin to a list of required plugin versions
-	// found in any loaded rules files.
-	std::map<std::string, std::list<std::string>> m_required_plugin_versions;
-
-	void populate_rule_result(unique_ptr<struct rule_result> &res, gen_event *ev);
 
 	//
 	// Here's how the sampling ratio and multiplier influence

userspace/engine/falco_engine_version.h

@@ -16,9 +16,9 @@ limitations under the License.
 
 // The version of rules/filter fields/etc supported by this Falco
 // engine.
-#define FALCO_ENGINE_VERSION (11)
+#define FALCO_ENGINE_VERSION (12)
 
 // This is the result of running "falco --list -N | sha256sum" and
 // represents the fields supported by this version of Falco. It's used
 // at build time to detect a changed set of fields.
-#define FALCO_FIELDS_CHECKSUM "4de812495f8529ac20bda2b9774462b15911a51df293d59fe9ccb6b922fdeb9d"
+#define FALCO_FIELDS_CHECKSUM "77c4c549181b8aac1b9698c0101ac61acb5b2faede84a2c4fecb34834c6de2b9"

userspace/engine/falco_rule.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019 The Falco Authors.
+Copyright (C) 2022 The Falco Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,7 +16,17 @@ limitations under the License.
 
 #pragma once
 
-#include "lua.h"
-
-int luaopen_lpeg (lua_State *L);
+#include <set>
+#include <string>
+#include "falco_common.h"
 
+struct falco_rule
+{
+	std::string source;
+	std::string name;
+	std::string description;
+	std::string output;
+	std::set<std::string> tags;
+	std::set<std::string> exception_fields;
+	falco_common::priority_type priority;
+};

userspace/engine/falco_utils.cpp

@@ -17,6 +17,7 @@ limitations under the License.
 
 */
 #include <cstring>
+#include <iomanip>
 
 #include "falco_utils.h"
 #include "banned.h" // This raises a compilation error when certain functions are used
@@ -27,6 +28,29 @@ namespace falco
 namespace utils
 {
 
+std::string wrap_text(const std::string& in, uint32_t indent, uint32_t line_len)
+{
+	std::istringstream is(in);
+	std::ostringstream os;
+	std::string word;
+	uint32_t len = 0;
+	while (is >> word)
+	{
+		if((len + word.length() + 1) <= (line_len-indent))
+		{
+			len += word.length() + 1;
+		}
+		else
+		{
+			os << std::endl;
+			os << std::left << std::setw(indent) << " ";
+			len = word.length() + 1;
+		}
+		os << word << " ";
+	}
+	return os.str();
+}
+
 uint32_t hardware_concurrency()
 {
 	auto hc = std::thread::hardware_concurrency();

userspace/engine/falco_utils.h

@@ -40,7 +40,7 @@ namespace falco
 namespace utils
 {
 
-std::string wrap_text(const std::string& str, uint32_t initial_pos, uint32_t indent, uint32_t line_len);
+std::string wrap_text(const std::string& in, uint32_t indent, uint32_t linelen);
 
 void readfile(const std::string& filename, std::string& data);
 

userspace/engine/filter_evttype_resolver.cpp

@@ -0,0 +1,164 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "filter_evttype_resolver.h"
+#include <sinsp.h>
+
+using namespace std;
+using namespace libsinsp::filter;
+
+extern sinsp_evttables g_infotables;
+
+static bool is_evttype_operator(const string& op)
+{
+	return op == "==" || op == "=" || op == "!=" || op == "in";
+}
+
+void filter_evttype_resolver::inversion(set<uint16_t>& types)
+{
+	set<uint16_t> all_types;
+	evttypes("", all_types);
+	if (types != all_types) // we don't invert the "all types" set
+	{
+		set<uint16_t> diff = types;
+		types.clear();
+		set_difference(
+			all_types.begin(), all_types.end(), diff.begin(), diff.end(),
+			inserter(types, types.begin()));
+	}
+}
+
+void filter_evttype_resolver::evttypes(string evtname, set<uint16_t>& out)
+{
+	// Fill in from 2 to PPM_EVENT_MAX-1. 0 and 1 are excluded as
+	// those are PPM_GENERIC_E/PPME_GENERIC_X
+	const struct ppm_event_info* etable = g_infotables.m_event_info;
+	for(uint16_t i = 2; i < PPM_EVENT_MAX; i++)
+	{
+		// Skip "old" event versions, unused events, or events not matching
+		// the requested evtname
+		if(!(etable[i].flags & (EF_OLD_VERSION | EF_UNUSED))
+			&& (evtname.empty() || string(etable[i].name) == evtname))
+		{
+			out.insert(i);
+		}
+	}
+}
+
+void filter_evttype_resolver::evttypes(ast::expr* filter, set<uint16_t>& out)
+{
+	m_expect_value = false;
+	m_last_node_evttypes.clear();
+	filter->accept(this);
+	out.insert(m_last_node_evttypes.begin(), m_last_node_evttypes.end());
+}
+
+void filter_evttype_resolver::evttypes(
+	shared_ptr<ast::expr> filter, set<uint16_t>& out)
+{
+	m_expect_value = false;
+	m_last_node_evttypes.clear();
+	filter.get()->accept(this);
+	out.insert(m_last_node_evttypes.begin(), m_last_node_evttypes.end());
+}
+
+// "and" nodes evttypes are the intersection of the evttypes of their children.
+// we initialize the set with "all event types"
+void filter_evttype_resolver::visit(ast::and_expr* e)
+{
+	set<uint16_t> types, inters;
+	evttypes("", types);
+	m_last_node_evttypes.clear();
+	for (auto &c : e->children)
+	{
+		inters.clear();
+		c->accept(this);
+		set_intersection(
+			types.begin(), types.end(),
+			m_last_node_evttypes.begin(), m_last_node_evttypes.end(),
+			inserter(inters, inters.begin()));
+		types = inters;
+	}
+	m_last_node_evttypes = types;
+}
+
+// "or" nodes evttypes are the union of the evttypes their children
+void filter_evttype_resolver::visit(ast::or_expr* e)
+{
+	set<uint16_t> types;
+	m_last_node_evttypes.clear();
+	for (auto &c : e->children)
+	{
+		c->accept(this);
+		types.insert(m_last_node_evttypes.begin(), m_last_node_evttypes.end());
+	}
+	m_last_node_evttypes = types;
+}
+
+void filter_evttype_resolver::visit(ast::not_expr* e)
+{
+	m_last_node_evttypes.clear();
+	e->child->accept(this);
+	inversion(m_last_node_evttypes);
+}
+
+void filter_evttype_resolver::visit(ast::binary_check_expr* e)
+{
+	m_last_node_evttypes.clear();
+	if (e->field == "evt.type" && is_evttype_operator(e->op))
+	{
+		m_expect_value = true;
+		e->value->accept(this);
+		m_expect_value = false;
+		if (e->op == "!=")
+		{
+			inversion(m_last_node_evttypes);
+		}
+		return;
+	}
+	evttypes("", m_last_node_evttypes);
+}
+
+void filter_evttype_resolver::visit(ast::unary_check_expr* e)
+{
+	m_last_node_evttypes.clear();
+	evttypes("", m_last_node_evttypes);
+}
+
+void filter_evttype_resolver::visit(ast::value_expr* e)
+{
+	m_last_node_evttypes.clear();
+	if (m_expect_value)
+	{
+		evttypes(e->value, m_last_node_evttypes);
+		return;
+	}
+	evttypes("", m_last_node_evttypes);
+}
+
+void filter_evttype_resolver::visit(ast::list_expr* e)
+{
+	m_last_node_evttypes.clear();
+	if (m_expect_value)
+	{
+		for (auto &v : e->values)
+		{
+			evttypes(v, m_last_node_evttypes);
+		}	
+		return;
+	}
+	evttypes("", m_last_node_evttypes);
+}

userspace/engine/filter_evttype_resolver.h

@@ -0,0 +1,69 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+#include <filter/parser.h>
+#include <string>
+#include <set>
+#include <memory>
+
+/*!
+	\brief Helper class for finding event types
+*/
+class filter_evttype_resolver: private libsinsp::filter::ast::expr_visitor
+{
+public:
+	/*!
+		\brief Collects the evttypes related to the provided event name.
+		The event types are inserted in the set provided as parameter.
+		The set is not cleared before inserting the elements.
+		\param evtname The event name used to search event types. If an empty
+		string is passed, all the available evttypes are collected
+		\param out The set to be filled with the evttypes
+	*/
+	void evttypes(std::string evtname, std::set<uint16_t>& out);
+
+	/*!
+		\brief Visits a filter AST and collects all the evttypes for which
+		the filter expression can be evaluated as true. The event types are
+		inserted in the set provided as parameter. The set is not cleared before
+		inserting the elements.
+		\param filter The filter AST to be explored
+		\param out The set to be filled with the evttypes
+	*/
+	void evttypes(libsinsp::filter::ast::expr* filter, std::set<uint16_t>& out);
+
+	/*!
+		\brief Overloaded version of evttypes() that supports filters wrapped
+		in shared pointers
+	*/
+	void evttypes(std::shared_ptr<libsinsp::filter::ast::expr> filter,
+		std::set<uint16_t>& out);
+
+private:
+	void visit(libsinsp::filter::ast::and_expr* e) override;
+	void visit(libsinsp::filter::ast::or_expr* e) override;
+	void visit(libsinsp::filter::ast::not_expr* e) override;
+	void visit(libsinsp::filter::ast::value_expr* e) override;
+	void visit(libsinsp::filter::ast::list_expr* e) override;
+	void visit(libsinsp::filter::ast::unary_check_expr* e) override;
+	void visit(libsinsp::filter::ast::binary_check_expr* e) override;
+	void inversion(std::set<uint16_t>& types);
+
+	bool m_expect_value;
+	std::set<uint16_t> m_last_node_evttypes;
+};

userspace/engine/filter_macro_resolver.cpp

@@ -0,0 +1,149 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "filter_macro_resolver.h"
+
+using namespace std;
+using namespace libsinsp::filter;
+
+bool filter_macro_resolver::run(libsinsp::filter::ast::expr*& filter)
+{
+	m_unknown_macros.clear();
+	m_resolved_macros.clear();
+	m_last_node_changed = false;
+	m_last_node = filter;
+	filter->accept(this);
+	if (m_last_node_changed)
+	{
+		delete filter;
+		filter = m_last_node;
+	}
+	return !m_resolved_macros.empty();
+}
+
+bool filter_macro_resolver::run(std::shared_ptr<libsinsp::filter::ast::expr>& filter)
+{
+	m_unknown_macros.clear();
+	m_resolved_macros.clear();
+	m_last_node_changed = false;
+	m_last_node = filter.get();
+	filter->accept(this);
+	if (m_last_node_changed)
+	{
+		filter.reset(m_last_node);
+	}
+	return !m_resolved_macros.empty();
+}
+
+void filter_macro_resolver::set_macro(
+		string name,
+		shared_ptr<libsinsp::filter::ast::expr> macro)
+{
+	m_macros[name] = macro;
+}
+
+set<string>& filter_macro_resolver::get_unknown_macros()
+{
+	return m_unknown_macros;
+}
+
+set<string>& filter_macro_resolver::get_resolved_macros()
+{
+	return m_resolved_macros;
+}
+
+void filter_macro_resolver::visit(ast::and_expr* e)
+{
+	for (size_t i = 0; i < e->children.size(); i++)
+	{
+		e->children[i]->accept(this);
+		if (m_last_node_changed)
+		{
+			delete e->children[i];
+			e->children[i] = m_last_node;
+		}
+	}
+	m_last_node = e;
+	m_last_node_changed = false;
+}
+
+void filter_macro_resolver::visit(ast::or_expr* e)
+{
+	for (size_t i = 0; i < e->children.size(); i++)
+	{
+		e->children[i]->accept(this);
+		if (m_last_node_changed)
+		{
+			delete e->children[i];
+			e->children[i] = m_last_node;
+		}
+	}
+	m_last_node = e;
+	m_last_node_changed = false;
+}
+
+void filter_macro_resolver::visit(ast::not_expr* e)
+{
+	e->child->accept(this);
+	if (m_last_node_changed)
+	{
+		delete e->child;
+		e->child = m_last_node;
+	}
+	m_last_node = e;
+	m_last_node_changed = false;
+}
+
+void filter_macro_resolver::visit(ast::list_expr* e)
+{
+	m_last_node = e;
+	m_last_node_changed = false;
+}
+
+void filter_macro_resolver::visit(ast::binary_check_expr* e)
+{
+	// avoid exploring checks, so that we can be sure that each
+	// value_expr* node visited is a macro identifier
+	m_last_node = e;
+	m_last_node_changed = false;
+}
+
+void filter_macro_resolver::visit(ast::unary_check_expr* e)
+{
+	m_last_node = e;
+	m_last_node_changed = false;
+}
+
+void filter_macro_resolver::visit(ast::value_expr* e)
+{
+	// we are supposed to get here only in case
+	// of identier-only children from either a 'not',
+	// an 'and' or an 'or'.
+	auto macro = m_macros.find(e->value);
+	if (macro != m_macros.end() && macro->second) // skip null-ptr macros
+	{
+		ast::expr* new_node = ast::clone(macro->second.get());
+		new_node->accept(this); // this sets m_last_node
+		m_last_node_changed = true;
+		m_resolved_macros.insert(e->value);
+	}
+	else
+	{
+		m_last_node = e;
+		m_last_node_changed = false;
+		m_unknown_macros.insert(e->value);
+	}
+}

userspace/engine/filter_macro_resolver.h

@@ -0,0 +1,93 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+#include <filter/parser.h>
+#include <string>
+#include <set>
+#include <map>
+#include <memory>
+
+/*!
+	\brief Helper class for substituting and resolving macro
+	references in parsed filters.
+*/
+class filter_macro_resolver: private libsinsp::filter::ast::expr_visitor
+{
+	public:
+		/*!
+			\brief Visits a filter AST and substitutes macro references
+			according with all the definitions added through set_macro(),
+			by replacing the reference with a clone of the macro AST.
+			\param filter The filter AST to be processed. Note that the pointer
+			is passed by reference and be modified in order to apply
+			the substutions. In that case, the old pointer is owned by this
+			class and is deleted automatically.
+			\return true if at least one of the defined macros is resolved
+		*/
+		bool run(libsinsp::filter::ast::expr*& filter);
+		
+		/*!
+			\brief Version of run() that works with shared pointers
+		*/
+		bool run(std::shared_ptr<libsinsp::filter::ast::expr>& filter);
+
+		/*!
+			\brief Defines a new macro to be substituted in filters. If called
+			multiple times for the same macro name, the previous definition
+			gets overridden. A macro can be undefined by setting a null
+			AST pointer.
+			\param name The name of the macro.
+			\param macro The AST of the macro.
+		*/
+		void set_macro(
+			std::string name,
+			std::shared_ptr<libsinsp::filter::ast::expr> macro);
+
+		/*!
+			\brief Returns a set containing the names of all the macros
+			substituted during the last invocation of run(). Should be
+			non-empty if the last invocation of run() returned true.
+		*/
+		std::set<std::string>& get_resolved_macros();
+
+		/*!
+			\brief Returns a set containing the names of all the macros
+			that remained unresolved during the last invocation of run().
+			A macro remains unresolved if it is found inside the processed
+			filter but it was not defined with set_macro();
+		*/
+		std::set<std::string>& get_unknown_macros();
+		
+	private:
+		void visit(libsinsp::filter::ast::and_expr* e) override;
+		void visit(libsinsp::filter::ast::or_expr* e) override;
+		void visit(libsinsp::filter::ast::not_expr* e) override;
+		void visit(libsinsp::filter::ast::value_expr* e) override;
+		void visit(libsinsp::filter::ast::list_expr* e) override;
+		void visit(libsinsp::filter::ast::unary_check_expr* e) override;
+		void visit(libsinsp::filter::ast::binary_check_expr* e) override;
+
+		bool m_last_node_changed;
+		libsinsp::filter::ast::expr* m_last_node;
+		std::set<std::string> m_unknown_macros;
+		std::set<std::string> m_resolved_macros;
+		std::map<
+			std::string,
+			std::shared_ptr<libsinsp::filter::ast::expr>
+		> m_macros;
+};

userspace/engine/formats.h

@@ -18,16 +18,7 @@ limitations under the License.
 
 #include <string>
 #include <map>
-
-extern "C"
-{
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
-}
-
 #include <gen_filter.h>
-
 #include "falco_engine.h"
 
 class falco_formats

userspace/engine/indexed_vector.h

@@ -0,0 +1,133 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+
+/*!
+	\brief Simple wrapper of std::vector that allows random access
+	through both numeric and string indexes with O(1) complexity
+*/
+template <typename T>
+class indexed_vector
+{
+public:
+	/*!
+		\brief Returns the number of elements
+	*/
+	virtual inline size_t size()
+	{
+		return m_entries.size();
+	}
+
+	/*!
+		\brief Returns true if the vector is empty
+	*/
+	virtual inline bool empty()
+	{
+		return m_entries.empty();
+	}
+
+	/*!
+		\brief Removes all the elements
+	*/
+	virtual inline void clear()
+	{
+		m_entries.clear();
+		m_index.clear();
+	}
+
+	/*!
+		\brief Inserts a new element in the vector with a given string index
+		and returns its numeric index. String indexes are unique in
+		the vector. If no element is already present with the given string
+		index, then the provided element is added to the vector and its
+		numeric index is assigned as the next free slot in the vector.
+		Otherwise, the existing element gets overwritten with the contents
+		of the provided one and the numeric index of the existing element
+		is returned.
+		\param entry Element to add in the vector
+		\param index String index of the element to be added in the vector
+		\return The numeric index assigned to the element
+	*/
+	virtual inline size_t insert(T& entry, const std::string& index)
+	{
+		size_t id;
+		auto prev = m_index.find(index);
+		if (prev != m_index.end()) {
+			id = prev->second;
+			m_entries[id] = entry;
+			return id;
+		}
+		id = m_entries.size();
+		m_entries.push_back(entry);
+		m_index[index] = id;
+		return id;
+	}
+
+	/*!
+		\brief Returns a pointer to the element at the given numeric index,
+		or nullptr if no element exists at the given index.
+	*/
+	virtual inline T* at(size_t id) const
+	{
+		if (id <= m_entries.size())
+		{
+			return (T* const) &m_entries[id];
+		}
+		return nullptr;
+	}
+
+	/*!
+		\brief Returns a pointer to the element at the given string index,
+		or nullptr if no element exists at the given index.
+	*/
+	virtual inline T* at(const std::string& index) const
+	{
+		auto it = m_index.find(index);
+		if (it != m_index.end()) {
+			return at(it->second);
+		}
+		return nullptr;
+	}
+
+	virtual inline typename std::vector<T>::iterator begin()
+	{
+		return m_entries.begin();
+	}
+
+	virtual inline typename std::vector<T>::iterator end()
+	{
+		return m_entries.end();
+	}
+
+	virtual inline typename std::vector<T>::const_iterator begin() const 
+	{
+		return m_entries.begin();
+	}
+
+	virtual inline typename std::vector<T>::const_iterator end() const
+	{
+		return m_entries.end();
+	}
+
+private:
+	std::vector<T> m_entries;
+	std::map<std::string, size_t> m_index;
+};

userspace/engine/json_evt.cpp

@@ -591,13 +591,15 @@ const json_event_filter_check::values_t &json_event_filter_check::extracted_valu
 
 bool json_event_filter_check::compare(gen_event *evt)
 {
-	auto jevt = (json_event *)evt;
+	auto jevt = (json_event *) evt;
+	std::vector<extract_value_t> values;
+	if (!extract(jevt, values))
+	{
+		return false;
+	}
+	auto evalues = (const extracted_values_t *) values[0].ptr;
 
-	uint32_t len;
-
-	auto evalues = (const extracted_values_t *) extract(jevt, &len);
 	values_set_t setvals;
-
 	switch(m_cmpop)
 	{
 	case CO_EQ:
@@ -712,7 +714,7 @@ void json_event_filter_check::add_extracted_value_num(int64_t val)
 	m_evalues.second.emplace(json_event_value(val));
 }
 
-uint8_t *json_event_filter_check::extract(gen_event *evt, uint32_t *len, bool sanitize_strings)
+bool json_event_filter_check::extract(gen_event *evt, std::vector<extract_value_t>& values, bool sanitize_strings)
 {
 	m_evalues.first.clear();
 	m_evalues.second.clear();
@@ -723,9 +725,8 @@ uint8_t *json_event_filter_check::extract(gen_event *evt, uint32_t *len, bool sa
 		m_evalues.second.clear();
 		add_extracted_value(no_value);
 	}
-
-	*len = sizeof(m_evalues);
-	return (uint8_t *)&m_evalues;
+	values.push_back({(uint8_t *)&m_evalues, sizeof(m_evalues)});
+	return true;
 }
 
 bool json_event_filter_check::extract_values(json_event *jevt)
@@ -1417,7 +1418,7 @@ json_event_filter_check *k8s_audit_filter_check::allocate_new()
 	return (json_event_filter_check *)chk;
 }
 
-json_event_filter::json_event_filter()
+json_event_filter::json_event_filter(): sinsp_filter(NULL)
 {
 }
 
@@ -1659,13 +1660,13 @@ void json_event_formatter::parse_format()
 
 void json_event_formatter::resolve_format(json_event *ev, std::list<std::pair<std::string, std::string>> &resolved)
 {
+	vector<extract_value_t> values;
 	for(auto tok : m_tokens)
 	{
 		if(tok.check)
 		{
-			uint32_t len;
-
-			(void) tok.check->extract(ev, &len);
+			values.clear();
+			tok.check->extract(ev, values);
 
 			const json_event_filter_check::values_t &evals =
 				tok.check->extracted_values();

userspace/engine/json_evt.h

@@ -27,7 +27,7 @@ limitations under the License.
 #include <nlohmann/json.hpp>
 
 #include "prefix_search.h"
-#include "gen_filter.h"
+#include <sinsp.h>
 
 class json_event : public gen_event
 {
@@ -179,8 +179,16 @@ public:
 	void add_filter_value(const char *str, uint32_t len, uint32_t i = 0);
 	bool compare(gen_event *evt);
 
-	// This always returns a const extracted_values_t *. The pointer points to m_evalues;
-	uint8_t* extract(gen_event *evt, uint32_t* len, bool sanitize_strings = true) final;
+	// This is adapted to support the new extract() method signature that
+	// supports extracting list of values, however json_evt was implemented
+	// to support this feature in the first place through the
+	// extracted_values_t structure. As such, for now this is only used for
+	// signature compliance, and always pushes a single value. The value pushed
+	// in the vector is a a const extracted_values_t* that points to the
+	// internal m_evalues. This is a temporary workaround to sync with the
+	// latest falcosecurity/libs development without re-designing the whole K8S
+	// support, which will eventually be refactored as a plugin in the future anyway.
+	bool extract(gen_event *evt, std::vector<extract_value_t>& values, bool sanitize_strings = true) final;
 
 	const std::string &field();
 	const std::string &idx();
@@ -375,7 +383,8 @@ public:
 					   json_event_filter_check &jchk);
 };
 
-class json_event_filter : public gen_event_filter
+
+class json_event_filter : public sinsp_filter
 {
 public:
 	json_event_filter();

userspace/engine/lua/CMakeLists.txt

@@ -1,21 +0,0 @@
-#
-# Copyright (C) 2021 The Falco Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-
-file(GLOB_RECURSE lua_files ${CMAKE_CURRENT_SOURCE_DIR} *.lua)
-
-add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/falco_engine_lua_files.cpp
-	COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/lua-to-cpp.sh ${CMAKE_CURRENT_SOURCE_DIR} ${LYAML_LUA_DIR} ${CMAKE_CURRENT_BINARY_DIR}
-	DEPENDS ${lua_files} ${CMAKE_CURRENT_SOURCE_DIR}/lua-to-cpp.sh lyaml)
-
-add_library(luafiles falco_engine_lua_files.cpp)
-
-target_include_directories(luafiles PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

userspace/engine/lua/README.md

@@ -1,6 +0,0 @@
-Installation
-------------
-
-The grammar uses the `lpeg` parser. For now install it using luarocks:
-`luarocks install lpeg`.
-

userspace/engine/lua/lua-to-cpp.sh

@@ -1,84 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-LUA_FILE_DIR=$1
-LYAML_LUA_DIR=$2
-OUTPUT_DIR=$3
-
-MODULE_SYMS=()
-CODE_SYMS=()
-
-function add_lua_file {
-    filename=$1
-    is_module=$2
-
-    # Take the basename of the file
-    BASE_NAME=$(basename ${file} .lua)
-    SYMBOL_NAME="${BASE_NAME}_lua_file_contents"
-    FILE_CONTENTS=$(<${file})
-
-    # Add a symbol to the .cc file containing the contents of the file
-    echo "const char *${SYMBOL_NAME}=R\"LUAFILE(${FILE_CONTENTS})LUAFILE\";" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp
-
-    # Add an extern reference to the .hh file
-    echo "extern const char *${SYMBOL_NAME};" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh
-
-    if [[ "${is_module}" == "true" ]]; then
-	# Determine the module name for the file
-	if [[ "${file}" == *"/"* ]]; then
-	    MODULE_NAME=$(echo ${file} | tr / . | sed -e 's/.lua//')
-	else
-	    MODULE_NAME=$(basename ${file} .lua)
-	fi
-
-	# Add the pair (string contents, module name) to MODULE_SYMS
-	PAIR=$(echo "{${SYMBOL_NAME},\"${MODULE_NAME}\"}")
-	MODULE_SYMS+=(${PAIR})
-    else
-	# Add the string to CODE_SYMS
-	CODE_SYMS+=(${SYMBOL_NAME})
-    fi
-}
-
-cat <<EOF > ${OUTPUT_DIR}/falco_engine_lua_files.cpp
-// Automatically generated. Do not edit
-#include "falco_engine_lua_files.hh"
-EOF
-
-cat <<EOF > ${OUTPUT_DIR}/falco_engine_lua_files.hh
-#pragma once
-// Automatically generated. Do not edit
-#include <list>
-#include <utility>
-EOF
-
-# lyaml and any files in the "modules" subdirectory are treated as lua
-# modules.
-pushd ${LYAML_LUA_DIR}
-for file in *.lua */*.lua; do
-    add_lua_file $file "true"
-done
-popd
-
-pushd ${LUA_FILE_DIR}/modules
-for file in *.lua; do
-    add_lua_file $file "true"
-done
-popd
-
-# Any .lua files in this directory are treated as code with functions
-# to execute.
-pushd ${LUA_FILE_DIR}
-for file in ${LUA_FILE_DIR}/*.lua; do
-    add_lua_file $file "false"
-done
-popd
-
-# Create a list of lua module (string, module name) pairs from MODULE_SYMS
-echo "extern std::list<std::pair<const char *,const char *>> lua_module_strings;" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh
-echo "std::list<std::pair<const char *,const char *>> lua_module_strings = {$(IFS=, ; echo "${MODULE_SYMS[*]}")};" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp
-
-# Create a list of lua code strings from CODE_SYMS
-echo "extern std::list<const char *> lua_code_strings;" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh
-echo "std::list<const char *> lua_code_strings = {$(IFS=, ; echo "${CODE_SYMS[*]}")};" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp

userspace/engine/lua/modules/compiler.lua

@@ -1,235 +0,0 @@
--- Copyright (C) 2019 The Falco Authors.
---
--- Licensed under the Apache License, Version 2.0 (the "License");
--- you may not use this file except in compliance with the License.
--- You may obtain a copy of the License at
---
---     http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
-
-local parser = require("parser")
-local compiler = {}
-
-compiler.trim = parser.trim
-
-function map(f, arr)
-   local res = {}
-   for i,v in ipairs(arr) do
-      res[i] = f(v)
-   end
-   return res
-end
-
-function foldr(f, acc, arr)
-   for i,v in pairs(arr) do
-      acc = f(acc, v)
-   end
-   return acc
-end
-
---[[
-
-   Given a map of macro definitions, traverse AST and replace macro references
-   with their definitions.
-
-   The AST is changed in-place.
-
-   The return value is a boolean which is true if any macro was
-   substituted. This allows a caller to re-traverse until no more macros are
-   found, a simple strategy for recursive resolutions (e.g. when a macro
-   definition uses another macro).
-
---]]
-
-function copy_ast_obj(obj)
-   if type(obj) ~= 'table' then return obj end
-   local res = {}
-   for k, v in pairs(obj) do res[copy_ast_obj(k)] = copy_ast_obj(v) end
-   return res
-end
-
-function expand_macros(ast, defs, changed)
-
-   if (ast.type == "Rule") then
-      return expand_macros(ast.filter, defs, changed)
-   elseif ast.type == "Filter" then
-      if (ast.value.type == "Macro") then
-         if (defs[ast.value.value] == nil) then
-	    return false, "Undefined macro '".. ast.value.value .. "' used in filter."
-         end
-	 defs[ast.value.value].used = true
-         ast.value = copy_ast_obj(defs[ast.value.value].ast)
-         changed = true
-	 return true, changed
-      end
-      return expand_macros(ast.value, defs, changed)
-
-   elseif ast.type == "BinaryBoolOp" then
-
-      if (ast.left.type == "Macro") then
-         if (defs[ast.left.value] == nil) then
-	    return false, "Undefined macro '".. ast.left.value .. "' used in filter."
-         end
-	 defs[ast.left.value].used = true
-         ast.left = copy_ast_obj(defs[ast.left.value].ast)
-         changed = true
-      end
-
-      if (ast.right.type == "Macro") then
-         if (defs[ast.right.value] == nil) then
-	    return false, "Undefined macro ".. ast.right.value .. " used in filter."
-         end
-	 defs[ast.right.value].used = true
-         ast.right = copy_ast_obj(defs[ast.right.value].ast)
-         changed = true
-      end
-
-      local status, changed_left = expand_macros(ast.left, defs, false)
-      if status == false then
-	 return false, changed_left
-      end
-      local status, changed_right = expand_macros(ast.right, defs, false)
-      if status == false then
-	 return false, changed_right
-      end
-      return true, changed or changed_left or changed_right
-
-   elseif ast.type == "UnaryBoolOp" then
-      if (ast.argument.type == "Macro") then
-         if (defs[ast.argument.value] == nil) then
-	    return false, "Undefined macro ".. ast.argument.value .. " used in filter."
-         end
-	 defs[ast.argument.value].used = true
-         ast.argument = copy_ast_obj(defs[ast.argument.value].ast)
-         changed = true
-      end
-      return expand_macros(ast.argument, defs, changed)
-   end
-   return true, changed
-end
-
-function get_filters(ast)
-
-   local filters = {}
-
-   function cb(node)
-      if node.type == "FieldName" then
-	 filters[node.value] = 1
-      end
-   end
-
-   parser.traverse_ast(ast.filter.value, {FieldName=1} , cb)
-
-   return filters
-end
-
-function compiler.expand_lists_in(source, list_defs)
-
-   for name, def in pairs(list_defs) do
-
-      local bpos = string.find(source, name, 1, true)
-
-      while bpos ~= nil do
-	 def.used = true
-
-	 local epos = bpos + string.len(name)
-
-	 -- The characters surrounding the name must be delimiters of beginning/end of string
-	 if (bpos == 1 or string.match(string.sub(source, bpos-1, bpos-1), "[%s(),=]")) and (epos > string.len(source) or string.match(string.sub(source, epos, epos), "[%s(),=]")) then
-	    new_source = ""
-
-	    if bpos > 1 then
-	       new_source = new_source..string.sub(source, 1, bpos-1)
-	    end
-
-	    sub = table.concat(def.items, ", ")
-
-	    new_source = new_source..sub
-
-	    if epos <= string.len(source) then
-	       new_source = new_source..string.sub(source, epos, string.len(source))
-	    end
-
-	    source = new_source
-	    bpos = bpos + (string.len(sub)-string.len(name))
-	 end
-
-	 bpos = string.find(source, name, bpos+1, true)
-      end
-   end
-
-   return source
-end
-
-function compiler.compile_macro(line, macro_defs, list_defs)
-
-   line = compiler.expand_lists_in(line, list_defs)
-
-   local ast, error_msg = parser.parse_filter(line)
-
-   if (error_msg) then
-      msg = "Compilation error when compiling \""..line.."\": ".. error_msg
-      return false, msg
-   end
-
-   -- Simply as a validation step, try to expand all macros in this
-   -- macro's condition. This changes the ast, so we make a copy
-   -- first.
-   local ast_copy = copy_ast_obj(ast)
-
-   if (ast.type == "Rule") then
-      -- Line is a filter, so expand macro references
-      repeat
-	 status, expanded = expand_macros(ast_copy, macro_defs, false)
-	 if status == false then
-	    msg = "Compilation error when compiling \""..line.."\": ".. expanded
-	    return false, msg
-	 end
-      until expanded == false
-
-   else
-      return false, "Unexpected top-level AST type: "..ast.type
-   end
-
-   return true, ast
-end
-
---[[
-   Parses a single filter, then expands macros using passed-in table of definitions. Returns resulting AST.
---]]
-function compiler.compile_filter(name, source, macro_defs, list_defs)
-
-   source = compiler.expand_lists_in(source, list_defs)
-
-   local ast, error_msg = parser.parse_filter(source)
-
-   if (error_msg) then
-      msg = "Compilation error when compiling \""..source.."\": "..error_msg
-      return false, msg
-   end
-
-   if (ast.type == "Rule") then
-      -- Line is a filter, so expand macro references
-      repeat
-	 status, expanded  = expand_macros(ast, macro_defs, false)
-	 if status == false then
-	    return false, expanded
-	 end
-      until expanded == false
-
-   else
-      return false, "Unexpected top-level AST type: "..ast.type
-   end
-
-   filters = get_filters(ast)
-
-   return true, ast, filters
-end
-
-
-return compiler

userspace/engine/lua/modules/parser.lua

@@ -1,307 +0,0 @@
--- Copyright (C) 2019 The Falco Authors.
---
--- Licensed under the Apache License, Version 2.0 (the "License");
--- you may not use this file except in compliance with the License.
--- You may obtain a copy of the License at
---
---     http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
---[[
-   Falco grammar and parser.
-
-   Much of the scaffolding and helpers was derived from Andre Murbach Maidl's Lua parser (https://github.com/andremm/lua-parser).
-
-   While this is based on the falcosecurity-libs filtering syntax (*), the Falco syntax is extended to support "macro" terms, which are just identifiers.
-
-   (*) There is currently one known difference with the syntax implemented in libsinsp: In libsinsp, field names cannot start with 'a', 'o', or 'n'. With this parser they can.
-
---]]
-local parser = {}
-
-local lpeg = require "lpeg"
-
-lpeg.locale(lpeg)
-
-local P, S, V = lpeg.P, lpeg.S, lpeg.V
-local C, Carg, Cb, Cc = lpeg.C, lpeg.Carg, lpeg.Cb, lpeg.Cc
-local Cf, Cg, Cmt, Cp, Ct = lpeg.Cf, lpeg.Cg, lpeg.Cmt, lpeg.Cp, lpeg.Ct
-local alpha, digit, alnum = lpeg.alpha, lpeg.digit, lpeg.alnum
-local xdigit = lpeg.xdigit
-local space = lpeg.space
-
--- error message auxiliary functions
-
--- creates an error message for the input string
-local function syntaxerror(errorinfo, pos, msg)
-   local error_msg = "%s: syntax error, %s"
-   return string.format(error_msg, pos, msg)
-end
-
--- gets the farthest failure position
-local function getffp(s, i, t)
-   return t.ffp or i, t
-end
-
--- gets the table that contains the error information
-local function geterrorinfo()
-   return Cmt(Carg(1), getffp) * (C(V "OneWord") + Cc("EOF")) / function(t, u)
-         t.unexpected = u
-         return t
-      end
-end
-
--- creates an error message using the farthest failure position
-local function errormsg()
-   return geterrorinfo() / function(t)
-         local p = t.ffp or 1
-         local msg = "unexpected '%s', expecting %s"
-         msg = string.format(msg, t.unexpected, t.expected)
-         return nil, syntaxerror(t, p, msg)
-      end
-end
-
--- reports a syntactic error
-local function report_error()
-   return errormsg()
-end
-
---- sets the farthest failure position and the expected tokens
-local function setffp(s, i, t, n)
-   if not t.ffp or i > t.ffp then
-      t.ffp = i
-      t.list = {}
-      t.list[n] = n
-      t.expected = "'" .. n .. "'"
-   elseif i == t.ffp then
-      if not t.list[n] then
-         t.list[n] = n
-         t.expected = "'" .. n .. "', " .. t.expected
-      end
-   end
-   return false
-end
-
-local function updateffp(name)
-   return Cmt(Carg(1) * Cc(name), setffp)
-end
-
--- regular combinators and auxiliary functions
-
-local function token(pat, name)
-   return pat * V "Skip" + updateffp(name) * P(false)
-end
-
-local function symb(str)
-   return token(P(str), str)
-end
-
-local function kw(str)
-   return token(P(str) * -V "idRest", str)
-end
-
-local function list(pat, sep)
-   return Ct(pat ^ -1 * (sep * pat ^ 0) ^ 0) / function(elements)
-         return {type = "List", elements = elements}
-      end
-end
-
---http://lua-users.org/wiki/StringTrim
-function trim(s)
-   if (type(s) ~= "string") then
-      return s
-   end
-   return (s:gsub("^%s*(.-)%s*$", "%1"))
-end
-parser.trim = trim
-
-local function terminal(tag)
-   -- Rather than trim the whitespace in this way, it would be nicer to exclude it from the capture...
-   return token(V(tag), tag) / function(tok)
-         val = tok
-         if tag ~= "String" then
-            val = trim(tok)
-         end
-         return {type = tag, value = val}
-      end
-end
-
-local function unaryboolop(op, e)
-   return {type = "UnaryBoolOp", operator = op, argument = e}
-end
-
-local function unaryrelop(e, op)
-   return {type = "UnaryRelOp", operator = op, argument = e}
-end
-
-local function binaryop(e1, op, e2)
-   if not op then
-      return e1
-   else
-      return {type = "BinaryBoolOp", operator = op, left = e1, right = e2}
-   end
-end
-
-local function bool(pat, sep)
-   return Cf(pat * Cg(sep * pat) ^ 0, binaryop)
-end
-
-local function rel(left, sep, right)
-   return left * sep * right / function(e1, op, e2)
-         return {type = "BinaryRelOp", operator = op, left = e1, right = e2}
-      end
-end
-
--- grammar
-
-local function filter(e)
-   return {type = "Filter", value = e}
-end
-
-local function rule(filter)
-   return {type = "Rule", filter = filter}
-end
-
-local G = {
-   V "Start", -- Entry rule
-   Start = V "Skip" * (V "Comment" + V "Rule" / rule) ^ -1 * -1 + report_error(),
-   -- Grammar
-   Comment = P "#" * P(1) ^ 0,
-   Rule = V "Filter" / filter * ((V "Skip") ^ -1),
-   Filter = V "OrExpression",
-   OrExpression = bool(V "AndExpression", V "OrOp"),
-   AndExpression = bool(V "NotExpression", V "AndOp"),
-   NotExpression = V "UnaryBoolOp" * V "NotExpression" / unaryboolop + V "ExistsExpression",
-   ExistsExpression = terminal "FieldName" * V "ExistsOp" / unaryrelop + V "MacroExpression",
-   MacroExpression = terminal "Macro" + V "RelationalExpression",
-   RelationalExpression = rel(terminal "FieldName", V "RelOp", V "Value") +
-      rel(terminal "FieldName", V "SetOp", V "InList") +
-      V "PrimaryExp",
-   PrimaryExp = symb("(") * V "Filter" * symb(")"),
-   FuncArgs = symb("(") * list(V "Value", symb(",")) * symb(")"),
-   -- Terminals
-   Value = terminal "Number" + terminal "String" + terminal "BareString",
-   InList = symb("(") * list(V "Value", symb(",")) * symb(")"),
-   -- Lexemes
-   Space = space ^ 1,
-   Skip = (V "Space") ^ 0,
-   idStart = alpha + P("_"),
-   idRest = alnum + P("_"),
-   Identifier = V "idStart" * V "idRest" ^ 0,
-   Macro = V "idStart" * V "idRest" ^ 0 * -P ".",
-   Int = digit ^ 1,
-   ArgString = (alnum + S ",.-_/*?~") ^ 1,
-   PortRangeString = (V "Int" + S ":,") ^ 1,
-   Index = V "PortRangeString" + V "Int" + V "ArgString",
-   FieldName = V "Identifier" * (P "." + V "Identifier") ^ 1 * (P "[" * V "Index" * P "]") ^ -1,
-   Name = C(V "Identifier") * -V "idRest",
-   Hex = (P("0x") + P("0X")) * xdigit ^ 1,
-   Expo = S("eE") * S("+-") ^ -1 * digit ^ 1,
-   Float = (((digit ^ 1 * P(".") * digit ^ 0) + (P(".") * digit ^ 1)) * V "Expo" ^ -1) + (digit ^ 1 * V "Expo"),
-   Number = C(V "Hex" + V "Float" + V "Int") * - V "idStart" / function(n)
-         return tonumber(n)
-      end,
-   String = (P '"' * C(((P "\\" * P(1)) + (P(1) - P '"')) ^ 0) * P '"' +
-      P "'" * C(((P "\\" * P(1)) + (P(1) - P "'")) ^ 0) * P "'"),
-   BareString = C((P(1) - S " (),=") ^ 1),
-   OrOp = kw("or") / "or",
-   AndOp = kw("and") / "and",
-   Colon = kw(":"),
-   RelOp = symb("=") / "=" + symb("==") / "==" + symb("!=") / "!=" + symb("<=") / "<=" + symb(">=") / ">=" +
-      symb("<") / "<" +
-      symb(">") / ">" +
-      symb("contains") / "contains" +
-      symb("icontains") / "icontains" +
-      symb("glob") / "glob" +
-      symb("startswith") / "startswith" +
-      symb("endswith") / "endswith",
-   SetOp = kw("in") / "in" + kw("intersects") / "intersects" + kw("pmatch") / "pmatch",
-   UnaryBoolOp = kw("not") / "not",
-   ExistsOp = kw("exists") / "exists",
-   -- for error reporting
-   OneWord = V "Name" + V "Number" + V "String" + P(1)
-}
-
---[[
-   Parses a single filter and returns the AST.
---]]
-function parser.parse_filter(subject)
-   local errorinfo = {subject = subject}
-   lpeg.setmaxstack(1000)
-   local ast, error_msg = lpeg.match(G, subject, nil, errorinfo)
-   return ast, error_msg
-end
-
-function print_ast(ast, level)
-   local t = ast.type
-   level = level or 0
-   local prefix = string.rep(" ", level * 4)
-   level = level + 1
-
-   if t == "Rule" then
-      print_ast(ast.filter, level)
-   elseif t == "Filter" then
-      print_ast(ast.value, level)
-   elseif t == "BinaryBoolOp" or t == "BinaryRelOp" then
-      print(prefix .. ast.operator)
-      print_ast(ast.left, level)
-      print_ast(ast.right, level)
-   elseif t == "UnaryRelOp" or t == "UnaryBoolOp" then
-      print(prefix .. ast.operator)
-      print_ast(ast.argument, level)
-   elseif t == "List" then
-      for i, v in ipairs(ast.elements) do
-         print_ast(v, level)
-      end
-   elseif t == "FieldName" or t == "Number" or t == "String" or t == "BareString" or t == "Macro" then
-      print(prefix .. t .. " " .. ast.value)
-   elseif t == "MacroDef" then
-      -- don't print for now
-   else
-      error("Unexpected type in print_ast: " .. t)
-   end
-end
-parser.print_ast = print_ast
-
--- Traverse the provided ast and call the provided callback function
--- for any nodes of the specified type. The callback function should
--- have the signature:
---     cb(ast_node, ctx)
--- ctx is optional.
-function traverse_ast(ast, node_types, cb, ctx)
-   local t = ast.type
-
-   if node_types[t] ~= nil then
-      cb(ast, ctx)
-   end
-
-   if t == "Rule" then
-      traverse_ast(ast.filter, node_types, cb, ctx)
-   elseif t == "Filter" then
-      traverse_ast(ast.value, node_types, cb, ctx)
-   elseif t == "BinaryBoolOp" or t == "BinaryRelOp" then
-      traverse_ast(ast.left, node_types, cb, ctx)
-      traverse_ast(ast.right, node_types, cb, ctx)
-   elseif t == "UnaryRelOp" or t == "UnaryBoolOp" then
-      traverse_ast(ast.argument, node_types, cb, ctx)
-   elseif t == "List" then
-      for i, v in ipairs(ast.elements) do
-         traverse_ast(v, node_types, cb, ctx)
-      end
-   elseif t == "MacroDef" then
-      traverse_ast(ast.value, node_types, cb, ctx)
-   elseif t == "FieldName" or t == "Number" or t == "String" or t == "BareString" or t == "Macro" then
-      -- do nothing, no traversal needed
-   else
-      error("Unexpected type in traverse_ast: " .. t)
-   end
-end
-parser.traverse_ast = traverse_ast
-
-return parser

userspace/engine/lua/parser-smoke.sh

@@ -1,85 +0,0 @@
-# Copyright (C) 2019 The Falco Authors.
-#
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-
-function error_exit_good
-{
-    echo "Error: '$1' did not compiler" 1>&2
-    exit 1
-}
-
-function error_exit_bad
-{
-    echo "Error: incorrect filter '$1' compiler ok" 1>&2
-    exit 1
-}
-
-
-function good
-{
-    lua5.1 test.lua "$1" 2> /dev/null || error_exit_good "$1"
-}
-
-function bad
-{
-    lua5.1 test.lua "$1" 2> /dev/null && error_exit_bad "$1"
-}
-
-# Filters
-good "  a"
-good "a and b"
-good "#a and b; a and b"
-good "#a and b; # ; ; a and b"
-good "(a)"
-good "(a and b)"
-good "(a.a exists and b)"
-good "(a.a exists) and (b)"
-good "a.a exists and b"
-good "a.a=1 or b.b=2 and c"
-good "not (a)"
-good "not (not (a))"
-good "not (a.b=1)"
-good "not (a.a exists)"
-good "not a"
-good "a.b = 1 and not a"
-good "not not a"
-good "(not not a)"
-good "not a.b=1"
-good "not a.a exists"
-good "notz and a and b"
-good "a.b = bla"
-good "a.b = 'bla'"
-good "a.b = not"
-good "a.b contains bla"
-good "a.b icontains 'bla'"
-good "a.g in (1, 'a', b)"
-good "a.g in ( 1 ,, , b)"
-good "evt.dir=> and fd.name=*.log"
-good "evt.dir=> and fd.name=/var/log/httpd.log"
-good "a.g in (1, 'a', b.c)"
-good "a.b = a.a"
-
-good "evt.arg[0] contains /bin"
-bad "evt.arg[a] contains /bin"
-bad "evt.arg[] contains /bin"
-
-bad "a.b = b = 1"
-bad "(a.b = 1"
-
-
-echo
-echo "All tests passed."
-exit 0

userspace/engine/rule_loader.cpp

@@ -0,0 +1,777 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "falco_engine.h"
+#include "rule_loader.h"
+#include "filter_macro_resolver.h"
+#include "filter_evttype_resolver.h"
+
+#define MAX_VISIBILITY		((uint32_t) -1)
+#define THROW(cond, err)    { if (cond) { throw falco_exception(err); } }
+
+static string s_container_info_fmt = "%container.info";
+static string s_default_extra_fmt  = "%container.name (id=%container.id)";
+
+using namespace std;
+using namespace libsinsp::filter;
+
+// todo(jasondellaluce): this breaks string escaping in lists and exceptions
+static void quote_item(string& e)
+{
+	if (e.find(" ") != string::npos && e[0] != '"' && e[0] != '\'')
+	{
+		e = '"' + e + '"';
+	}
+}
+
+static void paren_item(string& e)
+{
+	if(e[0] != '(')
+	{
+		e = '(' + e + ')';
+	}
+}
+
+static bool is_field_defined(
+	falco_engine *engine, const string& source, string field)
+{
+	auto factory = engine->get_filter_factory(source);
+	if(factory)
+	{
+		auto *chk = factory->new_filtercheck(field.c_str());
+		if (chk)
+		{
+			delete(chk);
+			return true;
+		}
+	}
+	return false;
+}
+
+// todo(jasondellaluce): add an helper in libsinsp for this
+static bool is_operator_defined(const string& op)
+{
+	static vector<string> ops = {"=", "==", "!=", "<=", ">=", "<", ">",
+		"contains", "icontains", "bcontains", "glob", "bstartswith",
+		"startswith", "endswith", "in", "intersects", "pmatch"};
+	return find(ops.begin(), ops.end(), op) != ops.end();
+}
+
+// todo(jasondellaluce): add an helper in libsinsp for this
+static bool is_operator_for_list(const string& op)
+{
+	return op == "in" || op == "intersects" || op == "pmatch";
+}
+
+static bool is_format_valid(
+	falco_engine* e, const string& src, const string& fmt, string& err)
+{
+	try
+	{
+		shared_ptr<gen_event_formatter> formatter;
+		formatter = e->create_formatter(src, fmt);
+		return true;
+	}
+	catch(exception &e)
+	{
+		err = e.what();
+		return false;
+	}
+}
+
+template <typename T>
+static inline void define_info(indexed_vector<T>& infos, T& info, uint32_t id)
+{
+	auto prev = infos.at(info.name);
+	if (prev)
+	{
+		info.index = prev->index;
+		info.visibility = id;
+		*prev = info;
+	}
+	else
+	{
+		info.index = id;
+		info.visibility = id;
+		infos.insert(info, info.name);
+	}
+}
+
+template <typename T>
+static inline void append_info(T* prev, T& info, uint32_t id)
+{
+	prev->visibility = id;
+	prev->ctx.append(info.ctx);
+}
+
+static void validate_exception_info(
+	rule_loader::configuration& cfg,
+	rule_loader::rule_exception_info &ex,
+	const string& source)
+{
+	if (ex.fields.is_list)
+	{
+		if (!ex.comps.is_valid())
+		{
+			ex.comps.is_list = true;
+			for (size_t i = 0; i < ex.fields.items.size(); i++)
+			{
+				ex.comps.items.push_back({false, "="});
+			}
+		}
+		THROW(ex.fields.items.size() != ex.comps.items.size(),
+			"Rule exception item " + ex.name 
+				+ ": fields and comps lists must have equal length");
+		for (auto &v : ex.comps.items)
+		{
+			THROW(!is_operator_defined(v.item),
+				"Rule exception item " + ex.name + ": comparison operator "
+					+ v.item + " is not a supported comparison operator");
+		}
+		for (auto &v : ex.fields.items)
+		{
+			THROW(!is_field_defined(cfg.engine, source, v.item),
+				"Rule exception item " + ex.name + ": field name "
+					+ v.item + " is not a supported filter field");
+		}
+	}
+	else
+	{
+		if (!ex.comps.is_valid())
+		{
+			ex.comps.is_list = false;
+			ex.comps.items.push_back({false, "in"});
+		}
+		THROW(ex.comps.is_list, "Rule exception item "
+			+ ex.name + ": fields and comps must both be strings");
+		THROW(!is_operator_defined(ex.comps.item),
+			"Rule exception item " + ex.name + ": comparison operator "
+				+ ex.comps.item + " is not a supported comparison operator");
+		THROW(!is_field_defined(cfg.engine, source, ex.fields.item),
+			"Rule exception item " + ex.name + ": field name "
+				+ ex.fields.item + " is not a supported filter field");
+	}
+}
+
+static void build_rule_exception_infos(
+	vector<rule_loader::rule_exception_info>& exceptions,
+	set<string>& exception_fields,
+	string& condition)
+{
+	string tmp;
+	for (auto &ex : exceptions)
+	{
+		string icond;
+		if(!ex.fields.is_list)
+		{
+			for (auto &val : ex.values)
+			{
+				THROW(val.is_list, "Expected values array for item "
+					+ ex.name + " to contain a list of strings");
+				icond += icond.empty()
+					? ("(" + ex.fields.item + " "
+						+ ex.comps.item + " (")
+					: ", ";
+				exception_fields.insert(ex.fields.item);
+				tmp = val.item;
+				quote_item(tmp);
+				icond += tmp;
+			}
+			icond += icond.empty() ? "" : "))";
+		}
+		else
+		{
+			icond = "(";
+			for (auto &values : ex.values)
+			{
+				THROW(ex.fields.items.size() != values.items.size(),
+					"Exception item " + ex.name
+						+ ": fields and values lists must have equal length");
+				icond += icond == "(" ? "" : " or ";
+				icond += "(";
+				uint32_t k = 0;
+				string istr;
+				for (auto &field : ex.fields.items)
+				{
+					icond += k == 0 ? "" : " and ";
+					if (values.items[k].is_list)
+					{
+						istr = "(";
+						for (auto &v : values.items[k].items)
+						{
+							tmp = v.item;
+							quote_item(tmp);
+							istr += istr == "(" ? "" : ", ";
+							istr += tmp;
+						}
+						istr += ")";
+					}
+					else
+					{
+						istr = values.items[k].item;
+						if(is_operator_for_list(ex.comps.items[k].item))
+						{
+							paren_item(istr);
+						}
+						else
+						{
+							quote_item(istr);
+						}
+					}
+					icond += " " + field.item;
+					icond += " " + ex.comps.items[k].item + " " + istr;
+					exception_fields.insert(field.item);
+					k++;
+				}
+				icond += ")";
+			}
+			icond += ")";
+			if (icond == "()")
+			{
+				icond = "";
+			}
+		}
+		condition += icond.empty() ? "" : " and not " + icond;
+	}
+}
+
+// todo(jasondellaluce): this breaks string escaping in lists
+static bool resolve_list(string& cnd, const rule_loader::list_info& list)
+{
+	static string blanks = " \t\n\r";
+	static string delims = blanks + "(),=";
+	string new_cnd;
+	size_t start, end;
+	bool used = false;
+	start = cnd.find(list.name);
+	while (start != string::npos)
+	{
+		// the characters surrounding the name must
+		// be delims of beginning/end of string
+		end = start + list.name.length();
+		if ((start == 0 || delims.find(cnd[start - 1]) != string::npos)
+			&& (end >= cnd.length() || delims.find(cnd[end]) != string::npos))
+		{
+			// shift pointers to consume all whitespaces
+			while (start > 0
+				&& blanks.find(cnd[start - 1]) != string::npos)
+			{
+				start--;
+			}
+			while (end < cnd.length()
+				&& blanks.find(cnd[end]) != string::npos)
+			{
+				end++;
+			}
+			// create substitution string by concatenating all values
+			string sub = "";
+			for (auto &v : list.items)
+			{
+				if (!sub.empty())
+				{
+					sub += ", ";
+				}
+				sub += v;
+			}
+			// if substituted list is empty, we need to
+			// remove a comma from the left or the right
+			if (sub.empty())
+			{
+				if (start > 0 && cnd[start - 1] == ',')
+				{
+					start--;
+				}
+				else if (end < cnd.length() && cnd[end] == ',')
+				{
+					end++;
+				}
+			}
+			// compose new string with substitution
+			new_cnd = "";
+			if (start > 0)
+			{
+				new_cnd += cnd.substr(0, start) + " ";
+			}
+			new_cnd += sub + " ";
+			if (end <= cnd.length())
+			{
+				new_cnd += cnd.substr(end);
+			}
+			cnd = new_cnd;
+			start += sub.length() + 1;
+			used = true;
+		}
+		start = cnd.find(list.name, start + 1);
+	}
+	return used;
+}
+
+static void resolve_macros(
+	indexed_vector<rule_loader::macro_info>& macros,
+	shared_ptr<ast::expr>& ast,
+	uint32_t visibility,
+	const string& on_unknown_err_prefix)
+{
+	filter_macro_resolver macro_resolver;
+	for (auto &m : macros)
+	{
+		if (m.index < visibility)
+		{
+			macro_resolver.set_macro(m.name, m.cond_ast);
+		}
+	}
+	macro_resolver.run(ast);
+	THROW(!macro_resolver.get_unknown_macros().empty(),
+		on_unknown_err_prefix + "Undefined macro '"
+			+ *macro_resolver.get_unknown_macros().begin()
+			+ "' used in filter.");
+	for (auto &m : macro_resolver.get_resolved_macros())
+	{
+		macros.at(m)->used = true;
+	}
+}
+
+// note: there is no visibility order between filter conditions and lists
+static shared_ptr<ast::expr> parse_condition(
+	string condition,
+	indexed_vector<rule_loader::list_info>& lists)
+{
+	for (auto &l : lists)
+	{
+		if (resolve_list(condition, l))
+		{
+			l.used = true;
+		}
+	}
+	libsinsp::filter::parser p(condition);
+	p.set_max_depth(1000);
+	try
+	{
+		shared_ptr<ast::expr> res_ptr(p.parse());
+		return res_ptr;
+	}
+	catch (const sinsp_exception& e)
+	{
+		throw falco_exception("Compilation error when compiling \"" 
+			+ condition + "\": " + to_string(p.get_pos().col) + ": " + e.what());
+	}
+}
+
+static shared_ptr<gen_event_filter> compile_condition(
+		falco_engine* engine,
+		uint32_t id,
+		shared_ptr<ast::expr> cnd,
+		string src,
+		string& err)
+{
+	try
+	{
+		auto factory = engine->get_filter_factory(src);
+		sinsp_filter_compiler compiler(factory, cnd.get());
+		compiler.set_check_id(id);
+		shared_ptr<gen_event_filter> ret(compiler.compile());
+		return ret;
+	}
+	catch (const sinsp_exception& e)
+	{
+		err = e.what();
+	}
+	catch (const falco_exception& e)
+	{
+		err = e.what();
+	}
+	return nullptr;
+}
+
+static void apply_output_substitutions(
+	rule_loader::configuration& cfg,
+	string& out)
+{
+	if (out.find(s_container_info_fmt) != string::npos)
+	{
+		if (cfg.replace_output_container_info)
+		{
+			out = replace(out, s_container_info_fmt, cfg.output_extra);
+			return;
+		}
+		out = replace(out, s_container_info_fmt, s_default_extra_fmt);
+	}
+	out += cfg.output_extra.empty() ? "" : " " + cfg.output_extra;
+}
+
+void rule_loader::clear()
+{
+	m_cur_index = 0;
+	m_rule_infos.clear();
+	m_list_infos.clear();
+	m_macro_infos.clear();
+	m_required_plugin_versions.clear();
+}
+
+bool rule_loader::is_plugin_compatible(
+		const string &name,
+		const string &version,
+		string &required_version)
+{
+	set<string> required_plugin_versions;
+	sinsp_plugin::version plugin_version(version);
+	if(!plugin_version.m_valid)
+	{
+		throw falco_exception(
+			string("Plugin version string ") + version + " not valid");
+	}
+	auto it = m_required_plugin_versions.find(name);
+	if (it != m_required_plugin_versions.end())
+	{
+		for (auto &rversion : it->second)
+		{
+			sinsp_plugin::version req_version(rversion);
+			if (!plugin_version.check(req_version))
+			{
+				required_version = rversion;
+				return false;
+			}
+		}
+	}
+	return true;
+}
+
+void rule_loader::define(configuration& cfg, engine_version_info& info)
+{
+	auto v = falco_engine::engine_version();
+	THROW(v < info.version, "Rules require engine version "
+		+ to_string(info.version) + ", but engine version is " + to_string(v));
+}
+
+void rule_loader::define(configuration& cfg, plugin_version_info& info)
+{
+	m_required_plugin_versions[info.name].insert(info.version);
+}
+
+void rule_loader::define(configuration& cfg, list_info& info)
+{
+	define_info(m_list_infos, info, m_cur_index++);
+}
+
+void rule_loader::append(configuration& cfg, list_info& info)
+{
+	auto prev = m_list_infos.at(info.name);
+	THROW(!prev, "List " + info.name +
+		" has 'append' key but no list by that name already exists");
+	prev->items.insert(prev->items.end(), info.items.begin(), info.items.end());
+	append_info(prev, info, m_cur_index++);
+}
+
+void rule_loader::define(configuration& cfg, macro_info& info)
+{
+	if (!cfg.engine->is_source_valid(info.source))
+	{
+		cfg.warnings.push_back("Macro " + info.name
+			+ ": warning (unknown-source): unknown source "
+			+ info.source + ", skipping");
+		return;
+	}
+	define_info(m_macro_infos, info, m_cur_index++);
+}
+
+void rule_loader::append(configuration& cfg, macro_info& info)
+{
+	auto prev = m_macro_infos.at(info.name);
+	THROW(!prev, "Macro " + info.name
+		+ " has 'append' key but no macro by that name already exists");
+	prev->cond += " ";
+	prev->cond += info.cond;
+	append_info(prev, info, m_cur_index++);
+}
+
+void rule_loader::define(configuration& cfg, rule_info& info)
+{
+	if (!cfg.engine->is_source_valid(info.source))
+	{
+		cfg.warnings.push_back("Rule " + info.name
+			+ ": warning (unknown-source): unknown source "
+			+ info.source + ", skipping");
+		return;
+	}
+
+	auto prev = m_macro_infos.at(info.name);
+	THROW(prev && prev->source != info.source,
+		"Rule " + info.name + " has been re-defined with a different source");
+
+	for (auto &ex : info.exceptions)
+	{
+		THROW(!ex.fields.is_valid(), "Rule exception item "
+			+ ex.name + ": must have fields property with a list of fields");
+		validate_exception_info(cfg, ex, info.source);
+	}
+
+	define_info(m_rule_infos, info, m_cur_index++);
+}
+
+void rule_loader::append(configuration& cfg, rule_info& info)
+{
+	auto prev = m_rule_infos.at(info.name);
+	THROW(!prev, "Rule " + info.name
+		+ " has 'append' key but no rule by that name already exists");
+	THROW(info.cond.empty() && info.exceptions.empty(),
+		"Appended rule must have exceptions or condition property");
+
+	if (!info.cond.empty())
+	{
+		prev->cond += " ";
+		prev->cond += info.cond;
+	}
+
+	for (auto &ex : info.exceptions)
+	{
+		auto prev_ex = find_if(prev->exceptions.begin(), prev->exceptions.end(),
+			[&ex](const rule_loader::rule_exception_info& i)
+				{ return i.name == ex.name; });
+		if (prev_ex == prev->exceptions.end())
+		{
+			THROW(!ex.fields.is_valid(), "Rule exception new item "
+				+ ex.name + ": must have fields property with a list of fields");
+			THROW(ex.values.empty(), "Rule exception new item "
+				+ ex.name + ": must have fields property with a list of values");
+			validate_exception_info(cfg, ex, prev->source);
+			prev->exceptions.push_back(ex);
+		}
+		else
+		{
+			THROW(ex.fields.is_valid(),
+				"Can not append exception fields to existing rule, only values");
+			THROW(ex.comps.is_valid(),
+				"Can not append exception comps to existing rule, only values");
+			prev_ex->values.insert(
+				prev_ex->values.end(), ex.values.begin(), ex.values.end());
+		}
+	}
+	append_info(prev, info, m_cur_index++);
+}
+
+void rule_loader::enable(configuration& cfg, rule_info& info)
+{
+	auto prev = m_rule_infos.at(info.name);
+	THROW(!prev, "Rule " + info.name
+		+ " has 'enabled' key but no rule by that name already exists");
+	prev->enabled = info.enabled;
+}
+
+void rule_loader::compile_list_infos(configuration& cfg, indexed_vector<list_info>& out)
+{
+	string tmp;
+	vector<string> used;
+	for (auto &list : m_list_infos)
+	{
+		try
+		{
+			list_info v = list;
+			v.items.clear();
+			for (auto &item : list.items)
+			{
+				auto ref = m_list_infos.at(item);
+				if (ref && ref->index < list.visibility)
+				{
+					used.push_back(ref->name);
+					for (auto val : ref->items)
+					{
+						quote_item(val);
+						v.items.push_back(val);
+					}
+				}
+				else
+				{
+					tmp = item;
+					quote_item(tmp);
+					v.items.push_back(tmp);
+				}
+			}
+			v.used = false;
+			out.insert(v, v.name);
+		}
+		catch (exception& e)
+		{
+			throw falco_exception(list.ctx.error(e.what()));
+		}
+	}
+	for (auto &v : used)
+	{
+		out.at(v)->used = true;
+	}
+}
+
+// note: there is a visibility ordering between macros
+void rule_loader::compile_macros_infos(
+	configuration& cfg,
+	indexed_vector<list_info>& lists,
+	indexed_vector<macro_info>& out)
+{
+	set<string> used;
+	context* info_ctx = NULL;
+	try
+	{
+		for (auto &m : m_macro_infos)
+		{
+			info_ctx = &m.ctx;
+			macro_info entry = m;
+			entry.cond_ast = parse_condition(m.cond, lists);
+			entry.used = false;
+			out.insert(entry, m.name);
+		}
+		for (auto &m : out)
+		{
+			info_ctx = &m.ctx;
+			resolve_macros(out, m.cond_ast, m.visibility,
+				"Compilation error when compiling \"" + m.cond + "\": ");
+		}
+	}
+	catch (exception& e)
+	{
+		throw falco_exception(info_ctx->error(e.what()));
+	}
+}
+
+
+void rule_loader::compile_rule_infos(
+	configuration& cfg,
+	indexed_vector<list_info>& lists,
+	indexed_vector<macro_info>& macros,
+	indexed_vector<falco_rule>& out)
+{
+	string err, condition;
+	for (auto &r : m_rule_infos)
+	{
+		try
+		{
+			if (r.priority > cfg.min_priority)
+			{
+				continue;
+			}
+
+			falco_rule rule;
+			condition = r.cond;
+			if (!r.exceptions.empty())
+			{
+				build_rule_exception_infos(
+					r.exceptions, rule.exception_fields, condition);
+			}
+			auto ast = parse_condition(condition, lists);
+			resolve_macros(macros, ast, MAX_VISIBILITY, "");
+			
+			rule.output = r.output;
+			if (r.source == falco_common::syscall_source)
+			{
+				apply_output_substitutions(cfg, rule.output);
+			}
+			THROW(!is_format_valid(cfg.engine, r.source, rule.output, err), 
+				"Invalid output format '" + rule.output + "': '" + err + "'");
+			
+			
+			rule.name = r.name;
+			rule.source = r.source;
+			rule.description = r.desc;
+			rule.priority = r.priority;
+			rule.tags = r.tags;
+			// note: indexes are 0-based, but 0 is not an acceptable rule_id
+			auto id = out.insert(rule, rule.name) + 1;
+			auto filter = compile_condition(cfg.engine, id, ast, rule.source, err);
+			if (!filter)
+			{
+				if (r.skip_if_unknown_filter
+					&& err.find("nonexistent field") != string::npos)
+				{
+					cfg.warnings.push_back(
+						"Rule " + rule.name + ": warning (unknown-field):");
+					continue;
+				}
+				else
+				{
+					throw falco_exception("Rule " + rule.name + ": error " + err);
+				}
+			}
+
+			set<uint16_t> evttypes;
+			if(rule.source == falco_common::syscall_source)
+			{
+				filter_evttype_resolver resolver;
+				resolver.evttypes(ast, evttypes);
+				if ((evttypes.empty() || evttypes.size() > 100)
+					&& r.warn_evttypes)
+				{
+					cfg.warnings.push_back(
+						"Rule " + rule.name + ": warning (no-evttype):\n" +
+						+ "		 matches too many evt.type values.\n"
+						+ "		 This has a significant performance penalty.");
+				}
+			}
+			else if (rule.source == "k8s_audit")
+			{
+				// todo(jasondellaluce): remove this case once k8saudit
+				// gets ported to a plugin
+				evttypes = { ppm_event_type::PPME_GENERIC_X };
+			}
+			else
+			{
+				evttypes = { ppm_event_type::PPME_PLUGINEVENT_E };
+			}
+
+			cfg.engine->add_filter(filter, rule.name, rule.source, evttypes, rule.tags);
+			cfg.engine->enable_rule(rule.name, r.enabled);
+		}
+		catch (exception& e)
+		{
+			throw falco_exception(r.ctx.error(e.what()));
+		}
+	}
+}
+
+bool rule_loader::compile(configuration& cfg, indexed_vector<falco_rule>& out)
+{
+	indexed_vector<list_info> lists;
+	indexed_vector<macro_info> macros;
+
+	// expand all lists, macros, and rules
+	try
+	{
+		compile_list_infos(cfg, lists);
+		compile_macros_infos(cfg, lists, macros);
+		compile_rule_infos(cfg, lists, macros, out);
+	}
+	catch (exception& e)
+	{
+		cfg.errors.push_back(e.what());
+		return false;
+	}
+
+	// print info on any dangling lists or macros that were not used anywhere
+	for (auto &m : macros)
+	{
+		if (!m.used)
+		{
+			cfg.warnings.push_back("macro " + m.name
+				+ " not referred to by any rule/macro");
+		}
+	}
+	for (auto &l : lists)
+	{
+		if (!l.used)
+		{
+			cfg.warnings.push_back("list " + l.name
+				+ " not referred to by any rule/macro/list");
+		}
+	}
+	return true;
+}

userspace/engine/rule_loader.h

@@ -0,0 +1,239 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+#include <yaml-cpp/yaml.h>
+#include "falco_rule.h"
+#include "indexed_vector.h"
+
+// todo(jasondellaluce): remove this cyclic dependency
+class falco_engine;
+
+
+/*!
+	\brief Ruleset loader of the falco engine
+*/
+class rule_loader
+{
+public:
+	/*!
+		\brief Represents a section of text from which a certain info
+		struct has been decoded
+	*/
+	struct context
+	{
+		std::string content;
+
+		/*!
+			\brief Wraps an error by adding info about the text section
+		*/
+		inline std::string error(std::string err)
+		{
+			err += "\n---\n";
+			err += trim(content);
+			err += "\n---";
+			return err;
+		}
+
+		/*!
+			\brief Appends another text section info to this one
+		*/
+		inline void append(context& m)
+		{
+			content += "\n\n";
+			content += m.content;
+		}
+	};
+
+	/*!
+		\brief Contains the info required to load rule definitions
+	*/
+	struct configuration
+	{
+		configuration(const std::string& cont): content(cont) {}
+		const std::string& content;
+		std::string output_extra;
+		bool replace_output_container_info;
+		falco_common::priority_type min_priority;
+		std::vector<std::string> warnings;
+		std::vector<std::string> errors;
+		falco_engine* engine;
+	};
+
+	/*!
+		\brief Represents infos about an engine version requirement
+	*/
+	struct engine_version_info
+	{
+		uint32_t version;
+	};
+
+	/*!
+		\brief Represents infos about a plugin version requirement
+	*/
+	struct plugin_version_info
+	{
+		std::string name;
+		std::string version;
+	};
+
+	/*!
+		\brief Represents infos about a list 
+	*/
+	struct list_info
+	{
+		context ctx;
+		bool used;
+		size_t index;
+		size_t visibility;
+		std::string name;
+		std::vector<std::string> items;
+	};
+
+	/*!
+		\brief Represents infos about a macro 
+	*/
+	struct macro_info
+	{
+		context ctx;
+		bool used;
+		size_t index;
+		size_t visibility;
+		std::string name;
+		std::string cond;
+		std::string source;
+		std::shared_ptr<libsinsp::filter::ast::expr> cond_ast;
+	};
+
+	/*!
+		\brief Represents infos about a single rule exception
+	*/
+	struct rule_exception_info
+	{
+		/*!
+			\brief This is necessary due to the dynamic-typed nature of
+			exceptions. Each of fields, comps, and values, can either be a
+			single value or a list of values. This is a simple hack to make
+			this easier to implement in C++, that is not non-dynamic-typed.
+		*/
+		struct entry {
+			bool is_list;
+			std::string item;
+			std::vector<entry> items;
+
+			inline bool is_valid() const
+			{
+				return (is_list && !items.empty())
+					|| (!is_list && !item.empty());
+			}
+		};
+
+		std::string name;
+		entry fields;
+		entry comps;
+		std::vector<entry> values;
+	};
+
+	/*!
+		\brief Represents infos about a rule 
+	*/
+	struct rule_info
+	{
+		context ctx;
+		size_t index;
+		size_t visibility;
+		std::string name;
+		std::string cond;
+		std::string source;
+		std::string desc;
+		std::string output;
+		std::set<std::string> tags;
+		std::vector<rule_exception_info> exceptions;
+		falco_common::priority_type priority;
+		bool enabled;
+		bool warn_evttypes;
+		bool skip_if_unknown_filter;
+	};
+
+	/*!
+		\brief Erases all the internal state and definitions
+	*/
+	virtual void clear();
+
+	/*!
+		\brief Returns true if the given plugin name and version are compatible
+		with the internal definitions. If false is returned, required_version is
+		filled with the required plugin version that didn't match.
+	*/
+	virtual bool is_plugin_compatible(
+		const std::string& name,
+		const std::string& version,
+		std::string& required_version);
+	
+	/*!
+		\brief Uses the internal state to compile a list of falco_rules
+	*/
+	bool compile(configuration& cfg, indexed_vector<falco_rule>& out);
+
+	/*!
+		\brief Defines an info block. If a similar info block is found
+		in the internal state (e.g. another rule with same name), then
+		the previous definition gets overwritten
+	*/
+	virtual void define(configuration& cfg, engine_version_info& info);
+	virtual void define(configuration& cfg, plugin_version_info& info);
+	virtual void define(configuration& cfg, list_info& info);
+	virtual void define(configuration& cfg, macro_info& info);
+	virtual void define(configuration& cfg, rule_info& info);
+
+	/*!
+		\brief Appends an info block to an existing one. An exception
+		is thrown if no existing definition can be matched with the appended
+		one
+	*/
+	virtual void append(configuration& cfg, list_info& info);
+	virtual void append(configuration& cfg, macro_info& info);
+	virtual void append(configuration& cfg, rule_info& info);
+
+	/*!
+		\brief Updates the 'enabled' flag of an existing definition
+	*/
+	virtual void enable(configuration& cfg, rule_info& info);
+
+private:
+	void compile_list_infos(
+		configuration& cfg,
+		indexed_vector<list_info>& out);
+	void compile_macros_infos(
+		configuration& cfg,
+		indexed_vector<list_info>& lists,
+		indexed_vector<macro_info>& out);
+	void compile_rule_infos(
+		configuration& cfg,
+		indexed_vector<list_info>& lists,
+		indexed_vector<macro_info>& macros,
+		indexed_vector<falco_rule>& out);
+
+	uint32_t m_cur_index;
+	indexed_vector<rule_info> m_rule_infos;
+	indexed_vector<macro_info> m_macro_infos;
+	indexed_vector<list_info> m_list_infos;
+	std::map<std::string, std::set<std::string>> m_required_plugin_versions;
+};

userspace/engine/rule_reader.cpp

@@ -0,0 +1,325 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "rule_reader.h"
+
+#define THROW(cond, err)    { if (cond) { throw falco_exception(err); } }
+
+static rule_loader::context yaml_get_context(
+		const string& content,
+		const vector<YAML::Node>& docs,
+		vector<YAML::Node>::iterator doc,
+		YAML::iterator node)
+{
+	rule_loader::context m;
+	YAML::Node item = *node++;
+	YAML::Node cur_doc = *doc++;
+	// include the "- " sequence mark
+	size_t from = item.Mark().pos - 2;
+	size_t to = 0;
+	if (node != cur_doc.end())
+	{
+		// end of item is beginning of next item
+		to = node->Mark().pos - 2;
+	}
+	else if (doc != docs.end())
+	{
+		// end of item is beginning of next doc
+		to = doc->Mark().pos - 4;
+	}
+	else
+	{
+		// end of item is end of file contents
+		to = content.length();
+	}
+	m.content = content.substr(from, to - from);
+	m.content = trim(m.content);
+	return m;
+}
+
+template <typename T>
+static bool decode_val(const YAML::Node& v, T& out)
+{
+	return v.IsDefined() && v.IsScalar() && YAML::convert<T>::decode(v, out);
+}
+
+template <typename T>
+static bool decode_seq(const YAML::Node& item, vector<T>& out)
+{
+	if (item.IsDefined() && item.IsSequence())
+	{
+		T value;
+		for(const YAML::Node& v : item)
+		{
+			THROW(!v.IsScalar() || !YAML::convert<T>::decode(v, value),
+				"Can't decode YAML sequence value: " + YAML::Dump(v));
+			out.push_back(value);
+		}
+		return true;
+	}
+	return false;
+}
+
+template <typename T>
+static bool decode_seq(const YAML::Node& item, set<T>& out)
+{
+	if (item.IsDefined() && item.IsSequence())
+	{
+		T value;
+		for(const YAML::Node& v : item)
+		{
+			THROW(!v.IsScalar() || !YAML::convert<T>::decode(v, value),
+				"Can't decode YAML sequence value: " + YAML::Dump(v));
+			out.insert(value);
+		}
+		return true;
+	}
+	return false;
+}
+
+static bool decode_exception_info_entry(
+	const YAML::Node& item,
+	rule_loader::rule_exception_info::entry& out)
+{
+	if (item.IsDefined())
+	{
+		if (item.IsScalar())
+		{
+			out.is_list = false;
+			if (YAML::convert<string>::decode(item, out.item))
+			{
+				return true;
+			}
+		}
+		if (item.IsSequence())
+		{
+			out.is_list = true;
+			rule_loader::rule_exception_info::entry tmp;
+			for(const YAML::Node& v : item)
+			{
+				if (!decode_exception_info_entry(v, tmp))
+				{
+					return false;
+				}
+				out.items.push_back(tmp);
+			}
+			return true;
+		}
+	}
+	return false;
+}
+
+static void read_rule_exceptions(
+	const YAML::Node& item,
+	rule_loader::rule_info& v)
+{
+	THROW(!item.IsSequence(), "Rule exceptions must be a sequence");
+	for (auto &ex : item)
+	{
+		rule_loader::rule_exception_info v_ex;
+		THROW(!decode_val(ex["name"], v_ex.name) || v_ex.name.empty(),
+			"Rule exception item must have name property");
+		// note: the legacy lua loader used to throw a "xxx must strings" error
+		decode_exception_info_entry(ex["fields"], v_ex.fields);
+		decode_exception_info_entry(ex["comps"], v_ex.comps);
+		if (ex["values"].IsDefined())
+		{
+			THROW(!ex["values"].IsSequence(),
+				"Rule exception values must be a sequence");
+			for (auto &val : ex["values"])
+			{
+				rule_loader::rule_exception_info::entry v_ex_val;
+				decode_exception_info_entry(val, v_ex_val);
+				v_ex.values.push_back(v_ex_val);
+			}
+		}
+		v.exceptions.push_back(v_ex);
+	}
+}
+
+static void read_item(
+		rule_loader::configuration& cfg,
+		rule_loader& loader,
+		const YAML::Node& item,
+		const rule_loader::context& ctx)
+{
+	if (item["required_engine_version"].IsDefined())
+	{
+		rule_loader::engine_version_info v;
+		THROW(!decode_val(item["required_engine_version"], v.version),
+			"Value of required_engine_version must be a number");
+		loader.define(cfg, v);
+	}
+	else if(item["required_plugin_versions"].IsDefined())
+	{
+		THROW(!item["required_plugin_versions"].IsSequence(),
+			"Value of required_plugin_versions must be a sequence");
+
+		for(const YAML::Node& plugin : item["required_plugin_versions"])
+		{
+			rule_loader::plugin_version_info v;
+			THROW(!decode_val(plugin["name"], v.name) || v.name.empty(),
+				"required_plugin_versions item must have name property");
+			THROW(!decode_val(plugin["version"], v.version) || v.version.empty(),
+				"required_plugin_versions item must have version property");
+			loader.define(cfg, v);
+		}
+	}
+	else if(item["list"].IsDefined())
+	{
+		rule_loader::list_info v;
+		v.ctx = ctx;
+		bool append = false;
+		THROW(!decode_val(item["list"], v.name) || v.name.empty(),
+			"List name is empty");
+		THROW(!decode_seq(item["items"], v.items),
+			"List must have property items");
+		if(decode_val(item["append"], append) && append)
+		{
+			loader.append(cfg, v);
+		}
+		else
+		{
+			loader.define(cfg, v);
+		}
+	}
+	else if(item["macro"].IsDefined())
+	{
+		rule_loader::macro_info v;
+		v.ctx = ctx;
+		bool append = false;
+		v.source = falco_common::syscall_source;
+		THROW(!decode_val(item["macro"], v.name) || v.name.empty(),
+			"Macro name is empty");
+		THROW(!decode_val(item["condition"], v.cond) || v.cond.empty(),
+			"Macro must have property condition");
+		decode_val(item["source"], v.source);
+		if(decode_val(item["append"], append) && append)
+		{
+			loader.append(cfg, v);
+		}
+		else
+		{
+			loader.define(cfg, v);
+		}
+	}
+	else if(item["rule"].IsDefined())
+	{
+		rule_loader::rule_info v;
+		v.ctx = ctx;
+		bool append = false;
+		v.enabled = true;
+		v.warn_evttypes = true;
+		v.skip_if_unknown_filter = false;
+		THROW(!decode_val(item["rule"], v.name) || v.name.empty(),
+			"Rule name is empty");
+		if(decode_val(item["append"], append) && append)
+		{
+			decode_val(item["condition"], v.cond);
+			if (item["exceptions"].IsDefined())
+			{
+				read_rule_exceptions(item["exceptions"], v);
+			}
+			loader.append(cfg, v);
+		}
+		else
+		{
+			string priority;
+			bool has_enabled = decode_val(item["enabled"], v.enabled);
+			bool has_defs = decode_val(item["condition"], v.cond)
+					&& decode_val(item["output"], v.output)
+					&& decode_val(item["desc"], v.desc)
+					&& decode_val(item["priority"], priority);
+			if (!has_defs)
+			{
+				THROW(!has_enabled, "Rule must have properties 'condition', 'output', 'desc', and 'priority'");
+				loader.enable(cfg, v);
+			}
+			else
+			{
+				v.output = trim(v.output);
+				v.source = falco_common::syscall_source;
+				THROW(!falco_common::parse_priority(priority, v.priority),
+					"Invalid priority");
+				decode_val(item["source"], v.source);
+				decode_val(item["warn_evttypes"], v.warn_evttypes);
+				decode_val(item["skip-if-unknown-filter"], v.skip_if_unknown_filter);
+				decode_seq(item["tags"], v.tags);
+				if (item["exceptions"].IsDefined())
+				{
+					read_rule_exceptions(item["exceptions"], v);
+				}
+				loader.define(cfg, v);
+			}
+		}
+	}
+	else
+	{
+		cfg.warnings.push_back("Unknown top level object");
+	}
+}
+
+bool rule_reader::load(rule_loader::configuration& cfg, rule_loader& loader)
+{
+	std::vector<YAML::Node> docs;
+	try
+	{
+		docs = YAML::LoadAll(cfg.content);
+	}
+	catch(const exception& e)
+	{
+		cfg.errors.push_back("Could not load YAML file: " + string(e.what()));
+		return false;
+	}
+	
+	for (auto doc = docs.begin(); doc != docs.end(); doc++)
+	{
+		if (doc->IsDefined() && !doc->IsNull())
+		{
+			if(!doc->IsMap() && !doc->IsSequence())
+			{
+				cfg.errors.push_back("Rules content is not yaml");
+				return false;
+			}
+			if(!doc->IsSequence())
+			{
+				cfg.errors.push_back(
+					"Rules content is not yaml array of objects");
+				return false;
+			}
+			for (auto it = doc->begin(); it != doc->end(); it++)
+			{
+				if (!it->IsNull())
+				{
+					auto ctx = yaml_get_context(cfg.content, docs, doc, it);
+					YAML::Node item = *it;
+					try
+					{
+						THROW(!item.IsMap(), "Unexpected element type. "
+							"Each element should be a yaml associative array.");
+						read_item(cfg, loader, item, ctx);
+					}
+					catch(const exception& e)
+					{
+						cfg.errors.push_back(ctx.error(e.what()));
+						return false;
+					}
+				}
+			}
+		}
+	}
+	return true;
+}

userspace/engine/rule_reader.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2019 The Falco Authors.
+Copyright (C) 2022 The Falco Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,7 +16,20 @@ limitations under the License.
 
 #pragma once
 
-#include "lua.h"
-
-int luaopen_yaml (lua_State *L);
+#include <map>
+#include <string>
+#include <vector>
+#include "rule_loader.h"
 
+/*!
+    \brief Reads the contents of a ruleset
+*/
+class rule_reader
+{
+public:
+    /*!
+		\brief Reads the contents of a ruleset and uses a loader to store
+        thew new definitions
+	*/
+	virtual bool load(rule_loader::configuration& cfg, rule_loader& loader);
+};

userspace/engine/rules.cpp

@@ -1,503 +0,0 @@
-/*
-Copyright (C) 2019 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include <sstream>
-
-#include "rules.h"
-
-extern "C" {
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
-}
-
-#include "falco_engine.h"
-#include "banned.h" // This raises a compilation error when certain functions are used
-
-const static struct luaL_Reg ll_falco_rules[] =
-	{
-		{"clear_filters", &falco_rules::clear_filters},
-		{"create_lua_parser", &falco_rules::create_lua_parser},
-		{"add_filter", &falco_rules::add_filter},
-		{"enable_rule", &falco_rules::enable_rule},
-		{"engine_version", &falco_rules::engine_version},
-		{"is_source_valid", &falco_rules::is_source_valid},
-		{"is_format_valid", &falco_rules::is_format_valid},
-		{"is_defined_field", &falco_rules::is_defined_field},
-		{NULL, NULL}};
-
-falco_rules::falco_rules(falco_engine *engine,
-			 lua_State *ls)
-	: m_engine(engine),
-	  m_ls(ls)
-{
-}
-
-void falco_rules::add_filter_factory(const std::string &source,
-				     std::shared_ptr<gen_event_filter_factory> factory)
-{
-	m_filter_factories[source] = factory;
-}
-
-void falco_rules::init(lua_State *ls)
-{
-	luaL_openlib(ls, "falco_rules", ll_falco_rules, 0);
-	lua_parser::register_callbacks(ls, "filter");
-}
-
-int falco_rules::clear_filters(lua_State *ls)
-{
-	if (! lua_islightuserdata(ls, -1))
-	{
-		lua_pushstring(ls, "Invalid arguments passed to clear_filters()");
-		lua_error(ls);
-	}
-
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -1);
-	rules->clear_filters();
-
-	return 0;
-}
-
-void falco_rules::clear_filters()
-{
-	m_engine->clear_filters();
-}
-
-int falco_rules::create_lua_parser(lua_State *ls)
-{
-	if (! lua_islightuserdata(ls, -2) ||
-	    ! lua_isstring(ls, -1))
-	{
-		lua_pushstring(ls, "Invalid argument passed to create_lua_parser()");
-		lua_error(ls);
-	}
-
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -2);
-	std::string source = lua_tostring(ls, -1);
-
-	std::string errstr;
-	lua_parser *lp = rules->create_lua_parser(source, errstr);
-
-	if(lp == NULL) {
-		lua_pushstring(ls, errstr.c_str());
-		lua_error(ls);
-	}
-
-	lua_pushlightuserdata(ls, lp);
-	return 1;
-}
-
-lua_parser *falco_rules::create_lua_parser(std::string &source, std::string &errstr)
-{
-	auto it = m_filter_factories.find(source);
-
-	if(it == m_filter_factories.end())
-	{
-		errstr = string("Unknown event source ") + source;
-		return NULL;
-	}
-
-	lua_parser *lp = new lua_parser(it->second);
-
-	return lp;
-}
-
-int falco_rules::add_filter(lua_State *ls)
-{
-	if (! lua_islightuserdata(ls, -5) ||
-	    ! lua_islightuserdata(ls, -4) ||
-	    ! lua_isstring(ls, -3) ||
-	    ! lua_isstring(ls, -2) ||
-	    ! lua_istable(ls, -1))
-	{
-		lua_pushstring(ls, "Invalid arguments passed to add_filter()");
-		lua_error(ls);
-	}
-
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -5);
-	lua_parser *lp = (lua_parser *) lua_topointer(ls, -4);
-	std::string rule = lua_tostring(ls, -3);
-	std::string source = lua_tostring(ls, -2);
-
-	set<string> tags;
-
-	lua_pushnil(ls);  /* first key */
-	while (lua_next(ls, -2) != 0) {
-                // key is at index -2, value is at index
-                // -1. We want the values.
-		tags.insert(lua_tostring(ls, -1));
-
-		// Remove value, keep key for next iteration
-		lua_pop(ls, 1);
-	}
-
-	// todo(jasondellaluce,leogr,fededp): temp workaround, remove when fixed in libs
-	size_t num_evttypes = 1; // assume plugin
-	if(source == "syscall" || source == "k8s_audit")
-	{
-		num_evttypes = lp->filter()->evttypes().size();
-	}
-
-	try
-	{
-		rules->add_filter(lp->filter(), rule, source, tags);
-	}
-	catch (exception &e)
-	{
-		std::string errstr = string("Could not add rule to falco engine: ") + e.what();
-		lua_pushstring(ls, errstr.c_str());
-		lua_error(ls);
-	}
-
-	delete lp;
-
-	lua_pushnumber(ls, num_evttypes);
-	return 1;
-}
-
-void falco_rules::add_filter(std::shared_ptr<gen_event_filter> filter, string &rule, string &source, set<string> &tags)
-{
-	m_engine->add_filter(filter, rule, source, tags);
-}
-
-int falco_rules::enable_rule(lua_State *ls)
-{
-	if (! lua_islightuserdata(ls, -3) ||
-	    ! lua_isstring(ls, -2) ||
-	    ! lua_isnumber(ls, -1))
-	{
-		lua_pushstring(ls, "Invalid arguments passed to enable_rule()");
-		lua_error(ls);
-	}
-
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -3);
-	const char *rulec = lua_tostring(ls, -2);
-	std::string rule = rulec;
-	bool enabled = (lua_tonumber(ls, -1) ? true : false);
-
-	rules->enable_rule(rule, enabled);
-
-	return 0;
-}
-
-void falco_rules::enable_rule(string &rule, bool enabled)
-{
-	m_engine->enable_rule(rule, enabled);
-}
-
-int falco_rules::engine_version(lua_State *ls)
-{
-	if (! lua_islightuserdata(ls, -1))
-	{
-		lua_pushstring(ls, "Invalid arguments passed to engine_version()");
-		lua_error(ls);
-	}
-
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -1);
-
-	lua_pushnumber(ls, rules->m_engine->engine_version());
-
-	return 1;
-}
-
-bool falco_rules::is_source_valid(const std::string &source)
-{
-	return m_engine->is_source_valid(source);
-}
-
-int falco_rules::is_source_valid(lua_State *ls)
-{
-	if (! lua_islightuserdata(ls, -2) ||
-	    ! lua_isstring(ls, -1))
-	{
-		lua_pushstring(ls, "Invalid arguments passed to is_source_valid");
-		lua_error(ls);
-	}
-
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -2);
-	string source = luaL_checkstring(ls, -1);
-
-	bool ret = rules->is_source_valid(source);
-
-	lua_pushboolean(ls, (ret ? 1 : 0));
-
-	return 1;
-}
-
-int falco_rules::is_format_valid(lua_State *ls)
-{
-	if (! lua_islightuserdata(ls, -3) ||
-	    ! lua_isstring(ls, -2) ||
-	    ! lua_isstring(ls, -1))
-	{
-		lua_pushstring(ls, "Invalid arguments passed to is_format_valid");
-		lua_error(ls);
-	}
-
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -3);
-	string source = luaL_checkstring(ls, -2);
-	string format = luaL_checkstring(ls, -1);
-	string errstr;
-
-	bool ret = rules->is_format_valid(source, format, errstr);
-
-	if (!ret)
-	{
-		lua_pushstring(ls, errstr.c_str());
-	}
-	else
-	{
-		lua_pushnil(ls);
-	}
-
-	return 1;
-}
-
-bool falco_rules::is_format_valid(const std::string &source, const std::string &format, std::string &errstr)
-{
-	bool ret = true;
-
-	try
-	{
-		std::shared_ptr<gen_event_formatter> formatter;
-
-		formatter = m_engine->create_formatter(source, format);
-	}
-	catch(exception &e)
-	{
-		std::ostringstream os;
-
-		os << "Invalid output format '"
-		   << format
-		   << "': '"
-		   << e.what()
-		   << "'";
-
-		errstr = os.str();
-		ret = false;
-	}
-
-	return ret;
-}
-
-int falco_rules::is_defined_field(lua_State *ls)
-{
-	if (! lua_islightuserdata(ls, -3) ||
-	    ! lua_isstring(ls, -2) ||
-	    ! lua_isstring(ls, -1))
-	{
-		lua_pushstring(ls, "Invalid arguments passed to is_defined_field");
-		lua_error(ls);
-	}
-
-	falco_rules *rules = (falco_rules *) lua_topointer(ls, -3);
-	string source = luaL_checkstring(ls, -2);
-	string fldname = luaL_checkstring(ls, -1);
-
-	bool ret = rules->is_defined_field(source, fldname);
-
-	lua_pushboolean(ls, (ret ? 1 : 0));
-
-	return 1;
-}
-
-bool falco_rules::is_defined_field(const std::string &source, const std::string &fldname)
-{
-	auto it = m_filter_factories.find(source);
-
-	if(it == m_filter_factories.end())
-	{
-		return false;
-	}
-
-	auto *chk = it->second->new_filtercheck(fldname.c_str());
-
-	if (chk == NULL)
-	{
-		return false;
-	}
-
-	delete(chk);
-
-	return true;
-}
-
-static std::list<std::string> get_lua_table_values(lua_State *ls, int idx)
-{
-	std::list<std::string> ret;
-
-	if (lua_isnil(ls, idx)) {
-		return ret;
-	}
-
-	lua_pushnil(ls);  /* first key */
-	while (lua_next(ls, idx-1) != 0) {
-                // key is at index -2, value is at index
-                // -1. We want the values.
-		if (! lua_isstring(ls, -1)) {
-			std::string err = "Non-string value in table of strings";
-			throw falco_exception(err);
-		}
-		ret.push_back(string(lua_tostring(ls, -1)));
-
-		// Remove value, keep key for next iteration
-		lua_pop(ls, 1);
-	}
-
-	return ret;
-}
-
-static void get_lua_table_list_values(lua_State *ls,
-				      int idx,
-				      std::map<std::string, std::list<std::string>> &required_plugin_versions)
-{
-	if (lua_isnil(ls, idx)) {
-		return;
-	}
-
-	lua_pushnil(ls);  /* first key */
-	while (lua_next(ls, idx-1) != 0) {
-                // key is at index -2, table of values is at index -1.
-		if (! lua_isstring(ls, -2)) {
-			std::string err = "Non-string key in table of strings";
-			throw falco_exception(err);
-		}
-
-		std::string key = string(lua_tostring(ls, -2));
-		std::list<std::string> vals = get_lua_table_values(ls, -1);
-
-		if (required_plugin_versions.find(key) == required_plugin_versions.end())
-		{
-			required_plugin_versions[key] = vals;
-		}
-		else
-		{
-			required_plugin_versions[key].insert(required_plugin_versions[key].end(),
-							     vals.begin(),
-							     vals.end());
-		}
-
-		// Remove value, keep key for next iteration
-		lua_pop(ls, 1);
-	}
-}
-
-
-void falco_rules::load_rules(const string &rules_content,
-			     bool verbose, bool all_events,
-			     string &extra, bool replace_container_info,
-			     falco_common::priority_type min_priority,
-			     uint64_t &required_engine_version,
-			     std::map<std::string, std::list<std::string>> &required_plugin_versions)
-{
-	lua_getglobal(m_ls, m_lua_load_rules.c_str());
-	if(lua_isfunction(m_ls, -1))
-	{
-		lua_pushstring(m_ls, rules_content.c_str());
-		lua_pushlightuserdata(m_ls, this);
-		lua_pushboolean(m_ls, (verbose ? 1 : 0));
-		lua_pushboolean(m_ls, (all_events ? 1 : 0));
-		lua_pushstring(m_ls, extra.c_str());
-		lua_pushboolean(m_ls, (replace_container_info ? 1 : 0));
-		lua_pushnumber(m_ls, min_priority);
-		if(lua_pcall(m_ls, 7, 5, 0) != 0)
-		{
-			const char* lerr = lua_tostring(m_ls, -1);
-
-			string err = "Error loading rules: " + string(lerr);
-
-			throw falco_exception(err);
-		}
-
-		// Returns:
-		// Load result: bool
-		// required engine version: will be nil when load result is false
-		// required_plugin_versions: will be nil when load result is false
-		// array of errors
-		// array of warnings
-		bool successful = lua_toboolean(m_ls, -5);
-		required_engine_version = lua_tonumber(m_ls, -4);
-		get_lua_table_list_values(m_ls, -3, required_plugin_versions);
-		std::list<std::string> errors = get_lua_table_values(m_ls, -2);
-		std::list<std::string> warnings = get_lua_table_values(m_ls, -1);
-
-		// Concatenate errors/warnings
-		std::ostringstream os;
-		if (errors.size() > 0)
-		{
-			os << errors.size() << " errors:" << std::endl;
-			for(auto err : errors)
-			{
-				os << err << std::endl;
-			}
-		}
-
-		if (warnings.size() > 0)
-		{
-			os << warnings.size() << " warnings:" << std::endl;
-			for(auto warn : warnings)
-			{
-				os << warn << std::endl;
-			}
-		}
-
-		if(!successful)
-		{
-			throw falco_exception(os.str());
-		}
-
-		if (verbose && os.str() != "") {
-			// We don't really have a logging callback
-			// from the falco engine, but this would be a
-			// good place to use it.
-			fprintf(stderr, "When reading rules content: %s", os.str().c_str());
-		}
-
-		lua_pop(m_ls, 4);
-
-	} else {
-		throw falco_exception("No function " + m_lua_load_rules + " found in lua rule module");
-	}
-}
-
-void falco_rules::describe_rule(std::string *rule)
-{
-	lua_getglobal(m_ls, m_lua_describe_rule.c_str());
-	if(lua_isfunction(m_ls, -1))
-	{
-		if (rule == NULL)
-		{
-			lua_pushnil(m_ls);
-		} else {
-			lua_pushstring(m_ls, rule->c_str());
-		}
-
-		if(lua_pcall(m_ls, 1, 0, 0) != 0)
-		{
-			const char* lerr = lua_tostring(m_ls, -1);
-			string err = "Could not describe " + (rule == NULL ? "all rules" : "rule " + *rule) + ": " + string(lerr);
-			throw falco_exception(err);
-		}
-	} else {
-		throw falco_exception("No function " + m_lua_describe_rule + " found in lua rule module");
-	}
-}
-
-
-falco_rules::~falco_rules()
-{
-}

userspace/engine/rules.h

@@ -1,86 +0,0 @@
-/*
-Copyright (C) 2019 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#pragma once
-
-#include <set>
-#include <memory>
-
-#include "sinsp.h"
-#include "filter.h"
-
-#include "lua_parser.h"
-
-#include "json_evt.h"
-#include "falco_common.h"
-
-class falco_engine;
-
-class falco_rules
-{
- public:
-	falco_rules(falco_engine *engine,
-		    lua_State *ls);
-	~falco_rules();
-
-	void add_filter_factory(const std::string &source,
-				std::shared_ptr<gen_event_filter_factory> factory);
-
-	void load_rules(const string &rules_content, bool verbose, bool all_events,
-			std::string &extra, bool replace_container_info,
-			falco_common::priority_type min_priority,
-			uint64_t &required_engine_version,
-			std::map<std::string, std::list<std::string>> &required_plugin_versions);
-	void describe_rule(string *rule);
-
-	bool is_source_valid(const std::string &source);
-
-	bool is_format_valid(const std::string &source, const std::string &format, std::string &errstr);
-
-	bool is_defined_field(const std::string &source, const std::string &field);
-
-	static void init(lua_State *ls);
-	static int clear_filters(lua_State *ls);
-	static int create_lua_parser(lua_State *ls);
-	static int add_filter(lua_State *ls);
-	static int enable_rule(lua_State *ls);
-	static int engine_version(lua_State *ls);
-
-	static int is_source_valid(lua_State *ls);
-
-	// err = falco_rules.is_format_valid(source, format_string)
-	static int is_format_valid(lua_State *ls);
-
-	// err = falco_rules.is_defined_field(source, field)
-	static int is_defined_field(lua_State *ls);
-
- private:
-	void clear_filters();
-	// XXX/mstemm can I make this a shared_ptr?
-	lua_parser * create_lua_parser(std::string &source, std::string &errstr);
-	void add_filter(std::shared_ptr<gen_event_filter> filter, string &rule, string &source, std::set<string> &tags);
-	void enable_rule(string &rule, bool enabled);
-
-	falco_engine *m_engine;
-	lua_State* m_ls;
-
-	// Maps from event source to an object that can create rules
-	// for that event source.
-	std::map<std::string, std::shared_ptr<gen_event_filter_factory>> m_filter_factories;
-
-	string m_lua_load_rules = "load_rules";
-	string m_lua_describe_rule = "describe_rule";
-};

userspace/engine/ruleset.cpp

@@ -66,16 +66,14 @@ void falco_ruleset::ruleset_filters::remove_wrapper_from_list(filter_wrapper_lis
 
 void falco_ruleset::ruleset_filters::add_filter(std::shared_ptr<filter_wrapper> wrap)
 {
-	std::set<uint16_t> fevttypes = wrap->evttypes();
-
-	if(fevttypes.empty())
+	if(wrap->evttypes.empty())
 	{
 		// Should run for all event types
 		add_wrapper_to_list(m_filter_all_event_types, wrap);
 	}
 	else
 	{
-		for(auto &etype : fevttypes)
+		for(auto &etype : wrap->evttypes)
 		{
 			if(m_filter_by_event_type.size() <= etype)
 			{
@@ -91,15 +89,13 @@ void falco_ruleset::ruleset_filters::add_filter(std::shared_ptr<filter_wrapper>
 
 void falco_ruleset::ruleset_filters::remove_filter(std::shared_ptr<filter_wrapper> wrap)
 {
-	std::set<uint16_t> fevttypes = wrap->evttypes();
-
-	if(fevttypes.empty())
+	if(wrap->evttypes.empty())
 	{
 		remove_wrapper_from_list(m_filter_all_event_types, wrap);
 	}
 	else
 	{
-		for(auto &etype : fevttypes)
+		for(auto &etype : wrap->evttypes)
 		{
 			if( etype < m_filter_by_event_type.size() )
 			{
@@ -147,14 +143,14 @@ void falco_ruleset::ruleset_filters::evttypes_for_ruleset(std::set<uint16_t> &ev
 
 	for(auto &wrap : m_filters)
 	{
-		auto fevttypes = wrap->evttypes();
-		evttypes.insert(fevttypes.begin(), fevttypes.end());
+		evttypes.insert(wrap->evttypes.begin(), wrap->evttypes.end());
 	}
 }
 
 void falco_ruleset::add(string &source,
 			string &name,
 			set<string> &tags,
+			set<uint16_t> &evttypes,
 			std::shared_ptr<gen_event_filter> filter)
 {
 	std::shared_ptr<filter_wrapper> wrap(new filter_wrapper());
@@ -162,6 +158,7 @@ void falco_ruleset::add(string &source,
 	wrap->name = name;
 	wrap->tags = tags;
 	wrap->filter = filter;
+	wrap->evttypes = evttypes;
 
 	m_filters.insert(wrap);
 }

userspace/engine/ruleset.h

@@ -37,6 +37,7 @@ public:
 	void add(string &source,
 		 std::string &name,
 		 std::set<std::string> &tags,
+		 set<uint16_t> &evttypes,
 		 std::shared_ptr<gen_event_filter> filter);
 
 	// rulesets are arbitrary numbers and should be managed by the caller.
@@ -77,18 +78,8 @@ private:
 		std::string source;
 		std::string name;
 		std::set<std::string> tags;
+		std::set<uint16_t> evttypes;
 		std::shared_ptr<gen_event_filter> filter;
-		std::set<uint16_t> evttypes()
-		{
-			// todo(jasondellaluce,leogr): temp workaround, remove when fixed in libs
-			if(source == "syscall" || source == "k8s_audit")
-			{
-				return filter->evttypes();
-			}
-			// else assume plugins
-			return {ppm_event_type::PPME_PLUGINEVENT_E};
-			// workaround end
-		}
 	};
 
 	typedef std::list<std::shared_ptr<filter_wrapper>> filter_wrapper_list;

userspace/engine/stats_manager.cpp

@@ -0,0 +1,82 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "stats_manager.h"
+#include "falco_common.h"
+
+using namespace std;
+
+void stats_manager::clear()
+{
+	m_total = 0;
+	m_by_rule_id.clear();
+	m_by_priority.clear();
+}
+
+void stats_manager::format(
+	const indexed_vector<falco_rule>& rules,
+	string& out)
+{
+	string fmt;
+	string name;
+	out = "Events detected: " + to_string(m_total) + "\n";
+	out += "Rule counts by severity:\n";
+	for (size_t i = 0; i < m_by_priority.size(); i++)
+	{
+		if (m_by_priority[i] > 0)
+		{
+			falco_common::format_priority(
+				(falco_common::priority_type) i, fmt, true);
+			transform(fmt.begin(), fmt.end(), fmt.begin(), ::toupper);
+			out += "   " + fmt;
+			out += ": " + to_string(m_by_priority[i]) + "\n";
+		}
+	}
+	out += "Triggered rules by rule name:\n";
+	for (size_t i = 0; i < m_by_rule_id.size(); i++)
+	{
+		if (m_by_rule_id[i] > 0)
+		{
+			out += "   " + rules.at(i)->name;
+			out += ": " + to_string(m_by_rule_id[i]) + "\n";
+		}
+	}
+}
+
+void stats_manager::on_event(
+		const indexed_vector<falco_rule>& rules,
+		uint32_t rule_id)
+{
+	auto *rule = rules.at(rule_id);
+	if (!rule)
+	{
+		throw falco_exception(
+			"on_event(): event with invalid rule_id: " + rule_id);
+	}
+	if (m_by_rule_id.size() <= rule_id)
+	{
+		m_by_rule_id.resize(rule_id + 1);
+		m_by_rule_id[rule_id] = 0;
+	}
+	if (m_by_priority.size() <= (size_t) rule->priority)
+	{
+		m_by_priority.resize((size_t) rule->priority + 1);
+		m_by_priority[(size_t) rule->priority] = 0;
+	}
+	m_total++;
+	m_by_rule_id[rule_id]++;
+	m_by_priority[(size_t) rule->priority]++;
+}

userspace/engine/stats_manager.h

@@ -0,0 +1,53 @@
+/*
+Copyright (C) 2022 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+#include <vector>
+#include <string>
+#include "falco_rule.h"
+#include "indexed_vector.h"
+
+/*!
+	\brief Manager for the internal statistics of the rule engine
+*/
+class stats_manager
+{
+public:
+	/*!
+		\brief Erases the internal state and statistics data
+	*/
+	virtual void clear();
+
+	/*!
+		\brief Callback for when a rule with a given index matches an event
+	*/
+	virtual void on_event(
+		const indexed_vector<falco_rule>& rules,
+		uint32_t index);
+
+	/*!
+		\brief Formats the internal statistics into the out string
+	*/
+	virtual void format(
+		const indexed_vector<falco_rule>& rules,
+		std::string& out);
+
+private:
+	uint64_t m_total;
+	std::vector<uint64_t> m_by_priority;
+	std::vector<uint64_t> m_by_rule_id;
+};

userspace/falco/CMakeLists.txt

@@ -44,11 +44,7 @@ set(
 set(
   FALCO_DEPENDENCIES
   string-view-lite
-  libyaml
   b64
-  luajit
-  lpeg
-  lyaml
   cxxopts
 )
 
@@ -56,7 +52,6 @@ set(
   FALCO_LIBRARIES
   falco_engine
   sinsp
-  "${LIBYAML_LIB}"
   "${YAMLCPP_LIB}"
 )
 
@@ -108,7 +103,6 @@ if(NOT MINIMAL_BUILD)
     "${PROTOBUF_LIB}"
     "${CARES_LIB}"
     "${OPENSSL_LIBRARIES}"
-    "${LIBYAML_LIB}"
     "${YAMLCPP_LIB}"
     "${CIVETWEB_LIB}"
     "${CIVETWEB_CPP_LIB}"

userspace/falco/configuration.cpp

@@ -190,17 +190,10 @@ void falco_configuration::init(string conf_filename, const vector<string> &cmdli
 	m_notifications_max_burst = m_config->get_scalar<uint32_t>("outputs.max_burst", 1000);
 
 	string priority = m_config->get_scalar<string>("priority", "debug");
-	vector<string>::iterator it;
-
-	auto comp = [priority](string &s) {
-		return (strcasecmp(s.c_str(), priority.c_str()) == 0);
-	};
-
-	if((it = std::find_if(falco_common::priority_names.begin(), falco_common::priority_names.end(), comp)) == falco_common::priority_names.end())
+	if (!falco_common::parse_priority(priority, m_min_priority))
 	{
 		throw logic_error("Unknown priority \"" + priority + "\"--must be one of emergency, alert, critical, error, warning, notice, informational, debug");
 	}
-	m_min_priority = (falco_common::priority_type)(it - falco_common::priority_names.begin());
 
 	m_buffered_outputs = m_config->get_scalar<bool>("buffered_outputs", false);
 	m_time_format_iso_8601 = m_config->get_scalar<bool>("time_format_iso_8601", false);

userspace/falco/falco.cpp

@@ -61,8 +61,10 @@ bool g_reopen_outputs = false;
 bool g_restart = false;
 bool g_daemonized = false;
 
-static std::string syscall_source = "syscall";
+static std::string syscall_source = falco_common::syscall_source;
+static std::size_t syscall_source_idx;
 static std::string k8s_audit_source = "k8s_audit";
+static std::size_t k8s_audit_source_idx;
 
 //
 // Helper functions
@@ -118,7 +120,7 @@ void read_k8s_audit_trace_file(falco_engine *engine,
 			continue;
 		}
 
-		if(!k8s_audit_handler::accept_data(engine, outputs, line, errstr))
+		if(!k8s_audit_handler::accept_data(engine, outputs, k8s_audit_source_idx, line, errstr))
 		{
 			falco_logger::log(LOG_ERR, "Could not read k8s audit event line #" + to_string(line_num) + ", \"" + line + "\": " + errstr + ", stopping");
 			return;
@@ -140,16 +142,16 @@ static std::string read_file(std::string filename)
 // Event processing loop
 //
 uint64_t do_inspect(falco_engine *engine,
-			falco_outputs *outputs,
-			sinsp* inspector,
-		        std::string &event_source,
-			falco_configuration &config,
-			syscall_evt_drop_mgr &sdropmgr,
-			uint64_t duration_to_tot_ns,
-			string &stats_filename,
-			uint64_t stats_interval,
-			bool all_events,
-			int &result)
+		    falco_outputs *outputs,
+		    sinsp *inspector,
+		    std::size_t event_source_idx,
+		    falco_configuration &config,
+		    syscall_evt_drop_mgr &sdropmgr,
+		    uint64_t duration_to_tot_ns,
+		    string &stats_filename,
+		    uint64_t stats_interval,
+		    bool all_events,
+		    int &result)
 {
 	uint64_t num_evts = 0;
 	int32_t rc;
@@ -207,7 +209,7 @@ uint64_t do_inspect(falco_engine *engine,
 			if(unlikely(ev == nullptr))
 			{
 				timeouts_since_last_success_or_msg++;
-				if(event_source == syscall_source &&
+				if(event_source_idx == syscall_source_idx &&
 				   (timeouts_since_last_success_or_msg > config.m_syscall_evt_timeout_max_consecutives))
 				{
 					std::string rule = "Falco internal: timeouts notification";
@@ -272,7 +274,7 @@ uint64_t do_inspect(falco_engine *engine,
 		// engine, which will match the event against the set
 		// of rules. If a match is found, pass the event to
 		// the outputs.
-		unique_ptr<falco_engine::rule_result> res = engine->process_event(event_source, ev);
+		unique_ptr<falco_engine::rule_result> res = engine->process_event(event_source_idx, ev);
 		if(res)
 		{
 			outputs->handle_event(res->evt, res->rule, res->source, res->priority_num, res->format, res->tags);
@@ -517,8 +519,8 @@ int falco_init(int argc, char **argv)
 		std::shared_ptr<gen_event_formatter_factory> syscall_formatter_factory(new sinsp_evt_formatter_factory(inspector));
 		std::shared_ptr<gen_event_formatter_factory> k8s_audit_formatter_factory(new json_event_formatter_factory(k8s_audit_filter_factory));
 
-		engine->add_source(syscall_source, syscall_filter_factory, syscall_formatter_factory);
-		engine->add_source(k8s_audit_source, k8s_audit_filter_factory, k8s_audit_formatter_factory);
+		syscall_source_idx = engine->add_source(syscall_source, syscall_filter_factory, syscall_formatter_factory);
+		k8s_audit_source_idx = engine->add_source(k8s_audit_source, k8s_audit_filter_factory, k8s_audit_formatter_factory);
 
 		for(const auto &src : app.options().disable_sources)
 		{
@@ -581,6 +583,7 @@ int falco_init(int argc, char **argv)
 		// plugin was found, the source is the source of that
 		// plugin.
 		std::string event_source = syscall_source;
+		std::size_t event_source_idx = syscall_source_idx;
 
 		// All filterchecks created by plugins go in this
 		// list. If we ever support multiple event sources at
@@ -628,7 +631,7 @@ int falco_init(int argc, char **argv)
 					inspector->set_input_plugin_open_params(p.m_open_params.c_str());
 				}
 
-				engine->add_source(event_source, plugin_filter_factory, plugin_formatter_factory);
+				event_source_idx = engine->add_source(event_source, plugin_filter_factory, plugin_formatter_factory);
 
 			} else {
 				extractor_plugins.push_back(plugin);
@@ -760,6 +763,9 @@ int falco_init(int argc, char **argv)
 			}
 		}
 
+		// Free-up memory for the rule loader, which is not used from now on
+		engine->clear_loader();
+
 		for (auto substring : app.options().disabled_rule_substrings)
 		{
 			falco_logger::log(LOG_INFO, "Disabling rules matching substring: " + substring + "\n");
@@ -1151,7 +1157,7 @@ int falco_init(int argc, char **argv)
 		{
 			std::string ssl_option = (config.m_webserver_ssl_enabled ? " (SSL)" : "");
 			falco_logger::log(LOG_INFO, "Starting internal webserver, listening on port " + to_string(config.m_webserver_listen_port) + ssl_option + "\n");
-			webserver.init(&config, engine, outputs);
+			webserver.init(&config, engine, outputs, k8s_audit_source_idx);
 			webserver.start();
 		}
 
@@ -1190,7 +1196,7 @@ int falco_init(int argc, char **argv)
 			num_evts = do_inspect(engine,
 					      outputs,
 					      inspector,
-					      event_source,
+					      event_source_idx,
 					      config,
 					      sdropmgr,
 					      uint64_t(app.options().duration_to_tot*ONE_SECOND_IN_NS),

userspace/falco/falco_outputs.cpp

@@ -159,24 +159,25 @@ void falco_outputs::handle_event(gen_event *evt, string &rule, string &source,
 	{
 		if(m_time_format_iso_8601)
 		{
-			sformat = "*%evt.time.iso8601: " + falco_common::priority_names[priority];
+			sformat = "*%evt.time.iso8601: ";
 		}
 		else
 		{
-			sformat = "*%evt.time: " + falco_common::priority_names[priority];
+			sformat = "*%evt.time: ";
 		}
 	}
 	else
 	{
 		if(m_time_format_iso_8601)
 		{
-			sformat = "*%jevt.time.iso8601: " + falco_common::priority_names[priority];
+			sformat = "*%jevt.time.iso8601: ";
 		}
 		else
 		{
-			sformat = "*%jevt.time: " + falco_common::priority_names[priority];
+			sformat = "*%jevt.time: ";
 		}
 	}
+	sformat += falco_common::format_priority(priority);
 
 	// if format starts with a *, remove it, as we added our own prefix
 	if(format[0] == '*')
@@ -188,7 +189,7 @@ void falco_outputs::handle_event(gen_event *evt, string &rule, string &source,
 		sformat += " " + format;
 	}
 
-	cmsg.msg = m_formats->format_event(evt, rule, source, falco_common::priority_names[priority], sformat, tags);
+	cmsg.msg = m_formats->format_event(evt, rule, source, falco_common::format_priority(priority), sformat, tags);
 	cmsg.fields = m_formats->get_field_values(evt, source, sformat);
 	cmsg.tags.insert(tags.begin(), tags.end());
 
@@ -225,7 +226,7 @@ void falco_outputs::handle_msg(uint64_t ts,
 		iso8601evttime += time_ns;
 
 		jmsg["output"] = msg;
-		jmsg["priority"] = falco_common::priority_names[priority];
+		jmsg["priority"] = falco_common::format_priority(priority);
 		jmsg["rule"] = rule;
 		jmsg["time"] = iso8601evttime;
 		jmsg["output_fields"] = output_fields;
@@ -238,7 +239,7 @@ void falco_outputs::handle_msg(uint64_t ts,
 		bool first = true;
 
 		sinsp_utils::ts_to_string(ts, &timestr, false, true);
-		cmsg.msg = timestr + ": " + falco_common::priority_names[priority] + " " + msg + " (";
+		cmsg.msg = timestr + ": " + falco_common::format_priority(priority) + " " + msg + " (";
 		for(auto &pair : output_fields)
 		{
 			if(first)

userspace/falco/outputs_grpc.cpp

@@ -67,7 +67,7 @@ void falco::outputs::output_grpc::output(const message *msg)
 
 	// priority
 	falco::schema::priority p = falco::schema::priority::EMERGENCY;
-	if(!falco::schema::priority_Parse(falco_common::priority_names[msg->priority], &p))
+	if(!falco::schema::priority_Parse(falco_common::format_priority(msg->priority), &p))
 	{
 		throw falco_exception("Unknown priority passed to output_grpc::output()");
 	}

userspace/falco/webserver.cpp

@@ -25,10 +25,8 @@ limitations under the License.
 using json = nlohmann::json;
 using namespace std;
 
-string k8s_audit_handler::m_k8s_audit_event_source = "k8s_audit";
-
-k8s_audit_handler::k8s_audit_handler(falco_engine *engine, falco_outputs *outputs):
-	m_engine(engine), m_outputs(outputs)
+k8s_audit_handler::k8s_audit_handler(falco_engine *engine, falco_outputs *outputs, std::size_t k8s_audit_event_source_idx):
+	m_engine(engine), m_outputs(outputs), m_k8s_audit_event_source_idx(k8s_audit_event_source_idx)
 {
 }
 
@@ -47,6 +45,7 @@ bool k8s_healthz_handler::handleGet(CivetServer *server, struct mg_connection *c
 
 bool k8s_audit_handler::accept_data(falco_engine *engine,
 				    falco_outputs *outputs,
+				    std::size_t k8s_audit_event_source_idx,
 				    std::string &data,
 				    std::string &errstr)
 {
@@ -89,7 +88,7 @@ bool k8s_audit_handler::accept_data(falco_engine *engine,
 
 		try
 		{
-			res = engine->process_event(m_k8s_audit_event_source, &jev);
+			res = engine->process_event(k8s_audit_event_source_idx, &jev);
 		}
 		catch(...)
 		{
@@ -120,7 +119,7 @@ bool k8s_audit_handler::accept_data(falco_engine *engine,
 
 bool k8s_audit_handler::accept_uploaded_data(std::string &post_data, std::string &errstr)
 {
-	return k8s_audit_handler::accept_data(m_engine, m_outputs, post_data, errstr);
+	return k8s_audit_handler::accept_data(m_engine, m_outputs, m_k8s_audit_event_source_idx, post_data, errstr);
 }
 
 bool k8s_audit_handler::handleGet(CivetServer *server, struct mg_connection *conn)
@@ -189,11 +188,13 @@ falco_webserver::~falco_webserver()
 
 void falco_webserver::init(falco_configuration *config,
 			   falco_engine *engine,
-			   falco_outputs *outputs)
+			   falco_outputs *outputs,
+			   std::size_t k8s_audit_event_source_idx)
 {
 	m_config = config;
 	m_engine = engine;
 	m_outputs = outputs;
+	m_k8s_audit_event_source_idx = k8s_audit_event_source_idx;
 }
 
 template<typename T, typename... Args>
@@ -253,7 +254,7 @@ void falco_webserver::start()
 		throw falco_exception("Could not create embedded webserver");
 	}
 
-	m_k8s_audit_handler = make_unique<k8s_audit_handler>(m_engine, m_outputs);
+	m_k8s_audit_handler = make_unique<k8s_audit_handler>(m_engine, m_outputs, m_k8s_audit_event_source_idx);
 	m_server->addHandler(m_config->m_webserver_k8s_audit_endpoint, *m_k8s_audit_handler);
 	m_k8s_healthz_handler = make_unique<k8s_healthz_handler>();
 	m_server->addHandler(m_config->m_webserver_k8s_healthz_endpoint, *m_k8s_healthz_handler);

userspace/falco/webserver.h

@@ -25,7 +25,7 @@ limitations under the License.
 class k8s_audit_handler : public CivetHandler
 {
 public:
-	k8s_audit_handler(falco_engine *engine, falco_outputs *outputs);
+	k8s_audit_handler(falco_engine *engine, falco_outputs *outputs, std::size_t k8s_audit_event_source_idx);
 	virtual ~k8s_audit_handler();
 
 	bool handleGet(CivetServer *server, struct mg_connection *conn);
@@ -33,13 +33,13 @@ public:
 
 	static bool accept_data(falco_engine *engine,
 				falco_outputs *outputs,
+				std::size_t k8s_audit_event_source_idx,
 				std::string &post_data, std::string &errstr);
 
-	static std::string m_k8s_audit_event_source;
-
 private:
 	falco_engine *m_engine;
 	falco_outputs *m_outputs;
+	std::size_t m_k8s_audit_event_source_idx;
 	bool accept_uploaded_data(std::string &post_data, std::string &errstr);
 };
 
@@ -65,7 +65,8 @@ public:
 
 	void init(falco_configuration *config,
 		  falco_engine *engine,
-		  falco_outputs *outputs);
+		  falco_outputs *outputs,
+		  std::size_t k8s_audit_event_source_idx);
 
 	void start();
 	void stop();
@@ -74,6 +75,7 @@ private:
 	falco_engine *m_engine;
 	falco_configuration *m_config;
 	falco_outputs *m_outputs;
+	std::size_t m_k8s_audit_event_source_idx;
 	unique_ptr<CivetServer> m_server;
 	unique_ptr<k8s_audit_handler> m_k8s_audit_handler;
 	unique_ptr<k8s_healthz_handler> m_k8s_healthz_handler;