wip

Signed-off-by: Federico Di Pierro <nierro92@gmail.com>

.github/workflows/ci.yml

@@ -16,12 +16,22 @@ jobs:
   fetch-version:
     uses: ./.github/workflows/reusable_fetch_version.yaml
 
-  build-dev-packages:
+  build-dev-packages-sanitizers-x86_64:
     needs: [fetch-version]
     uses: ./.github/workflows/reusable_build_packages.yaml
     with:
       arch: x86_64
       version: ${{ needs.fetch-version.outputs.version }}
+      build_type: Debug
+      sanitizers: true
+
+  build-dev-packages-x86_64:
+    needs: [fetch-version]
+    uses: ./.github/workflows/reusable_build_packages.yaml
+    with:
+      arch: x86_64
+      version: ${{ needs.fetch-version.outputs.version }}
+      build_type: Release
 
   build-dev-packages-arm64:
     needs: [fetch-version]
@@ -29,21 +39,26 @@ jobs:
     with:
       arch: aarch64
       version: ${{ needs.fetch-version.outputs.version }}
+      build_type: Debug
+      sanitizers: false
 
   test-dev-packages:
-    needs: [fetch-version, build-dev-packages]
+    needs: [fetch-version, build-dev-packages-sanitizers-x86_64]
     uses: ./.github/workflows/reusable_test_packages.yaml
-    strategy:
-      fail-fast: false
-      matrix:
-        static: ["static", ""]
+    # The musl build job is currently disabled because we link libelf dynamically and it is
+    # not possible to dynamically link with musl
+    # strategy:
+    #   fail-fast: false
+    #   matrix:
+    #     static: ["static", ""]
     with:
       arch: x86_64
-      static: ${{ matrix.static != '' && true || false }}
+      sanitizers: true
+      # static: ${{ matrix.static != '' && true || false }}
       version: ${{ needs.fetch-version.outputs.version }}
 
   test-dev-packages-arm64:
-    needs: [fetch-version, build-dev-packages]
+    needs: [fetch-version, build-dev-packages-arm64]
     uses: ./.github/workflows/reusable_test_packages.yaml
     strategy:
       fail-fast: false
@@ -80,6 +95,7 @@ jobs:
       arch: x86_64
       git_ref: ${{ github.event.pull_request.head.sha }}
       minimal: false
+      sanitizers: true
       build_type: Debug
       cmd: "echo $(build/userspace/falco/falco -c ./falco.yaml --version | grep 'Engine:' | awk '{print $2}') $(echo $(build/userspace/falco/falco -c ./falco.yaml --version | grep 'Schema version:' | awk '{print $3}') $(build/userspace/falco/falco -c ./falco.yaml --list --markdown | grep '^`' | sort) $(build/userspace/falco/falco -c ./falco.yaml --list-events | sort) | sha256sum)"
 

.github/workflows/codeql.yaml

@@ -21,7 +21,7 @@ on:
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     permissions:
       actions: read
       contents: read
@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@ddccb873888234080b77e9bc2d4764d5ccaaccf9 # v2.21.9
+      uses: github/codeql-action/init@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -60,16 +60,11 @@ jobs:
 
     - name: Prepare project
       run: |
-          mkdir build
-          pushd build
-          cmake -DBUILD_BPF=On ..
-          popd
+          cmake -B build -S . -DBUILD_BPF=On -DUSE_BUNDLED_DEPS=Off -DUSE_BUNDLED_NLOHMANN_JSON=On -DUSE_BUNDLED_CXXOPTS=On -DUSE_BUNDLED_CPPHTTPLIB=On
 
     - name: Build
       run: |
-          pushd build
-          KERNELDIR=/lib/modules/$(uname -r)/build make -j4 all
-          popd
+          KERNELDIR=/lib/modules/$(uname -r)/build cmake --build build -j4
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@ddccb873888234080b77e9bc2d4764d5ccaaccf9 # v2.21.9
+      uses: github/codeql-action/analyze@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5

.github/workflows/master.yaml

@@ -31,13 +31,15 @@ jobs:
   test-dev-packages:
     needs: [fetch-version, build-dev-packages]
     uses: ./.github/workflows/reusable_test_packages.yaml
-    strategy:
-      fail-fast: false
-      matrix:
-        static: ["static", ""]
+    # The musl build job is currently disabled because we link libelf dynamically and it is
+    # not possible to dynamically link with musl
+    # strategy:
+    #   fail-fast: false
+    #   matrix:
+    #     static: ["static", ""]
     with:
       arch: x86_64
-      static: ${{ matrix.static != '' && true || false }}
+      # static: ${{ matrix.static != '' && true || false }}
       version: ${{ needs.fetch-version.outputs.version }}
   
   test-dev-packages-arm64:

.github/workflows/release.yaml

@@ -69,13 +69,16 @@ jobs:
   test-packages:
     needs: [release-settings, build-packages]
     uses: ./.github/workflows/reusable_test_packages.yaml
-    strategy:
-      fail-fast: false
-      matrix:
-        static: ["static", ""]
+
+    # The musl build job is currently disabled because we link libelf dynamically and it is
+    # not possible to dynamically link with musl
+    # strategy:
+    #   fail-fast: false
+    #   matrix:
+    #     static: ["static", ""]
     with:
       arch: x86_64
-      static: ${{ matrix.static != '' && true || false }}
+      # static: ${{ matrix.static != '' && true || false }}
       version: ${{ github.event.release.tag_name }}
   
   test-packages-arm64:
@@ -137,7 +140,7 @@ jobs:
         run: |
           cp .github/release_template.md release-body.md
           LIBS_VERS=$(cat cmake/modules/falcosecurity-libs.cmake | grep 'set(FALCOSECURITY_LIBS_VERSION' | tail -n1 | grep -o '[[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*')
-          DRIVER_VERS=$(cat cmake/modules/driver.cmake | grep 'set(DRIVER_VERSION' | tail -n1 | grep -o '[[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*')
+          DRIVER_VERS=$(cat cmake/modules/driver.cmake | grep 'set(DRIVER_VERSION' | tail -n1 | grep -o '[[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*+driver')
           sed -i s/LIBSVER/$LIBS_VERS/g release-body.md
           sed -i s/DRIVERVER/$DRIVER_VERS/g release-body.md
           
@@ -147,7 +150,7 @@ jobs:
           sed -i s/FALCOVER/${{ github.event.release.tag_name }}/g release-body.md
           
       - name: Generate release notes
-        uses: leodido/rn2md@1378404a058ecf86701f3ab533d487333fc675a7
+        uses: leodido/rn2md@9c351d81278644c0e17b1ca68edbdba305276c73
         with:
           milestone: ${{ github.event.release.tag_name }}
           output: ./notes.md

.github/workflows/reusable_build_dev.yaml

@@ -14,6 +14,11 @@ on:
         description: Minimal build
         required: true
         type: boolean
+      sanitizers:
+        description: Enable sanitizer support
+        required: false
+        default: false
+        type: boolean
       build_type:
         description: One of 'Debug' or 'Release'
         required: true
@@ -53,22 +58,22 @@ jobs:
 
       - name: Prepare project
         run: |
-          mkdir build
-          pushd build
-          cmake \
+          cmake -B build -S .\
             -DBUILD_FALCO_UNIT_TESTS=On \
             -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} \
             -DBUILD_BPF=${{ inputs.minimal == true && 'OFF' || 'ON' }} \
             -DBUILD_DRIVER=${{ inputs.minimal == true && 'OFF' || 'ON' }} \
             -DMINIMAL_BUILD=${{ inputs.minimal == true && 'ON' || 'OFF' }} \
-            ..
-          popd
+            -DUSE_ASAN=${{ inputs.sanitizers == true && 'ON' || 'OFF' }} \
+            -DUSE_UBSAN=${{ inputs.sanitizers == true && 'ON' || 'OFF' }} \
+            -DUSE_BUNDLED_DEPS=Off \
+            -DUSE_BUNDLED_NLOHMANN_JSON=On \
+            -DUSE_BUNDLED_CXXOPTS=On \
+            -DUSE_BUNDLED_CPPHTTPLIB=On \
 
       - name: Build
         run: |
-          pushd build
-          KERNELDIR=/lib/modules/$(uname -r)/build make -j4 all
-          popd
+          KERNELDIR=/lib/modules/$(uname -r)/build cmake --build build -j4
 
       - name: Run unit tests
         run: |

.github/workflows/reusable_build_packages.yaml

@@ -10,6 +10,16 @@ on:
         description: The Falco version to use when building packages
         required: true
         type: string
+      build_type:
+        description: The build type
+        required: false
+        type: string
+        default: 'Release'
+      sanitizers:
+        description: enable sanitizer support
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   build-modern-bpf-skeleton:
@@ -20,24 +30,24 @@ jobs:
       # Always install deps before invoking checkout action, to properly perform a full clone.
       - name: Install build dependencies
         run: |
-          dnf install -y bpftool ca-certificates cmake make automake gcc gcc-c++ kernel-devel clang git pkg-config autoconf automake libbpf-devel
-    
+          dnf install -y bpftool ca-certificates cmake make automake gcc gcc-c++ kernel-devel clang git pkg-config autoconf automake libbpf-devel elfutils-libelf-devel
+
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
 
       - name: Build modern BPF skeleton
         run: |
-          mkdir skeleton-build && cd skeleton-build
-          cmake -DUSE_BUNDLED_DEPS=ON -DBUILD_FALCO_MODERN_BPF=ON -DCREATE_TEST_TARGETS=Off -DFALCO_VERSION=${{ inputs.version }} ..
-          make ProbeSkeleton -j6
-          
+          cmake -B skeleton-build -S . \
+                -DUSE_BUNDLED_DEPS=ON -DBUILD_FALCO_MODERN_BPF=ON -DCREATE_TEST_TARGETS=Off -DFALCO_VERSION=${{ inputs.version }}
+          cmake --build skeleton-build --target ProbeSkeleton -j6
+
       - name: Upload skeleton
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
           name: bpf_probe_${{ inputs.arch }}.skel.h
           path: skeleton-build/skel_dir/bpf_probe.skel.h
           retention-days: 1
-          
+
   build-packages:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
     runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
@@ -50,77 +60,74 @@ jobs:
           yum -y install centos-release-scl
           yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++
           source /opt/rh/devtoolset-9/enable
-          yum install -y wget git make m4 rpm-build perl-IPC-Cmd
-    
+          yum install -y wget git make m4 rpm-build elfutils-libelf-devel perl-IPC-Cmd devtoolset-9-libasan-devel devtoolset-9-libubsan-devel
+
       - name: Checkout
         # It is not possible to upgrade the checkout action to versions >= v4.0.0 because of incompatibilities with centos 7's libc.
         uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
-    
+
       - name: Download skeleton
         uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           name: bpf_probe_${{ inputs.arch }}.skel.h
           path: /tmp
-          
+
       - name: Install updated cmake
         run: |
-          curl -L -o /tmp/cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.22.5/cmake-3.22.5-linux-$(uname -m).tar.gz
-          gzip -d /tmp/cmake.tar.gz
-          tar -xpf /tmp/cmake.tar --directory=/tmp
-          cp -R /tmp/cmake-3.22.5-linux-$(uname -m)/* /usr
-          rm -rf /tmp/cmake-3.22.5-linux-$(uname -m)
-    
+          curl -L https://github.com/Kitware/CMake/releases/download/v3.22.5/cmake-3.22.5-linux-$(uname -m).tar.gz \
+            | tar --directory=/usr --strip-components=1 -xzp
+
       - name: Prepare project
         run: |
-          mkdir build && cd build
           source /opt/rh/devtoolset-9/enable
-          cmake \
-              -DCMAKE_BUILD_TYPE=Release \
+          cmake -B build -S . \
+              -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} \
               -DUSE_BUNDLED_DEPS=On \
               -DFALCO_ETC_DIR=/etc/falco \
               -DBUILD_FALCO_MODERN_BPF=ON \
               -DMODERN_BPF_SKEL_DIR=/tmp \
               -DBUILD_DRIVER=Off \
               -DBUILD_BPF=Off \
-              -DFALCO_VERSION=${{ inputs.version }} \
-              ..
-              
+              -DUSE_ASAN=${{ (inputs.sanitizers == true && inputs.arch == 'x86_64' && 'ON') || 'OFF' }} \
+              -DFALCO_VERSION=${{ inputs.version }}
+
       - name: Build project
         run: |
-          cd build
           source /opt/rh/devtoolset-9/enable
-          make falco -j6
-      
+          cmake --build build --target falco -j6
+
       - name: Build packages
         run: |
-          cd build
           source /opt/rh/devtoolset-9/enable
-          make package
+          cmake --build build --target package
 
       - name: Upload Falco tar.gz package
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
-          name: falco-${{ inputs.version }}-${{ inputs.arch }}.tar.gz
+          name: falco-${{ inputs.version }}-${{ inputs.arch }}${{ inputs.sanitizers == true && '-sanitizers' || '' }}.tar.gz
           path: |
             ${{ github.workspace }}/build/falco-*.tar.gz
-            
+
       - name: Upload Falco deb package
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
-          name: falco-${{ inputs.version }}-${{ inputs.arch }}.deb
+          name: falco-${{ inputs.version }}-${{ inputs.arch }}${{ inputs.sanitizers == true && '-sanitizers' || '' }}.deb
           path: |
             ${{ github.workspace }}/build/falco-*.deb
-      
+
       - name: Upload Falco rpm package
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
-          name: falco-${{ inputs.version }}-${{ inputs.arch }}.rpm
+          name: falco-${{ inputs.version }}-${{ inputs.arch }}${{ inputs.sanitizers == true && '-sanitizers' || '' }}.rpm
           path: |
             ${{ github.workspace }}/build/falco-*.rpm
-            
+
+  # The musl build job is currently disabled because we link libelf dynamically and it is
+  # not possible to dynamically link with musl
   build-musl-package:
     # x86_64 only for now
-    if: ${{ inputs.arch == 'x86_64' }}
+    # if: ${{ inputs.arch == 'x86_64' }}
+    if: false
     runs-on: ubuntu-latest
     container: alpine:3.17
     steps:
@@ -128,32 +135,33 @@ jobs:
       - name: Install build dependencies
         run: |
           apk add g++ gcc cmake make git bash perl linux-headers autoconf automake m4 libtool elfutils-dev libelf-static patch binutils bpftool clang
-    
+
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
         with:
           fetch-depth: 0
-          
+
       - name: Prepare project
         run: |
-          mkdir build && cd build
-          cmake -DCPACK_GENERATOR=TGZ -DBUILD_BPF=Off -DBUILD_DRIVER=Off -DCMAKE_BUILD_TYPE=Release -DUSE_BUNDLED_DEPS=On -DUSE_BUNDLED_LIBELF=Off -DBUILD_LIBSCAP_MODERN_BPF=ON -DMUSL_OPTIMIZED_BUILD=On -DFALCO_ETC_DIR=/etc/falco ../ -DFALCO_VERSION=${{ inputs.version }}
-          
+          cmake -B build -S . \
+                -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} \
+                -DCPACK_GENERATOR=TGZ \
+                -DBUILD_BPF=Off -DBUILD_DRIVER=Off \
+                -DUSE_BUNDLED_DEPS=On -DUSE_BUNDLED_LIBELF=Off -DBUILD_LIBSCAP_MODERN_BPF=ON -DMUSL_OPTIMIZED_BUILD=On -DFALCO_ETC_DIR=/etc/falco -DFALCO_VERSION=${{ inputs.version }}
+
       - name: Build project
         run: |
-          cd build
-          make -j6 all
-      
+          cmake --build build -j6
+
       - name: Build packages
         run: |
-          cd build
-          make -j6 package
+          cmake --build build -j6 --target package
 
       - name: Rename static package
         run: |
           cd build
-          mv falco-${{ inputs.version }}-x86_64.tar.gz falco-${{ inputs.version }}-static-x86_64.tar.gz 
-          
+          mv falco-${{ inputs.version }}-x86_64.tar.gz falco-${{ inputs.version }}-static-x86_64.tar.gz
+
       - name: Upload Falco static package
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
@@ -175,41 +183,39 @@ jobs:
         uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1
         with:
           node-version: 14
-    
+
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
         with:
           fetch-depth: 0
-          
+
       - name: Prepare project
         run: |
-          mkdir build && cd build
-          emcmake cmake \
+          emcmake cmake -B build -S . \
             -DBUILD_BPF=Off \
             -DBUILD_DRIVER=Off \
             -DBUILD_LIBSCAP_MODERN_BPF=OFF \
-            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} \
             -DUSE_BUNDLED_DEPS=On \
             -DFALCO_ETC_DIR=/etc/falco \
             -DBUILD_FALCO_UNIT_TESTS=On \
-            -DFALCO_VERSION=${{ inputs.version }} \
-            ..
-          
+            -DFALCO_VERSION=${{ inputs.version }}
+
       - name: Build project
         run: |
           cd build
           emmake make -j6 all
-      
+
       - name: Run unit Tests
         run: |
           cd build
           node ./unit_tests/falco_unit_tests.js
-      
+
       - name: Build packages
         run: |
           cd build
           emmake make -j6 package
-          
+
       - name: Upload Falco WASM package
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
@@ -226,19 +232,18 @@ jobs:
         with:
           fetch-depth: 0
 
+      # NOTE: Backslash doesn't work as line continuation on Windows.
       - name: Prepare project
         run: |
-          mkdir build
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=Release -DMINIMAL_BUILD=On -DUSE_BUNDLED_DEPS=On -DBUILD_FALCO_UNIT_TESTS=On -DFALCO_VERSION=${{ inputs.version }} ..
+          cmake -B build -S . -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} -DMINIMAL_BUILD=On -DUSE_BUNDLED_DEPS=On -DBUILD_FALCO_UNIT_TESTS=On -DFALCO_VERSION=${{ inputs.version }}
 
       - name: Build project
         run: |
-          cmake --build build --target package --config Release
+          cmake --build build --target package --config ${{ inputs.build_type }}
 
       - name: Run unit Tests
         run: |
-          build/unit_tests/Release/falco_unit_tests.exe
+          build/unit_tests/${{ inputs.build_type }}/falco_unit_tests.exe
 
       - name: Upload Falco win32 installer
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
@@ -251,7 +256,7 @@ jobs:
         with:
           name: falco-${{ inputs.version }}-win32.exe
           path: |
-            ${{ github.workspace }}/build/userspace/falco/Release/falco.exe
+            ${{ github.workspace }}/build/userspace/falco/${{ inputs.build_type }}/falco.exe
 
   build-macos-package:
     if: ${{ inputs.arch == 'x86_64' }}
@@ -264,9 +269,8 @@ jobs:
 
       - name: Prepare project
         run: |
-          mkdir build
-          cd build
-          cmake -DMINIMAL_BUILD=On -DUSE_BUNDLED_DEPS=On -DBUILD_FALCO_UNIT_TESTS=On -DFALCO_VERSION=${{ inputs.version }} ..
+          cmake -B build -S . \
+                -DMINIMAL_BUILD=On -DUSE_BUNDLED_DEPS=On -DBUILD_FALCO_UNIT_TESTS=On -DFALCO_VERSION=${{ inputs.version }}
 
       - name: Build project
         run: |

.github/workflows/reusable_fetch_version.yaml

@@ -5,34 +5,33 @@ on:
       version:
         description: "Falco version"
         value: ${{ jobs.fetch-version.outputs.version }}
-    
+
 jobs:
   # We need to use an ubuntu-latest to fetch Falco version because
   # Falco version is computed by some cmake scripts that do git sorceries
   # to get the current version.
-  # But centos7 jobs have a git version too old and actions/checkout does not 
+  # But centos7 jobs have a git version too old and actions/checkout does not
   # fully clone the repo, but uses http rest api instead.
   fetch-version:
     runs-on: ubuntu-latest
     # Map the job outputs to step outputs
     outputs:
-      version: ${{ steps.store_version.outputs.version }}  
+      version: ${{ steps.store_version.outputs.version }}
     steps:
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
         with:
           fetch-depth: 0
-          
+
       - name: Install build dependencies
         run: |
-          sudo apt update 
+          sudo apt update
           sudo apt install -y cmake build-essential
-      
+
       - name: Configure project
         run: |
-          mkdir build && cd build
-          cmake -DUSE_BUNDLED_DEPS=On ..
-          
+          cmake -B build -S . -DUSE_BUNDLED_DEPS=On -DUSE_DYNAMIC_LIBELF=Off
+
       - name: Load and store Falco version output
         id: store_version
         run: |

.github/workflows/reusable_publish_packages.yaml

@@ -65,12 +65,21 @@ jobs:
           name: falco-${{ inputs.version }}-aarch64.tar.gz
           path: /tmp/falco-build-bin
 
+      # The musl build job is currently disabled because we link libelf dynamically and it is
+      # not possible to dynamically link with musl
       - name: Download static binary x86_64
+        if: false
         uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
           name: falco-${{ inputs.version }}-static-x86_64.tar.gz
           path: /tmp/falco-build-bin-static
 
+      - name: Download WASM package
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: falco-${{ inputs.version }}-wasm.tar.gz
+          path: /tmp/falco-wasm
+
       - name: Import gpg key
         env:
           GPG_KEY: ${{ secrets.GPG_KEY }}
@@ -81,6 +90,10 @@ jobs:
           rpmsign --define '_gpg_name Falcosecurity Package Signing' --addsign /tmp/falco-build-rpm/falco-*.rpm
           rpm --qf %{SIGPGP:pgpsig} -qp /tmp/falco-build-rpm/falco-*.rpm | grep SHA256
 
+      - name: Publish wasm
+        run: |
+          ./scripts/publish-wasm -f /tmp/falco-wasm/falco-${{ inputs.version }}-wasm.tar.gz
+          
       - name: Publish rpm
         run: |
           ./scripts/publish-rpm -f /tmp/falco-build-rpm/falco-${{ inputs.version }}-x86_64.rpm -f /tmp/falco-build-rpm/falco-${{ inputs.version }}-aarch64.rpm -r rpm${{ inputs.bucket_suffix }}
@@ -89,11 +102,14 @@ jobs:
         run: |
           ./scripts/publish-bin -f /tmp/falco-build-bin/falco-${{ inputs.version }}-x86_64.tar.gz -r bin${{ inputs.bucket_suffix }} -a x86_64
           ./scripts/publish-bin -f /tmp/falco-build-bin/falco-${{ inputs.version }}-aarch64.tar.gz -r bin${{ inputs.bucket_suffix }} -a aarch64
-          
+
+      # The musl build job is currently disabled because we link libelf dynamically and it is
+      # not possible to dynamically link with musl
       - name: Publish static
+        if: false
         run: |
           ./scripts/publish-bin -f /tmp/falco-build-bin-static/falco-${{ inputs.version }}-static-x86_64.tar.gz -r bin${{ inputs.bucket_suffix }} -a x86_64
-          
+
   publish-packages-deb:
     runs-on: ubuntu-latest
     container: docker.io/debian:stable

.github/workflows/reusable_test_packages.yaml

@@ -15,27 +15,21 @@ on:
         description: The Falco version to use when testing packages
         required: true
         type: string
+      sanitizers:
+        description: Use sanitizer enabled build
+        required: false
+        default: false
+        type: boolean
 
 jobs:
   test-packages:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
     runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
     steps:
-      - name: Checkout
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
-        with:
-          fetch-depth: 0
-          submodules: 'true'
-      
-      - name: Setup Go
-        uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
-        with:
-          go-version: '>=1.17.0'
-
       - name: Download binary
         uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
         with:
-          name: falco-${{ inputs.version }}${{ inputs.static && '-static' || '' }}-${{ inputs.arch }}.tar.gz
+          name: falco-${{ inputs.version }}${{ inputs.static && '-static' || '' }}-${{ inputs.arch }}${{ inputs.sanitizers == true && '-sanitizers' || '' }}.tar.gz
       
       - name: Install Falco package
         run: |
@@ -43,54 +37,26 @@ jobs:
           tar -xvf $(ls falco-*.tar.gz)
           cd falco-${{ inputs.version }}-${{ inputs.arch }}
           sudo cp -r * /
-      
-      # Note: most probably the plugin related tests should be moved to the plugin repo sooner or later.
-      - name: Install needed artifacts using falcoctl
-        if: ${{ inputs.static == false }}
-        run: |
-          sudo mkdir -p /usr/share/falco/plugins
-          sudo falcoctl artifact install k8saudit-rules
-          sudo falcoctl artifact install cloudtrail-rules
      
       # We only run driver loader tests on x86_64
-      - name: Install dependencies for falco-driver-loader tests
+      - name: Install kernel headers for falco-driver-loader tests
         if: ${{ inputs.arch == 'x86_64' }}
         run: |
           sudo apt update -y
-          sudo apt install -y --no-install-recommends build-essential clang make llvm gcc dkms linux-headers-$(uname -r)
+          sudo apt install -y --no-install-recommends linux-headers-$(uname -r)
 
-      - name: Install go-junit-report
+      # Some builds use sanitizers, we always install support for them so they can run
+      - name: Install sanitizer support
         run: |
-          pushd submodules/falcosecurity-testing
-          go install github.com/jstemmer/go-junit-report/v2@latest
-          popd
-  
-      - name: Generate regression test files
-        run: |
-          pushd submodules/falcosecurity-testing
-          go generate ./...
-          popd
-
-      - name: Run regression tests
-        env:
-          # fixme(leogr): this is a workaround for https://github.com/falcosecurity/falco/issues/2784
-          HOST_ROOT: ""
-        run: |
-          pushd submodules/falcosecurity-testing
-          ./build/falco.test -falco-static=${{ inputs.static && 'true' || 'false' }} -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
-          if ${{ inputs.static && 'false' || 'true' }}; then
-            ./build/falcoctl.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
-            ./build/k8saudit.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
-            if ${{ inputs.arch == 'x86_64' && 'true' || 'false' }}; then
-                sudo ./build/falco-driver-loader.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
-            fi    
-          fi
-          cat ./report.txt | go-junit-report -set-exit-code > report.xml
-          popd
-
-      - name: Test Summary
-        if: always() # run this even if previous step fails
-        uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f # v2.1
+          sudo apt update -y
+          sudo apt install -y libasan5 libubsan1
+          
+      - name: Run tests
+        uses: falcosecurity/testing@main    
         with:
-          paths: "submodules/falcosecurity-testing/report.xml"
-          show: "fail"
+          test-falco: 'true'
+          test-falcoctl: 'true'
+          test-k8saudit: 'true'
+          static: ${{ inputs.static && 'true' || 'false' }}
+          test-drivers: ${{ inputs.arch == 'x86_64' && 'true' || 'false' }}
+          show-all: 'true'

.github/workflows/scorecard.yaml

@@ -43,7 +43,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@e38b1902ae4f44df626f11ba0734b14fb91f8f86 # v2.1.2
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
         with:
           results_file: results.sarif
           results_format: sarif

.github/workflows/staticanalysis.yaml

@@ -19,10 +19,11 @@ jobs:
 
       - name: Build and run cppcheck 🏎️
         run: |
-          mkdir build
-          cd build && cmake -DUSE_BUNDLED_DEPS=On -DBUILD_WARNINGS_AS_ERRORS=ON -DCREATE_TEST_TARGETS=Off -DCMAKE_BUILD_TYPE="release" -DBUILD_BPF=Off -DBUILD_DRIVER=Off ..
-          make -j4 cppcheck
-          make -j4 cppcheck_htmlreport
+          cmake -B build -S . \
+                -DCMAKE_BUILD_TYPE="release" \
+                -DUSE_BUNDLED_DEPS=On -DUSE_DYNAMIC_LIBELF=Off -DBUILD_WARNINGS_AS_ERRORS=ON -DCREATE_TEST_TARGETS=Off -DBUILD_BPF=Off -DBUILD_DRIVER=Off
+          cmake --build build -j4 --target cppcheck
+          cmake --build build -j4 --target cppcheck_htmlreport
 
       - name: Upload reports ⬆️
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3

.gitmodules

@@ -2,7 +2,3 @@
 	path = submodules/falcosecurity-rules
 	url = https://github.com/falcosecurity/rules.git
 	branch = main
-[submodule "submodules/falcosecurity-testing"]
-	path = submodules/falcosecurity-testing
-	url = https://github.com/falcosecurity/testing.git
-	branch = main

CHANGELOG.md

@@ -1,5 +1,159 @@
 # Change Log
 
+## v0.37.1
+
+Released on 2024-02-13
+
+### Major Changes
+
+* new(docker): added option for insecure http driver download to falco and driver-loader images [[#3058](https://github.com/falcosecurity/falco/pull/3058)] - [@toamto94](https://github.com/toamto94)
+
+### Minor Changes
+
+* update(cmake): bumped falcoctl to v0.7.2 [[#3076](https://github.com/falcosecurity/falco/pull/3076)] - [@FedeDP](https://github.com/FedeDP)
+* update(build): link libelf dynamically [[#3048](https://github.com/falcosecurity/falco/pull/3048)] - [@LucaGuerra](https://github.com/LucaGuerra)
+
+### Bug Fixes
+
+* fix(userspace/engine): always consider all rules (even the ones below min_prio) in m_rule_stats_manager [[#3060](https://github.com/falcosecurity/falco/pull/3060)] - [@FedeDP](https://github.com/FedeDP)
+
+### Non user-facing changes
+
+* Added http headers option for driver download in docker images [[#3075](https://github.com/falcosecurity/falco/pull/3075)] - [@toamto94](https://github.com/toamto94)
+* fix(build): install libstdc++ in the Wolfi image [[#3053](https://github.com/falcosecurity/falco/pull/3053)] - [@LucaGuerra](https://github.com/LucaGuerra)
+
+## v0.37.0
+
+Released on 2024-01-30
+
+### Breaking Changes
+
+- The deprecated `rate-limiter` mechanism is removed as it is no longer used.
+  - the deprecated `outputs.rate` Falco config is removed.
+  - the deprecated `outputs.max_burst` Falco config is removed.
+- The deprecated `--userspace` CLI option is removed as it is no longer used.
+- The `falco-driver-loader` script will be removed and embedded into falcoctl. The new falcoctl driven implementation will drop:
+  - `--source-only` CLI option.
+  - `BPF_USE_LOCAL_KERNEL_SOURCES` environment variable.
+  - `DRIVER_CURL_OPTIONS` environment variable.
+  - `FALCO_BPF_PROBE` environment variable is not used by the new falcoctl driver loader, since it is already deprecated and will be removed in the next major version.
+  
+  Some env vars were renamed:
+  - `DRIVERS_REPO` env variable has been replaced by `FALCOCTL_DRIVER_NAME`  or `--name` command line argument for `falcoctl driver` command
+  - `DRIVERS_NAME` env variable has been replaced by `FALCOCTL_DRIVER_REPOS`, or `--repo` command line argument for `falcoctl driver` command
+  - `DRIVER_KERNEL_RELEASE` env variable has been replaced by `--kernelrelease` command line argument for `falcoctl driver install` command
+  - `DRIVER_KERNEL_VERSION` env variable has been replaced by `--kernelversion` command line argument for `falcoctl driver install` command
+  - `DRIVER_INSECURE_DOWNLOAD` env variable has been replaced by `--http-insecure` command line argument for `falcoctl driver install` command
+- Remove `-K/-k` options from Falco in favor of the new `k8smeta` plugin.
+- Drop plugins shipped with Falco since plugins are now be managed by falcoctl.
+- Falco 0.37.0 allows environment variables to be expanded even if they are part of a string. This introduces small breaking changes:
+  - Previously, environment variables used in YAML that were empty or defined as `“”` would be expanded to the default value. This was not consistent with the way YAML was handled in other cases, where we only returned the default values if the node was not defined. Now expanded env vars retain the same behavior of all other variables.
+  - Falco 0.37.0 will return default value for nodes that cannot be parsed to chosen type.
+  - `program_output` command will be env-expanded at init time, instead of letting `popen` and thus the `sh` shell expand it. This is technically a breaking change even if no behavioral change is expected. Also, you can avoid env var expansion by using `${{FOO}}` instead of `${FOO}`. It will resolve to `${FOO}` and won't be resolved to the env var value.
+
+### Major Changes
+
+* new!: dropped falco-driver-loader script in favor of new falcoctl driver command [[#2905](https://github.com/falcosecurity/falco/pull/2905)] - [@FedeDP](https://github.com/FedeDP)
+* update!: bump libs to latest and deprecation of k8s metadata options and configs [[#2914](https://github.com/falcosecurity/falco/pull/2914)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* cleanup(falco)!: remove `outputs.rate` and `outputs.max_burst` from Falco config [[#2841](https://github.com/falcosecurity/falco/pull/2841)] - [@Andreagit97](https://github.com/Andreagit97)
+* cleanup(falco)!: remove `--userspace` support [[#2839](https://github.com/falcosecurity/falco/pull/2839)] - [@Andreagit97](https://github.com/Andreagit97)
+* new(engine): add selective overrides for Falco rules [[#2981](https://github.com/falcosecurity/falco/pull/2981)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* feat(userspace/falco): falco administrators can now configure the http output to compress the data sent as well as enable keep alive for the connection. Two new fields (compress_uploads and keep_alive) in the http_output block of the `falco.yaml` file can be used for that purpose. Both are disabled by default. [[#2974](https://github.com/falcosecurity/falco/pull/2974)] - [@sgaist](https://github.com/sgaist)
+* new(userspace): support env variable expansion in all yaml, even inside strings. [[#2918](https://github.com/falcosecurity/falco/pull/2918)] - [@FedeDP](https://github.com/FedeDP)
+* new(scripts): add a way to enforce driver kind and falcoctl enablement when installing Falco from packages and dialog is not present. [[#2773](https://github.com/falcosecurity/falco/pull/2773)] - [@vjjmiras](https://github.com/vjjmiras)
+* new(falco): print system info when Falco starts [[#2927](https://github.com/falcosecurity/falco/pull/2927)] - [@Andreagit97](https://github.com/Andreagit97)
+* new: driver selection in falco.yaml [[#2413](https://github.com/falcosecurity/falco/pull/2413)] - [@therealbobo](https://github.com/therealbobo)
+* new(build): enable compilation on win32 and macOS. [[#2889](https://github.com/falcosecurity/falco/pull/2889)] - [@therealbobo](https://github.com/therealbobo)
+* feat(userspace/falco): falco administrators can now configure the address on which the webserver listen using the new listen_address field in the webserver block of the `falco.yaml` file. [[#2890](https://github.com/falcosecurity/falco/pull/2890)] - [@sgaist](https://github.com/sgaist)
+
+### Minor Changes
+
+* update(userspace/falco): add `engine_version_semver` key in `/versions` endpoint [[#2899](https://github.com/falcosecurity/falco/pull/2899)] - [@loresuso](https://github.com/loresuso)
+* update: default ruleset upgrade to version 3.0 [[#3034](https://github.com/falcosecurity/falco/pull/3034)] - [@leogr](https://github.com/leogr)
+* update!(config): soft deprecation of drop stats counters in `syscall_event_drops` [[#3015](https://github.com/falcosecurity/falco/pull/3015)] - [@incertum](https://github.com/incertum)
+* update(cmake): bumped falcoctl tool to v0.7.1. [[#3030](https://github.com/falcosecurity/falco/pull/3030)] - [@FedeDP](https://github.com/FedeDP)
+* update(rule_loader): deprecate the `append` flag in Falco rules [[#2992](https://github.com/falcosecurity/falco/pull/2992)] - [@Andreagit97](https://github.com/Andreagit97)
+* cleanup!(cmake): drop bundled plugins in Falco [[#2997](https://github.com/falcosecurity/falco/pull/2997)] - [@FedeDP](https://github.com/FedeDP)
+* update(config): clarify deprecation notices + list all env vars [[#2988](https://github.com/falcosecurity/falco/pull/2988)] - [@incertum](https://github.com/incertum)
+* update: now the `watch_config_files` config option monitors file/directory moving and deletion, too [[#2965](https://github.com/falcosecurity/falco/pull/2965)] - [@NitroCao](https://github.com/NitroCao)
+* update(userspace): enhancements in rule description feature [[#2934](https://github.com/falcosecurity/falco/pull/2934)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* update(userspace/falco): add libsinsp state metrics option [[#2883](https://github.com/falcosecurity/falco/pull/2883)] - [@incertum](https://github.com/incertum)
+* update(doc): Add Thought Machine as adopters [[#2919](https://github.com/falcosecurity/falco/pull/2919)] - [@RichardoC](https://github.com/RichardoC)
+* update(docs): add Wireshark/Logray as adopter [[#2867](https://github.com/falcosecurity/falco/pull/2867)] - [@geraldcombs](https://github.com/geraldcombs)
+* update: engine_version in semver representation [[#2838](https://github.com/falcosecurity/falco/pull/2838)] - [@loresuso](https://github.com/loresuso)
+* update(userspace/engine): modularize rule compiler, fix and enrich rule descriptions [[#2817](https://github.com/falcosecurity/falco/pull/2817)] - [@jasondellaluce](https://github.com/jasondellaluce)
+
+### Bug Fixes
+
+* fix(userspace/metric): minor fixes in new libsinsp state metrics handling [[#3033](https://github.com/falcosecurity/falco/pull/3033)] - [@incertum](https://github.com/incertum)
+* fix(userspace/engine): avoid storing escaped strings in engine defs [[#3028](https://github.com/falcosecurity/falco/pull/3028)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* fix(userspace/engine): cache latest rules compilation output [[#2900](https://github.com/falcosecurity/falco/pull/2900)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* fix(userspace/engine): solve description of macro-only rules [[#2898](https://github.com/falcosecurity/falco/pull/2898)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* fix(userspace/engine): fix memory leak [[#2877](https://github.com/falcosecurity/falco/pull/2877)] - [@therealbobo](https://github.com/therealbobo)
+
+### Non user-facing changes
+
+* fix: nlohmann_json lib include path [[#3032](https://github.com/falcosecurity/falco/pull/3032)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* chore: bump falco rules [[#3021](https://github.com/falcosecurity/falco/pull/3021)] - [@Andreagit97](https://github.com/Andreagit97)
+* chore: bump Falco to libs 0.14.1 [[#3020](https://github.com/falcosecurity/falco/pull/3020)] - [@Andreagit97](https://github.com/Andreagit97)
+* chore(build): remove outdated development libs [[#2946](https://github.com/falcosecurity/falco/pull/2946)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* chore(falco): bump Falco to `000d576` libs commit [[#2944](https://github.com/falcosecurity/falco/pull/2944)] - [@Andreagit97](https://github.com/Andreagit97)
+* fix(gha): update rpmsign [[#2856](https://github.com/falcosecurity/falco/pull/2856)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* build(deps): Bump submodules/falcosecurity-rules from `424b258` to `1221b9e` [[#3000](https://github.com/falcosecurity/falco/pull/3000)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `2ac430b` to `c39d31a` [[#3019](https://github.com/falcosecurity/falco/pull/3019)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* cleanup(falco.yaml): rename `none` in `nodriver` [[#3012](https://github.com/falcosecurity/falco/pull/3012)] - [@Andreagit97](https://github.com/Andreagit97)
+* update(config): graduate outputs_queue to stable [[#3016](https://github.com/falcosecurity/falco/pull/3016)] - [@incertum](https://github.com/incertum)
+* update(cmake): bump falcoctl to v0.7.0. [[#3009](https://github.com/falcosecurity/falco/pull/3009)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `1221b9e` to `2ac430b` [[#3007](https://github.com/falcosecurity/falco/pull/3007)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* chore(ci): bumped rn2md to latest master. [[#3006](https://github.com/falcosecurity/falco/pull/3006)] - [@FedeDP](https://github.com/FedeDP)
+* chore: bump Falco to latest libs [[#3002](https://github.com/falcosecurity/falco/pull/3002)] - [@Andreagit97](https://github.com/Andreagit97)
+* chore: bump driver version [[#2998](https://github.com/falcosecurity/falco/pull/2998)] - [@Andreagit97](https://github.com/Andreagit97)
+* Add addl source related methods [[#2939](https://github.com/falcosecurity/falco/pull/2939)] - [@mstemm](https://github.com/mstemm)
+* build(deps): Bump submodules/falcosecurity-rules from `cd33bc3` to `424b258` [[#2993](https://github.com/falcosecurity/falco/pull/2993)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* cleanup(engine): clarify deprecation notice for engines [[#2987](https://github.com/falcosecurity/falco/pull/2987)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* update(cmake): bumped falcoctl to v0.7.0-rc1. [[#2983](https://github.com/falcosecurity/falco/pull/2983)] - [@FedeDP](https://github.com/FedeDP)
+* chore(ci): revert #2961. [[#2984](https://github.com/falcosecurity/falco/pull/2984)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-testing from `930170b` to `9b9630e` [[#2980](https://github.com/falcosecurity/falco/pull/2980)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* chore: bump Falco to latest libs [[#2977](https://github.com/falcosecurity/falco/pull/2977)] - [@Andreagit97](https://github.com/Andreagit97)
+* build(deps): Bump submodules/falcosecurity-rules from `262f569` to `cd33bc3` [[#2976](https://github.com/falcosecurity/falco/pull/2976)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* Allow enabling rules by ruleset id in addition to name [[#2920](https://github.com/falcosecurity/falco/pull/2920)] - [@mstemm](https://github.com/mstemm)
+* chore(ci): enable aarch64 falco driver loader tests. [[#2961](https://github.com/falcosecurity/falco/pull/2961)] - [@FedeDP](https://github.com/FedeDP)
+* chore(unit_tests): added more tests for yaml env vars expansion. [[#2972](https://github.com/falcosecurity/falco/pull/2972)] - [@FedeDP](https://github.com/FedeDP)
+* chore(falco.yaml): use HOME env var for ebpf probe path. [[#2971](https://github.com/falcosecurity/falco/pull/2971)] - [@FedeDP](https://github.com/FedeDP)
+* chore: bump falco to latest libs [[#2970](https://github.com/falcosecurity/falco/pull/2970)] - [@Andreagit97](https://github.com/Andreagit97)
+* build(deps): Bump submodules/falcosecurity-rules from `dd38952` to `262f569` [[#2969](https://github.com/falcosecurity/falco/pull/2969)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* update(readme): add actuated.dev badge [[#2967](https://github.com/falcosecurity/falco/pull/2967)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(cmake,docker): bumped falcoctl to v0.7.0-beta5. [[#2968](https://github.com/falcosecurity/falco/pull/2968)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `64e2adb` to `dd38952` [[#2959](https://github.com/falcosecurity/falco/pull/2959)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* fix(docker): small fixes in docker entrypoints for new driver loader. [[#2966](https://github.com/falcosecurity/falco/pull/2966)] - [@FedeDP](https://github.com/FedeDP)
+* chore(build): allow usage of non-bundled nlohmann-json [[#2947](https://github.com/falcosecurity/falco/pull/2947)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* update(ci): enable actuated.dev [[#2945](https://github.com/falcosecurity/falco/pull/2945)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* cleanup: fix several warnings from a Clang build [[#2948](https://github.com/falcosecurity/falco/pull/2948)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* chore(docker/falco): add back some deps to falco docker image. [[#2932](https://github.com/falcosecurity/falco/pull/2932)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-testing from `92c313f` to `5248e6d` [[#2937](https://github.com/falcosecurity/falco/pull/2937)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `e206c1a` to `8f0520f` [[#2904](https://github.com/falcosecurity/falco/pull/2904)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* cleanup(falco): remove decode_uri as it is no longer used [[#2933](https://github.com/falcosecurity/falco/pull/2933)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* update(engine): port decode_uri in falco engine [[#2912](https://github.com/falcosecurity/falco/pull/2912)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(falco): update to libs on nov 28th [[#2929](https://github.com/falcosecurity/falco/pull/2929)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* cleanup(falco): remove `init` in the configuration constructor [[#2917](https://github.com/falcosecurity/falco/pull/2917)] - [@Andreagit97](https://github.com/Andreagit97)
+* build(deps): Bump submodules/falcosecurity-rules from `8f0520f` to `64e2adb` [[#2908](https://github.com/falcosecurity/falco/pull/2908)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* cleanup(userspace/engine): remove legacy k8saudit implementation [[#2913](https://github.com/falcosecurity/falco/pull/2913)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* fix(gha): disable branch protection rule trigger for scorecard [[#2911](https://github.com/falcosecurity/falco/pull/2911)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(gha): set cosign-installer to v3.1.2 [[#2901](https://github.com/falcosecurity/falco/pull/2901)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* new(docs): sync changelog for 0.36.2. [[#2894](https://github.com/falcosecurity/falco/pull/2894)] - [@FedeDP](https://github.com/FedeDP)
+* Run OpenSSF Scorecard in pipeline [[#2888](https://github.com/falcosecurity/falco/pull/2888)] - [@maxgio92](https://github.com/maxgio92)
+* cleanup: replace banned.h with semgrep [[#2881](https://github.com/falcosecurity/falco/pull/2881)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(gha): upgrade GitHub actions [[#2876](https://github.com/falcosecurity/falco/pull/2876)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* build(deps): Bump submodules/falcosecurity-rules from `a22d0d7` to `e206c1a` [[#2865](https://github.com/falcosecurity/falco/pull/2865)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `d119706` to `a22d0d7` [[#2860](https://github.com/falcosecurity/falco/pull/2860)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* fix(gha): use fedora instead of centos 7 for package publishing [[#2854](https://github.com/falcosecurity/falco/pull/2854)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(gha): pin versions to hashes [[#2849](https://github.com/falcosecurity/falco/pull/2849)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* build(deps): Bump submodules/falcosecurity-rules from `c366d5b` to `d119706` [[#2847](https://github.com/falcosecurity/falco/pull/2847)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* new(ci): properly link libs and driver releases linked to a Falco release [[#2846](https://github.com/falcosecurity/falco/pull/2846)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `7a7cf24` to `c366d5b` [[#2842](https://github.com/falcosecurity/falco/pull/2842)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `77ba57a` to `7a7cf24` [[#2836](https://github.com/falcosecurity/falco/pull/2836)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* chore(ci): bumped rn2md to latest master. [[#2844](https://github.com/falcosecurity/falco/pull/2844)] - [@FedeDP](https://github.com/FedeDP)
+
 ## v0.36.2
 
 Released on 2023-10-27

CMakeLists.txt

@@ -15,13 +15,22 @@ cmake_minimum_required(VERSION 3.5.1)
 
 project(falco)
 
-option(USE_BUNDLED_DEPS "Bundle hard to find dependencies into the Falco binary" OFF)
+option(USE_BUNDLED_DEPS "Bundle hard to find dependencies into the Falco binary" ON)
+option(USE_DYNAMIC_LIBELF "Dynamically link libelf" ON)
 option(BUILD_WARNINGS_AS_ERRORS "Enable building with -Wextra -Werror flags" OFF)
 option(MINIMAL_BUILD "Build a minimal version of Falco, containing only the engine and basic input/output (EXPERIMENTAL)" OFF)
 option(MUSL_OPTIMIZED_BUILD "Enable if you want a musl optimized build" OFF)
 option(BUILD_FALCO_UNIT_TESTS "Build falco unit tests" OFF)
+option(USE_ASAN "Build with AddressSanitizer" OFF)
+option(USE_UBSAN "Build with UndefinedBehaviorSanitizer" OFF)
+option(UBSAN_HALT_ON_ERROR "Halt on error when building with UBSan" ON)
 
 if(WIN32)
+  if(POLICY CMP0091)
+    # Needed for CMAKE_MSVC_RUNTIME_LIBRARY
+    # https://cmake.org/cmake/help/latest/policy/CMP0091.html
+    cmake_policy(SET CMP0091 NEW)
+  endif()
 	set(CPACK_GENERATOR "NSIS") # this needs NSIS installed, and available
 elseif (APPLE)
 	set(CPACK_GENERATOR "DragNDrop")
@@ -155,12 +164,15 @@ endif()
 if(WIN32)
 	set(FALCO_INSTALL_CONF_FILE "%PROGRAMFILES%/${PACKAGE_NAME}-${FALCO_VERSION}/etc/falco/falco.yaml")
 	install(FILES falco.yaml DESTINATION etc/falco/ COMPONENT "${FALCO_COMPONENT_NAME}")
+    install(DIRECTORY DESTINATION etc/falco/config.d COMPONENT "${FALCO_COMPONENT_NAME}")
 elseif(APPLE)
 	set(FALCO_INSTALL_CONF_FILE "/etc/falco/falco.yaml")
 	install(FILES falco.yaml DESTINATION etc/falco/ COMPONENT "${FALCO_COMPONENT_NAME}")
+    install(DIRECTORY DESTINATION etc/falco/config.d COMPONENT "${FALCO_COMPONENT_NAME}")
 else()
 	set(FALCO_INSTALL_CONF_FILE "/etc/falco/falco.yaml")
 	install(FILES falco.yaml DESTINATION "${FALCO_ETC_DIR}" COMPONENT "${FALCO_COMPONENT_NAME}")
+    install(DIRECTORY DESTINATION "${FALCO_ETC_DIR}/config.d" COMPONENT "${FALCO_COMPONENT_NAME}")
 endif()
 
 if(NOT MINIMAL_BUILD)
@@ -178,7 +190,6 @@ include(rules)
 include(static-analysis)
 
 # Shared build variables
-set(FALCO_SINSP_LIBRARY sinsp)
 set(FALCO_SHARE_DIR share/falco)
 set(FALCO_ABSOLUTE_SHARE_DIR "${CMAKE_INSTALL_PREFIX}/${FALCO_SHARE_DIR}")
 set(FALCO_BIN_DIR bin)

README.md

@@ -10,15 +10,15 @@
 
 At its core, Falco is a kernel monitoring and detection agent that observes events, such as syscalls, based on custom rules. Falco can enhance these events by integrating metadata from the container runtime and Kubernetes. The collected events can be analyzed off-host in SIEM or data lake systems.
 
-Falco, originally created by [Sysdig](https://sysdig.com), is an incubating project under the [Cloud Native Computing Foundation](https://cncf.io) (CNCF) used in production by various [organisations](https://github.com/falcosecurity/falco/blob/master/ADOPTERS.md).
+Falco, originally created by [Sysdig](https://sysdig.com), is a **graduated project** under the [Cloud Native Computing Foundation](https://cncf.io) (CNCF) used in production by various [organisations](https://github.com/falcosecurity/falco/blob/master/ADOPTERS.md).
 
 For detailed technical information and insights into the cyber threats that Falco can detect, visit the official [Falco](https://falco.org/) website.
 
-For comprehensive information on the latest updates and changes to the project, please refer to the [change log](CHANGELOG.md). Additionally, we have documented the [release process](RELEASE.md) for delivering new versions of Falco.
+For comprehensive information on the latest updates and changes to the project, please refer to the [Change Log](CHANGELOG.md). Additionally, we have documented the [Release Process](RELEASE.md) for delivering new versions of Falco.
 
 ## Falco Repo: Powering the Core of The Falco Project
 
-This is the main Falco repository which contains the source code for building the Falco binary. By utilizing its [libraries](https://github.com/falcosecurity/libs) and the [falco.yaml](falco.yaml) configuration file, this repository forms the foundation of Falco's functionality. The Falco repository is closely interconnected with the following *core* repositories:
+This is the main Falco repository which contains the source code for building the Falco binary. By utilizing its [libs](https://github.com/falcosecurity/libs) and the [falco.yaml](falco.yaml) configuration file, this repository forms the foundation of Falco's functionality. The Falco repository is closely interconnected with the following *core* repositories:
 
 - [falcosecurity/libs](https://github.com/falcosecurity/libs): Falco's libraries are key to its fundamental operations, making up the greater portion of the source code of the Falco binary and providing essential features such as kernel drivers.
 - [falcosecurity/rules](https://github.com/falcosecurity/rules): Contains the official ruleset for Falco, providing pre-defined detection rules for various security threats and abnormal behaviors.
@@ -29,7 +29,7 @@ For more information, visit the official hub of The Falco Project: [falcosecurit
 
 ## Getting Started with Falco
 
-Carefully review and follow the [official guide and documentation](https://falco.org/docs/getting-started/).
+Carefully review and follow the [Official Documentation](https://falco.org/docs/install-operate/).
 
 Considerations and guidance for Falco adopters:
 
@@ -46,11 +46,11 @@ Considerations and guidance for Falco adopters:
 
 ## How to Contribute
 
-Please refer to the [contributing guide](https://github.com/falcosecurity/.github/blob/main/CONTRIBUTING.md) and the [code of conduct](https://github.com/falcosecurity/evolution/blob/main/CODE_OF_CONDUCT.md) for more information on how to contribute.
+Please refer to the [Contributing](https://github.com/falcosecurity/.github/blob/main/CONTRIBUTING.md) guide and the [Code of Conduct](https://github.com/falcosecurity/evolution/blob/main/CODE_OF_CONDUCT.md) for more information on how to contribute.
 
 ## Join the Community
 
-To get involved with the Falco Project please visit the [community repository](https://github.com/falcosecurity/community) to find more information and ways to get involved.
+To get involved with the Falco Project please visit the [Community](https://github.com/falcosecurity/community) repository to find more information and ways to get involved.
 
 If you have any questions about Falco or contributing, do not hesitate to file an issue or contact the Falco maintainers and community members for assistance.
 
@@ -64,7 +64,7 @@ How to reach out?
 
 Full reports of various security audits can be found [here](./audits/).
 
-In addition, you can refer to the [falco security](https://github.com/falcosecurity/falco/security) and [libs security](https://github.com/falcosecurity/libs/security) sections for detailed updates on security advisories and policies.
+In addition, you can refer to the [falco](https://github.com/falcosecurity/falco/security) and [libs](https://github.com/falcosecurity/libs/security) security sections for detailed updates on security advisories and policies.
 
 To report security vulnerabilities, please follow the community process outlined in the documentation found [here](https://github.com/falcosecurity/.github/blob/main/SECURITY.md).
 
@@ -76,8 +76,49 @@ Stay updated with Falco's evolving capabilities by exploring the [Falco Roadmap]
 
 Falco is licensed to you under the [Apache 2.0](./COPYING) open source license.
 
+## Testing
+
+<details>
+	<summary>Expand Testing Instructions</summary>
+
+Falco's [Build Falco from source](https://falco.org/docs/install-operate/source/) is the go-to resource to understand how to build Falco from source. In addition, the [falcosecurity/libs](https://github.com/falcosecurity/libs) repository offers additional valuable information about tests and debugging of Falco's underlying libraries and kernel drivers.
+
+Here's an example of a `cmake` command that will enable everything you need for all unit tests of this repository:
+
+```bash
+cmake \
+-DUSE_BUNDLED_DEPS=ON \
+-DBUILD_LIBSCAP_GVISOR=ON \
+-DBUILD_BPF=ON \
+-DBUILD_DRIVER=ON \
+-DBUILD_FALCO_MODERN_BPF=ON \
+-DCREATE_TEST_TARGETS=ON \
+-DBUILD_FALCO_UNIT_TESTS=ON ..;
+```
+
+Build and run the unit test suite:
+
+```bash
+nproc=$(grep processor /proc/cpuinfo | tail -n 1 | awk '{print $3}');
+make -j$(($nproc-1)) falco_unit_tests;
+# Run the tests
+sudo ./unit_tests/falco_unit_tests;
+```
+
+Optionally, build the driver of your choice and test run the Falco binary to perform manual tests.
+
+Lastly, The Falco Project has moved its Falco regression tests to [falcosecurity/testing](https://github.com/falcosecurity/testing).
+
+
+</details>
+
+</br>
+
 ## Why is Falco in C++ rather than Go or {language}?
 
+<details>
+	<summary>Expand Information</summary>
+
 1. The first lines of code at the base of Falco were written some time ago, where Go didn't yet have the same level of maturity and adoption as today.
 2. The Falco execution model is sequential and mono-thread due to the statefulness requirements of the tool, and so most of the concurrency-related selling points of the Go runtime would not be leveraged at all.
 3. The Falco code deals with very low-level programming in many places (e.g. some headers are shared with the eBPF probe and the Kernel module), and we all know that interfacing Go with C is possible but brings tons of complexity and tradeoffs to the table.
@@ -89,6 +130,8 @@ Falco is licensed to you under the [Apache 2.0](./COPYING) open source license.
 9. Memory safety is definitely a concern and we try our best to keep an high level of quality even though C++ is quite error prone. For instance, we try to use smart pointers whenever possible, we build the libraries with an address sanitizer in our CI, we run Falco through Valgrind before each release, and have ways to stress-test it to detect performance regressions or weird memory usage (e.g. https://github.com/falcosecurity/event-generator). On top of that, we also have third parties auditing the codebase by time to time. None of this make a perfect safety standpoint of course, but we try to maximize our odds. Go would definitely make our life easier from this perspective, however the tradeoffs never made it worth it so far due to the points above.
 10. The C++ codebase of falcosecurity/libs, which is at the core of Falco, is quite large and complex. Porting all that code to another language would be a major effort requiring lots of development resource and with an high chance of failure and regression. As such, our approach so far has been to choose refactors and code polishing instead, up until we'll reach an optimal level of stability, quality, and modularity, on that portion of code. This would allow further developments to be smoother and more feasibile in the future.
 
+</details>
+</br>
 
 ## Resources
 
@@ -99,3 +142,5 @@ Falco is licensed to you under the [Apache 2.0](./COPYING) open source license.
  - [Repositories Guidelines](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md)
  - [Repositories List](https://github.com/falcosecurity/evolution/blob/main/README.md#repositories)
  - [Adopters List](https://github.com/falcosecurity/falco/blob/master/ADOPTERS.md)
+ - [Install and Operate](https://falco.org/docs/install-operate/)
+ - [Troubleshooting](https://falco.org/docs/troubleshooting/)

cmake/modules/CompilerFlags.cmake

@@ -42,7 +42,8 @@ endif()
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(FALCO_SECURITY_FLAGS "")
 if(LINUX)
-  set(FALCO_SECURITY_FLAGS "${FALCO_SECURITY_FLAGS} -Wl,-z,relro,-z,now -fstack-protector-strong")
+  set(FALCO_SECURITY_FLAGS "${FALCO_SECURITY_FLAGS} -fstack-protector-strong")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,relro,-z,now")
 endif()
 
 
@@ -52,17 +53,29 @@ if(NOT MSVC)
 		set(FALCO_SECURITY_FLAGS "${FALCO_SECURITY_FLAGS} -D_FORTIFY_SOURCE=2")
 	endif()
 
+	if(USE_ASAN)
+		set(FALCO_SECURITY_FLAGS "${FALCO_SECURITY_FLAGS} -fsanitize=address")
+	endif()
+
+	if(USE_UBSAN)
+		set(FALCO_SECURITY_FLAGS "${FALCO_SECURITY_FLAGS} -fsanitize=undefined")
+		if(UBSAN_HALT_ON_ERROR)
+			set(FALCO_SECURITY_FLAGS "${FALCO_SECURITY_FLAGS} -fno-sanitize-recover=undefined")
+		endif()
+	endif()
+
 	set(CMAKE_COMMON_FLAGS "${FALCO_SECURITY_FLAGS} -Wall -ggdb ${FALCO_EXTRA_FEATURE_FLAGS} ${MINIMAL_BUILD_FLAGS} ${MUSL_FLAGS}")
 
 	if(BUILD_WARNINGS_AS_ERRORS)
 		set(CMAKE_SUPPRESSED_WARNINGS
 			"-Wno-unused-parameter -Wno-unused-variable -Wno-unused-but-set-variable -Wno-missing-field-initializers -Wno-sign-compare -Wno-type-limits -Wno-implicit-fallthrough -Wno-format-truncation -Wno-stringop-truncation -Wno-stringop-overflow -Wno-restrict"
 		)
-		set(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -Wextra -Werror ${CMAKE_SUPPRESSED_WARNINGS}")
+		set(CMAKE_COMPILE_WARNING_AS_ERROR ON)
+		set(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -Wextra ${CMAKE_SUPPRESSED_WARNINGS}")
 	endif()
 
 	set(CMAKE_C_FLAGS "${CMAKE_COMMON_FLAGS}")
-	set(CMAKE_CXX_FLAGS "-std=c++17 ${CMAKE_COMMON_FLAGS} -Wno-class-memaccess")
+	set(CMAKE_CXX_FLAGS "-std=c++17 ${CMAKE_COMMON_FLAGS}")
 
 	set(CMAKE_C_FLAGS_DEBUG "${FALCO_EXTRA_DEBUG_FLAGS}")
 	set(CMAKE_CXX_FLAGS_DEBUG "${FALCO_EXTRA_DEBUG_FLAGS}")
@@ -72,6 +85,7 @@ if(NOT MSVC)
 
 else() # MSVC
 	set(MINIMAL_BUILD ON)
+	set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 
 	# The WIN32_LEAN_AND_MEAN define avoids possible macro pollution
 	# when a libsinsp consumer includes the windows.h header.
@@ -82,7 +96,7 @@ else() # MSVC
 		_CRT_SECURE_NO_WARNINGS
 		WIN32
 		MINIMAL_BUILD
-		WIN32_LEAN_AND_MEAN 
+		WIN32_LEAN_AND_MEAN
 	)
 
 	set(FALCOSECURITY_LIBS_COMMON_FLAGS "/EHsc /W3 /Zi /std:c++17")

cmake/modules/cpp-httplib.cmake

@@ -12,22 +12,15 @@
 # specific language governing permissions and limitations under the License.
 #
 
-#
-# cpp-httplib (https://github.com/yhirose/cpp-httplib)
-#
-if(CPPHTTPLIB_INCLUDE)
-	# we already have cpp-httplib
+option(USE_BUNDLED_CPPHTTPLIB "Enable building of the bundled cpp-httplib" ${USE_BUNDLED_DEPS})
+
+if(USE_BUNDLED_CPPHTTPLIB)
+    include(FetchContent)
+    FetchContent_Declare(cpp-httplib
+        URL https://github.com/yhirose/cpp-httplib/archive/refs/tags/v0.15.3.tar.gz
+        URL_HASH SHA256=2121bbf38871bb2aafb5f7f2b9b94705366170909f434428352187cb0216124e
+    )
+    FetchContent_MakeAvailable(cpp-httplib)
 else()
-	set(CPPHTTPLIB_SRC "${PROJECT_BINARY_DIR}/cpp-httplib-prefix/src/cpp-httplib")
-	set(CPPHTTPLIB_INCLUDE "${CPPHTTPLIB_SRC}")
-
-	message(STATUS "Using bundled cpp-httplib in '${CPPHTTPLIB_SRC}'")
-
-	ExternalProject_Add(cpp-httplib
-		PREFIX "${PROJECT_BINARY_DIR}/cpp-httplib-prefix"
-		URL "https://github.com/yhirose/cpp-httplib/archive/refs/tags/v0.13.1.tar.gz"
-		URL_HASH "SHA256=9b837d290b61e3f0c4239da0b23bbf14c382922e2bf2a9bac21c1e3feabe1ff9"
-		CONFIGURE_COMMAND ""
-		BUILD_COMMAND ""
-		INSTALL_COMMAND "")	
+    find_package(httplib CONFIG REQUIRED)
 endif()

cmake/modules/cxxopts.cmake

@@ -12,13 +12,32 @@
 # specific language governing permissions and limitations under the License.
 #
 
-set(CXXOPTS_SRC "${PROJECT_BINARY_DIR}/cxxopts-prefix/src/cxxopts/")
-set(CXXOPTS_INCLUDE_DIR "${CXXOPTS_SRC}/include")
+#
+# cxxopts (https://github.com/jarro2783/cxxopts)
+#
 
-ExternalProject_Add(
+option(USE_BUNDLED_CXXOPTS "Enable building of the bundled cxxopts" ${USE_BUNDLED_DEPS})
+
+if(CXXOPTS_INCLUDE_DIR)
+    # we already have cxxopts
+elseif(NOT USE_BUNDLED_CXXOPTS)
+    find_package(cxxopts CONFIG REQUIRED)
+    get_target_property(CXXOPTS_INCLUDE_DIR cxxopts::cxxopts INTERFACE_INCLUDE_DIRECTORIES)
+else()
+    set(CXXOPTS_SRC "${PROJECT_BINARY_DIR}/cxxopts-prefix/src/cxxopts/")
+    set(CXXOPTS_INCLUDE_DIR "${CXXOPTS_SRC}/include")
+
+    message(STATUS "Using bundled cxxopts in ${CXXOPTS_SRC}")
+
+    ExternalProject_Add(
         cxxopts
         URL "https://github.com/jarro2783/cxxopts/archive/refs/tags/v3.0.0.tar.gz"
         URL_HASH "SHA256=36f41fa2a46b3c1466613b63f3fa73dc24d912bc90d667147f1e43215a8c6d00"
-	CONFIGURE_COMMAND ""
-	BUILD_COMMAND ""
-	INSTALL_COMMAND "")
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND "")
+endif()
+
+if(NOT TARGET cxxopts)
+    add_custom_target(cxxopts)
+endif()

cmake/modules/driver.cmake

@@ -34,8 +34,8 @@ else()
   # In case you want to test against another driver version (or branch, or commit) just pass the variable -
   # ie., `cmake -DDRIVER_VERSION=dev ..`
   if(NOT DRIVER_VERSION)
-    set(DRIVER_VERSION "7.0.0+driver")
-    set(DRIVER_CHECKSUM "SHA256=9f2a0f14827c0d9d1c3d1abe45b8f074dea531ebeca9859363a92f0d2475757e")
+    set(DRIVER_VERSION "91e8aa8cf0f1952c301d16a34ee11830803426c3")
+    #set(DRIVER_CHECKSUM "SHA256=130dbf120245c39da2dd85cd8a3ebdec49463d00f7b474230795089cc73a6450")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

cmake/modules/falcoctl.cmake

@@ -16,14 +16,14 @@ include(ExternalProject)
 
 string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} FALCOCTL_SYSTEM_NAME)
 
-set(FALCOCTL_VERSION "0.7.0")
+set(FALCOCTL_VERSION "0.8.0-rc1")
 
 if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
     set(FALCOCTL_SYSTEM_PROC_GO "amd64")
-    set(FALCOCTL_HASH "d9ccff287bffd847752f2ec2d65566032f097a38219c6ca87dbcf1cd0fe3cbe4")
+    set(FALCOCTL_HASH "120a45be26be26ed9795926e3d382440ce010b8ec31fda8a0118ec3d42fc7bc7")
 else() # aarch64
     set(FALCOCTL_SYSTEM_PROC_GO "arm64")
-    set(FALCOCTL_HASH "5db283cd0ba15c875ef8b95037f18c01a95d683fdc177a4f5f1b5b92450b6602")
+    set(FALCOCTL_HASH "f2c15e7a4c210ee9e3a521f4b6f6ed3096db6bf52426d369a9fe2a1cd3a4a4c2")
 endif()
 
 ExternalProject_Add(
@@ -35,3 +35,4 @@ ExternalProject_Add(
         INSTALL_COMMAND "")
 
 install(PROGRAMS "${PROJECT_BINARY_DIR}/falcoctl-prefix/src/falcoctl/falcoctl" DESTINATION "${FALCO_BIN_DIR}" COMPONENT "${FALCO_COMPONENT_NAME}")
+install(DIRECTORY DESTINATION "${FALCO_ABSOLUTE_SHARE_DIR}/plugins" COMPONENT "${FALCO_COMPONENT_NAME}")

cmake/modules/falcosecurity-libs.cmake

@@ -35,8 +35,8 @@ else()
   # In case you want to test against another falcosecurity/libs version (or branch, or commit) just pass the variable -
   # ie., `cmake -DFALCOSECURITY_LIBS_VERSION=dev ..`
   if(NOT FALCOSECURITY_LIBS_VERSION)
-    set(FALCOSECURITY_LIBS_VERSION "0.14.1")
-    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=defdea24bf3b176c63f10900d3716fe4373151965cc09d3fe67a31a3a9af0b13")
+    set(FALCOSECURITY_LIBS_VERSION "91e8aa8cf0f1952c301d16a34ee11830803426c3")
+    #set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=130dbf120245c39da2dd85cd8a3ebdec49463d00f7b474230795089cc73a6450")
   endif()
 
   # cd /path/to/build && cmake /path/to/source
@@ -67,12 +67,10 @@ set(SCAP_HOST_ROOT_ENV_VAR_NAME "HOST_ROOT")
 set(SCAP_HOSTNAME_ENV_VAR "FALCO_HOSTNAME")
 set(SINSP_AGENT_CGROUP_MEM_PATH_ENV_VAR "FALCO_CGROUP_MEM_PATH")
 
-if(NOT LIBSCAP_DIR)
-  set(LIBSCAP_DIR "${FALCOSECURITY_LIBS_SOURCE_DIR}")
+if(NOT LIBS_DIR)
+  set(LIBS_DIR "${FALCOSECURITY_LIBS_SOURCE_DIR}")
 endif()
 
-set(LIBSINSP_DIR "${FALCOSECURITY_LIBS_SOURCE_DIR}")
-
 # configure gVisor support
 set(BUILD_LIBSCAP_GVISOR ${BUILD_FALCO_GVISOR} CACHE BOOL "")
 
@@ -85,10 +83,13 @@ set(BUILD_LIBSCAP_EXAMPLES OFF CACHE BOOL "")
 
 set(USE_BUNDLED_TBB ON CACHE BOOL "")
 set(USE_BUNDLED_JSONCPP ON CACHE BOOL "")
-set(USE_BUNDLED_NLOHMANN_JSON ON CACHE BOOL "")
 set(USE_BUNDLED_VALIJSON ON CACHE BOOL "")
 set(USE_BUNDLED_RE2 ON CACHE BOOL "")
 set(USE_BUNDLED_UTHASH ON CACHE BOOL "")
+if(USE_DYNAMIC_LIBELF)
+  set(USE_BUNDLED_LIBELF OFF CACHE BOOL "")
+  set(USE_SHARED_LIBELF ON CACHE BOOL "")
+endif()
 
 list(APPEND CMAKE_MODULE_PATH "${FALCOSECURITY_LIBS_SOURCE_DIR}/cmake/modules")
 

cmake/modules/njson.cmake

@@ -12,15 +12,15 @@
 # specific language governing permissions and limitations under the License.
 #
 
-if(USE_BUNDLED_NLOHMANN_JSON)
-    ExternalProject_Add(njson
-        URL "https://github.com/nlohmann/json/archive/v3.3.0.tar.gz"
-        URL_HASH "SHA256=2fd1d207b4669a7843296c41d3b6ac5b23d00dec48dba507ba051d14564aa801"
-        CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PROJECT_BINARY_DIR}/njson-prefix -DJSON_BuildTests=OFF -DBUILD_TESTING=OFF
-    )
+option(USE_BUNDLED_NLOHMANN_JSON "Enable building of the bundled nlohmann-json" ${USE_BUNDLED_DEPS})
 
-    set(nlohmann_json_DIR ${PROJECT_BINARY_DIR}/njson-prefix/include)
+if(USE_BUNDLED_NLOHMANN_JSON)
+    include(FetchContent)
+    FetchContent_Declare(nlohmann_json
+        URL https://github.com/nlohmann/json/archive/v3.11.3.tar.gz
+        URL_HASH SHA256=0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406
+    )
+    FetchContent_MakeAvailable(nlohmann_json)
 else()
     find_package(nlohmann_json CONFIG REQUIRED)
-    add_custom_target(njson)
 endif()

cmake/modules/rules.cmake

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 #
-# Copyright (C) 2023 The Falco Authors.
+# Copyright (C) 2024 The Falco Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 # the License. You may obtain a copy of the License at
@@ -16,7 +16,7 @@ include(GNUInstallDirs)
 include(ExternalProject)
 
 # falco_rules.yaml
-set(FALCOSECURITY_RULES_FALCO_VERSION "falco-rules-3.0.0-rc1")
+set(FALCOSECURITY_RULES_FALCO_VERSION "falco-rules-3.0.0")
 set(FALCOSECURITY_RULES_FALCO_CHECKSUM "SHA256=2e91799fee49c2daf58fb482e47410a21433eb116e02cde18206f7af87449ddb")
 set(FALCOSECURITY_RULES_FALCO_PATH "${PROJECT_BINARY_DIR}/falcosecurity-rules-falco-prefix/src/falcosecurity-rules-falco/falco_rules.yaml")
 ExternalProject_Add(

cmake/modules/yaml-cpp.cmake

@@ -11,31 +11,16 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 #
-mark_as_advanced(YAMLCPP_INCLUDE_DIR YAMLCPP_LIB)
-if(NOT USE_BUNDLED_DEPS)
-  find_path(YAMLCPP_INCLUDE_DIR NAMES yaml-cpp/yaml.h)
-  find_library(YAMLCPP_LIB NAMES yaml-cpp)
-  if(YAMLCPP_INCLUDE_DIR AND YAMLCPP_LIB)
-    message(STATUS "Found yamlcpp: include: ${YAMLCPP_INCLUDE_DIR}, lib: ${YAMLCPP_LIB}")
-  else()
-    message(FATAL_ERROR "Couldn't find system yamlcpp")
-  endif()
-  add_custom_target(yamlcpp)
+
+option(USE_BUNDLED_YAMLCPP "Enable building of the bundled yamlcpp" ${USE_BUNDLED_DEPS})
+
+if(USE_BUNDLED_YAMLCPP)
+    include(FetchContent)
+    FetchContent_Declare(yamlcpp
+        URL https://github.com/jbeder/yaml-cpp/archive/refs/tags/0.8.0.tar.gz
+        URL_HASH SHA256=fbe74bbdcee21d656715688706da3c8becfd946d92cd44705cc6098bb23b3a16
+    )
+    FetchContent_MakeAvailable(yamlcpp)
 else()
-  set(YAMLCPP_SRC "${PROJECT_BINARY_DIR}/yamlcpp-prefix/src/yamlcpp")
-  message(STATUS "Using bundled yaml-cpp in '${YAMLCPP_SRC}'")
-  if(NOT WIN32)
-    set(YAMLCPP_LIB "${YAMLCPP_SRC}/libyaml-cpp.a")
-  else()
-    set(YAMLCPP_LIB "${YAMLCPP_SRC}/${CMAKE_BUILD_TYPE}/yaml-cpp.lib")
-  endif()
-  set(YAMLCPP_INCLUDE_DIR "${YAMLCPP_SRC}/include")
-  ExternalProject_Add(
-    yamlcpp
-    URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-0.7.0.tar.gz"
-    URL_HASH "SHA256=43e6a9fcb146ad871515f0d0873947e5d497a1c9c60c58cb102a97b47208b7c3"
-    BUILD_BYPRODUCTS ${YAMLCPP_LIB}
-    CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DYAML_MSVC_SHARED_RT=Off -DYAML_BUILD_SHARED_LIBS=Off -DYAML_CPP_BUILD_TESTS=Off -DYAML_CPP_BUILD_TOOLS=OFF -DYAML_CPP_BUILD_CONTRIB=OFF -DCMAKE_DEBUG_POSTFIX=''
-    BUILD_IN_SOURCE 1
-    INSTALL_COMMAND "")
+    find_package(yaml-cpp CONFIG REQUIRED)
 endif()

docker/builder/README.md

@@ -1,6 +0,0 @@
-# Builder folder
-
-* We use `modern-falco-builder.Dockerfile` to build Falco with the modern probe and return it as a Dockerfile output. This Dockerfile doesn't generate a Docker image but returns as output (through the `--output` command):
-  * Falco `tar.gz`.
-  * Falco `deb` package.
-  * Falco `rpm` package.

docker/builder/modern-falco-builder.Dockerfile

@@ -1,43 +0,0 @@
-
-FROM centos:7 AS build-stage
-
-# To build Falco you need to pass the cmake option
-ARG CMAKE_OPTIONS=""
-ARG MAKE_JOBS=6
-
-# Install all the dependencies
-WORKDIR /
-
-RUN yum -y install centos-release-scl; \
-    yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++; \
-    source scl_source enable devtoolset-9; \
-    yum install -y git wget make m4 rpm-build
-
-# With some previous cmake versions it fails when downloading `zlib` with curl in the libs building phase
-RUN curl -L -o /tmp/cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.22.5/cmake-3.22.5-linux-$(uname -m).tar.gz; \
-    gzip -d /tmp/cmake.tar.gz; \
-    tar -xpf /tmp/cmake.tar --directory=/tmp; \
-    cp -R /tmp/cmake-3.22.5-linux-$(uname -m)/* /usr; \
-    rm -rf /tmp/cmake-3.22.5-linux-$(uname -m)/
-
-# Copy Falco folder from the build context
-COPY . /source
-WORKDIR /build/release
-
-RUN source scl_source enable devtoolset-9; \
-    cmake ${CMAKE_OPTIONS} /source; \
-    make falco -j${MAKE_JOBS}
-RUN make package
-
-# We need `make all` for integration tests.
-RUN make all -j${MAKE_JOBS}
-
-FROM scratch AS export-stage
-
-LABEL org.opencontainers.image.source="https://github.com/falcosecurity/falco"
-
-ARG DEST_BUILD_DIR="/build"
-
-COPY --from=build-stage /build/release/falco-*.tar.gz /packages/
-COPY --from=build-stage /build/release/falco-*.deb /packages/
-COPY --from=build-stage /build/release/falco-*.rpm /packages/

docker/driver-loader-legacy/Dockerfile

@@ -92,7 +92,7 @@ RUN rm -rf /usr/bin/clang \
 RUN curl -s https://falco.org/repo/falcosecurity-packages.asc | apt-key add - \
 	&& echo "deb https://download.falco.org/packages/${VERSION_BUCKET} stable main" | tee -a /etc/apt/sources.list.d/falcosecurity.list \
 	&& apt-get update -y \
-	&& if [ "$FALCO_VERSION" = "latest" ]; then apt-get install -y --no-install-recommends falco; else apt-get install -y --no-install-recommends falco=${FALCO_VERSION}; fi \
+	&& if [ "$FALCO_VERSION" = "latest" ]; then FALCO_DRIVER_CHOICE=none apt-get install -y --no-install-recommends falco; else FALCO_DRIVER_CHOICE=none apt-get install -y --no-install-recommends falco=${FALCO_VERSION}; fi \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/*
 

docker/driver-loader-legacy/docker-entrypoint.sh

@@ -24,19 +24,23 @@ print_usage() {
 	echo "  docker run -i -t --privileged -v /root/.falco:/root/.falco -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro -v /etc:/host/etc:ro falcosecurity/falco-driver-loader-legacy:latest [driver] [options]"
 	echo ""
 	echo "Available drivers:"
-	echo "  kmod           kernel module (default)"
+	echo "  auto	       leverage automatic driver selection logic (default)"
+	echo "  modern_ebpf    modern eBPF CORE probe"
+	echo "  kmod           kernel module"
 	echo "  ebpf           eBPF probe"
 	echo ""
 	echo "Options:"
-	echo "  --help         show this help message"
-	echo "  --clean        try to remove an already present driver installation"
-	echo "  --compile      try to compile the driver locally (default true)"
-	echo "  --download     try to download a prebuilt driver (default true)"
-	echo "  --print-env    skip execution and print env variables for other tools to consume"
+	echo "  --help           show this help message"
+	echo "  --clean          try to remove an already present driver installation"
+	echo "  --compile        try to compile the driver locally (default true)"
+	echo "  --download       try to download a prebuilt driver (default true)"
+ 	echo "  --http-insecure	 enable insecure downloads"
+	echo "  --print-env      skip execution and print env variables for other tools to consume"
 	echo ""
 	echo "Environment variables:"
-	echo "  FALCOCTL_DRIVER_REPOS             specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
-	echo "  FALCOCTL_DRIVER_NAME              specify a different name for the driver"
+	echo "  FALCOCTL_DRIVER_REPOS         specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
+	echo "  FALCOCTL_DRIVER_NAME          specify a different name for the driver"
+	echo "  FALCOCTL_DRIVER_HTTP_HEADERS  specify comma separated list of http headers for driver download (e.g. 'x-emc-namespace: default,Proxy-Authenticate: Basic')"
 	echo ""
 }
 
@@ -50,18 +54,18 @@ done
 
 ENABLE_COMPILE="false"
 ENABLE_DOWNLOAD="false"
-has_driver=
+HTTP_INSECURE="false"
+driver=
 has_opts=
 while test $# -gt 0; do
 	case "$1" in
-		kmod|ebpf)
-			if [ -n "$has_driver" ]; then
+		auto|kmod|ebpf|modern_ebpf)
+			if [ -n "$driver" ]; then
 				>&2 echo "Only one driver per invocation"
 				print_usage
 				exit 1
 			else
-				/usr/bin/falcoctl driver config --type $1
-				has_driver="true"
+			  driver=$1
 			fi
 			;;
 		-h|--help)
@@ -80,10 +84,8 @@ while test $# -gt 0; do
 			ENABLE_DOWNLOAD="true"
 			has_opts="true"
 			;;
-		--source-only)
-		    >&2 echo "Support dropped in Falco 0.37.0."
-			print_usage
-			exit 1
+		--http-insecure)
+			HTTP_INSECURE="true"
 			;;
 		--print-env)
 			/usr/bin/falcoctl driver printenv
@@ -103,9 +105,22 @@ while test $# -gt 0; do
     shift
 done
 
+# No opts passed, enable both compile and download
 if [ -z "$has_opts" ]; then
-	ENABLE_COMPILE="true"
-	ENABLE_DOWNLOAD="true"
+    ENABLE_COMPILE="true"
+    ENABLE_DOWNLOAD="true"
 fi
 
-/usr/bin/falcoctl driver install --compile=$ENABLE_COMPILE --download=$ENABLE_DOWNLOAD
+# Default value: auto
+if [ -z "$driver" ]; then
+  driver="auto"
+fi
+
+if [ "$driver" != "auto" ]; then
+  /usr/bin/falcoctl driver config --type $driver
+else
+  # Needed because we need to configure Falco to start with correct driver
+  /usr/bin/falcoctl driver config --type modern_ebpf --type kmod --type ebpf
+fi
+
+/usr/bin/falcoctl driver install --compile=$ENABLE_COMPILE --download=$ENABLE_DOWNLOAD --http-insecure=$HTTP_INSECURE --http-headers="$FALCOCTL_DRIVER_HTTP_HEADERS"

docker/driver-loader/docker-entrypoint.sh

@@ -24,19 +24,25 @@ print_usage() {
 	echo "  docker run -i -t --privileged -v /root/.falco:/root/.falco -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro -v /etc:/host/etc:ro falcosecurity/falco-driver-loader:latest [driver] [options]"
 	echo ""
 	echo "Available drivers:"
-	echo "  kmod           kernel module (default)"
+	echo "  auto	       leverage automatic driver selection logic (default)"
+	echo "	modern_ebpf    modern eBPF CORE probe"
+	echo "  kmod           kernel module"
 	echo "  ebpf           eBPF probe"
 	echo ""
 	echo "Options:"
-	echo "  --help         show this help message"
-	echo "  --clean        try to remove an already present driver installation"
-	echo "  --compile      try to compile the driver locally (default true)"
-	echo "  --download     try to download a prebuilt driver (default true)"
-	echo "  --print-env    skip execution and print env variables for other tools to consume"
+	echo "  --help                   show this help message"
+	echo "  --clean                  try to remove an already present driver installation"
+	echo "  --compile                try to compile the driver locally (default true)"
+	echo "  --download               try to download a prebuilt driver (default true)"
+	echo "  --kernel-release <value> set the kernel release"
+	echo "  --kernel-version <value> set the kernel version"
+ 	echo "  --http-insecure	         enable insecure downloads"
+	echo "  --print-env              skip execution and print env variables for other tools to consume"
 	echo ""
 	echo "Environment variables:"
-	echo "  FALCOCTL_DRIVER_REPOS             specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
-	echo "  FALCOCTL_DRIVER_NAME              specify a different name for the driver"
+	echo "  FALCOCTL_DRIVER_REPOS         specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
+	echo "  FALCOCTL_DRIVER_NAME          specify a different name for the driver"
+	echo "  FALCOCTL_DRIVER_HTTP_HEADERS  specify comma separated list of http headers for driver download (e.g. 'x-emc-namespace: default,Proxy-Authenticate: Basic')"
 	echo ""
 }
 
@@ -50,18 +56,20 @@ done
 
 ENABLE_COMPILE="false"
 ENABLE_DOWNLOAD="false"
-has_driver=
+HTTP_INSECURE="false"
+driver=
 has_opts=
+extra_args=
+
 while test $# -gt 0; do
 	case "$1" in
-		kmod|ebpf)
-			if [ -n "$has_driver" ]; then
+		auto|kmod|ebpf|modern_ebpf)
+			if [ -n "$driver" ]; then
 				>&2 echo "Only one driver per invocation"
 				print_usage
 				exit 1
 			else
-				/usr/bin/falcoctl driver config --type $1
-				has_driver="true"
+				driver=$1
 			fi
 			;;
 		-h|--help)
@@ -80,10 +88,16 @@ while test $# -gt 0; do
 			ENABLE_DOWNLOAD="true"
 			has_opts="true"
 			;;
-		--source-only)
-		    >&2 echo "Support dropped in Falco 0.37.0."
-			print_usage
-			exit 1
+		--http-insecure)
+			HTTP_INSECURE="true"
+			;;
+		--kernel-release)
+			extra_args+="--kernelrelease=$2 "
+			shift
+			;;
+		--kernel-version)
+			extra_args+="--kernelversion=$2 "
+			shift
 			;;
 		--print-env)
 			/usr/bin/falcoctl driver printenv
@@ -103,9 +117,22 @@ while test $# -gt 0; do
     shift
 done
 
+# No opts passed, enable both compile and download
 if [ -z "$has_opts" ]; then
-	ENABLE_COMPILE="true"
-	ENABLE_DOWNLOAD="true"
+    ENABLE_COMPILE="true"
+    ENABLE_DOWNLOAD="true"
 fi
 
-/usr/bin/falcoctl driver install --compile=$ENABLE_COMPILE --download=$ENABLE_DOWNLOAD
+# Default value: auto
+if [ -z "$driver" ]; then
+  driver="auto"
+fi
+
+if [ "$driver" != "auto" ]; then
+  /usr/bin/falcoctl driver config --type $driver
+else
+  # Needed because we need to configure Falco to start with correct driver
+  /usr/bin/falcoctl driver config --type modern_ebpf --type kmod --type ebpf
+fi
+
+/usr/bin/falcoctl driver install --compile=$ENABLE_COMPILE --download=$ENABLE_DOWNLOAD --http-insecure=$HTTP_INSECURE --http-headers="$FALCOCTL_DRIVER_HTTP_HEADERS" $extra_args

docker/falco/Dockerfile

@@ -44,7 +44,7 @@ RUN apt-get update \
 RUN curl -s https://falco.org/repo/falcosecurity-packages.asc | apt-key add - \
 	&& echo "deb https://download.falco.org/packages/${VERSION_BUCKET} stable main" | tee -a /etc/apt/sources.list.d/falcosecurity.list \
 	&& apt-get update -y \
-	&& if [ "$FALCO_VERSION" = "latest" ]; then apt-get install -y --no-install-recommends falco; else apt-get install -y --no-install-recommends falco=${FALCO_VERSION}; fi \
+	&& if [ "$FALCO_VERSION" = "latest" ]; then FALCO_DRIVER_CHOICE=none apt-get install -y --no-install-recommends falco; else FALCO_DRIVER_CHOICE=none apt-get install -y --no-install-recommends falco=${FALCO_VERSION}; fi \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/*
 

docker/falco/docker-entrypoint.sh

@@ -24,19 +24,23 @@ print_usage() {
 	echo "  docker run -i -t --privileged -v /root/.falco:/root/.falco -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro -v /etc:/host/etc:ro -e 'FALCO_DRIVER_LOADER_OPTIONS=[driver] [options]' falcosecurity/falco:latest"
 	echo ""
 	echo "Available FALCO_DRIVER_LOADER_OPTIONS drivers:"
-	echo "  kmod           kernel module (default)"
+	echo "  auto	       leverage automatic driver selection logic (default)"
+	echo "  modern_ebpf    modern eBPF CORE probe"
+	echo "  kmod           kernel module"
 	echo "  ebpf           eBPF probe"
 	echo ""
 	echo "FALCO_DRIVER_LOADER_OPTIONS options:"
-	echo "  --help         show this help message"
-	echo "  --clean        try to remove an already present driver installation"
-	echo "  --compile      try to compile the driver locally (default true)"
-	echo "  --download     try to download a prebuilt driver (default true)"
-	echo "  --print-env    skip execution and print env variables for other tools to consume"
+	echo "  --help           show this help message"
+	echo "  --clean          try to remove an already present driver installation"
+	echo "  --compile        try to compile the driver locally (default true)"
+	echo "  --download       try to download a prebuilt driver (default true)"
+ 	echo "  --http-insecure	 enable insecure downloads"
+	echo "  --print-env      skip execution and print env variables for other tools to consume"
 	echo ""
 	echo "Environment variables:"
-	echo "  FALCOCTL_DRIVER_REPOS             specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
-	echo "  FALCOCTL_DRIVER_NAME              specify a different name for the driver"
+	echo "  FALCOCTL_DRIVER_REPOS         specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
+	echo "  FALCOCTL_DRIVER_NAME          specify a different name for the driver"
+	echo "  FALCOCTL_DRIVER_HTTP_HEADERS  specify comma separated list of http headers for driver download (e.g. 'x-emc-namespace: default,Proxy-Authenticate: Basic')"
 	echo ""
 }
 
@@ -57,19 +61,19 @@ if [[ -z "${SKIP_DRIVER_LOADER}" ]]; then
 
     ENABLE_COMPILE="false"
     ENABLE_DOWNLOAD="false"
-    has_driver=
+    HTTP_INSECURE="false"
+    driver=
     has_opts=
     for opt in "${falco_driver_loader_option_arr[@]}"
     do
         case "$opt" in
-            kmod|ebpf)
-                if [ -n "$has_driver" ]; then
+            auto|kmod|ebpf|modern_ebpf)
+                if [ -n "$driver" ]; then
                     >&2 echo "Only one driver per invocation"
                     print_usage
                     exit 1
                 else
-                    /usr/bin/falcoctl driver config --type $opt
-                    has_driver="true"
+                    driver=$opt
                 fi
                 ;;
             -h|--help)
@@ -88,32 +92,44 @@ if [[ -z "${SKIP_DRIVER_LOADER}" ]]; then
                 ENABLE_DOWNLOAD="true"
                 has_opts="true"
                 ;;
-            --source-only)
-                >&2 echo "Support dropped in Falco 0.37.0."
-                print_usage
-                exit 1
-                ;;    
+            --http-insecure)
+                HTTP_INSECURE="true"
+                ;;
             --print-env)
                 /usr/bin/falcoctl driver printenv
                 exit 0
                 ;;
             --*)
-                >&2 echo "Unknown option: $1"
+                >&2 echo "Unknown option: $opt"
                 print_usage
                 exit 1
                 ;;
             *)
-                >&2 echo "Unknown driver: $1"
+                >&2 echo "Unknown driver: $opt"
                 print_usage
                 exit 1
                 ;;
         esac
     done
+
+    # No opts passed, enable both compile and download
     if [ -z "$has_opts" ]; then
         ENABLE_COMPILE="true"
         ENABLE_DOWNLOAD="true"
     fi
-    /usr/bin/falcoctl driver install --compile=$ENABLE_COMPILE --download=$ENABLE_DOWNLOAD
+
+    # Default value: auto
+    if [ -z "$driver" ]; then
+      driver="auto"
+    fi
+
+    if [ "$driver" != "auto" ]; then
+      /usr/bin/falcoctl driver config --type $driver
+    else
+      # Needed because we need to configure Falco to start with correct driver
+      /usr/bin/falcoctl driver config --type modern_ebpf --type kmod --type ebpf
+    fi
+    /usr/bin/falcoctl driver install --compile=$ENABLE_COMPILE --download=$ENABLE_DOWNLOAD --http-insecure=$HTTP_INSECURE --http-headers="$FALCOCTL_DRIVER_HTTP_HEADERS"
 
 fi
 

docker/no-driver/Dockerfile

@@ -28,7 +28,7 @@ LABEL org.opencontainers.image.source="https://github.com/falcosecurity/falco"
 LABEL usage="docker run -i -t --privileged -v /var/run/docker.sock:/host/var/run/docker.sock -v /dev:/host/dev -v /proc:/host/proc:ro --name NAME IMAGE"
 # NOTE: for the "least privileged" use case, please refer to the official documentation
 
-RUN apt-get -y update && apt-get -y install ca-certificates curl jq \
+RUN apt-get -y update && apt-get -y install ca-certificates curl jq libelf1 \
     && apt clean -y && rm -rf /var/lib/apt/lists/*
 
 ENV HOST_ROOT /host

docker/no-driver/Dockerfile.distroless

@@ -6,7 +6,7 @@ ARG VERSION_BUCKET=bin
 ENV FALCO_VERSION=${FALCO_VERSION}
 ENV VERSION_BUCKET=${VERSION_BUCKET}
 
-RUN apk update && apk add build-base gcc curl ca-certificates jq
+RUN apk update && apk add build-base gcc curl ca-certificates jq elfutils
 
 WORKDIR /
 
@@ -21,7 +21,7 @@ RUN FALCO_VERSION_URLENCODED=$(echo -n ${FALCO_VERSION}|jq -sRr @uri) && \
 RUN sed -e 's/time_format_iso_8601: false/time_format_iso_8601: true/' < /falco/etc/falco/falco.yaml > /falco/etc/falco/falco.yaml.new \
     && mv /falco/etc/falco/falco.yaml.new /falco/etc/falco/falco.yaml
 
-FROM cgr.dev/chainguard/glibc-dynamic
+FROM cgr.dev/chainguard/wolfi-base
 
 LABEL maintainer="cncf-falco-dev@lists.cncf.io"
 LABEL org.opencontainers.image.source="https://github.com/falcosecurity/falco"
@@ -29,6 +29,8 @@ LABEL org.opencontainers.image.source="https://github.com/falcosecurity/falco"
 LABEL usage="docker run -i -t --privileged -v /var/run/docker.sock:/host/var/run/docker.sock -v /dev:/host/dev -v /proc:/host/proc:ro --name NAME IMAGE"
 # NOTE: for the "least privileged" use case, please refer to the official documentation
 
+RUN apk update && apk add libelf libstdc++
+
 ENV HOST_ROOT /host
 ENV HOME /root
 

falco.yaml

@@ -25,8 +25,10 @@
 #
 #     (Falco command-line arguments)
 #     (Falco environment variables)
+# Falco config files
+#     configs_files
 # Falco rules files
-#     rules_file
+#     rules_files
 # Falco engine
 #     engine
 # Falco plugins
@@ -61,14 +63,12 @@
 # Falco logging / alerting / metrics related to software functioning (advanced)
 #     output_timeout
 #     syscall_event_timeouts
-#     syscall_event_drops
+#     syscall_event_drops -> [CHANGE NOTICE] Automatic notifications will be simplified in Falco 0.38! If you depend on the detailed drop counters payload, use 'metrics.output_rule' along with 'metrics.kernel_event_counters_enabled' instead
 #     metrics
 # Falco performance tuning (advanced)
-#     syscall_buf_size_preset [DEPRECATED] -> Replaced by `engine.<driver>.buf_size_preset` starting Falco 0.38!
-#     syscall_drop_failed_exit [DEPRECATED] -> Replaced by `engine.<driver>.drop_failed_exit` starting Falco 0.38! 
 #     base_syscalls
-#     modern_bpf.cpus_for_each_syscall_buffer [DEPRECATED] -> Replaced by `engine.modern_ebpf.cpus_for_each_buffer` starting Falco 0.38! 
-
+# Falco libs
+#     falco_libs
 
 ################################
 # Falco command-line arguments #
@@ -96,10 +96,6 @@
 #   when deploying Falco over a container with read-only host mounts instead of
 #   directly on the host. Defaults to "/host". 
 #
-# - !!! [DEPRECATED] FALCO_BPF_PROBE: Specify a custom path to the BPF object code file (`bpf`
-#   driver). This is not needed for the modern_bpf driver. 
-#   -> Replaced by `engine.kind: ebpf` and `engine.ebpf` starting Falco 0.38!
-#
 # - FALCO_HOSTNAME: Customize the hostname output field logged by Falco by
 #   setting the "FALCO_HOSTNAME" environment variable.
 #
@@ -120,15 +116,34 @@
 # disabling the automatic artifacts followed by falcoctl.
 
 
+#######################
+# Falco configs files #
+#######################
+
+# [Stable] `configs_files`
+#
+# Falco will load additional configs files specified here.
+# Their loading is assumed to be made *after* main config file has been processed,
+# exactly in the order they are specified.
+# Therefore, loaded config files *can* override values from main config file.
+# Also, nested include is not allowed, ie: included config files won't be able to include other config files.
+#
+# Like for 'rules_files', specifying a folder will load all the configs files present in it in a lexicographical order.
+configs_files:
+  - /etc/falco/config.d
+
 #####################
 # Falco rules files #
 #####################
 
-# [Stable] `rules_file`
+# [Stable] `rules_files`
+
+NOTICE: Before Falco 0.38, this config key was `rules_file` (singular form), which is now deprecated in favor of `rules_files` (plural form).
 #
 # Falco rules can be specified using files or directories, which are loaded at
-# startup. The name "rules_file" is maintained for backwards compatibility. If
-# the entry is a file, it will be read directly. If the entry is a directory,
+# startup.
+#
+# If the entry is a file, it will be read directly. If the entry is a directory,
 # all files within that directory will be read in alphabetical order.
 #
 # The falco_rules.yaml file ships with the Falco package and is overridden with
@@ -157,7 +172,7 @@
 # "first match wins" principle. However, enabling the `all` matching option may result
 # in a performance penalty. We recommend carefully testing this alternative setting  
 # before deploying it in production. Read more under the `rule_matching` configuration.
-rules_file:
+rules_files:
   - /etc/falco/falco_rules.yaml
   - /etc/falco/falco_rules.local.yaml
   - /etc/falco/rules.d
@@ -328,7 +343,7 @@ rules_file:
 # buffers (higher `cpus_for_each_buffer`) can lower the memory footprint.
 #
 engine:
-  kind: kmod
+  kind: modern_ebpf
   kmod:
     buf_size_preset: 4
     drop_failed_exit: false
@@ -793,7 +808,7 @@ output_timeout: 2000
 syscall_event_timeouts:
   max_consecutives: 1000
 
-# [Stable] `syscall_event_drops`
+# [Stable] `syscall_event_drops` -> [CHANGE NOTICE] Automatic notifications will be simplified in Falco 0.38! If you depend on the detailed drop counters payload, use 'metrics.output_rule' along with 'metrics.kernel_event_counters_enabled' instead
 #
 # Generates "Falco internal: syscall event drop" rule output when `priority=debug` at minimum
 #
@@ -993,76 +1008,6 @@ metrics:
 # Falco performance tuning (advanced) #
 #######################################
 
-# [DEPRECATED] `syscall_buf_size_preset` -> Replaced by `engine.<driver>.buf_size_preset` starting Falco 0.38!
-#
-# Deprecated in favor of engine.{kmod,ebpf,modern_ebpf}.buf_size_preset.
-# This config is evaluated only if the default `engine` config block is not changed,
-# otherwise it is ignored.
-#
-# --- [Description]
-#
-# The syscall buffer index determines the size of the shared space between Falco
-# and its drivers. This shared space serves as a temporary storage for syscall
-# events, allowing them to be transferred from the kernel to the userspace
-# efficiently. The buffer size for each online CPU is determined by the buffer
-# index, and each CPU has its own dedicated buffer. Adjusting this index allows
-# you to control the overall size of the syscall buffers.
-#
-# --- [Usage]
-#
-# The index 0 is reserved, and each subsequent index corresponds to an
-# increasing size in bytes. For example, index 1 corresponds to a size of 1 MB,
-# index 2 corresponds to 2 MB, and so on:
-#
-# [(*), 1 MB, 2 MB, 4 MB, 8 MB, 16 MB, 32 MB, 64 MB, 128 MB, 256 MB, 512 MB]
-#   ^    ^     ^     ^     ^     ^      ^      ^       ^       ^       ^
-#   |    |     |     |     |     |      |      |       |       |       |
-#   0    1     2     3     4     5      6      7       8       9       10
-#
-#
-# The buffer dimensions in bytes are determined by the following requirements:
-# (1) a power of 2.
-# (2) a multiple of your system_page_dimension.
-# (3) greater than `2 * (system_page_dimension).
-#
-# The buffer size constraints may limit the usability of certain indexes. Let's
-# consider an example to illustrate this:
-#
-# If your system has a page size of 1 MB, the first available buffer size would
-# be 4 MB because 2 MB is exactly equal to 2 * (system_page_size), which is not
-# sufficient as we require more than 2 * (system_page_size). In this example, it
-# is evident that if the page size is 1 MB, the first index that can be used is 3.
-#
-# However, in most cases, these constraints do not pose a limitation, and all
-# indexes from 1 to 10 can be used. You can check your system's page size using
-# the Falco `--page-size` command-line option.
-# 
-# --- [Suggestions]
-#
-# The buffer size was previously fixed at 8 MB (index 4). You now have the
-# option to adjust the size based on your needs. Increasing the size, such as to
-# 16 MB (index 5), can reduce syscall drops in heavy production systems, but may
-# impact performance. Decreasing the size can speed up the system but may
-# increase syscall drops. It's important to note that the buffer size is mapped
-# twice in the process' virtual memory, so a buffer of 8 MB will result in a 16
-# MB area in virtual memory. Use this parameter with caution and only modify it
-# if the default size is not suitable for your use case.
-syscall_buf_size_preset: 4
-
-# [DEPRECATED] `syscall_drop_failed_exit` -> Replaced by `engine.<driver>.drop_failed_exit` starting Falco 0.38! 
-#
-# Deprecated in favor of engine.{kmod,ebpf,modern_ebpf}.drop_failed_exit.
-# This config is evaluated only if the default `engine` config block is not changed,
-# otherwise it is ignored.
-#
-# Enabling this option in Falco allows it to drop failed system call exit events
-# in the kernel drivers before pushing them onto the ring buffer. This
-# optimization can result in lower CPU usage and more efficient utilization of
-# the ring buffer, potentially reducing the number of event losses. However, it
-# is important to note that enabling this option also means sacrificing some
-# visibility into the system.
-syscall_drop_failed_exit: false
-
 # [Stable] `base_syscalls`, use with caution, read carefully
 #
 # --- [Description]
@@ -1178,89 +1123,28 @@ base_syscalls:
   custom_set: []
   repair: false
 
-# [DEPRECATED] `modern_bpf.cpus_for_each_syscall_buffer`, modern_bpf only -> Replaced by `engine.modern_ebpf.cpus_for_each_buffer` starting Falco 0.38! 
+##############
+# Falco libs #
+##############
+
+# [Experimental] `falco_libs` - Potentially subject to more frequent changes
 #
-# Deprecated in favor of engine.modern_ebpf.cpus_for_each_buffer.
-# This config is evaluated only if the default `engine` config block is not changed,
-# otherwise it is ignored.
+# `thread_table_size`
 #
-# --- [Description]
+# Set the maximum number of entries (the absolute maximum value can only be MAX UINT32) 
+# for Falco's internal threadtable (process cache). Please note that Falco operates at a 
+# granular level, focusing on individual threads. Falco rules reference the thread leader 
+# as the process. The size of the threadtable should typically be much higher than the 
+# number of currently alive processes. The default value should work well on modern 
+# infrastructures and be sufficient to absorb bursts.
 #
-# The modern_bpf driver in Falco utilizes the new BPF ring buffer, which has a
-# different memory footprint compared to the current BPF driver that uses the
-# perf buffer. The Falco core maintainers have discussed the differences and
-# their implications, particularly in Kubernetes environments where limits need
-# to be carefully set to avoid interference with the Falco daemonset deployment
-# from the OOM killer. Based on guidance received from the kernel mailing list,
-# it is recommended to assign multiple CPUs to one buffer instead of allocating
-# a buffer for each CPU individually. This helps optimize resource allocation
-# and prevent potential issues related to memory usage.
-#
-# This is an index that controls how many CPUs you want to assign to a single
-# syscall buffer (ring buffer). By default, for modern_bpf every syscall buffer
-# is associated to 2 CPUs, so the mapping is 1:2. The modern BPF probe allows
-# you to choose different mappings, for example, changing the value to `1`
-# results in a 1:1 mapping and would mean one syscall buffer for each CPU (this
-# is the default for the `bpf` driver).
-#
-# --- [Usage]
-#
-# You can choose an index from 0 to MAX_NUMBER_ONLINE_CPUs to set the dimension
-# of the syscall buffers. The value 0 represents a single buffer shared among
-# all online CPUs. It serves as a flexible option when the exact number of
-# online CPUs is unknown. Here's an example to illustrate this:
-#
-# Consider a system with 7 online CPUs:
-#
-#          CPUs     0  X  2  3  X  X  6  7  8  9   (X means offline CPU)
-#
-# - `1` means a syscall buffer for each CPU so 7 buffers
-#
-#          CPUs     0  X  2  3  X  X  6  7  8  9   (X means offline CPU)
-#                   |     |  |        |  |  |  |
-#       BUFFERs     0     1  2        3  4  5  6
-#
-# - `2` (Default value) means a syscall buffer for each CPU pair, so 4 buffers
-#
-#          CPUs     0  X  2  3  X  X  6  7  8  9   (X means offline CPU)
-#                   |     |  |        |  |  |  |
-#       BUFFERs     0     0  1        1  2  2  3
-#
-# Please note that in this example, there are 4 buffers in total. Three of the
-# buffers are associated with pairs of CPUs, while the last buffer is mapped to
-# a single CPU. This arrangement is necessary because we have an odd number of
-# CPUs.
-#
-# - `0` or `MAX_NUMBER_ONLINE_CPUs` mean a syscall buffer shared between all
-#   CPUs, so 1 buffer
-#
-#          CPUs     0  X  2  3  X  X  6  7  8  9   (X means offline CPU)
-#                   |     |  |        |  |  |  |
-#       BUFFERs     0     0  0        0  0  0  0
-#
-# Moreover, you have the option to combine this parameter with
-# `syscall_buf_size_preset` index. For instance, you can create a large shared
-# syscall buffer of 512 MB (using syscall_buf_size_preset=10) that is
-# allocated among all the online CPUs.
-#
-# --- [Suggestions]
-#
-# The default choice of index 2 (one syscall buffer for each CPU pair) was made
-# because the modern bpf probe utilizes a different memory allocation strategy
-# compared to the other two drivers (bpf and kernel module). However, you have
-# the flexibility to experiment and find the optimal configuration for your
-# system.
-# 
-# When considering a fixed syscall_buf_size_preset and a fixed buffer dimension:
-# - Increasing this configs value results in lower number of buffers and you can
-#   speed up your system and reduce memory usage
-# - However, using too few buffers may increase contention in the kernel,
-#   leading to a slowdown.
-# 
-# If you have low event throughputs and minimal drops, reducing the number of
-# buffers (higher `cpus_for_each_syscall_buffer`) can lower the memory footprint.
-modern_bpf:
-  cpus_for_each_syscall_buffer: 2
+# Reducing its size can help in better memory management, but as a consequence, your 
+# process tree may be more frequently disrupted due to missing threads. You can explore 
+# `metrics.state_counters_enabled` to measure how the internal state handling is performing, 
+# and the fields called `n_drops_full_threadtable` or `n_store_evts_drops` will inform you
+# if you should increase this value for optimal performance.
+falco_libs:
+  thread_table_size: 262144
 
 # [Stable] Guidance for Kubernetes container engine command-line args settings
 #

proposals/20230620-anomaly-detection-framework.md

@@ -0,0 +1,112 @@
+# On Host Anomaly Detection Framework - New `anomalydetection` Plugin
+
+## Motivation
+
+**A Wind of Change for Threat Detection**
+
+Feel that light breeze? That is the continued advancement of cloud native security blowing steady. But despite our progress, threat actors are outpacing our innovation constantly finding new ways to thwart and tornado past our achievements — rule-based detections focus on what we *think* attackers will do, not on what they *are* doing and generate enough alerts to bury security analysts in a sandstorm of poor signal-to-noise. Can this dynamic be blown back to shift the information asymmetry in favor of defenders?
+
+This framework lays the foundation on how to create high-value, kernel signals that are difficult to bypass - but not in the traditional way. Advanced data analytics is an emerging crosswind that enables us to soar past attackers by detecting deviations in current behavior from past behavior. 
+
+## Benefits to the Ecosystem
+
+Advanced data analytics enables us to combine the intricacies of the Linux kernel with on-host anomaly detection in cloud native and cloud environments to determine patterns of past behavior in running applications. By detecting deviations in current behavior from past behavior, we can shift the focus away from relying solely on signatures and rule matching to catch attackers.
+
+Threat detection in open source and more importantly cloud native is constrained by the amount of rules we can write and the signatures we know to look for in our environments. But these have the same problem: they assume our attackers don't change what they're doing. The reality is attackers are not limited to the ways, means, and methods they employ to expose, manipulate, or even destroy our data, systems, and organizations. 
+
+This framework leverages an attacker's mindset applied to detection engineering: observing and learning about our targets to create more rich and actionable alerts so we can catch them earlier and more often - regardless if it's behavior we know about, or something we haven't seen yet.
+
+## Elevator Pitch
+
+When Falco processes events in userspace, its rules engine filters the events while the parsers simultaneously update and maintain an internal state. This state includes a process tree cache that enhances Falco alerts by providing contextual information derived from previous events. The goal is to enhance the "state engine" even further and provide an option for monitoring the behavior of applications over time.
+
+To achieve this, end users define a "behavior profile" in the configuration by combining existing event fields such as process name, file descriptor (fd), executable path, parent lineage, cmdline, and others. During event parsing on the hot path, Falco compresses and stores this information in a "filter" - an efficient probabilistic data structure that optimizes space, time, robustness and accuracy. As time progresses, Falco provides more accurate estimates of application behavior counts and identifies events as rare or heavy hitters. Instead of analyzing the original event stream, you can write Falco rules based on pre-filtered data. 
+
+This approach introduces a novel threat detection framework that analyzes abnormal application behavior in real-time, derived and observed in a data-driven fashion, without requiring operator reconfiguration of Falco. It complements the operator's expertise and extends capabilities similar to our current practices. The new capability draws inspiration from big data stream and database query optimizations, ensuring that Falco maintains a streamlined real-time one-pass stream with zero allocations.
+
+Similar to Falco rules, the analysis of events may require multiple behavior profiles of different dimensions based on sets of events. These profiles can either vote in parallel or in a cascading fashion, a common practice in established algorithms. This is just the beginning and and paves the way for more sophisticated approaches, such as running Falco in a DAST-like capacity to build a pre-state pattern file on a workload with test data and soften the cold-start via distributing it to production.
+
+## Challenges and Considerations
+
+First, The Falco Project is committed to continuously ensuring access to the most accurate data possible for on-host threat detection. As an example, recent efforts involved expanding kernel signal logging, such as verifying if an execve call is linked to a file descriptor existing exclusively in memory or improving the efficient and reliable resolution of symlinks for executable paths. Therefore, the proposed anomaly detection framework operates under the assumption of having the *correct* data, thereby complementing the ongoing efforts to expand logging coverage and improve its quality. In summary, the primary focus of the framework is to derive increased value from the existing *right* data that is currently available.
+
+There is a common perception that attacks on running cloud applications, as well as their indicators of compromise, are typically rare when the appropriate data or combination of signals is considered. While this holds true, there are inherent challenges in applying this concept of rarity to robust data analytics approaches. 
+
+On the one hand, this is due to the diverse range of attacks and attack vectors. An attacker may introduce a new malicious binary (which is comparatively easier to detect using traditional rules and high-value kernel signals) after gaining initial access. Alternatively, they may exploit existing binaries, shell built-ins, and employ obfuscation techniques to "live off the land". The Turing completeness of the latter scenario, in particular, leads to an infinite number of attack possibilities.
+
+However, what poses even more challenges in anomaly detection lies not necessarily in the nature of attacks but rather in identifying the right signals and their appropriate combinations for robust analytics to distinguish between normal and anomalous behavior. This challenge becomes particularly evident when considering the natural fluctuations in application behavior over time and the occurrence of ad-hoc legitimate debugging activities. Such fluctuations can arise from various factors, including routine deployment updates. Moreover, certain applications may produce random file names or execute arbitrary executable paths as part of their regular operations, adding to the challenge of anomaly detection. This is compounded by the inherent "cold start" issue when initially observing an application. In such cases, the algorithms must demonstrate flexibility and robustness by recognizing and encoding consistent patterns, similar to how humans can identify the sameness by examining combinations of file names, command arguments, parent process lineage, and other attributes. Furthermore, factors like data inconsistency and the diverse forms of data representations (comprising a mix of numeric data and strings with varying meanings) further complicate the task.
+
+We believe it is important to incorporate operator heuristics or domain knowledge into the algorithm's definition of rarity. For example, while current algorithms are capable of generating human faces, they used to frequently produce images with different eye colors. However, if we were to inform the machine that humans typically have matching eye colors, it could easily correct this discrepancy. This highlights the role of the security engineer as a guiding hand to the algorithms, both in terms of handling noise tolerance and choosing the appropriate data to be ingested into the algorithm. This is crucial as machines are currently limited in their ability to draw meaningful observations from limited data and constrained memory. In summary, this is where the fusion of data-driven anomaly detection and rules matching will come into play.
+
+Lastly, the value proposition of conducting real-time anomaly analysis on the host lies in the unique options it offers, which cannot be achieved through alternative methods. On the host, we can observe anomalies based on all relevant and observed kernel events. In contrast, sending a large volume of kernel events to a centralized system would be impractical, resulting in significant costs for data pipeline management and data lake compute expenses.
+
+## Initial Scope
+
+The initial scope is to implement the Count Min Sketch algorithm using n shared sketches and expose its count estimates as new filterchecks for use in Falco rules. An MVP can be explored in this libs draft PR [wip: new(userspace/libsinsp): MVP CountMinSketch Powered Probabilistic Counting and Filtering](https://github.com/falcosecurity/libs/pull/1453). Moreover, the initial anomaly detection framework will include a transparent `plugin` user interface for defining application behavior profiles and utilizing sketch count estimates in Falco rules. The primary direct benefit lies in establishing a safety boundary for Falco rules in production environments, allowing for broader rule monitoring while preventing Falco rules from blowing up in production.
+
+Furthermore, The Falco Project will provide adopters with valuable initial use cases, recommended thresholds, and callouts for known issues. One important consideration is the identification of SRE anti-patterns. Another consideration is to provide *very clear* guidance to adopters for setting and configuring parameters, including recommended minimums. Additionally, guidance should be provided on indicators to look for in order to determine if adjustments need to be made and in which direction, particularly when defining application behavior profiles.
+
+## High-Level Technical Design of a New `anomalydetection` Plugin
+
+This document provides a high-level proposal with limited technical details.
+
+*Probabilistic Data Structures*
+
+One option for implementing the probabilistic filter is by utilizing a robust two-dimensional probabilistic data structure known as the Count Min Sketch. This data structure is widely employed in distributed stream processing frameworks such as Apache Spark, Apache Storm, Apache Flink, and others, as well as databases like Redis and PostgreSQL.
+
+Technical details and implications are extensively covered in numerous research papers and textbooks. Therefore, here are some key points to consider in order to make informed choices:
+
+- The challenges posed by both hard and soft collisions can be mitigated by using multiple non-cryptographic hash functions, which has been mathematically proven to be effective.
+- Despite providing one-sided error bounds and preventing undercounting, the sketchy data structure requires adopters to define a tolerance level for overcounting. This tolerance level determines what qualifies as rare or noteworthy.
+- To enhance accuracy and reduce estimation errors, consider debiasing data (e.g. Count Min Sketch with Conservative Updates) or applying a logarithmic scale to address kernel event data skew. The logarithmic scale may suit threat detection, targeting low-frequency or long-tail attack-related items. However, only use if performance overhead is acceptable.
+- Use larger shared sketches and incorporate container IDs as part of the behavior profiles to differentiate between workloads / applications. Conversely, use separate sketches for distinct behavior profiles, also known as the "what we are counting".
+- ... and numerous other aspects that will be discussed in subsequent implementation PRs.
+
+*Plumbing and Interface*
+
+The ultimate goal is to introduce these new capabilities as plugin. A significant amount of work will be dedicated to addressing the necessary plumbing required to support the new framework and integrate it with the existing rules filtering, `libsinsp` and `plugin` mechanisms. This integration aims to provide a user-friendly interface that allows users to easily configure and utilize the opt-in framework for different use cases. 
+
+For instance, the interface should empower end users to define error tolerances and, consequently, sketch dimensions, along with other tuning parameters, bounds, and settings. Ultimately, it should enable the definition of n behavior profiles to facilitate the use of count estimates in Falco rules.
+
+## What this Framework is Not
+
+- This framework is not intended to function as an event aggregator or enhancer, such as netflow data. Its sole purpose is to serve as an anomaly filter for individual events, utilizing the existing sinsp state, the newly built state through sketches, and the current rules engine.
+- The development of this framework will not be swayed by overly specific use cases that limit its broader adoption and coverage.
+- While it may not offer flawless attack threat detection from the beginning, it serves as an initial step towards comprehensive event logging and analysis, capturing all events that exhibit any form of new or changing behavior we observe. Therefore, initially, the greatest value lies in combining it with regular Falco rules based on the anomaly-filtered event stream.
+
+## Why now?
+
+In case you haven't noticed, advanced data analytics is quite the big deal these days, and we can leverage robust established algorithms used in real production settings across various industries. The novelty lies in addressing the specific data encoding challenges unique to the field of cybersecurity, not re-inventing already established algorithms.
+
+Furthermore, over the past several Falco releases, we have significantly improved stability, configurability, and capabilities. Notably, the plugins system has been refined over the past year to efficiently access the complete `libsinsp` state, now also featuring an improved CPP SDK. Additionally, it now seamlessly collaborates with the existing primary syscalls event source, deviating from its original purpose of processing new data sources. This improvement allows for more intuitive functionality, as demonstrated by the new `k8smeta` plugin. Now is the opportune time to further enhance proven threat detection capabilities and expand the plugins system even more. 
+
+*Initial community feedback concerning the KubeCon NA 2023 Full Talk*
+
+- Overall, the feedback for [A Wind of Change for Threat Detection](https://kccncna2023.sched.com/event/1R2mX/a-wind-of-change-for-threat-detection-melissa-kilby-apple) was very positive and appreciative, particularly regarding the direct real-life benefits (a safety boundary for Falco rules enabling broader monitoring that won't blow up in production). Suggestions for future development included integrating the sketch directly into the kernel driver (which would be a remarkable achievement if feasible). Lastly, people have inquired about the timeline for the availability of this feature.
+- Refer to the [KubeCon NA 2023 Slides](https://static.sched.com/hosted_files/kccncna2023/c5/A%20Wind%20of%20Change%20for%20Threat%20Detection%20-%20Melissa%20Kilby%20-%20KubeCon%20NA%202023.pdf) or [attached PDF](kubeconna23-anomaly-detection-slides.pdf) for more information. Here's the [Talk Recording](https://www.youtube.com/watch?v=1y1m9Vz93Yo) (please note that the first four minutes of the video are missing, but the slides and audio recordings are complete).
+
+*Falco Community Call - January 17, 2024*
+
+See dedicated [HackMD](https://hackmd.io/Ss0_1avySUuxArBQm-oaGQ?view):
+
+- While not blocking the start of the plugin or an alpha dev version, there's feedback from @jasondellaluce that plugins cannot access the existing `libsinsp` filtercheck. It would be advantageous to enable this access to avoid reimplementing them and the constant risk of falling out of sync with `libs`. @leogr mentioned that supporting this over time should be possible.
+- We have discussed the plugins config and are currently undecided on whether the definition of the behavior profile per sketch, meaning the fields that are string concatenated together and counted, should reside in the plugins config or in the rules files. The latter would potentially require a new rules component. Final decisions will be deferred to a later stage to ensure the config is intuitive, and we want to guarantee proper sketch definition when attempting to run Falco rules using the `anomalydetection` plugin.  
+- One use case, namely determining if a rule has previously occurred in a container, could be addressed by this framework as well. However, we are currently unsure how to expose the rule names, as `libsinsp` is not aware of them. This may be an optimization we can address later and does not block the development of an initial version.
+- Future use cases might involve counting distinct values, utilizing the hyper log log algorithm. However, there will be additional technical challenges to overcome.
+- Finally, just to reiterate some feedback from the KubeCon talk, there's a suggestion that, perhaps in the future, we could pass intelligence back and forth between the drivers and userspace. This idea has been discussed independently, especially in the context of kernel-side filtering. However, such capabilities would be a long-term consideration.
+
+## Proposed Timelines
+
+- Falco 0.37.0: Design details and scaffolding
+- Falco 0.38.0: Experimental release
+- Falco 0.39.0: First release
+
+## Resources / References
+
+- [Probabilistic Data Structures and Algorithms
+for Big Data Applications](https://www.gakhov.com/books/pdsa.html) book
+- [Count Min Sketch blog 1](https://towardsdatascience.com/big-data-with-sketchy-structures-part-1-the-count-min-sketch-b73fb3a33e2a)
+- [Count Min Sketch blog 2](https://www.synnada.ai/blog/probabilistic-data-structures-in-streaming-count-min-sketch)
+- [Count Min Log Sketch](https://arxiv.org/pdf/1502.04885.pdf) paper
+- [Count Min Sketch with Conservative Updates](https://hal.science/hal-03613957/document#:~:text=Count%2DMin%20Sketch%20with%20Conservative%20Updates%20(CMS%2DCU),because%20of%20its%20inherent%20difficulty) paper
+- [xxHash](https://github.com/Cyan4973/xxHash) as new dependency for fast and reliable hashing (using xxh3)

proposals/20231220-features-adoption-and-deprecation.md

@@ -0,0 +1,219 @@
+# Features Adoption and Deprecation Policies Proposal
+
+This proposal aims to introduce a balance between maintaining adopter trust and the need for The Falco Project to evolve. Historically, Falco has favored rapid evolution over providing long-term support for features and interfaces. However, some project subsystems have been implicitly assumed not to allow backward-incompatible changes (e.g., we have almost never removed a condition syntax field). These implicit conventions have never been formalized, and decisions in this regard have been left unspecified.
+
+## Goals
+
+- Establish adopter expectations on the operational cost of using Falco.
+- Provide a clear path for features to be adopted and dismissed.
+- Allow quick evolution and experimentation without disrupting our adopters' deployments.
+- Detail the process for introducing new features, following a "sandbox" to "incubating" to "stable" progression.
+- Define the scope of the policy, including which aspects of Falco are covered (e.g., command line flags, configuration files, rules syntax).
+- Establish stages for deprecating features, aligning with the project's current status (pre- and post-1.0 stages).
+- Adopt a semantic versioning (semver) approach.
+
+## Non-Goals
+
+- Define the number of previous releases that will receive patches or security updates and the duration of this support.
+- Define the criteria for Falco 1.0.
+
+## Scope
+
+The proposed policies apply to Falco, its subsystems (e.g., rules, the plugin system), and all [core projects](https://github.com/falcosecurity/evolution#core) which are deemed [stable](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#stable), thus officially supported by The Falco Project.
+
+## Definitions
+
+### Feature Changes
+
+A feature is a distinct and specific functionality or characteristic of Falco and its core components that provides value to the user by enabling them to perform particular tasks. Features encompass aspects such as functionality, user value, usability, integrability, scalability, configurability, and discoverability. Features can range from essential user interface elements to complex, multifunctional operations.
+
+A feature change refers to any modification or update to an existing feature or the addition of a new feature. This does not include documentation, Falco compatibility across different environments, platforms, systems, or other software or hardware, bug fixing (stated it does not require a feature change to overcome the problem), and performance (unless a change produces a measurable effect on usability).
+
+### Behavior Changes
+
+A behavior change refers to alterations in how Falco, or a specific feature within it, operates or responds under certain conditions. Unlike feature changes, behavior changes are more about tweaking the underlying logic or the way existing features interact or perform, particularly the expected behavior of Falco when run with the default configuration.
+
+Behaviors are generally documented. Any modification that does not meet the conditions and expectations of an already documented feature is assumed to be a behavior change.
+
+Undocumented behaviors may be included in this definition if there's strong evidence or suspicion that users rely on those undocumented behaviors.
+
+### User-Facing Changes
+
+User-facing changes refer to any feature changes, behavior changes, modifications, or additions that are directly noticeable and interactable by the end users. These changes affect how Falco operates from the user's perspective (notably any change that can lead to user disruption). Unlike internal changes (i.e., code refactoring, CI, maintenance-related changes), which are under-the-hood improvements not directly visible to the user, user-facing changes are evident in the Falco and its core components interface and functionality.
+
+### CLI/Config Area
+
+Falco is comprised of the Falco binary and other programs and tools cooperating (notably [falcoctl](https://github.com/falcosecurity/falcoctl)). These programs are the primary user interface for Falco. Any feature or behavior changes to the following elements of these programs are assumed to be user-facing changes to the CLI/Config area:
+
+- Program name.
+- Distribution mechanism and packaging (e.g., a container image).
+- Command line flags and options.
+- Environment variables.
+- Configurations.
+- Elements that affect the program's lifecycle (e.g., the effect of sending a SIGINT to the program).
+- Elements that allow scripting, automation, or interaction with other programs (e.g., piping and redirection).
+- Program inputs, excluding elements explicitly governed by other areas (e.g., [Falco rules](#rules-area)).
+- Program outputs excluding elements explicitly governed by other areas (e.g., [Falco outputs/alerts](#outputs-alerts-area)).
+
+### Rules System Area
+
+Rules are the primary input for Falco. Any feature or behavior changes to the following aspects or elements are assumed to be user-facing changes to the rules system area:
+
+- Syntax.
+- File format.
+- Schema (i.e., supported fields).
+- Elements that affect the way users can implement rules.
+- Elements that affect the way rules are triggered.
+
+However, any change related to the rule's output when triggered (i.e., the alert) is out of scope for this area (see next section).
+
+Note that this area does not include changes related to the ruleset files. Ruleset distributions follow their own [Rules Maturity Framework](https://github.com/falcosecurity/rules/blob/main/CONTRIBUTING.md#rules-maturity-framework) policies. 
+
+### Outputs/Alerts Area
+
+Alerts, delivered through Falco output channels, are Falco's primary output. The way and the format in which alerts are produced can have a significant impact on adopters. For example, removing a supported rule field also impacts this area, as adopters may have relied on that field when consuming Falco output.
+
+Any feature or behavior changes to the following aspects or elements are assumed to be user-facing changes to the Outputs/Alerts area:
+
+- Output and logging formats.
+- Schema of outputted data (i.e., supported fields).
+- Falco output channels.
+- Any element that might be consumed from the output.
+
+### Subsystem APIs (Plugins, gRPC Output, Metrics, etc.) Area
+
+Falco is also comprised of several subsystems providing specific APIs. These subsystems notably include plugin system API, gRPC output API, and metrics API.
+
+In the context of this proposal, only changes to **public APIs** are assumed to be user-facing changes to this area.
+
+Public APIs are defined as those supporting Falco functioning and explicitly intended for user usage. Internal APIs consumed by Falco or other tools are out of scope for this area. For instance, the driver APIs or libs APIs are intended to be mainly consumed by Falco and not by users.
+
+### Platform Support Area
+
+Platform support for Falco encompasses the range of platforms, systems, and environments it is designed to operate in. Platform support may significantly vary by Falco's data sources and use cases. For example, its compatibility differs when utilized for Kubernetes audit events versus system call events. Additionally, platform support can be influenced by deployment methods (e.g., directly on a host versus within Kubernetes) or configurations (e.g., running in privileged versus least privileged mode).
+
+Given the diversity of potential platforms and setups, only those explicitly listed in Falco's documentation are considered officially supported. While Falco may function on other platforms, official support is guaranteed solely for documented ones.
+
+Therefore, changes in platform compatibility or behavior that are documented explicitly assumed to be user-facing changes to the Platform Support area.
+
+### Release Cycle
+
+In the context of this proposal, a release cycle is the period between two consecutive major or minor releases of Falco. Hotfix/Patch releases must not be counted.
+
+The actual duration of a release cycle can vary. Still, it's assumed to be about 16 weeks (as per our current defined [Release Cycles and Development Iterations](https://github.com/falcosecurity/falco/blob/master/proposals/20230511-roadmap-management.md#release-cycles-and-development-iterations)). In case of future modification to the Falco release schedule, a period of minimum 3 months must be assumed.
+
+## Proposal
+
+### Maturation Levels
+
+Maturation levels (inspired by those we already have in place for [repositories](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#status)) are used to characterize the maturity of a feature. Each feature will have an assigned level  at any specific time (i.e., a Falco release). Levels are shown in the table below.
+
+| Maturity Level | Intended for |
+| --- | --- |
+| Sandbox |  Experimental/alpha features, not recommended for production use, can be removed at any time without further notice. |
+| Incubating | Beta features, long-term support is not guaranteed. |
+| Stable | General Availability (GA) features for which long-term support is expected. |
+| Deprecated | See the [deprecation policy](#Deprecation-policy) section below. |
+
+### Adoption Policy
+
+The adoption policy applies to any backward compatible user-facing changes which add functionalities. For non-backward compatible changes see the [deprecation policy](#Deprecation-policy) below.
+
+**Adoption rules**:
+1. A feature can be introduced at only one of the following levels:
+   - Sandbox: The feature must be opt-in (e.g., not enabled by default), labeled as *Sandbox* and the user must be proactively informed by the experimental nature of the feature (i.e. emitting a notice when the feature is being enabled).
+   - Incubating: The feature must be labeled as *Incubating*.
+2. Any functionality additions to an existing feature are inherently introduced at the same level as the feature itself unless logically separable (for instance, a sub-feature that may be enabled separately).
+3. A feature can be promoted *from Sandbox to Incubating* or *from Incubating to Stable* only after at least one release cycle has passed without user-facing changes to the feature.
+4. A feature cannot be demoted to a previous level.
+
+
+_Note about behaviors_:
+This policy indirectly applies to behaviors, too. Behavior changes are assumed to be a consequence of a feature change. The adoption level of a documented behavior is considered to be the same as the related feature. Furthermore, behavior changes are particularly relevant in the context of deprecation (see the next section).
+
+
+### Deprecation Policy
+
+The deprecation policy applies to any non-backward compatible user-facing changes. Any other changes introduced in a backward-compatible manner does not fall under the scope of this deprecation policy.
+
+**Deprecation rules**:
+1. Sandbox features can be removed or changed at any time without notice. No deprecation period is required.
+2. Incubating or Stable features and documented behaviors must enter a deprecation period and function for no less than the indicated release cycles (see tables below) after their announced deprecation. 
+   - If the change affects the feature partially, the deprecation applies only to that feature part. 
+   - If the change removes the feature entirely, the deprecation applies to the entire feature.
+3. At least for the entire deprecation period, the feature must be labeled as *Deprecated* in all relevant documentation, and:
+   - for deprecated configurations or CLI elements, a warning must be emitted warnings when the feature is being enabled or used;
+   - for deprecated APIs, when technically feasible, the API should be signal the deprecation status (this may vary depending on the specific subsystem);
+   - for deprecated behaviors the documentation must highlight the _before_ and _after_ behavior, alongside with a prominent deprecation notice.
+4. Any Pull Request introducing a deprecation notice must be labeled and include a note in the format `DEPRECATION NOTICE: ...`.
+5. Any Pull Request introducing a breaking change due to the end of the deprecation notice period must be labeled and include a note in the format `BREAKING CHANGE: ...`. 
+   - It is also recommended for code commits that introduce a breaking change to follow the related [conventional commit spec](https://www.conventionalcommits.org/en/v1.0.0/#specification).
+
+The minimum deprecation period length depends on the affected area. If a single change spans multiple areas, the area with the most extended deprecation period is assumed. Longer deprecation periods are allowed if the feature is deemed to be particularly critical or widely used.
+
+#### Deprecation Period Lengths
+
+_The units represent the number of releases._
+
+##### Before Falco 1.0
+
+| Area           | Stable | Incubating |
+| -------------- | ------ | ---------- |
+| *all areas*    | 1      | 0          |
+
+##### Since Falco 1.0 onward
+
+| Area           | Stable | Incubating |
+| -------------- | ------ | ---------- |
+| Behaviors      | 2      | 1          |
+| Rules System   | 2      | 1          |
+| Output/Alerts  | 2      | 1          |
+| Platform       | 2      | 1          |
+| CLI/Config     | 1      | 1          |
+| Subsystem APIs | 1      | 0          |
+
+### Examples
+
+**Example 1** Let's consider a feature _foo_ in the Output/Alerts Area introduced in Falco 1.0.0 and labeled as *Incubating*. The feature is promoted to *Stable* in Falco 1.1.0 (because the feature did not get any user-facing change).
+Subsequently, maintainers decide that backward-compatible changes must be introduced in _foo_ to improve its functionality. The part of the feature to be changed is labeled as *Deprecated* in Falco 1.2.0, and the deprecation period starts. The non-backward compatible change is then introduced in Falco 1.4.0.
+
+**Example 2** The `--bar` flag in the CLI/Config Area has been introduced since Falco 1.1.0 and is labeled as *Stable*. Before releasing Falco 1.5.0, maintainers realize `--bar` is redundant and should be removed. The flag is labeled as *Deprecated* in Falco 1.5.0, and the deprecation period starts. The flag is removed in Falco 1.6.0.
+
+### Exceptions
+
+- Ruleset in the official distributions follow the [Rules Maturity Framework](https://github.com/falcosecurity/rules/blob/main/CONTRIBUTING.md#rules-maturity-framework) policies.
+- Subsystems or subcomponents may have additional criteria and exceptions. Stated other criteria and exceptions must not directly affect the main Falco distribution (e.g., *falcoctl* can have a different release cycle and different policies; however, if Falco relies on a specific *falcoctl* feature, that *falcoctl* feature adoption and deprecation must be strictly compatible with the rules described in this proposal).
+- Internal APIs are out of scope for this policy. Their adoption models and deprecation policies might be regulated separately.
+- Different parties may provide plugins, and each plugin may have a different maturity level. Only those plugins officially maintained by The Falco Project and identified as "core" for Falco are in scope for this policy; all others are excluded.
+- Any other exceptions to the rules provided by this policy require a formal core maintainer majority vote.
+
+### Versioning
+
+Regarding the above policies, component versioning must adhere to [Semantic Versioning 2.0.0](https://semver.org/). However, in the context of Falco core components, the scope extends beyond the strict API definition and includes any user-facing changes.
+
+Thus, given a version number `MAJOR.MINOR.PATCH` increment the:
+
+- *MAJOR* version when the deprecation period of one or more _stable_ features ends, thus introducing incompatible user-facing or API changes.
+- *MINOR* version when adding functionality in a backward-compatible manner.
+- *PATCH* version when making backward-compatible bug fixes.
+
+Moreover, *MAJOR* version zero (0.y.z) is for versioning stabilization (i.e., before defining the public set of user-facing features and APIs). At this stage, the *MINOR* version is allowed to be incremented instead of the *MAJOR* version.
+
+### Documentation
+
+Documentation must be tied to a specific release and reflect the adoption level status of a feature at that specific release. In particular:
+
+- Deprecated items must be labeled `DEPRECATED` in all relevant documentation.
+- Stable items must be sufficiently documented. Explicitly labeling the Stable status is not required or recommended.
+- Incubating items must be sufficiently documented and labeled  `INCUBATING` in all relevant documentation.
+- Sandbox items may be partially documented and labeled `SANDBOX` in all relevant documentation, if any. The relevant documentation must also explicitly state the experimental nature of the item.
+
+## Transition Phases
+
+Since software components may need to adapt to implement the requirements this proposal mandates, we assume the following stages are required to transition from the current state to the desired state fully:
+
+- Within Falco 0.38, at least stable features must be identified, and the adoption policy and relevant documentation should be implemented in Falco. Exceptions may be made temporarily for the deprecation policy.
+- Within subsequent releases and no later than Falco 1.0.0 (still not scheduled to date), all the policies must be strictly implemented in Falco and documented in [falco.org](falco.org). The [Rules Maturity Framework](https://github.com/falcosecurity/rules/blob/main/CONTRIBUTING.md#rules-maturity-framework) must be adapted to ensure it aligns with the spirit of this proposal. Exceptions may be made temporarily for other [core projects](https://github.com/falcosecurity/evolution#core) with [stable](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#stable) status, assuming exceptions don't severely affect the main Falco distribution.
+- Within Falco 1.1.0, all the policies must be strictly implemented in Falco and in all [core projects](https://github.com/falcosecurity/evolution#core) with [stable](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#stable) status.
+
+During the transition phases, maintainers can fine-tune these policies and add further exceptions, eventually. After this initial transition phases, the policy is assumed to be established. From then on, any policy modifications, updates, and exceptions must be subject to a core maintainer majority vote to ensure the policy remains relevant and practical.

scripts/CMakeLists.txt

@@ -47,7 +47,18 @@ endif()
 if (NOT WIN32 AND NOT APPLE AND NOT EMSCRIPTEN AND NOT MUSL_OPTIMIZED_BUILD)
 	if(NOT DEFINED FALCOCTL_ETC_DIR)
 		set(FALCOCTL_ETC_DIR "${CMAKE_INSTALL_FULL_SYSCONFDIR}/falcoctl")
+    endif()
+    set(FALCOCTL_DRIVER_TYPES_LIST "")
+	if (BUILD_FALCO_MODERN_BPF)
+		list(APPEND FALCOCTL_DRIVER_TYPES_LIST "modern_ebpf")
 	endif()
+	if (BUILD_DRIVER)
+		list(APPEND FALCOCTL_DRIVER_TYPES_LIST "kmod")
+	endif()
+	if (BUILD_BPF)
+		list(APPEND FALCOCTL_DRIVER_TYPES_LIST "ebpf")
+	endif()
+	string(REPLACE ";" ", " FALCOCTL_DRIVER_TYPES "${FALCOCTL_DRIVER_TYPES_LIST}")
 	configure_file(${CMAKE_CURRENT_SOURCE_DIR}/falcoctl/falcoctl.yaml.in ${PROJECT_BINARY_DIR}/scripts/falcoctl/falcoctl.yaml)
 	install(FILES ${PROJECT_BINARY_DIR}/scripts/falcoctl/falcoctl.yaml DESTINATION "${FALCOCTL_ETC_DIR}" COMPONENT "${FALCO_COMPONENT_NAME}")
 endif()

scripts/debian/postinst.in

@@ -17,7 +17,8 @@
 # limitations under the License.
 #
 
-chosen_driver=
+# By default, we use the automatic selection for drivers
+chosen_driver="auto"
 chosen_unit=
 CHOICE=
 
@@ -38,43 +39,56 @@ systemctl --system disable 'falcoctl-artifact-follow.service' || true
 systemctl --system unmask falcoctl-artifact-follow.service || true
 
 if [ "$1" = "configure" ]; then
+	# "auto" case is not managed here since it is already the default, so no CHOICE=2
         case $FALCO_DRIVER_CHOICE in
-              kmod)
-                  CHOICE=2
+              none)
+                  CHOICE=1
                   ;;
-              ebpf)
+              kmod)
                   CHOICE=3
                   ;;
-              modern_ebpf)
+              ebpf)
                   CHOICE=4
                   ;;
+              modern_ebpf)
+                  CHOICE=5
+                  ;;
         esac     
         if [ -z $CHOICE ] && [ -x /usr/bin/dialog ] && [ "${FALCO_FRONTEND}" != "noninteractive" ]; then          
             # If dialog is installed, create a dialog to let users choose the correct driver for them
             CHOICE=$(dialog --clear --title "Falco drivers" --menu "Choose your preferred driver:" 12 55 4 \
                     1 "Manual configuration (no unit is started)" \
-                    2 "Kmod" \
-                    3 "eBPF" \
-                    4 "Modern eBPF" \
+                    2 "Automatic selection" \
+                    3 "Kmod" \
+                    4 "eBPF" \
+                    5 "Modern eBPF" \
                     2>&1 >/dev/tty)
-        fi         
+        fi
+        # "auto" case is not managed here since it is already the default, so no CHOICE=2
         case $CHOICE in
-            2)
-                chosen_driver="kmod"
-                chosen_unit="kmod"
+            1)
+                chosen_driver=""
                 ;;
             3)
-                chosen_driver="ebpf"
-                chosen_unit="bpf"
+                chosen_driver="kmod"
                 ;;
             4)
+                chosen_driver="ebpf"
+                ;;
+            5)
                 chosen_driver="modern_ebpf"
-                chosen_unit="modern-bpf"
                 ;;
         esac
-        if [ -n "$CHOICE" ]; then
-            echo "[POST-INSTALL] Configure falcoctl driver type:"
-            falcoctl driver config --type $chosen_driver
+        if [ -n "$chosen_driver" ]; then
+            echo "[POST-INSTALL] Configure falcoctl '$chosen_driver' driver type:"
+            if [ "$chosen_driver" = "auto" ];  then
+                # Configure falcoctl to enable all drivers
+                falcoctl driver config --type "modern_ebpf" --type "kmod" --type "ebpf"
+                # Load the actually automatic chosen driver
+                chosen_driver=$(falcoctl driver printenv | grep DRIVER= | cut -d'"' -f2)
+            else
+                falcoctl driver config --type "$chosen_driver"
+            fi
             CHOICE=
             case $FALCOCTL_ENABLED in
                 no)
@@ -108,10 +122,15 @@ case "$chosen_driver" in
       # Only compile for kmod, in this way we use dkms
       echo "[POST-INSTALL] Call 'falcoctl driver install for kmod:"
       falcoctl driver install --download=false
+      chosen_unit="kmod"
       ;;
     "ebpf")
       echo "[POST-INSTALL] Call 'falcoctl driver install for ebpf':"
       falcoctl driver install
+      chosen_unit="bpf"
+      ;;
+    "modern_ebpf")
+      chosen_unit="modern-bpf"
       ;;
 esac
 

scripts/falcoctl/falcoctl.yaml.in

@@ -1,5 +1,5 @@
 driver:
-    type: "kmod"
+    type: [@FALCOCTL_DRIVER_TYPES@]
     name: "@DRIVER_NAME@"
     repos:
         - "@DRIVERS_REPO@"
@@ -10,7 +10,7 @@ artifact:
     every: 6h0m0s
     falcoVersions: http://localhost:8765/versions
     refs:
-    - falco-rules:0
+    - falco-rules:3
 indexes:
 - name: falcosecurity
   url: https://falcosecurity.github.io/falcoctl/index.yaml

scripts/publish-deb

@@ -121,6 +121,47 @@ update_repo() {
     popd > /dev/null
 }
 
+reduce_dir_size() {
+    local DIR=$1
+    local MAX_SIZE_GB=$2
+    local EXTENSION=$3
+    local MAX_SIZE=$((MAX_SIZE_GB*1024*1024))  # Convert GB to KB for du command
+
+    # Check if directory exists
+    if [[ ! -d "$DIR" ]]; then
+        echo "The directory $DIR does not exist."
+        return 1
+    fi
+
+    # Calculate current directory size in KB
+    local CUR_SIZE=$(du -sk "$DIR" | cut -f1)
+
+    # Check if we need to delete any files
+    if ((CUR_SIZE <= MAX_SIZE)); then
+        return 0
+    fi
+
+    # Calculate size to delete in bytes
+    local DEL_SIZE=$(( (CUR_SIZE - MAX_SIZE) * 1024 ))
+
+    local ACC_SIZE=0
+    find "$DIR" -maxdepth 1 -type f -name "*.$EXTENSION" -printf "%T+ %s %p\n" | sort | while read -r date size file; do
+        if ((ACC_SIZE + size < DEL_SIZE)); then
+            rm "$file"
+            ACC_SIZE=$((ACC_SIZE + size))
+            
+            local asc_file="$file.asc"
+            if [[ -e "$asc_file" ]]; then
+                local asc_size=$(stat --format="%s" "$asc_file")
+                rm "$asc_file"
+                ACC_SIZE=$((ACC_SIZE + asc_size))
+            fi
+        else
+            break
+        fi
+    done
+}
+
 # parse options
 while getopts ":f::r::s" opt; do
     case "${opt}" in
@@ -188,6 +229,11 @@ if [ "${sign_all}" ]; then
   sign_repo ${tmp_repo_path} ${debSuite}
 fi
 
+# remove old dev packages if necessary
+if [[ ${repo} == "deb-dev" ]]; then
+  reduce_dir_size "${tmp_repo_path}/${debSuite}" 10 deb
+fi
+
 # update the repo by adding new packages
 if ! [ ${#files[@]} -eq 0 ]; then
   for file in "${files[@]}"; do
@@ -211,4 +257,10 @@ fi
 
 # sync dists
 aws s3 sync ${tmp_repo_path}/dists ${s3_bucket_repo}/dists --delete --acl public-read
-aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/dists/*
+aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/dists/*
+
+# delete packages that have been pruned
+# the dryrun option is there so we can check that we're doing the right thing, can be removed after testing
+if [[ ${repo} == "deb-dev" ]]; then
+  aws s3 sync "${tmp_repo_path}/${debSuite}" ${s3_bucket_repo} --dryrun --delete --acl public-read
+fi

scripts/publish-rpm

@@ -53,6 +53,47 @@ update_repo() {
     popd > /dev/null
 }
 
+reduce_dir_size() {
+    local DIR=$1
+    local MAX_SIZE_GB=$2
+    local EXTENSION=$3
+    local MAX_SIZE=$((MAX_SIZE_GB*1024*1024))  # Convert GB to KB for du command
+
+    # Check if directory exists
+    if [[ ! -d "$DIR" ]]; then
+        echo "The directory $DIR does not exist."
+        return 1
+    fi
+
+    # Calculate current directory size in KB
+    local CUR_SIZE=$(du -sk "$DIR" | cut -f1)
+
+    # Check if we need to delete any files
+    if ((CUR_SIZE <= MAX_SIZE)); then
+        return 0
+    fi
+
+    # Calculate size to delete in bytes
+    local DEL_SIZE=$(( (CUR_SIZE - MAX_SIZE) * 1024 ))
+
+    local ACC_SIZE=0
+    find "$DIR" -maxdepth 1 -type f -name "*.$EXTENSION" -printf "%T+ %s %p\n" | sort | while read -r date size file; do
+        if ((ACC_SIZE + size < DEL_SIZE)); then
+            rm "$file"
+            ACC_SIZE=$((ACC_SIZE + size))
+            
+            local asc_file="$file.asc"
+            if [[ -e "$asc_file" ]]; then
+                local asc_size=$(stat --format="%s" "$asc_file")
+                rm "$asc_file"
+                ACC_SIZE=$((ACC_SIZE + asc_size))
+            fi
+        else
+            break
+        fi
+    done
+}
+
 # parse options
 while getopts ":f::r::s" opt; do
     case "${opt}" in
@@ -115,6 +156,11 @@ if [ "${sign_all}" ]; then
   sign_repo ${tmp_repo_path}
 fi
 
+# remove old dev packages if necessary
+if [[ ${repo} == "rpm-dev" ]]; then
+  reduce_dir_size ${tmp_repo_path} 10 rpm
+fi
+
 # update the repo by adding new packages
 if ! [ ${#files[@]} -eq 0 ]; then
   for file in "${files[@]}"; do
@@ -138,4 +184,10 @@ fi
 
 # sync repodata
 aws s3 sync ${tmp_repo_path}/repodata ${s3_bucket_repo}/repodata --delete --acl public-read
-aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/repodata/*
+aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/repodata/*
+
+# delete packages that have been pruned
+# the dryrun option is there so we can check that we're doing the right thing, can be removed after testing
+if [[ ${repo} == "rpm-dev" ]]; then
+  aws s3 sync ${tmp_repo_path} ${s3_bucket_repo} --dryrun --delete --acl public-read
+fi

scripts/publish-wasm

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -e
+
+usage() {
+    echo "usage: $0 -f <package.tar.gz>"
+    exit 1
+}
+
+# parse options
+while getopts ":f:" opt; do
+    case "${opt}" in
+        f )
+          file=${OPTARG}
+          ;;
+        \?)
+          echo "invalid option: ${OPTARG}" >&2
+          exit 1
+          ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -z "${file}" ]; then
+    usage
+fi
+
+repo="wasm-dev" 
+
+# settings
+s3_bucket_repo="s3://falco-distribution/packages/${repo}"
+cloudfront_path="/packages/${repo}"
+
+# publish
+package=$(basename -- ${file})
+echo "Publishing ${package} to ${s3_bucket_repo}..."
+aws s3 cp ${file} ${s3_bucket_repo}/${package} --acl public-read
+
+aws cloudfront create-invalidation --distribution-id ${AWS_CLOUDFRONT_DIST_ID} --paths ${cloudfront_path}/${package}

scripts/rpm/postinstall.in

@@ -16,7 +16,8 @@
 # limitations under the License.
 #
 
-chosen_driver=
+# By default, we use the automatic selection for drivers
+chosen_driver="auto"
 chosen_unit=
 CHOICE=
 
@@ -37,43 +38,56 @@ systemctl --system disable 'falcoctl-artifact-follow.service' || true
 systemctl --system unmask falcoctl-artifact-follow.service || true
 
 if [ $1 -ge 1 ]; then
+	# "auto" case is not managed here since it is already the default, so no CHOICE=2
         case $FALCO_DRIVER_CHOICE in
-              kmod)
-                  CHOICE=2
+              none)
+                  CHOICE=1
                   ;;
-              ebpf)
+              kmod)
                   CHOICE=3
                   ;;
-              modern_ebpf)
+              ebpf)
                   CHOICE=4
                   ;;
+              modern_ebpf)
+                  CHOICE=5
+                  ;;
         esac     
         if [ -z $CHOICE ] && [ -x /usr/bin/dialog ] && [ "${FALCO_FRONTEND}" != "noninteractive" ]; then          
             # If dialog is installed, create a dialog to let users choose the correct driver for them
             CHOICE=$(dialog --clear --title "Falco drivers" --menu "Choose your preferred driver:" 12 55 4 \
                     1 "Manual configuration (no unit is started)" \
-                    2 "Kmod" \
-                    3 "eBPF" \
-                    4 "Modern eBPF" \
+                    2 "Automatic selection" \
+                    3 "Kmod" \
+                    4 "eBPF" \
+                    5 "Modern eBPF" \
                     2>&1 >/dev/tty)
-        fi         
+        fi
+        # "auto" case is not managed here since it is already the default, so no CHOICE=2
         case $CHOICE in
-            2)
-                chosen_driver="kmod"
-                chosen_unit="kmod"
+            1)
+                chosen_driver=""
                 ;;
             3)
-                chosen_driver="ebpf"
-                chosen_unit="bpf"
+                chosen_driver="kmod"
                 ;;
             4)
+                chosen_driver="ebpf"
+                ;;
+            5)
                 chosen_driver="modern_ebpf"
-                chosen_unit="modern-bpf"
                 ;;
         esac
-        if [ -n "$CHOICE" ]; then
-            echo "[POST-INSTALL] Configure falcoctl driver type:"
-            falcoctl driver config --type $chosen_driver
+        if [ -n "$chosen_driver" ]; then
+            echo "[POST-INSTALL] Configure falcoctl '$chosen_driver' driver type:"
+            if [ "$chosen_driver" = "auto" ];  then
+                # Configure falcoctl to enable all drivers
+                falcoctl driver config --type "modern_ebpf" --type "kmod" --type "ebpf"
+                # Load the actually automatic chosen driver
+                chosen_driver=$(falcoctl driver printenv | grep DRIVER= | cut -d'"' -f2)
+            else
+                falcoctl driver config --type "$chosen_driver"
+            fi
             CHOICE=
             case $FALCOCTL_ENABLED in
                 no)
@@ -105,12 +119,17 @@ systemctl --system daemon-reload || true
 case "$chosen_driver" in
     "kmod")
       # Only compile for kmod, in this way we use dkms
-       echo "[POST-INSTALL] Call 'falcoctl driver install for kmod:"
-       falcoctl driver install --download=false
+      echo "[POST-INSTALL] Call 'falcoctl driver install for kmod:"
+      falcoctl driver install --download=false
+      chosen_unit="kmod"
       ;;
     "ebpf")
       echo "[POST-INSTALL] Call 'falcoctl driver install for ebpf':"
       falcoctl driver install
+      chosen_unit="bpf"
+      ;;
+    "modern_ebpf")
+      chosen_unit="modern-bpf"
       ;;
 esac
 

unit_tests/CMakeLists.txt

@@ -19,68 +19,61 @@ include(FetchContent)
 FetchContent_Declare(
   googletest
   GIT_REPOSITORY https://github.com/google/googletest.git
-  GIT_TAG        release-1.12.1
+  GIT_TAG        v1.14.0
 )
 
 FetchContent_MakeAvailable(googletest)
 
-file(GLOB_RECURSE ENGINE_TESTS ${CMAKE_CURRENT_SOURCE_DIR}/engine/*.cpp)
-file(GLOB_RECURSE FALCO_TESTS ${CMAKE_CURRENT_SOURCE_DIR}/falco/*.cpp)
-
 # Create a libscap_test_var.h file with some variables used by our tests
 # for example the kmod path or the bpf path.
 configure_file (
-    "${CMAKE_CURRENT_SOURCE_DIR}/falco_test_var.h.in"
-    "${CMAKE_CURRENT_BINARY_DIR}/falco_test_var.h"
+    ${CMAKE_CURRENT_SOURCE_DIR}/falco_test_var.h.in
+    ${CMAKE_CURRENT_BINARY_DIR}/falco_test_var.h
 )
 
-set(FALCO_UNIT_TESTS_SOURCES
-  "${ENGINE_TESTS}"
-  falco/test_configuration.cpp
-  falco/app/actions/test_select_event_sources.cpp
-  falco/app/actions/test_load_config.cpp
+add_executable(falco_unit_tests
+    test_falco_engine.cpp
+    engine/test_add_source.cpp
+    engine/test_alt_rule_loader.cpp
+    engine/test_enable_rule.cpp
+    engine/test_falco_utils.cpp
+    engine/test_filter_details_resolver.cpp
+    engine/test_filter_macro_resolver.cpp
+    engine/test_filter_warning_resolver.cpp
+    engine/test_plugin_requirements.cpp
+    engine/test_rule_loader.cpp
+    engine/test_rulesets.cpp
+    falco/test_configuration.cpp
+    falco/app/actions/test_select_event_sources.cpp
+    falco/app/actions/test_load_config.cpp
 )
 
 if (CMAKE_SYSTEM_NAME MATCHES "Linux")
-  list(APPEND FALCO_UNIT_TESTS_SOURCES
-    falco/test_atomic_signal_handler.cpp
-    falco/app/actions/test_configure_interesting_sets.cpp
-    falco/app/actions/test_configure_syscall_buffer_num.cpp)
+    target_sources(falco_unit_tests
+    PRIVATE
+        falco/test_atomic_signal_handler.cpp
+        falco/app/actions/test_configure_interesting_sets.cpp
+        falco/app/actions/test_configure_syscall_buffer_num.cpp
+    )
 endif()
 
-set(FALCO_UNIT_TESTS_INCLUDES
-    PRIVATE
+target_include_directories(falco_unit_tests
+PRIVATE
     ${CMAKE_SOURCE_DIR}/userspace
     ${CMAKE_BINARY_DIR}/userspace/falco # we need it to include indirectly `config_falco.h` file
     ${CMAKE_SOURCE_DIR}/userspace/engine # we need it to include indirectly `falco_common.h` file
-	"${CMAKE_CURRENT_BINARY_DIR}" # we need it to include `falco_test_var.h`
-)
-
-set(FALCO_UNIT_TESTS_DEPENDENCIES
-  gtest
-  gtest_main
-  falco_application
+    ${CMAKE_CURRENT_BINARY_DIR} # we need it to include `falco_test_var.h`
 )
 
 get_target_property(FALCO_APPLICATION_LIBRARIES falco_application LINK_LIBRARIES)
 
-set(FALCO_UNIT_TESTS_LIBRARIES
-  gtest
-  gtest_main
-  falco_application
-  ${FALCO_APPLICATION_LIBRARIES}
+target_link_libraries(falco_unit_tests
+    falco_application
+    GTest::gtest
+    GTest::gtest_main
+    ${FALCO_APPLICATION_LIBRARIES}
 )
 
-message(STATUS "FALCO_UNIT_TESTS_SOURCES: ${FALCO_UNIT_TESTS_SOURCES}")
-message(STATUS "FALCO_UNIT_TESTS_INCLUDES: ${FALCO_UNIT_TESTS_INCLUDES}")
-message(STATUS "FALCO_UNIT_TESTS_DEPENDENCIES: ${FALCO_UNIT_TESTS_DEPENDENCIES}")
-message(STATUS "FALCO_UNIT_TESTS_LIBRARIES: ${FALCO_UNIT_TESTS_LIBRARIES}")
-
-add_executable(falco_unit_tests ${FALCO_UNIT_TESTS_SOURCES})
-target_include_directories(falco_unit_tests ${FALCO_UNIT_TESTS_INCLUDES})
-target_link_libraries(falco_unit_tests ${FALCO_UNIT_TESTS_LIBRARIES})
-add_dependencies(falco_unit_tests ${FALCO_UNIT_TESTS_DEPENDENCIES})
-
 if (EMSCRIPTEN)
 	target_compile_options(falco_unit_tests PRIVATE "-sDISABLE_EXCEPTION_CATCHING=0")
 	target_link_options(falco_unit_tests PRIVATE "-sDISABLE_EXCEPTION_CATCHING=0")

unit_tests/engine/test_add_source.cpp

@@ -31,7 +31,7 @@ namespace
 class test_ruleset_factory : public evttype_index_ruleset_factory
 {
 public:
-	test_ruleset_factory(std::shared_ptr<gen_event_filter_factory> factory):
+	explicit test_ruleset_factory(std::shared_ptr<sinsp_filter_factory> factory):
 		evttype_index_ruleset_factory(factory)
 	{
 		ruleset = evttype_index_ruleset_factory::new_ruleset();
@@ -54,12 +54,9 @@ TEST(AddSource, basic)
 	sinsp inspector;
 	sinsp_filter_check_list filterchecks;
 
-	auto filter_factory = std::shared_ptr<gen_event_filter_factory>(
-		new sinsp_filter_factory(&inspector, filterchecks));
-	auto formatter_factory = std::shared_ptr<gen_event_formatter_factory>(
-		new sinsp_evt_formatter_factory(&inspector, filterchecks));
-	test_ruleset_factory *test_factory = new test_ruleset_factory(filter_factory);
-	auto ruleset_factory = std::shared_ptr<filter_ruleset_factory>(test_factory);
+	auto filter_factory = std::make_shared<sinsp_filter_factory>(&inspector, filterchecks);
+	auto formatter_factory = std::make_shared<sinsp_evt_formatter_factory>(&inspector, filterchecks);
+	auto ruleset_factory = std::make_shared<test_ruleset_factory>(filter_factory);
 
 	falco_source syscall_source;
 	syscall_source.name = syscall_source_name;
@@ -84,6 +81,6 @@ TEST(AddSource, basic)
 	ASSERT_EQ(engine.ruleset_factory_for_source(syscall_source_name), ruleset_factory);
 	ASSERT_EQ(engine.ruleset_factory_for_source(source_idx), ruleset_factory);
 
-	ASSERT_EQ(engine.ruleset_for_source(syscall_source_name), test_factory->ruleset);
-	ASSERT_EQ(engine.ruleset_for_source(source_idx), test_factory->ruleset);
+	ASSERT_EQ(engine.ruleset_for_source(syscall_source_name), ruleset_factory->ruleset);
+	ASSERT_EQ(engine.ruleset_for_source(source_idx), ruleset_factory->ruleset);
 }

unit_tests/engine/test_alt_rule_loader.cpp

@@ -0,0 +1,354 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include <sinsp.h>
+#include <filter_check_list.h>
+#include <filter.h>
+#include <eventformatter.h>
+
+#include <falco_engine.h>
+#include "indexed_vector.h"
+#include "evttype_index_ruleset.h"
+
+#include "rule_loader_reader.h"
+#include "rule_loader_collector.h"
+#include "rule_loader_compiler.h"
+
+namespace
+{
+
+struct test_object_info
+{
+	std::string name;
+	std::string property;
+};
+
+struct test_compile_output : public rule_loader::compile_output
+{
+	test_compile_output() = default;
+	~test_compile_output() = default;
+
+	std::set<std::string> defined_test_properties;
+};
+
+class test_compiler : public rule_loader::compiler
+{
+public:
+	test_compiler() = default;
+	virtual ~test_compiler() = default;
+
+	std::unique_ptr<rule_loader::compile_output> new_compile_output() override
+	{
+		return std::make_unique<test_compile_output>();
+	}
+
+	void compile(
+		rule_loader::configuration& cfg,
+		const rule_loader::collector& col,
+		rule_loader::compile_output& out) const override;
+};
+
+class test_collector : public rule_loader::collector
+{
+public:
+	test_collector() = default;
+	virtual ~test_collector() = default;
+
+	indexed_vector<test_object_info> test_object_infos;
+};
+
+class test_reader : public rule_loader::reader
+{
+public:
+	test_reader() = default;
+	virtual ~test_reader() = default;
+
+protected:
+	rule_loader::context create_context(const YAML::Node& item,
+					    const rule_loader::context& parent)
+	{
+		return rule_loader::context(item,
+					    rule_loader::context::EXTENSION_ITEM,
+					    "test object",
+					    parent);
+	};
+
+	void read_item(rule_loader::configuration& cfg,
+		       rule_loader::collector& collector,
+		       const YAML::Node& item,
+		       const rule_loader::context& parent) override
+	{
+		test_collector& test_col =
+			dynamic_cast<test_collector&>(collector);
+
+		if(item["test_object"].IsDefined())
+		{
+			rule_loader::context tmp = create_context(item, parent);
+			test_object_info obj;
+			std::string name;
+			std::string property;
+
+			decode_val(item, "test_object", name, tmp);
+			decode_val(item, "property", property, tmp);
+
+			obj.name = name;
+			obj.property = property;
+
+			test_col.test_object_infos.insert(obj, obj.name);
+		}
+		else
+		{
+			rule_loader::reader::read_item(cfg, collector, item, parent);
+		}
+	};
+};
+
+class test_ruleset : public evttype_index_ruleset
+{
+public:
+	explicit test_ruleset(std::shared_ptr<sinsp_filter_factory> factory):
+		evttype_index_ruleset(factory){};
+	virtual ~test_ruleset() = default;
+
+	void add_compile_output(
+		const rule_loader::compile_output& compile_output,
+		falco_common::priority_type min_priority,
+		const std::string& source)
+	{
+
+		evttype_index_ruleset::add_compile_output(compile_output,
+							  min_priority,
+							  source);
+
+		std::shared_ptr<filter_ruleset> ruleset;
+		get_engine_state().get_ruleset(source, ruleset);
+		EXPECT_EQ(this, ruleset.get());
+
+		const test_compile_output& test_output =
+			dynamic_cast<const test_compile_output&>(compile_output);
+
+		defined_properties = test_output.defined_test_properties;
+	};
+
+	std::set<std::string> defined_properties;
+};
+
+class test_ruleset_factory : public filter_ruleset_factory
+{
+public:
+	explicit test_ruleset_factory(std::shared_ptr<sinsp_filter_factory> factory):
+		m_filter_factory(factory)
+	{
+	}
+
+	virtual ~test_ruleset_factory() = default;
+
+	inline std::shared_ptr<filter_ruleset> new_ruleset() override
+	{
+		return std::make_shared<test_ruleset>(m_filter_factory);
+	}
+
+	std::shared_ptr<sinsp_filter_factory> m_filter_factory;
+};
+}; // namespace
+
+void test_compiler::compile(
+	rule_loader::configuration& cfg,
+	const rule_loader::collector& col,
+	rule_loader::compile_output& out) const
+{
+	rule_loader::compiler::compile(cfg, col, out);
+
+	const test_collector& test_col =
+		dynamic_cast<const test_collector&>(col);
+
+	test_compile_output& test_output =
+		dynamic_cast<test_compile_output&>(out);
+
+	for(auto& test_obj : test_col.test_object_infos)
+	{
+		test_output.defined_test_properties.insert(test_obj.property);
+	}
+}
+
+static std::string content = R"END(
+
+- test_object: test
+  property: my-value
+
+- test_object: test2
+  property: other-value
+
+- list: shell_binaries
+  items: [sh, bash]
+
+- macro: spawned_process
+  condition: evt.type=execve and proc.name in (shell_binaries)
+
+- rule: test info rule
+  desc: A test info rule
+  condition: spawned_process
+  output: A test info rule matched (evt.type=%evt.type proc.name=%proc.name)
+  priority: INFO
+  source: syscall
+  tags: [process]
+
+- rule: test k8s_audit rule
+  desc: A k8s audit test rule
+  condition: ka.target.resource=deployments
+  output: A k8s audit rule matched (ka.verb=%ka.verb resource=%ka.target.resource)
+  priority: INFO
+  source: k8s_audit
+  tags: [process]
+
+- rule: test debug rule
+  desc: A test debug rule
+  condition: spawned_process and proc.name="bash"
+  output: A test debug rule matched (evt.type=%evt.type proc.name=%proc.name)
+  priority: DEBUG
+  source: syscall
+  tags: [process]
+)END";
+
+static std::string syscall_source_name = "syscall";
+
+static std::shared_ptr<rule_loader::configuration> create_configuration(sinsp& inspector,
+									sinsp_filter_check_list& filterchecks,
+									indexed_vector<falco_source>& sources)
+{
+	auto filter_factory = std::make_shared<sinsp_filter_factory>(&inspector, filterchecks);
+	auto formatter_factory = std::make_shared<sinsp_evt_formatter_factory>(&inspector, filterchecks);
+	auto ruleset_factory = std::make_shared<evttype_index_ruleset_factory>(filter_factory);
+
+	falco_source syscall_source;
+	syscall_source.name = syscall_source_name;
+	syscall_source.ruleset = ruleset_factory->new_ruleset();
+	syscall_source.ruleset_factory = ruleset_factory;
+	syscall_source.filter_factory = filter_factory;
+	syscall_source.formatter_factory = formatter_factory;
+
+	sources.insert(syscall_source, syscall_source_name);
+
+	return std::make_shared<rule_loader::configuration>(content,
+	                                                    sources,
+	                                                    "test configuration");
+}
+
+static void load_rules(sinsp& inspector,
+		       sinsp_filter_check_list& filterchecks,
+		       std::unique_ptr<rule_loader::compile_output>& compile_output,
+		       indexed_vector<falco_source>& sources)
+{
+	std::shared_ptr<rule_loader::configuration> cfg = create_configuration(inspector, filterchecks, sources);
+
+	rule_loader::reader reader;
+	rule_loader::collector collector;
+	rule_loader::compiler compiler;
+
+	EXPECT_TRUE(reader.read(*(cfg.get()), collector));
+
+	compile_output = compiler.new_compile_output();
+
+	compiler.compile(*(cfg.get()), collector, *(compile_output.get()));
+}
+
+TEST(engine_loader_alt_loader, load_rules)
+{
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+	std::unique_ptr<rule_loader::compile_output> compile_output;
+	indexed_vector<falco_source> sources;
+
+	load_rules(inspector, filterchecks, compile_output, sources);
+
+	// Note that the k8s_audit rule will be skipped as load_rules
+	// only adds a syscall source.
+	EXPECT_EQ(compile_output->lists.size(), 1);
+	EXPECT_TRUE(compile_output->lists.at("shell_binaries") != nullptr);
+
+	EXPECT_EQ(compile_output->macros.size(), 1);
+	EXPECT_TRUE(compile_output->macros.at("spawned_process") != nullptr);
+
+	EXPECT_EQ(compile_output->rules.size(), 2);
+	EXPECT_TRUE(compile_output->rules.at("test info rule") != nullptr);
+	EXPECT_TRUE(compile_output->rules.at("test debug rule") != nullptr);
+}
+
+TEST(engine_loader_alt_loader, pass_compile_output_to_ruleset)
+{
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+	std::unique_ptr<rule_loader::compile_output> compile_output;
+	indexed_vector<falco_source> sources;
+
+	load_rules(inspector, filterchecks, compile_output, sources);
+
+	std::shared_ptr<filter_ruleset> ruleset = sources.at(syscall_source_name)->ruleset;
+
+	ruleset->add_compile_output(*(compile_output.get()),
+				    falco_common::PRIORITY_INFORMATIONAL,
+				    syscall_source_name);
+
+	// Enable all rules for a ruleset id. Because the compile
+	// output contained one rule with priority >= INFO, that rule
+	// should be enabled.
+	bool match_exact = true;
+	uint16_t ruleset_id = 0;
+	ruleset->enable("", match_exact, ruleset_id);
+
+	EXPECT_EQ(ruleset->enabled_count(ruleset_id), 1);
+}
+
+TEST(engine_loader_alt_loader, falco_engine_alternate_loader)
+{
+	falco_engine engine;
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+
+	auto filter_factory = std::make_shared<sinsp_filter_factory>(&inspector, filterchecks);
+	auto formatter_factory = std::make_shared<sinsp_evt_formatter_factory>(&inspector, filterchecks);
+	auto ruleset_factory = std::make_shared<test_ruleset_factory>(filter_factory);
+
+	engine.add_source(syscall_source_name, filter_factory, formatter_factory, ruleset_factory);
+
+	auto reader = std::make_shared<test_reader>();
+	auto collector = std::make_shared<test_collector>();
+	auto compiler = std::make_shared<test_compiler>();
+
+	engine.set_rule_reader(reader);
+	engine.set_rule_collector(collector);
+	engine.set_rule_compiler(compiler);
+
+	EXPECT_EQ(reader, engine.get_rule_reader());
+	EXPECT_EQ(collector, engine.get_rule_collector());
+	EXPECT_EQ(compiler, engine.get_rule_compiler());
+
+	engine.load_rules(content, "test_rules.yaml");
+
+	EXPECT_EQ(collector->test_object_infos.size(), 2);
+
+	std::shared_ptr<filter_ruleset> ruleset = engine.ruleset_for_source(syscall_source_name);
+	std::set<std::string>& defined_properties = std::dynamic_pointer_cast<test_ruleset>(ruleset)->defined_properties;
+
+	EXPECT_TRUE(defined_properties.find("my-value") != defined_properties.end());
+	EXPECT_TRUE(defined_properties.find("other-value") != defined_properties.end());
+	EXPECT_TRUE(defined_properties.find("not-exists-value") == defined_properties.end());
+};

unit_tests/engine/test_enable_rule.cpp

@@ -23,7 +23,7 @@ limitations under the License.
 #include <filter_check_list.h>
 #include <filter.h>
 
-#include <falco_engine.h>
+#include "../test_falco_engine.h"
 
 static std::string single_rule = R"END(
 - rule: test rule
@@ -52,200 +52,167 @@ static const std::string ruleset_2 = "ruleset-2";
 static const std::string ruleset_3 = "ruleset-3";
 static const std::string ruleset_4 = "ruleset-4";
 
-static void load_rules(falco_engine& engine, sinsp& inspector, sinsp_filter_check_list& filterchecks)
+TEST_F(test_falco_engine, enable_rule_name)
 {
-	std::unique_ptr<falco::load_result> res;
-
-	auto filter_factory = std::shared_ptr<gen_event_filter_factory>(
-		new sinsp_filter_factory(&inspector, filterchecks));
-	auto formatter_factory = std::shared_ptr<gen_event_formatter_factory>(
-		new sinsp_evt_formatter_factory(&inspector, filterchecks));
-
-	engine.add_source("syscall", filter_factory, formatter_factory);
-
-	res = engine.load_rules(single_rule, "single_rule.yaml");
-
-	EXPECT_TRUE(res->successful());
-}
-
-TEST(EnableRule, enable_rule_name)
-{
-	falco_engine engine;
-	sinsp inspector;
-	sinsp_filter_check_list filterchecks;
-
-	load_rules(engine, inspector, filterchecks);
+	load_rules(single_rule, "single_rule.yaml");
 
 	// No rules should be enabled yet for any custom rulesets
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
 
 	// Enable for first ruleset, only that ruleset should have an
 	// enabled rule afterward
-	engine.enable_rule("test", true, ruleset_1);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	m_engine->enable_rule("test", true, ruleset_1);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
 
 	// Enable for second ruleset
-	engine.enable_rule("test", true, ruleset_2);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	m_engine->enable_rule("test", true, ruleset_2);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
 
 	// When the substring is blank, all rules are enabled
 	// (including the disabled rule)
-	engine.enable_rule("", true, ruleset_3);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_3));
+	m_engine->enable_rule("", true, ruleset_3);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(2, m_engine->num_rules_for_ruleset(ruleset_3));
 
 	// Now disable for second ruleset
-	engine.enable_rule("test", false, ruleset_2);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_3));
+	m_engine->enable_rule("test", false, ruleset_2);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(2, m_engine->num_rules_for_ruleset(ruleset_3));
 }
 
-TEST(EnableRule, enable_rule_tags)
+TEST_F(test_falco_engine, enable_rule_tags)
 {
-	falco_engine engine;
-	sinsp inspector;
-	sinsp_filter_check_list filterchecks;
 	std::set<std::string> process_tags = {"process"};
 
-	load_rules(engine, inspector, filterchecks);
+	load_rules(single_rule, "single_rule.yaml");
 
 	// No rules should be enabled yet for any custom rulesets
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
 
 	// Enable for first ruleset, only that ruleset should have an
 	// enabled rule afterward
-	engine.enable_rule_by_tag(process_tags, true, ruleset_1);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	m_engine->enable_rule_by_tag(process_tags, true, ruleset_1);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
 
 	// Enable for second ruleset
-	engine.enable_rule_by_tag(process_tags, true, ruleset_2);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
+	m_engine->enable_rule_by_tag(process_tags, true, ruleset_2);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_2));
 
 	// Now disable for second ruleset
-	engine.enable_rule_by_tag(process_tags, false, ruleset_2);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	m_engine->enable_rule_by_tag(process_tags, false, ruleset_2);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
 }
 
-TEST(EnableRule, enable_disabled_rule_by_tag)
+TEST_F(test_falco_engine, enable_disabled_rule_by_tag)
 {
-	falco_engine engine;
-	sinsp inspector;
-	sinsp_filter_check_list filterchecks;
 	std::set<std::string> exec_process_tags = {"exec process"};
 
-	load_rules(engine, inspector, filterchecks);
+	load_rules(single_rule, "single_rule.yaml");
 
 	// Only the first rule should be enabled
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(default_ruleset));
 
 	// Enable the disabled rule by tag
-	engine.enable_rule_by_tag(exec_process_tags, true);
+	m_engine->enable_rule_by_tag(exec_process_tags, true);
 
 	// Both rules should be enabled now
-	EXPECT_EQ(2, engine.num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(2, m_engine->num_rules_for_ruleset(default_ruleset));
 }
 
-TEST(EnableRule, enable_rule_id)
+TEST_F(test_falco_engine, enable_rule_id)
 {
-	falco_engine engine;
-	sinsp inspector;
-	sinsp_filter_check_list filterchecks;
 	uint16_t ruleset_1_id;
 	uint16_t ruleset_2_id;
 	uint16_t ruleset_3_id;
 
-	load_rules(engine, inspector, filterchecks);
+	load_rules(single_rule, "single_rule.yaml");
 
 	// The cases are identical to above, just using ruleset ids
 	// instead of names.
 
-	ruleset_1_id = engine.find_ruleset_id(ruleset_1);
-	ruleset_2_id = engine.find_ruleset_id(ruleset_2);
-	ruleset_3_id = engine.find_ruleset_id(ruleset_3);
+	ruleset_1_id = m_engine->find_ruleset_id(ruleset_1);
+	ruleset_2_id = m_engine->find_ruleset_id(ruleset_2);
+	ruleset_3_id = m_engine->find_ruleset_id(ruleset_3);
 
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
 
-	engine.enable_rule("test rule", true, ruleset_1_id);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	m_engine->enable_rule("test rule", true, ruleset_1_id);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
 
-	engine.enable_rule("test rule", true, ruleset_2_id);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	m_engine->enable_rule("test rule", true, ruleset_2_id);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
 
-	engine.enable_rule("", true, ruleset_3_id);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_3));
+	m_engine->enable_rule("", true, ruleset_3_id);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(2, m_engine->num_rules_for_ruleset(ruleset_3));
 
-	engine.enable_rule("test", false, ruleset_2_id);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_3));
+	m_engine->enable_rule("test", false, ruleset_2_id);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(2, m_engine->num_rules_for_ruleset(ruleset_3));
 }
 
-TEST(EnableRule, enable_rule_name_exact)
+TEST_F(test_falco_engine, enable_rule_name_exact)
 {
-	falco_engine engine;
-	sinsp inspector;
-	sinsp_filter_check_list filterchecks;
+	load_rules(single_rule, "single_rule.yaml");
 
-	load_rules(engine, inspector, filterchecks);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_4));
 
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_4));
+	m_engine->enable_rule_exact("test rule", true, ruleset_1);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_4));
 
-	engine.enable_rule_exact("test rule", true, ruleset_1);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_4));
-
-	engine.enable_rule_exact("test rule", true, ruleset_2);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_4));
+	m_engine->enable_rule_exact("test rule", true, ruleset_2);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_4));
 
 	// This should **not** enable as this is a substring and not
 	// an exact match.
-	engine.enable_rule_exact("test", true, ruleset_3);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_4));
+	m_engine->enable_rule_exact("test", true, ruleset_3);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_4));
 
-	engine.enable_rule_exact("", true, ruleset_4);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
-	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_4));
+	m_engine->enable_rule_exact("", true, ruleset_4);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(2, m_engine->num_rules_for_ruleset(ruleset_4));
 
-	engine.enable_rule("test rule", false, ruleset_2);
-	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
-	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
-	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_4));
+	m_engine->enable_rule("test rule", false, ruleset_2);
+	EXPECT_EQ(1, m_engine->num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, m_engine->num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(2, m_engine->num_rules_for_ruleset(ruleset_4));
 }

unit_tests/engine/test_filter_macro_resolver.cpp

@@ -8,7 +8,7 @@ You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
 
-Unless ASSERT_EQd by applicable law or agreed to in writing, software
+Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and

unit_tests/engine/test_filter_warning_resolver.cpp

@@ -8,7 +8,7 @@ You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
 
-Unless ASSERTd by applicable law or agreed to in writing, software
+Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and

unit_tests/engine/test_plugin_requirements.cpp

@@ -23,15 +23,15 @@ static bool check_requirements(std::string& err,
 			       const std::vector<falco_engine::plugin_version_requirement>& plugins,
 			       const std::string& ruleset_content)
 {
-	std::unique_ptr<falco_engine> e(new falco_engine());
+	falco_engine e;
 	falco::load_result::rules_contents_t c = {{"test", ruleset_content}};
 
-	auto res = e->load_rules(c.begin()->second, c.begin()->first);
+	auto res = e.load_rules(c.begin()->second, c.begin()->first);
 	if(!res->successful())
 	{
 		return false;
 	}
-	return e->check_plugin_requirements(plugins, err);
+	return e.check_plugin_requirements(plugins, err);
 }
 
 TEST(PluginRequirements, check_plugin_requirements_success)

unit_tests/engine/test_rule_loader.cpp

@@ -1,125 +1,11 @@
 #include <gtest/gtest.h>
 
-#include "falco_engine.h"
-#include "rule_loader_reader.h"
-#include "rule_loader_compiler.h"
-#include "rule_loading_messages.h"
-
-class engine_loader_test : public ::testing::Test {
-protected:
-	void SetUp() override
-	{
-		m_sample_ruleset = "sample-ruleset";
-		m_sample_source = falco_common::syscall_source;
-
-		// create a falco engine ready to load the ruleset
-		m_inspector.reset(new sinsp());
-		m_engine.reset(new falco_engine());
-		m_filter_factory = std::shared_ptr<gen_event_filter_factory>(
-			new sinsp_filter_factory(m_inspector.get(), m_filterlist));
-		m_formatter_factory = std::shared_ptr<gen_event_formatter_factory>(
-			new sinsp_evt_formatter_factory(m_inspector.get(), m_filterlist));
-		m_engine->add_source(m_sample_source, m_filter_factory, m_formatter_factory);
-	}
-
-	void TearDown() override
-	{
-
-	}
-
-	bool load_rules(std::string rules_content, std::string rules_filename)
-	{
-		bool ret = false;
-		falco::load_result::rules_contents_t rc = {{rules_filename, rules_content}};
-		m_load_result = m_engine->load_rules(rules_content, rules_filename);
-		m_load_result_string = m_load_result->as_string(true, rc);
-		m_load_result_json = m_load_result->as_json(rc);
-		ret = m_load_result->successful();
-
-		if (ret)
-		{
-			m_engine->enable_rule("", true, m_sample_ruleset);
-		}
-
-		return ret;
-	}
-
-	// This must be kept in line with the (private) falco_engine::s_default_ruleset
-	uint64_t num_rules_for_ruleset(std::string ruleset = "falco-default-ruleset")
-	{
-		return m_engine->num_rules_for_ruleset(ruleset);
-	}
-
-	bool has_warnings()
-	{
-		return m_load_result->has_warnings();
-	}
-
-	bool check_warning_message(std::string warning_msg)
-	{
-		if(!m_load_result->has_warnings())
-		{
-			return false;
-		}
-
-		for(auto &warn : m_load_result_json["warnings"])
-		{
-			std::string msg = warn["message"];
-			// Debug:
-			// printf("msg: %s\n", msg.c_str());
-			if(msg.find(warning_msg) != std::string::npos)
-			{
-				return true;
-			}
-		}
-
-		return false;
-	}
-
-	bool check_error_message(std::string error_msg)
-	{
-		// if the loading is successful there are no errors
-		if(m_load_result->successful())
-		{
-			return false;
-		}
-
-		for(auto &err : m_load_result_json["errors"])
-		{
-			std::string msg = err["message"];
-			// Debug:
-			// printf("msg: %s\n", msg.c_str());
-			if(msg.find(error_msg) != std::string::npos)
-			{
-				return true;
-			}
-		}
-
-		return false;
-	}
-
-	std::string get_compiled_rule_condition(std::string rule_name = "")
-	{
-		auto rule_description = m_engine->describe_rule(&rule_name, {});
-		return rule_description["rules"][0]["details"]["condition_compiled"].template get<std::string>();
-	}
-
-	std::string m_sample_ruleset;
-	std::string m_sample_source;
-	sinsp_filter_check_list m_filterlist;
-	std::shared_ptr<gen_event_filter_factory> m_filter_factory;
-	std::shared_ptr<gen_event_formatter_factory> m_formatter_factory;
-	std::unique_ptr<falco_engine> m_engine;
-	std::unique_ptr<falco::load_result> m_load_result;
-	std::string m_load_result_string;
-	nlohmann::json m_load_result_json;
-	std::unique_ptr<sinsp> m_inspector;
-};
+#include "../test_falco_engine.h"
 
 std::string s_sample_ruleset = "sample-ruleset";
 std::string s_sample_source = falco_common::syscall_source;
 
-TEST_F(engine_loader_test, list_append)
+TEST_F(test_falco_engine, list_append)
 {
     std::string rules_content = R"END(
 - list: shell_binaries
@@ -141,7 +27,7 @@ TEST_F(engine_loader_test, list_append)
 	ASSERT_EQ(get_compiled_rule_condition("legit_rule"),"(evt.type = open and proc.name in (ash, bash, csh, ksh, sh, tcsh, zsh, dash, pwsh))");
 }
 
-TEST_F(engine_loader_test, condition_append)
+TEST_F(test_falco_engine, condition_append)
 {
     std::string rules_content = R"END(
 - macro: interactive
@@ -165,7 +51,7 @@ TEST_F(engine_loader_test, condition_append)
 	ASSERT_EQ(get_compiled_rule_condition("legit_rule"),"(evt.type = open and (((proc.aname = sshd and proc.name != sshd) or proc.name = systemd-logind or proc.name = login) or proc.name = ssh))");
 }
 
-TEST_F(engine_loader_test, rule_override_append)
+TEST_F(test_falco_engine, rule_override_append)
 {
     std::string rules_content = R"END(
 - rule: legit_rule
@@ -201,7 +87,7 @@ TEST_F(engine_loader_test, rule_override_append)
 	 	"legit rule description with append");
 }
 
-TEST_F(engine_loader_test, rule_append)
+TEST_F(test_falco_engine, rule_append)
 {
     std::string rules_content = R"END(
 - rule: legit_rule
@@ -223,7 +109,7 @@ TEST_F(engine_loader_test, rule_append)
 	ASSERT_EQ(get_compiled_rule_condition("legit_rule"),"(evt.type = open and proc.name = cat)");
 }
 
-TEST_F(engine_loader_test, rule_override_replace)
+TEST_F(test_falco_engine, rule_override_replace)
 {
     std::string rules_content = R"END(
 - rule: legit_rule
@@ -254,7 +140,7 @@ TEST_F(engine_loader_test, rule_override_replace)
 	 	"a replaced legit description");
 }
 
-TEST_F(engine_loader_test, rule_override_append_replace)
+TEST_F(test_falco_engine, rule_override_append_replace)
 {
     std::string rules_content = R"END(
 - rule: legit_rule
@@ -290,7 +176,7 @@ TEST_F(engine_loader_test, rule_override_append_replace)
 	 	"Warning");
 }
 
-TEST_F(engine_loader_test, rule_incorrect_override_type)
+TEST_F(test_falco_engine, rule_incorrect_override_type)
 {
     std::string rules_content = R"END(
 - rule: failing_rule
@@ -309,14 +195,12 @@ TEST_F(engine_loader_test, rule_incorrect_override_type)
     priority: append
 )END";
 
-	std::string rule_name = "failing_rule";
-
 	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
 	ASSERT_TRUE(check_error_message("Key 'priority' cannot be appended to, use 'replace' instead"));
 	ASSERT_TRUE(std::string(m_load_result_json["errors"][0]["context"]["snippet"]).find("priority: append") != std::string::npos);
 }
 
-TEST_F(engine_loader_test, rule_incorrect_append_override)
+TEST_F(test_falco_engine, rule_incorrect_append_override)
 {
     std::string rules_content = R"END(
 - rule: failing_rule
@@ -334,8 +218,6 @@ TEST_F(engine_loader_test, rule_incorrect_append_override)
     condition: append
 )END";
 
-	std::string rule_name = "failing_rule";
-
 	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
 	
 	// We should have at least one warning because the 'append' flag is deprecated.
@@ -344,7 +226,7 @@ TEST_F(engine_loader_test, rule_incorrect_append_override)
 	ASSERT_TRUE(check_error_message(ERROR_OVERRIDE_APPEND));
 }
 
-TEST_F(engine_loader_test, macro_override_append_before_macro_definition)
+TEST_F(test_falco_engine, macro_override_append_before_macro_definition)
 {
     std::string rules_content = R"END(
 
@@ -369,7 +251,7 @@ TEST_F(engine_loader_test, macro_override_append_before_macro_definition)
 	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_MACRO));
 }
 
-TEST_F(engine_loader_test, macro_override_replace_before_macro_definition)
+TEST_F(test_falco_engine, macro_override_replace_before_macro_definition)
 {
     std::string rules_content = R"END(
 
@@ -394,7 +276,7 @@ TEST_F(engine_loader_test, macro_override_replace_before_macro_definition)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"evt.type in (open, openat)");	
 }
 
-TEST_F(engine_loader_test, macro_append_before_macro_definition)
+TEST_F(test_falco_engine, macro_append_before_macro_definition)
 {
     std::string rules_content = R"END(
 
@@ -418,7 +300,7 @@ TEST_F(engine_loader_test, macro_append_before_macro_definition)
 	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_MACRO));
 }
 
-TEST_F(engine_loader_test, macro_override_append_after_macro_definition)
+TEST_F(test_falco_engine, macro_override_append_after_macro_definition)
 {
     std::string rules_content = R"END(
 
@@ -443,7 +325,7 @@ TEST_F(engine_loader_test, macro_override_append_after_macro_definition)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) or evt.type = openat2)");
 }
 
-TEST_F(engine_loader_test, macro_append_after_macro_definition)
+TEST_F(test_falco_engine, macro_append_after_macro_definition)
 {
     std::string rules_content = R"END(
 
@@ -467,7 +349,7 @@ TEST_F(engine_loader_test, macro_append_after_macro_definition)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) or evt.type = openat2)");
 }
 
-TEST_F(engine_loader_test, rule_override_append_before_rule_definition)
+TEST_F(test_falco_engine, rule_override_append_before_rule_definition)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -487,7 +369,7 @@ TEST_F(engine_loader_test, rule_override_append_before_rule_definition)
 	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_RULE_APPEND));
 }
 
-TEST_F(engine_loader_test, rule_override_replace_before_rule_definition)
+TEST_F(test_falco_engine, rule_override_replace_before_rule_definition)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -507,7 +389,7 @@ TEST_F(engine_loader_test, rule_override_replace_before_rule_definition)
 	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_RULE_REPLACE));
 }
 
-TEST_F(engine_loader_test, rule_append_before_rule_definition)
+TEST_F(test_falco_engine, rule_append_before_rule_definition)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -526,7 +408,7 @@ TEST_F(engine_loader_test, rule_append_before_rule_definition)
 	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_RULE_APPEND));
 }
 
-TEST_F(engine_loader_test, rule_override_append_after_rule_definition)
+TEST_F(test_falco_engine, rule_override_append_after_rule_definition)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -545,7 +427,7 @@ TEST_F(engine_loader_test, rule_override_append_after_rule_definition)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) and proc.name = cat)");
 }
 
-TEST_F(engine_loader_test, rule_append_after_rule_definition)
+TEST_F(test_falco_engine, rule_append_after_rule_definition)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -563,7 +445,7 @@ TEST_F(engine_loader_test, rule_append_after_rule_definition)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) and proc.name = cat)");
 }
 
-TEST_F(engine_loader_test, list_override_append_wrong_key)
+TEST_F(test_falco_engine, list_override_append_wrong_key)
 {
 	// todo: maybe we want to manage some non-existent keys
 	// Please note how the non-existent key 'non-existent keys' is ignored.
@@ -591,7 +473,7 @@ TEST_F(engine_loader_test, list_override_append_wrong_key)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid))");
 }
 
-TEST_F(engine_loader_test, list_override_append_before_list_definition)
+TEST_F(test_falco_engine, list_override_append_before_list_definition)
 {
     std::string rules_content = R"END(
 - list: dev_creation_binaries
@@ -615,7 +497,7 @@ TEST_F(engine_loader_test, list_override_append_before_list_definition)
 	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_LIST));
 }
 
-TEST_F(engine_loader_test, list_override_replace_before_list_definition)
+TEST_F(test_falco_engine, list_override_replace_before_list_definition)
 {
     std::string rules_content = R"END(
 - list: dev_creation_binaries
@@ -639,7 +521,7 @@ TEST_F(engine_loader_test, list_override_replace_before_list_definition)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid))");
 }
 
-TEST_F(engine_loader_test, list_append_before_list_definition)
+TEST_F(test_falco_engine, list_append_before_list_definition)
 {
     std::string rules_content = R"END(
 - list: dev_creation_binaries
@@ -662,7 +544,7 @@ TEST_F(engine_loader_test, list_append_before_list_definition)
 	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_LIST));
 }
 
-TEST_F(engine_loader_test, list_override_append_after_list_definition)
+TEST_F(test_falco_engine, list_override_append_after_list_definition)
 {
     std::string rules_content = R"END(
 - list: dev_creation_binaries
@@ -685,7 +567,7 @@ TEST_F(engine_loader_test, list_override_append_after_list_definition)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid, csi-provisioner, csi-attacher))");
 }
 
-TEST_F(engine_loader_test, list_append_after_list_definition)
+TEST_F(test_falco_engine, list_append_after_list_definition)
 {
     std::string rules_content = R"END(
 - list: dev_creation_binaries
@@ -706,7 +588,7 @@ TEST_F(engine_loader_test, list_append_after_list_definition)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid, csi-provisioner, csi-attacher))");
 }
 
-TEST_F(engine_loader_test, rule_override_without_field)
+TEST_F(test_falco_engine, rule_override_without_field)
 {
     std::string rules_content = R"END(
 - rule: failing_rule
@@ -722,13 +604,11 @@ TEST_F(engine_loader_test, rule_override_without_field)
     condition: append
 )END";
 
-	std::string rule_name = "failing_rule";
-
 	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
 	ASSERT_TRUE(check_error_message("An append override for 'condition' was specified but 'condition' is not defined"));
 }
 
-TEST_F(engine_loader_test, rule_override_extra_field)
+TEST_F(test_falco_engine, rule_override_extra_field)
 {
     std::string rules_content = R"END(
 - rule: failing_rule
@@ -746,13 +626,11 @@ TEST_F(engine_loader_test, rule_override_extra_field)
     condition: append
 )END";
 
-	std::string rule_name = "failing_rule";
-
 	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
 	ASSERT_TRUE(check_error_message("Unexpected key 'priority'"));
 }
 
-TEST_F(engine_loader_test, missing_enabled_key_with_override)
+TEST_F(test_falco_engine, missing_enabled_key_with_override)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -776,7 +654,7 @@ TEST_F(engine_loader_test, missing_enabled_key_with_override)
 	ASSERT_TRUE(check_error_message("'enabled' was specified but 'enabled' is not defined"));
 }
 
-TEST_F(engine_loader_test, rule_override_with_enabled)
+TEST_F(test_falco_engine, rule_override_with_enabled)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -802,7 +680,7 @@ TEST_F(engine_loader_test, rule_override_with_enabled)
 	EXPECT_EQ(num_rules_for_ruleset(), 1);
 }
 
-TEST_F(engine_loader_test, rule_not_enabled)
+TEST_F(test_falco_engine, rule_not_enabled)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -818,7 +696,7 @@ TEST_F(engine_loader_test, rule_not_enabled)
 	EXPECT_EQ(num_rules_for_ruleset(), 0);
 }
 
-TEST_F(engine_loader_test, rule_enabled_warning)
+TEST_F(test_falco_engine, rule_enabled_warning)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -839,7 +717,7 @@ TEST_F(engine_loader_test, rule_enabled_warning)
 }
 
 // todo!: Probably we shouldn't allow this syntax
-TEST_F(engine_loader_test, rule_enabled_is_ignored_by_append)
+TEST_F(test_falco_engine, rule_enabled_is_ignored_by_append)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -862,7 +740,7 @@ TEST_F(engine_loader_test, rule_enabled_is_ignored_by_append)
 }
 
 // todo!: Probably we shouldn't allow this syntax
-TEST_F(engine_loader_test, rewrite_rule)
+TEST_F(test_falco_engine, rewrite_rule)
 {
     std::string rules_content = R"END(
 - rule: test_rule
@@ -888,7 +766,7 @@ TEST_F(engine_loader_test, rewrite_rule)
 	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"proc.name = cat");
 }
 
-TEST_F(engine_loader_test, required_engine_version_semver)
+TEST_F(test_falco_engine, required_engine_version_semver)
 {
     std::string rules_content = R"END(
 - required_engine_version: 0.26.0
@@ -906,7 +784,7 @@ TEST_F(engine_loader_test, required_engine_version_semver)
 	ASSERT_FALSE(has_warnings());
 }
 
-TEST_F(engine_loader_test, required_engine_version_not_semver)
+TEST_F(test_falco_engine, required_engine_version_not_semver)
 {
     std::string rules_content = R"END(
 - required_engine_version: 26
@@ -924,7 +802,7 @@ TEST_F(engine_loader_test, required_engine_version_not_semver)
 	ASSERT_FALSE(has_warnings());
 }
 
-TEST_F(engine_loader_test, required_engine_version_invalid)
+TEST_F(test_falco_engine, required_engine_version_invalid)
 {
     std::string rules_content = R"END(
 - required_engine_version: seven
@@ -941,3 +819,160 @@ TEST_F(engine_loader_test, required_engine_version_invalid)
 	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
 	ASSERT_TRUE(check_error_message("Unable to parse engine version"));
 }
+
+// checks for issue described in https://github.com/falcosecurity/falco/pull/3028
+TEST_F(test_falco_engine, list_value_with_escaping)
+{
+    std::string rules_content = R"END(
+- list: my_list
+  items: [non_escaped_val, "escaped val"]
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(m_load_result->successful());
+  ASSERT_TRUE(m_load_result->has_warnings()); // a warning for the unused list
+
+  auto rule_description = m_engine->describe_rule(nullptr, {});
+  ASSERT_TRUE(m_load_result->successful());
+  ASSERT_EQ(rule_description["rules"].size(), 0);
+  ASSERT_EQ(rule_description["macros"].size(), 0);
+  ASSERT_EQ(rule_description["lists"].size(), 1);
+
+  // escaped values must not be interpreted as list refs by mistake
+  ASSERT_EQ(rule_description["lists"][0]["details"]["lists"].size(), 0);
+
+  // values should be escaped correctly
+  ASSERT_EQ(rule_description["lists"][0]["details"]["items_compiled"].size(), 2);
+  ASSERT_EQ(rule_description["lists"][0]["details"]["items_compiled"][0].template get<std::string>(), "non_escaped_val");
+  ASSERT_EQ(rule_description["lists"][0]["details"]["items_compiled"][1].template get<std::string>(), "escaped val");
+}
+
+TEST_F(test_falco_engine, exceptions_condition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule
+  condition: proc.cmdline contains curl or proc.cmdline contains wget
+  output: command=%proc.cmdline
+  priority: INFO
+  exceptions:
+    - name: test_exception
+      fields: [proc.cmdline]
+      comps: [contains]
+      values:
+        - [curl 127.0.0.1]
+)END";
+
+  ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+  ASSERT_EQ(get_compiled_rule_condition("test_rule"),"((proc.cmdline contains curl or proc.cmdline contains wget) and not proc.cmdline contains \"curl 127.0.0.1\")");
+}
+
+TEST_F(test_falco_engine, macro_name_invalid)
+{
+    std::string rules_content = R"END(
+- macro: test-macro
+  condition: evt.type = close
+)END";
+
+  ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+  ASSERT_TRUE(check_warning_message("Macro has an invalid name. Macro names should match a regular expression"));
+}
+
+TEST_F(test_falco_engine, list_name_invalid)
+{
+    std::string rules_content = R"END(
+- list: test list
+  items: [open, openat, openat2]
+
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type in (test list)
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+)END";
+
+  ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+  ASSERT_TRUE(check_warning_message("List has an invalid name. List names should match a regular expression"));
+}
+
+// The appended exception has a purposely miswritten field (value),
+// simulating a typo or an incorrect usage.
+TEST_F(test_falco_engine, exceptions_append_no_values)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule
+  condition: proc.cmdline contains curl
+  output: command=%proc.cmdline
+  priority: INFO
+  exceptions:
+    - name: test_exception
+      fields: [proc.cmdline]
+      comps: [contains]
+      values:
+        - [curl 127.0.0.1]
+
+- rule: test_rule
+  exceptions:
+    - name: test_exception
+      value: curl 1.1.1.1
+  append: true
+)END";
+
+  ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+  ASSERT_TRUE(check_warning_message("Overriding/appending exception with no values"));
+}
+
+TEST_F(test_falco_engine, exceptions_override_no_values)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule
+  condition: proc.cmdline contains curl
+  output: command=%proc.cmdline
+  priority: INFO
+  exceptions:
+    - name: test_exception
+      fields: [proc.cmdline]
+      comps: [contains]
+      values:
+        - [curl 127.0.0.1]
+
+- rule: test_rule
+  exceptions:
+    - name: test_exception
+      value: curl 1.1.1.1
+  override:
+    exceptions: append
+)END";
+
+  ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+  ASSERT_TRUE(check_warning_message("Overriding/appending exception with no values"));
+}
+
+TEST_F(test_falco_engine, exceptions_names_not_unique)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule
+  condition: proc.cmdline contains curl
+  output: command=%proc.cmdline
+  priority: INFO
+  exceptions:
+    - name: test_exception
+      fields: [proc.cmdline]
+      comps: [contains]
+      values:
+        - [curl 127.0.0.1]
+    - name: test_exception
+      fields: [proc.cmdline]
+      comps: [endswith]
+      values:
+        - [curl 127.0.0.1]
+)END";
+
+  ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+  ASSERT_TRUE(check_warning_message("Multiple definitions of exception"));
+}

unit_tests/engine/test_rulesets.cpp

@@ -23,41 +23,39 @@ limitations under the License.
 #define RULESET_2 2
 
 /* Helpers methods */
-static std::shared_ptr<gen_event_filter_factory> create_factory(filter_check_list& list)
+static std::shared_ptr<sinsp_filter_factory> create_factory(sinsp* inspector, filter_check_list& list)
 {
-	std::shared_ptr<gen_event_filter_factory> ret(new sinsp_filter_factory(NULL, list));
-	return ret;
+	return std::make_shared<sinsp_filter_factory>(inspector, list);
 }
 
-static std::shared_ptr<filter_ruleset> create_ruleset(std::shared_ptr<gen_event_filter_factory> f)
+static std::shared_ptr<filter_ruleset> create_ruleset(std::shared_ptr<sinsp_filter_factory> f)
 {
-	std::shared_ptr<filter_ruleset> ret(new evttype_index_ruleset(f));
-	return ret;
+	return std::make_shared<evttype_index_ruleset>(f);
 }
 
-static std::shared_ptr<libsinsp::filter::ast::expr> create_ast(std::shared_ptr<gen_event_filter_factory> f)
+static std::shared_ptr<libsinsp::filter::ast::expr> create_ast(std::shared_ptr<sinsp_filter_factory> f)
 {
 	libsinsp::filter::parser parser("evt.type=open");
-	std::shared_ptr<libsinsp::filter::ast::expr> ret(parser.parse());
-	return ret;
+	return parser.parse();
 }
 
-static std::shared_ptr<gen_event_filter> create_filter(
-	std::shared_ptr<gen_event_filter_factory> f,
-	std::shared_ptr<libsinsp::filter::ast::expr> ast)
+static std::shared_ptr<sinsp_filter> create_filter(
+	std::shared_ptr<sinsp_filter_factory> f,
+	libsinsp::filter::ast::expr* ast)
 {
-	sinsp_filter_compiler compiler(f, ast.get());
-	std::shared_ptr<gen_event_filter> filter(compiler.compile());
-	return filter;
+	sinsp_filter_compiler compiler(f, ast);
+	return std::shared_ptr<sinsp_filter>(compiler.compile());
 }
 
 TEST(Ruleset, enable_disable_rules_using_names)
 {
+	sinsp inspector;
+
 	sinsp_filter_check_list filterlist;
-	auto f = create_factory(filterlist);
+	auto f = create_factory(&inspector, filterlist);
 	auto r = create_ruleset(f);
 	auto ast = create_ast(f);
-	auto filter = create_filter(f, ast);
+	auto filter = create_filter(f, ast.get());
 
 	falco_rule rule_A = {};
 	rule_A.name = "rule_A";
@@ -120,11 +118,13 @@ TEST(Ruleset, enable_disable_rules_using_names)
 
 TEST(Ruleset, enable_disable_rules_using_tags)
 {
+	sinsp inspector;
+
 	sinsp_filter_check_list filterlist;
-	auto f = create_factory(filterlist);
+	auto f = create_factory(&inspector, filterlist);
 	auto r = create_ruleset(f);
 	auto ast = create_ast(f);
-	auto filter = create_filter(f, ast);
+	auto filter = create_filter(f, ast.get());
 
 	falco_rule rule_A = {};
 	rule_A.name = "rule_A";

unit_tests/falco/app/actions/test_configure_interesting_sets.cpp

@@ -16,7 +16,9 @@ limitations under the License.
 
 */
 
-#include <falco_engine.h>
+#include "../../../test_falco_engine.h"
+
+#include <utility>
 
 #include <falco/app/app.h>
 #include "app_action_helpers.h"
@@ -57,14 +59,11 @@ static strset_t s_sample_nonsyscall_filters = {
 	"evt.type in (procexit, switch, pluginevent, container)"};
 
 
-// todo(jasondellaluce): once we have deeper and more modular
-// control on the falco engine, make this a little nicer
-static std::shared_ptr<falco_engine> mock_engine_from_filters(const strset_t& filters)
+static std::string ruleset_from_filters(const strset_t& filters)
 {
-	// craft a fake ruleset with the given filters
-	int n_rules = 0;
 	std::string dummy_rules;
 	falco::load_result::rules_contents_t content = {{"dummy_rules.yaml", dummy_rules}};
+	int n_rules = 0;
 	for (const auto& f : filters)
 	{
 		n_rules++;
@@ -76,28 +75,18 @@ static std::shared_ptr<falco_engine> mock_engine_from_filters(const strset_t& fi
 			+ "  priority: CRITICAL\n\n";
 	}
 
-	// create a falco engine and load the ruleset
-	sinsp_filter_check_list filterlist;
-	std::shared_ptr<falco_engine> res(new falco_engine());
-	auto filter_factory = std::shared_ptr<gen_event_filter_factory>(
-			new sinsp_filter_factory(nullptr, filterlist));
-	auto formatter_factory = std::shared_ptr<gen_event_formatter_factory>(
-			new sinsp_evt_formatter_factory(nullptr, filterlist));
-	res->add_source(s_sample_source, filter_factory, formatter_factory);
-	res->load_rules(dummy_rules, "dummy_rules.yaml");
-	res->enable_rule("", true, s_sample_ruleset);
-	return res;
+	return dummy_rules;
 }
 
-TEST(ConfigureInterestingSets, engine_codes_syscalls_set)
+TEST_F(test_falco_engine, engine_codes_syscalls_set)
 {
+	load_rules(ruleset_from_filters(s_sample_filters), "dummy_ruleset.yaml");
 
-	auto engine = mock_engine_from_filters(s_sample_filters);
-	auto enabled_count = engine->num_rules_for_ruleset(s_sample_ruleset);
+	auto enabled_count = m_engine->num_rules_for_ruleset(s_sample_ruleset);
 	ASSERT_EQ(enabled_count, s_sample_filters.size());
 
 	// test if event code names were extracted from each rule in test ruleset.
-	auto rules_event_set = engine->event_codes_for_ruleset(s_sample_source);
+	auto rules_event_set = m_engine->event_codes_for_ruleset(s_sample_source);
 	auto rules_event_names = libsinsp::events::event_set_to_names(rules_event_set);
 	ASSERT_NAMES_EQ(rules_event_names, strset_t({
 		"connect", "accept", "accept4", "umount2", "open", "ptrace", "mmap", "execve", "read", "container", "asyncevent"}));
@@ -105,30 +94,31 @@ TEST(ConfigureInterestingSets, engine_codes_syscalls_set)
 	// test if sc code names were extracted from each rule in test ruleset.
 	// note, this is not supposed to contain "container", as that's an event
 	// not mapped through the ppm_sc_code enumerative.
-	auto rules_sc_set = engine->sc_codes_for_ruleset(s_sample_source);
+	auto rules_sc_set = m_engine->sc_codes_for_ruleset(s_sample_source);
 	auto rules_sc_names = libsinsp::events::sc_set_to_event_names(rules_sc_set);
 	ASSERT_NAMES_EQ(rules_sc_names, strset_t({
 		"connect", "accept", "accept4", "umount2", "open", "ptrace", "mmap", "execve", "read"}));
 }
 
-TEST(ConfigureInterestingSets, preconditions_postconditions)
+TEST_F(test_falco_engine, preconditions_postconditions)
 {
-	auto mock_engine = mock_engine_from_filters(s_sample_filters);
+	load_rules(ruleset_from_filters(s_sample_filters), "dummy_ruleset.yaml");
+
 	falco::app::state s1;
 
-	s1.engine = mock_engine;
-	s1.config = nullptr;
+	s1.engine = nullptr;
+	s1.config = std::make_shared<falco_configuration>();
+	s1.options.all_events = false;
 	auto result = falco::app::actions::configure_interesting_sets(s1);
 	ASSERT_FALSE(result.success);
 	ASSERT_NE(result.errstr, "");
 
-	s1.engine = nullptr;
-	s1.config = std::make_shared<falco_configuration>();
+	s1.engine = m_engine;
+	s1.config = nullptr;
 	result = falco::app::actions::configure_interesting_sets(s1);
 	ASSERT_FALSE(result.success);
 	ASSERT_NE(result.errstr, "");
 
-	s1.engine = mock_engine;
 	s1.config = std::make_shared<falco_configuration>();
 	result = falco::app::actions::configure_interesting_sets(s1);
 	ASSERT_TRUE(result.success);
@@ -141,17 +131,18 @@ TEST(ConfigureInterestingSets, preconditions_postconditions)
 	ASSERT_EQ(prev_selection_size, s1.selected_sc_set.size());
 }
 
-TEST(ConfigureInterestingSets, engine_codes_nonsyscalls_set)
+TEST_F(test_falco_engine, engine_codes_nonsyscalls_set)
 {
 	auto filters = s_sample_filters;
 	filters.insert(s_sample_generic_filters.begin(), s_sample_generic_filters.end());
 	filters.insert(s_sample_nonsyscall_filters.begin(), s_sample_nonsyscall_filters.end());
 
-	auto engine = mock_engine_from_filters(filters);
-	auto enabled_count = engine->num_rules_for_ruleset(s_sample_ruleset);
+	load_rules(ruleset_from_filters(filters), "dummy_ruleset.yaml");
+
+	auto enabled_count = m_engine->num_rules_for_ruleset(s_sample_ruleset);
 	ASSERT_EQ(enabled_count, filters.size());
 
-	auto rules_event_set = engine->event_codes_for_ruleset(s_sample_source);
+	auto rules_event_set = m_engine->event_codes_for_ruleset(s_sample_source);
 	auto rules_event_names = libsinsp::events::event_set_to_names(rules_event_set);
 	// note: including even one generic event will cause PPME_GENERIC_E to be
 	// included in the ruleset's event codes. As such, when translating to names,
@@ -164,7 +155,7 @@ TEST(ConfigureInterestingSets, engine_codes_nonsyscalls_set)
 	expected_names.insert(generic_names.begin(), generic_names.end());
 	ASSERT_NAMES_EQ(rules_event_names, expected_names);
 
-	auto rules_sc_set = engine->sc_codes_for_ruleset(s_sample_source);
+	auto rules_sc_set = m_engine->sc_codes_for_ruleset(s_sample_source);
 	auto rules_sc_names = libsinsp::events::sc_set_to_event_names(rules_sc_set);
 	ASSERT_NAMES_EQ(rules_sc_names, strset_t({
 		"connect", "accept", "accept4", "umount2", "open", "ptrace", "mmap", "execve", "read",
@@ -172,11 +163,13 @@ TEST(ConfigureInterestingSets, engine_codes_nonsyscalls_set)
 	}));
 }
 
-TEST(ConfigureInterestingSets, selection_not_allevents)
+TEST_F(test_falco_engine, selection_not_allevents)
 {
+	load_rules(ruleset_from_filters(s_sample_filters), "dummy_ruleset.yaml");
+
 	falco::app::state s2;
 	// run app action with fake engine and without the `-A` option
-	s2.engine = mock_engine_from_filters(s_sample_filters);
+	s2.engine = m_engine;
 	s2.options.all_events = false;
 
 	ASSERT_EQ(s2.options.all_events, false);
@@ -217,11 +210,13 @@ TEST(ConfigureInterestingSets, selection_not_allevents)
 	ASSERT_EQ(s2.selected_sc_set, union_set);
 }
 
-TEST(ConfigureInterestingSets, selection_allevents)
+TEST_F(test_falco_engine, selection_allevents)
 {
+	load_rules(ruleset_from_filters(s_sample_filters), "dummy_ruleset.yaml");
+
 	falco::app::state s3;
 	// run app action with fake engine and with the `-A` option
-	s3.engine = mock_engine_from_filters(s_sample_filters);
+	s3.engine = m_engine;
 	s3.options.all_events = true;
 	auto result = falco::app::actions::configure_interesting_sets(s3);
 	ASSERT_TRUE(result.success);
@@ -250,14 +245,15 @@ TEST(ConfigureInterestingSets, selection_allevents)
 	ASSERT_EQ(s3.selected_sc_set, union_set);
 }
 
-TEST(ConfigureInterestingSets, selection_generic_evts)
+TEST_F(test_falco_engine, selection_generic_evts)
 {
 	falco::app::state s4;
 	// run app action with fake engine and without the `-A` option
 	s4.options.all_events = false;
 	auto filters = s_sample_filters;
 	filters.insert(s_sample_generic_filters.begin(), s_sample_generic_filters.end());
-	s4.engine = mock_engine_from_filters(filters);
+	load_rules(ruleset_from_filters(filters), "dummy_ruleset.yaml");
+	s4.engine = m_engine;
 	auto result = falco::app::actions::configure_interesting_sets(s4);
 	ASSERT_TRUE(result.success);
 	ASSERT_EQ(result.errstr, "");
@@ -282,12 +278,14 @@ TEST(ConfigureInterestingSets, selection_generic_evts)
 //   (either default or custom positive set)
 // - events in the custom negative set are removed from the selected set
 // - if `-A` is not set, events from the IO set are removed from the selected set
-TEST(ConfigureInterestingSets, selection_custom_base_set)
+TEST_F(test_falco_engine, selection_custom_base_set)
 {
+	load_rules(ruleset_from_filters(s_sample_filters), "dummy_ruleset.yaml");
+
 	falco::app::state s5;
 	// run app action with fake engine and without the `-A` option
 	s5.options.all_events = true;
-	s5.engine = mock_engine_from_filters(s_sample_filters);
+	s5.engine = m_engine;
 	auto default_base_set = libsinsp::events::sinsp_state_sc_set();
 
 	// non-empty custom base set (both positive and negative)
@@ -365,12 +363,14 @@ TEST(ConfigureInterestingSets, selection_custom_base_set)
 	ASSERT_NAMES_NOCONTAIN(selected_sc_names, unexpected_sc_names);
 }
 
-TEST(ConfigureInterestingSets, selection_custom_base_set_repair)
+TEST_F(test_falco_engine, selection_custom_base_set_repair)
 {
+	load_rules(ruleset_from_filters(s_sample_filters), "dummy_ruleset.yaml");
+
 	falco::app::state s6;
 	// run app action with fake engine and without the `-A` option
 	s6.options.all_events = false;
-	s6.engine = mock_engine_from_filters(s_sample_filters);
+	s6.engine = m_engine;
 
 	// note: here we use file syscalls (e.g. open, openat) and have a custom
 	// positive set, so we expect syscalls such as "close" to be selected as
@@ -393,12 +393,14 @@ TEST(ConfigureInterestingSets, selection_custom_base_set_repair)
 	ASSERT_NAMES_NOCONTAIN(selected_sc_names, unexpected_sc_names);
 }
 
-TEST(ConfigureInterestingSets, selection_empty_custom_base_set_repair)
+TEST_F(test_falco_engine, selection_empty_custom_base_set_repair)
 {
+	load_rules(ruleset_from_filters(s_sample_filters), "dummy_ruleset.yaml");
+
 	falco::app::state s7;
 	// run app action with fake engine and with the `-A` option
 	s7.options.all_events = true;
-	s7.engine = mock_engine_from_filters(s_sample_filters);
+	s7.engine = m_engine;
 
 	// simulate empty custom set but repair option set.
 	s7.config->m_base_syscalls_custom_set = {};

unit_tests/falco/app/actions/test_configure_syscall_buffer_num.cpp

@@ -8,7 +8,7 @@ You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
 
-Unless ASSERTd by applicable law or agreed to in writing, software
+Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and

unit_tests/falco/app/actions/test_load_config.cpp

@@ -8,7 +8,7 @@ You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
 
-Unless ASSERTd by applicable law or agreed to in writing, software
+Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
@@ -19,10 +19,10 @@ limitations under the License.
 #include "falco_test_var.h"
 
 #ifndef __EMSCRIPTEN__
-TEST(ActionLoadConfig, check_engine_config_is_correctly_parsed)
+TEST(ActionLoadConfig, check_kmod_engine_config)
 {
 	falco::app::state s = {};
-	s.options.conf_filename = NEW_ENGINE_CONFIG_CHANGED;
+	s.options.conf_filename = TEST_ENGINE_KMOD_CONFIG;
 	EXPECT_ACTION_OK(falco::app::actions::load_config(s));
 
 	// Check that the engine is the kmod
@@ -45,105 +45,26 @@ TEST(ActionLoadConfig, check_engine_config_is_correctly_parsed)
 
 	EXPECT_TRUE(s.config->m_gvisor.m_config.empty());
 	EXPECT_TRUE(s.config->m_gvisor.m_root.empty());
-
-	// Check that deprecated configs are always set since
-	EXPECT_EQ(s.config->m_syscall_buf_size_preset, 6);
-	EXPECT_EQ(s.config->m_cpus_for_each_syscall_buffer, 7);
-	EXPECT_TRUE(s.config->m_syscall_drop_failed_exit);
 }
 
-// Equal to the one above but checks that the command line options are not parsed
-TEST(ActionLoadConfig, check_command_line_options_are_not_used)
+TEST(ActionLoadConfig, check_modern_engine_config)
 {
-	falco::app::state s;
-	s.options.modern_bpf = true;
-	s.options.conf_filename = NEW_ENGINE_CONFIG_CHANGED;
+	falco::app::state s = {};
+	s.options.conf_filename = TEST_ENGINE_MODERN_CONFIG;
 	EXPECT_ACTION_OK(falco::app::actions::load_config(s));
 
-	// Check that the engine is the kmod
-	EXPECT_TRUE(s.config->m_engine_mode == engine_kind_t::KMOD);
+	// Check that the engine is the modern ebpf
+	EXPECT_TRUE(s.config->m_engine_mode == engine_kind_t::MODERN_EBPF);
 
-	// Check that kmod params are the ones specified in the config
-	EXPECT_EQ(s.config->m_kmod.m_buf_size_preset, 2);
-	EXPECT_FALSE(s.config->m_kmod.m_drop_failed_exit);
-
-	// Check that all other engine params are empty
-	EXPECT_TRUE(s.config->m_ebpf.m_probe_path.empty());
-	EXPECT_EQ(s.config->m_ebpf.m_buf_size_preset, 0);
-	EXPECT_FALSE(s.config->m_ebpf.m_drop_failed_exit);
-
-	EXPECT_EQ(s.config->m_modern_ebpf.m_cpus_for_each_buffer, 0);
-	EXPECT_EQ(s.config->m_modern_ebpf.m_buf_size_preset, 0);
-	EXPECT_FALSE(s.config->m_modern_ebpf.m_drop_failed_exit);
-
-	EXPECT_TRUE(s.config->m_replay.m_capture_file.empty());
-
-	EXPECT_TRUE(s.config->m_gvisor.m_config.empty());
-	EXPECT_TRUE(s.config->m_gvisor.m_root.empty());
-
-	// Check that deprecated configs are always set since
-	EXPECT_EQ(s.config->m_syscall_buf_size_preset, 6);
-	EXPECT_EQ(s.config->m_cpus_for_each_syscall_buffer, 7);
-	EXPECT_TRUE(s.config->m_syscall_drop_failed_exit);
-}
-
-TEST(ActionLoadConfig, check_kmod_with_syscall_configs)
-{
-	falco::app::state s;
-	s.options.conf_filename = NEW_ENGINE_CONFIG_UNCHANGED;
-	EXPECT_ACTION_OK(falco::app::actions::load_config(s));
-
-	// Check that the engine is the kmod
-	EXPECT_TRUE(s.config->m_engine_mode == engine_kind_t::KMOD);
-
-	// Kmod params should be populated with the syscall configs
-	// since the `engine` block is untouched.
-	EXPECT_EQ(s.config->m_kmod.m_buf_size_preset, 6);
-	EXPECT_TRUE(s.config->m_kmod.m_drop_failed_exit);
-
-	// Check that all other engine params are empty
-	EXPECT_TRUE(s.config->m_ebpf.m_probe_path.empty());
-	EXPECT_EQ(s.config->m_ebpf.m_buf_size_preset, 0);
-	EXPECT_FALSE(s.config->m_ebpf.m_drop_failed_exit);
-
-	EXPECT_EQ(s.config->m_modern_ebpf.m_cpus_for_each_buffer, 0);
-	EXPECT_EQ(s.config->m_modern_ebpf.m_buf_size_preset, 0);
-	EXPECT_FALSE(s.config->m_modern_ebpf.m_drop_failed_exit);
-
-	EXPECT_TRUE(s.config->m_replay.m_capture_file.empty());
-
-	EXPECT_TRUE(s.config->m_gvisor.m_config.empty());
-	EXPECT_TRUE(s.config->m_gvisor.m_root.empty());
-
-	// Check that deprecated configs are populated
-	EXPECT_EQ(s.config->m_syscall_buf_size_preset, 6);
-	EXPECT_EQ(s.config->m_cpus_for_each_syscall_buffer, 3);
-	EXPECT_TRUE(s.config->m_syscall_drop_failed_exit);
-}
-
-TEST(ActionLoadConfig, check_override_command_line_modern)
-{
-	falco::app::state s;
-	// The command line options should be correctly applied since the
-	// config is unchanged
-	s.options.modern_bpf = true;
-	s.options.conf_filename = NEW_ENGINE_CONFIG_UNCHANGED;
-	EXPECT_ACTION_OK(falco::app::actions::load_config(s));
-
-	// Check that the engine is the kmod
-	EXPECT_TRUE(s.is_modern_ebpf());
-
-	// Check that the modern ebpf engine uses the default syscall configs
-	// and not the ones in the `engine` block
-	EXPECT_EQ(s.config->m_modern_ebpf.m_cpus_for_each_buffer, 3);
-	EXPECT_EQ(s.config->m_modern_ebpf.m_buf_size_preset, 6);
+	// Check that modern ebpf params are the ones specified in the config
+	EXPECT_EQ(s.config->m_modern_ebpf.m_cpus_for_each_buffer, 1);
+	EXPECT_EQ(s.config->m_modern_ebpf.m_buf_size_preset, 4);
 	EXPECT_TRUE(s.config->m_modern_ebpf.m_drop_failed_exit);
 
-	// Kmod params should be always populated since the kmod is the default
-	EXPECT_EQ(s.config->m_kmod.m_buf_size_preset, 6);
-	EXPECT_TRUE(s.config->m_kmod.m_drop_failed_exit);
-
 	// Check that all other engine params are empty
+	EXPECT_EQ(s.config->m_kmod.m_buf_size_preset, 0);
+	EXPECT_FALSE(s.config->m_kmod.m_drop_failed_exit);
+
 	EXPECT_TRUE(s.config->m_ebpf.m_probe_path.empty());
 	EXPECT_EQ(s.config->m_ebpf.m_buf_size_preset, 0);
 	EXPECT_FALSE(s.config->m_ebpf.m_drop_failed_exit);
@@ -152,45 +73,6 @@ TEST(ActionLoadConfig, check_override_command_line_modern)
 
 	EXPECT_TRUE(s.config->m_gvisor.m_config.empty());
 	EXPECT_TRUE(s.config->m_gvisor.m_root.empty());
-
-	// Check that deprecated configs are populated
-	EXPECT_EQ(s.config->m_syscall_buf_size_preset, 6);
-	EXPECT_EQ(s.config->m_cpus_for_each_syscall_buffer, 3);
-	EXPECT_TRUE(s.config->m_syscall_drop_failed_exit);
 }
 
-TEST(ActionLoadConfig, check_override_command_line_gvisor)
-{
-	falco::app::state s;
-	// The command line options should be correctly applied since the
-	// config is unchanged
-	s.options.gvisor_config = "config";
-	s.options.conf_filename = NEW_ENGINE_CONFIG_UNCHANGED;
-	EXPECT_ACTION_OK(falco::app::actions::load_config(s));
-
-	// Check that the engine is the kmod
-	EXPECT_TRUE(s.is_gvisor());
-	EXPECT_EQ(s.config->m_gvisor.m_config, "config");
-	EXPECT_TRUE(s.config->m_gvisor.m_root.empty());
-
-	// Kmod params should be always populated since the kmod is the default
-	EXPECT_EQ(s.config->m_kmod.m_buf_size_preset, 6);
-	EXPECT_TRUE(s.config->m_kmod.m_drop_failed_exit);
-
-	// Check that all other engine params are empty
-	EXPECT_TRUE(s.config->m_ebpf.m_probe_path.empty());
-	EXPECT_EQ(s.config->m_ebpf.m_buf_size_preset, 0);
-	EXPECT_FALSE(s.config->m_ebpf.m_drop_failed_exit);
-
-	EXPECT_EQ(s.config->m_modern_ebpf.m_cpus_for_each_buffer, 0);
-	EXPECT_EQ(s.config->m_modern_ebpf.m_buf_size_preset, 0);
-	EXPECT_FALSE(s.config->m_modern_ebpf.m_drop_failed_exit);
-
-	EXPECT_TRUE(s.config->m_replay.m_capture_file.empty());
-
-	// Check that deprecated configs are populated
-	EXPECT_EQ(s.config->m_syscall_buf_size_preset, 6);
-	EXPECT_EQ(s.config->m_cpus_for_each_syscall_buffer, 3);
-	EXPECT_TRUE(s.config->m_syscall_drop_failed_exit);
-}
 #endif

unit_tests/falco/app/actions/test_select_event_sources.cpp

@@ -8,7 +8,7 @@ You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
 
-Unless ASSERTd by applicable law or agreed to in writing, software
+Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and

unit_tests/falco/test_atomic_signal_handler.cpp

@@ -8,23 +8,23 @@ You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
 
-Unless ASSERTd by applicable law or agreed to in writing, software
+Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include <gtest/gtest.h>
-#include <future>
-#include <thread>
-#include <vector>
-#include <memory>
-#include <chrono>
- 
 #include <falco/atomic_signal_handler.h>
 #include <falco/logger.h>
 
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <future>
+#include <memory>
+#include <vector>
+
 TEST(AtomicSignalHandler, lock_free_implementation)
 {
 	ASSERT_TRUE(falco::atomic_signal_handler().is_lock_free());
@@ -33,55 +33,50 @@ TEST(AtomicSignalHandler, lock_free_implementation)
 TEST(AtomicSignalHandler, handle_once_wait_consistency)
 {
 	constexpr const auto thread_num = 10;
-	constexpr const auto thread_wait_sec = 2;
-	constexpr const auto handler_wait_sec = 1;
+	constexpr const std::chrono::seconds thread_wait_sec{2};
+	constexpr const std::chrono::seconds handler_wait_sec{1};
 
 	// have a shared signal handler
 	falco::atomic_signal_handler handler;
 
 	// launch a bunch of threads all syncing on the same handler
-	typedef struct
+	struct task_result_t
 	{
 		bool handled;
-		uint64_t duration_secs;
-	} task_result_t;
+		std::chrono::seconds duration_secs;
+	};
+
 	std::vector<std::future<task_result_t>> futures;
-	std::vector<std::unique_ptr<std::thread>> threads;
 	for (int i = 0; i < thread_num; i++)
 	{
-		std::packaged_task<task_result_t()> task([&handler, &thread_wait_sec]{
-			auto start = std::chrono::high_resolution_clock::now();
-			task_result_t res;
-			res.handled = false;
-			while (!handler.handled())
-			{
-				if (handler.triggered())
+		futures.emplace_back(std::async(std::launch::async,
+			[&handler, thread_wait_sec]() {
+				auto start = std::chrono::high_resolution_clock::now();
+				task_result_t res;
+				res.handled = false;
+				while (!handler.handled())
 				{
-					res.handled = handler.handle([&thread_wait_sec]{
-						std::this_thread::sleep_for (std::chrono::seconds(thread_wait_sec));
-					});
+					if (handler.triggered())
+					{
+						res.handled = handler.handle([thread_wait_sec]() {
+							std::this_thread::sleep_for(thread_wait_sec);
+						});
+					}
 				}
-			}
-			auto diff = std::chrono::high_resolution_clock::now() - start;
-			res.duration_secs = std::chrono::duration_cast<std::chrono::seconds>(diff).count();
-			return res;
-		});
-		futures.push_back(task.get_future());
-		threads.emplace_back();
-		threads[i].reset(new std::thread(std::move(task)));
+				auto diff = std::chrono::high_resolution_clock::now() - start;
+				res.duration_secs = std::chrono::duration_cast<std::chrono::seconds>(diff);
+				return res;
+			}));
 	}
 
 	// wait a bit, then trigger the signal handler from the main thread
 	auto total_handled = 0;
 	auto start = std::chrono::high_resolution_clock::now();
-	std::this_thread::sleep_for (std::chrono::seconds(handler_wait_sec));
+	std::this_thread::sleep_for(handler_wait_sec);
 	handler.trigger();
 	for (int i = 0; i < thread_num; i++)
 	{
-		// we need to check that all threads didn't quit before
-		// the handle() function finished executing
-		futures[i].wait();
-		threads[i]->join();
+		// wait for all threads to finish and get the results from the futures
 		auto res = futures[i].get();
 		if (res.handled)
 		{
@@ -92,7 +87,7 @@ TEST(AtomicSignalHandler, handle_once_wait_consistency)
 
 	// check that the total time is consistent with the expectations
 	auto diff = std::chrono::high_resolution_clock::now() - start;
-	auto secs = std::chrono::duration_cast<std::chrono::seconds>(diff).count();
+	auto secs = std::chrono::duration_cast<std::chrono::seconds>(diff);
 	ASSERT_GE(secs, thread_wait_sec + handler_wait_sec);
 
 	// check that only one thread handled the signal

unit_tests/falco/test_configs/engine_kmod_config.yaml

@@ -20,16 +20,15 @@
 # Falco engine #
 ################
 
-# Unchanged
 engine:
   kind: kmod
   kmod:
-    buf_size_preset: 4
+    buf_size_preset: 2
     drop_failed_exit: false
   ebpf:
     probe: /path/to/probe.o
-    buf_size_preset: 4
-    drop_failed_exit: false
+    buf_size_preset: 7
+    drop_failed_exit: true
   modern_ebpf:
     cpus_for_each_buffer: 2
     buf_size_preset: 4
@@ -39,15 +38,3 @@ engine:
   gvisor:
     config: /path/to/gvisor_config.yaml
     root: ""
-
-#######################################
-# Falco performance tuning (advanced) #
-#######################################
-
-# The `engine` config is unchanged so these configs are used
-syscall_buf_size_preset: 6
-
-syscall_drop_failed_exit: true
-
-modern_bpf:
-  cpus_for_each_syscall_buffer: 3

unit_tests/falco/test_configs/engine_modern_config.yaml

@@ -21,32 +21,20 @@
 ################
 
 engine:
-  kind: kmod
+  kind: modern_ebpf
   kmod:
-    buf_size_preset: 2 # changed default value
-    drop_failed_exit: false
+    buf_size_preset: 1
+    drop_failed_exit: true
   ebpf:
     probe: /path/to/probe.o
     buf_size_preset: 4
     drop_failed_exit: false
   modern_ebpf:
-    cpus_for_each_buffer: 2
-    buf_size_preset: 4
-    drop_failed_exit: false
+    cpus_for_each_buffer: 1
+    # missing `buf_size_preset` should be defaulted
+    drop_failed_exit: true
   replay:
     capture_file: /path/to/file.scap
   gvisor:
     config: /path/to/gvisor_config.yaml
     root: ""
-
-#######################################
-# Falco performance tuning (advanced) #
-#######################################
-
-# These configs should be ignored since we have changed the `engine` config
-syscall_buf_size_preset: 6
-
-syscall_drop_failed_exit: true
-
-modern_bpf:
-  cpus_for_each_syscall_buffer: 7

unit_tests/falco/test_configuration.cpp

@@ -8,7 +8,7 @@ You may obtain a copy of the License at
 
     http://www.apache.org/licenses/LICENSE-2.0
 
-Unless ASSERTd by applicable law or agreed to in writing, software
+Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
@@ -109,6 +109,484 @@ TEST(Configuration, modify_yaml_fields)
 	ASSERT_EQ(conf.get_scalar<bool>(key, false), true);
 }
 
+TEST(Configuration, configuration_config_files_secondary_fail)
+{
+	/* Test that a secondary config file is not able to include anything, triggering an exception. */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - conf_3.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_4.yaml\n"
+					   "foo2: bar2\n"
+					   "base_value_2:\n"
+					   "    id: 2\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_ANY_THROW(falco_config.init("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+}
+
+TEST(Configuration, configuration_config_files_ok)
+{
+	/* Test that every included config file was correctly parsed */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - conf_3.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo3: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+	const std::string conf_yaml_4 =
+		"base_value_4:\n"
+		"    id: 4\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	outfile.open("conf_4.yaml");
+	outfile << conf_yaml_4;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(loaded_conf_files.size(), 3);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(falco_config.config.is_defined("foo3"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo3", ""), "bar3");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_3.id", 0), 3);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value_3.name", ""), "foo3");
+	ASSERT_FALSE(falco_config.config.is_defined("base_value_4.id")); // conf_4 is not included
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+	std::filesystem::remove("conf_4.yaml");
+}
+
+
+TEST(Configuration, configuration_config_files_relative_main)
+{
+	/*
+	 * Test that relative path are treated as relative to cwd and not to main config folder,
+	 * and that absolute includes are ok too.
+	 */
+	const auto temp_main = std::filesystem::temp_directory_path() / "main.yaml";
+	// So, conf_2 will be looked up in the same folder as main config file,
+	// while conf_3, since is absolute, will be looked up in the absolute path (and found!).
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+		"  - conf_2.yaml\n"
+		"  - " +
+		std::filesystem::current_path().string() + "/conf_3.yaml\n"
+							   "foo: bar\n"
+							   "base_value:\n"
+							   "    id: 1\n"
+							   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo3: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+
+	std::ofstream outfile(temp_main.string());
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init(temp_main.string(), loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(loaded_conf_files.size(), 3);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_3.id", 0), 3);
+
+	std::filesystem::remove(temp_main.string());
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+}
+
+TEST(Configuration, configuration_config_files_override)
+{
+	/* Test that included config files are able to override configs from main file */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - conf_3.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"base_value:\n"
+		"    id: 3\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(loaded_conf_files.size(), 3);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 3); // overridden!
+	ASSERT_FALSE(falco_config.config.is_defined("base_value.name"));	// no more present since entire `base_value` block was overridden
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_FALSE(falco_config.config.is_defined("base_value_3.id")); // not defined
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+}
+
+TEST(Configuration, configuration_config_files_unexistent)
+{
+	/* Test that including an unexistent file just skips it */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+		"  - conf_5.yaml\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main
+	ASSERT_EQ(loaded_conf_files.size(), 1);
+
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+
+	std::filesystem::remove("main.yaml");
+}
+
+TEST(Configuration, configuration_config_files_scalar_configs_files)
+{
+	/* Test that a single file can be included as a scalar (thanks to get_sequence_from_node magic) */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ": conf_2.yaml\n"
+		"foo: bar\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2
+	ASSERT_EQ(loaded_conf_files.size(), 2);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+}
+
+TEST(Configuration, configuration_config_files_empty_configs_files)
+{
+	/* Test that empty includes list is accepted */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+		"foo: bar\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main
+	ASSERT_EQ(loaded_conf_files.size(), 1);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+
+	std::filesystem::remove("main.yaml");
+}
+
+TEST(Configuration, configuration_config_files_self)
+{
+	/* Test that main config file cannot include itself */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ": main.yaml\n"
+		"foo: bar\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_ANY_THROW(falco_config.init("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	std::filesystem::remove("main.yaml");
+}
+
+TEST(Configuration, configuration_config_files_directory)
+{
+	/*
+	 * Test that when main config file includes a config directory,
+	 * the config directory is parsed in lexicographic order,
+	 * and only regular files are parsed.
+	 */
+	// Main config includes whole temp directory
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ": " + std::filesystem::temp_directory_path().string() + "/test\n"
+		   "foo: bar\n"
+		   "base_value:\n"
+		   "    id: 1\n"
+		   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo2: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+	const std::string conf_yaml_4 =
+		"foo4: bar4\n";
+
+	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test");
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open(std::filesystem::temp_directory_path()/"test/conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open(std::filesystem::temp_directory_path()/"test/conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	// Create a directory and create a config inside it. We will later check that it was not parsed
+	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test" / "foo");
+	outfile.open(std::filesystem::temp_directory_path()/"test/foo/conf_4.yaml");
+	outfile << conf_yaml_4;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2 + conf_3.
+	// test/foo is not parsed.
+	ASSERT_EQ(loaded_conf_files.size(), 3);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_3.id", 0), 3);
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar3");
+	ASSERT_FALSE(falco_config.config.is_defined("foo4"));
+
+	std::filesystem::remove("main");
+	std::filesystem::remove_all(std::filesystem::temp_directory_path()/"test");
+}
+
+TEST(Configuration, configuration_config_files_cmdline)
+{
+	/* Test that we support including configs files from cmdline option */
+	const std::string main_conf_yaml =
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	// Pass "configs_files=..." cmdline option
+	std::vector<std::string> cmdline_config_options;
+	cmdline_config_options.push_back((yaml_helper::configs_key+"=conf_2.yaml"));
+
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2
+	ASSERT_EQ(loaded_conf_files.size(), 2);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+}
+
 TEST(Configuration, configuration_environment_variables)
 {
     // Set an environment variable for testing purposes

unit_tests/falco_test_var.h.in

@@ -1,4 +1,4 @@
 #pragma once
 
-#define NEW_ENGINE_CONFIG_CHANGED "${CMAKE_SOURCE_DIR}/unit_tests/falco/test_configs/new_engine_config_changed.yaml"
-#define NEW_ENGINE_CONFIG_UNCHANGED "${CMAKE_SOURCE_DIR}/unit_tests/falco/test_configs/new_engine_config_unchanged.yaml"
+#define TEST_ENGINE_KMOD_CONFIG "${CMAKE_SOURCE_DIR}/unit_tests/falco/test_configs/engine_kmod_config.yaml"
+#define TEST_ENGINE_MODERN_CONFIG "${CMAKE_SOURCE_DIR}/unit_tests/falco/test_configs/engine_modern_config.yaml"

unit_tests/test_falco_engine.cpp

@@ -0,0 +1,87 @@
+#include "test_falco_engine.h"
+
+test_falco_engine::test_falco_engine()
+{
+	// create a falco engine ready to load the ruleset
+	m_filter_factory = std::make_shared<sinsp_filter_factory>(&m_inspector, m_filterlist);
+	m_formatter_factory = std::make_shared<sinsp_evt_formatter_factory>(&m_inspector, m_filterlist);
+	m_engine = std::make_shared<falco_engine>();
+	m_engine->add_source(m_sample_source, m_filter_factory, m_formatter_factory);
+}
+
+bool test_falco_engine::load_rules(const std::string& rules_content, const std::string& rules_filename)
+{
+	bool ret = false;
+	falco::load_result::rules_contents_t rc = {{rules_filename, rules_content}};
+	m_load_result = m_engine->load_rules(rules_content, rules_filename);
+	m_load_result_string = m_load_result->as_string(true, rc);
+	m_load_result_json = m_load_result->as_json(rc);
+	ret = m_load_result->successful();
+
+	if (ret)
+	{
+		m_engine->enable_rule("", true, m_sample_ruleset);
+	}
+
+	return ret;
+}
+
+// This must be kept in line with the (private) falco_engine::s_default_ruleset
+uint64_t test_falco_engine::num_rules_for_ruleset(const std::string& ruleset)
+{
+	return m_engine->num_rules_for_ruleset(ruleset);
+}
+
+bool test_falco_engine::has_warnings() const
+{
+	return m_load_result->has_warnings();
+}
+
+bool test_falco_engine::check_warning_message(const std::string& warning_msg) const
+{
+	if(!m_load_result->has_warnings())
+	{
+		return false;
+	}
+
+	for(const auto &warn : m_load_result_json["warnings"])
+	{
+		std::string msg = warn["message"];
+		// Debug:
+		// printf("msg: %s\n", msg.c_str());
+		if(msg.find(warning_msg) != std::string::npos)
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool test_falco_engine::check_error_message(const std::string& error_msg) const
+{
+	// if the loading is successful there are no errors
+	if(m_load_result->successful())
+	{
+		return false;
+	}
+
+	for(const auto &err : m_load_result_json["errors"])
+	{
+		std::string msg = err["message"];
+		// Debug:
+		// printf("msg: %s\n", msg.c_str());
+		if(msg.find(error_msg) != std::string::npos)
+		{
+			return true;
+		}
+	}
+
+	return false;
+}
+
+std::string test_falco_engine::get_compiled_rule_condition(std::string rule_name) const
+{
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	return rule_description["rules"][0]["details"]["condition_compiled"].template get<std::string>();
+}

unit_tests/test_falco_engine.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "falco_engine.h"
+#include "rule_loader_reader.h"
+#include "rule_loader_compiler.h"
+#include "rule_loading_messages.h"
+
+#include <gtest/gtest.h>
+
+class test_falco_engine : public testing::Test
+{
+protected:
+	test_falco_engine();
+
+	bool load_rules(const std::string& rules_content, const std::string& rules_filename);
+	// This must be kept in line with the (private) falco_engine::s_default_ruleset
+	uint64_t num_rules_for_ruleset(const std::string& ruleset = "falco-default-ruleset");
+	bool has_warnings() const;
+	bool check_warning_message(const std::string& warning_msg) const;
+	bool check_error_message(const std::string& error_msg) const;
+	std::string get_compiled_rule_condition(std::string rule_name = "") const;
+
+	std::string m_sample_ruleset = "sample-ruleset";
+	std::string m_sample_source = falco_common::syscall_source;
+	sinsp m_inspector;
+	sinsp_filter_check_list m_filterlist;
+	std::shared_ptr<sinsp_filter_factory> m_filter_factory;
+	std::shared_ptr<sinsp_evt_formatter_factory> m_formatter_factory;
+	std::shared_ptr<falco_engine> m_engine;
+	std::unique_ptr<falco::load_result> m_load_result;
+	std::string m_load_result_string;
+	nlohmann::json m_load_result_json;
+};

userspace/engine/CMakeLists.txt

@@ -16,6 +16,7 @@ add_library(falco_engine STATIC
     falco_engine.cpp
     falco_load_result.cpp
     falco_utils.cpp
+    filter_ruleset.cpp
     evttype_index_ruleset.cpp
     formats.cpp
     filter_details_resolver.cpp
@@ -32,19 +33,15 @@ if (EMSCRIPTEN)
 	target_compile_options(falco_engine PRIVATE "-sDISABLE_EXCEPTION_CATCHING=0")
 endif()
 
-add_dependencies(falco_engine yamlcpp njson)
-
 target_include_directories(falco_engine
 PUBLIC
-    ${LIBSCAP_INCLUDE_DIRS}
-    ${LIBSINSP_INCLUDE_DIRS}
-    ${PROJECT_BINARY_DIR}/userspace/engine
-    ${nlohmann_json_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}
     ${TBB_INCLUDE_DIR}
-    ${YAMLCPP_INCLUDE_DIR}
 )
 
 target_link_libraries(falco_engine
-    ${FALCO_SINSP_LIBRARY}
-    ${YAMLCPP_LIB}
+PUBLIC
+    sinsp
+    nlohmann_json::nlohmann_json
+    yaml-cpp
 )

userspace/engine/evttype_index_ruleset.cpp

@@ -20,7 +20,7 @@ limitations under the License.
 #include <algorithm>
 
 evttype_index_ruleset::evttype_index_ruleset(
-	std::shared_ptr<gen_event_filter_factory> f): m_filter_factory(f)
+	std::shared_ptr<sinsp_filter_factory> f): m_filter_factory(f)
 {
 }
 
@@ -111,11 +111,11 @@ uint64_t evttype_index_ruleset::ruleset_filters::num_filters()
 	return m_filters.size();
 }
 
-bool evttype_index_ruleset::ruleset_filters::run(gen_event *evt, falco_rule& match)
+bool evttype_index_ruleset::ruleset_filters::run(sinsp_evt *evt, falco_rule& match)
 {
     if(evt->get_type() < m_filter_by_event_type.size())
     {
-        for(auto &wrap : m_filter_by_event_type[evt->get_type()])
+        for(const auto &wrap : m_filter_by_event_type[evt->get_type()])
         {
             if(wrap->filter->run(evt))
             {
@@ -126,7 +126,7 @@ bool evttype_index_ruleset::ruleset_filters::run(gen_event *evt, falco_rule& mat
     }
 
 	// Finally, try filters that are not specific to an event type.
-	for(auto &wrap : m_filter_all_event_types)
+	for(const auto &wrap : m_filter_all_event_types)
 	{
 		if(wrap->filter->run(evt))
 		{
@@ -138,13 +138,13 @@ bool evttype_index_ruleset::ruleset_filters::run(gen_event *evt, falco_rule& mat
 	return false;
 }
 
-bool evttype_index_ruleset::ruleset_filters::run(gen_event *evt, std::vector<falco_rule>& matches)
+bool evttype_index_ruleset::ruleset_filters::run(sinsp_evt *evt, std::vector<falco_rule>& matches)
 {
 	bool match_found = false;
 
 	if(evt->get_type() < m_filter_by_event_type.size())
 	{
-		for(auto &wrap : m_filter_by_event_type[evt->get_type()])
+		for(const auto &wrap : m_filter_by_event_type[evt->get_type()])
 		{
 			if(wrap->filter->run(evt))
 			{
@@ -160,7 +160,7 @@ bool evttype_index_ruleset::ruleset_filters::run(gen_event *evt, std::vector<fal
 	}
 
 	// Finally, try filters that are not specific to an event type.
-	for(auto &wrap : m_filter_all_event_types)
+	for(const auto &wrap : m_filter_all_event_types)
 	{
 		if(wrap->filter->run(evt))
 		{
@@ -175,7 +175,7 @@ bool evttype_index_ruleset::ruleset_filters::run(gen_event *evt, std::vector<fal
 libsinsp::events::set<ppm_sc_code> evttype_index_ruleset::ruleset_filters::sc_codes()
 {
 	libsinsp::events::set<ppm_sc_code> res;
-	for(auto &wrap : m_filters)
+	for(const auto &wrap : m_filters)
 	{
 		res.insert(wrap->sc_codes.begin(), wrap->sc_codes.end());
 	}
@@ -185,7 +185,7 @@ libsinsp::events::set<ppm_sc_code> evttype_index_ruleset::ruleset_filters::sc_co
 libsinsp::events::set<ppm_event_code> evttype_index_ruleset::ruleset_filters::event_codes()
 {
 	libsinsp::events::set<ppm_event_code> res;
-	for(auto &wrap : m_filters)
+	for(const auto &wrap : m_filters)
 	{
 		res.insert(wrap->event_codes.begin(), wrap->event_codes.end());
 	}
@@ -194,12 +194,12 @@ libsinsp::events::set<ppm_event_code> evttype_index_ruleset::ruleset_filters::ev
 
 void evttype_index_ruleset::add(
 		const falco_rule& rule,
-		std::shared_ptr<gen_event_filter> filter,
+		std::shared_ptr<sinsp_filter> filter,
 		std::shared_ptr<libsinsp::filter::ast::expr> condition)
 {
 	try
 	{
-		std::shared_ptr<filter_wrapper> wrap(new filter_wrapper());
+		auto wrap = std::make_shared<filter_wrapper>();
 		wrap->rule = rule;
 		wrap->filter = filter;
 		if(rule.source == falco_common::syscall_source)
@@ -230,8 +230,7 @@ void evttype_index_ruleset::clear()
 {
 	for (size_t i = 0; i < m_rulesets.size(); i++)
 	{
-		std::shared_ptr<ruleset_filters> r(new ruleset_filters());
-		m_rulesets[i] = r;
+		m_rulesets[i] = std::make_shared<ruleset_filters>();
 	}
 	m_filters.clear();
 }
@@ -332,7 +331,7 @@ uint64_t evttype_index_ruleset::enabled_count(uint16_t ruleset_id)
 	return m_rulesets[ruleset_id]->num_filters();
 }
 
-bool evttype_index_ruleset::run(gen_event *evt, falco_rule& match, uint16_t ruleset_id)
+bool evttype_index_ruleset::run(sinsp_evt *evt, falco_rule& match, uint16_t ruleset_id)
 {
 	if(m_rulesets.size() < (size_t)ruleset_id + 1)
 	{
@@ -342,7 +341,7 @@ bool evttype_index_ruleset::run(gen_event *evt, falco_rule& match, uint16_t rule
 	return m_rulesets[ruleset_id]->run(evt, match);
 }
 
-bool evttype_index_ruleset::run(gen_event *evt, std::vector<falco_rule>& matches, uint16_t ruleset_id)
+bool evttype_index_ruleset::run(sinsp_evt *evt, std::vector<falco_rule>& matches, uint16_t ruleset_id)
 {
 	if(m_rulesets.size() < (size_t)ruleset_id + 1)
 	{
@@ -369,7 +368,7 @@ libsinsp::events::set<ppm_sc_code> evttype_index_ruleset::enabled_sc_codes(uint1
 	}
 	return m_rulesets[ruleset]->sc_codes();
 }
-	
+
 libsinsp::events::set<ppm_event_code> evttype_index_ruleset::enabled_event_codes(uint16_t ruleset)
 {
 	if(m_rulesets.size() < (size_t)ruleset + 1)

userspace/engine/evttype_index_ruleset.h

@@ -24,11 +24,9 @@ limitations under the License.
 #include <map>
 
 #include "filter_ruleset.h"
-#include "sinsp.h"
-#include "filter.h"
-#include "event.h"
-
-#include "gen_filter.h"
+#include <libsinsp/sinsp.h>
+#include <libsinsp/filter.h>
+#include <libsinsp/event.h>
 
 /*!
 	\brief A filter_ruleset that indexes enabled rules by event type,
@@ -37,18 +35,18 @@ limitations under the License.
 class evttype_index_ruleset: public filter_ruleset
 {
 public:
-	evttype_index_ruleset(std::shared_ptr<gen_event_filter_factory> factory);
+	explicit evttype_index_ruleset(std::shared_ptr<sinsp_filter_factory> factory);
 	virtual ~evttype_index_ruleset();
 
 	void add(
 		const falco_rule& rule,
-		std::shared_ptr<gen_event_filter> filter,
+		std::shared_ptr<sinsp_filter> filter,
 		std::shared_ptr<libsinsp::filter::ast::expr> condition) override;
 
 	void clear() override;
 
-	bool run(gen_event *evt, falco_rule& match, uint16_t ruleset_id) override;
-	bool run(gen_event *evt, std::vector<falco_rule>&matches, uint16_t ruleset_id) override;
+	bool run(sinsp_evt *evt, falco_rule& match, uint16_t ruleset_id) override;
+	bool run(sinsp_evt *evt, std::vector<falco_rule>&matches, uint16_t ruleset_id) override;
 
 	uint64_t enabled_count(uint16_t ruleset_id) override;
 
@@ -103,7 +101,7 @@ private:
 		falco_rule rule;
 		libsinsp::events::set<ppm_sc_code> sc_codes;
 		libsinsp::events::set<ppm_event_code> event_codes;
-		std::shared_ptr<gen_event_filter> filter;
+		std::shared_ptr<sinsp_filter> filter;
 	};
 
 	typedef std::list<std::shared_ptr<filter_wrapper>> filter_wrapper_list;
@@ -122,11 +120,11 @@ private:
 
 		// Evaluate an event against the ruleset and return the first rule
 		// that matched.
-		bool run(gen_event *evt, falco_rule& match);
+		bool run(sinsp_evt *evt, falco_rule& match);
 
-		//  Evaluate an event against the ruleset and return all the 
-		//	matching rules. 
-		bool run(gen_event *evt, std::vector<falco_rule>& matches);
+		//  Evaluate an event against the ruleset and return all the
+		//	matching rules.
+		bool run(sinsp_evt *evt, std::vector<falco_rule>& matches);
 
 		libsinsp::events::set<ppm_sc_code> sc_codes();
 
@@ -153,24 +151,22 @@ private:
 	// All filters added. The set of enabled filters is held in m_rulesets
 	std::set<std::shared_ptr<filter_wrapper>> m_filters;
 
-	std::shared_ptr<gen_event_filter_factory> m_filter_factory;
+	std::shared_ptr<sinsp_filter_factory> m_filter_factory;
 	std::vector<std::string> m_ruleset_names;
 };
 
 class evttype_index_ruleset_factory: public filter_ruleset_factory
 {
 public:
-	inline evttype_index_ruleset_factory(
-		std::shared_ptr<gen_event_filter_factory> factory
+	inline explicit evttype_index_ruleset_factory(
+		std::shared_ptr<sinsp_filter_factory> factory
 	): m_filter_factory(factory) { }
 
 	inline std::shared_ptr<filter_ruleset> new_ruleset() override
 	{
-		std::shared_ptr<filter_ruleset> ret(
-			new evttype_index_ruleset(m_filter_factory));
-		return ret;
+		return std::make_shared<evttype_index_ruleset>(m_filter_factory);
 	}
 
 private:
-	std::shared_ptr<gen_event_filter_factory> m_filter_factory;
+	std::shared_ptr<sinsp_filter_factory> m_filter_factory;
 };

userspace/engine/falco_common.cpp

@@ -33,7 +33,7 @@ static std::vector<std::string> rule_matching_names = {
 	"all"
 };
 
-bool falco_common::parse_priority(std::string v, priority_type& out)
+bool falco_common::parse_priority(const std::string& v, priority_type& out)
 {
 	for (size_t i = 0; i < priority_names.size(); i++)
 	{
@@ -50,7 +50,7 @@ bool falco_common::parse_priority(std::string v, priority_type& out)
 	return false;
 }
 
-falco_common::priority_type falco_common::parse_priority(std::string v)
+falco_common::priority_type falco_common::parse_priority(const std::string& v)
 {
 	falco_common::priority_type out;
 	if (!parse_priority(v, out))
@@ -87,7 +87,7 @@ std::string falco_common::format_priority(priority_type v, bool shortfmt)
 	return out;
 }
 
-bool falco_common::parse_rule_matching(std::string v, rule_matching& out)
+bool falco_common::parse_rule_matching(const std::string& v, rule_matching& out)
 {
 	for (size_t i = 0; i < rule_matching_names.size(); i++)
 	{

userspace/engine/falco_common.h

@@ -20,7 +20,7 @@ limitations under the License.
 #include <string>
 #include <exception>
 #include <mutex>
-#include <sinsp.h>
+#include <libsinsp/sinsp.h>
 
 //
 // equivalent to an "unbounded queue" in TBB terms or largest long value
@@ -28,33 +28,17 @@ limitations under the License.
 //
 #define DEFAULT_OUTPUTS_QUEUE_CAPACITY_UNBOUNDED_MAX_LONG_VALUE std::ptrdiff_t(~size_t(0) / 2)
 
+#define DEFAULT_FALCO_LIBS_THREAD_TABLE_SIZE 262144
+
 //
 // Most falco_* classes can throw exceptions. Unless directly related
 // to low-level failures like inability to open file, etc, they will
 // be of this type.
 //
 
-struct falco_exception : std::exception
+struct falco_exception : std::runtime_error
 {
-	falco_exception()
-	{
-	}
-
-	virtual ~falco_exception() throw()
-	{
-	}
-
-	falco_exception(std::string error_str)
-	{
-		m_error_str = error_str;
-	}
-
-	char const* what() const throw()
-	{
-		return m_error_str.c_str();
-	}
-
-	std::string m_error_str;
+	using std::runtime_error::runtime_error;
 };
 
 namespace falco_common
@@ -75,8 +59,8 @@ namespace falco_common
 		PRIORITY_DEBUG = 7
 	};
 	
-	bool parse_priority(std::string v, priority_type& out);
-	priority_type parse_priority(std::string v);
+	bool parse_priority(const std::string& v, priority_type& out);
+	priority_type parse_priority(const std::string& v);
 	bool format_priority(priority_type v, std::string& out, bool shortfmt=false);
 	std::string format_priority(priority_type v, bool shortfmt=false);
 
@@ -86,5 +70,5 @@ namespace falco_common
 		ALL = 1
 	};
 
-	bool parse_rule_matching(std::string v, rule_matching& out);
+	bool parse_rule_matching(const std::string& v, rule_matching& out);
 };

userspace/engine/falco_engine.cpp

@@ -27,23 +27,22 @@ limitations under the License.
 #include <string>
 #include <fstream>
 #include <functional>
+#include <memory>
 #include <utility>
 #include <vector>
 
 #include <nlohmann/json.hpp>
 
-#include <sinsp.h>
-#include <plugin.h>
+#include <libsinsp/sinsp.h>
+#include <libsinsp/plugin.h>
+#include <libsinsp/utils.h>
 
 #include "falco_engine.h"
 #include "falco_utils.h"
 #include "falco_engine_version.h"
-#include "rule_loader_reader.h"
-#include "rule_loader_compiler.h"
 
 #include "formats.h"
 
-#include "utils.h"
 #include "evttype_index_ruleset.h"
 
 const std::string falco_engine::s_default_ruleset = "falco-default-ruleset";
@@ -53,6 +52,9 @@ using namespace falco;
 falco_engine::falco_engine(bool seed_rng)
 	: m_syscall_source(NULL),
 	  m_syscall_source_idx(SIZE_MAX),
+	  m_rule_reader(std::make_shared<rule_loader::reader>()),
+	  m_rule_collector(std::make_shared<rule_loader::collector>()),
+	  m_rule_compiler(std::make_shared<rule_loader::compiler>()),
 	  m_next_ruleset_id(0),
 	  m_min_priority(falco_common::PRIORITY_DEBUG),
 	  m_sampling_ratio(1), m_sampling_multiplier(0),
@@ -64,12 +66,14 @@ falco_engine::falco_engine(bool seed_rng)
 	}
 
 	m_default_ruleset_id = find_ruleset_id(s_default_ruleset);
+
+	fill_engine_state_funcs(m_engine_state);
 }
 
 falco_engine::~falco_engine()
 {
 	m_rules.clear();
-	m_rule_collector.clear();
+	m_rule_collector->clear();
 	m_rule_stats_manager.clear();
 	m_sources.clear();
 }
@@ -79,14 +83,44 @@ sinsp_version falco_engine::engine_version()
 	return sinsp_version(FALCO_ENGINE_VERSION);
 }
 
+void falco_engine::set_rule_reader(std::shared_ptr<rule_loader::reader> reader)
+{
+	m_rule_reader = reader;
+}
+
+std::shared_ptr<rule_loader::reader> falco_engine::get_rule_reader()
+{
+	return m_rule_reader;
+}
+
+void falco_engine::set_rule_collector(std::shared_ptr<rule_loader::collector> collector)
+{
+	m_rule_collector = collector;
+}
+
+std::shared_ptr<rule_loader::collector> falco_engine::get_rule_collector()
+{
+	return m_rule_collector;
+}
+
+void falco_engine::set_rule_compiler(std::shared_ptr<rule_loader::compiler> compiler)
+{
+	m_rule_compiler = compiler;
+}
+
+std::shared_ptr<rule_loader::compiler> falco_engine::get_rule_compiler()
+{
+	return m_rule_compiler;
+}
+
 // Return a key that uniquely represents a field class.
 // For now, we assume name + shortdesc is unique.
-static std::string fieldclass_key(const gen_event_filter_factory::filter_fieldclass_info &fld_info)
+static std::string fieldclass_key(const sinsp_filter_factory::filter_fieldclass_info &fld_info)
 {
 	return fld_info.name + fld_info.shortdesc;
 }
 
-void falco_engine::list_fields(std::string &source, bool verbose, bool names_only, bool markdown) const
+void falco_engine::list_fields(const std::string &source, bool verbose, bool names_only, bool markdown) const
 {
 	// Maps from field class name + short desc to list of event
 	// sources for which this field class can be used.
@@ -164,47 +198,48 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 	cfg.replace_output_container_info = m_replace_container_info;
 
 	// read rules YAML file and collect its definitions
-	rule_loader::reader reader;
-	if (reader.read(cfg, m_rule_collector))
+	if(m_rule_reader->read(cfg, *(m_rule_collector.get())))
 	{
 		// compile the definitions (resolve macro/list refs, exceptions, ...)
-		m_last_compile_output = std::make_unique<rule_loader::compiler::compile_output>();
-		rule_loader::compiler().compile(cfg, m_rule_collector, *m_last_compile_output.get());
+		m_last_compile_output = m_rule_compiler->new_compile_output();
+		m_rule_compiler->compile(cfg, *(m_rule_collector.get()), *m_last_compile_output.get());
 
 		// clear the rules known by the engine and each ruleset
 		m_rules.clear();
 		for (auto &src : m_sources)
+		// add rules to each ruleset
 		{
-			src.ruleset = src.ruleset_factory->new_ruleset();
+			src.ruleset = create_ruleset(src.ruleset_factory);
+			src.ruleset->add_compile_output(*(m_last_compile_output.get()),
+							m_min_priority,
+							src.name);
 		}
 
 		// add rules to the engine and the rulesets
 		for (const auto& rule : m_last_compile_output->rules)
 		{
-			// skip the rule if below the minimum priority
-			if (rule.priority > m_min_priority)
-			{
-				continue;
-			}
-
-			auto info = m_rule_collector.rules().at(rule.name);
+			auto info = m_rule_collector->rules().at(rule.name);
 			if (!info)
 			{
 				// this is just defensive, it should never happen
 				throw falco_exception("can't find internal rule info at name: " + name);
 			}
 
-			// the rule is ok, we can add it to the engine and the rulesets
-			// note: the compiler should guarantee that the rule's condition
-			// is a valid sinsp filter
 			auto source = find_source(rule.source);
-			std::shared_ptr<gen_event_filter> filter(
-				sinsp_filter_compiler(source->filter_factory, rule.condition.get()).compile());
 			auto rule_id = m_rules.insert(rule, rule.name);
-			m_rules.at(rule_id)->id = rule_id;
-			source->ruleset->add(rule, filter, rule.condition);
+			if (rule_id != rule.id)
+			{
+				throw falco_exception("Incompatible ID for rule: " + rule.name +
+						      " | compiled ID: " + std::to_string(rule.id) +
+						      " | stats_mgr ID: " + std::to_string(rule_id));
+			}
 
 			// By default rules are enabled/disabled for the default ruleset
+			// skip the rule if below the minimum priority
+			if (rule.priority > m_min_priority)
+			{
+				continue;
+			}
 			if(info->enabled)
 			{
 				source->ruleset->enable(rule.name, true, m_default_ruleset_id);
@@ -325,7 +360,7 @@ uint64_t falco_engine::num_rules_for_ruleset(const std::string &ruleset)
 	return ret;
 }
 
-void falco_engine::evttypes_for_ruleset(std::string &source, std::set<uint16_t> &evttypes, const std::string &ruleset)
+void falco_engine::evttypes_for_ruleset(const std::string &source, std::set<uint16_t> &evttypes, const std::string &ruleset)
 {
 	find_source(source)->ruleset->enabled_evttypes(evttypes, find_ruleset_id(ruleset));
 }
@@ -340,14 +375,14 @@ libsinsp::events::set<ppm_event_code> falco_engine::event_codes_for_ruleset(cons
 	return find_source(source)->ruleset->enabled_event_codes(find_ruleset_id(ruleset));
 }
 
-std::shared_ptr<gen_event_formatter> falco_engine::create_formatter(const std::string &source,
+std::shared_ptr<sinsp_evt_formatter> falco_engine::create_formatter(const std::string &source,
 								    const std::string &output) const
 {
 	return find_source(source)->formatter_factory->create_formatter(output);
 }
 
 std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_event(std::size_t source_idx,
-	gen_event *ev, uint16_t ruleset_id, falco_common::rule_matching strategy)
+	sinsp_evt *ev, uint16_t ruleset_id, falco_common::rule_matching strategy)
 {
 	// note: there are no thread-safety guarantees on the filter_ruleset::run()
 	// method, but the thread-safety assumptions of falco_engine::process_event()
@@ -387,7 +422,7 @@ std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_ev
 	}
 
 	auto res = std::make_unique<std::vector<falco_engine::rule_result>>();
-	for(auto rule : source->m_rules)
+	for(const auto& rule : source->m_rules)
 	{
 		rule_result rule_result;
 		rule_result.evt = ev;
@@ -405,19 +440,18 @@ std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_ev
 }
 
 std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_event(std::size_t source_idx,
-	gen_event *ev, falco_common::rule_matching strategy)
+	sinsp_evt *ev, falco_common::rule_matching strategy)
 {
 	return process_event(source_idx, ev, m_default_ruleset_id, strategy);
 }
 
 std::size_t falco_engine::add_source(const std::string &source,
-				     std::shared_ptr<gen_event_filter_factory> filter_factory,
-				     std::shared_ptr<gen_event_formatter_factory> formatter_factory)
+				     std::shared_ptr<sinsp_filter_factory> filter_factory,
+				     std::shared_ptr<sinsp_evt_formatter_factory> formatter_factory)
 {
 	// evttype_index_ruleset is the default ruleset implementation
-	std::shared_ptr<filter_ruleset_factory> ruleset_factory(
-		new evttype_index_ruleset_factory(filter_factory));
-	size_t idx = add_source(source, filter_factory, formatter_factory, ruleset_factory);
+	size_t idx = add_source(source, filter_factory, formatter_factory,
+	                        std::make_shared<evttype_index_ruleset_factory>(filter_factory));
 
 	if(source == falco_common::syscall_source)
 	{
@@ -428,8 +462,8 @@ std::size_t falco_engine::add_source(const std::string &source,
 }
 
 std::size_t falco_engine::add_source(const std::string &source,
-	std::shared_ptr<gen_event_filter_factory> filter_factory,
-	std::shared_ptr<gen_event_formatter_factory> formatter_factory,
+	std::shared_ptr<sinsp_filter_factory> filter_factory,
+	std::shared_ptr<sinsp_evt_formatter_factory> formatter_factory,
 	std::shared_ptr<filter_ruleset_factory> ruleset_factory)
 {
 	falco_source src;
@@ -437,7 +471,7 @@ std::size_t falco_engine::add_source(const std::string &source,
 	src.filter_factory = filter_factory;
 	src.formatter_factory = formatter_factory;
 	src.ruleset_factory = ruleset_factory;
-	src.ruleset = ruleset_factory->new_ruleset();
+	src.ruleset = create_ruleset(src.ruleset_factory);
 	return m_sources.insert(src, source);
 }
 
@@ -451,7 +485,7 @@ template <typename T> inline nlohmann::json sequence_to_json_array(const T& seq)
 	return ret;
 }
 
-nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
+nlohmann::json falco_engine::describe_rule(std::string *rule_name, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
 	// use previously-loaded collector definitions and the compiled
 	// output of rules, macros, and lists.
@@ -462,15 +496,15 @@ nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector<
 
 	// use collected and compiled info to print a json output
 	nlohmann::json output;
-	if(!rule)
+	if(!rule_name)
 	{
 		// Store required engine version
-		auto required_engine_version = m_rule_collector.required_engine_version();
+		auto required_engine_version = m_rule_collector->required_engine_version();
 		output["required_engine_version"] = required_engine_version.version.as_string();
 
 		// Store required plugin versions
 		nlohmann::json plugin_versions = nlohmann::json::array();
-		auto required_plugin_versions = m_rule_collector.required_plugin_versions();
+		auto required_plugin_versions = m_rule_collector->required_plugin_versions();
 		for(const auto& req : required_plugin_versions)
 		{
 			nlohmann::json r;
@@ -493,51 +527,51 @@ nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector<
 
 		// Store information about rules
 		nlohmann::json rules_array = nlohmann::json::array();
-		for(const auto& r : m_last_compile_output->rules)
+		for(const auto& rule : m_last_compile_output->rules)
 		{
-			auto info = m_rule_collector.rules().at(r.name);
-			nlohmann::json rule;
-			get_json_details(rule, r, *info, plugins);
-			rules_array.push_back(std::move(rule));
+			auto info = m_rule_collector->rules().at(rule.name);
+			nlohmann::json details;
+			get_json_details(details, rule, *info, plugins);
+			rules_array.push_back(std::move(details));
 		}
 		output["rules"] = std::move(rules_array);
 
 		// Store information about macros
 		nlohmann::json macros_array = nlohmann::json::array();
-		for(const auto &m : m_last_compile_output->macros)
+		for(const auto &macro : m_last_compile_output->macros)
 		{
-			auto info = m_rule_collector.macros().at(m.name);
-			nlohmann::json macro;
-			get_json_details(macro, m, *info, plugins);
-			macros_array.push_back(std::move(macro));
+			auto info = m_rule_collector->macros().at(macro.name);
+			nlohmann::json details;
+			get_json_details(details, macro, *info, plugins);
+			macros_array.push_back(std::move(details));
 		}
 		output["macros"] = std::move(macros_array);
 
 		// Store information about lists
 		nlohmann::json lists_array = nlohmann::json::array();
-		for(const auto &l : m_last_compile_output->lists)
+		for(const auto &list : m_last_compile_output->lists)
 		{
-			auto info = m_rule_collector.lists().at(l.name);
-			nlohmann::json list;
-			get_json_details(list, l, *info, plugins);
-			lists_array.push_back(std::move(list));
+			auto info = m_rule_collector->lists().at(list.name);
+			nlohmann::json details;
+			get_json_details(details, list, *info, plugins);
+			lists_array.push_back(std::move(details));
 		}
 		output["lists"] = std::move(lists_array);
 	}
 	else
 	{
 		// build json information for just the specified rule
-		auto ri = m_rule_collector.rules().at(*rule);
+		auto ri = m_rule_collector->rules().at(*rule_name);
 		if(ri == nullptr || ri->unknown_source)
 		{
-			throw falco_exception("Rule \"" + *rule + "\" is not loaded");
+			throw falco_exception("Rule \"" + *rule_name + "\" is not loaded");
 		}
-		auto r = m_rules.at(ri->name);
+		auto rule = m_rules.at(ri->name);
 
-		nlohmann::json rule;
-		get_json_details(rule, *r, *ri, plugins);
+		nlohmann::json details;
+		get_json_details(details, *rule, *ri, plugins);
 		nlohmann::json rules_array = nlohmann::json::array();
-		rules_array.push_back(std::move(rule));
+		rules_array.push_back(std::move(details));
 		output["rules"] = std::move(rules_array);
 	}
 
@@ -571,12 +605,12 @@ void falco_engine::get_json_details(
 	filter_details details;
 	filter_details compiled_details;
 	nlohmann::json json_details;
-	for(const auto &m : m_rule_collector.macros())
+	for(const auto &m : m_rule_collector->macros())
 	{
 		details.known_macros.insert(m.name);
 		compiled_details.known_macros.insert(m.name);
 	}
-	for(const auto &l : m_rule_collector.lists())
+	for(const auto &l : m_rule_collector->lists())
 	{
 		details.known_lists.insert(l.name);
 		compiled_details.known_lists.insert(l.name);
@@ -654,13 +688,13 @@ void falco_engine::get_json_details(
 
 void falco_engine::get_json_details(
 	nlohmann::json& out,
-	const falco_macro& m,
+	const falco_macro& macro,
 	const rule_loader::macro_info& info,
 	const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
 	nlohmann::json macro_info;
 
-	macro_info["name"] = m.name;
+	macro_info["name"] = macro.name;
 	macro_info["condition"] = info.cond;
 	out["info"] = std::move(macro_info);
 
@@ -672,20 +706,20 @@ void falco_engine::get_json_details(
 	filter_details details;
 	filter_details compiled_details;
 	nlohmann::json json_details;
-	for(const auto &m : m_rule_collector.macros())
+	for(const auto &m : m_rule_collector->macros())
 	{
 		details.known_macros.insert(m.name);
 		compiled_details.known_macros.insert(m.name);
 	}
-	for(const auto &l : m_rule_collector.lists())
+	for(const auto &l : m_rule_collector->lists())
 	{
 		details.known_lists.insert(l.name);
 		compiled_details.known_lists.insert(l.name);
 	}
 	filter_details_resolver().run(ast.get(), details);
-	filter_details_resolver().run(m.condition.get(), compiled_details);
+	filter_details_resolver().run(macro.condition.get(), compiled_details);
 
-	out["details"]["used"] = m.used;
+	out["details"]["used"] = macro.used;
 	out["details"]["macros"] = sequence_to_json_array(details.macros);
 	out["details"]["lists"] = sequence_to_json_array(details.lists);
 	out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators);
@@ -693,11 +727,11 @@ void falco_engine::get_json_details(
 
 	// Store event types
 	nlohmann::json events;
-	get_json_evt_types(events, "", m.condition.get());
+	get_json_evt_types(events, "", macro.condition.get());
 	out["details"]["events"] = std::move(events);
 
 	// Store compiled condition
-	out["details"]["condition_compiled"] = libsinsp::filter::ast::as_string(m.condition.get());
+	out["details"]["condition_compiled"] = libsinsp::filter::ast::as_string(macro.condition.get());
 
 	// Compute the plugins that are actually used by this macro.
 	// Note: macros have no specific source, we need to set an empty list of used
@@ -860,22 +894,22 @@ bool falco_engine::is_source_valid(const std::string &source) const
 	return m_sources.at(source) != nullptr;
 }
 
-std::shared_ptr<gen_event_filter_factory> falco_engine::filter_factory_for_source(const std::string& source)
+std::shared_ptr<sinsp_filter_factory> falco_engine::filter_factory_for_source(const std::string& source)
 {
 	return find_source(source)->filter_factory;
 }
 
-std::shared_ptr<gen_event_filter_factory> falco_engine::filter_factory_for_source(std::size_t source_idx)
+std::shared_ptr<sinsp_filter_factory> falco_engine::filter_factory_for_source(std::size_t source_idx)
 {
 	return find_source(source_idx)->filter_factory;
 }
 
-std::shared_ptr<gen_event_formatter_factory> falco_engine::formatter_factory_for_source(const std::string& source)
+std::shared_ptr<sinsp_evt_formatter_factory> falco_engine::formatter_factory_for_source(const std::string& source)
 {
 	return find_source(source)->formatter_factory;
 }
 
-std::shared_ptr<gen_event_formatter_factory> falco_engine::formatter_factory_for_source(std::size_t source_idx)
+std::shared_ptr<sinsp_evt_formatter_factory> falco_engine::formatter_factory_for_source(std::size_t source_idx)
 {
 	return find_source(source_idx)->formatter_factory;
 }
@@ -958,7 +992,7 @@ bool falco_engine::check_plugin_requirements(
 		std::string& err) const
 {
 	err = "";
-	for (const auto &alternatives : m_rule_collector.required_plugin_versions())
+	for(const auto &alternatives : m_rule_collector->required_plugin_versions())
 	{
 		if (!check_plugin_requirement_alternatives(plugins, alternatives, err))
 		{
@@ -977,6 +1011,31 @@ bool falco_engine::check_plugin_requirements(
 	return true;
 }
 
+std::shared_ptr<filter_ruleset> falco_engine::create_ruleset(std::shared_ptr<filter_ruleset_factory> &ruleset_factory)
+{
+	auto ret = ruleset_factory->new_ruleset();
+
+	ret->set_engine_state(m_engine_state);
+
+	return ret;
+}
+
+void falco_engine::fill_engine_state_funcs(filter_ruleset::engine_state_funcs &engine_state)
+{
+	engine_state.get_ruleset = [this](const std::string &source_name, std::shared_ptr<filter_ruleset> &ruleset) -> bool
+	{
+		const falco_source *src = m_sources.at(source_name);
+		if(src == nullptr)
+		{
+			return false;
+		}
+
+		ruleset = src->ruleset;
+
+		return true;
+	};
+};
+
 void falco_engine::complete_rule_loading() const
 {
 	for (const auto &src : m_sources)
@@ -995,7 +1054,7 @@ void falco_engine::set_sampling_multiplier(double sampling_multiplier)
 	m_sampling_multiplier = sampling_multiplier;
 }
 
-void falco_engine::set_extra(std::string &extra, bool replace_container_info)
+void falco_engine::set_extra(const std::string &extra, bool replace_container_info)
 {
 	m_extra = extra;
 	m_replace_container_info = replace_container_info;

userspace/engine/falco_engine.h

@@ -15,12 +15,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Gen filtering TODO
-//  - DONE Clean up use/sharing of factories amongst engine-related classes.
-//  - DONE Fix outputs to actually use factories
-//  - Review gen_filter apis to see if they have only the required interfaces
-//  - Fix json filterchecks to split json and evt.time filterchecks.
-
 #pragma once
 
 #include <atomic>
@@ -30,17 +24,16 @@ limitations under the License.
 
 #include <nlohmann/json.hpp>
 
-#include "gen_filter.h"
 #include "filter_ruleset.h"
 #include "rule_loader.h"
+#include "rule_loader_reader.h"
 #include "rule_loader_collector.h"
+#include "rule_loader_compiler.h"
 #include "stats_manager.h"
 #include "falco_common.h"
 #include "falco_source.h"
 #include "falco_load_result.h"
 #include "filter_details_resolver.h"
-#include "rule_loader_reader.h"
-#include "rule_loader_compiler.h"
 
 //
 // This class acts as the primary interface between a program and the
@@ -51,7 +44,7 @@ limitations under the License.
 class falco_engine
 {
 public:
-	falco_engine(bool seed_rng=true);
+	explicit falco_engine(bool seed_rng=true);
 	virtual ~falco_engine();
 
 	// A given engine has a version which identifies the fields
@@ -71,7 +64,18 @@ public:
 
 	// Print to stdout (using printf) a description of each field supported by this engine.
 	// If source is non-empty, only fields for the provided source are printed.
-	void list_fields(std::string &source, bool verbose, bool names_only, bool markdown) const;
+	void list_fields(const std::string &source, bool verbose, bool names_only, bool markdown) const;
+
+	// Provide an alternate rule reader, collector, and compiler
+	// to compile any rules provided via load_rules*
+	void set_rule_reader(std::shared_ptr<rule_loader::reader> reader);
+	std::shared_ptr<rule_loader::reader> get_rule_reader();
+
+	void set_rule_collector(std::shared_ptr<rule_loader::collector> collector);
+	std::shared_ptr<rule_loader::collector> get_rule_collector();
+
+	void set_rule_compiler(std::shared_ptr<rule_loader::compiler> compiler);
+	std::shared_ptr<rule_loader::compiler> get_rule_compiler();
 
 	//
 	// Load rules and returns a result object.
@@ -137,7 +141,7 @@ public:
 	// Print details on the given rule. If rule is NULL, print
 	// details on all rules.
 	//
-	nlohmann::json describe_rule(std::string *rule, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
+	nlohmann::json describe_rule(std::string *rule_name, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
 
 	//
 	// Print statistics on how many events matched each rule.
@@ -164,12 +168,12 @@ public:
 	// add k8s/container information to outputs when
 	// available.
 	//
-	void set_extra(std::string &extra, bool replace_container_info);
+	void set_extra(const std::string &extra, bool replace_container_info);
 
 	// Represents the result of matching an event against a set of
 	// rules.
 	struct rule_result {
-		gen_event *evt;
+		sinsp_evt *evt;
 		std::string rule;
 		std::string source;
 		falco_common::priority_type priority_num;
@@ -203,7 +207,7 @@ public:
 	// concurrently with the same source_idx would inherently cause data races
 	// and lead to undefined behavior.
 	std::unique_ptr<std::vector<rule_result>> process_event(std::size_t source_idx,
-		gen_event *ev, uint16_t ruleset_id, falco_common::rule_matching strategy);
+		sinsp_evt *ev, uint16_t ruleset_id, falco_common::rule_matching strategy);
 
 	//
 	// Wrapper assuming the default ruleset.
@@ -211,7 +215,7 @@ public:
 	// This inherits the same thread-safety guarantees.
 	//
 	std::unique_ptr<std::vector<rule_result>> process_event(std::size_t source_idx,
-		gen_event *ev, falco_common::rule_matching strategy);
+		sinsp_evt *ev, falco_common::rule_matching strategy);
 
 	//
 	// Configure the engine to support events with the provided
@@ -219,16 +223,16 @@ public:
 	// Return source index for fast lookup.
 	//
 	std::size_t add_source(const std::string &source,
-			       std::shared_ptr<gen_event_filter_factory> filter_factory,
-			       std::shared_ptr<gen_event_formatter_factory> formatter_factory);
+			       std::shared_ptr<sinsp_filter_factory> filter_factory,
+			       std::shared_ptr<sinsp_evt_formatter_factory> formatter_factory);
 
 	//
 	// Equivalent to above, but allows specifying a ruleset factory
 	// for the newly added source.
 	//
 	std::size_t add_source(const std::string &source,
-			       std::shared_ptr<gen_event_filter_factory> filter_factory,
-			       std::shared_ptr<gen_event_formatter_factory> formatter_factory,
+			       std::shared_ptr<sinsp_filter_factory> filter_factory,
+			       std::shared_ptr<sinsp_evt_formatter_factory> formatter_factory,
 			       std::shared_ptr<filter_ruleset_factory> ruleset_factory);
 
 	// Return whether or not there is a valid filter/formatter
@@ -239,15 +243,15 @@ public:
 	// Given a source, return a formatter factory that can create
 	// filters for events of that source.
 	//
-	std::shared_ptr<gen_event_filter_factory> filter_factory_for_source(const std::string& source);
-	std::shared_ptr<gen_event_filter_factory> filter_factory_for_source(std::size_t source_idx);
+	std::shared_ptr<sinsp_filter_factory> filter_factory_for_source(const std::string& source);
+	std::shared_ptr<sinsp_filter_factory> filter_factory_for_source(std::size_t source_idx);
 
 	//
 	// Given a source, return a formatter factory that can create
 	// formatters for an event.
 	//
-	std::shared_ptr<gen_event_formatter_factory> formatter_factory_for_source(const std::string& source);
-	std::shared_ptr<gen_event_formatter_factory> formatter_factory_for_source(std::size_t source_idx);
+	std::shared_ptr<sinsp_evt_formatter_factory> formatter_factory_for_source(const std::string& source);
+	std::shared_ptr<sinsp_evt_formatter_factory> formatter_factory_for_source(std::size_t source_idx);
 
 	//
 	// Given a source, return a ruleset factory that can create
@@ -267,7 +271,7 @@ public:
 	// typing-improved `enabled_event_codes` and `enabled_sc_codes` instead
 	// todo(jasondellaluce): remove this in future code refactors
 	//
-	void evttypes_for_ruleset(std::string &source,
+	void evttypes_for_ruleset(const std::string &source,
 				  std::set<uint16_t> &evttypes,
 				  const std::string &ruleset = s_default_ruleset);
 
@@ -289,10 +293,10 @@ public:
 
 	//
 	// Given a source and output string, return an
-	// gen_event_formatter that can format output strings for an
+	// sinsp_evt_formatter that can format output strings for an
 	// event.
 	//
-	std::shared_ptr<gen_event_formatter> create_formatter(const std::string &source,
+	std::shared_ptr<sinsp_evt_formatter> create_formatter(const std::string &source,
 							      const std::string &output) const;
 
 	// The rule loader definition is aliased as it is exactly what we need
@@ -310,6 +314,14 @@ public:
 		std::string& err) const;
 
 private:
+	// Create a ruleset using the provided factory and set the
+	// engine state funcs for it.
+	std::shared_ptr<filter_ruleset> create_ruleset(std::shared_ptr<filter_ruleset_factory>& ruleset_factory);
+
+	// Functions to retrieve state from this engine
+	void fill_engine_state_funcs(filter_ruleset::engine_state_funcs& engine_state);
+
+	filter_ruleset::engine_state_funcs m_engine_state;
 
 	// Throws falco_exception if the file can not be read
 	void read_file(const std::string& filename, std::string& contents);
@@ -395,15 +407,17 @@ private:
 		const std::unordered_set<std::string>& fields,
 		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
 
-	rule_loader::collector m_rule_collector;
 	indexed_vector<falco_rule> m_rules;
+	std::shared_ptr<rule_loader::reader> m_rule_reader;
+	std::shared_ptr<rule_loader::collector> m_rule_collector;
+	std::shared_ptr<rule_loader::compiler> m_rule_compiler;
 	stats_manager m_rule_stats_manager;
 
 	uint16_t m_next_ruleset_id;
 	std::map<std::string, uint16_t> m_known_rulesets;
 	falco_common::priority_type m_min_priority;
 
-	std::unique_ptr<rule_loader::compiler::compile_output> m_last_compile_output;
+	std::unique_ptr<rule_loader::compile_output> m_last_compile_output;
 
 	//
 	// Here's how the sampling ratio and multiplier influence

userspace/engine/falco_engine_version.h

@@ -20,7 +20,7 @@ limitations under the License.
 
 // The version of this Falco engine
 #define FALCO_ENGINE_VERSION_MAJOR 0
-#define FALCO_ENGINE_VERSION_MINOR 31
+#define FALCO_ENGINE_VERSION_MINOR 39
 #define FALCO_ENGINE_VERSION_PATCH 0
 
 #define FALCO_ENGINE_VERSION \
@@ -34,4 +34,4 @@ limitations under the License.
 // It represents the fields supported by this version of Falco,
 // the event types, and the underlying driverevent schema. It's used to
 // detetect changes in engine version in our CI jobs.
-#define FALCO_ENGINE_CHECKSUM "7c512927c89f594f024f2ff181077c780c4fe6e9dd4cee3f20a9ef208a356e4e"
+#define FALCO_ENGINE_CHECKSUM "12cb01b5d2379547819a37a4656f1cef2952893ea1225de79614c673f386f645"

userspace/engine/falco_load_result.cpp

@@ -23,7 +23,8 @@ static const std::string error_codes[] = {
 	"LOAD_ERR_YAML_VALIDATE",
 	"LOAD_ERR_COMPILE_CONDITION",
 	"LOAD_ERR_COMPILE_OUTPUT",
-	"LOAD_ERR_VALIDATE"
+	"LOAD_ERR_VALIDATE",
+	"LOAD_ERR_EXTENSION"
 };
 
 const std::string& falco::load_result::error_code_str(error_code ec)
@@ -37,7 +38,8 @@ static const std::string error_strings[] = {
 	"Error validating internal structure of YAML file",
 	"Error compiling condition",
 	"Error compiling output",
-	"Error validating rule/macro/list/exception objects"
+	"Error validating rule/macro/list/exception objects",
+	"Error in extension item"
 };
 
 const std::string& falco::load_result::error_str(error_code ec)
@@ -51,7 +53,8 @@ static const std::string error_descs[] = {
 	"This occurs when the internal structure of the YAML file is incorrect. Examples include not consisting of a sequence of maps, a given rule/macro/list item not having required keys, values not having the right type (e.g. the items property of a list not being a sequence), etc.",
 	"This occurs when a condition string can not be compiled to a filter object.",
 	"This occurs when an output string can not be compiled to an output object.",
-	"This occurs when a rule/macro/list item is incorrect. Examples include a condition field referring to an undefined macro, falco engine/plugin version mismatches, items with append without any existing item, exception fields/comps having different lengths, etc."
+	"This occurs when a rule/macro/list item is incorrect. Examples include a condition field referring to an undefined macro, falco engine/plugin version mismatches, items with append without any existing item, exception fields/comps having different lengths, etc.",
+	"This occurs when there is an error in an extension item"
 };
 
 const std::string& falco::load_result::error_desc(error_code ec)
@@ -67,7 +70,12 @@ static const std::string warning_codes[] = {
 	"LOAD_UNUSED_MACRO",
 	"LOAD_UNUSED_LIST",
 	"LOAD_UNKNOWN_ITEM",
-	"LOAD_DEPRECATED_ITEM"
+	"LOAD_DEPRECATED_ITEM",
+	"LOAD_WARNING_EXTENSION",
+	"LOAD_APPEND_NO_VALUES",
+	"LOAD_EXCEPTION_NAME_NOT_UNIQUE",
+	"LOAD_INVALID_MACRO_NAME",
+	"LOAD_INVALID_LIST_NAME"
 };
 
 const std::string& falco::load_result::warning_code_str(warning_code wc)
@@ -83,7 +91,12 @@ static const std::string warning_strings[] = {
 	"Unused macro",
 	"Unused list",
 	"Unknown rules file item",
-	"Used deprecated item"
+	"Used deprecated item",
+	"Warning in extension item",
+	"Overriding/appending with no values",
+	"Multiple exceptions defined with the same name",
+	"Invalid macro name",
+	"Invalid list name"
 };
 
 const std::string& falco::load_result::warning_str(warning_code wc)
@@ -99,7 +112,12 @@ static const std::string warning_descs[] = {
 	"A macro is defined in the rules content but is not used by any other macro or rule.",
 	"A list is defined in the rules content but is not used by any other list, macro, or rule.",
 	"An unknown top-level object is in the rules content. It will be ignored.",
-	"A deprecated item is employed by lists, macros, or rules."
+	"A deprecated item is employed by lists, macros, or rules.",
+	"An extension item has a warning",
+	"A rule exception is overriding/appending with no values",
+	"A rule is defining multiple exceptions with the same name",
+	"A macro is defined with an invalid name",
+	"A list is defined with an invalid name"
 };
 
 const std::string& falco::load_result::warning_desc(warning_code wc)

userspace/engine/falco_load_result.h

@@ -34,7 +34,8 @@ public:
 		LOAD_ERR_YAML_VALIDATE,
 		LOAD_ERR_COMPILE_CONDITION,
 		LOAD_ERR_COMPILE_OUTPUT,
-		LOAD_ERR_VALIDATE
+		LOAD_ERR_VALIDATE,
+		LOAD_ERR_EXTENSION
 	};
 
 	// The error code as a string
@@ -55,7 +56,12 @@ public:
 		LOAD_UNUSED_MACRO,
 		LOAD_UNUSED_LIST,
 		LOAD_UNKNOWN_ITEM,
-		LOAD_DEPRECATED_ITEM
+		LOAD_DEPRECATED_ITEM,
+		LOAD_WARNING_EXTENSION,
+		LOAD_APPEND_NO_VALUES,
+		LOAD_EXCEPTION_NAME_NOT_UNIQUE,
+		LOAD_INVALID_MACRO_NAME,
+		LOAD_INVALID_LIST_NAME
 	};
 
 	virtual ~load_result() = default;

userspace/engine/falco_rule.h

@@ -21,7 +21,7 @@ limitations under the License.
 #include <string>
 #include "falco_common.h"
 
-#include <filter/ast.h>
+#include <libsinsp/filter/ast.h>
 
 /*!
 	\brief Represents a list in the Falco Engine.
@@ -83,4 +83,5 @@ struct falco_rule
 	std::set<std::string> exception_fields;
 	falco_common::priority_type priority;
 	std::shared_ptr<libsinsp::filter::ast::expr> condition;
+	std::shared_ptr<sinsp_filter> filter;
 };

userspace/engine/falco_source.h

@@ -49,8 +49,8 @@ struct falco_source
 	std::string name;
 	std::shared_ptr<filter_ruleset> ruleset;
 	std::shared_ptr<filter_ruleset_factory> ruleset_factory;
-	std::shared_ptr<gen_event_filter_factory> filter_factory;
-	std::shared_ptr<gen_event_formatter_factory> formatter_factory;
+	std::shared_ptr<sinsp_filter_factory> filter_factory;
+	std::shared_ptr<sinsp_evt_formatter_factory> formatter_factory;
 
 	// Used by the filter_ruleset interface. Filled in when a rule
 	// matches an event.
@@ -58,10 +58,8 @@ struct falco_source
 
 	inline bool is_field_defined(const std::string& field) const
 	{
-		auto *chk = filter_factory->new_filtercheck(field.c_str());
-		if (chk)
+		if (filter_factory->new_filtercheck(field.c_str()) != nullptr)
 		{
-			delete(chk);
 			return true;
 		}
 		return false;

userspace/engine/falco_utils.cpp

@@ -17,14 +17,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 */
-#include <cstring>
-#include <iomanip>
 
 #include "falco_utils.h"
-#include "utils.h"
+#include <libsinsp/utils.h>
 
 #include <re2/re2.h>
 
+#include <cstring>
+#include <fstream>
+#include <iomanip>
+#include <thread>
+
 #define RGX_PROMETHEUS_TIME_DURATION "^((?P<y>[0-9]+)y)?((?P<w>[0-9]+)w)?((?P<d>[0-9]+)d)?((?P<h>[0-9]+)h)?((?P<m>[0-9]+)m)?((?P<s>[0-9]+)s)?((?P<ms>[0-9]+)ms)?$"
 
 // using pre-compiled regex
@@ -145,7 +148,7 @@ uint32_t hardware_concurrency()
 
 void readfile(const std::string& filename, std::string& data)
 {
-	std::ifstream file(filename.c_str(), std::ios::in);
+	std::ifstream file(filename, std::ios::in);
 
 	if(file.is_open())
 	{

userspace/engine/falco_utils.h

@@ -20,30 +20,11 @@ limitations under the License.
 
 #pragma once
 
-#include <sstream>
-#include <fstream>
-#include <iostream>
-#include <string>
-#include <thread>
-#include <unordered_set>
-#include <set>
-#include <vector>
+#include <cstdint>
 #include <string>
 
-#ifdef __GNUC__
-#define likely(x) __builtin_expect(!!(x), 1)
-#define unlikely(x) __builtin_expect(!!(x), 0)
-#else
-#define likely(x) (x)
-#define unlikely(x) (x)
-#endif
-
-namespace falco
+namespace falco::utils
 {
-
-namespace utils
-{
-
 uint64_t parse_prometheus_interval(std::string interval_str);
 
 std::string wrap_text(const std::string& in, uint32_t indent, uint32_t linelen);
@@ -57,5 +38,4 @@ namespace network
 static const std::string UNIX_SCHEME("unix://");
 bool is_unix_scheme(const std::string& url);
 } // namespace network
-} // namespace utils
-} // namespace falco
+} // namespace falco::utils

userspace/engine/filter_details_resolver.h

@@ -17,7 +17,7 @@ limitations under the License.
 
 #pragma once
 
-#include <filter/parser.h>
+#include <libsinsp/filter/parser.h>
 #include <string>
 #include <unordered_set>
 #include <unordered_map>
@@ -57,7 +57,7 @@ public:
 private:
 	struct visitor : public libsinsp::filter::ast::expr_visitor
 	{
-		visitor(filter_details& details) :
+		explicit visitor(filter_details& details) :
 			m_details(details),
 			m_expect_list(false),
 			m_expect_macro(false),

userspace/engine/filter_macro_resolver.cpp

@@ -20,23 +20,6 @@ limitations under the License.
 
 using namespace libsinsp::filter;
 
-bool filter_macro_resolver::run(libsinsp::filter::ast::expr*& filter)
-{
-	m_unknown_macros.clear();
-	m_resolved_macros.clear();
-	m_errors.clear();
-
-	visitor v(m_errors, m_unknown_macros, m_resolved_macros, m_macros);
-	v.m_node_substitute = nullptr;
-	filter->accept(&v);
-	if (v.m_node_substitute)
-	{
-		delete filter;
-		filter = v.m_node_substitute.release();
-	}
-	return !m_resolved_macros.empty();
-}
-
 bool filter_macro_resolver::run(std::shared_ptr<libsinsp::filter::ast::expr>& filter)
 {
 	m_unknown_macros.clear();
@@ -54,8 +37,8 @@ bool filter_macro_resolver::run(std::shared_ptr<libsinsp::filter::ast::expr>& fi
 }
 
 void filter_macro_resolver::set_macro(
-		std::string name,
-		std::shared_ptr<libsinsp::filter::ast::expr> macro)
+		const std::string& name,
+		const std::shared_ptr<libsinsp::filter::ast::expr>& macro)
 {
 	m_macros[name] = macro;
 }

userspace/engine/filter_macro_resolver.h

@@ -17,7 +17,7 @@ limitations under the License.
 
 #pragma once
 
-#include <filter/parser.h>
+#include <libsinsp/filter/parser.h>
 #include <string>
 #include <unordered_set>
 #include <unordered_map>
@@ -40,11 +40,6 @@ class filter_macro_resolver
 			class and is deleted automatically.
 			\return true if at least one of the defined macros is resolved
 		*/
-		bool run(libsinsp::filter::ast::expr*& filter);
-
-		/*!
-			\brief Version of run() that works with shared pointers
-		*/
 		bool run(std::shared_ptr<libsinsp::filter::ast::expr>& filter);
 
 		/*!
@@ -56,8 +51,8 @@ class filter_macro_resolver
 			\param macro The AST of the macro.
 		*/
 		void set_macro(
-			std::string name,
-			std::shared_ptr<libsinsp::filter::ast::expr> macro);
+			const std::string& name,
+			const std::shared_ptr<libsinsp::filter::ast::expr>& macro);
 
 		/*!
 		    \brief used in get_{resolved,unknown}_macros and get_errors
@@ -86,6 +81,18 @@ class filter_macro_resolver
 		*/
 		const std::vector<value_info>& get_errors() const;
 
+		/*!
+			\brief Clears the resolver by resetting all state related to
+			known macros and everything related to the previous resolution run.
+		*/
+		inline void clear()
+		{
+			m_errors.clear();
+			m_unknown_macros.clear();
+			m_resolved_macros.clear();
+			m_macros.clear();
+		}
+
 	private:
 		typedef std::unordered_map<
 			std::string,

userspace/engine/filter_ruleset.cpp

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "filter_ruleset.h"
+
+void filter_ruleset::set_engine_state(const filter_ruleset::engine_state_funcs& engine_state)
+{
+	m_engine_state = engine_state;
+}
+
+filter_ruleset::engine_state_funcs& filter_ruleset::get_engine_state()
+{
+	return m_engine_state;
+}

userspace/engine/filter_ruleset.h

@@ -18,21 +18,34 @@ limitations under the License.
 #pragma once
 
 #include "falco_rule.h"
-#include <filter/ast.h>
-#include <filter.h>
-#include <event.h>
-#include <gen_filter.h>
-#include <events/sinsp_events.h>
+#include "rule_loader_compile_output.h"
+#include <libsinsp/filter/ast.h>
+#include <libsinsp/filter.h>
+#include <libsinsp/event.h>
+#include <libsinsp/events/sinsp_events.h>
 
 /*!
 	\brief Manages a set of rulesets. A ruleset is a set of
 	enabled rules that is able to process events and find matches for those rules.
 */
+
 class filter_ruleset
 {
 public:
+	// A set of functions that can be used to retrieve state from
+	// the falco engine that created this ruleset.
+	struct engine_state_funcs
+	{
+		using ruleset_retriever_func_t = std::function<bool(const std::string &, std::shared_ptr<filter_ruleset> &ruleset)>;
+
+		ruleset_retriever_func_t get_ruleset;
+	};
+
 	virtual ~filter_ruleset() = default;
 
+	void set_engine_state(const engine_state_funcs &engine_state);
+	engine_state_funcs &get_engine_state();
+
 	/*!
 		\brief Adds a rule and its filtering filter + condition inside the manager.
 	        This method only adds the rule inside the internal collection,
@@ -47,9 +60,38 @@ public:
 	*/
 	virtual void add(
 		const falco_rule& rule,
-		std::shared_ptr<gen_event_filter> filter,
+		std::shared_ptr<sinsp_filter> filter,
 		std::shared_ptr<libsinsp::filter::ast::expr> condition) = 0;
 
+	/*!
+		\brief Adds all rules contained in the provided
+		rule_loader::compile_output struct. Only
+		those rules with the provided source and those rules
+		with priority >= min_priority should be added. The
+		intent is that this replaces add(). However, we retain
+		add() for backwards compatibility. Any rules added via
+		add() are also added to this ruleset. The default
+		implementation iterates over rules and calls add(),
+		but can be overridden.
+		\param rule The compile output.
+		\param min_priority Only add rules with priority above this priority.
+		\param source Only add rules with source equal to this source.
+	*/
+	virtual void add_compile_output(
+		const rule_loader::compile_output& compile_output,
+		falco_common::priority_type min_priority,
+		const std::string& source)
+	{
+		for (const auto& rule : compile_output.rules)
+		{
+			if(rule.priority <= min_priority &&
+			   rule.source == source)
+			{
+				add(rule, rule.filter, rule.condition);
+			}
+		}
+	};
+
 	/*!
 		\brief Erases the internal state. All rules are disabled in each
 		ruleset, and all the rules defined with add() are removed.
@@ -71,7 +113,7 @@ public:
 		\param ruleset_id The id of the ruleset to be used
 	*/
 	virtual bool run(
-		gen_event *evt,
+		sinsp_evt *evt,
 		falco_rule& match,
 		uint16_t ruleset_id) = 0;
 	
@@ -84,7 +126,7 @@ public:
 		\param ruleset_id The id of the ruleset to be used
 	*/
 	virtual bool run(
-		gen_event *evt,
+		sinsp_evt *evt,
 		std::vector<falco_rule>& matches,
 		uint16_t ruleset_id) = 0;
 
@@ -181,6 +223,9 @@ public:
 	virtual void disable_tags(
 		const std::set<std::string> &tags,
 		uint16_t ruleset_id) = 0;
+
+private:
+	engine_state_funcs m_engine_state;
 };
 
 /*!

userspace/engine/filter_warning_resolver.cpp

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include <sinsp.h>
+#include <libsinsp/sinsp.h>
 #include "filter_warning_resolver.h"
 
 using namespace falco;

userspace/engine/filter_warning_resolver.h

@@ -17,7 +17,7 @@ limitations under the License.
 
 #pragma once
 
-#include <filter/parser.h>
+#include <libsinsp/filter/parser.h>
 #include <string>
 #include <set>
 #include <memory>

userspace/engine/formats.cpp

@@ -33,20 +33,20 @@ falco_formats::~falco_formats()
 {
 }
 
-std::string falco_formats::format_event(gen_event *evt, const std::string &rule, const std::string &source,
-				   const std::string &level, const std::string &format, std::set<std::string> &tags,
+std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
+				   const std::string &level, const std::string &format, const std::set<std::string> &tags,
 				   const std::string &hostname) const
 {
 	std::string line;
 
-	std::shared_ptr<gen_event_formatter> formatter;
+	std::shared_ptr<sinsp_evt_formatter> formatter;
 
 	formatter = m_falco_engine->create_formatter(source, format);
 
 	// Format the original output string, regardless of output format
-	formatter->tostring_withformat(evt, line, gen_event_formatter::OF_NORMAL);
+	formatter->tostring_withformat(evt, line, sinsp_evt_formatter::OF_NORMAL);
 
-	if(formatter->get_output_format() == gen_event_formatter::OF_JSON)
+	if(formatter->get_output_format() == sinsp_evt_formatter::OF_JSON)
 	{
 		std::string json_line;
 
@@ -101,7 +101,7 @@ std::string falco_formats::format_event(gen_event *evt, const std::string &rule,
 			}
 			else
 			{
-				for (auto &tag : tags)
+				for (const auto &tag : tags)
 				{
 					rule_tags[rule_tags_idx++] = tag;
 				}
@@ -128,13 +128,13 @@ std::string falco_formats::format_event(gen_event *evt, const std::string &rule,
 		line = full_line;
 	}
 
-	return line.c_str();
+	return line;
 }
 
-std::map<std::string, std::string> falco_formats::get_field_values(gen_event *evt, const std::string &source,
+std::map<std::string, std::string> falco_formats::get_field_values(sinsp_evt *evt, const std::string &source,
 						    const std::string &format) const
 {
-	std::shared_ptr<gen_event_formatter> formatter;
+	std::shared_ptr<sinsp_evt_formatter> formatter;
 
 	formatter = m_falco_engine->create_formatter(source, format);
 

userspace/engine/formats.h

@@ -19,7 +19,6 @@ limitations under the License.
 
 #include <string>
 #include <map>
-#include <gen_filter.h>
 #include "falco_engine.h"
 
 class falco_formats
@@ -30,11 +29,11 @@ public:
 		      bool json_include_tags_property);
 	virtual ~falco_formats();
 
-	std::string format_event(gen_event *evt, const std::string &rule, const std::string &source,
-				 const std::string &level, const std::string &format, std::set<std::string> &tags,
+	std::string format_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
+				 const std::string &level, const std::string &format, const std::set<std::string> &tags,
 				 const std::string &hostname) const;
 
-	std::map<std::string, std::string> get_field_values(gen_event *evt, const std::string &source,
+	std::map<std::string, std::string> get_field_values(sinsp_evt *evt, const std::string &source,
 					     const std::string &format) const ;
 
 protected:

userspace/engine/rule_loader.cpp

@@ -42,7 +42,8 @@ static const std::string item_type_strings[] = {
 	"rule output",
 	"rule output expression",
 	"rule priority",
-	"overrides"
+	"overrides",
+	"extension item"
 };
 
 const std::string& rule_loader::context::item_type_as_string(enum item_type it)
@@ -80,12 +81,12 @@ rule_loader::context::context(const libsinsp::filter::ast::pos_info& pos,
 	// Contexts based on conditions don't use the
 	// filename. Instead the "name" is just the condition, and
 	// uses a short prefix of the condition.
-	std::string name = "\"" + (
+	std::string condition_name = "\"" + (
 		condition.length() > 20
 		? condition.substr(0, 20 - 3) + "...\""
 		: condition + "\"");
-	std::replace(name.begin(), name.end(), '\n', ' ');
-	std::replace(name.begin(), name.end(), '\r', ' ');
+	std::replace(condition_name.begin(), condition_name.end(), '\n', ' ');
+	std::replace(condition_name.begin(), condition_name.end(), '\r', ' ');
 
 	std::string item_name = "";
 
@@ -99,7 +100,7 @@ rule_loader::context::context(const libsinsp::filter::ast::pos_info& pos,
 	condpos.line = pos.line + lastpos.pos.line;
 	condpos.column = pos.col + lastpos.pos.column;
 
-	init(name, condpos, rule_loader::context::CONDITION_EXPRESSION, item_name, parent);
+	init(condition_name, condpos, rule_loader::context::CONDITION_EXPRESSION, item_name, parent);
 }
 
 const std::string& rule_loader::context::name() const
@@ -139,7 +140,7 @@ std::string rule_loader::context::as_string()
 
 	bool first = true;
 
-	for(auto& loc : m_locs)
+	for(const auto& loc : m_locs)
 	{
 		os << (first ? "In " : "    ");
 		first = false;
@@ -173,7 +174,7 @@ nlohmann::json rule_loader::context::as_json()
 		throw falco_exception("rule_loader::context without location?");
 	}
 
-	for(auto& loc : m_locs)
+	for(const auto& loc : m_locs)
 	{
 		nlohmann::json jloc, jpos;
 

userspace/engine/rule_loader.h

@@ -25,7 +25,7 @@ limitations under the License.
 #include "falco_source.h"
 #include "falco_load_result.h"
 #include "indexed_vector.h"
-#include "version.h"
+#include <libsinsp/version.h>
 
 namespace rule_loader
 {
@@ -58,7 +58,8 @@ namespace rule_loader
 			RULE_OUTPUT,
 			RULE_OUTPUT_EXPRESSION,
 			RULE_PRIORITY,
-			OVERRIDE
+			OVERRIDE,
+			EXTENSION_ITEM
 		};
 
 		static const std::string& item_type_as_string(enum item_type it);
@@ -225,7 +226,7 @@ namespace rule_loader
 	class result : public falco::load_result
 	{
 	public:
-		result(const std::string &name);
+		explicit result(const std::string &name);
 		virtual ~result() = default;
 		result(result&&) = default;
 		result& operator = (result&&) = default;
@@ -292,7 +293,7 @@ namespace rule_loader
 	struct engine_version_info
 	{
 		engine_version_info() : ctx("no-filename-given"), version("0.0.0") { };
-		engine_version_info(context &ctx);
+		explicit engine_version_info(context &ctx);
 		~engine_version_info() = default;
 		engine_version_info(engine_version_info&&) = default;
 		engine_version_info& operator = (engine_version_info&&) = default;
@@ -328,7 +329,7 @@ namespace rule_loader
 		// a default constructor. This allows it to be used
 		// by falco_engine, which aliases the type.
 		plugin_version_info();
-		plugin_version_info(context &ctx);
+		explicit plugin_version_info(context &ctx);
 		~plugin_version_info() = default;
 		plugin_version_info(plugin_version_info&&) = default;
 		plugin_version_info& operator = (plugin_version_info&&) = default;
@@ -344,7 +345,7 @@ namespace rule_loader
 	*/
 	struct list_info
 	{
-		list_info(context &ctx);
+		explicit list_info(context &ctx);
 		~list_info() = default;
 		list_info(list_info&&) = default;
 		list_info& operator = (list_info&&) = default;
@@ -363,7 +364,7 @@ namespace rule_loader
 	*/
 	struct macro_info
 	{
-		macro_info(context &ctx);
+		explicit macro_info(context &ctx);
 		~macro_info() = default;
 		macro_info(macro_info&&) = default;
 		macro_info& operator = (macro_info&&) = default;
@@ -383,7 +384,7 @@ namespace rule_loader
 	*/
 	struct rule_exception_info
 	{
-		rule_exception_info(context &ctx);
+		explicit rule_exception_info(context &ctx);
 		~rule_exception_info() = default;
 		rule_exception_info(rule_exception_info&&) = default;
 		rule_exception_info& operator = (rule_exception_info&&) = default;
@@ -428,7 +429,7 @@ namespace rule_loader
 	*/
 	struct rule_info
 	{
-		rule_info(context &ctx);
+		explicit rule_info(context &ctx);
 		~rule_info() = default;
 		rule_info(rule_info&&) = default;
 		rule_info& operator = (rule_info&&) = default;
@@ -460,7 +461,7 @@ namespace rule_loader
 
 	struct rule_update_info
 	{
-		rule_update_info(context &ctx);
+		explicit rule_update_info(context &ctx);
 		~rule_update_info() = default;
 		rule_update_info(rule_update_info&&) = default;
 		rule_update_info& operator = (rule_update_info&&) = default;

userspace/engine/rule_loader_collector.cpp

@@ -16,7 +16,7 @@ limitations under the License.
 */
 
 #include <string>
-#include <version.h>
+#include <libsinsp/version.h>
 
 #include "falco_engine.h"
 #include "rule_loader_collector.h"
@@ -78,7 +78,7 @@ static void validate_exception_info(
 		THROW(ex.fields.items.size() != ex.comps.items.size(),
 		       "Fields and comps lists must have equal length",
 		       ex.ctx);
-		for (auto &v : ex.comps.items)
+		for (const auto &v : ex.comps.items)
 		{
 			THROW(!is_operator_defined(v.item),
 			      std::string("'") + v.item + "' is not a supported comparison operator",
@@ -86,7 +86,7 @@ static void validate_exception_info(
 		}
 		if (source)
 		{
-			for (auto &v : ex.fields.items)
+			for (const auto &v : ex.fields.items)
 			{
 				THROW(!source->is_field_defined(v.item),
 					std::string("'") + v.item + "' is not a supported filter field",
@@ -212,12 +212,12 @@ void rule_loader::collector::append(configuration& cfg, macro_info& info)
 
 void rule_loader::collector::define(configuration& cfg, rule_info& info)
 {
-	auto prev = m_rule_infos.at(info.name);
+	const auto* prev = m_rule_infos.at(info.name);
 	THROW(prev && prev->source != info.source,
 		"Rule has been re-defined with a different source",
 		info.ctx);
 
-	auto source = cfg.sources.at(info.source);
+	const auto* source = cfg.sources.at(info.source);
 	if (!source)
 	{
 		info.unknown_source = true;
@@ -248,7 +248,7 @@ void rule_loader::collector::append(configuration& cfg, rule_update_info& info)
 
 	// note: source can be nullptr in case we've collected a
 	// rule for which the source is unknown
-	falco_source* source = nullptr;
+	const falco_source* source = nullptr;
 	if (!prev->unknown_source)
 	{
 		// note: if the source is not unknown, this should not return nullptr
@@ -330,7 +330,7 @@ void rule_loader::collector::selective_replace(configuration& cfg, rule_update_i
 
 	// note: source can be nullptr in case we've collected a
 	// rule for which the source is unknown
-	falco_source* source = nullptr;
+	const falco_source* source = nullptr;
 	if (!prev->unknown_source)
 	{
 		// note: if the source is not unknown, this should not return nullptr

userspace/engine/rule_loader_compile_output.h

@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#pragma once
+
+#include "indexed_vector.h"
+#include "falco_rule.h"
+
+namespace rule_loader
+{
+	struct compile_output
+	{
+		compile_output() = default;
+		virtual ~compile_output() = default;
+		compile_output(compile_output&&) = default;
+		compile_output& operator = (compile_output&&) = default;
+		compile_output(const compile_output&) = default;
+		compile_output& operator = (const compile_output&) = default;
+
+		indexed_vector<falco_list> lists;
+		indexed_vector<falco_macro> macros;
+		indexed_vector<falco_rule> rules;
+	};
+};
+

userspace/engine/rule_loader_compiler.cpp

@@ -21,7 +21,6 @@ limitations under the License.
 #include <vector>
 
 #include "rule_loader_compiler.h"
-#include "filter_macro_resolver.h"
 #include "filter_warning_resolver.h"
 
 #define MAX_VISIBILITY		((uint32_t) -1)
@@ -60,7 +59,7 @@ static bool is_format_valid(const falco_source& source, std::string fmt, std::st
 {
 	try
 	{
-		std::shared_ptr<gen_event_formatter> formatter;
+		std::shared_ptr<sinsp_evt_formatter> formatter;
 		formatter = source.formatter_factory->create_formatter(fmt);
 		return true;
 	}
@@ -77,6 +76,7 @@ static void build_rule_exception_infos(
 	std::string& condition)
 {
 	std::string tmp;
+	condition = "(" + condition + ")";
 	for (const auto &ex : exceptions)
 	{
 		std::string icond;
@@ -181,6 +181,7 @@ static bool resolve_list(std::string& cnd, const falco_list& list)
 {
 	static std::string blanks = " \t\n\r";
 	static std::string delims = blanks + "(),=";
+	std::string tmp;
 	std::string new_cnd;
 	size_t start, end;
 	bool used = false;
@@ -212,7 +213,9 @@ static bool resolve_list(std::string& cnd, const falco_list& list)
 				{
 					sub += ", ";
 				}
-				sub += v;
+				tmp = v;
+				quote_item(tmp);
+				sub += tmp;
 			}
 			// if substituted list is empty, we need to
 			// remove a comma from the left or the right
@@ -247,7 +250,8 @@ static bool resolve_list(std::string& cnd, const falco_list& list)
 	return used;
 }
 
-static void resolve_macros(
+static inline void resolve_macros(
+	filter_macro_resolver& macro_resolver,
 	const indexed_vector<rule_loader::macro_info>& infos,
 	indexed_vector<falco_macro>& macros,
 	std::shared_ptr<ast::expr>& ast,
@@ -255,7 +259,7 @@ static void resolve_macros(
 	uint32_t visibility,
 	const rule_loader::context &ctx)
 {
-	filter_macro_resolver macro_resolver;
+	macro_resolver.clear();
 	for (const auto &m : infos)
 	{
 		if (m.index < visibility)
@@ -339,39 +343,35 @@ void rule_loader::compiler::compile_list_infos(
 		const collector& col,
 		indexed_vector<falco_list>& out) const
 {
-	std::string tmp;
-	std::list<std::string> used;
-	falco_list v;
+	std::list<std::string> used_names;
+	falco_list infos;
 	for (const auto &list : col.lists())
 	{
-		v.name = list.name;
-		v.items.clear();
+		infos.name = list.name;
+		infos.items.clear();
 		for (const auto &item : list.items)
 		{
 			const auto ref = col.lists().at(item);
 			if (ref && ref->index < list.visibility)
 			{
-				used.push_back(ref->name);
-				for (auto val : ref->items)
+				used_names.push_back(ref->name);
+				for (const auto &val : ref->items)
 				{
-					quote_item(val);
-					v.items.push_back(val);
+					infos.items.push_back(val);
 				}
 			}
 			else
 			{
-				tmp = item;
-				quote_item(tmp);
-				v.items.push_back(tmp);
+				infos.items.push_back(item);
 			}
 		}
-		v.used = false;
-		auto list_id = out.insert(v, v.name);
+		infos.used = false;
+		auto list_id = out.insert(infos, infos.name);
 		out.at(list_id)->id = list_id;
 	}
-	for (const auto &v : used)
+	for (const auto &name : used_names)
 	{
-		out.at(v)->used = true;
+		out.at(name)->used = true;
 	}
 }
 
@@ -392,10 +392,11 @@ void rule_loader::compiler::compile_macros_infos(
 		out.at(macro_id)->id = macro_id;
 	}
 
+	filter_macro_resolver macro_resolver;
 	for (auto &m : out)
 	{
-		auto info = macro_info_from_name(col, m.name);
-		resolve_macros(col.macros(), out, m.condition, info->cond, info->visibility, info->ctx);
+		const auto* info = macro_info_from_name(col, m.name);
+		resolve_macros(macro_resolver, col.macros(), out, m.condition, info->cond, info->visibility, info->ctx);
 	}
 }
 
@@ -406,17 +407,74 @@ static bool err_is_unknown_type_or_field(const std::string& err)
 		|| err.find("unknown event type") != std::string::npos;
 }
 
-void rule_loader::compiler::compile_rule_infos(
-		configuration& cfg,
-		const collector& col,
-		indexed_vector<falco_list>& lists,
-		indexed_vector<falco_macro>& macros,
-		indexed_vector<falco_rule>& out) const
+bool rule_loader::compiler::compile_condition(
+	configuration& cfg,
+	filter_macro_resolver& macro_resolver,
+	indexed_vector<falco_list>& lists,
+	const indexed_vector<rule_loader::macro_info>& macros,
+	const std::string& condition,
+	std::shared_ptr<sinsp_filter_factory> filter_factory,
+	const rule_loader::context& cond_ctx,
+	const rule_loader::context& parent_ctx,
+	bool allow_unknown_fields,
+	indexed_vector<falco_macro>& macros_out,
+	std::shared_ptr<libsinsp::filter::ast::expr>& ast_out,
+	std::shared_ptr<sinsp_filter>& filter_out) const
 {
-	std::string err, condition;
 	std::set<falco::load_result::load_result::warning_code> warn_codes;
 	filter_warning_resolver warn_resolver;
-	for (const auto &r : col.rules())
+	ast_out = parse_condition(condition, lists, cond_ctx);
+	resolve_macros(macro_resolver, macros, macros_out, ast_out, condition, MAX_VISIBILITY, parent_ctx);
+
+	// check for warnings in the filtering condition
+	if(warn_resolver.run(ast_out.get(), warn_codes))
+	{
+		for(const auto& w : warn_codes)
+		{
+			cfg.res->add_warning(w, "", parent_ctx);
+		}
+	}
+
+	// validate the rule's condition: we compile it into a sinsp filter
+	// on-the-fly and we throw an exception with details on failure
+	sinsp_filter_compiler compiler(filter_factory, ast_out.get());
+	try
+	{
+		filter_out = compiler.compile();
+	}
+	catch(const sinsp_exception& e)
+	{
+		// skip the rule silently if skip_if_unknown_filter is true and
+		// we encountered some specific kind of errors
+		std::string err = e.what();
+		if(err_is_unknown_type_or_field(err) && allow_unknown_fields)
+		{
+			cfg.res->add_warning(
+				falco::load_result::load_result::LOAD_UNKNOWN_FILTER,
+				err,
+				cond_ctx);
+			return false;
+		}
+		rule_loader::context ctx(compiler.get_pos(), condition, cond_ctx);
+		throw rule_loader::rule_load_exception(
+			falco::load_result::load_result::LOAD_ERR_COMPILE_CONDITION,
+			err,
+			ctx);
+	}
+
+	return true;
+}
+
+void rule_loader::compiler::compile_rule_infos(
+	configuration& cfg,
+	const collector& col,
+	indexed_vector<falco_list>& lists,
+	indexed_vector<falco_macro>& macros,
+	indexed_vector<falco_rule>& out) const
+{
+	std::string err, condition;
+	filter_macro_resolver macro_resolver;
+	for(const auto& r : col.rules())
 	{
 		// skip the rule if it has an unknown source
 		if (r.unknown_source)
@@ -440,25 +498,10 @@ void rule_loader::compiler::compile_rule_infos(
 			build_rule_exception_infos(
 				r.exceptions, rule.exception_fields, condition);
 		}
-		rule.condition = parse_condition(condition, lists, r.cond_ctx);
-		resolve_macros(col.macros(), macros, rule.condition, condition, MAX_VISIBILITY, r.ctx);
-
-		// check for warnings in the filtering condition
-		warn_codes.clear();
-		if (warn_resolver.run(rule.condition.get(), warn_codes))
-		{
-			for (const auto &w : warn_codes)
-			{
-				cfg.res->add_warning(w, "", r.ctx);
-			}
-		}
 
 		// build rule output message
 		rule.output = r.output;
-		if (r.source == falco_common::syscall_source)
-		{
-			apply_output_substitutions(cfg, rule.output);
-		}
+		apply_output_substitutions(cfg, rule.output);
 
 		// validate the rule's output
 		if(!is_format_valid(*cfg.sources.at(r.source), rule.output, err))
@@ -479,31 +522,20 @@ void rule_loader::compiler::compile_rule_infos(
 				r.output_ctx);
 		}
 
-		// validate the rule's condition: we compile it into a sinsp filter
-		// on-the-fly and we throw an exception with details on failure
-		sinsp_filter_compiler compiler(cfg.sources.at(r.source)->filter_factory, rule.condition.get());
-		try
+		if (!compile_condition(cfg,
+				  macro_resolver,
+				  lists,
+				  col.macros(),
+				  condition,
+				  cfg.sources.at(r.source)->filter_factory,
+				  r.cond_ctx,
+				  r.ctx,
+				  r.skip_if_unknown_filter,
+				  macros,
+				  rule.condition,
+				  rule.filter))
 		{
-			std::shared_ptr<sinsp_filter> sfPtr(compiler.compile());
-		}
-		catch (const sinsp_exception& e)
-		{
-			// skip the rule silently if skip_if_unknown_filter is true and
-			// we encountered some specific kind of errors
-			std::string err = e.what();
-			if (err_is_unknown_type_or_field(err) && r.skip_if_unknown_filter)
-			{
-				cfg.res->add_warning(
-					falco::load_result::load_result::LOAD_UNKNOWN_FILTER,
-					err,
-					r.cond_ctx);
-				continue;
-			}
-			rule_loader::context ctx(compiler.get_pos(), condition, r.cond_ctx);
-			throw rule_loader::rule_load_exception(
-				falco::load_result::load_result::LOAD_ERR_COMPILE_CONDITION,
-				err,
-				ctx);
+			continue;
 		}
 
 		// populate set of event types and emit an special warning
@@ -530,6 +562,11 @@ void rule_loader::compiler::compile_rule_infos(
 	}
 }
 
+std::unique_ptr<rule_loader::compile_output> rule_loader::compiler::new_compile_output()
+{
+	return std::make_unique<compile_output>();
+}
+
 void rule_loader::compiler::compile(
 		configuration& cfg,
 		const collector& col,

userspace/engine/rule_loader_compiler.h

@@ -18,7 +18,9 @@ limitations under the License.
 #pragma once
 
 #include "rule_loader.h"
+#include "rule_loader_compile_output.h"
 #include "rule_loader_collector.h"
+#include "filter_macro_resolver.h"
 #include "indexed_vector.h"
 #include "falco_rule.h"
 
@@ -31,23 +33,6 @@ namespace rule_loader
 class compiler
 {
 public:
-	/*!
-		\brief The output of a compilation.
-	*/
-	struct compile_output
-	{
-		compile_output() = default;
-		virtual ~compile_output() = default;
-		compile_output(compile_output&&) = default;
-		compile_output& operator = (compile_output&&) = default;
-		compile_output(const compile_output&) = default;
-		compile_output& operator = (const compile_output&) = default;
-
-		indexed_vector<falco_list> lists;
-		indexed_vector<falco_macro> macros;
-		indexed_vector<falco_rule> rules;
-	};
-
 	compiler() = default;
 	virtual ~compiler() = default;
 	compiler(compiler&&) = default;
@@ -55,6 +40,10 @@ public:
 	compiler(const compiler&) = default;
 	compiler& operator = (const compiler&) = default;
 
+	// Return a new result object, suitable for passing to
+	// compile().
+        virtual std::unique_ptr<compile_output> new_compile_output();
+
 	/*!
 		\brief Compiles a list of falco rules
 	*/
@@ -62,6 +51,28 @@ public:
 		configuration& cfg,
 		const collector& col,
 		compile_output& out) const;
+protected:
+	 /*!
+                \brief Compile a single condition expression,
+                including expanding macro and list references.
+
+		returns true if the condition could be compiled, and sets
+		ast_out/filter_out with the compiled filter + ast. Returns false if
+		the condition could not be compiled and should be skipped.
+        */
+	bool compile_condition(
+		configuration& cfg,
+		filter_macro_resolver& macro_resolver,
+		indexed_vector<falco_list>& lists,
+		const indexed_vector<rule_loader::macro_info>& macros,
+		const std::string& condition,
+		std::shared_ptr<sinsp_filter_factory> filter_factory,
+		const rule_loader::context& cond_ctx,
+		const rule_loader::context& parent_ctx,
+		bool allow_unknown_fields,
+		indexed_vector<falco_macro>& macros_out,
+		std::shared_ptr<libsinsp::filter::ast::expr>& ast_out,
+		std::shared_ptr<sinsp_filter>& filter_out) const;
 
 private:
 	void compile_list_infos(

userspace/engine/rule_loader_reader.cpp

@@ -23,9 +23,22 @@ limitations under the License.
 #include "rule_loader_reader.h"
 #include "falco_engine_version.h"
 #include "rule_loading_messages.h"
+#include <libsinsp/logger.h>
+
+#include <re2/re2.h>
 
 #define THROW(cond, err, ctx)    { if ((cond)) { throw rule_loader::rule_load_exception(falco::load_result::LOAD_ERR_YAML_VALIDATE, (err), (ctx)); } }
 
+// Sinsp Filter grammar tokens taken from "libsinsp/filter/parser.h"
+// These regular expressions are used here to check for invalid macro/list names
+// todo(mrgian): to avoid code duplication we can move regex definitions in libsinsp/filter/parser.h
+// and include it here instead of redefining them.
+#define RGX_IDENTIFIER "([a-zA-Z]+[a-zA-Z0-9_]*)"
+#define RGX_BARESTR    "([^()\"'[:space:]=,]+)"
+
+static re2::RE2 s_rgx_identifier(RGX_IDENTIFIER, re2::RE2::POSIX);
+static re2::RE2 s_rgx_barestr(RGX_BARESTR, re2::RE2::POSIX);
+
 // Don't call this directly, call decode_val/decode_optional_val instead.
 template <typename T>
 static void decode_val_generic(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx, bool optional)
@@ -56,21 +69,27 @@ static void decode_val_generic(const YAML::Node& item, const char *key, std::opt
 }
 
 template <typename T>
-static void decode_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx)
+void rule_loader::reader::decode_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx)
 {
 	bool optional = false;
 
 	decode_val_generic(item, key, out, ctx, optional);
 }
 
+template void rule_loader::reader::decode_val<std::string>(const YAML::Node& item, const char *key, std::string& out, const rule_loader::context& ctx);
+
 template <typename T>
-static void decode_optional_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx)
+void rule_loader::reader::decode_optional_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx)
 {
 	bool optional = true;
 
 	decode_val_generic(item, key, out, ctx, optional);
 }
 
+template void rule_loader::reader::decode_optional_val<std::string>(const YAML::Node& item, const char *key, std::string& out, const rule_loader::context& ctx);
+
+template void rule_loader::reader::decode_optional_val<bool>(const YAML::Node& item, const char *key, bool& out, const rule_loader::context& ctx);
+
 // Don't call this directly, call decode_items/decode_tags instead.
 template <typename T>
 static void decode_seq(const YAML::Node& item, const char *key,
@@ -263,6 +282,7 @@ static void decode_exception_values(
 }
 
 static void read_rule_exceptions(
+	rule_loader::configuration& cfg,
 	const YAML::Node& item,
 	std::vector<rule_loader::rule_exception_info>& exceptions,
 	const rule_loader::context& parent,
@@ -289,13 +309,22 @@ static void read_rule_exceptions(
 		rule_loader::context tmp(ex, rule_loader::context::EXCEPTION, "", exes_ctx);
 
 		THROW(!ex.IsMap(), "Rule exception must be a mapping", tmp);
-		decode_val(ex, "name", name, tmp);
+		rule_loader::reader::decode_val(ex, "name", name, tmp);
 
 		// Now use a real context including the exception name.
 		rule_loader::context ex_ctx(ex, rule_loader::context::EXCEPTION, name, parent);
 		rule_loader::rule_exception_info v_ex(ex_ctx);
 		v_ex.name = name;
 
+		// Check if an exception with the same name has already been defined
+		for (auto &exception : exceptions)
+		{
+			if(v_ex.name == exception.name)
+			{
+				cfg.res->add_warning(falco::load_result::LOAD_EXCEPTION_NAME_NOT_UNIQUE, "Multiple definitions of exception '" + v_ex.name + "' in the same rule", ex_ctx);
+			}
+		}
+
 		// note: the legacy lua loader used to throw a "xxx must strings" error
 
 		// fields are optional when append is true
@@ -307,7 +336,7 @@ static void read_rule_exceptions(
 			rule_loader::context vals_ctx(exvals, rule_loader::context::EXCEPTION_VALUES, "", ex_ctx);
 			THROW(!exvals.IsSequence(),
 			       "Rule exception values must be a sequence", vals_ctx);
-			for (auto &val : exvals)
+			for (const auto &val : exvals)
 			{
 				rule_loader::context vctx(val, rule_loader::context::EXCEPTION_VALUE, "", vals_ctx);
 				rule_loader::rule_exception_info::entry v_ex_val;
@@ -315,19 +344,24 @@ static void read_rule_exceptions(
 				decode_exception_values(val, v_ex_val, vctx);
 				v_ex.values.push_back(v_ex_val);
 			}
+		} 
+		else if (append)
+		{
+			cfg.res->add_warning(falco::load_result::LOAD_APPEND_NO_VALUES, "Overriding/appending exception with no values", ex_ctx);
 		}
 		exceptions.push_back(v_ex);
 	}
 }
 
 static void read_rule_exceptions(
+	rule_loader::configuration& cfg,
 	const YAML::Node& item,
 	std::optional<std::vector<rule_loader::rule_exception_info>>& exceptions,
 	const rule_loader::context& parent,
 	bool append)
 {
 	std::vector<rule_loader::rule_exception_info> decoded;
-	read_rule_exceptions(item, decoded, parent, append);
+	read_rule_exceptions(cfg, item, decoded, parent, append);
 	exceptions = decoded;
 }
 
@@ -346,15 +380,17 @@ inline static bool check_update_expected(std::set<std::string>& expected_keys, c
 	return true;
 }
 
-static void read_item(
+void rule_loader::reader::read_item(
 	rule_loader::configuration& cfg,
 	rule_loader::collector& collector,
 	const YAML::Node& item,
 	const rule_loader::context& parent)
 {
-	rule_loader::context tmp(item, rule_loader::context::RULES_CONTENT_ITEM, "", parent);
-	THROW(!item.IsMap(), "Unexpected element type. "
-	      "Each element should be a yaml associative array.", tmp);
+	{
+		rule_loader::context tmp(item, rule_loader::context::RULES_CONTENT_ITEM, "", parent);
+		THROW(!item.IsMap(), "Unexpected element type. "
+		      "Each element should be a yaml associative array.", tmp);
+	}
 
 	if (item["required_engine_version"].IsDefined())
 	{
@@ -433,6 +469,13 @@ static void read_item(
 		decode_val(item, "list", name, tmp);
 
 		rule_loader::context ctx(item, rule_loader::context::LIST, name, parent);
+
+		bool invalid_name = !re2::RE2::FullMatch(name, s_rgx_barestr);
+		if(invalid_name)
+		{
+			cfg.res->add_warning(falco::load_result::LOAD_INVALID_LIST_NAME, "List has an invalid name. List names should match a regular expression: " RGX_BARESTR, ctx);
+		}
+
 		rule_loader::list_info v(ctx);
 
 		bool append = false;
@@ -473,6 +516,13 @@ static void read_item(
 		decode_val(item, "macro", name, tmp);
 
 		rule_loader::context ctx(item, rule_loader::context::MACRO, name, parent);
+
+		bool invalid_name = !re2::RE2::FullMatch(name, s_rgx_identifier);
+		if(invalid_name)
+		{
+			cfg.res->add_warning(falco::load_result::LOAD_INVALID_MACRO_NAME, "Macro has an invalid name. Macro names should match a regular expression: " RGX_IDENTIFIER, ctx);
+		}
+
 		rule_loader::macro_info v(ctx);
 		v.name = name;
 
@@ -568,7 +618,7 @@ static void read_item(
 
 				if (check_update_expected(expected_keys, override_append, "append", "exceptions", ctx))
 				{
-					read_rule_exceptions(item, v.exceptions, ctx, true);
+					read_rule_exceptions(cfg, item, v.exceptions, ctx, true);
 				}
 
 				if (check_update_expected(expected_keys, override_append, "append", "output", ctx))
@@ -600,7 +650,7 @@ static void read_item(
 
 				if (check_update_expected(expected_keys, override_replace, "replace", "exceptions", ctx))
 				{
-					read_rule_exceptions(item, v.exceptions, ctx, true);
+					read_rule_exceptions(cfg, item, v.exceptions, ctx, true);
 				}
 
 				if (check_update_expected(expected_keys, override_replace, "replace", "output", ctx))
@@ -647,7 +697,7 @@ static void read_item(
 			}
 
 			// if any expected key has not been defined throw an error
-			for (auto &key : expected_keys) {
+			for (const auto &key : expected_keys) {
 				rule_loader::context keyctx(item[key], rule_loader::context::OVERRIDE, key, ctx);
 				THROW(true, "Unexpected key '" + key + "': no corresponding entry under 'override' is defined.", keyctx);
 			}
@@ -665,7 +715,7 @@ static void read_item(
 
 			if(item["exceptions"].IsDefined())
 			{
-				read_rule_exceptions(item, v.exceptions, ctx, true);
+				read_rule_exceptions(cfg, item, v.exceptions, ctx, true);
 			}
 
 			// TODO restore this error and update testing
@@ -721,7 +771,7 @@ static void read_item(
 				decode_optional_val(item, "warn_evttypes", v.warn_evttypes, ctx);
 				decode_optional_val(item, "skip-if-unknown-filter", v.skip_if_unknown_filter, ctx);
 				decode_tags(item, v.tags, ctx);
-				read_rule_exceptions(item, v.exceptions, ctx, has_append_flag);
+				read_rule_exceptions(cfg, item, v.exceptions, ctx, has_append_flag);
 				collector.define(cfg, v);
 			}
 		}

userspace/engine/rule_loader_reader.h

@@ -19,8 +19,8 @@ limitations under the License.
 
 #include "rule_loader.h"
 #include "rule_loader_collector.h"
-#include "logger.h"
-#include "version.h"
+#include <libsinsp/logger.h>
+#include <libsinsp/version.h>
 #include "falco_engine_version.h"
 
 namespace rule_loader
@@ -57,6 +57,19 @@ public:
 			+ std::to_string(minor) + "." 
 			+ std::to_string(FALCO_ENGINE_VERSION_PATCH));
 	}
+
+	template <typename T>
+	static void decode_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx);
+
+	template <typename T>
+	static void decode_optional_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx);
+
+protected:
+
+	virtual void read_item(rule_loader::configuration& cfg,
+			       rule_loader::collector& collector,
+			       const YAML::Node& item,
+			       const rule_loader::context& parent);
 };
 
 }; // namespace rule_loader

userspace/falco/CMakeLists.txt

@@ -14,8 +14,7 @@
 
 configure_file(config_falco.h.in config_falco.h)
 
-set(
-  FALCO_SOURCES
+add_library(falco_application STATIC
   app/app.cpp
   app/options.cpp
   app/restart_handler.cpp
@@ -49,6 +48,7 @@ set(
   app/actions/start_webserver.cpp
   app/actions/validate_rules_files.cpp
   app/actions/create_requested_paths.cpp
+  app/actions/close_inspectors.cpp
   configuration.cpp
   logger.cpp
   falco_outputs.cpp
@@ -62,16 +62,12 @@ set(
 set(
   FALCO_INCLUDE_DIRECTORIES
   "${PROJECT_SOURCE_DIR}/userspace/engine"
-  "${PROJECT_BINARY_DIR}/userspace/falco"
+  "${CMAKE_CURRENT_SOURCE_DIR}"
+  "${CMAKE_CURRENT_BINARY_DIR}"
   "${PROJECT_BINARY_DIR}/driver/src"
   "${CXXOPTS_INCLUDE_DIR}"
-  "${YAMLCPP_INCLUDE_DIR}"
-  "${CMAKE_CURRENT_BINARY_DIR}"
-  "${CMAKE_CURRENT_SOURCE_DIR}"
 )
 
-list(APPEND FALCO_INCLUDE_DIRECTORIES "${FALCO_EXTRA_INCLUDE_DIRS}")
-
 set(
   FALCO_DEPENDENCIES
   cxxopts
@@ -81,33 +77,27 @@ set(
   FALCO_LIBRARIES
   falco_engine
   sinsp
-  "${YAMLCPP_LIB}"
+  yaml-cpp
 )
 
-if(USE_BUNDLED_DEPS)
-  list(APPEND FALCO_DEPENDENCIES yamlcpp)
-endif()
-
 if(NOT WIN32)
-  list(
-    APPEND FALCO_SOURCES
-	outputs_program.cpp
-	outputs_syslog.cpp
-)
+  target_sources(falco_application
+  PRIVATE
+    outputs_program.cpp
+    outputs_syslog.cpp
+  )
 endif()
 
 if(CMAKE_SYSTEM_NAME MATCHES "Linux" AND NOT MINIMAL_BUILD)
-  list(
-    APPEND FALCO_SOURCES
+  target_sources(falco_application
+  PRIVATE
     outputs_grpc.cpp
     outputs_http.cpp
     webserver.cpp
     grpc_context.cpp
-    grpc_server_impl.cpp
     grpc_request_context.cpp
     grpc_server.cpp
     grpc_context.cpp
-    grpc_server_impl.cpp
     ${CMAKE_CURRENT_BINARY_DIR}/version.grpc.pb.cc
     ${CMAKE_CURRENT_BINARY_DIR}/version.pb.cc
     ${CMAKE_CURRENT_BINARY_DIR}/outputs.grpc.pb.cc
@@ -117,7 +107,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux" AND NOT MINIMAL_BUILD)
 
   list(
     APPEND FALCO_INCLUDE_DIRECTORIES
-    "${CPPHTTPLIB_INCLUDE}"
     "${OPENSSL_INCLUDE_DIR}"
     "${GRPC_INCLUDE}"
     "${GRPCPP_INCLUDE}"
@@ -130,29 +119,24 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux" AND NOT MINIMAL_BUILD)
     list(APPEND FALCO_LIBRARIES "${GRPC_LIBRARIES}")
   endif()
 
-  list(APPEND FALCO_DEPENDENCIES cpp-httplib)
-
   list(
     APPEND FALCO_LIBRARIES
+    httplib::httplib
     "${GRPCPP_LIB}"
     "${GRPC_LIB}"
     "${GPR_LIB}"
     "${PROTOBUF_LIB}"
     "${CARES_LIB}"
     "${OPENSSL_LIBRARIES}"
-    "${YAMLCPP_LIB}"
   )
 endif()
 
-add_library(
-  falco_application STATIC
-  ${FALCO_SOURCES}
-)
-
 if (EMSCRIPTEN)
 	target_compile_options(falco_application PRIVATE "-sDISABLE_EXCEPTION_CATCHING=0")
 endif()
 
+target_compile_definitions(falco_application PRIVATE CPPHTTPLIB_OPENSSL_SUPPORT)
+
 add_dependencies(falco_application ${FALCO_DEPENDENCIES})
 
 target_link_libraries(

userspace/falco/app/actions/actions.h

@@ -26,29 +26,29 @@ namespace actions {
 
 falco::app::run_result configure_interesting_sets(falco::app::state& s);
 falco::app::run_result configure_syscall_buffer_size(falco::app::state& s);
-falco::app::run_result configure_syscall_buffer_num(falco::app::state& s);
+falco::app::run_result configure_syscall_buffer_num(const falco::app::state& s);
 falco::app::run_result create_requested_paths(falco::app::state& s);
 falco::app::run_result create_signal_handlers(falco::app::state& s);
-falco::app::run_result pidfile(falco::app::state& s);
+falco::app::run_result pidfile(const falco::app::state& s);
 falco::app::run_result init_falco_engine(falco::app::state& s);
 falco::app::run_result init_inspectors(falco::app::state& s);
 falco::app::run_result init_outputs(falco::app::state& s);
 falco::app::run_result list_fields(falco::app::state& s);
-falco::app::run_result list_plugins(falco::app::state& s);
-falco::app::run_result load_config(falco::app::state& s);
+falco::app::run_result list_plugins(const falco::app::state& s);
+falco::app::run_result load_config(const falco::app::state& s);
 falco::app::run_result load_plugins(falco::app::state& s);
 falco::app::run_result load_rules_files(falco::app::state& s);
 falco::app::run_result print_generated_gvisor_config(falco::app::state& s);
 falco::app::run_result print_help(falco::app::state& s);
-falco::app::run_result print_ignored_events(falco::app::state& s);
-falco::app::run_result print_kernel_version(falco::app::state& s);
-falco::app::run_result print_page_size(falco::app::state& s);
-falco::app::run_result print_plugin_info(falco::app::state& s);
+falco::app::run_result print_ignored_events(const falco::app::state& s);
+falco::app::run_result print_kernel_version(const falco::app::state& s);
+falco::app::run_result print_page_size(const falco::app::state& s);
+falco::app::run_result print_plugin_info(const falco::app::state& s);
 falco::app::run_result print_support(falco::app::state& s);
 falco::app::run_result print_syscall_events(falco::app::state& s);
 falco::app::run_result print_version(falco::app::state& s);
 falco::app::run_result process_events(falco::app::state& s);
-falco::app::run_result require_config_file(falco::app::state& s);
+falco::app::run_result require_config_file(const falco::app::state& s);
 falco::app::run_result select_event_sources(falco::app::state& s);
 falco::app::run_result start_grpc_server(falco::app::state& s);
 falco::app::run_result start_webserver(falco::app::state& s);
@@ -56,6 +56,7 @@ falco::app::run_result stop_grpc_server(falco::app::state& s);
 falco::app::run_result stop_webserver(falco::app::state& s);
 falco::app::run_result unregister_signal_handlers(falco::app::state& s);
 falco::app::run_result validate_rules_files(falco::app::state& s);
+falco::app::run_result close_inspectors(falco::app::state& s);
 
 }; // namespace actions
 }; // namespace app

userspace/falco/app/actions/close_inspectors.cpp

@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "actions.h"
+#include "helpers.h"
+
+using namespace falco::app;
+using namespace falco::app::actions;
+
+falco::app::run_result falco::app::actions::close_inspectors(falco::app::state& s)
+{
+	falco_logger::log(falco_logger::level::DEBUG, "closing inspectors");
+
+	if (s.offline_inspector != nullptr)
+	{
+		s.offline_inspector->close();
+	}
+
+	for (const auto &src : s.loaded_sources)
+	{
+		auto src_info = s.source_infos.at(src);
+
+		if (src_info->inspector != nullptr)
+		{
+			src_info->inspector->close();
+		}
+	}
+
+	return run_result::ok();
+}