new(docs): add changelog for 0.37.0

Signed-off-by: Andrea Terzolo <andreaterzolo3@gmail.com>

.github/workflows/ci.yml

@@ -23,6 +23,13 @@ jobs:
       arch: x86_64
       version: ${{ needs.fetch-version.outputs.version }}
 
+  build-dev-packages-arm64:
+    needs: [fetch-version]
+    uses: ./.github/workflows/reusable_build_packages.yaml
+    with:
+      arch: aarch64
+      version: ${{ needs.fetch-version.outputs.version }}
+
   test-dev-packages:
     needs: [fetch-version, build-dev-packages]
     uses: ./.github/workflows/reusable_test_packages.yaml
@@ -35,6 +42,16 @@ jobs:
       static: ${{ matrix.static != '' && true || false }}
       version: ${{ needs.fetch-version.outputs.version }}
 
+  test-dev-packages-arm64:
+    needs: [fetch-version, build-dev-packages]
+    uses: ./.github/workflows/reusable_test_packages.yaml
+    strategy:
+      fail-fast: false
+    with:
+      arch: aarch64
+      static: ${{ matrix.static != '' && true || false }}
+      version: ${{ needs.fetch-version.outputs.version }}
+
   build-dev-minimal:
     uses: ./.github/workflows/reusable_build_dev.yaml
     with:
@@ -42,7 +59,15 @@ jobs:
       git_ref: ${{ github.event.pull_request.head.sha }}
       minimal: true
       build_type: Debug
-  
+
+  build-dev-minimal-arm64:
+    uses: ./.github/workflows/reusable_build_dev.yaml
+    with:
+      arch: aarch64
+      git_ref: ${{ github.event.pull_request.head.sha }}
+      minimal: true
+      build_type: Debug
+
   # builds using system deps, checking out the PR's code
   # note: this also runs a command that generates an output of form: "<engine_version> <some_hash>",
   # of which <some_hash> is computed by hashing in order the following:

.github/workflows/codeql.yaml

@@ -48,7 +48,7 @@ jobs:
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
-        
+
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         # queries: security-extended,security-and-quality
 
@@ -56,7 +56,7 @@ jobs:
       run: sudo apt update -y
 
     - name: Install build dependencies
-      run: sudo DEBIAN_FRONTEND=noninteractive apt install libssl-dev libyaml-dev libc-ares-dev libprotobuf-dev protobuf-compiler libjq-dev libyaml-cpp-dev libgrpc++-dev protobuf-compiler-grpc rpm libelf-dev cmake build-essential libcurl4-openssl-dev linux-headers-$(uname -r) clang llvm git -y
+      run: sudo DEBIAN_FRONTEND=noninteractive apt install libssl-dev libc-ares-dev libprotobuf-dev protobuf-compiler libyaml-cpp-dev libgrpc++-dev protobuf-compiler-grpc rpm libelf-dev cmake build-essential libcurl4-openssl-dev linux-headers-$(uname -r) clang llvm git -y
 
     - name: Prepare project
       run: |

.github/workflows/release.yaml

@@ -147,7 +147,7 @@ jobs:
           sed -i s/FALCOVER/${{ github.event.release.tag_name }}/g release-body.md
           
       - name: Generate release notes
-        uses: leodido/rn2md@0669e5f3b21492c11c2db43cd6e267566f5880f3
+        uses: leodido/rn2md@1378404a058ecf86701f3ab533d487333fc675a7
         with:
           milestone: ${{ github.event.release.tag_name }}
           output: ./notes.md

.github/workflows/reusable_build_dev.yaml

@@ -27,14 +27,13 @@ on:
         required: false
         default: ''
         type: string
-       
+
 jobs:
   build-and-test:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
-    runs-on: ${{ (inputs.arch == 'aarch64' && fromJSON('[ "self-hosted", "linux", "ARM64" ]')) || 'ubuntu-22.04' }}
-    container: ${{ (inputs.arch == 'aarch64' && 'ubuntu:22.04') || '' }}
+    runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-22.04' }}
     outputs:
-      cmdout: ${{ steps.run_cmd.outputs.out }}  
+      cmdout: ${{ steps.run_cmd.outputs.out }}
     steps:
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
@@ -44,14 +43,14 @@ jobs:
 
       - name: Update base image
         run: sudo apt update -y
-      
+
       - name: Install build dependencies
-        run: sudo DEBIAN_FRONTEND=noninteractive apt install libjq-dev libelf-dev libyaml-cpp-dev cmake build-essential git -y
+        run: sudo DEBIAN_FRONTEND=noninteractive apt install libelf-dev libyaml-cpp-dev cmake build-essential git -y
 
       - name: Install build dependencies (non-minimal)
         if: inputs.minimal != true
-        run: sudo DEBIAN_FRONTEND=noninteractive apt install libssl-dev libyaml-dev libc-ares-dev libprotobuf-dev protobuf-compiler libgrpc++-dev protobuf-compiler-grpc rpm libcurl4-openssl-dev linux-headers-$(uname -r) clang llvm -y
-      
+        run: sudo DEBIAN_FRONTEND=noninteractive apt install libssl-dev libc-ares-dev libprotobuf-dev protobuf-compiler libgrpc++-dev protobuf-compiler-grpc rpm libcurl4-openssl-dev linux-headers-$(uname -r) clang llvm -y
+
       - name: Prepare project
         run: |
           mkdir build
@@ -74,7 +73,7 @@ jobs:
       - name: Run unit tests
         run: |
           pushd build
-          sudo ./unit_tests/falco_unit_tests 
+          sudo ./unit_tests/falco_unit_tests
           popd
 
       - name: Run command

.github/workflows/reusable_build_docker.yaml

@@ -27,7 +27,7 @@ on:
 jobs:
   build-docker:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
-    runs-on: ${{ (inputs.arch == 'aarch64' && fromJSON('[ "self-hosted", "linux", "ARM64" ]')) || 'ubuntu-latest' }}
+    runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
     env:
       TARGETARCH: ${{ (inputs.arch == 'aarch64' && 'arm64') || 'amd64' }}
     steps:

.github/workflows/reusable_build_packages.yaml

@@ -14,7 +14,7 @@ on:
 jobs:
   build-modern-bpf-skeleton:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
-    runs-on: ${{ (inputs.arch == 'aarch64' && fromJSON('[ "self-hosted", "linux", "ARM64" ]')) || 'ubuntu-latest' }}
+    runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
     container: fedora:latest
     steps:
       # Always install deps before invoking checkout action, to properly perform a full clone.
@@ -40,7 +40,7 @@ jobs:
           
   build-packages:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
-    runs-on: ${{ (inputs.arch == 'aarch64' && fromJSON('[ "self-hosted", "linux", "ARM64" ]')) || 'ubuntu-latest' }}
+    runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
     needs: [build-modern-bpf-skeleton]
     container: centos:7
     steps:

.github/workflows/reusable_test_packages.yaml

@@ -19,7 +19,7 @@ on:
 jobs:
   test-packages:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
-    runs-on: ${{ (inputs.arch == 'aarch64' && fromJSON('[ "self-hosted", "linux", "ARM64" ]')) || 'ubuntu-latest' }}
+    runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
     steps:
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
@@ -43,9 +43,17 @@ jobs:
           tar -xvf $(ls falco-*.tar.gz)
           cd falco-${{ inputs.version }}-${{ inputs.arch }}
           sudo cp -r * /
-
-      # x86_64 job run on ubuntu-22.04 and here we can install kernel-headers
-      - name: Install dependencies for falco-driver-loader tests on x86
+      
+      # Note: most probably the plugin related tests should be moved to the plugin repo sooner or later.
+      - name: Install needed artifacts using falcoctl
+        if: ${{ inputs.static == false }}
+        run: |
+          sudo mkdir -p /usr/share/falco/plugins
+          sudo falcoctl artifact install k8saudit-rules
+          sudo falcoctl artifact install cloudtrail-rules
+     
+      # We only run driver loader tests on x86_64
+      - name: Install dependencies for falco-driver-loader tests
         if: ${{ inputs.arch == 'x86_64' }}
         run: |
           sudo apt update -y
@@ -63,8 +71,6 @@ jobs:
           go generate ./...
           popd
 
-      # Right now we are not able to install kernel-headers on our ARM64 self-hosted runner.
-      # For this reason, we disable the falco-driver-loader tests, which require kernel headers on the host.
       - name: Run regression tests
         env:
           # fixme(leogr): this is a workaround for https://github.com/falcosecurity/falco/issues/2784
@@ -77,7 +83,7 @@ jobs:
             ./build/k8saudit.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
             if ${{ inputs.arch == 'x86_64' && 'true' || 'false' }}; then
                 sudo ./build/falco-driver-loader.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
-            fi
+            fi    
           fi
           cat ./report.txt | go-junit-report -set-exit-code > report.xml
           popd

.gitignore

@@ -5,3 +5,4 @@
 .vscode/*
 
 *.idea*
+CMakeUserPresets.json

CHANGELOG.md

@@ -1,5 +1,137 @@
 # Change Log
 
+## v0.37.0
+
+Released on 2024-01-30
+
+### Breaking Changes
+
+- The deprecated `rate-limiter` mechanism is removed as it is no longer used.
+  - the deprecated `outputs.rate` Falco config is removed.
+  - the deprecated `outputs.max_burst` Falco config is removed.
+- The deprecated `--userspace` CLI option is removed as it is no longer used.
+- The `falco-driver-loader` script will be removed and embedded into falcoctl. The new falcoctl driven implementation will drop:
+  - `--source-only` CLI option.
+  - `BPF_USE_LOCAL_KERNEL_SOURCES` environment variable.
+  - `DRIVER_CURL_OPTIONS` environment variable.
+  - `FALCO_BPF_PROBE` environment variable is not used by the new falcoctl driver loader, since it is already deprecated and will be removed in the next major version.
+  
+  Some env vars were renamed:
+  - `DRIVERS_REPO` env variable has been replaced by `FALCOCTL_DRIVER_NAME`  or `--name` command line argument for `falcoctl driver` command
+  - `DRIVERS_NAME` env variable has been replaced by `FALCOCTL_DRIVER_REPOS`, or `--repo` command line argument for `falcoctl driver` command
+  - `DRIVER_KERNEL_RELEASE` env variable has been replaced by `--kernelrelease` command line argument for `falcoctl driver install` command
+  - `DRIVER_KERNEL_VERSION` env variable has been replaced by `--kernelversion` command line argument for `falcoctl driver install` command
+  - `DRIVER_INSECURE_DOWNLOAD` env variable has been replaced by `--http-insecure` command line argument for `falcoctl driver install` command
+- Remove `-K/-k` options from Falco in favor of the new `k8smeta` plugin.
+- Drop plugins shipped with Falco since plugins are now be managed by falcoctl.
+- Falco 0.37.0 allows environment variables to be expanded even if they are part of a string. This introduces small breaking changes:
+  - Previously, environment variables used in YAML that were empty or defined as `“”` would be expanded to the default value. This was not consistent with the way YAML was handled in other cases, where we only returned the default values if the node was not defined. Now expanded env vars retain the same behavior of all other variables.
+  - Falco 0.37.0 will return default value for nodes that cannot be parsed to chosen type.
+  - `program_output` command will be env-expanded at init time, instead of letting `popen` and thus the `sh` shell expand it. This is technically a breaking change even if no behavioral change is expected. Also, you can avoid env var expansion by using `${{FOO}}` instead of `${FOO}`. It will resolve to `${FOO}` and won't be resolved to the env var value.
+
+### Major Changes
+
+* new!: dropped falco-driver-loader script in favor of new falcoctl driver command [[#2905](https://github.com/falcosecurity/falco/pull/2905)] - [@FedeDP](https://github.com/FedeDP)
+* update!: bump libs to latest and deprecation of k8s metadata options and configs [[#2914](https://github.com/falcosecurity/falco/pull/2914)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* cleanup(falco)!: remove `outputs.rate` and `outputs.max_burst` from Falco config [[#2841](https://github.com/falcosecurity/falco/pull/2841)] - [@Andreagit97](https://github.com/Andreagit97)
+* cleanup(falco)!: remove `--userspace` support [[#2839](https://github.com/falcosecurity/falco/pull/2839)] - [@Andreagit97](https://github.com/Andreagit97)
+* new(engine): add selective overrides for Falco rules [[#2981](https://github.com/falcosecurity/falco/pull/2981)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* feat(userspace/falco): falco administrators can now configure the http output to compress the data sent as well as enable keep alive for the connection. Two new fields (compress_uploads and keep_alive) in the http_output block of the `falco.yaml` file can be used for that purpose. Both are disabled by default. [[#2974](https://github.com/falcosecurity/falco/pull/2974)] - [@sgaist](https://github.com/sgaist)
+* new(userspace): support env variable expansion in all yaml, even inside strings. [[#2918](https://github.com/falcosecurity/falco/pull/2918)] - [@FedeDP](https://github.com/FedeDP)
+* new(scripts): add a way to enforce driver kind and falcoctl enablement when installing Falco from packages and dialog is not present. [[#2773](https://github.com/falcosecurity/falco/pull/2773)] - [@vjjmiras](https://github.com/vjjmiras)
+* new(falco): print system info when Falco starts [[#2927](https://github.com/falcosecurity/falco/pull/2927)] - [@Andreagit97](https://github.com/Andreagit97)
+* new: driver selection in falco.yaml [[#2413](https://github.com/falcosecurity/falco/pull/2413)] - [@therealbobo](https://github.com/therealbobo)
+* new(build): enable compilation on win32 and macOS. [[#2889](https://github.com/falcosecurity/falco/pull/2889)] - [@therealbobo](https://github.com/therealbobo)
+* feat(userspace/falco): falco administrators can now configure the address on which the webserver listen using the new listen_address field in the webserver block of the `falco.yaml` file. [[#2890](https://github.com/falcosecurity/falco/pull/2890)] - [@sgaist](https://github.com/sgaist)
+
+### Minor Changes
+
+* update(userspace/falco): add `engine_version_semver` key in `/versions` endpoint [[#2899](https://github.com/falcosecurity/falco/pull/2899)] - [@loresuso](https://github.com/loresuso)
+* update: default ruleset upgrade to version 3.0 [[#3034](https://github.com/falcosecurity/falco/pull/3034)] - [@leogr](https://github.com/leogr)
+* update!(config): soft deprecation of drop stats counters in `syscall_event_drops` [[#3015](https://github.com/falcosecurity/falco/pull/3015)] - [@incertum](https://github.com/incertum)
+* update(cmake): bumped falcoctl tool to v0.7.1. [[#3030](https://github.com/falcosecurity/falco/pull/3030)] - [@FedeDP](https://github.com/FedeDP)
+* update(rule_loader): deprecate the `append` flag in Falco rules [[#2992](https://github.com/falcosecurity/falco/pull/2992)] - [@Andreagit97](https://github.com/Andreagit97)
+* cleanup!(cmake): drop bundled plugins in Falco [[#2997](https://github.com/falcosecurity/falco/pull/2997)] - [@FedeDP](https://github.com/FedeDP)
+* update(config): clarify deprecation notices + list all env vars [[#2988](https://github.com/falcosecurity/falco/pull/2988)] - [@incertum](https://github.com/incertum)
+* update: now the `watch_config_files` config option monitors file/directory moving and deletion, too [[#2965](https://github.com/falcosecurity/falco/pull/2965)] - [@NitroCao](https://github.com/NitroCao)
+* update(userspace): enhancements in rule description feature [[#2934](https://github.com/falcosecurity/falco/pull/2934)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* update(userspace/falco): add libsinsp state metrics option [[#2883](https://github.com/falcosecurity/falco/pull/2883)] - [@incertum](https://github.com/incertum)
+* update(doc): Add Thought Machine as adopters [[#2919](https://github.com/falcosecurity/falco/pull/2919)] - [@RichardoC](https://github.com/RichardoC)
+* update(docs): add Wireshark/Logray as adopter [[#2867](https://github.com/falcosecurity/falco/pull/2867)] - [@geraldcombs](https://github.com/geraldcombs)
+* update: engine_version in semver representation [[#2838](https://github.com/falcosecurity/falco/pull/2838)] - [@loresuso](https://github.com/loresuso)
+* update(userspace/engine): modularize rule compiler, fix and enrich rule descriptions [[#2817](https://github.com/falcosecurity/falco/pull/2817)] - [@jasondellaluce](https://github.com/jasondellaluce)
+
+### Bug Fixes
+
+* fix(userspace/metric): minor fixes in new libsinsp state metrics handling [[#3033](https://github.com/falcosecurity/falco/pull/3033)] - [@incertum](https://github.com/incertum)
+* fix(userspace/engine): avoid storing escaped strings in engine defs [[#3028](https://github.com/falcosecurity/falco/pull/3028)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* fix(userspace/engine): cache latest rules compilation output [[#2900](https://github.com/falcosecurity/falco/pull/2900)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* fix(userspace/engine): solve description of macro-only rules [[#2898](https://github.com/falcosecurity/falco/pull/2898)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* fix(userspace/engine): fix memory leak [[#2877](https://github.com/falcosecurity/falco/pull/2877)] - [@therealbobo](https://github.com/therealbobo)
+
+### Non user-facing changes
+
+* fix: nlohmann_json lib include path [[#3032](https://github.com/falcosecurity/falco/pull/3032)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* chore: bump falco rules [[#3021](https://github.com/falcosecurity/falco/pull/3021)] - [@Andreagit97](https://github.com/Andreagit97)
+* chore: bump Falco to libs 0.14.1 [[#3020](https://github.com/falcosecurity/falco/pull/3020)] - [@Andreagit97](https://github.com/Andreagit97)
+* chore(build): remove outdated development libs [[#2946](https://github.com/falcosecurity/falco/pull/2946)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* chore(falco): bump Falco to `000d576` libs commit [[#2944](https://github.com/falcosecurity/falco/pull/2944)] - [@Andreagit97](https://github.com/Andreagit97)
+* fix(gha): update rpmsign [[#2856](https://github.com/falcosecurity/falco/pull/2856)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* build(deps): Bump submodules/falcosecurity-rules from `424b258` to `1221b9e` [[#3000](https://github.com/falcosecurity/falco/pull/3000)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `2ac430b` to `c39d31a` [[#3019](https://github.com/falcosecurity/falco/pull/3019)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* cleanup(falco.yaml): rename `none` in `nodriver` [[#3012](https://github.com/falcosecurity/falco/pull/3012)] - [@Andreagit97](https://github.com/Andreagit97)
+* update(config): graduate outputs_queue to stable [[#3016](https://github.com/falcosecurity/falco/pull/3016)] - [@incertum](https://github.com/incertum)
+* update(cmake): bump falcoctl to v0.7.0. [[#3009](https://github.com/falcosecurity/falco/pull/3009)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `1221b9e` to `2ac430b` [[#3007](https://github.com/falcosecurity/falco/pull/3007)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* chore(ci): bumped rn2md to latest master. [[#3006](https://github.com/falcosecurity/falco/pull/3006)] - [@FedeDP](https://github.com/FedeDP)
+* chore: bump Falco to latest libs [[#3002](https://github.com/falcosecurity/falco/pull/3002)] - [@Andreagit97](https://github.com/Andreagit97)
+* chore: bump driver version [[#2998](https://github.com/falcosecurity/falco/pull/2998)] - [@Andreagit97](https://github.com/Andreagit97)
+* Add addl source related methods [[#2939](https://github.com/falcosecurity/falco/pull/2939)] - [@mstemm](https://github.com/mstemm)
+* build(deps): Bump submodules/falcosecurity-rules from `cd33bc3` to `424b258` [[#2993](https://github.com/falcosecurity/falco/pull/2993)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* cleanup(engine): clarify deprecation notice for engines [[#2987](https://github.com/falcosecurity/falco/pull/2987)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* update(cmake): bumped falcoctl to v0.7.0-rc1. [[#2983](https://github.com/falcosecurity/falco/pull/2983)] - [@FedeDP](https://github.com/FedeDP)
+* chore(ci): revert #2961. [[#2984](https://github.com/falcosecurity/falco/pull/2984)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-testing from `930170b` to `9b9630e` [[#2980](https://github.com/falcosecurity/falco/pull/2980)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* chore: bump Falco to latest libs [[#2977](https://github.com/falcosecurity/falco/pull/2977)] - [@Andreagit97](https://github.com/Andreagit97)
+* build(deps): Bump submodules/falcosecurity-rules from `262f569` to `cd33bc3` [[#2976](https://github.com/falcosecurity/falco/pull/2976)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* Allow enabling rules by ruleset id in addition to name [[#2920](https://github.com/falcosecurity/falco/pull/2920)] - [@mstemm](https://github.com/mstemm)
+* chore(ci): enable aarch64 falco driver loader tests. [[#2961](https://github.com/falcosecurity/falco/pull/2961)] - [@FedeDP](https://github.com/FedeDP)
+* chore(unit_tests): added more tests for yaml env vars expansion. [[#2972](https://github.com/falcosecurity/falco/pull/2972)] - [@FedeDP](https://github.com/FedeDP)
+* chore(falco.yaml): use HOME env var for ebpf probe path. [[#2971](https://github.com/falcosecurity/falco/pull/2971)] - [@FedeDP](https://github.com/FedeDP)
+* chore: bump falco to latest libs [[#2970](https://github.com/falcosecurity/falco/pull/2970)] - [@Andreagit97](https://github.com/Andreagit97)
+* build(deps): Bump submodules/falcosecurity-rules from `dd38952` to `262f569` [[#2969](https://github.com/falcosecurity/falco/pull/2969)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* update(readme): add actuated.dev badge [[#2967](https://github.com/falcosecurity/falco/pull/2967)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(cmake,docker): bumped falcoctl to v0.7.0-beta5. [[#2968](https://github.com/falcosecurity/falco/pull/2968)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `64e2adb` to `dd38952` [[#2959](https://github.com/falcosecurity/falco/pull/2959)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* fix(docker): small fixes in docker entrypoints for new driver loader. [[#2966](https://github.com/falcosecurity/falco/pull/2966)] - [@FedeDP](https://github.com/FedeDP)
+* chore(build): allow usage of non-bundled nlohmann-json [[#2947](https://github.com/falcosecurity/falco/pull/2947)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* update(ci): enable actuated.dev [[#2945](https://github.com/falcosecurity/falco/pull/2945)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* cleanup: fix several warnings from a Clang build [[#2948](https://github.com/falcosecurity/falco/pull/2948)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* chore(docker/falco): add back some deps to falco docker image. [[#2932](https://github.com/falcosecurity/falco/pull/2932)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-testing from `92c313f` to `5248e6d` [[#2937](https://github.com/falcosecurity/falco/pull/2937)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `e206c1a` to `8f0520f` [[#2904](https://github.com/falcosecurity/falco/pull/2904)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* cleanup(falco): remove decode_uri as it is no longer used [[#2933](https://github.com/falcosecurity/falco/pull/2933)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* update(engine): port decode_uri in falco engine [[#2912](https://github.com/falcosecurity/falco/pull/2912)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(falco): update to libs on nov 28th [[#2929](https://github.com/falcosecurity/falco/pull/2929)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* cleanup(falco): remove `init` in the configuration constructor [[#2917](https://github.com/falcosecurity/falco/pull/2917)] - [@Andreagit97](https://github.com/Andreagit97)
+* build(deps): Bump submodules/falcosecurity-rules from `8f0520f` to `64e2adb` [[#2908](https://github.com/falcosecurity/falco/pull/2908)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* cleanup(userspace/engine): remove legacy k8saudit implementation [[#2913](https://github.com/falcosecurity/falco/pull/2913)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* fix(gha): disable branch protection rule trigger for scorecard [[#2911](https://github.com/falcosecurity/falco/pull/2911)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(gha): set cosign-installer to v3.1.2 [[#2901](https://github.com/falcosecurity/falco/pull/2901)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* new(docs): sync changelog for 0.36.2. [[#2894](https://github.com/falcosecurity/falco/pull/2894)] - [@FedeDP](https://github.com/FedeDP)
+* Run OpenSSF Scorecard in pipeline [[#2888](https://github.com/falcosecurity/falco/pull/2888)] - [@maxgio92](https://github.com/maxgio92)
+* cleanup: replace banned.h with semgrep [[#2881](https://github.com/falcosecurity/falco/pull/2881)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(gha): upgrade GitHub actions [[#2876](https://github.com/falcosecurity/falco/pull/2876)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* build(deps): Bump submodules/falcosecurity-rules from `a22d0d7` to `e206c1a` [[#2865](https://github.com/falcosecurity/falco/pull/2865)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `d119706` to `a22d0d7` [[#2860](https://github.com/falcosecurity/falco/pull/2860)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* fix(gha): use fedora instead of centos 7 for package publishing [[#2854](https://github.com/falcosecurity/falco/pull/2854)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(gha): pin versions to hashes [[#2849](https://github.com/falcosecurity/falco/pull/2849)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* build(deps): Bump submodules/falcosecurity-rules from `c366d5b` to `d119706` [[#2847](https://github.com/falcosecurity/falco/pull/2847)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* new(ci): properly link libs and driver releases linked to a Falco release [[#2846](https://github.com/falcosecurity/falco/pull/2846)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `7a7cf24` to `c366d5b` [[#2842](https://github.com/falcosecurity/falco/pull/2842)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `77ba57a` to `7a7cf24` [[#2836](https://github.com/falcosecurity/falco/pull/2836)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* chore(ci): bumped rn2md to latest master. [[#2844](https://github.com/falcosecurity/falco/pull/2844)] - [@FedeDP](https://github.com/FedeDP)
+
 ## v0.36.2
 
 Released on 2023-10-27

CMakeLists.txt

@@ -180,7 +180,6 @@ include(static-analysis)
 # Shared build variables
 set(FALCO_SINSP_LIBRARY sinsp)
 set(FALCO_SHARE_DIR share/falco)
-set(FALCO_PLUGINS_DIR ${FALCO_SHARE_DIR}/plugins)
 set(FALCO_ABSOLUTE_SHARE_DIR "${CMAKE_INSTALL_PREFIX}/${FALCO_SHARE_DIR}")
 set(FALCO_BIN_DIR bin)
 
@@ -189,7 +188,6 @@ add_subdirectory(userspace/engine)
 add_subdirectory(userspace/falco)
 
 if(NOT WIN32 AND NOT APPLE AND NOT EMSCRIPTEN AND NOT MUSL_OPTIMIZED_BUILD)
-  include(plugins)
   include(falcoctl)
 endif()
 

README.md

@@ -2,7 +2,7 @@
 
 [![Latest release](https://img.shields.io/github/v/release/falcosecurity/falco?style=for-the-badge)](https://github.com/falcosecurity/falco/releases/latest) [![Supported Architectures](https://img.shields.io/badge/ARCHS-x86__64%7Caarch64-blueviolet?style=for-the-badge)](https://github.com/falcosecurity/falco/releases/latest) [![License](https://img.shields.io/github/license/falcosecurity/falco?style=for-the-badge)](COPYING) [![Docs](https://img.shields.io/badge/docs-latest-green.svg?style=for-the-badge)](https://falco.org/docs)
 
-[![Falco Core Repository](https://github.com/falcosecurity/evolution/blob/main/repos/badges/falco-core-blue.svg)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#core-scope) [![Stable](https://img.shields.io/badge/status-stable-brightgreen?style=for-the-badge)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#stable)  [![OpenSSF Best Practices](https://img.shields.io/cii/summary/2317?label=OpenSSF%20Best%20Practices&style=for-the-badge)](https://bestpractices.coreinfrastructure.org/projects/2317)
+[![Falco Core Repository](https://github.com/falcosecurity/evolution/blob/main/repos/badges/falco-core-blue.svg)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#core-scope) [![Stable](https://img.shields.io/badge/status-stable-brightgreen?style=for-the-badge)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#stable)  [![OpenSSF Best Practices](https://img.shields.io/cii/summary/2317?label=OpenSSF%20Best%20Practices&style=for-the-badge)](https://bestpractices.coreinfrastructure.org/projects/2317) <a href="https://actuated.dev/"><img alt="Arm CI sponsored by Actuated" src="https://docs.actuated.dev/images/actuated-badge.png" width="120px"></img></a>
 
 [![Falco](https://falco.org/img/brand/falco-horizontal-color.svg)](https://falco.org)
 

cmake/modules/CPackConfig.cmake

@@ -44,11 +44,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux") # only Linux has drivers
     "${CMAKE_CURRENT_BINARY_DIR};${DRIVER_COMPONENT_NAME};${DRIVER_COMPONENT_NAME};/")
 endif()
 
-if(NOT WIN32 AND NOT APPLE AND NOT EMSCRIPTEN AND NOT MUSL_OPTIMIZED_BUILD) # static builds do not have plugins
-  list(APPEND CPACK_INSTALL_CMAKE_PROJECTS
-    "${CMAKE_CURRENT_BINARY_DIR};${PLUGINS_COMPONENT_NAME};${PLUGINS_COMPONENT_NAME};/")
-endif()
-
 if(NOT CPACK_GENERATOR)
   if (CMAKE_SYSTEM_NAME MATCHES "Linux")
     set(CPACK_GENERATOR DEB RPM TGZ)

cmake/modules/driver.cmake

@@ -34,8 +34,8 @@ else()
   # In case you want to test against another driver version (or branch, or commit) just pass the variable -
   # ie., `cmake -DDRIVER_VERSION=dev ..`
   if(NOT DRIVER_VERSION)
-    set(DRIVER_VERSION "7cbc03a535ead9d530f7b77ffd68766d5e22da74")
-    set(DRIVER_CHECKSUM "SHA256=94d110ad1738cce2635fd15d41701bea5e061fd9a5a4be3f2ee8ec7a28fe50cc")
+    set(DRIVER_VERSION "7.0.0+driver")
+    set(DRIVER_CHECKSUM "SHA256=9f2a0f14827c0d9d1c3d1abe45b8f074dea531ebeca9859363a92f0d2475757e")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

cmake/modules/falcoctl.cmake

@@ -16,14 +16,14 @@ include(ExternalProject)
 
 string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} FALCOCTL_SYSTEM_NAME)
 
-set(FALCOCTL_VERSION "0.6.2")
+set(FALCOCTL_VERSION "0.7.1")
 
 if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
     set(FALCOCTL_SYSTEM_PROC_GO "amd64")
-    set(FALCOCTL_HASH "2d06d7577dbae91fb085f71477ff6e22076a815978bddd036984fa077236a515")
+    set(FALCOCTL_HASH "f142507c0e2b1e7dc03fd0b1ec36b479eb171f1f58c17f90d2d8edeb00668ef5")
 else() # aarch64
     set(FALCOCTL_SYSTEM_PROC_GO "arm64")
-    set(FALCOCTL_HASH "0b711a1b3499f479d999f4f4d2c94fc4f0bc23a2506711b613e6eedb0593631b")
+    set(FALCOCTL_HASH "93e4800b68e21057da82c8c7aafa0970598594d62cd9929ebb9b38a9c02159a6")
 endif()
 
 ExternalProject_Add(

cmake/modules/falcosecurity-libs.cmake

@@ -26,17 +26,17 @@ if(FALCOSECURITY_LIBS_SOURCE_DIR)
 else()
   # FALCOSECURITY_LIBS_REPO accepts a repository name (<org name>/<repo name>) alternative to the falcosecurity/libs repository.
   # In case you want to test against a fork of falcosecurity/libs just pass the variable -
-  # ie., `cmake -DFALCOSECURITY_LIBS_REPO=<your-gh-handle>/libs ..` 
+  # ie., `cmake -DFALCOSECURITY_LIBS_REPO=<your-gh-handle>/libs ..`
   if (NOT FALCOSECURITY_LIBS_REPO)
     set(FALCOSECURITY_LIBS_REPO "falcosecurity/libs")
   endif()
 
   # FALCOSECURITY_LIBS_VERSION accepts a git reference (branch name, commit hash, or tag) to the falcosecurity/libs repository.
   # In case you want to test against another falcosecurity/libs version (or branch, or commit) just pass the variable -
-  # ie., `cmake -DFALCOSECURITY_LIBS_VERSION=dev ..` 
+  # ie., `cmake -DFALCOSECURITY_LIBS_VERSION=dev ..`
   if(NOT FALCOSECURITY_LIBS_VERSION)
-    set(FALCOSECURITY_LIBS_VERSION "7cbc03a535ead9d530f7b77ffd68766d5e22da74")
-    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=94d110ad1738cce2635fd15d41701bea5e061fd9a5a4be3f2ee8ec7a28fe50cc")
+    set(FALCOSECURITY_LIBS_VERSION "0.14.2")
+    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=b4ae7bbf3ad031deee4a808145f9fd64b71c537406cc5a92f512823b2e52cdd5")
   endif()
 
   # cd /path/to/build && cmake /path/to/source
@@ -84,12 +84,11 @@ set(CREATE_TEST_TARGETS OFF CACHE BOOL "")
 set(BUILD_LIBSCAP_EXAMPLES OFF CACHE BOOL "")
 
 set(USE_BUNDLED_TBB ON CACHE BOOL "")
-set(USE_BUNDLED_B64 ON CACHE BOOL "")
 set(USE_BUNDLED_JSONCPP ON CACHE BOOL "")
+set(USE_BUNDLED_NLOHMANN_JSON ON CACHE BOOL "")
 set(USE_BUNDLED_VALIJSON ON CACHE BOOL "")
 set(USE_BUNDLED_RE2 ON CACHE BOOL "")
 set(USE_BUNDLED_UTHASH ON CACHE BOOL "")
-set(USE_BUNDLED_TINYDIR ON CACHE BOOL "")
 
 list(APPEND CMAKE_MODULE_PATH "${FALCOSECURITY_LIBS_SOURCE_DIR}/cmake/modules")
 

cmake/modules/libyaml.cmake

@@ -1,28 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-#
-# Copyright (C) 2023 The Falco Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-#
-
-set(LIBYAML_SRC "${PROJECT_BINARY_DIR}/libyaml-prefix/src/libyaml")
-set(LIBYAML_INSTALL_DIR "${LIBYAML_SRC}/target")
-message(STATUS "Using bundled libyaml in '${LIBYAML_SRC}'")
-set(LIBYAML_LIB "${LIBYAML_SRC}/src/.libs/libyaml.a")
-externalproject_add(
-  libyaml
-  URL "https://github.com/yaml/libyaml/releases/download/0.2.5/yaml-0.2.5.tar.gz"
-  URL_HASH "SHA256=c642ae9b75fee120b2d96c712538bd2cf283228d2337df2cf2988e3c02678ef4"
-  CONFIGURE_COMMAND ./configure --prefix=${LIBYAML_INSTALL_DIR} CFLAGS=-fPIC CPPFLAGS=-fPIC --enable-static=true --enable-shared=false
-  BUILD_COMMAND ${CMD_MAKE}
-  BUILD_IN_SOURCE 1
-  BUILD_BYPRODUCTS ${LIBYAML_LIB}
-  INSTALL_COMMAND ${CMD_MAKE} install
-)

cmake/modules/njson.cmake

@@ -12,24 +12,16 @@
 # specific language governing permissions and limitations under the License.
 #
 
-#
-# nlohmann-json
-#
-if(NJSON_INCLUDE)
-    # Adding the custom target we can use it with `add_dependencies()`
-    if(NOT TARGET njson)
-        add_custom_target(njson)
-    endif()
-else()
-    # We always use the bundled version
-    set(NJSON_SRC "${PROJECT_BINARY_DIR}/njson-prefix/src/njson")
-    set(NJSON_INCLUDE "${NJSON_SRC}/single_include")
-    ExternalProject_Add(
-        njson
+if(USE_BUNDLED_NLOHMANN_JSON)
+    ExternalProject_Add(njson
         URL "https://github.com/nlohmann/json/archive/v3.3.0.tar.gz"
         URL_HASH "SHA256=2fd1d207b4669a7843296c41d3b6ac5b23d00dec48dba507ba051d14564aa801"
-        CONFIGURE_COMMAND ""
-        BUILD_COMMAND ""
-        INSTALL_COMMAND "")
-    message(STATUS "Using bundled nlohmann-json in '${NJSON_SRC}'")
+        CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PROJECT_BINARY_DIR}/njson-prefix -DJSON_BuildTests=OFF -DBUILD_TESTING=OFF
+    )
+
+    set(nlohmann_json_INCLUDE_DIRS ${PROJECT_BINARY_DIR}/njson-prefix/include)
+else()
+    find_package(nlohmann_json CONFIG REQUIRED)
+    get_target_property(nlohmann_json_INCLUDE_DIRS nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES)
+    add_custom_target(njson)
 endif()

cmake/modules/plugins.cmake

@@ -1,98 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-#
-# Copyright (C) 2023 The Falco Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-#
-
-include(ExternalProject)
-
-# 'stable' or 'dev'
-set(PLUGINS_DOWNLOAD_BUCKET "stable")
-
-string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} PLUGINS_SYSTEM_NAME)
-
-if(NOT DEFINED PLUGINS_COMPONENT_NAME)
-    set(PLUGINS_COMPONENT_NAME "${CMAKE_PROJECT_NAME}-plugins")
-endif()
-
-# k8saudit
-set(PLUGIN_K8S_AUDIT_VERSION "0.6.1")
-if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
-    set(PLUGIN_K8S_AUDIT_HASH "e2908ebf2c03feecd26307ceab55aec9cae1cbc63d6aa05e147d8786e7670fb0")
-else() # aarch64
-    set(PLUGIN_K8S_AUDIT_HASH "8987a995fa09518aebc488ba549448166d605596c2d6478c10415a9d9f5f05dd")
-endif()
-
-ExternalProject_Add(
-  k8saudit-plugin
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/k8saudit-${PLUGIN_K8S_AUDIT_VERSION}-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
-  URL_HASH "SHA256=${PLUGIN_K8S_AUDIT_HASH}"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/k8saudit-plugin-prefix/src/k8saudit-plugin/libk8saudit.so" DESTINATION "${FALCO_PLUGINS_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")
-
-ExternalProject_Add(
-  k8saudit-rules
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/k8saudit-rules-${PLUGIN_K8S_AUDIT_VERSION}.tar.gz"
-  URL_HASH "SHA256=36321b3f1d7969926073a4d40bbbb7b4b28805b038c067f140795210ab641161"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/k8saudit-rules-prefix/src/k8saudit-rules/k8s_audit_rules.yaml" DESTINATION "${FALCO_ETC_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")
-
-# cloudtrail
-set(PLUGIN_CLOUDTRAIL_VERSION "0.9.0")
-if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
-    set(PLUGIN_CLOUDTRAIL_HASH "c8dc8ea5337aa9475042e6441320a5188bbf76977e3a69dd34a49a6251f8e9ad")
-else() # aarch64
-    set(PLUGIN_CLOUDTRAIL_HASH "bea12e81409c3df5698f7ab6a740ee9698b9dd1275b5985810daf70ac505c810")
-endif()
-
-ExternalProject_Add(
-  cloudtrail-plugin
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/cloudtrail-${PLUGIN_CLOUDTRAIL_VERSION}-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
-  URL_HASH "SHA256=${PLUGIN_CLOUDTRAIL_HASH}"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so" DESTINATION "${FALCO_PLUGINS_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")
-
-ExternalProject_Add(
-  cloudtrail-rules
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/cloudtrail-rules-${PLUGIN_CLOUDTRAIL_VERSION}.tar.gz"
-  URL_HASH "SHA256=b0c2b6c78d61cc3e7fb66445bcd8f763d15eb4a24f518385377e704aacec6b3f"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/cloudtrail-rules-prefix/src/cloudtrail-rules/aws_cloudtrail_rules.yaml" DESTINATION "${FALCO_ETC_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")
-
-# json
-set(PLUGIN_JSON_VERSION "0.7.1")
-if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
-    set(PLUGIN_JSON_HASH "3177fd667b384df2ffd2ae3260bda867c407c09d3fbcae841af204b82c1341c1")
-else() # aarch64
-    set(PLUGIN_JSON_HASH "3b5d0a9190bfd08e21915f997f88ca314f2027564a022eb88eef80ff4e2c77fa")
-endif()
-
-ExternalProject_Add(
-  json-plugin
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/json-${PLUGIN_JSON_VERSION}-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
-  URL_HASH "SHA256=${PLUGIN_JSON_HASH}"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/json-plugin-prefix/src/json-plugin/libjson.so" DESTINATION "${FALCO_PLUGINS_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")

cmake/modules/rules.cmake

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 #
-# Copyright (C) 2023 The Falco Authors.
+# Copyright (C) 2024 The Falco Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 # the License. You may obtain a copy of the License at
@@ -16,8 +16,8 @@ include(GNUInstallDirs)
 include(ExternalProject)
 
 # falco_rules.yaml
-set(FALCOSECURITY_RULES_FALCO_VERSION "falco-rules-2.0.0")
-set(FALCOSECURITY_RULES_FALCO_CHECKSUM "SHA256=48b6c5ae7a619a320eb51dbe036d1bc78622ab692956c9493390678874757b32")
+set(FALCOSECURITY_RULES_FALCO_VERSION "falco-rules-3.0.0")
+set(FALCOSECURITY_RULES_FALCO_CHECKSUM "SHA256=2e91799fee49c2daf58fb482e47410a21433eb116e02cde18206f7af87449ddb")
 set(FALCOSECURITY_RULES_FALCO_PATH "${PROJECT_BINARY_DIR}/falcosecurity-rules-falco-prefix/src/falcosecurity-rules-falco/falco_rules.yaml")
 ExternalProject_Add(
   falcosecurity-rules-falco

docker/driver-loader-legacy/docker-entrypoint.sh

@@ -18,6 +18,28 @@
 #
 
 
+print_usage() {
+	echo ""
+	echo "Usage:"
+	echo "  docker run -i -t --privileged -v /root/.falco:/root/.falco -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro -v /etc:/host/etc:ro falcosecurity/falco-driver-loader-legacy:latest [driver] [options]"
+	echo ""
+	echo "Available drivers:"
+	echo "  kmod           kernel module (default)"
+	echo "  ebpf           eBPF probe"
+	echo ""
+	echo "Options:"
+	echo "  --help         show this help message"
+	echo "  --clean        try to remove an already present driver installation"
+	echo "  --compile      try to compile the driver locally (default true)"
+	echo "  --download     try to download a prebuilt driver (default true)"
+	echo "  --print-env    skip execution and print env variables for other tools to consume"
+	echo ""
+	echo "Environment variables:"
+	echo "  FALCOCTL_DRIVER_REPOS             specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
+	echo "  FALCOCTL_DRIVER_NAME              specify a different name for the driver"
+	echo ""
+}
+
 echo "* Setting up /usr/src links from host"
 
 for i in "$HOST_ROOT/usr/src"/*
@@ -26,4 +48,64 @@ do
     ln -s "$i" "/usr/src/$base"
 done
 
-/usr/bin/falco-driver-loader "$@"
+ENABLE_COMPILE="false"
+ENABLE_DOWNLOAD="false"
+has_driver=
+has_opts=
+while test $# -gt 0; do
+	case "$1" in
+		kmod|ebpf)
+			if [ -n "$has_driver" ]; then
+				>&2 echo "Only one driver per invocation"
+				print_usage
+				exit 1
+			else
+				/usr/bin/falcoctl driver config --type $1
+				has_driver="true"
+			fi
+			;;
+		-h|--help)
+			print_usage
+			exit 0
+			;;
+		--clean)
+			/usr/bin/falcoctl driver cleanup
+			exit 0
+			;;
+		--compile)
+			ENABLE_COMPILE="true"
+			has_opts="true"
+			;;
+		--download)
+			ENABLE_DOWNLOAD="true"
+			has_opts="true"
+			;;
+		--source-only)
+		    >&2 echo "Support dropped in Falco 0.37.0."
+			print_usage
+			exit 1
+			;;
+		--print-env)
+			/usr/bin/falcoctl driver printenv
+			exit 0
+			;;
+		--*)
+			>&2 echo "Unknown option: $1"
+			print_usage
+			exit 1
+			;;
+		*)
+			>&2 echo "Unknown driver: $1"
+			print_usage
+			exit 1
+			;;
+	esac
+    shift
+done
+
+if [ -z "$has_opts" ]; then
+	ENABLE_COMPILE="true"
+	ENABLE_DOWNLOAD="true"
+fi
+
+/usr/bin/falcoctl driver install --compile=$ENABLE_COMPILE --download=$ENABLE_DOWNLOAD

docker/driver-loader/docker-entrypoint.sh

@@ -18,6 +18,28 @@
 #
 
 
+print_usage() {
+	echo ""
+	echo "Usage:"
+	echo "  docker run -i -t --privileged -v /root/.falco:/root/.falco -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro -v /etc:/host/etc:ro falcosecurity/falco-driver-loader:latest [driver] [options]"
+	echo ""
+	echo "Available drivers:"
+	echo "  kmod           kernel module (default)"
+	echo "  ebpf           eBPF probe"
+	echo ""
+	echo "Options:"
+	echo "  --help         show this help message"
+	echo "  --clean        try to remove an already present driver installation"
+	echo "  --compile      try to compile the driver locally (default true)"
+	echo "  --download     try to download a prebuilt driver (default true)"
+	echo "  --print-env    skip execution and print env variables for other tools to consume"
+	echo ""
+	echo "Environment variables:"
+	echo "  FALCOCTL_DRIVER_REPOS             specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
+	echo "  FALCOCTL_DRIVER_NAME              specify a different name for the driver"
+	echo ""
+}
+
 echo "* Setting up /usr/src links from host"
 
 for i in "$HOST_ROOT/usr/src"/*
@@ -26,4 +48,64 @@ do
     ln -s "$i" "/usr/src/$base"
 done
 
-/usr/bin/falco-driver-loader "$@"
+ENABLE_COMPILE="false"
+ENABLE_DOWNLOAD="false"
+has_driver=
+has_opts=
+while test $# -gt 0; do
+	case "$1" in
+		kmod|ebpf)
+			if [ -n "$has_driver" ]; then
+				>&2 echo "Only one driver per invocation"
+				print_usage
+				exit 1
+			else
+				/usr/bin/falcoctl driver config --type $1
+				has_driver="true"
+			fi
+			;;
+		-h|--help)
+			print_usage
+			exit 0
+			;;
+		--clean)
+			/usr/bin/falcoctl driver cleanup
+			exit 0
+			;;
+		--compile)
+			ENABLE_COMPILE="true"
+			has_opts="true"
+			;;
+		--download)
+			ENABLE_DOWNLOAD="true"
+			has_opts="true"
+			;;
+		--source-only)
+		    >&2 echo "Support dropped in Falco 0.37.0."
+			print_usage
+			exit 1
+			;;
+		--print-env)
+			/usr/bin/falcoctl driver printenv
+			exit 0
+			;;
+		--*)
+			>&2 echo "Unknown option: $1"
+			print_usage
+			exit 1
+			;;
+		*)
+			>&2 echo "Unknown driver: $1"
+			print_usage
+			exit 1
+			;;
+	esac
+    shift
+done
+
+if [ -z "$has_opts" ]; then
+	ENABLE_COMPILE="true"
+	ENABLE_DOWNLOAD="true"
+fi
+
+/usr/bin/falcoctl driver install --compile=$ENABLE_COMPILE --download=$ENABLE_DOWNLOAD

docker/falco/Dockerfile

@@ -19,17 +19,26 @@ RUN cp /etc/skel/.bashrc /root && cp /etc/skel/.profile /root
 
 RUN apt-get update \
 	&& apt-get install -y --no-install-recommends \
+	bc \
+	bison \
 	ca-certificates \
 	clang \
 	curl \
 	dkms \
+	dwarves \
+	flex \
 	gcc \
 	gcc-11 \
 	gnupg2 \
 	jq \
-	libelf1 \
+	libc6-dev \
+	libelf-dev \
+	libssl-dev \
 	llvm \
 	make \
+	netcat-openbsd \
+	patchelf \
+	xz-utils \
 	&& rm -rf /var/lib/apt/lists/*
 
 RUN curl -s https://falco.org/repo/falcosecurity-packages.asc | apt-key add - \

docker/falco/docker-entrypoint.sh

@@ -17,6 +17,29 @@
 # limitations under the License.
 #
 
+
+print_usage() {
+	echo ""
+	echo "Usage:"
+	echo "  docker run -i -t --privileged -v /root/.falco:/root/.falco -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro -v /etc:/host/etc:ro -e 'FALCO_DRIVER_LOADER_OPTIONS=[driver] [options]' falcosecurity/falco:latest"
+	echo ""
+	echo "Available FALCO_DRIVER_LOADER_OPTIONS drivers:"
+	echo "  kmod           kernel module (default)"
+	echo "  ebpf           eBPF probe"
+	echo ""
+	echo "FALCO_DRIVER_LOADER_OPTIONS options:"
+	echo "  --help         show this help message"
+	echo "  --clean        try to remove an already present driver installation"
+	echo "  --compile      try to compile the driver locally (default true)"
+	echo "  --download     try to download a prebuilt driver (default true)"
+	echo "  --print-env    skip execution and print env variables for other tools to consume"
+	echo ""
+	echo "Environment variables:"
+	echo "  FALCOCTL_DRIVER_REPOS             specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
+	echo "  FALCOCTL_DRIVER_NAME              specify a different name for the driver"
+	echo ""
+}
+
 # Set the SKIP_DRIVER_LOADER variable to skip loading the driver
 
 if [[ -z "${SKIP_DRIVER_LOADER}" ]]; then
@@ -29,9 +52,69 @@ if [[ -z "${SKIP_DRIVER_LOADER}" ]]; then
     done
 
     # convert the optional space-separated env variable FALCO_DRIVER_LOADER_OPTIONS to array, prevent 
-    # shell expansion and use it as argument list for falco-driver-loader
+    # shell expansion and use it as argument list for falcoctl
     read -a falco_driver_loader_option_arr <<< $FALCO_DRIVER_LOADER_OPTIONS
-    /usr/bin/falco-driver-loader "${falco_driver_loader_option_arr[@]}"
+
+    ENABLE_COMPILE="false"
+    ENABLE_DOWNLOAD="false"
+    has_driver=
+    has_opts=
+    for opt in "${falco_driver_loader_option_arr[@]}"
+    do
+        case "$opt" in
+            kmod|ebpf)
+                if [ -n "$has_driver" ]; then
+                    >&2 echo "Only one driver per invocation"
+                    print_usage
+                    exit 1
+                else
+                    /usr/bin/falcoctl driver config --type $opt
+                    has_driver="true"
+                fi
+                ;;
+            -h|--help)
+                print_usage
+                exit 0
+                ;;    
+            --clean)
+                /usr/bin/falcoctl driver cleanup
+                exit 0
+                ;;
+            --compile)
+                ENABLE_COMPILE="true"
+                has_opts="true"
+                ;;
+            --download)
+                ENABLE_DOWNLOAD="true"
+                has_opts="true"
+                ;;
+            --source-only)
+                >&2 echo "Support dropped in Falco 0.37.0."
+                print_usage
+                exit 1
+                ;;    
+            --print-env)
+                /usr/bin/falcoctl driver printenv
+                exit 0
+                ;;
+            --*)
+                >&2 echo "Unknown option: $1"
+                print_usage
+                exit 1
+                ;;
+            *)
+                >&2 echo "Unknown driver: $1"
+                print_usage
+                exit 1
+                ;;
+        esac
+    done
+    if [ -z "$has_opts" ]; then
+        ENABLE_COMPILE="true"
+        ENABLE_DOWNLOAD="true"
+    fi
+    /usr/bin/falcoctl driver install --compile=$ENABLE_COMPILE --download=$ENABLE_DOWNLOAD
+
 fi
 
 exec "$@"

docker/no-driver/Dockerfile

@@ -15,7 +15,7 @@ RUN curl -L -o falco.tar.gz \
     tar -xvf falco.tar.gz && \
     rm -f falco.tar.gz && \
     mv falco-${FALCO_VERSION}-$(uname -m) falco && \
-    rm -rf /falco/usr/src/falco-* /falco/usr/bin/falco-driver-loader
+    rm -rf /falco/usr/src/falco-*
 
 RUN sed -e 's/time_format_iso_8601: false/time_format_iso_8601: true/' < /falco/etc/falco/falco.yaml > /falco/etc/falco/falco.yaml.new \
     && mv /falco/etc/falco/falco.yaml.new /falco/etc/falco/falco.yaml

docker/no-driver/Dockerfile.distroless

@@ -16,7 +16,7 @@ RUN FALCO_VERSION_URLENCODED=$(echo -n ${FALCO_VERSION}|jq -sRr @uri) && \
     tar -xvf falco.tar.gz && \
     rm -f falco.tar.gz && \
     mv falco-${FALCO_VERSION}-$(uname -m) falco && \
-    rm -rf /falco/usr/src/falco-* /falco/usr/bin/falco-driver-loader
+    rm -rf /falco/usr/src/falco-*
 
 RUN sed -e 's/time_format_iso_8601: false/time_format_iso_8601: true/' < /falco/etc/falco/falco.yaml > /falco/etc/falco/falco.yaml.new \
     && mv /falco/etc/falco/falco.yaml.new /falco/etc/falco/falco.yaml

falco.yaml

@@ -41,7 +41,6 @@
 #     json_include_output_property
 #     json_include_tags_property
 #     buffered_outputs
-#     outputs (throttling)
 #     rule_matching
 #     outputs_queue
 # Falco outputs channels
@@ -62,13 +61,13 @@
 # Falco logging / alerting / metrics related to software functioning (advanced)
 #     output_timeout
 #     syscall_event_timeouts
-#     syscall_event_drops
+#     syscall_event_drops -> [CHANGE NOTICE] Automatic notifications will be simplified in Falco 0.38! If you depend on the detailed drop counters payload, use 'metrics.output_rule' along with 'metrics.kernel_event_counters_enabled' instead
 #     metrics
 # Falco performance tuning (advanced)
-#     syscall_buf_size_preset [DEPRECATED]
-#     syscall_drop_failed_exit [DEPRECATED]
+#     syscall_buf_size_preset [DEPRECATED] -> Replaced by `engine.<driver>.buf_size_preset` starting Falco 0.38!
+#     syscall_drop_failed_exit [DEPRECATED] -> Replaced by `engine.<driver>.drop_failed_exit` starting Falco 0.38! 
 #     base_syscalls
-#     modern_bpf.cpus_for_each_syscall_buffer [DEPRECATED]
+#     modern_bpf.cpus_for_each_syscall_buffer [DEPRECATED] -> Replaced by `engine.modern_ebpf.cpus_for_each_buffer` starting Falco 0.38! 
 
 
 ################################
@@ -80,9 +79,9 @@
 # configuration options from this config file as command-line arguments by using
 # the `-o` flag followed by the option name and value. In the following example,
 # three config options (`json_output`, `log_level`, and
-# `modern_bpf.cpus_for_each_syscall_buffer`) are passed as command-line
+# `engine.kind`) are passed as command-line
 # arguments with their corresponding values: falco -o "json_output=true"
-# -o "log_level=debug" -o "modern_bpf.cpus_for_each_syscall_buffer=4" 
+# -o "log_level=debug" -o "engine.kind=kmod" 
 # Please note that command-line arguments take precedence over the options
 # specified in this config file.
 
@@ -93,16 +92,32 @@
 
 # Customize Falco settings using environment variables:
 #
-# - "HOST_ROOT": Specifies the prefix to the underlying host `/proc` filesystem
+# - HOST_ROOT: Specifies the prefix to the underlying host `/proc` filesystem
 #   when deploying Falco over a container with read-only host mounts instead of
 #   directly on the host. Defaults to "/host". 
-# - "FALCO_BPF_PROBE": DEPRECATED. Specify a custom path to the BPF object code file (`bpf`
+#
+# - !!! [DEPRECATED] FALCO_BPF_PROBE: Specify a custom path to the BPF object code file (`bpf`
 #   driver). This is not needed for the modern_bpf driver. 
-# - "FALCO_HOSTNAME": Customize the hostname output field logged by Falco by
+#   -> Replaced by `engine.kind: ebpf` and `engine.ebpf` starting Falco 0.38!
+#
+# - FALCO_HOSTNAME: Customize the hostname output field logged by Falco by
 #   setting the "FALCO_HOSTNAME" environment variable.
-# - "FALCO_CGROUP_MEM_PATH": Specifies the file path holding the container
+#
+# - FALCO_CGROUP_MEM_PATH: Specifies the file path holding the container
 #   memory usage metric for the `metrics` feature. Defaults to
 #   "/sys/fs/cgroup/memory/memory.usage_in_bytes" (Kubernetes).
+#
+# - SKIP_DRIVER_LOADER is used by the Falco fat image to skip the driver loading part.
+#
+# - FALCO_FRONTEND is useful when set to noninteractive to skip the dialog choice during 
+# the installation of Falco deb/rpm packages. This setting is somewhat similar to DEBIAN_FRONTEND.
+#
+# - FALCO_DRIVER_CHOICE is useful when set to kmod, ebpf, or modern_ebpf (matching the names 
+# used in engine.kind in the Falco config) during the installation of Falco deb/rpm packages. 
+# It skips the dialog choice but retains the driver configuration.
+#
+# - FALCOCTL_ENABLED is useful when set to 'no' during the installation of Falco deb/rpm packages, 
+# disabling the automatic artifacts followed by falcoctl.
 
 
 #####################
@@ -164,7 +179,8 @@ rules_file:
 # - `modern_ebpf`: Modern eBPF (CO-RE eBPF probe)
 # - `gvisor`: gVisor (gVisor sandbox)
 # - `replay`: Replay a scap trace file
-# - `none`: No event producer loaded, useful to run with plugins.
+# - `nodriver`: No driver is injected into the system.
+#   This is useful to debug and to run plugins with 'syscall' source.
 #
 # Only one engine can be specified in the `kind` key.
 # Moreover, for each engine multiple options might be available,
@@ -318,7 +334,7 @@ engine:
     drop_failed_exit: false
   ebpf:
     # path to the elf file to load.
-    probe: /root/.falco/falco-bpf.o
+    probe: ${HOME}/.falco/falco-bpf.o
     buf_size_preset: 4
     drop_failed_exit: false
   modern_ebpf:
@@ -482,7 +498,7 @@ buffered_outputs: false
 # deploying it in production.
 rule_matching: first
 
-# [Experimental] `outputs_queue`
+# [Stable] `outputs_queue`
 #
 # Falco utilizes tbb::concurrent_bounded_queue for handling outputs, and this parameter
 # allows you to customize the queue capacity. Please refer to the official documentation:
@@ -566,6 +582,8 @@ http_output:
   client_key: "/etc/ssl/certs/client.key"
   # Whether to echo server answers to stdout
   echo: false
+  compress_uploads: false
+  keep_alive: false
 
 # [Stable] `program_output`
 #
@@ -775,7 +793,7 @@ output_timeout: 2000
 syscall_event_timeouts:
   max_consecutives: 1000
 
-# [Stable] `syscall_event_drops`
+# [Stable] `syscall_event_drops` -> [CHANGE NOTICE] Automatic notifications will be simplified in Falco 0.38! If you depend on the detailed drop counters payload, use 'metrics.output_rule' along with 'metrics.kernel_event_counters_enabled' instead
 #
 # Generates "Falco internal: syscall event drop" rule output when `priority=debug` at minimum
 #
@@ -917,13 +935,22 @@ syscall_event_drops:
 # number of CPUs to determine overall usage. Memory metrics are provided in raw
 # units (`kb` for `RSS`, `PSS` and `VSZ` or `bytes` for `container_memory_used`)
 # and can be uniformly converted to megabytes (MB) using the
-# `convert_memory_to_mb` functionality. In environments such as Kubernetes, it
-# is crucial to track Falco's container memory usage. To customize the path of
-# the memory metric file, you can create an environment variable named
-# `FALCO_CGROUP_MEM_PATH` and set it to the desired file path. By default, Falco
-# uses the file `/sys/fs/cgroup/memory/memory.usage_in_bytes` to monitor
-# container memory usage, which aligns with Kubernetes'
-# `container_memory_working_set_bytes` metric.
+# `convert_memory_to_mb` functionality. In environments such as Kubernetes when 
+# deployed as daemonset, it is crucial to track Falco's container memory usage. 
+# To customize the path of the memory metric file, you can create an environment 
+# variable named `FALCO_CGROUP_MEM_PATH` and set it to the desired file path. By 
+# default, Falco uses the file `/sys/fs/cgroup/memory/memory.usage_in_bytes` to 
+# monitor container memory usage, which aligns with Kubernetes' 
+# `container_memory_working_set_bytes` metric. Finally, we emit the overall host 
+# CPU and memory usages, along with the total number of processes and open file 
+# descriptors (fds) on the host, obtained from the proc file system unrelated to 
+# Falco's monitoring. These metrics help assess Falco's usage in relation to the 
+# server's workload intensity.
+#
+# `state_counters_enabled`: Emit counters related to Falco's state engine, including 
+# added, removed threads or file descriptors (fds), and failed lookup, store, or 
+# retrieve actions in relation to Falco's underlying process cache table (threadtable). 
+# We also log the number of currently cached containers if applicable.
 #
 # `kernel_event_counters_enabled`: Emit kernel side event and drop counters, as
 # an alternative to `syscall_event_drops`, but with some differences. These
@@ -956,6 +983,7 @@ metrics:
   output_rule: true
   # output_file: /tmp/falco_stats.jsonl
   resource_utilization_enabled: true
+  state_counters_enabled: true
   kernel_event_counters_enabled: true
   libbpf_stats_enabled: true
   convert_memory_to_mb: true
@@ -965,7 +993,7 @@ metrics:
 # Falco performance tuning (advanced) #
 #######################################
 
-# [DEPRECATED] `syscall_buf_size_preset`
+# [DEPRECATED] `syscall_buf_size_preset` -> Replaced by `engine.<driver>.buf_size_preset` starting Falco 0.38!
 #
 # Deprecated in favor of engine.{kmod,ebpf,modern_ebpf}.buf_size_preset.
 # This config is evaluated only if the default `engine` config block is not changed,
@@ -1021,7 +1049,7 @@ metrics:
 # if the default size is not suitable for your use case.
 syscall_buf_size_preset: 4
 
-# [DEPRECATED] `syscall_drop_failed_exit`
+# [DEPRECATED] `syscall_drop_failed_exit` -> Replaced by `engine.<driver>.drop_failed_exit` starting Falco 0.38! 
 #
 # Deprecated in favor of engine.{kmod,ebpf,modern_ebpf}.drop_failed_exit.
 # This config is evaluated only if the default `engine` config block is not changed,
@@ -1150,7 +1178,7 @@ base_syscalls:
   custom_set: []
   repair: false
 
-# [DEPRECATED] `modern_bpf.cpus_for_each_syscall_buffer`, modern_bpf only
+# [DEPRECATED] `modern_bpf.cpus_for_each_syscall_buffer`, modern_bpf only -> Replaced by `engine.modern_ebpf.cpus_for_each_buffer` starting Falco 0.38! 
 #
 # Deprecated in favor of engine.modern_ebpf.cpus_for_each_buffer.
 # This config is evaluated only if the default `engine` config block is not changed,

scripts/CMakeLists.txt

@@ -41,11 +41,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux")
 	configure_file(rpm/postinstall.in rpm/postinstall COPYONLY)
 	configure_file(rpm/postuninstall.in rpm/postuninstall COPYONLY)
 	configure_file(rpm/preuninstall.in rpm/preuninstall COPYONLY)
-
-	# driver loader
-	configure_file(falco-driver-loader falco-driver-loader @ONLY)
-	install(PROGRAMS ${PROJECT_BINARY_DIR}/scripts/falco-driver-loader
-		DESTINATION ${FALCO_BIN_DIR} COMPONENT "${FALCO_COMPONENT_NAME}")
 endif()
 
 # Install Falcoctl config file
@@ -53,5 +48,6 @@ if (NOT WIN32 AND NOT APPLE AND NOT EMSCRIPTEN AND NOT MUSL_OPTIMIZED_BUILD)
 	if(NOT DEFINED FALCOCTL_ETC_DIR)
 		set(FALCOCTL_ETC_DIR "${CMAKE_INSTALL_FULL_SYSCONFDIR}/falcoctl")
 	endif()
-	install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/falcoctl/falcoctl.yaml DESTINATION "${FALCOCTL_ETC_DIR}" COMPONENT "${FALCO_COMPONENT_NAME}")
+	configure_file(${CMAKE_CURRENT_SOURCE_DIR}/falcoctl/falcoctl.yaml.in ${PROJECT_BINARY_DIR}/scripts/falcoctl/falcoctl.yaml)
+	install(FILES ${PROJECT_BINARY_DIR}/scripts/falcoctl/falcoctl.yaml DESTINATION "${FALCOCTL_ETC_DIR}" COMPONENT "${FALCO_COMPONENT_NAME}")
 endif()

scripts/debian/postinst.in

@@ -18,6 +18,8 @@
 #
 
 chosen_driver=
+chosen_unit=
+CHOICE=
 
 # Every time we call this script we want to stat from a clean state.
 echo "[POST-INSTALL] Disable all possible 'falco' services:"
@@ -36,39 +38,63 @@ systemctl --system disable 'falcoctl-artifact-follow.service' || true
 systemctl --system unmask falcoctl-artifact-follow.service || true
 
 if [ "$1" = "configure" ]; then
-        if [ -x /usr/bin/dialog ] && [ "${FALCO_FRONTEND}" != "noninteractive" ]; then
-          # If dialog is installed, create a dialog to let users choose the correct driver for them
-          CHOICE=$(dialog --clear --title "Falco drivers" --menu "Choose your preferred driver:" 12 55 4 \
-                1 "Manual configuration (no unit is started)" \
-                2 "Kmod" \
-                3 "eBPF" \
-                4 "Modern eBPF" \
-                2>&1 >/dev/tty)
-          case $CHOICE in
-                  2)
-                      chosen_driver="kmod"
-                      ;;
-                  3)
-                      chosen_driver="bpf"
-                      ;;
-                  4)
-                      chosen_driver="modern-bpf"
-                      ;;
-          esac
-          if [ -n "$chosen_driver" ]; then
-            CHOICE=$(dialog --clear --title "Falcoctl" --menu "Do you want to follow automatic ruleset updates?" 10 40 2 \
-                    1 "Yes" \
-                    2 "No" \
+        case $FALCO_DRIVER_CHOICE in
+              kmod)
+                  CHOICE=2
+                  ;;
+              ebpf)
+                  CHOICE=3
+                  ;;
+              modern_ebpf)
+                  CHOICE=4
+                  ;;
+        esac     
+        if [ -z $CHOICE ] && [ -x /usr/bin/dialog ] && [ "${FALCO_FRONTEND}" != "noninteractive" ]; then          
+            # If dialog is installed, create a dialog to let users choose the correct driver for them
+            CHOICE=$(dialog --clear --title "Falco drivers" --menu "Choose your preferred driver:" 12 55 4 \
+                    1 "Manual configuration (no unit is started)" \
+                    2 "Kmod" \
+                    3 "eBPF" \
+                    4 "Modern eBPF" \
                     2>&1 >/dev/tty)
+        fi         
+        case $CHOICE in
+            2)
+                chosen_driver="kmod"
+                chosen_unit="kmod"
+                ;;
+            3)
+                chosen_driver="ebpf"
+                chosen_unit="bpf"
+                ;;
+            4)
+                chosen_driver="modern_ebpf"
+                chosen_unit="modern-bpf"
+                ;;
+        esac
+        if [ -n "$CHOICE" ]; then
+            echo "[POST-INSTALL] Configure falcoctl driver type:"
+            falcoctl driver config --type $chosen_driver
+            CHOICE=
+            case $FALCOCTL_ENABLED in
+                no)
+                    CHOICE=2
+                    ;;
+            esac
+            if [ -z $CHOICE ] &&  [ -x /usr/bin/dialog ] && [ "${FALCO_FRONTEND}" != "noninteractive" ]; then
+                CHOICE=$(dialog --clear --title "Falcoctl" --menu "Do you want to follow automatic ruleset updates?" 10 40 2 \
+                        1 "Yes" \
+                        2 "No" \
+                        2>&1 >/dev/tty)
+            fi  
             case $CHOICE in
                 2)
-                # we don't want falcoctl enabled, we mask it
-                systemctl --system mask falcoctl-artifact-follow.service || true
+                    # we don't want falcoctl enabled, we mask it
+                    systemctl --system mask falcoctl-artifact-follow.service || true
                 ;;
             esac
-          fi
-          clear
-        fi
+        fi    
+        clear
 fi
 
 set -e
@@ -76,25 +102,25 @@ set -e
 echo "[POST-INSTALL] Trigger deamon-reload:"
 systemctl --system daemon-reload || true
 
-# If needed, try to load/compile the driver through falco-driver-loader
+# If needed, try to load/compile the driver through falcoctl
 case "$chosen_driver" in
     "kmod")
       # Only compile for kmod, in this way we use dkms
-      echo "[POST-INSTALL] Call 'falco-driver-loader --compile module':"
-      falco-driver-loader --compile module
+      echo "[POST-INSTALL] Call 'falcoctl driver install for kmod:"
+      falcoctl driver install --download=false
       ;;
-    "bpf")
-      echo "[POST-INSTALL] Call 'falco-driver-loader bpf':"
-      falco-driver-loader bpf
+    "ebpf")
+      echo "[POST-INSTALL] Call 'falcoctl driver install for ebpf':"
+      falcoctl driver install
       ;;
 esac
 
 if [ "$1" = "configure" ] || [ "$1" = "abort-upgrade" ] || [ "$1" = "abort-deconfigure" ] || [ "$1" = "abort-remove" ] ; then
-        if [ -n "$chosen_driver" ]; then
+        if [ -n "$chosen_unit" ]; then
             # we do this in 2 steps because `enable --now` is not always supported
-            echo "[POST-INSTALL] Enable 'falco-$chosen_driver.service':"
-            systemctl --system enable "falco-$chosen_driver.service" || true
-            echo "[POST-INSTALL] Start 'falco-$chosen_driver.service':"
-            systemctl --system start "falco-$chosen_driver.service" || true           
+            echo "[POST-INSTALL] Enable 'falco-$chosen_unit.service':"
+            systemctl --system enable "falco-$chosen_unit.service" || true
+            echo "[POST-INSTALL] Start 'falco-$chosen_unit.service':"
+            systemctl --system start "falco-$chosen_unit.service" || true
         fi
 fi

scripts/debian/prerm.in

@@ -31,7 +31,7 @@ case "$1" in
       systemctl --system stop 'falco-custom.service' || true
       systemctl --system stop 'falcoctl-artifact-follow.service' || true
 
-      echo "[PRE-REMOVE] Call 'falco-driver-loader --clean:'"
-      falco-driver-loader --clean
+      echo "[PRE-REMOVE] Call 'falcoctl driver cleanup:'"
+      falcoctl driver cleanup
     ;;
 esac

scripts/falco-driver-loader

@@ -1,866 +0,0 @@
-#!/usr/bin/env bash
-# SPDX-License-Identifier: Apache-2.0
-#
-# Copyright (C) 2023 The Falco Authors.
-#
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Simple script that desperately tries to load the kernel instrumentation by
-# looking for it in a bunch of ways. Convenient when running Falco inside
-# a container or in other weird environments.
-#
-
-#
-# Returns 1 if $cos_ver > $base_ver, 0 otherwise
-#
-cos_version_greater() {
-	if [[ $cos_ver == "${base_ver}" ]]; then
-		return 0
-	fi
-
-	#
-	# COS build numbers are in the format x.y.z
-	#
-	a=$(echo "${cos_ver}" | cut -d. -f1)
-	b=$(echo "${cos_ver}" | cut -d. -f2)
-	c=$(echo "${cos_ver}" | cut -d. -f3)
-
-	d=$(echo "${base_ver}" | cut -d. -f1)
-	e=$(echo "${base_ver}" | cut -d. -f2)
-	f=$(echo "${base_ver}" | cut -d. -f3)
-
-	# Test the first component
-	if [[ $a -gt $d ]]; then
-		return 1
-	elif [[ $d -gt $a ]]; then
-		return 0
-	fi
-
-	# Test the second component
-	if [[ $b -gt $e ]]; then
-		return 1
-	elif [[ $e -gt $b ]]; then
-		return 0
-	fi
-
-	# Test the third component
-	if [[ $c -gt $f ]]; then
-		return 1
-	elif [[ $f -gt $c ]]; then
-		return 0
-	fi
-
-	# If we get here, probably malformatted version string?
-
-	return 0
-}
-
-get_kernel_config() {
-	if [ -f /proc/config.gz ]; then
-		echo "* Found kernel config at /proc/config.gz"
-		KERNEL_CONFIG_PATH=/proc/config.gz
-	elif [ -f "/boot/config-${KERNEL_RELEASE}" ]; then
-		echo "* Found kernel config at /boot/config-${KERNEL_RELEASE}"
-		KERNEL_CONFIG_PATH=/boot/config-${KERNEL_RELEASE}
-	elif [ -n "${HOST_ROOT}" ] && [ -f "${HOST_ROOT}/boot/config-${KERNEL_RELEASE}" ]; then
-		echo "* Found kernel config at ${HOST_ROOT}/boot/config-${KERNEL_RELEASE}"
-		KERNEL_CONFIG_PATH="${HOST_ROOT}/boot/config-${KERNEL_RELEASE}"
-	elif [ -f "/usr/lib/ostree-boot/config-${KERNEL_RELEASE}" ]; then
-		echo "* Found kernel config at /usr/lib/ostree-boot/config-${KERNEL_RELEASE}"
-		KERNEL_CONFIG_PATH="/usr/lib/ostree-boot/config-${KERNEL_RELEASE}"
-	elif [ -n "${HOST_ROOT}" ] && [ -f "${HOST_ROOT}/usr/lib/ostree-boot/config-${KERNEL_RELEASE}" ]; then
-		echo "* Found kernel config at ${HOST_ROOT}/usr/lib/ostree-boot/config-${KERNEL_RELEASE}"
-		KERNEL_CONFIG_PATH="${HOST_ROOT}/usr/lib/ostree-boot/config-${KERNEL_RELEASE}"
-	elif [ -f "/lib/modules/${KERNEL_RELEASE}/config" ]; then
-		# This code works both for native host and containers assuming that
-		# Dockerfile sets up the desired symlink /lib/modules -> $HOST_ROOT/lib/modules
-		echo "* Found kernel config at /lib/modules/${KERNEL_RELEASE}/config"
-		KERNEL_CONFIG_PATH="/lib/modules/${KERNEL_RELEASE}/config"
-	fi
-
-	if [ -z "${KERNEL_CONFIG_PATH}" ]; then
-		>&2 echo "Cannot find kernel config"
-		exit 1
-	fi
-
-	if [[ "${KERNEL_CONFIG_PATH}" == *.gz ]]; then
-		HASH=$(zcat "${KERNEL_CONFIG_PATH}" | md5sum - | cut -d' ' -f1)
-	else
-		HASH=$(md5sum "${KERNEL_CONFIG_PATH}" | cut -d' ' -f1)
-	fi
-}
-
-get_target_id() {
-	if [ -f "${HOST_ROOT}/etc/os-release" ]; then
-		# freedesktop.org and systemd
-		# shellcheck source=/dev/null
-		source "${HOST_ROOT}/etc/os-release"
-		OS_ID=$ID
-	elif [ -f "${HOST_ROOT}/etc/debian_version" ]; then
-		# Older debian distros
-		# fixme > Can this happen on older Ubuntu?
-		OS_ID=debian
-	elif [ -f "${HOST_ROOT}/etc/centos-release" ]; then
-		# Older CentOS distros
-		OS_ID=centos
-	elif [ -f "${HOST_ROOT}/etc/redhat-release" ]; then
-		# Older RHEL distros
-		OS_ID=rhel
-	else
-		# No target id can be determinand
-		TARGET_ID="undetermined"
-		return
-	fi
-
-	# Overwrite the OS_ID if /etc/VERSION file is present.
-	# Not sure if there is a better way to detect minikube.
-	if [ -f "${HOST_ROOT}/etc/VERSION" ]; then
-		OS_ID=minikube
-	fi
-
-	case "${OS_ID}" in
-	("amzn")
-		case "${VERSION_ID}" in
-		("2")
-			TARGET_ID="amazonlinux2"
-			;;
-		("2022")
-			TARGET_ID="amazonlinux2022"
-			;;
-		("2023")
-			TARGET_ID="amazonlinux2023"
-			;;
-		(*)
-			TARGET_ID="amazonlinux"
-			;;
-		esac
-		;;
-	("debian")
-		# Workaround: debian kernelreleases might now be actual kernel running;
-		# instead, they might be the Debian kernel package
-		# providing the compatible kernel ABI
-		# See https://lists.debian.org/debian-user/2017/03/msg00485.html
-		# Real kernel release is embedded inside the kernel version.
-		# Moreover, kernel arch, when present, is attached to the former,
-		# therefore make sure to properly take it and attach it to the latter.
-		# Moreover, we support 3 flavors for debian kernels: cloud, rt and normal.
-		# KERNEL-RELEASE will have a `-rt`, or `-cloud` if we are in one of these flavors.
-		# Manage it to download the correct driver.
-		#
-		# Example: KERNEL_RELEASE="5.10.0-0.deb10.22-rt-amd64" and `uname -v`="5.10.178-3"
-		# should lead to: KERNEL_RELEASE="5.10.178-3-rt-amd64"
-		TARGET_ID=$(echo "${OS_ID}" | tr '[:upper:]' '[:lower:]')
-		local ARCH_extra=""
-		if [[ $KERNEL_RELEASE =~ -?(rt-|cloud-|)(amd64|arm64) ]];
-		then
-			ARCH_extra="-${BASH_REMATCH[1]}${BASH_REMATCH[2]}"
-		fi
-		if [[ ${DRIVER_KERNEL_VERSION} =~ ([0-9]+\.[0-9]+\.[0-9]+\-[0-9]+) ]];
-		then
-			KERNEL_RELEASE="${BASH_REMATCH[1]}${ARCH_extra}"
-		fi
-		;;
-	("ubuntu")
-		# Extract the flavor from the kernelrelease
-		# Examples:
-		# 5.0.0-1028-aws-5.0 -> ubuntu-aws
-		# 5.15.0-1009-aws -> ubuntu-aws
-		if [[ $KERNEL_RELEASE =~ -([a-zA-Z]+)(-.*)?$ ]];
-		then
-			TARGET_ID="ubuntu-${BASH_REMATCH[1]}"
-		else
-			TARGET_ID="ubuntu-generic"
-		fi
-
-
-		# In the case that the kernelversion isn't just a number
-		# we keep also the remaining part excluding `-Ubuntu`.
-		# E.g.:
-		# from the following `uname -v` result
-		# `#26~22.04.1-Ubuntu SMP Mon Apr 24 01:58:15 UTC 2023`
-		# we obtain the kernelversion`26~22.04.1`
-		if [[ ${DRIVER_KERNEL_VERSION} =~ (^\#[0-9]+\~[^-]*-Ubuntu .*$) ]];
-		then
-			KERNEL_VERSION=$(echo "${DRIVER_KERNEL_VERSION}" | sed 's/#\([^-\\ ]*\).*/\1/g')
-		fi
-		;;
-	("flatcar")
-		KERNEL_RELEASE="${VERSION_ID}"
-		TARGET_ID=$(echo "${OS_ID}" | tr '[:upper:]' '[:lower:]')
-		;;
-	("minikube")
-		TARGET_ID=$(echo "${OS_ID}" | tr '[:upper:]' '[:lower:]')
-		# Extract the minikube version. Ex. With minikube version equal to "v1.26.0-1655407986-14197" the extracted version
-		# will be "1.26.0"
-		if [[ $(cat ${HOST_ROOT}/etc/VERSION) =~ ([0-9]+(\.[0-9]+){2}) ]]; then
-			# kernel version for minikube is always in "1_minikubeversion" format. Ex "1_1.26.0".
-			KERNEL_VERSION="1_${BASH_REMATCH[1]}"
-		else
-			echo "* Unable to extract minikube version from ${HOST_ROOT}/etc/VERSION"
-			exit 1
-		fi
-		;;
-	("bottlerocket")
-		TARGET_ID=$(echo "${OS_ID}" | tr '[:upper:]' '[:lower:]')
-		# variant_id has been sourced from os-release. Get only the first variant part
-		if [[ -n ${VARIANT_ID} ]];  then
-			# take just first part (eg: VARIANT_ID=aws-k8s-1.15 -> aws)
-			VARIANT_ID_CUT=${VARIANT_ID%%-*}
-		fi
-		# version_id has been sourced from os-release. Build a kernel version like: 1_1.11.0-aws
-		KERNEL_VERSION="1_${VERSION_ID}-${VARIANT_ID_CUT}"
-		;;
-	("talos")
-		TARGET_ID=$(echo "${OS_ID}" | tr '[:upper:]' '[:lower:]')
-		# version_id has been sourced from os-release. Build a kernel version like: 1_1.4.1
-		KERNEL_VERSION="1_${VERSION_ID}"
-		;;
-	(*)
-		TARGET_ID=$(echo "${OS_ID}" | tr '[:upper:]' '[:lower:]')
-		;;
-	esac
-}
-
-flatcar_relocate_tools() {
-	local -a tools=(
-		scripts/basic/fixdep
-		scripts/mod/modpost
-		tools/objtool/objtool
-	)
-	local -r hostld=$(ls /host/usr/lib64/ld-linux-*.so.*)
-	local -r kdir=/lib/modules/$(ls /lib/modules/)/build
-	echo "** Found host dl interpreter: ${hostld}"
-	for host_tool in ${tools[@]}; do
-		t=${host_tool}
-		tool=$(basename $t)
-		tool_dir=$(dirname $t)
-		host_tool=${kdir}/${host_tool}
-		if [ ! -f ${host_tool} ]; then
-			continue
-		fi
-		umount ${host_tool} 2>/dev/null || true
-		mkdir -p /tmp/${tool_dir}/
-		cp -a ${host_tool} /tmp/${tool_dir}/
-		echo "** Setting host dl interpreter for $host_tool"
-		patchelf --set-interpreter ${hostld} --set-rpath /host/usr/lib64 /tmp/${tool_dir}/${tool}
-		mount -o bind /tmp/${tool_dir}/${tool} ${host_tool}
-	done
-}
-
-load_kernel_module_compile() {
-	# Skip dkms on UEK hosts because it will always fail
-	if [[ ${DRIVER_KERNEL_RELEASE} == *uek* ]]; then
-		>&2 echo "Skipping because the dkms install always fail (on UEK hosts)"
-		return
-	fi
-
-	if ! hash dkms >/dev/null 2>&1; then
-		>&2 echo "This program requires dkms"
-		return
-	fi
-
-	if [ "${TARGET_ID}" == "flatcar" ]; then
-		KERNEL_RELEASE=${DRIVER_KERNEL_RELEASE}
-		echo "* Flatcar detected (version ${VERSION_ID}); relocating kernel tools"
-		flatcar_relocate_tools
-	fi
-
-	# Try to compile using all the available gcc versions
-	for CURRENT_GCC in $(ls "$(dirname "$(which gcc)")"/gcc*); do
-		# Filter away gcc-{ar,nm,...}
-		# Only gcc compiler has `-print-search-dirs` option.
-		${CURRENT_GCC} -print-search-dirs 2>&1 | grep "install:"
-		if [ "$?" -ne "0" ]; then
-			continue
-		fi
-		echo "* Trying to dkms install ${DRIVER_NAME} module with GCC ${CURRENT_GCC}"
-		echo "#!/usr/bin/env bash" > "${TMPDIR}/falco-dkms-make"
-		echo "make CC=${CURRENT_GCC} \$@" >> "${TMPDIR}/falco-dkms-make"
-		chmod +x "${TMPDIR}/falco-dkms-make"
-		if dkms install --directive="MAKE='${TMPDIR}/falco-dkms-make'" -m "${DRIVER_NAME}" -v "${DRIVER_VERSION}" -k "${KERNEL_RELEASE}" 2>/dev/null; then
-			echo "* ${DRIVER_NAME} module installed in dkms"
-			KO_FILE="/var/lib/dkms/${DRIVER_NAME}/${DRIVER_VERSION}/${KERNEL_RELEASE}/${ARCH}/module/${DRIVER_NAME}"
-			if [ -f "$KO_FILE.ko" ]; then
-				KO_FILE="$KO_FILE.ko"
-			elif [ -f "$KO_FILE.ko.gz" ]; then
-				KO_FILE="$KO_FILE.ko.gz"
-			elif [ -f "$KO_FILE.ko.xz" ]; then
-				KO_FILE="$KO_FILE.ko.xz"
-			elif [ -f "$KO_FILE.ko.zst" ]; then
-				KO_FILE="$KO_FILE.ko.zst"
-			else
-				>&2 echo "${DRIVER_NAME} module file not found"
-				return
-			fi
-			echo "* ${DRIVER_NAME} module found: ${KO_FILE}"
-			echo "* Trying to insmod"
-			chcon -t modules_object_t "$KO_FILE" > /dev/null 2>&1 || true
-			if insmod "$KO_FILE" > /dev/null 2>&1; then
-				echo "* Success: ${DRIVER_NAME} module found and loaded in dkms"
-				exit 0
-			fi
-			echo "* Unable to insmod ${DRIVER_NAME} module"
-		else
-			DKMS_LOG="/var/lib/dkms/${DRIVER_NAME}/${DRIVER_VERSION}/build/make.log"
-			if [ -f "${DKMS_LOG}" ]; then
-				echo "* Running dkms build failed, dumping ${DKMS_LOG} (with GCC ${CURRENT_GCC})"
-				cat "${DKMS_LOG}"
-			else
-				echo "* Running dkms build failed, couldn't find ${DKMS_LOG} (with GCC ${CURRENT_GCC})"
-			fi
-		fi
-	done
-}
-
-load_kernel_module_download() {
-	local FALCO_KERNEL_MODULE_FILENAME="${DRIVER_NAME}_${TARGET_ID}_${KERNEL_RELEASE}_${KERNEL_VERSION}.ko"
-	local URL=$(echo "${1}/${DRIVER_VERSION}/${ARCH}/${FALCO_KERNEL_MODULE_FILENAME}" | sed s/+/%2B/g)
-
-	echo "* Trying to download a prebuilt ${DRIVER_NAME} module from ${URL}"
-	if curl -L --create-dirs ${FALCO_DRIVER_CURL_OPTIONS} -o "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${FALCO_KERNEL_MODULE_FILENAME}" "${URL}"; then
-		echo "* Download succeeded"
-		chcon -t modules_object_t "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${FALCO_KERNEL_MODULE_FILENAME}" > /dev/null 2>&1 || true
-		if insmod "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${FALCO_KERNEL_MODULE_FILENAME}"; then
-			echo "* Success: ${DRIVER_NAME} module found and inserted"
-			exit 0
-		fi
-		>&2 echo "Unable to insmod the prebuilt ${DRIVER_NAME} module"
-	else
-		>&2 echo "Unable to find a prebuilt ${DRIVER_NAME} module"
-		return
-	fi
-}
-
-print_clean_termination() {
-	echo
-	echo "[SUCCESS] Cleaning phase correctly terminated."
-	echo
-	echo "================ Cleaning phase ================"
-	echo
-}
-
-print_filename_components() {
-	echo " - driver name: ${DRIVER_NAME}"
-	echo " - target identifier: ${TARGET_ID}"
-	echo " - kernel release: ${KERNEL_RELEASE}"
-	echo " - kernel version: ${KERNEL_VERSION}"
-}
-
-print_as_env_vars() {
-	echo "ARCH=\"${ARCH}\""
-	echo "KERNEL_RELEASE=\"${KERNEL_RELEASE}\""
-	echo "KERNEL_VERSION=\"${KERNEL_VERSION}\""
-	echo "ENABLE_COMPILE=\"${ENABLE_COMPILE}\""
-	echo "ENABLE_DOWNLOAD=\"${ENABLE_DOWNLOAD}\""
-	echo "TARGET_ID=\"${TARGET_ID}\""
-	echo "DRIVER=\"${DRIVER}\""
-	echo "DRIVERS_REPO=\"${DRIVERS_REPO}\""
-	echo "DRIVER_VERSION=\"${DRIVER_VERSION}\""
-	echo "DRIVER_NAME=\"${DRIVER_NAME}\""
-	echo "FALCO_VERSION=\"${FALCO_VERSION}\""
-}
-
-clean_kernel_module() {
-	echo
-	echo "================ Cleaning phase ================"
-	echo
-
-	if ! hash lsmod > /dev/null 2>&1; then
-		>&2 echo "This program requires lsmod."
-		exit 1
-	fi
-
-	if ! hash rmmod > /dev/null 2>&1; then
-		>&2 echo "This program requires rmmod."
-		exit 1
-	fi
-
-	KMOD_NAME=$(echo "${DRIVER_NAME}" | tr "-" "_")
-	echo "* 1. Check if kernel module '${KMOD_NAME}' is still loaded:"
-
-	if ! lsmod | cut -d' ' -f1 | grep -qx "${KMOD_NAME}"; then
-		echo "- OK! There is no '${KMOD_NAME}' module loaded."
-		echo
-	fi
-
-	# Wait 50s = MAX_RMMOD_WAIT * 5s
-	MAX_RMMOD_WAIT=10
-	# Remove kernel module if is still loaded.
-	while lsmod | cut -d' ' -f1 | grep -qx "${KMOD_NAME}" && [ $MAX_RMMOD_WAIT -gt 0 ]; do
-		echo "- Kernel module '${KMOD_NAME}' is still loaded."
-		echo "- Trying to unload it with 'rmmod ${KMOD_NAME}'..."
-		if rmmod ${KMOD_NAME}; then
-			echo "- OK! Unloading '${KMOD_NAME}' module succeeded."
-			echo
-		else
-			echo "- Nothing to do...'falco-driver-loader' will wait until you remove the kernel module to have a clean termination."
-			echo "- Check that no process is using the kernel module with 'lsmod | grep ${KMOD_NAME}'."
-			echo "- Sleep 5 seconds..."
-			echo
-			((--MAX_RMMOD_WAIT))
-			sleep 5
-		fi
-	done
-
-	if [ ${MAX_RMMOD_WAIT} -eq 0 ]; then
-		echo "[WARNING] '${KMOD_NAME}' module is still loaded, you could have incompatibility issues."
-		echo
-	fi
-
-	if ! hash dkms >/dev/null 2>&1; then
-		echo "- Skipping dkms remove (dkms not found)."
-		print_clean_termination
-		return
-	fi
-
-	# Remove all versions of this module from dkms.
-	echo "* 2. Check all versions of kernel module '${KMOD_NAME}' in dkms:"
-	DRIVER_VERSIONS=$(dkms status -m "${KMOD_NAME}" | tr -d "," | tr -d ":" | tr "/" " " | cut -d' ' -f2)
-	if [ -z "${DRIVER_VERSIONS}" ]; then
-		echo "- OK! There are no '${KMOD_NAME}' module versions in dkms."
-	else
-		echo "- There are some versions of '${KMOD_NAME}' module in dkms."
-		echo
-		echo "* 3. Removing all the following versions from dkms:"
-		echo "${DRIVER_VERSIONS}"
-		echo
-	fi
-
-	for CURRENT_VER in ${DRIVER_VERSIONS}; do
-		echo "- Removing ${CURRENT_VER}..."
-		if dkms remove -m ${KMOD_NAME} -v "${CURRENT_VER}" --all; then
-			echo
-			echo "- OK! Removing '${CURRENT_VER}' succeeded."
-			echo
-		else
-			echo "[WARNING] Removing '${KMOD_NAME}' version '${CURRENT_VER}' failed."
-		fi
-	done
-
-	print_clean_termination
-}
-
-load_kernel_module() {
-	clean_kernel_module
-
-	echo "* Looking for a ${DRIVER_NAME} module locally (kernel ${KERNEL_RELEASE})"
-
-	local FALCO_KERNEL_MODULE_FILENAME="${DRIVER_NAME}_${TARGET_ID}_${KERNEL_RELEASE}_${KERNEL_VERSION}.ko"
-	echo "* Filename '${FALCO_KERNEL_MODULE_FILENAME}' is composed of:"
-	print_filename_components
-
-	if [ -f "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${FALCO_KERNEL_MODULE_FILENAME}" ]; then
-		echo "* Found a prebuilt ${DRIVER_NAME} module at ${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${FALCO_KERNEL_MODULE_FILENAME}, loading it"
-		chcon -t modules_object_t "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${FALCO_KERNEL_MODULE_FILENAME}" > /dev/null 2>&1 || true
-		insmod "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${FALCO_KERNEL_MODULE_FILENAME}" && echo "* Success: ${DRIVER_NAME} module found and inserted"
-		exit $?
-	fi
-
-	if [ -n "$ENABLE_DOWNLOAD" ]; then
-		IFS=", " read -r -a urls <<< "${DRIVERS_REPO}"
-		for url in "${urls[@]}"; do
-			load_kernel_module_download $url
-		done
-	fi
-
-	if [ -n "$ENABLE_COMPILE" ]; then
-		load_kernel_module_compile
-	fi
-
-	# Last try (might load a previous driver version)
-	echo "* Trying to load a system ${DRIVER_NAME} module, if present"
-	if modprobe "${DRIVER_NAME}" > /dev/null 2>&1; then
-		echo "* Success: ${DRIVER_NAME} module found and loaded with modprobe"
-		exit 0
-	fi
-
-	# Not able to download a prebuilt module nor to compile one on-the-fly
-	>&2 echo "Consider compiling your own ${DRIVER_NAME} driver and loading it or getting in touch with the Falco community"
-	exit 1
-}
-
-load_bpf_probe_compile() {
-	local BPF_KERNEL_SOURCES_URL=""
-	local STRIP_COMPONENTS=1
-
-	customize_kernel_build() {
-		if [ -n "${KERNEL_EXTRA_VERSION}" ]; then
-			sed -i "s/LOCALVERSION=\"\"/LOCALVERSION=\"${KERNEL_EXTRA_VERSION}\"/" .config
-		fi
-		make olddefconfig > /dev/null
-		make modules_prepare > /dev/null
-	}
-
-	if [ "${TARGET_ID}" == "flatcar" ]; then
-		KERNEL_RELEASE=${DRIVER_KERNEL_RELEASE}
-		echo "* Flatcar detected (version ${VERSION_ID}); relocating kernel tools"
-		flatcar_relocate_tools
-	fi
-
-	if [ "${TARGET_ID}" == "cos" ]; then
-		echo "* COS detected (build ${BUILD_ID}), using COS kernel headers"
-
-		BPF_KERNEL_SOURCES_URL="https://storage.googleapis.com/cos-tools/${BUILD_ID}/kernel-headers.tgz"
-		KERNEL_EXTRA_VERSION="+"
-		STRIP_COMPONENTS=0
-
-		customize_kernel_build() {
-			pushd usr/src/* > /dev/null || exit
-
-			# Note: this overrides the KERNELDIR set while untarring the tarball
-			KERNELDIR=$(pwd)
-			export KERNELDIR
-
-			sed -i '/^#define randomized_struct_fields_start	struct {$/d' include/linux/compiler-clang.h
-			sed -i '/^#define randomized_struct_fields_end	};$/d' include/linux/compiler-clang.h
-
-			popd > /dev/null || exit
-
-			# Might need to configure our own sources depending on COS version
-			cos_ver=${BUILD_ID}
-			base_ver=11553.0.0
-
-			cos_version_greater
-			greater_ret=$?
-
-			if [[ greater_ret -eq 1 ]]; then
-			export KBUILD_EXTRA_CPPFLAGS=-DCOS_73_WORKAROUND
-			fi
-		}
-	fi
-
-	if [ "${TARGET_ID}" == "minikube" ]; then
-		MINIKUBE_VERSION="$(cat "${HOST_ROOT}/etc/VERSION")"
-		echo "* Minikube detected (${MINIKUBE_VERSION}), using linux kernel sources for minikube kernel"
-		local kernel_version
-		kernel_version=${DRIVER_KERNEL_RELEASE}
-		local -r kernel_version_major=$(echo "${kernel_version}" | cut -d. -f1)
-		local -r kernel_version_minor=$(echo "${kernel_version}" | cut -d. -f2)
-		local -r kernel_version_patch=$(echo "${kernel_version}" | cut -d. -f3)
-
-		if [ "${kernel_version_patch}" == "0" ]; then
-			kernel_version="${kernel_version_major}.${kernel_version_minor}"
-		fi
-
-		BPF_KERNEL_SOURCES_URL="http://mirrors.edge.kernel.org/pub/linux/kernel/v${kernel_version_major}.x/linux-${kernel_version}.tar.gz"
-	fi
-
-	if [ -n "${BPF_USE_LOCAL_KERNEL_SOURCES}" ]; then
-		local -r kernel_version_major=$(echo "${DRIVER_KERNEL_RELEASE}" | cut -d. -f1)
-		local -r kernel_version=$(echo "${DRIVER_KERNEL_RELEASE}" | cut -d- -f1)
-		KERNEL_EXTRA_VERSION="-$(echo "${DRIVER_KERNEL_RELEASE}" | cut -d- -f2)"
-
-		echo "* Using downloaded kernel sources for kernel version ${kernel_version}..."
-
-		BPF_KERNEL_SOURCES_URL="http://mirrors.edge.kernel.org/pub/linux/kernel/v${kernel_version_major}.x/linux-${kernel_version}.tar.gz"
-	fi
-
-	if [ -n "${BPF_KERNEL_SOURCES_URL}" ]; then
-		get_kernel_config
-
-		echo "* Downloading ${BPF_KERNEL_SOURCES_URL}"
-
-		mkdir -p /tmp/kernel
-		cd /tmp/kernel || exit
-		cd "$(mktemp -d -p /tmp/kernel)" || exit
-		if ! curl -L -o kernel-sources.tgz --create-dirs ${FALCO_DRIVER_CURL_OPTIONS} "${BPF_KERNEL_SOURCES_URL}"; then
-			>&2 echo "Unable to download the kernel sources"
-			return
-		fi
-
-		echo "* Extracting kernel sources"
-
-		mkdir kernel-sources && tar xf kernel-sources.tgz -C kernel-sources --strip-components "${STRIP_COMPONENTS}"
-
-		cd kernel-sources || exit
-		KERNELDIR=$(pwd)
-		export KERNELDIR
-
-		if [[ "${KERNEL_CONFIG_PATH}" == *.gz ]]; then
-			zcat "${KERNEL_CONFIG_PATH}" > .config
-		else
-			cat "${KERNEL_CONFIG_PATH}" > .config
-		fi
-
-		echo "* Configuring kernel"
-		customize_kernel_build
-	fi
-
-	echo "* Trying to compile the eBPF probe (${BPF_PROBE_FILENAME})"
-
-	make -C "/usr/src/${DRIVER_NAME}-${DRIVER_VERSION}/bpf" > /dev/null
-
-	mkdir -p "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}"
-	mv "/usr/src/${DRIVER_NAME}-${DRIVER_VERSION}/bpf/probe.o" "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}"
-
-	if [ -n "${BPF_KERNEL_SOURCES_URL}" ]; then
-		rm -r /tmp/kernel
-	fi
-
-}
-
-load_bpf_probe_download() {
-	local URL
-	URL=$(echo "${1}/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}" | sed s/+/%2B/g)
-
-	echo "* Trying to download a prebuilt eBPF probe from ${URL}"
-
-	if ! curl -L --create-dirs ${FALCO_DRIVER_CURL_OPTIONS} -o "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}" "${URL}"; then
-		>&2 echo "Unable to find a prebuilt ${DRIVER_NAME} eBPF probe"
-		return 1
-	fi
-	return 0
-}
-
-load_bpf_probe() {
-
-	if [ ! -d /sys/kernel/debug/tracing ]; then
-		echo "* Mounting debugfs"
-		mount -t debugfs nodev /sys/kernel/debug
-	fi
-
-	BPF_PROBE_FILENAME="${DRIVER_NAME}_${TARGET_ID}_${KERNEL_RELEASE}_${KERNEL_VERSION}.o"
-	echo "* Filename '${BPF_PROBE_FILENAME}' is composed of:"
-	print_filename_components
-
-	if [ -n "$ENABLE_DOWNLOAD" ]; then
-		if [ -f "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}" ]; then
-			echo "* Skipping download, eBPF probe is already present in ${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}"
-		else
-			IFS=", " read -r -a urls <<< "${DRIVERS_REPO}"
-			for url in "${urls[@]}"; do
-				load_bpf_probe_download $url
-				if [ $? -eq 0 ]; then
-                	break
-                fi
-			done
-		fi
-	fi
-
-	if [ -n "$ENABLE_COMPILE" ]; then
-		if [ -f "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}" ]; then
-			echo "* Skipping compilation, eBPF probe is already present in ${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}"
-		else
-			load_bpf_probe_compile
-		fi
-	fi
-
-	if [ -f "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}" ]; then
-		echo "* eBPF probe located in ${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}"
-
-		ln -sf "${HOME}/.falco/${DRIVER_VERSION}/${ARCH}/${BPF_PROBE_FILENAME}" "${HOME}/.falco/${DRIVER_NAME}-bpf.o" \
-			&& echo "* Success: eBPF probe symlinked to ${HOME}/.falco/${DRIVER_NAME}-bpf.o"
-		exit $?
-	else
-		>&2 echo "Unable to load the ${DRIVER_NAME} eBPF probe"
-		exit 1
-	fi
-}
-
-print_usage() {
-	echo ""
-	echo "Usage:"
-	echo "  falco-driver-loader [driver] [options]"
-	echo ""
-	echo "Available drivers:"
-	echo "  module        kernel module (default)"
-	echo "  bpf           eBPF probe"
-	echo ""
-	echo "Options:"
-	echo "  --help         show brief help"
-	echo "  --clean        try to remove an already present driver installation"
-	echo "  --compile      try to compile the driver locally (default true)"
-	echo "  --download     try to download a prebuilt driver (default true)"
-	echo "  --source-only  skip execution and allow sourcing in another script using `. falco-driver-loader`"
-	echo "  --print-env    skip execution and print env variables for other tools to consume"
-	echo ""
-	echo "Environment variables:"
-	echo "  DRIVERS_REPO             specify different URL(s) where to look for prebuilt Falco drivers (comma separated)"
-	echo "  DRIVER_NAME              specify a different name for the driver"
-	echo "  DRIVER_INSECURE_DOWNLOAD whether you want to allow insecure downloads or not"
-	echo "  DRIVER_CURL_OPTIONS      specify additional options to be passed to curl command used to download Falco drivers"
-	echo "  DRIVER_KERNEL_RELEASE    specify the kernel release for which to download/build the driver in the same format used by 'uname -r' (e.g. '6.1.0-10-cloud-amd64')"
-	echo "  DRIVER_KERNEL_VERSION    specify the kernel version for which to download/build the driver in the same format used by 'uname -v' (e.g. '#1 SMP PREEMPT_DYNAMIC Debian 6.1.38-2 (2023-07-27)')"
-	echo ""
-	echo "Versions:"
-	echo "  Falco version  ${FALCO_VERSION}"
-	echo "  Driver version ${DRIVER_VERSION}"
-	echo ""
-}
-
-ARCH=$(uname -m)
-
-DRIVER_KERNEL_RELEASE=${DRIVER_KERNEL_RELEASE:-$(uname -r)}
-KERNEL_RELEASE=${DRIVER_KERNEL_RELEASE}
-
-if ! hash sed > /dev/null 2>&1; then
-	>&2 echo "This program requires sed"
-	exit 1
-fi
-
-DRIVER_KERNEL_VERSION=${DRIVER_KERNEL_VERSION:-$(uname -v)}
-KERNEL_VERSION=$(echo "${DRIVER_KERNEL_VERSION}" | sed 's/#\([[:digit:]]\+\).*/\1/')
-
-DRIVERS_REPO=${DRIVERS_REPO:-"@DRIVERS_REPO@"}
-
-FALCO_DRIVER_CURL_OPTIONS="-fsS --connect-timeout 5 --max-time 60 --retry 3 --retry-max-time 120"
-
-if [ -n "$DRIVER_INSECURE_DOWNLOAD" ]
-then
-	FALCO_DRIVER_CURL_OPTIONS+=" -k"
-fi
-
-FALCO_DRIVER_CURL_OPTIONS+=" "${DRIVER_CURL_OPTIONS}
-
-if [[ -z "$MAX_RMMOD_WAIT" ]]; then
-	MAX_RMMOD_WAIT=60
-fi
-
-DRIVER_VERSION=${DRIVER_VERSION:-"@DRIVER_VERSION@"}
-DRIVER_NAME=${DRIVER_NAME:-"@DRIVER_NAME@"}
-FALCO_VERSION="@FALCO_VERSION@"
-
-TARGET_ID=
-get_target_id
-
-DRIVER="module"
-if [ -v FALCO_BPF_PROBE ]; then
-	DRIVER="bpf"
-fi
-
-TMPDIR=${TMPDIR:-"/tmp"}
-
-ENABLE_COMPILE=
-ENABLE_DOWNLOAD=
-
-clean=
-has_args=
-has_opts=
-print_env=
-source_only=
-while test $# -gt 0; do
-	case "$1" in
-		module|bpf)
-			if [ -n "$has_args" ]; then
-				>&2 echo "Only one driver per invocation"
-				print_usage
-				exit 1
-			else
-				DRIVER="$1"
-				has_args="true"
-				shift
-			fi
-			;;
-		-h|--help)
-			print_usage
-			exit 0
-			;;
-		--clean)
-			clean="true"
-			shift
-			;;
-		--compile)
-			ENABLE_COMPILE="yes"
-			has_opts="true"
-			shift
-			;;
-		--download)
-			ENABLE_DOWNLOAD="yes"
-			has_opts="true"
-			shift
-			;;
-		--source-only)
-			source_only="true"
-			shift
-			;;
-		--print-env)
-			print_env="true"
-			shift
-			;;
-		--*)
-			>&2 echo "Unknown option: $1"
-			print_usage
-			exit 1
-			;;
-		*)
-			>&2 echo "Unknown driver: $1"
-			print_usage
-			exit 1
-			;;
-	esac
-done
-
-if [ -z "$has_opts" ]; then
-	ENABLE_COMPILE="yes"
-	ENABLE_DOWNLOAD="yes"
-fi
-
-if [ -n "$source_only" ]; then
-	# Return or exit, depending if we've been sourced.
-	(return 0 2>/dev/null) && return || exit 0
-fi
-
-if [ -n "$print_env" ]; then
-	print_as_env_vars
-	exit 0
-fi
-
-echo "* Running falco-driver-loader for: falco version=${FALCO_VERSION}, driver version=${DRIVER_VERSION}, arch=${ARCH}, kernel release=${KERNEL_RELEASE}, kernel version=${KERNEL_VERSION}"
-
-if [ "$(id -u)" != 0 ]; then
-	>&2 echo "This program must be run as root (or with sudo)"
-	exit 1
-fi
-
-if [ "$TARGET_ID" = "undetermined" ]; then
-	if [ -n "$ENABLE_COMPILE" ]; then
-		ENABLE_DOWNLOAD=
-		>&2 echo "Detected an unsupported target system, please get in touch with the Falco community. Trying to compile anyway."
-	else
-		>&2 echo "Detected an unsupported target system, please get in touch with the Falco community."
-		exit 1
-	fi
-fi
-
-if [ -n "$clean" ]; then
-	if [ -n "$has_opts" ]; then
-		>&2 echo "Cannot use --clean with other options"
-		exit 1
-	fi
-
-	echo "* Running falco-driver-loader with: driver=$DRIVER, clean=yes"
-	case $DRIVER in
-	module)
-		clean_kernel_module
-		;;
-	bpf)
-		>&2 echo "--clean not supported for driver=bpf"
-		exit 1
-	esac
-else
-	if ! hash curl > /dev/null 2>&1; then
-		>&2 echo "This program requires curl"
-		exit 1
-	fi
-
-	echo "* Running falco-driver-loader with: driver=$DRIVER, compile=${ENABLE_COMPILE:-"no"}, download=${ENABLE_DOWNLOAD:-"no"}"
-	case $DRIVER in
-		module)
-			load_kernel_module
-			;;
-		bpf)
-			load_bpf_probe
-			;;
-	esac
-fi

scripts/falcoctl/falcoctl.yaml.in

@@ -1,3 +1,10 @@
+driver:
+    type: "kmod"
+    name: "@DRIVER_NAME@"
+    repos:
+        - "@DRIVERS_REPO@"
+    version: "@DRIVER_VERSION@"
+    hostroot: "/"
 artifact:
   follow:
     every: 6h0m0s

scripts/rpm/postinstall.in

@@ -17,6 +17,8 @@
 #
 
 chosen_driver=
+chosen_unit=
+CHOICE=
 
 # Every time we call this script we want to stat from a clean state.
 echo "[POST-INSTALL] Disable all possible enabled 'falco' service:"
@@ -35,7 +37,18 @@ systemctl --system disable 'falcoctl-artifact-follow.service' || true
 systemctl --system unmask falcoctl-artifact-follow.service || true
 
 if [ $1 -ge 1 ]; then
-        if [ -x /usr/bin/dialog ] && [ "${FALCO_FRONTEND}" != "noninteractive" ]; then
+        case $FALCO_DRIVER_CHOICE in
+              kmod)
+                  CHOICE=2
+                  ;;
+              ebpf)
+                  CHOICE=3
+                  ;;
+              modern_ebpf)
+                  CHOICE=4
+                  ;;
+        esac     
+        if [ -z $CHOICE ] && [ -x /usr/bin/dialog ] && [ "${FALCO_FRONTEND}" != "noninteractive" ]; then          
             # If dialog is installed, create a dialog to let users choose the correct driver for them
             CHOICE=$(dialog --clear --title "Falco drivers" --menu "Choose your preferred driver:" 12 55 4 \
                     1 "Manual configuration (no unit is started)" \
@@ -43,31 +56,44 @@ if [ $1 -ge 1 ]; then
                     3 "eBPF" \
                     4 "Modern eBPF" \
                     2>&1 >/dev/tty)
-            case $CHOICE in
-                2)
-                    chosen_driver="kmod"
-                    ;;
-                3)
-                    chosen_driver="bpf"
-                    ;;
-                4)
-                    chosen_driver="modern-bpf"
+        fi         
+        case $CHOICE in
+            2)
+                chosen_driver="kmod"
+                chosen_unit="kmod"
+                ;;
+            3)
+                chosen_driver="ebpf"
+                chosen_unit="bpf"
+                ;;
+            4)
+                chosen_driver="modern_ebpf"
+                chosen_unit="modern-bpf"
+                ;;
+        esac
+        if [ -n "$CHOICE" ]; then
+            echo "[POST-INSTALL] Configure falcoctl driver type:"
+            falcoctl driver config --type $chosen_driver
+            CHOICE=
+            case $FALCOCTL_ENABLED in
+                no)
+                    CHOICE=2
                     ;;
             esac
-            if [ -n "$chosen_driver" ]; then
-              CHOICE=$(dialog --clear --title "Falcoctl" --menu "Do you want to follow automatic ruleset updates?" 10 40 2 \
-                      1 "Yes" \
-                      2 "No" \
-                      2>&1 >/dev/tty)
-              case $CHOICE in
-                  2)
-                      # we don't want falcoctl enabled, we mask it
-                      systemctl --system mask falcoctl-artifact-follow.service || true
-                  ;;
-              esac
-            fi
-            clear
-        fi
+            if [ -z $CHOICE ] &&  [ -x /usr/bin/dialog ] && [ "${FALCO_FRONTEND}" != "noninteractive" ]; then
+                CHOICE=$(dialog --clear --title "Falcoctl" --menu "Do you want to follow automatic ruleset updates?" 10 40 2 \
+                        1 "Yes" \
+                        2 "No" \
+                        2>&1 >/dev/tty)
+            fi  
+            case $CHOICE in
+                2)
+                    # we don't want falcoctl enabled, we mask it
+                    systemctl --system mask falcoctl-artifact-follow.service || true
+                ;;
+            esac
+        fi    
+        clear
 fi
 
 set -e
@@ -75,16 +101,16 @@ set -e
 echo "[POST-INSTALL] Trigger deamon-reload:"
 systemctl --system daemon-reload || true
 
-# If needed, try to load/compile the driver through falco-driver-loader
+# If needed, try to load/compile the driver through falcoctl
 case "$chosen_driver" in
     "kmod")
       # Only compile for kmod, in this way we use dkms
-      echo "[POST-INSTALL] Call 'falco-driver-loader --compile module':"
-      falco-driver-loader --compile module
+       echo "[POST-INSTALL] Call 'falcoctl driver install for kmod:"
+       falcoctl driver install --download=false
       ;;
-    "bpf")
-      echo "[POST-INSTALL] Call 'falco-driver-loader bpf':"
-      falco-driver-loader bpf
+    "ebpf")
+      echo "[POST-INSTALL] Call 'falcoctl driver install for ebpf':"
+      falcoctl driver install
       ;;
 esac
 
@@ -95,14 +121,14 @@ esac
 # systemd_post macro expands to
 # if postinst:
 #   `systemd-update-helper install-system-units <service>`
-%systemd_post "falco-$chosen_driver.service"
+%systemd_post "falco-$chosen_unit.service"
 
 # post install/upgrade mirrored from .deb
 if [ $1 -ge 1 ]; then
-        if [ -n "$chosen_driver" ]; then
-            echo "[POST-INSTALL] Enable 'falco-$chosen_driver.service':"
-            systemctl --system enable "falco-$chosen_driver.service" || true
-            echo "[POST-INSTALL] Start 'falco-$chosen_driver.service':"
-            systemctl --system start "falco-$chosen_driver.service" || true
+        if [ -n "$chosen_unit" ]; then
+            echo "[POST-INSTALL] Enable 'falco-$chosen_unit.service':"
+            systemctl --system enable "falco-$chosen_unit.service" || true
+            echo "[POST-INSTALL] Start 'falco-$chosen_unit.service':"
+            systemctl --system start "falco-$chosen_unit.service" || true
         fi
 fi

scripts/rpm/preuninstall.in

@@ -25,8 +25,8 @@ systemctl --system stop 'falco-modern-bpf.service' || true
 systemctl --system stop 'falco-custom.service' || true
 systemctl --system stop 'falcoctl-artifact-follow.service' || true
 
-echo "[PRE-REMOVE] Call 'falco-driver-loader --clean:'"
-falco-driver-loader --clean
+echo "[PRE-REMOVE] Call 'falcoctl driver cleanup:'"
+falcoctl driver cleanup
 
 # validate rpm macros by `rpm -qp --scripts <rpm>`
 # RPM scriptlets: https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_systemd

scripts/systemd/falco-bpf.service

@@ -7,8 +7,7 @@ Wants=falcoctl-artifact-follow.service
 [Service]
 Type=simple
 User=root
-Environment=FALCO_BPF_PROBE=
-ExecStart=/usr/bin/falco
+ExecStart=/usr/bin/falco -o engine.kind=ebpf
 ExecReload=kill -1 $MAINPID
 UMask=0077
 TimeoutSec=30

scripts/systemd/falco-kmod.service

@@ -9,7 +9,7 @@ Wants=falcoctl-artifact-follow.service
 [Service]
 Type=simple
 User=root
-ExecStart=/usr/bin/falco
+ExecStart=/usr/bin/falco -o engine.kind=kmod
 ExecReload=kill -1 $MAINPID
 UMask=0077
 TimeoutSec=30

scripts/systemd/falco-modern-bpf.service

@@ -7,7 +7,7 @@ Wants=falcoctl-artifact-follow.service
 [Service]
 Type=simple
 User=root
-ExecStart=/usr/bin/falco --modern-bpf
+ExecStart=/usr/bin/falco -o engine.kind=modern_ebpf
 ExecReload=kill -1 $MAINPID
 UMask=0077
 TimeoutSec=30

unit_tests/engine/test_add_source.cpp

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+
+#include <falco_engine.h>
+#include <evttype_index_ruleset.h>
+
+static std::string syscall_source_name = "syscall";
+
+// A variant of evttype_index_ruleset_factory that uses a singleton
+// for the underlying ruleset. This allows testing of
+// ruleset_for_source
+
+namespace
+{
+class test_ruleset_factory : public evttype_index_ruleset_factory
+{
+public:
+	test_ruleset_factory(std::shared_ptr<gen_event_filter_factory> factory):
+		evttype_index_ruleset_factory(factory)
+	{
+		ruleset = evttype_index_ruleset_factory::new_ruleset();
+	}
+
+	virtual ~test_ruleset_factory() = default;
+
+	inline std::shared_ptr<filter_ruleset> new_ruleset() override
+	{
+		return ruleset;
+	}
+
+	std::shared_ptr<filter_ruleset> ruleset;
+};
+}; // namespace
+
+TEST(AddSource, basic)
+{
+	falco_engine engine;
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+
+	auto filter_factory = std::shared_ptr<gen_event_filter_factory>(
+		new sinsp_filter_factory(&inspector, filterchecks));
+	auto formatter_factory = std::shared_ptr<gen_event_formatter_factory>(
+		new sinsp_evt_formatter_factory(&inspector, filterchecks));
+	test_ruleset_factory *test_factory = new test_ruleset_factory(filter_factory);
+	auto ruleset_factory = std::shared_ptr<filter_ruleset_factory>(test_factory);
+
+	falco_source syscall_source;
+	syscall_source.name = syscall_source_name;
+	syscall_source.ruleset = ruleset_factory->new_ruleset();
+	syscall_source.ruleset_factory = ruleset_factory;
+	syscall_source.filter_factory = filter_factory;
+	syscall_source.formatter_factory = formatter_factory;
+
+	size_t source_idx = engine.add_source(syscall_source_name,
+					      filter_factory,
+					      formatter_factory,
+					      ruleset_factory);
+
+	ASSERT_TRUE(engine.is_source_valid(syscall_source_name));
+
+	ASSERT_EQ(engine.filter_factory_for_source(syscall_source_name), filter_factory);
+	ASSERT_EQ(engine.filter_factory_for_source(source_idx), filter_factory);
+
+	ASSERT_EQ(engine.formatter_factory_for_source(syscall_source_name), formatter_factory);
+	ASSERT_EQ(engine.formatter_factory_for_source(source_idx), formatter_factory);
+
+	ASSERT_EQ(engine.ruleset_factory_for_source(syscall_source_name), ruleset_factory);
+	ASSERT_EQ(engine.ruleset_factory_for_source(source_idx), ruleset_factory);
+
+	ASSERT_EQ(engine.ruleset_for_source(syscall_source_name), test_factory->ruleset);
+	ASSERT_EQ(engine.ruleset_for_source(source_idx), test_factory->ruleset);
+}

unit_tests/engine/test_enable_rule.cpp

@@ -0,0 +1,251 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <string>
+
+#include <gtest/gtest.h>
+
+#include <sinsp.h>
+#include <filter_check_list.h>
+#include <filter.h>
+
+#include <falco_engine.h>
+
+static std::string single_rule = R"END(
+- rule: test rule
+  desc: A test rule
+  condition: evt.type=execve
+  output: A test rule matched (evt.type=%evt.type)
+  priority: INFO
+  source: syscall
+  tags: [process]
+
+- rule: disabled rule
+  desc: A disabled rule
+  condition: evt.type=execve
+  output: A disabled rule matched (evt.type=%evt.type)
+  priority: INFO
+  source: syscall
+  enabled: false
+  tags: [exec process]
+)END";
+
+// This must be kept in line with the (private) falco_engine::s_default_ruleset
+static const std::string default_ruleset = "falco-default-ruleset";
+
+static const std::string ruleset_1 = "ruleset-1";
+static const std::string ruleset_2 = "ruleset-2";
+static const std::string ruleset_3 = "ruleset-3";
+static const std::string ruleset_4 = "ruleset-4";
+
+static void load_rules(falco_engine& engine, sinsp& inspector, sinsp_filter_check_list& filterchecks)
+{
+	std::unique_ptr<falco::load_result> res;
+
+	auto filter_factory = std::shared_ptr<gen_event_filter_factory>(
+		new sinsp_filter_factory(&inspector, filterchecks));
+	auto formatter_factory = std::shared_ptr<gen_event_formatter_factory>(
+		new sinsp_evt_formatter_factory(&inspector, filterchecks));
+
+	engine.add_source("syscall", filter_factory, formatter_factory);
+
+	res = engine.load_rules(single_rule, "single_rule.yaml");
+
+	EXPECT_TRUE(res->successful());
+}
+
+TEST(EnableRule, enable_rule_name)
+{
+	falco_engine engine;
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+
+	load_rules(engine, inspector, filterchecks);
+
+	// No rules should be enabled yet for any custom rulesets
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+
+	// Enable for first ruleset, only that ruleset should have an
+	// enabled rule afterward
+	engine.enable_rule("test", true, ruleset_1);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+
+	// Enable for second ruleset
+	engine.enable_rule("test", true, ruleset_2);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+
+	// When the substring is blank, all rules are enabled
+	// (including the disabled rule)
+	engine.enable_rule("", true, ruleset_3);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_3));
+
+	// Now disable for second ruleset
+	engine.enable_rule("test", false, ruleset_2);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_3));
+}
+
+TEST(EnableRule, enable_rule_tags)
+{
+	falco_engine engine;
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+	std::set<std::string> process_tags = {"process"};
+
+	load_rules(engine, inspector, filterchecks);
+
+	// No rules should be enabled yet for any custom rulesets
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+
+	// Enable for first ruleset, only that ruleset should have an
+	// enabled rule afterward
+	engine.enable_rule_by_tag(process_tags, true, ruleset_1);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+
+	// Enable for second ruleset
+	engine.enable_rule_by_tag(process_tags, true, ruleset_2);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
+
+	// Now disable for second ruleset
+	engine.enable_rule_by_tag(process_tags, false, ruleset_2);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+}
+
+TEST(EnableRule, enable_disabled_rule_by_tag)
+{
+	falco_engine engine;
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+	std::set<std::string> exec_process_tags = {"exec process"};
+
+	load_rules(engine, inspector, filterchecks);
+
+	// Only the first rule should be enabled
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
+
+	// Enable the disabled rule by tag
+	engine.enable_rule_by_tag(exec_process_tags, true);
+
+	// Both rules should be enabled now
+	EXPECT_EQ(2, engine.num_rules_for_ruleset(default_ruleset));
+}
+
+TEST(EnableRule, enable_rule_id)
+{
+	falco_engine engine;
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+	uint16_t ruleset_1_id;
+	uint16_t ruleset_2_id;
+	uint16_t ruleset_3_id;
+
+	load_rules(engine, inspector, filterchecks);
+
+	// The cases are identical to above, just using ruleset ids
+	// instead of names.
+
+	ruleset_1_id = engine.find_ruleset_id(ruleset_1);
+	ruleset_2_id = engine.find_ruleset_id(ruleset_2);
+	ruleset_3_id = engine.find_ruleset_id(ruleset_3);
+
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+
+	engine.enable_rule("test rule", true, ruleset_1_id);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+
+	engine.enable_rule("test rule", true, ruleset_2_id);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+
+	engine.enable_rule("", true, ruleset_3_id);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_3));
+
+	engine.enable_rule("test", false, ruleset_2_id);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_3));
+}
+
+TEST(EnableRule, enable_rule_name_exact)
+{
+	falco_engine engine;
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+
+	load_rules(engine, inspector, filterchecks);
+
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(default_ruleset));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_4));
+
+	engine.enable_rule_exact("test rule", true, ruleset_1);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_4));
+
+	engine.enable_rule_exact("test rule", true, ruleset_2);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_4));
+
+	// This should **not** enable as this is a substring and not
+	// an exact match.
+	engine.enable_rule_exact("test", true, ruleset_3);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_4));
+
+	engine.enable_rule_exact("", true, ruleset_4);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_4));
+
+	engine.enable_rule("test rule", false, ruleset_2);
+	EXPECT_EQ(1, engine.num_rules_for_ruleset(ruleset_1));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_2));
+	EXPECT_EQ(0, engine.num_rules_for_ruleset(ruleset_3));
+	EXPECT_EQ(2, engine.num_rules_for_ruleset(ruleset_4));
+}

unit_tests/engine/test_falco_utils.cpp

@@ -72,22 +72,3 @@ TEST(FalcoUtils, parse_prometheus_interval)
 	 */
 	ASSERT_EQ(falco::utils::parse_prometheus_interval("200"), 0UL);
 }
-
-TEST(FalcoUtils, decode_url)
-{
-	ASSERT_EQ(
-		falco::utils::decode_uri("https://www.example.com?key1=value+1&key2=value%40%21%242&key3=value%253", true),
-		"https://www.example.com?key1=value 1&key2=value@!$2&key3=value%3");
-	
-	ASSERT_EQ(
-		falco::utils::decode_uri("https://download.falco.org/?prefix=driver/3.0.1%2Bdriver/x86_64/", true),
-		"https://download.falco.org/?prefix=driver/3.0.1+driver/x86_64/");
-
-	ASSERT_EQ(
-		falco::utils::decode_uri("https://example.com/hello%20world", true),
-		"https://example.com/hello world");
-
-	ASSERT_EQ(
-		falco::utils::decode_uri("https://example.com/helloworld", true),
-		"https://example.com/helloworld");
-}

unit_tests/engine/test_filter_macro_resolver.cpp

@@ -37,17 +37,17 @@ TEST(MacroResolver, should_resolve_macros_on_a_filter_AST)
 {
 	libsinsp::filter::ast::pos_info macro_pos(12, 85, 27);
 
-	std::shared_ptr<libsinsp::filter::ast::expr> macro = std::move(libsinsp::filter::ast::unary_check_expr::create("test.field", "", "exists"));
+	std::shared_ptr<libsinsp::filter::ast::expr> macro = libsinsp::filter::ast::unary_check_expr::create("test.field", "", "exists");
 
 	std::vector<std::unique_ptr<libsinsp::filter::ast::expr>> filter_and;
 	filter_and.push_back(libsinsp::filter::ast::unary_check_expr::create("evt.name", "", "exists"));
 	filter_and.push_back(libsinsp::filter::ast::not_expr::create(libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos)));
-	std::shared_ptr<libsinsp::filter::ast::expr> filter = std::move(libsinsp::filter::ast::and_expr::create(filter_and));
+	std::shared_ptr<libsinsp::filter::ast::expr> filter = libsinsp::filter::ast::and_expr::create(filter_and);
 
 	std::vector<std::unique_ptr<libsinsp::filter::ast::expr>> expected_and;
 	expected_and.push_back(libsinsp::filter::ast::unary_check_expr::create("evt.name", "", "exists"));
 	expected_and.push_back(libsinsp::filter::ast::not_expr::create(clone(macro.get())));
-	std::shared_ptr<libsinsp::filter::ast::expr> expected = std::move(libsinsp::filter::ast::and_expr::create(expected_and));
+	std::shared_ptr<libsinsp::filter::ast::expr> expected = libsinsp::filter::ast::and_expr::create(expected_and);
 
 	filter_macro_resolver resolver;
 	resolver.set_macro(MACRO_NAME, macro);
@@ -71,9 +71,9 @@ TEST(MacroResolver, should_resolve_macros_on_a_filter_AST_single_node)
 {
 	libsinsp::filter::ast::pos_info macro_pos(12, 85, 27);
 
-	std::shared_ptr<libsinsp::filter::ast::expr> macro = std::move(libsinsp::filter::ast::unary_check_expr::create("test.field", "", "exists"));
+	std::shared_ptr<libsinsp::filter::ast::expr> macro = libsinsp::filter::ast::unary_check_expr::create("test.field", "", "exists");
 
-	std::shared_ptr<libsinsp::filter::ast::expr> filter = std::move(libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos));
+	std::shared_ptr<libsinsp::filter::ast::expr> filter = libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos);
 
 	filter_macro_resolver resolver;
 	resolver.set_macro(MACRO_NAME, macro);
@@ -102,18 +102,18 @@ TEST(MacroResolver, should_resolve_macros_on_a_filter_AST_multiple_macros)
 	libsinsp::filter::ast::pos_info a_macro_pos(11, 75, 43);
 	libsinsp::filter::ast::pos_info b_macro_pos(91, 21, 9);
 
-	std::shared_ptr<libsinsp::filter::ast::expr> a_macro = std::move(libsinsp::filter::ast::unary_check_expr::create("one.field", "", "exists"));
-	std::shared_ptr<libsinsp::filter::ast::expr> b_macro = std::move(libsinsp::filter::ast::unary_check_expr::create("another.field", "", "exists"));
+	std::shared_ptr<libsinsp::filter::ast::expr> a_macro = libsinsp::filter::ast::unary_check_expr::create("one.field", "", "exists");
+	std::shared_ptr<libsinsp::filter::ast::expr> b_macro = libsinsp::filter::ast::unary_check_expr::create("another.field", "", "exists");
 
 	std::vector<std::unique_ptr<libsinsp::filter::ast::expr>> filter_or;
 	filter_or.push_back(libsinsp::filter::ast::value_expr::create(MACRO_A_NAME, a_macro_pos));
 	filter_or.push_back(libsinsp::filter::ast::value_expr::create(MACRO_B_NAME, b_macro_pos));
-	std::shared_ptr<libsinsp::filter::ast::expr> filter = std::move(libsinsp::filter::ast::or_expr::create(filter_or));
+	std::shared_ptr<libsinsp::filter::ast::expr> filter = libsinsp::filter::ast::or_expr::create(filter_or);
 
 	std::vector<std::unique_ptr<libsinsp::filter::ast::expr>> expected_or;
 	expected_or.push_back(clone(a_macro.get()));
 	expected_or.push_back(clone(b_macro.get()));
-	std::shared_ptr<libsinsp::filter::ast::expr> expected_filter = std::move(libsinsp::filter::ast::or_expr::create(expected_or));
+	std::shared_ptr<libsinsp::filter::ast::expr> expected_filter = libsinsp::filter::ast::or_expr::create(expected_or);
 
 	filter_macro_resolver resolver;
 	resolver.set_macro(MACRO_A_NAME, a_macro);
@@ -149,17 +149,17 @@ TEST(MacroResolver, should_resolve_macros_on_a_filter_AST_nested_macros)
 	std::vector<std::unique_ptr<libsinsp::filter::ast::expr>> a_macro_and;
 	a_macro_and.push_back(libsinsp::filter::ast::unary_check_expr::create("one.field", "", "exists"));
 	a_macro_and.push_back(libsinsp::filter::ast::value_expr::create(MACRO_B_NAME, b_macro_pos));
-	std::shared_ptr<libsinsp::filter::ast::expr> a_macro = std::move(libsinsp::filter::ast::and_expr::create(a_macro_and));
+	std::shared_ptr<libsinsp::filter::ast::expr> a_macro = libsinsp::filter::ast::and_expr::create(a_macro_and);
 
-	std::shared_ptr<libsinsp::filter::ast::expr> b_macro = std::move(
-		libsinsp::filter::ast::unary_check_expr::create("another.field", "", "exists"));
+	std::shared_ptr<libsinsp::filter::ast::expr> b_macro =
+		libsinsp::filter::ast::unary_check_expr::create("another.field", "", "exists");
 
-	std::shared_ptr<libsinsp::filter::ast::expr> filter = std::move(libsinsp::filter::ast::value_expr::create(MACRO_A_NAME, a_macro_pos));
+	std::shared_ptr<libsinsp::filter::ast::expr> filter = libsinsp::filter::ast::value_expr::create(MACRO_A_NAME, a_macro_pos);
 
 	std::vector<std::unique_ptr<libsinsp::filter::ast::expr>> expected_and;
 	expected_and.push_back(libsinsp::filter::ast::unary_check_expr::create("one.field", "", "exists"));
 	expected_and.push_back(libsinsp::filter::ast::unary_check_expr::create("another.field", "", "exists"));
-	std::shared_ptr<libsinsp::filter::ast::expr> expected_filter = std::move(libsinsp::filter::ast::and_expr::create(expected_and));
+	std::shared_ptr<libsinsp::filter::ast::expr> expected_filter = libsinsp::filter::ast::and_expr::create(expected_and);
 
 	filter_macro_resolver resolver;
 	resolver.set_macro(MACRO_A_NAME, a_macro);
@@ -196,7 +196,7 @@ TEST(MacroResolver, should_find_unknown_macros)
 	std::vector<std::unique_ptr<libsinsp::filter::ast::expr>> filter_and;
 	filter_and.push_back(libsinsp::filter::ast::unary_check_expr::create("evt.name", "", "exists"));
 	filter_and.push_back(libsinsp::filter::ast::not_expr::create(libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos)));
-	std::shared_ptr<libsinsp::filter::ast::expr> filter = std::move(libsinsp::filter::ast::and_expr::create(filter_and));
+	std::shared_ptr<libsinsp::filter::ast::expr> filter = libsinsp::filter::ast::and_expr::create(filter_and);
 
 	filter_macro_resolver resolver;
 	ASSERT_FALSE(resolver.run(filter));
@@ -214,9 +214,9 @@ TEST(MacroResolver, should_find_unknown_nested_macros)
 	std::vector<std::unique_ptr<libsinsp::filter::ast::expr>> a_macro_and;
 	a_macro_and.push_back(libsinsp::filter::ast::unary_check_expr::create("one.field", "", "exists"));
 	a_macro_and.push_back(libsinsp::filter::ast::value_expr::create(MACRO_B_NAME, b_macro_pos));
-	std::shared_ptr<libsinsp::filter::ast::expr> a_macro = std::move(libsinsp::filter::ast::and_expr::create(a_macro_and));
+	std::shared_ptr<libsinsp::filter::ast::expr> a_macro = libsinsp::filter::ast::and_expr::create(a_macro_and);
 
-	std::shared_ptr<libsinsp::filter::ast::expr> filter = std::move(libsinsp::filter::ast::value_expr::create(MACRO_A_NAME, a_macro_pos));
+	std::shared_ptr<libsinsp::filter::ast::expr> filter = libsinsp::filter::ast::value_expr::create(MACRO_A_NAME, a_macro_pos);
 	auto expected_filter = clone(a_macro.get());
 
 	filter_macro_resolver resolver;
@@ -237,9 +237,9 @@ TEST(MacroResolver, should_undefine_macro)
 	libsinsp::filter::ast::pos_info macro_pos_1(12, 9, 3);
 	libsinsp::filter::ast::pos_info macro_pos_2(9, 6, 3);
 
-	std::shared_ptr<libsinsp::filter::ast::expr> macro = std::move(libsinsp::filter::ast::unary_check_expr::create("test.field", "", "exists"));
-	std::shared_ptr<libsinsp::filter::ast::expr> a_filter = std::move(libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos_1));
-	std::shared_ptr<libsinsp::filter::ast::expr> b_filter = std::move(libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos_2));
+	std::shared_ptr<libsinsp::filter::ast::expr> macro = libsinsp::filter::ast::unary_check_expr::create("test.field", "", "exists");
+	std::shared_ptr<libsinsp::filter::ast::expr> a_filter = libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos_1);
+	std::shared_ptr<libsinsp::filter::ast::expr> b_filter = libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos_2);
 	filter_macro_resolver resolver;
 
 	resolver.set_macro(MACRO_NAME, macro);
@@ -262,8 +262,8 @@ TEST(MacroResolver, should_undefine_macro)
 TEST(MacroResolver, should_clone_macro_AST)
 {
 	libsinsp::filter::ast::pos_info macro_pos(5, 2, 8888);
-	std::shared_ptr<libsinsp::filter::ast::unary_check_expr> macro = std::move(libsinsp::filter::ast::unary_check_expr::create("test.field", "", "exists"));
-	std::shared_ptr<libsinsp::filter::ast::expr> filter = std::move(libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos));
+	std::shared_ptr<libsinsp::filter::ast::unary_check_expr> macro = libsinsp::filter::ast::unary_check_expr::create("test.field", "", "exists");
+	std::shared_ptr<libsinsp::filter::ast::expr> filter = libsinsp::filter::ast::value_expr::create(MACRO_NAME, macro_pos);
 	filter_macro_resolver resolver;
 
 	resolver.set_macro(MACRO_NAME, macro);

unit_tests/engine/test_rule_loader.cpp

@@ -0,0 +1,970 @@
+#include <gtest/gtest.h>
+
+#include "falco_engine.h"
+#include "rule_loader_reader.h"
+#include "rule_loader_compiler.h"
+#include "rule_loading_messages.h"
+
+class engine_loader_test : public ::testing::Test {
+protected:
+	void SetUp() override
+	{
+		m_sample_ruleset = "sample-ruleset";
+		m_sample_source = falco_common::syscall_source;
+
+		// create a falco engine ready to load the ruleset
+		m_inspector.reset(new sinsp());
+		m_engine.reset(new falco_engine());
+		m_filter_factory = std::shared_ptr<gen_event_filter_factory>(
+			new sinsp_filter_factory(m_inspector.get(), m_filterlist));
+		m_formatter_factory = std::shared_ptr<gen_event_formatter_factory>(
+			new sinsp_evt_formatter_factory(m_inspector.get(), m_filterlist));
+		m_engine->add_source(m_sample_source, m_filter_factory, m_formatter_factory);
+	}
+
+	void TearDown() override
+	{
+
+	}
+
+	bool load_rules(std::string rules_content, std::string rules_filename)
+	{
+		bool ret = false;
+		falco::load_result::rules_contents_t rc = {{rules_filename, rules_content}};
+		m_load_result = m_engine->load_rules(rules_content, rules_filename);
+		m_load_result_string = m_load_result->as_string(true, rc);
+		m_load_result_json = m_load_result->as_json(rc);
+		ret = m_load_result->successful();
+
+		if (ret)
+		{
+			m_engine->enable_rule("", true, m_sample_ruleset);
+		}
+
+		return ret;
+	}
+
+	// This must be kept in line with the (private) falco_engine::s_default_ruleset
+	uint64_t num_rules_for_ruleset(std::string ruleset = "falco-default-ruleset")
+	{
+		return m_engine->num_rules_for_ruleset(ruleset);
+	}
+
+	bool has_warnings()
+	{
+		return m_load_result->has_warnings();
+	}
+
+	bool check_warning_message(std::string warning_msg)
+	{
+		if(!m_load_result->has_warnings())
+		{
+			return false;
+		}
+
+		for(auto &warn : m_load_result_json["warnings"])
+		{
+			std::string msg = warn["message"];
+			// Debug:
+			// printf("msg: %s\n", msg.c_str());
+			if(msg.find(warning_msg) != std::string::npos)
+			{
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	bool check_error_message(std::string error_msg)
+	{
+		// if the loading is successful there are no errors
+		if(m_load_result->successful())
+		{
+			return false;
+		}
+
+		for(auto &err : m_load_result_json["errors"])
+		{
+			std::string msg = err["message"];
+			// Debug:
+			// printf("msg: %s\n", msg.c_str());
+			if(msg.find(error_msg) != std::string::npos)
+			{
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	std::string get_compiled_rule_condition(std::string rule_name = "")
+	{
+		auto rule_description = m_engine->describe_rule(&rule_name, {});
+		return rule_description["rules"][0]["details"]["condition_compiled"].template get<std::string>();
+	}
+
+	std::string m_sample_ruleset;
+	std::string m_sample_source;
+	sinsp_filter_check_list m_filterlist;
+	std::shared_ptr<gen_event_filter_factory> m_filter_factory;
+	std::shared_ptr<gen_event_formatter_factory> m_formatter_factory;
+	std::unique_ptr<falco_engine> m_engine;
+	std::unique_ptr<falco::load_result> m_load_result;
+	std::string m_load_result_string;
+	nlohmann::json m_load_result_json;
+	std::unique_ptr<sinsp> m_inspector;
+};
+
+std::string s_sample_ruleset = "sample-ruleset";
+std::string s_sample_source = falco_common::syscall_source;
+
+TEST_F(engine_loader_test, list_append)
+{
+    std::string rules_content = R"END(
+- list: shell_binaries
+  items: [ash, bash, csh, ksh, sh, tcsh, zsh, dash]
+
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open and proc.name in (shell_binaries)
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- list: shell_binaries
+  items: [pwsh]
+  override:
+    items: append
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+	ASSERT_EQ(get_compiled_rule_condition("legit_rule"),"(evt.type = open and proc.name in (ash, bash, csh, ksh, sh, tcsh, zsh, dash, pwsh))");
+}
+
+TEST_F(engine_loader_test, condition_append)
+{
+    std::string rules_content = R"END(
+- macro: interactive
+  condition: >
+    ((proc.aname=sshd and proc.name != sshd) or
+    proc.name=systemd-logind or proc.name=login)
+
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open and interactive
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- macro: interactive
+  condition: or proc.name = ssh
+  override:
+    condition: append
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+	ASSERT_EQ(get_compiled_rule_condition("legit_rule"),"(evt.type = open and (((proc.aname = sshd and proc.name != sshd) or proc.name = systemd-logind or proc.name = login) or proc.name = ssh))");
+}
+
+TEST_F(engine_loader_test, rule_override_append)
+{
+    std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: legit_rule
+  desc: with append
+  condition: and proc.name = cat
+  output: proc=%proc.name
+  override:
+    desc: append
+    condition: append
+    output: append
+)END";
+
+	std::string rule_name = "legit_rule";
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	// Here we don't use the deprecated `append` flag, so we don't expect the warning.
+	ASSERT_FALSE(check_warning_message(WARNING_APPEND));
+
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	ASSERT_EQ(rule_description["rules"][0]["info"]["condition"].template get<std::string>(),
+	 	"evt.type=open and proc.name = cat");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["output"].template get<std::string>(),
+	 	"user=%user.name command=%proc.cmdline file=%fd.name proc=%proc.name");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["description"].template get<std::string>(),
+	 	"legit rule description with append");
+}
+
+TEST_F(engine_loader_test, rule_append)
+{
+    std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: legit_rule
+  condition: and proc.name = cat
+  append: true
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	// We should have at least one warning because the 'append' flag is deprecated.
+	ASSERT_TRUE(check_warning_message(WARNING_APPEND));
+
+	ASSERT_EQ(get_compiled_rule_condition("legit_rule"),"(evt.type = open and proc.name = cat)");
+}
+
+TEST_F(engine_loader_test, rule_override_replace)
+{
+    std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: legit_rule
+  desc: a replaced legit description
+  condition: evt.type = close
+  override:
+    desc: replace
+    condition: replace
+)END";
+
+	std::string rule_name = "legit_rule";
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	ASSERT_EQ(rule_description["rules"][0]["info"]["condition"].template get<std::string>(),
+	 	"evt.type = close");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["output"].template get<std::string>(),
+	 	"user=%user.name command=%proc.cmdline file=%fd.name");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["description"].template get<std::string>(),
+	 	"a replaced legit description");
+}
+
+TEST_F(engine_loader_test, rule_override_append_replace)
+{
+    std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: legit_rule
+  desc: a replaced legit description
+  condition: and proc.name = cat
+  priority: WARNING
+  override:
+    desc: replace
+    condition: append
+    priority: replace
+)END";
+
+	std::string rule_name = "legit_rule";
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	ASSERT_EQ(rule_description["rules"][0]["info"]["condition"].template get<std::string>(),
+	 	"evt.type = close and proc.name = cat");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["output"].template get<std::string>(),
+	 	"user=%user.name command=%proc.cmdline file=%fd.name");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["description"].template get<std::string>(),
+	 	"a replaced legit description");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["priority"].template get<std::string>(),
+	 	"Warning");
+}
+
+TEST_F(engine_loader_test, rule_incorrect_override_type)
+{
+    std::string rules_content = R"END(
+- rule: failing_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: failing_rule
+  desc: an appended incorrect field
+  condition: and proc.name = cat
+  priority: WARNING
+  override:
+    desc: replace
+    condition: append
+    priority: append
+)END";
+
+	std::string rule_name = "failing_rule";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("Key 'priority' cannot be appended to, use 'replace' instead"));
+	ASSERT_TRUE(std::string(m_load_result_json["errors"][0]["context"]["snippet"]).find("priority: append") != std::string::npos);
+}
+
+TEST_F(engine_loader_test, rule_incorrect_append_override)
+{
+    std::string rules_content = R"END(
+- rule: failing_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: failing_rule
+  desc: an appended incorrect field
+  condition: and proc.name = cat
+  append: true
+  override:
+    desc: replace
+    condition: append
+)END";
+
+	std::string rule_name = "failing_rule";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	
+	// We should have at least one warning because the 'append' flag is deprecated.
+	ASSERT_TRUE(check_warning_message(WARNING_APPEND));
+	
+	ASSERT_TRUE(check_error_message(ERROR_OVERRIDE_APPEND));
+}
+
+TEST_F(engine_loader_test, macro_override_append_before_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  override:
+    condition: append
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a macro override before the macro definition.
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_MACRO));
+}
+
+TEST_F(engine_loader_test, macro_override_replace_before_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  override:
+    condition: replace
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// The first override defines a macro that is overridden by the second macro definition
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"evt.type in (open, openat)");	
+}
+
+TEST_F(engine_loader_test, macro_append_before_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  append: true
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a macro override before the macro definition.
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_MACRO));
+}
+
+TEST_F(engine_loader_test, macro_override_append_after_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  override:
+    condition: append
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a macro override before the macro definition.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) or evt.type = openat2)");
+}
+
+TEST_F(engine_loader_test, macro_append_after_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  append: true
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a macro override before the macro definition.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) or evt.type = openat2)");
+}
+
+TEST_F(engine_loader_test, rule_override_append_before_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  condition: and proc.name = cat
+  override:
+    condition: append
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_RULE_APPEND));
+}
+
+TEST_F(engine_loader_test, rule_override_replace_before_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  condition: and proc.name = cat
+  override:
+    condition: replace
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_RULE_REPLACE));
+}
+
+TEST_F(engine_loader_test, rule_append_before_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  condition: and proc.name = cat
+  append: true
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_RULE_APPEND));
+}
+
+TEST_F(engine_loader_test, rule_override_append_after_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+- rule: test_rule
+  condition: and proc.name = cat
+  override:
+    condition: append
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) and proc.name = cat)");
+}
+
+TEST_F(engine_loader_test, rule_append_after_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+- rule: test_rule
+  condition: and proc.name = cat
+  append: true
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) and proc.name = cat)");
+}
+
+TEST_F(engine_loader_test, list_override_append_wrong_key)
+{
+	// todo: maybe we want to manage some non-existent keys
+	// Please note how the non-existent key 'non-existent keys' is ignored.
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  override_written_wrong:
+    items: append
+
+- list: dev_creation_binaries
+  items: [blkid]
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// Since there is a wrong key in the first list definition the `override` is not
+	// considered. so in this situation, we are defining the list 2 times. The 
+	// second one overrides the first one.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid))");
+}
+
+TEST_F(engine_loader_test, list_override_append_before_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  override:
+    items: append
+
+- list: dev_creation_binaries
+  items: [blkid]
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a list override before the list definition.
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_LIST));
+}
+
+TEST_F(engine_loader_test, list_override_replace_before_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  override:
+    items: replace
+
+- list: dev_creation_binaries
+  items: [blkid]
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// With override replace we define a first list that then is overridden by the second one.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid))");
+}
+
+TEST_F(engine_loader_test, list_append_before_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  append: true
+
+- list: dev_creation_binaries
+  items: [blkid]
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a list append before the list definition.
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_LIST));
+}
+
+TEST_F(engine_loader_test, list_override_append_after_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: [blkid]
+
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  override:
+    items: append
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid, csi-provisioner, csi-attacher))");
+}
+
+TEST_F(engine_loader_test, list_append_after_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: [blkid]
+
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  append: true
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid, csi-provisioner, csi-attacher))");
+}
+
+TEST_F(engine_loader_test, rule_override_without_field)
+{
+    std::string rules_content = R"END(
+- rule: failing_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: failing_rule
+  desc: an appended incorrect field
+  override:
+    desc: replace
+    condition: append
+)END";
+
+	std::string rule_name = "failing_rule";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("An append override for 'condition' was specified but 'condition' is not defined"));
+}
+
+TEST_F(engine_loader_test, rule_override_extra_field)
+{
+    std::string rules_content = R"END(
+- rule: failing_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: failing_rule
+  desc: an appended incorrect field
+  condition: and proc.name = cat
+  priority: WARNING
+  override:
+    desc: replace
+    condition: append
+)END";
+
+	std::string rule_name = "failing_rule";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("Unexpected key 'priority'"));
+}
+
+TEST_F(engine_loader_test, missing_enabled_key_with_override)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  desc: missing enabled key
+  condition: and proc.name = cat
+  override:
+    desc: replace
+    condition: append
+    enabled: replace
+)END";
+
+	// In the rule override we miss `enabled: true`
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("'enabled' was specified but 'enabled' is not defined"));
+}
+
+TEST_F(engine_loader_test, rule_override_with_enabled)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  desc: correct override
+  condition: and proc.name = cat
+  enabled: true
+  override:
+    desc: replace
+    condition: append
+    enabled: replace
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_FALSE(has_warnings());
+	// The rule should be enabled at the end.
+	EXPECT_EQ(num_rules_for_ruleset(), 1);
+}
+
+TEST_F(engine_loader_test, rule_not_enabled)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: rule not enabled
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_FALSE(has_warnings());
+	EXPECT_EQ(num_rules_for_ruleset(), 0);
+}
+
+TEST_F(engine_loader_test, rule_enabled_warning)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  enabled: true
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_warning_message(WARNING_ENABLED));
+	// The rule should be enabled at the end.
+	EXPECT_EQ(num_rules_for_ruleset(), 1);
+}
+
+// todo!: Probably we shouldn't allow this syntax
+TEST_F(engine_loader_test, rule_enabled_is_ignored_by_append)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  condition: and proc.name = cat
+  append: true
+  enabled: true
+)END";
+
+	// 'enabled' is ignored by the append, this syntax is not supported
+	// so the rule is not enabled.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	EXPECT_EQ(num_rules_for_ruleset(), 0);
+}
+
+// todo!: Probably we shouldn't allow this syntax
+TEST_F(engine_loader_test, rewrite_rule)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  desc: redefined rule syntax
+  condition: proc.name = cat
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: WARNING
+  enabled: true
+)END";
+
+	// The above syntax is not supported, we cannot override the content
+	// of a rule in this way.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	// In this case the rule is completely overridden but this syntax is not supported.
+	EXPECT_EQ(num_rules_for_ruleset(), 1);
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"proc.name = cat");
+}
+
+TEST_F(engine_loader_test, required_engine_version_semver)
+{
+    std::string rules_content = R"END(
+- required_engine_version: 0.26.0
+
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_FALSE(has_warnings());
+}
+
+TEST_F(engine_loader_test, required_engine_version_not_semver)
+{
+    std::string rules_content = R"END(
+- required_engine_version: 26
+
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_FALSE(has_warnings());
+}
+
+TEST_F(engine_loader_test, required_engine_version_invalid)
+{
+    std::string rules_content = R"END(
+- required_engine_version: seven
+
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+)END";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("Unable to parse engine version"));
+}
+
+// checks for issue described in https://github.com/falcosecurity/falco/pull/3028
+TEST_F(engine_loader_test, list_value_with_escaping)
+{
+    std::string rules_content = R"END(
+- list: my_list
+  items: [non_escaped_val, "escaped val"]
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(m_load_result->successful());
+  ASSERT_TRUE(m_load_result->has_warnings()); // a warning for the unused list
+
+  auto rule_description = m_engine->describe_rule(nullptr, {});
+  ASSERT_TRUE(m_load_result->successful());
+  ASSERT_EQ(rule_description["rules"].size(), 0);
+  ASSERT_EQ(rule_description["macros"].size(), 0);
+  ASSERT_EQ(rule_description["lists"].size(), 1);
+
+  // escaped values must not be interpreted as list refs by mistake
+  ASSERT_EQ(rule_description["lists"][0]["details"]["lists"].size(), 0);
+
+  // values should be escaped correctly
+  ASSERT_EQ(rule_description["lists"][0]["details"]["items_compiled"].size(), 2);
+  ASSERT_EQ(rule_description["lists"][0]["details"]["items_compiled"][0].template get<std::string>(), "non_escaped_val");
+  ASSERT_EQ(rule_description["lists"][0]["details"]["items_compiled"][1].template get<std::string>(), "escaped val");
+}

unit_tests/falco/test_configuration.cpp

@@ -114,17 +114,35 @@ TEST(Configuration, configuration_environment_variables)
     // Set an environment variable for testing purposes
     std::string env_var_value = "envVarValue";
     std::string env_var_name = "ENV_VAR";
-    std::string default_value = "default";
     SET_ENV_VAR(env_var_name.c_str(), env_var_value.c_str());
-    yaml_helper conf;
 
-    std::string sample_yaml =
+    std::string embedded_env_var_value = "${ENV_VAR}";
+    std::string embedded_env_var_name = "ENV_VAR_EMBEDDED";
+    SET_ENV_VAR(embedded_env_var_name.c_str(), embedded_env_var_value.c_str());
+
+    std::string bool_env_var_value = "true";
+    std::string bool_env_var_name = "ENV_VAR_BOOL";
+    SET_ENV_VAR(bool_env_var_name.c_str(), bool_env_var_value.c_str());
+
+    std::string int_env_var_value = "12";
+    std::string int_env_var_name = "ENV_VAR_INT";
+    SET_ENV_VAR(int_env_var_name.c_str(), int_env_var_value.c_str());
+
+    std::string empty_env_var_value = "";
+    std::string empty_env_var_name = "ENV_VAR_EMPTY";
+    SET_ENV_VAR(empty_env_var_name.c_str(), empty_env_var_value.c_str());
+
+    std::string default_value = "default";
+    std::string env_var_sample_yaml =
         "base_value:\n"
         "    id: $ENV_VAR\n"
         "    name: '${ENV_VAR}'\n"
         "    string: my_string\n"
         "    invalid: $${ENV_VAR}\n"
-		"    invalid_env: $$ENV_VAR\n"
+	"    invalid_env: $$ENV_VAR\n"
+	"    invalid_double_env: $${ENV_VAR}$${ENV_VAR}\n"
+	"    invalid_embedded_env: $${${ENV_VAR}}\n"
+	"    invalid_valid_env: $${ENV_VAR}${ENV_VAR}\n"
         "    escaped: \"${ENV_VAR}\"\n"
         "    subvalue:\n"
         "        subvalue2:\n"
@@ -133,52 +151,135 @@ TEST(Configuration, configuration_environment_variables)
         "    sample_list:\n"
         "        - ${ENV_VAR}\n"
         "        - ' ${ENV_VAR}'\n"
-        "        - $UNSED_XX_X_X_VAR\n";
-    conf.load_from_string(sample_yaml);
+	"        - '${ENV_VAR} '\n"
+        "        - $UNSED_XX_X_X_VAR\n"
+        "paths:\n"
+	"    - ${ENV_VAR}/foo\n"
+	"    - $ENV_VAR/foo\n"
+	"    - /foo/${ENV_VAR}/\n"
+	"    - /${ENV_VAR}/${ENV_VAR}${ENV_VAR}/foo\n"
+	"    - ${ENV_VAR_EMBEDDED}/foo\n"
+	"is_test: ${ENV_VAR_BOOL}\n"
+	"num_test: ${ENV_VAR_INT}\n"
+	"empty_test: ${ENV_VAR_EMPTY}\n"
+	"plugins:\n"
+	"  - name: k8saudit\n"
+	"    library_path: /foo/${ENV_VAR}/libk8saudit.so\n"
+	"    open_params: ${ENV_VAR_INT}\n";
+
+    yaml_helper conf;
+    conf.load_from_string(env_var_sample_yaml);
 
     /* Check if the base values are defined */
     ASSERT_TRUE(conf.is_defined("base_value"));
     ASSERT_TRUE(conf.is_defined("base_value_2"));
+    ASSERT_TRUE(conf.is_defined("paths"));
     ASSERT_FALSE(conf.is_defined("unknown_base_value"));
 
     /* Test fetching of a regular string without any environment variable */
-    std::string base_value_string = conf.get_scalar<std::string>("base_value.string", default_value);
+    auto base_value_string = conf.get_scalar<std::string>("base_value.string", default_value);
     ASSERT_EQ(base_value_string, "my_string");
 
     /* Test fetching of escaped environment variable format. Should return the string as-is after stripping the leading `$` */
-    std::string base_value_invalid = conf.get_scalar<std::string>("base_value.invalid", default_value);
+    auto base_value_invalid = conf.get_scalar<std::string>("base_value.invalid", default_value);
     ASSERT_EQ(base_value_invalid, "${ENV_VAR}");
 
-	/* Test fetching of invalid escaped environment variable format. Should return the string as-is */
-    std::string base_value_invalid_env = conf.get_scalar<std::string>("base_value.invalid_env", default_value);
+    /* Test fetching of invalid escaped environment variable format. Should return the string as-is */
+    auto base_value_invalid_env = conf.get_scalar<std::string>("base_value.invalid_env", default_value);
     ASSERT_EQ(base_value_invalid_env, "$$ENV_VAR");
 
+    /* Test fetching of 2 escaped environment variables side by side. Should return the string as-is after stripping the leading `$` */
+    auto base_value_double_invalid = conf.get_scalar<std::string>("base_value.invalid_double_env", default_value);
+    ASSERT_EQ(base_value_double_invalid, "${ENV_VAR}${ENV_VAR}");
+
+    /*
+     * Test fetching of escaped environment variable format with inside an env variable.
+     * Should return the string as-is after stripping the leading `$` with the resolved env variable within
+     */
+    auto base_value_embedded_invalid = conf.get_scalar<std::string>("base_value.invalid_embedded_env", default_value);
+    ASSERT_EQ(base_value_embedded_invalid, "${" + env_var_value + "}");
+
+    /*
+     * Test fetching of an escaped env variable plus an env variable side by side.
+     * Should return the escaped one trimming the leading `$` plus the second one resolved.
+     */
+    auto base_value_valid_invalid = conf.get_scalar<std::string>("base_value.invalid_valid_env", default_value);
+    ASSERT_EQ(base_value_valid_invalid, "${ENV_VAR}" + env_var_value);
+
     /* Test fetching of strings that contain environment variables */
-    std::string base_value_id = conf.get_scalar<std::string>("base_value.id", default_value);
+    auto base_value_id = conf.get_scalar<std::string>("base_value.id", default_value);
     ASSERT_EQ(base_value_id, "$ENV_VAR"); // Does not follow the `${VAR}` format, so it should be treated as a regular string
 
-    std::string base_value_name = conf.get_scalar<std::string>("base_value.name", default_value);
+    auto base_value_name = conf.get_scalar<std::string>("base_value.name", default_value);
     ASSERT_EQ(base_value_name, env_var_value); // Proper environment variable format
 
-    std::string base_value_escaped = conf.get_scalar<std::string>("base_value.escaped", default_value);
+    auto base_value_escaped = conf.get_scalar<std::string>("base_value.escaped", default_value);
     ASSERT_EQ(base_value_escaped, env_var_value); // Environment variable within quotes
 
-    /* Test fetching of an undefined environment variable. Expected to return the default value.*/
-    std::string unknown_boolean = conf.get_scalar<std::string>("base_value.subvalue.subvalue2.boolean", default_value);
-    ASSERT_EQ(unknown_boolean, default_value);
+    /* Test fetching of an undefined environment variable. Resolves to empty string. */
+    auto unknown_boolean = conf.get_scalar<std::string>("base_value.subvalue.subvalue2.boolean", default_value);
+    ASSERT_EQ(unknown_boolean, "");
 
     /* Test fetching of environment variables from a list */
-    std::string base_value_2_list_0 = conf.get_scalar<std::string>("base_value_2.sample_list[0]", default_value);
+    auto base_value_2_list_0 = conf.get_scalar<std::string>("base_value_2.sample_list[0]", default_value);
     ASSERT_EQ(base_value_2_list_0, env_var_value); // Proper environment variable format
 
-    std::string base_value_2_list_1 = conf.get_scalar<std::string>("base_value_2.sample_list[1]", default_value);
-    ASSERT_EQ(base_value_2_list_1, " ${ENV_VAR}"); // Environment variable preceded by a space, hence treated as a regular string
+    auto base_value_2_list_1 = conf.get_scalar<std::string>("base_value_2.sample_list[1]", default_value);
+    ASSERT_EQ(base_value_2_list_1, " " + env_var_value); // Environment variable preceded by a space, still extracted env var with leading space
 
-    std::string base_value_2_list_2 = conf.get_scalar<std::string>("base_value_2.sample_list[2]", default_value);
-    ASSERT_EQ(base_value_2_list_2, "$UNSED_XX_X_X_VAR"); // Does not follow the `${VAR}` format, so should be treated as a regular string
+    auto base_value_2_list_2 = conf.get_scalar<std::string>("base_value_2.sample_list[2]", default_value);
+    ASSERT_EQ(base_value_2_list_2, env_var_value + " "); // Environment variable followed by a space, still extracted env var with trailing space
 
-    /* Clear the set environment variable after testing */
-    SET_ENV_VAR(env_var_name.c_str(), env_var_value.c_str());
+    auto base_value_2_list_3 = conf.get_scalar<std::string>("base_value_2.sample_list[3]", default_value);
+    ASSERT_EQ(base_value_2_list_3, "$UNSED_XX_X_X_VAR"); // Does not follow the `${VAR}` format, so should be treated as a regular string
+
+    /* Test expansion of environment variables within strings */
+    auto path_list_0 = conf.get_scalar<std::string>("paths[0]", default_value);
+    ASSERT_EQ(path_list_0, env_var_value + "/foo"); // Even if env var is part of bigger string, it gets expanded
+
+    auto path_list_1 = conf.get_scalar<std::string>("paths[1]", default_value);
+    ASSERT_EQ(path_list_1, "$ENV_VAR/foo"); // Does not follow the `${VAR}` format, so should be treated as a regular string
+
+    auto path_list_2 = conf.get_scalar<std::string>("paths[2]", default_value);
+    ASSERT_EQ(path_list_2, "/foo/" + env_var_value + "/"); // Even when env var is in the middle of a string. it gets expanded
+
+    auto path_list_3 = conf.get_scalar<std::string>("paths[3]", default_value);
+    ASSERT_EQ(path_list_3, "/" + env_var_value + "/" + env_var_value + env_var_value + "/foo"); // Even when the string contains multiple env vars they are correctly expanded
+
+    auto path_list_4 = conf.get_scalar<std::string>("paths[4]", default_value);
+    ASSERT_EQ(path_list_4, env_var_value + "/foo"); // Even when the env var contains another env var, it gets correctly double-expanded
+
+    /* Check that variable expansion is type-aware */
+    auto boolean = conf.get_scalar<bool>("is_test", false);
+    ASSERT_EQ(boolean, true); // `true` can be parsed to bool.
+
+    auto boolean_as_str = conf.get_scalar<std::string>("is_test", "false");
+    ASSERT_EQ(boolean_as_str, "true"); // `true` can be parsed to string.
+
+    auto boolean_as_int = conf.get_scalar<int32_t>("is_test", 0);
+    ASSERT_EQ(boolean_as_int, 0); // `true` cannot be parsed to integer.
+
+    auto integer = conf.get_scalar<int32_t>("num_test", -1);
+    ASSERT_EQ(integer, 12);
+
+    // An env var that resolves to an empty string returns ""
+    auto empty_default_str = conf.get_scalar<std::string>("empty_test", default_value);
+    ASSERT_EQ(empty_default_str, "");
+
+    std::list<falco_configuration::plugin_config> plugins;
+    conf.get_sequence<std::list<falco_configuration::plugin_config>>(plugins, std::string("plugins"));
+    std::vector<falco_configuration::plugin_config> m_plugins{ std::make_move_iterator(std::begin(plugins)),
+							      std::make_move_iterator(std::end(plugins)) };
+    ASSERT_EQ(m_plugins[0].m_name, "k8saudit");
+    ASSERT_EQ(m_plugins[0].m_library_path, "/foo/" + env_var_value + "/libk8saudit.so");
+    ASSERT_EQ(m_plugins[0].m_open_params, "12");
+
+    /* Clear the set environment variables after testing */
+    SET_ENV_VAR(env_var_name.c_str(), "");
+    SET_ENV_VAR(embedded_env_var_name.c_str(), "");
+    SET_ENV_VAR(bool_env_var_name.c_str(), "");
+    SET_ENV_VAR(int_env_var_name.c_str(), "");
+    SET_ENV_VAR(empty_env_var_name.c_str(), "");
 }
 
 TEST(Configuration, configuration_webserver_ip)

userspace/engine/CMakeLists.txt

@@ -11,7 +11,7 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
-set(FALCO_ENGINE_SOURCE_FILES
+add_library(falco_engine STATIC
     falco_common.cpp
     falco_engine.cpp
     falco_load_result.cpp
@@ -25,9 +25,8 @@ set(FALCO_ENGINE_SOURCE_FILES
     rule_loader.cpp
     rule_loader_reader.cpp
     rule_loader_collector.cpp
-    rule_loader_compiler.cpp)
-
-add_library(falco_engine STATIC ${FALCO_ENGINE_SOURCE_FILES})
+    rule_loader_compiler.cpp
+)
 
 if (EMSCRIPTEN)
 	target_compile_options(falco_engine PRIVATE "-sDISABLE_EXCEPTION_CATCHING=0")
@@ -35,14 +34,17 @@ endif()
 
 add_dependencies(falco_engine yamlcpp njson)
 
-target_include_directories(
-    falco_engine
-    PUBLIC
-      "${NJSON_INCLUDE}"
-      "${TBB_INCLUDE_DIR}"
-      "${LIBSCAP_INCLUDE_DIRS}"
-      "${LIBSINSP_INCLUDE_DIRS}"
-      "${YAMLCPP_INCLUDE_DIR}"
-      "${PROJECT_BINARY_DIR}/userspace/engine")
+target_include_directories(falco_engine
+PUBLIC
+    ${LIBSCAP_INCLUDE_DIRS}
+    ${LIBSINSP_INCLUDE_DIRS}
+    ${PROJECT_BINARY_DIR}/userspace/engine
+    ${nlohmann_json_INCLUDE_DIRS}
+    ${TBB_INCLUDE_DIR}
+    ${YAMLCPP_INCLUDE_DIR}
+)
 
-target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${YAMLCPP_LIB}")
+target_link_libraries(falco_engine
+    ${FALCO_SINSP_LIBRARY}
+    ${YAMLCPP_LIB}
+)

userspace/engine/falco_engine.cpp

@@ -79,26 +79,6 @@ sinsp_version falco_engine::engine_version()
 	return sinsp_version(FALCO_ENGINE_VERSION);
 }
 
-const falco_source* falco_engine::find_source(const std::string& name) const
-{
-	auto ret = m_sources.at(name);
-	if(!ret)
-	{
-		throw falco_exception("Unknown event source " + name);
-	}
-	return ret;
-}
-
-const falco_source* falco_engine::find_source(std::size_t index) const
-{
-	auto ret = m_sources.at(index);
-	if(!ret)
-	{
-		throw falco_exception("Unknown event source index " + std::to_string(index));
-	}
-	return ret;
-}
-
 // Return a key that uniquely represents a field class.
 // For now, we assume name + shortdesc is unique.
 static std::string fieldclass_key(const gen_event_filter_factory::filter_fieldclass_info &fld_info)
@@ -177,15 +157,6 @@ void falco_engine::list_fields(std::string &source, bool verbose, bool names_onl
 	}
 }
 
-void falco_engine::load_rules(const std::string &rules_content, bool verbose, bool all_events)
-{
-	static const std::string no_name = "N/A";
-
-	std::unique_ptr<load_result> res = load_rules(rules_content, no_name);
-
-	interpret_load_result(res, no_name, rules_content, verbose);
-}
-
 std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_content, const std::string &name)
 {
 	rule_loader::configuration cfg(rules_content, m_sources, name);
@@ -195,7 +166,7 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 	// read rules YAML file and collect its definitions
 	rule_loader::reader reader;
 	if (reader.read(cfg, m_rule_collector))
-	{	
+	{
 		// compile the definitions (resolve macro/list refs, exceptions, ...)
 		m_last_compile_output = std::make_unique<rule_loader::compiler::compile_output>();
 		rule_loader::compiler().compile(cfg, m_rule_collector, *m_last_compile_output.get());
@@ -257,47 +228,15 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 	return std::move(cfg.res);
 }
 
-void falco_engine::load_rules_file(const std::string &rules_filename, bool verbose, bool all_events)
-{
-	std::string rules_content;
-
-	read_file(rules_filename, rules_content);
-
-	std::unique_ptr<load_result> res = load_rules(rules_content, rules_filename);
-
-	interpret_load_result(res, rules_filename, rules_content, verbose);
-}
-
-std::unique_ptr<load_result> falco_engine::load_rules_file(const std::string &rules_filename)
-{
-	std::string rules_content;
-
-	try {
-		read_file(rules_filename, rules_content);
-	}
-	catch (falco_exception &e)
-	{
-		rule_loader::context ctx(rules_filename);
-
-		std::unique_ptr<rule_loader::result> res(new rule_loader::result(rules_filename));
-
-		res->add_error(load_result::LOAD_ERR_FILE_READ, e.what(), ctx);
-
-// Old gcc versions (e.g. 4.8.3) won't allow move elision but newer versions
-// (e.g. 10.2.1) would complain about the redundant move.
-#if __GNUC__ > 4
-		return res;
-#else
-		return std::move(res);
-#endif
-	}
-
-	return load_rules(rules_content, rules_filename);
-}
-
 void falco_engine::enable_rule(const std::string &substring, bool enabled, const std::string &ruleset)
 {
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
+
+	enable_rule(substring, enabled, ruleset_id);
+}
+
+void falco_engine::enable_rule(const std::string &substring, bool enabled, const uint16_t ruleset_id)
+{
 	bool match_exact = false;
 
 	for(const auto &it : m_sources)
@@ -316,6 +255,12 @@ void falco_engine::enable_rule(const std::string &substring, bool enabled, const
 void falco_engine::enable_rule_exact(const std::string &rule_name, bool enabled, const std::string &ruleset)
 {
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
+
+	enable_rule_exact(rule_name, enabled, ruleset_id);
+}
+
+void falco_engine::enable_rule_exact(const std::string &rule_name, bool enabled, const uint16_t ruleset_id)
+{
 	bool match_exact = true;
 
 	for(const auto &it : m_sources)
@@ -335,6 +280,11 @@ void falco_engine::enable_rule_by_tag(const std::set<std::string> &tags, bool en
 {
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
 
+	enable_rule_by_tag(tags, enabled, ruleset_id);
+}
+
+void falco_engine::enable_rule_by_tag(const std::set<std::string> &tags, bool enabled, const uint16_t ruleset_id)
+{
 	for(const auto &it : m_sources)
 	{
 		if(enabled)
@@ -384,7 +334,7 @@ libsinsp::events::set<ppm_sc_code> falco_engine::sc_codes_for_ruleset(const std:
 {
 	return find_source(source)->ruleset->enabled_sc_codes(find_ruleset_id(ruleset));
 }
-	
+
 libsinsp::events::set<ppm_event_code> falco_engine::event_codes_for_ruleset(const std::string &source, const std::string &ruleset)
 {
 	return find_source(source)->ruleset->enabled_event_codes(find_ruleset_id(ruleset));
@@ -405,21 +355,7 @@ std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_ev
 	// source_idx, which means that at any time each filter_ruleset will only
 	// be accessed by a single thread.
 
-	const falco_source *source;
-
-	if(source_idx == m_syscall_source_idx)
-	{
-		if(m_syscall_source == NULL)
-		{
-			m_syscall_source = find_source(m_syscall_source_idx);
-		}
-
-		source = m_syscall_source;
-	}
-	else
-	{
-		source = find_source(source_idx);
-	}
+	const falco_source *source = find_source(source_idx);
 
 	if(should_drop_evt() || !source)
 	{
@@ -505,122 +441,88 @@ std::size_t falco_engine::add_source(const std::string &source,
 	return m_sources.insert(src, source);
 }
 
-template <typename T> inline Json::Value sequence_to_json_array(const T& seq)
+template <typename T> inline nlohmann::json sequence_to_json_array(const T& seq)
 {
-	Json::Value ret = Json::arrayValue;
-	for (auto it = seq.begin(); it != seq.end(); it++)
+	nlohmann::json ret = nlohmann::json::array();
+	for (const auto& v : seq)
 	{
-		ret.append(*it);
+		ret.push_back(v);
 	}
 	return ret;
 }
 
-void falco_engine::describe_rule(std::string *rule, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins, bool json) const
+nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
 	// use previously-loaded collector definitions and the compiled
 	// output of rules, macros, and lists.
 	if (m_last_compile_output == nullptr)
 	{
-		throw falco_exception("rules most be loaded before describing them");
-	}
-
-	if(!json)
-	{
-		static const char *rule_fmt = "%-50s %s\n";
-		fprintf(stdout, rule_fmt, "Rule", "Description");
-		fprintf(stdout, rule_fmt, "----", "-----------");
-		if(!rule)
-		{
-			for(auto &r : m_rules)
-			{
-				auto str = falco::utils::wrap_text(r.description, 51, 110) + "\n";
-				fprintf(stdout, rule_fmt, r.name.c_str(), str.c_str());
-			}
-		}
-		else
-		{
-			auto r = m_rules.at(*rule);
-			if(r == nullptr)
-			{
-				return;
-			}
-			auto str = falco::utils::wrap_text(r->description, 51, 110) + "\n";
-			fprintf(stdout, rule_fmt, r->name.c_str(), str.c_str());
-		}
-
-		return;
+		throw falco_exception("rules must be loaded before describing them");
 	}
 
 	// use collected and compiled info to print a json output
-	Json::FastWriter writer;
-	std::string json_str;
+	nlohmann::json output;
 	if(!rule)
 	{
-		// In this case we build json information about
-		// all rules, macros and lists
-		Json::Value output;
-
 		// Store required engine version
 		auto required_engine_version = m_rule_collector.required_engine_version();
 		output["required_engine_version"] = required_engine_version.version.as_string();
 
 		// Store required plugin versions
-		Json::Value plugin_versions = Json::arrayValue;
+		nlohmann::json plugin_versions = nlohmann::json::array();
 		auto required_plugin_versions = m_rule_collector.required_plugin_versions();
 		for(const auto& req : required_plugin_versions)
 		{
-			Json::Value r;
+			nlohmann::json r;
 			r["name"] = req.at(0).name;
 			r["version"] = req.at(0).version;
 
-			Json::Value alternatives = Json::arrayValue;
+			nlohmann::json alternatives = nlohmann::json::array();
 			for(size_t i = 1; i < req.size(); i++)
 			{
-				Json::Value alternative;
+				nlohmann::json alternative;
 				alternative["name"] = req[i].name;
 				alternative["version"] = req[i].version;
-				alternatives.append(alternative);
+				alternatives.push_back(std::move(alternative));
 			}
-			r["alternatives"] = alternatives;
-			
-			plugin_versions.append(r);
+			r["alternatives"] = std::move(alternatives);
+
+			plugin_versions.push_back(std::move(r));
 		}
-		output["required_plugin_versions"] = plugin_versions;
+		output["required_plugin_versions"] = std::move(plugin_versions);
 
 		// Store information about rules
-		Json::Value rules_array = Json::arrayValue;
+		nlohmann::json rules_array = nlohmann::json::array();
 		for(const auto& r : m_last_compile_output->rules)
 		{
 			auto info = m_rule_collector.rules().at(r.name);
-			Json::Value rule;
+			nlohmann::json rule;
 			get_json_details(rule, r, *info, plugins);
-			rules_array.append(rule);
+			rules_array.push_back(std::move(rule));
 		}
-		output["rules"] = rules_array;
-		
+		output["rules"] = std::move(rules_array);
+
 		// Store information about macros
-		Json::Value macros_array = Json::arrayValue;
+		nlohmann::json macros_array = nlohmann::json::array();
 		for(const auto &m : m_last_compile_output->macros)
 		{
 			auto info = m_rule_collector.macros().at(m.name);
-			Json::Value macro;
+			nlohmann::json macro;
 			get_json_details(macro, m, *info, plugins);
-			macros_array.append(macro);
+			macros_array.push_back(std::move(macro));
 		}
-		output["macros"] = macros_array;
+		output["macros"] = std::move(macros_array);
 
-		// Store information about lists 
-		Json::Value lists_array = Json::arrayValue;
+		// Store information about lists
+		nlohmann::json lists_array = nlohmann::json::array();
 		for(const auto &l : m_last_compile_output->lists)
 		{
 			auto info = m_rule_collector.lists().at(l.name);
-			Json::Value list;
+			nlohmann::json list;
 			get_json_details(list, l, *info, plugins);
-			lists_array.append(list);			
+			lists_array.push_back(std::move(list));
 		}
-		output["lists"] = lists_array;
-
-		json_str = writer.write(output);
+		output["lists"] = std::move(lists_array);
 	}
 	else
 	{
@@ -631,21 +533,24 @@ void falco_engine::describe_rule(std::string *rule, const std::vector<std::share
 			throw falco_exception("Rule \"" + *rule + "\" is not loaded");
 		}
 		auto r = m_rules.at(ri->name);
-		Json::Value rule; 
+
+		nlohmann::json rule;
 		get_json_details(rule, *r, *ri, plugins);
-		json_str = writer.write(rule);
+		nlohmann::json rules_array = nlohmann::json::array();
+		rules_array.push_back(std::move(rule));
+		output["rules"] = std::move(rules_array);
 	}
 
-	fprintf(stdout, "%s", json_str.c_str());
+	return output;
 }
 
 void falco_engine::get_json_details(
-	Json::Value &out,
+	nlohmann::json &out,
 	const falco_rule &r,
 	const rule_loader::rule_info &info,
 	const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
-	Json::Value rule_info;
+	nlohmann::json rule_info;
 
 	// Fill general rule information
 	rule_info["name"] = r.name;
@@ -656,7 +561,7 @@ void falco_engine::get_json_details(
 	rule_info["enabled"] = info.enabled;
 	rule_info["source"] = r.source;
 	rule_info["tags"] = sequence_to_json_array(info.tags);
-	out["info"] = rule_info;
+	out["info"] = std::move(rule_info);
 
 	// Parse rule condition and build the non-compiled AST
 	// Assumption: no error because rules have already been loaded.
@@ -665,7 +570,7 @@ void falco_engine::get_json_details(
 	// get details related to the condition's filter
 	filter_details details;
 	filter_details compiled_details;
-	Json::Value json_details;
+	nlohmann::json json_details;
 	for(const auto &m : m_rule_collector.macros())
 	{
 		details.known_macros.insert(m.name);
@@ -720,15 +625,15 @@ void falco_engine::get_json_details(
 		else
 		{
 			exception_operators.insert(e.comps.item);
-		}	
+		}
 	}
 	out["details"]["exception_names"] = sequence_to_json_array(exception_names);
 	out["details"]["exception_operators"] = sequence_to_json_array(exception_operators);
 
 	// Store event types
-	Json::Value events;
+	nlohmann::json events;
 	get_json_evt_types(events, info.source, r.condition.get());
-	out["details"]["events"] = events;
+	out["details"]["events"] = std::move(events);
 
 	// Store compiled condition and output
 	out["details"]["condition_compiled"] = libsinsp::filter::ast::as_string(r.condition.get());
@@ -739,25 +644,25 @@ void falco_engine::get_json_details(
 	// - The fields used in the rule's condition, output, and exceptions
 	// - The evt types used in the rule's condition checks, that can potentially
 	//   match plugin-provided async events
-	Json::Value used_plugins;
+	nlohmann::json used_plugins;
 	// note: making a union of conditions's and output's fields
 	// note: the condition's AST accounts for all the resolved refs and exceptions
 	compiled_details.fields.insert(out_fields.begin(), out_fields.end());
 	get_json_used_plugins(used_plugins, info.source, compiled_details.evtnames, compiled_details.fields, plugins);
-	out["details"]["plugins"] = used_plugins;
+	out["details"]["plugins"] = std::move(used_plugins);
 }
 
 void falco_engine::get_json_details(
-	Json::Value& out,
+	nlohmann::json& out,
 	const falco_macro& m,
 	const rule_loader::macro_info& info,
 	const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
-	Json::Value macro_info;
+	nlohmann::json macro_info;
 
 	macro_info["name"] = m.name;
 	macro_info["condition"] = info.cond;
-	out["info"] = macro_info;
+	out["info"] = std::move(macro_info);
 
 	// Parse the macro condition and build the non-compiled AST
 	// Assumption: no exception because rules have already been loaded.
@@ -766,7 +671,7 @@ void falco_engine::get_json_details(
 	// get details related to the condition's filter
 	filter_details details;
 	filter_details compiled_details;
-	Json::Value json_details;
+	nlohmann::json json_details;
 	for(const auto &m : m_rule_collector.macros())
 	{
 		details.known_macros.insert(m.name);
@@ -787,9 +692,9 @@ void falco_engine::get_json_details(
 	out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields);
 
 	// Store event types
-	Json::Value events;
+	nlohmann::json events;
 	get_json_evt_types(events, "", m.condition.get());
-	out["details"]["events"] = events;
+	out["details"]["events"] = std::move(events);
 
 	// Store compiled condition
 	out["details"]["condition_compiled"] = libsinsp::filter::ast::as_string(m.condition.get());
@@ -800,20 +705,20 @@ void falco_engine::get_json_details(
 	// if a macro uses a plugin's field, we can't be sure which plugin actually
 	// is used until we resolve the macro ref in a rule providing a source for
 	// disambiguation.
-	out["details"]["plugins"] = Json::arrayValue;
+	out["details"]["plugins"] = nlohmann::json::array();
 }
 
 void falco_engine::get_json_details(
-	Json::Value& out,
+	nlohmann::json& out,
 	const falco_list& l,
 	const rule_loader::list_info& info,
 	const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const
 {
-	Json::Value list_info;
+	nlohmann::json list_info;
 	list_info["name"] = l.name;
 
 	// note: the syntactic definitions still has the list refs unresolved
-	Json::Value items = Json::arrayValue;
+	nlohmann::json items = nlohmann::json::array();
 	std::unordered_set<std::string> lists;
 	for(const auto &i : info.items)
 	{
@@ -825,19 +730,19 @@ void falco_engine::get_json_details(
 			lists.insert(i);
 			continue;
 		}
-		items.append(i);
+		items.push_back(std::move(i));
 	}
 
-	list_info["items"] = items;
-	out["info"] = list_info;
+	list_info["items"] = std::move(items);
+	out["info"] = std::move(list_info);
 	out["details"]["used"] = l.used;
 	out["details"]["lists"] = sequence_to_json_array(lists);
 	out["details"]["items_compiled"] = sequence_to_json_array(l.items);
-	out["details"]["plugins"] = Json::arrayValue; // always empty
+	out["details"]["plugins"] = nlohmann::json::array(); // always empty
 }
 
 void falco_engine::get_json_evt_types(
-	Json::Value& out,
+	nlohmann::json& out,
 	const std::string& source,
 	libsinsp::filter::ast::expr* ast) const
 {
@@ -860,7 +765,7 @@ void falco_engine::get_json_evt_types(
 }
 
 void falco_engine::get_json_used_plugins(
-	Json::Value& out,
+	nlohmann::json& out,
 	const std::string& source,
 	const std::unordered_set<std::string>& evtnames,
 	const std::unordered_set<std::string>& fields,
@@ -869,14 +774,17 @@ void falco_engine::get_json_used_plugins(
 	// note: condition and output fields may have an argument, so
 	// we need to isolate the field names
 	std::unordered_set<std::string> fieldnames;
-	for (auto f: fields)
+	for (const auto &f: fields)
 	{
 		auto argpos = f.find('[');
 		if (argpos != std::string::npos)
 		{
-			f = f.substr(0, argpos);
+			fieldnames.insert(f.substr(0, argpos));
+		}
+		else
+		{
+			fieldnames.insert(f);
 		}
-		fieldnames.insert(f);
 	}
 
 	std::unordered_set<std::string> used_plugins;
@@ -952,6 +860,50 @@ bool falco_engine::is_source_valid(const std::string &source) const
 	return m_sources.at(source) != nullptr;
 }
 
+std::shared_ptr<gen_event_filter_factory> falco_engine::filter_factory_for_source(const std::string& source)
+{
+	return find_source(source)->filter_factory;
+}
+
+std::shared_ptr<gen_event_filter_factory> falco_engine::filter_factory_for_source(std::size_t source_idx)
+{
+	return find_source(source_idx)->filter_factory;
+}
+
+std::shared_ptr<gen_event_formatter_factory> falco_engine::formatter_factory_for_source(const std::string& source)
+{
+	return find_source(source)->formatter_factory;
+}
+
+std::shared_ptr<gen_event_formatter_factory> falco_engine::formatter_factory_for_source(std::size_t source_idx)
+{
+	return find_source(source_idx)->formatter_factory;
+}
+
+std::shared_ptr<filter_ruleset_factory> falco_engine::ruleset_factory_for_source(const std::string& source)
+{
+	return find_source(source)->ruleset_factory;
+}
+
+std::shared_ptr<filter_ruleset_factory> falco_engine::ruleset_factory_for_source(std::size_t source_idx)
+{
+	return find_source(source_idx)->ruleset_factory;
+}
+
+std::shared_ptr<filter_ruleset> falco_engine::ruleset_for_source(const std::string& source_name)
+{
+	const falco_source *source = find_source(source_name);
+
+	return source->ruleset;
+}
+
+std::shared_ptr<filter_ruleset> falco_engine::ruleset_for_source(std::size_t source_idx)
+{
+	const falco_source *source = find_source(source_idx);
+
+	return source->ruleset;
+}
+
 void falco_engine::read_file(const std::string& filename, std::string& contents)
 {
 	std::ifstream is;
@@ -966,29 +918,6 @@ void falco_engine::read_file(const std::string& filename, std::string& contents)
 			std::istreambuf_iterator<char>());
 }
 
-void falco_engine::interpret_load_result(std::unique_ptr<load_result>& res,
-					 const std::string& rules_filename,
-					 const std::string& rules_content,
-					 bool verbose)
-{
-	falco::load_result::rules_contents_t rc = {{rules_filename, rules_content}};
-
-	if(!res->successful())
-	{
-		// The output here is always the full e.g. "verbose" output.
-		throw falco_exception(res->as_string(true, rc).c_str());
-	}
-
-	if(verbose && res->has_warnings())
-	{
-		// Here, verbose controls whether to additionally
-		// "log" e.g. print to stderr. What's logged is always
-		// non-verbose so it fits on a single line.
-		// todo(jasondellaluce): introduce a logging callback in Falco
-		fprintf(stderr, "%s\n", res->as_string(false, rc).c_str());
-	}
-}
-
 static bool check_plugin_requirement_alternatives(
 		const std::vector<falco_engine::plugin_version_requirement>& plugins,
 		const rule_loader::plugin_version_info::requirement_alternatives& alternatives,

userspace/engine/falco_engine.h

@@ -74,15 +74,8 @@ public:
 	void list_fields(std::string &source, bool verbose, bool names_only, bool markdown) const;
 
 	//
-	// Load rules either directly or from a filename.
+	// Load rules and returns a result object.
 	//
-	void load_rules_file(const std::string &rules_filename, bool verbose, bool all_events);
-	void load_rules(const std::string &rules_content, bool verbose, bool all_events);
-
-	//
-	// Identical to above, but returns a result object instead of
-	// throwing exceptions on error.
-	std::unique_ptr<falco::load_result> load_rules_file(const std::string &rules_filename);
 	std::unique_ptr<falco::load_result> load_rules(const std::string &rules_content, const std::string &name);
 
 	//
@@ -96,15 +89,23 @@ public:
 	//
 	void enable_rule(const std::string &substring, bool enabled, const std::string &ruleset = s_default_ruleset);
 
+	// Same as above but providing a ruleset id instead
+	void enable_rule(const std::string &substring, bool enabled, const uint16_t ruleset_id);
 
 	// Like enable_rule, but the rule name must be an exact match.
 	void enable_rule_exact(const std::string &rule_name, bool enabled, const std::string &ruleset = s_default_ruleset);
 
+	// Same as above but providing a ruleset id instead
+	void enable_rule_exact(const std::string &rule_name, bool enabled, const uint16_t ruleset_id);
+
 	//
 	// Enable/Disable any rules with any of the provided tags (set, exact matches only)
 	//
 	void enable_rule_by_tag(const std::set<std::string> &tags, bool enabled, const std::string &ruleset = s_default_ruleset);
 
+	// Same as above but providing a ruleset id instead
+	void enable_rule_by_tag(const std::set<std::string> &tags, bool enabled, const uint16_t ruleset_id);
+
 	//
 	// Must be called after the engine has been configured and all rulesets
 	// have been loaded and enabled/disabled.
@@ -136,7 +137,7 @@ public:
 	// Print details on the given rule. If rule is NULL, print
 	// details on all rules.
 	//
-	void describe_rule(std::string *rule, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins, bool json) const;
+	nlohmann::json describe_rule(std::string *rule, const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
 
 	//
 	// Print statistics on how many events matched each rule.
@@ -234,6 +235,31 @@ public:
 	// factory for this source.
 	bool is_source_valid(const std::string &source) const;
 
+	//
+	// Given a source, return a formatter factory that can create
+	// filters for events of that source.
+	//
+	std::shared_ptr<gen_event_filter_factory> filter_factory_for_source(const std::string& source);
+	std::shared_ptr<gen_event_filter_factory> filter_factory_for_source(std::size_t source_idx);
+
+	//
+	// Given a source, return a formatter factory that can create
+	// formatters for an event.
+	//
+	std::shared_ptr<gen_event_formatter_factory> formatter_factory_for_source(const std::string& source);
+	std::shared_ptr<gen_event_formatter_factory> formatter_factory_for_source(std::size_t source_idx);
+
+	//
+	// Given a source, return a ruleset factory that can create
+	// rulesets for that source.
+	//
+	std::shared_ptr<filter_ruleset_factory> ruleset_factory_for_source(const std::string& source);
+	std::shared_ptr<filter_ruleset_factory> ruleset_factory_for_source(std::size_t source_idx);
+
+	// Return the filter_ruleset used for a given source.
+	std::shared_ptr<filter_ruleset> ruleset_for_source(const std::string& source);
+	std::shared_ptr<filter_ruleset> ruleset_for_source(std::size_t source_idx);
+
 	//
 	// Given an event source and ruleset, fill in a bitset
 	// containing the event types for which this ruleset can run.
@@ -288,17 +314,46 @@ private:
 	// Throws falco_exception if the file can not be read
 	void read_file(const std::string& filename, std::string& contents);
 
-	// For load_rules methods that throw exceptions on error,
-	// interpret a load_result and throw an exception if needed.
-	void interpret_load_result(std::unique_ptr<falco::load_result>& res,
-				   const std::string& rules_filename,
-				   const std::string& rules_content,
-				   bool verbose);
-
 	indexed_vector<falco_source> m_sources;
 
-	const falco_source* find_source(std::size_t index) const;
-	const falco_source* find_source(const std::string& name) const;
+	inline const falco_source* find_source(std::size_t index)
+	{
+		const falco_source *source;
+
+		if(index == m_syscall_source_idx)
+		{
+			if(m_syscall_source == NULL)
+			{
+				m_syscall_source = m_sources.at(m_syscall_source_idx);
+				if(!m_syscall_source)
+				{
+					throw falco_exception("Unknown event source index " + std::to_string(index));
+				}
+			}
+
+			source = m_syscall_source;
+		}
+		else
+		{
+			source = m_sources.at(index);
+			if(!source)
+			{
+				throw falco_exception("Unknown event source index " + std::to_string(index));
+			}
+		}
+
+		return source;
+	}
+
+	inline const falco_source* find_source(const std::string& name) const
+	{
+		auto ret = m_sources.at(name);
+		if(!ret)
+		{
+			throw falco_exception("Unknown event source " + name);
+		}
+		return ret;
+	}
 
 	// To allow the engine to be extremely fast for syscalls (can
 	// be > 1M events/sec), we save the syscall source/source_idx
@@ -315,26 +370,26 @@ private:
 
 	// Retrieve json details from rules, macros, lists
 	void get_json_details(
-		Json::Value& out,
+		nlohmann::json& out,
 		const falco_rule& r,
 		const rule_loader::rule_info& info,
 		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
 	void get_json_details(
-		Json::Value& out,
+		nlohmann::json& out,
 		const falco_macro& m,
 		const rule_loader::macro_info& info,
 		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
 	void get_json_details(
-		Json::Value& out,
+		nlohmann::json& out,
 		const falco_list& l,
 		const rule_loader::list_info& info,
 		const std::vector<std::shared_ptr<sinsp_plugin>>& plugins) const;
 	void get_json_evt_types(
-		Json::Value& out,
+		nlohmann::json& out,
 		const std::string& source,
 		libsinsp::filter::ast::expr* ast) const;
 	void get_json_used_plugins(
-		Json::Value& out,
+		nlohmann::json& out,
 		const std::string& source,
 		const std::unordered_set<std::string>& evttypes,
 		const std::unordered_set<std::string>& fields,
@@ -379,4 +434,3 @@ private:
 	std::string m_extra;
 	bool m_replace_container_info;
 };
-

userspace/engine/falco_engine_version.h

@@ -20,7 +20,7 @@ limitations under the License.
 
 // The version of this Falco engine
 #define FALCO_ENGINE_VERSION_MAJOR 0
-#define FALCO_ENGINE_VERSION_MINOR 27
+#define FALCO_ENGINE_VERSION_MINOR 31
 #define FALCO_ENGINE_VERSION_PATCH 0
 
 #define FALCO_ENGINE_VERSION \
@@ -34,4 +34,4 @@ limitations under the License.
 // It represents the fields supported by this version of Falco,
 // the event types, and the underlying driverevent schema. It's used to
 // detetect changes in engine version in our CI jobs.
-#define FALCO_ENGINE_CHECKSUM "dbc34e88ab420320994d85f155dee6baff2dd018aacc00e249f897edc8b1e0f4"
+#define FALCO_ENGINE_CHECKSUM "7c512927c89f594f024f2ff181077c780c4fe6e9dd4cee3f20a9ef208a356e4e"

userspace/engine/falco_load_result.cpp

@@ -66,7 +66,8 @@ static const std::string warning_codes[] = {
 	"LOAD_UNKNOWN_FILTER",
 	"LOAD_UNUSED_MACRO",
 	"LOAD_UNUSED_LIST",
-	"LOAD_UNKNOWN_ITEM"
+	"LOAD_UNKNOWN_ITEM",
+	"LOAD_DEPRECATED_ITEM"
 };
 
 const std::string& falco::load_result::warning_code_str(warning_code wc)
@@ -81,7 +82,8 @@ static const std::string warning_strings[] = {
 	"Unknown field or event-type in condition or output",
 	"Unused macro",
 	"Unused list",
-	"Unknown rules file item"
+	"Unknown rules file item",
+	"Used deprecated item"
 };
 
 const std::string& falco::load_result::warning_str(warning_code wc)
@@ -96,7 +98,8 @@ static const std::string warning_descs[] = {
 	"A rule condition or output refers to a field or evt.type that does not exist. This is normally an error, but if a rule has a skip-if-unknown-filter property, the error is downgraded to a warning.",
 	"A macro is defined in the rules content but is not used by any other macro or rule.",
 	"A list is defined in the rules content but is not used by any other list, macro, or rule.",
-	"An unknown top-level object is in the rules content. It will be ignored."
+	"An unknown top-level object is in the rules content. It will be ignored.",
+	"A deprecated item is employed by lists, macros, or rules."
 };
 
 const std::string& falco::load_result::warning_desc(warning_code wc)

userspace/engine/falco_load_result.h

@@ -54,7 +54,8 @@ public:
 		LOAD_UNKNOWN_FILTER,
 		LOAD_UNUSED_MACRO,
 		LOAD_UNUSED_LIST,
-		LOAD_UNKNOWN_ITEM
+		LOAD_UNKNOWN_ITEM,
+		LOAD_DEPRECATED_ITEM
 	};
 
 	virtual ~load_result() = default;

userspace/engine/falco_utils.cpp

@@ -20,7 +20,6 @@ limitations under the License.
 #include <cstring>
 #include <iomanip>
 
-#include "falco_common.h"
 #include "falco_utils.h"
 #include "utils.h"
 
@@ -161,80 +160,6 @@ void readfile(const std::string& filename, std::string& data)
 	return;
 }
 
-// URI-decodes the given string by replacing percent-encoded
-// characters with the actual character. Returns the decoded string.
-//
-// When plus_as_space is true, non-encoded plus signs in the query are decoded as spaces.
-// (http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.1)
-std::string decode_uri(const std::string& str, bool plus_as_space)
-{
-	std::string decoded_str;
-	bool in_query = false;
-	std::string::const_iterator it  = str.begin();
-	std::string::const_iterator end = str.end();
-	while(it != end)
-	{
-		char c = *it++;
-		if(c == '?')
-		{
-			in_query = true;
-		}
-		// spaces may be encoded as plus signs in the query
-		if(in_query && plus_as_space && c == '+')
-		{
-			c = ' ';
-		}
-		else if(c == '%')
-		{
-			if (it == end)
-			{
-				throw falco_exception("URI encoding: no hex digit following percent sign in " + str);
-			}
-			char hi = *it++;
-			if (it == end)
-			{
-				throw falco_exception("URI encoding: two hex digits must follow percent sign in " + str);
-			}
-			char lo = *it++;
-			if (hi >= '0' && hi <= '9')
-			{
-				c = hi - '0';
-			}
-			else if (hi >= 'A' && hi <= 'F')
-			{
-				c = hi - 'A' + 10;
-			}
-			else if (hi >= 'a' && hi <= 'f')
-			{
-				c = hi - 'a' + 10;
-			}
-			else
-			{
-				throw falco_exception("URI encoding: not a hex digit found in " + str);
-			}
-			c *= 16;
-			if (lo >= '0' && lo <= '9')
-			{
-				c += lo - '0';
-			}
-			else if (lo >= 'A' && lo <= 'F')
-			{
-				c += lo - 'A' + 10;
-			}
-			else if (lo >= 'a' && lo <= 'f')
-			{
-				c += lo - 'a' + 10;
-			}
-			else
-			{
-				throw falco_exception("URI encoding: not a hex digit");
-			}
-		}
-		decoded_str += c;
-	}
-	return decoded_str;
-}
-
 namespace network
 {
 bool is_unix_scheme(const std::string& url)

userspace/engine/falco_utils.h

@@ -52,8 +52,6 @@ void readfile(const std::string& filename, std::string& data);
 
 uint32_t hardware_concurrency();
 
-std::string decode_uri(const std::string& str, bool plus_as_space);
-
 namespace network
 {
 static const std::string UNIX_SCHEME("unix://");

userspace/engine/filter_details_resolver.h

@@ -22,12 +22,12 @@ limitations under the License.
 #include <unordered_set>
 #include <unordered_map>
 
-struct filter_details 
+struct filter_details
 {
 	// input macros and lists
 	std::unordered_set<std::string> known_macros;
 	std::unordered_set<std::string> known_lists;
-	
+
 	// output details
 	std::unordered_set<std::string> fields;
 	std::unordered_set<std::string> macros;
@@ -47,25 +47,23 @@ public:
 	/*!
 		\brief Visits a filter AST and stores details about macros, lists,
 		fields and operators used.
-		\param filter The filter AST to be processed. 
-		\param details Helper structure used to state known macros and 
+		\param filter The filter AST to be processed.
+		\param details Helper structure used to state known macros and
 		lists on input, and to store all the retrieved details as output.
 	*/
 	void run(libsinsp::filter::ast::expr* filter,
 		filter_details& details);
-	
+
 private:
 	struct visitor : public libsinsp::filter::ast::expr_visitor
 	{
-		visitor(filter_details& details) : 
+		visitor(filter_details& details) :
 			m_details(details),
 			m_expect_list(false),
 			m_expect_macro(false),
 			m_expect_evtname(false) {}
 		visitor(visitor&&) = default;
-		visitor& operator = (visitor&&) = default;
 		visitor(const visitor&) = delete;
-		visitor& operator = (const visitor&) = delete;
 
 		void visit(libsinsp::filter::ast::and_expr* e) override;
 		void visit(libsinsp::filter::ast::or_expr* e) override;

userspace/engine/filter_macro_resolver.h

@@ -61,7 +61,7 @@ class filter_macro_resolver
 
 		/*!
 		    \brief used in get_{resolved,unknown}_macros and get_errors
-			to represent an identifier/string value along with an AST position. 
+			to represent an identifier/string value along with an AST position.
 		*/
 		typedef std::pair<std::string,libsinsp::filter::ast::pos_info> value_info;
 
@@ -103,10 +103,6 @@ class filter_macro_resolver
 					m_unknown_macros(unknown_macros),
 					m_resolved_macros(resolved_macros),
 					m_macros(macros) {}
-			visitor(visitor&&) = default;
-			visitor& operator = (visitor&&) = default;
-			visitor(const visitor&) = delete;
-			visitor& operator = (const visitor&) = delete;
 
 			std::vector<std::string> m_macros_path;
 			std::unique_ptr<libsinsp::filter::ast::expr> m_node_substitute;

userspace/engine/rule_loader.cpp

@@ -41,7 +41,8 @@ static const std::string item_type_strings[] = {
 	"condition expression",
 	"rule output",
 	"rule output expression",
-	"rule priority"
+	"rule priority",
+	"overrides"
 };
 
 const std::string& rule_loader::context::item_type_as_string(enum item_type it)
@@ -553,6 +554,11 @@ rule_loader::rule_info::rule_info(context &ctx)
 {
 }
 
+rule_loader::rule_update_info::rule_update_info(context &ctx)
+	: ctx(ctx), cond_ctx(ctx)
+{
+}
+
 rule_loader::rule_load_exception::rule_load_exception(falco::load_result::error_code ec, const std::string& msg, const context& ctx)
 	: ec(ec), msg(msg), ctx(ctx)
 {
@@ -562,10 +568,8 @@ rule_loader::rule_load_exception::~rule_load_exception()
 {
 }
 
-const char* rule_loader::rule_load_exception::what()
+const char* rule_loader::rule_load_exception::what() const noexcept
 {
-	errstr = falco::load_result::error_code_str(ec) + ": "
-		+ msg.c_str();
-
-	return errstr.c_str();
+	// const + noexcept: can't use functions that change the object or throw
+	return msg.c_str();
 }

userspace/engine/rule_loader.h

@@ -19,6 +19,7 @@ limitations under the License.
 
 #include <string>
 #include <vector>
+#include <optional>
 #include <yaml-cpp/yaml.h>
 #include <nlohmann/json.hpp>
 #include "falco_source.h"
@@ -56,7 +57,8 @@ namespace rule_loader
 			CONDITION_EXPRESSION,
 			RULE_OUTPUT,
 			RULE_OUTPUT_EXPRESSION,
-			RULE_PRIORITY
+			RULE_PRIORITY,
+			OVERRIDE
 		};
 
 		static const std::string& item_type_as_string(enum item_type it);
@@ -209,18 +211,12 @@ namespace rule_loader
 	public:
 		rule_load_exception(falco::load_result::error_code ec, const std::string& msg, const context& ctx);
 		virtual ~rule_load_exception();
-		rule_load_exception(rule_load_exception&&) = default;
-		rule_load_exception& operator = (rule_load_exception&&) = default;
-		rule_load_exception(const rule_load_exception&) = default;
-		rule_load_exception& operator = (const rule_load_exception&) = default;
 
-		const char* what();
+		const char* what() const noexcept override;
 
 		falco::load_result::error_code ec;
 		std::string msg;
 		context ctx;
-
-		std::string errstr;
 	};
 
 	/*!
@@ -278,10 +274,6 @@ namespace rule_loader
 			{
 				res.reset(new result(name));
 			}
-		configuration(configuration&&) = default;
-		configuration& operator = (configuration&&) = default;
-		configuration(const configuration&) = delete;
-		configuration& operator = (const configuration&) = delete;
 
 		// inputs
 		const std::string& content;
@@ -461,4 +453,38 @@ namespace rule_loader
 		bool warn_evttypes;
 		bool skip_if_unknown_filter;
 	};
+
+	/*!
+		\brief Represents infos about a rule update (append or replace) request
+	*/
+
+	struct rule_update_info
+	{
+		rule_update_info(context &ctx);
+		~rule_update_info() = default;
+		rule_update_info(rule_update_info&&) = default;
+		rule_update_info& operator = (rule_update_info&&) = default;
+		rule_update_info(const rule_update_info&) = default;
+		rule_update_info& operator = (const rule_update_info&) = default;
+
+		bool has_any_value()
+		{
+			return cond.has_value() || output.has_value() || desc.has_value() || tags.has_value() ||
+				   exceptions.has_value() || priority.has_value() || enabled.has_value() ||
+				   warn_evttypes.has_value() || skip_if_unknown_filter.has_value();
+		}
+
+		context ctx;
+		context cond_ctx;
+		std::string name;
+		std::optional<std::string> cond;
+		std::optional<std::string> output;
+		std::optional<std::string> desc;
+		std::optional<std::set<std::string>> tags;
+		std::optional<std::vector<rule_exception_info>> exceptions;
+		std::optional<falco_common::priority_type> priority;
+		std::optional<bool> enabled;
+		std::optional<bool> warn_evttypes;
+		std::optional<bool> skip_if_unknown_filter;
+	};
 };

userspace/engine/rule_loader_collector.cpp

@@ -20,6 +20,7 @@ limitations under the License.
 
 #include "falco_engine.h"
 #include "rule_loader_collector.h"
+#include "rule_loading_messages.h"
 
 #define THROW(cond, err, ctx)    { if ((cond)) { throw rule_loader::rule_load_exception(falco::load_result::LOAD_ERR_VALIDATE, (err), (ctx)); } }
 
@@ -48,8 +49,14 @@ static inline void define_info(indexed_vector<T>& infos, T& info, uint32_t id)
 	}
 }
 
-template <typename T>
-static inline void append_info(T* prev, T& info, uint32_t id)
+template <typename T, typename U>
+static inline void append_info(T* prev, U& info, uint32_t id)
+{
+	prev->visibility = id;
+}
+
+template <typename T, typename U>
+static inline void replace_info(T* prev, U& info, uint32_t id)
 {
 	prev->visibility = id;
 }
@@ -184,9 +191,7 @@ void rule_loader::collector::define(configuration& cfg, list_info& info)
 void rule_loader::collector::append(configuration& cfg, list_info& info)
 {
 	auto prev = m_list_infos.at(info.name);
-	THROW(!prev,
-	       "List has 'append' key but no list by that name already exists",
-	       info.ctx);
+	THROW(!prev, ERROR_NO_PREVIOUS_LIST, info.ctx);
 	prev->items.insert(prev->items.end(), info.items.begin(), info.items.end());
 	append_info(prev, info, m_cur_index++);
 }
@@ -199,9 +204,7 @@ void rule_loader::collector::define(configuration& cfg, macro_info& info)
 void rule_loader::collector::append(configuration& cfg, macro_info& info)
 {
 	auto prev = m_macro_infos.at(info.name);
-	THROW(!prev,
-	       "Macro has 'append' key but no macro by that name already exists",
-	       info.ctx);
+	THROW(!prev, ERROR_NO_PREVIOUS_MACRO, info.ctx);
 	prev->cond += " ";
 	prev->cond += info.cond;
 	append_info(prev, info, m_cur_index++);
@@ -233,15 +236,14 @@ void rule_loader::collector::define(configuration& cfg, rule_info& info)
 	define_info(m_rule_infos, info, m_cur_index++);
 }
 
-void rule_loader::collector::append(configuration& cfg, rule_info& info)
+void rule_loader::collector::append(configuration& cfg, rule_update_info& info)
 {
 	auto prev = m_rule_infos.at(info.name);
 
-	THROW(!prev,
-	       "Rule has 'append' key but no rule by that name already exists",
-	       info.ctx);
-	THROW(info.cond.empty() && info.exceptions.empty(),
+	THROW(!prev, ERROR_NO_PREVIOUS_RULE_APPEND, info.ctx);
+	THROW(!info.has_any_value(),
 	       "Appended rule must have exceptions or condition property",
+	       // "Appended rule must have at least one field that can be appended to", // TODO replace with this and update testing
 	       info.ctx);
 
 	// note: source can be nullptr in case we've collected a
@@ -256,43 +258,136 @@ void rule_loader::collector::append(configuration& cfg, rule_info& info)
 	    	info.ctx);
 	}
 
-	if (!info.cond.empty())
+	if (info.cond.has_value() && !info.cond->empty())
 	{
 		prev->cond += " ";
-		prev->cond += info.cond;
+		prev->cond += *info.cond;
 	}
 
-	for (auto &ex : info.exceptions)
+	if (info.output.has_value() && !info.output->empty())
 	{
-		auto prev_ex = find_if(prev->exceptions.begin(), prev->exceptions.end(),
-			[&ex](const rule_loader::rule_exception_info& i)
-				{ return i.name == ex.name; });
-		if (prev_ex == prev->exceptions.end())
+		prev->output += " ";
+		prev->output += *info.output;
+	}
+
+	if (info.desc.has_value() && !info.desc->empty())
+	{
+		prev->desc += " ";
+		prev->desc += *info.desc;
+	}
+
+	if (info.tags.has_value())
+	{
+		for (auto &tag: *info.tags)
 		{
-			THROW(!ex.fields.is_valid(),
-			       "Rule exception must have fields property with a list of fields",
-			       ex.ctx);
-			THROW(ex.values.empty(),
-			       "Rule exception must have values property with a list of values",
-			       ex.ctx);
-			validate_exception_info(source, ex);
-			prev->exceptions.push_back(ex);
-		}
-		else
-		{
-			THROW(ex.fields.is_valid(),
-			       "Can not append exception fields to existing exception, only values",
-			       ex.ctx);
-			THROW(ex.comps.is_valid(),
-			       "Can not append exception comps to existing exception, only values",
-			       ex.ctx);
-			prev_ex->values.insert(
-				prev_ex->values.end(), ex.values.begin(), ex.values.end());
+			prev->tags.insert(tag);
 		}
 	}
+
+	if (info.exceptions.has_value())
+	{
+		for (auto &ex : *info.exceptions)
+		{
+			auto prev_ex = find_if(prev->exceptions.begin(), prev->exceptions.end(),
+				[&ex](const rule_loader::rule_exception_info& i)
+					{ return i.name == ex.name; });
+			if (prev_ex == prev->exceptions.end())
+			{
+				THROW(!ex.fields.is_valid(),
+					"Rule exception must have fields property with a list of fields",
+					ex.ctx);
+				THROW(ex.values.empty(),
+					"Rule exception must have values property with a list of values",
+					ex.ctx);
+				validate_exception_info(source, ex);
+				prev->exceptions.push_back(ex);
+			}
+			else
+			{
+				THROW(ex.fields.is_valid(),
+					"Can not append exception fields to existing exception, only values",
+					ex.ctx);
+				THROW(ex.comps.is_valid(),
+					"Can not append exception comps to existing exception, only values",
+					ex.ctx);
+				prev_ex->values.insert(
+					prev_ex->values.end(), ex.values.begin(), ex.values.end());
+			}
+		}
+	}
+
 	append_info(prev, info, m_cur_index++);
 }
 
+void rule_loader::collector::selective_replace(configuration& cfg, rule_update_info& info)
+{
+	auto prev = m_rule_infos.at(info.name);
+
+	THROW(!prev, ERROR_NO_PREVIOUS_RULE_REPLACE, info.ctx);
+	THROW(!info.has_any_value(),
+	       "The rule must have at least one field that can be replaced",
+	       info.ctx);
+
+	// note: source can be nullptr in case we've collected a
+	// rule for which the source is unknown
+	falco_source* source = nullptr;
+	if (!prev->unknown_source)
+	{
+		// note: if the source is not unknown, this should not return nullptr
+		source = cfg.sources.at(prev->source);
+		THROW(!source,
+	    	std::string("Unknown source ") + prev->source,
+	    	info.ctx);
+	}
+
+	if (info.cond.has_value())
+	{
+		prev->cond = *info.cond;
+	}
+
+	if (info.output.has_value())
+	{
+		prev->output = *info.output;
+	}
+
+	if (info.desc.has_value())
+	{
+		prev->desc = *info.desc;
+	}
+
+	if (info.tags.has_value())
+	{
+		prev->tags = *info.tags;
+	}
+
+	if (info.exceptions.has_value())
+	{
+		prev->exceptions = *info.exceptions;
+	}
+
+	if (info.priority.has_value())
+	{
+		prev->priority = *info.priority;
+	}
+
+	if (info.enabled.has_value())
+	{
+		prev->enabled = *info.enabled;
+	}
+
+	if (info.warn_evttypes.has_value())
+	{
+		prev->warn_evttypes = *info.warn_evttypes;
+	}
+
+	if (info.skip_if_unknown_filter.has_value())
+	{
+		prev->skip_if_unknown_filter = *info.skip_if_unknown_filter;
+	}
+
+	replace_info(prev, info, m_cur_index++);
+}
+
 void rule_loader::collector::enable(configuration& cfg, rule_info& info)
 {
 	auto prev = m_rule_infos.at(info.name);

userspace/engine/rule_loader_collector.h

@@ -85,13 +85,18 @@ public:
 	*/
 	virtual void append(configuration& cfg, list_info& info);
 	virtual void append(configuration& cfg, macro_info& info);
-	virtual void append(configuration& cfg, rule_info& info);
+	virtual void append(configuration& cfg, rule_update_info& info);
 
 	/*!
 		\brief Updates the 'enabled' flag of an existing definition
 	*/
 	virtual void enable(configuration& cfg, rule_info& info);
 
+	/*!
+		\brief Selectively replaces some fields of an existing definition
+	*/
+	virtual void selective_replace(configuration& cfg, rule_update_info& info);
+
 private:
 	uint32_t m_cur_index;
 	indexed_vector<rule_info> m_rule_infos;

userspace/engine/rule_loader_compiler.cpp

@@ -50,12 +50,6 @@ static void paren_item(std::string& e)
 	}
 }
 
-static inline bool is_operator_defined(const std::string& op)
-{
-	auto ops = libsinsp::filter::parser::supported_operators();
-	return find(ops.begin(), ops.end(), op) != ops.end();
-}
-
 static inline bool is_operator_for_list(const std::string& op)
 {
 	auto ops = libsinsp::filter::parser::supported_operators(true);
@@ -83,12 +77,12 @@ static void build_rule_exception_infos(
 	std::string& condition)
 {
 	std::string tmp;
-	for (auto &ex : exceptions)
+	for (const auto &ex : exceptions)
 	{
 		std::string icond;
 		if(!ex.fields.is_list)
 		{
-			for (auto &val : ex.values)
+			for (const auto &val : ex.values)
 			{
 				THROW(val.is_list,
 				       "Expected values array to contain a list of strings",
@@ -107,7 +101,7 @@ static void build_rule_exception_infos(
 		else
 		{
 			icond = "(";
-			for (auto &values : ex.values)
+			for (const auto &values : ex.values)
 			{
 				THROW(ex.fields.items.size() != values.items.size(),
 				       "Fields and values lists must have equal length",
@@ -116,13 +110,13 @@ static void build_rule_exception_infos(
 				icond += "(";
 				uint32_t k = 0;
 				std::string istr;
-				for (auto &field : ex.fields.items)
+				for (const auto &field : ex.fields.items)
 				{
 					icond += k == 0 ? "" : " and ";
 					if (values.items[k].is_list)
 					{
 						istr = "(";
-						for (auto &v : values.items[k].items)
+						for (const auto &v : values.items[k].items)
 						{
 							tmp = v.item;
 							quote_item(tmp);
@@ -187,6 +181,7 @@ static bool resolve_list(std::string& cnd, const falco_list& list)
 {
 	static std::string blanks = " \t\n\r";
 	static std::string delims = blanks + "(),=";
+	std::string tmp;
 	std::string new_cnd;
 	size_t start, end;
 	bool used = false;
@@ -212,13 +207,15 @@ static bool resolve_list(std::string& cnd, const falco_list& list)
 			}
 			// create substitution string by concatenating all values
 			std::string sub = "";
-			for (auto &v : list.items)
+			for (const auto &v : list.items)
 			{
 				if (!sub.empty())
 				{
 					sub += ", ";
 				}
-				sub += v;
+				tmp = v;
+				quote_item(tmp);
+				sub += tmp;
 			}
 			// if substituted list is empty, we need to
 			// remove a comma from the left or the right
@@ -262,7 +259,7 @@ static void resolve_macros(
 	const rule_loader::context &ctx)
 {
 	filter_macro_resolver macro_resolver;
-	for (auto &m : infos)
+	for (const auto &m : infos)
 	{
 		if (m.index < visibility)
 		{
@@ -287,7 +284,7 @@ static void resolve_macros(
 		THROW(true, errmsg, cond_ctx);
 	}
 
-	for (auto &it : macro_resolver.get_resolved_macros())
+	for (const auto &it : macro_resolver.get_resolved_macros())
 	{
 		macros.at(it.first)->used = true;
 	}
@@ -345,37 +342,33 @@ void rule_loader::compiler::compile_list_infos(
 		const collector& col,
 		indexed_vector<falco_list>& out) const
 {
-	std::string tmp;
-	std::vector<std::string> used;
-	for (auto &list : col.lists())
+	std::list<std::string> used;
+	falco_list v;
+	for (const auto &list : col.lists())
 	{
-		falco_list v;
 		v.name = list.name;
 		v.items.clear();
-		for (auto &item : list.items)
+		for (const auto &item : list.items)
 		{
 			const auto ref = col.lists().at(item);
 			if (ref && ref->index < list.visibility)
 			{
 				used.push_back(ref->name);
-				for (auto val : ref->items)
+				for (const auto &val : ref->items)
 				{
-					quote_item(val);
 					v.items.push_back(val);
 				}
 			}
 			else
 			{
-				tmp = item;
-				quote_item(tmp);
-				v.items.push_back(tmp);
+				v.items.push_back(item);
 			}
 		}
 		v.used = false;
 		auto list_id = out.insert(v, v.name);
 		out.at(list_id)->id = list_id;
 	}
-	for (auto &v : used)
+	for (const auto &v : used)
 	{
 		out.at(v)->used = true;
 	}
@@ -388,7 +381,7 @@ void rule_loader::compiler::compile_macros_infos(
 		indexed_vector<falco_list>& lists,
 		indexed_vector<falco_macro>& out) const
 {
-	for (auto &m : col.macros())
+	for (const auto &m : col.macros())
 	{
 		falco_macro entry;
 		entry.name = m.name;
@@ -422,7 +415,7 @@ void rule_loader::compiler::compile_rule_infos(
 	std::string err, condition;
 	std::set<falco::load_result::load_result::warning_code> warn_codes;
 	filter_warning_resolver warn_resolver;
-	for (auto &r : col.rules())
+	for (const auto &r : col.rules())
 	{
 		// skip the rule if it has an unknown source
 		if (r.unknown_source)
@@ -453,7 +446,7 @@ void rule_loader::compiler::compile_rule_infos(
 		warn_codes.clear();
 		if (warn_resolver.run(rule.condition.get(), warn_codes))
 		{
-			for (auto &w : warn_codes)
+			for (const auto &w : warn_codes)
 			{
 				cfg.res->add_warning(w, "", r.ctx);
 			}
@@ -555,7 +548,7 @@ void rule_loader::compiler::compile(
 	}
 
 	// print info on any dangling lists or macros that were not used anywhere
-	for (auto &m : out.macros)
+	for (const auto &m : out.macros)
 	{
 		if (!m.used)
 		{
@@ -565,7 +558,7 @@ void rule_loader::compiler::compile(
 				macro_info_from_name(col, m.name)->ctx);
 		}
 	}
-	for (auto &l : out.lists)
+	for (const auto &l : out.lists)
 	{
 		if (!l.used)
 		{

userspace/engine/rule_loader_reader.cpp

@@ -17,10 +17,12 @@ limitations under the License.
 
 #include <string>
 #include <vector>
+#include <set>
+#include <sstream>
 
 #include "rule_loader_reader.h"
 #include "falco_engine_version.h"
-#include "logger.h"
+#include "rule_loading_messages.h"
 
 #define THROW(cond, err, ctx)    { if ((cond)) { throw rule_loader::rule_load_exception(falco::load_result::LOAD_ERR_YAML_VALIDATE, (err), (ctx)); } }
 
@@ -45,6 +47,14 @@ static void decode_val_generic(const YAML::Node& item, const char *key, T& out,
 	THROW(!YAML::convert<T>::decode(val, out), "Can't decode YAML scalar value", valctx);
 }
 
+template <typename T>
+static void decode_val_generic(const YAML::Node& item, const char *key, std::optional<T>& out, const rule_loader::context& ctx, bool optional)
+{
+	T decoded;
+	decode_val_generic(item, key, decoded, ctx, optional);
+	out = decoded;
+}
+
 template <typename T>
 static void decode_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx)
 {
@@ -115,6 +125,73 @@ static void decode_tags(const YAML::Node& item, std::set<T>& out,
 	decode_seq(item, "tags", inserter, ctx, optional);
 }
 
+template <typename T>
+static void decode_tags(const YAML::Node& item, std::optional<std::set<T>>& out,
+		       const rule_loader::context& ctx)
+{
+	std::set<T> decoded;
+	decode_tags(item, decoded, ctx);
+	out = decoded;
+}
+
+static void decode_overrides(const YAML::Node& item,
+				std::set<std::string>& overridable_append,
+				std::set<std::string>& overridable_replace,
+				std::set<std::string>& out_append,
+				std::set<std::string>& out_replace,
+				const rule_loader::context& ctx)
+{
+	const YAML::Node& val = item["override"];
+
+	if(!val.IsDefined())
+	{
+		return;
+	}
+
+	rule_loader::context overridectx(item, rule_loader::context::OVERRIDE, "", ctx);
+
+	for(YAML::const_iterator it=val.begin();it!=val.end();++it)
+	{
+		std::string key = it->first.as<std::string>();
+		std::string operation = it->second.as<std::string>();
+
+		bool is_overridable_append = overridable_append.find(it->first.as<std::string>()) != overridable_append.end();
+		bool is_overridable_replace = overridable_replace.find(it->first.as<std::string>()) != overridable_replace.end();
+
+		if (operation == "append")
+		{
+			rule_loader::context keyctx(it->first, rule_loader::context::OVERRIDE, key, overridectx);
+			THROW(!is_overridable_append, std::string("Key '") + key + std::string("' cannot be appended to, use 'replace' instead"), keyctx);
+
+			out_append.insert(key);
+		}
+		else if (operation == "replace")
+		{
+			rule_loader::context keyctx(it->first, rule_loader::context::OVERRIDE, key, overridectx);
+			THROW(!is_overridable_replace, std::string("Key '") + key + std::string("' cannot be replaced"), keyctx);
+
+			out_replace.insert(key);
+		}
+		else
+		{
+			rule_loader::context operationctx(it->second, rule_loader::context::VALUE_FOR, key, overridectx);
+			std::stringstream err_ss;
+			err_ss << "Invalid override operation for key '" << key << "': '" << operation << "'. "
+				   << "Allowed values are: ";
+			if (is_overridable_append)
+			{
+				err_ss << "append ";
+			}
+			if (is_overridable_replace)
+			{
+				err_ss << "replace ";
+			}
+
+			THROW(true, err_ss.str(), operationctx);
+		}
+	}
+}
+
 // Don't call this directly, call decode_exception_{fields,comps,values} instead
 static void decode_exception_info_entry(
 	const YAML::Node& item,
@@ -187,7 +264,7 @@ static void decode_exception_values(
 
 static void read_rule_exceptions(
 	const YAML::Node& item,
-	rule_loader::rule_info& v,
+	std::vector<rule_loader::rule_exception_info>& exceptions,
 	const rule_loader::context& parent,
 	bool append)
 {
@@ -239,10 +316,36 @@ static void read_rule_exceptions(
 				v_ex.values.push_back(v_ex_val);
 			}
 		}
-		v.exceptions.push_back(v_ex);
+		exceptions.push_back(v_ex);
 	}
 }
 
+static void read_rule_exceptions(
+	const YAML::Node& item,
+	std::optional<std::vector<rule_loader::rule_exception_info>>& exceptions,
+	const rule_loader::context& parent,
+	bool append)
+{
+	std::vector<rule_loader::rule_exception_info> decoded;
+	read_rule_exceptions(item, decoded, parent, append);
+	exceptions = decoded;
+}
+
+inline static bool check_update_expected(std::set<std::string>& expected_keys, const std::set<std::string>& overrides, const std::string& override_type, const std::string& key, const rule_loader::context& ctx)
+{
+	if (overrides.find(key) == overrides.end())
+	{
+		return false;
+	}
+	
+	THROW(expected_keys.find(key) == expected_keys.end(),
+		std::string("An ") + override_type + " override for '" + key + "' was specified but '" + key + "' is not defined", ctx);
+
+	expected_keys.erase(key);
+
+	return true;
+}
+
 static void read_item(
 	rule_loader::configuration& cfg,
 	rule_loader::collector& collector,
@@ -337,6 +440,21 @@ static void read_item(
 		decode_items(item, v.items, ctx);
 
 		decode_optional_val(item, "append", append, ctx);
+		if(append)
+		{
+			cfg.res->add_warning(falco::load_result::LOAD_DEPRECATED_ITEM, WARNING_APPEND, ctx);
+		}
+
+		std::set<std::string> override_append, override_replace;
+		std::set<std::string> overridable {"items"};
+		decode_overrides(item, overridable, overridable, override_append, override_replace, ctx);
+		bool has_overrides = !override_append.empty() || !override_replace.empty();
+
+		THROW(append && has_overrides, ERROR_OVERRIDE_APPEND, ctx);
+
+		// Since a list only has items, if we have chosen to append them we can append the entire object
+		// otherwise we just want to redefine the list.
+		append |= override_append.find("items") != override_append.end();
 
 		if(append)
 		{
@@ -365,6 +483,21 @@ static void read_item(
 		v.cond_ctx = rule_loader::context(item["condition"], rule_loader::context::MACRO_CONDITION, "", ctx);
 
 		decode_optional_val(item, "append", append, ctx);
+		if(append)
+		{
+			cfg.res->add_warning(falco::load_result::LOAD_DEPRECATED_ITEM, WARNING_APPEND, ctx);
+		}
+
+		std::set<std::string> override_append, override_replace;
+		std::set<std::string> overridable {"condition"};
+		decode_overrides(item, overridable, overridable, override_append, override_replace, ctx);
+		bool has_overrides = !override_append.empty() || !override_replace.empty();
+
+		THROW((append && has_overrides), ERROR_OVERRIDE_APPEND, ctx);
+
+		// Since a macro only has a condition, if we have chosen to append to it we can append the entire object
+		// otherwise we just want to redefine the macro.
+		append |= override_append.find("condition") != override_append.end();
 
 		if(append)
 		{
@@ -384,28 +517,172 @@ static void read_item(
 		decode_val(item, "rule", name, tmp);
 
 		rule_loader::context ctx(item, rule_loader::context::RULE, name, parent);
-		rule_loader::rule_info v(ctx);
-		v.name = name;
 
-		bool append = false;
-		v.enabled = true;
-		v.warn_evttypes = true;
-		v.skip_if_unknown_filter = false;
-
-		decode_optional_val(item, "append", append, ctx);
-
-		if(append)
+		bool has_append_flag = false;
+		decode_optional_val(item, "append", has_append_flag, ctx);
+		if(has_append_flag)
 		{
-			decode_optional_val(item, "condition", v.cond, ctx);
+			cfg.res->add_warning(falco::load_result::LOAD_DEPRECATED_ITEM, WARNING_APPEND, ctx);
+		}
+
+		std::set<std::string> override_append, override_replace;
+		std::set<std::string> overridable_append {"condition", "output", "desc", "tags", "exceptions"};
+		std::set<std::string> overridable_replace {
+			"condition", "output", "desc", "priority", "tags", "exceptions", "enabled", "warn_evttypes", "skip-if-unknown-filter"};
+		decode_overrides(item, overridable_append, overridable_replace, override_append, override_replace, ctx);
+		bool has_overrides_append = !override_append.empty();
+		bool has_overrides_replace = !override_replace.empty();
+		bool has_overrides = has_overrides_append || has_overrides_replace;
+
+		THROW((has_append_flag && has_overrides), ERROR_OVERRIDE_APPEND, ctx);
+
+		if(has_overrides)
+		{
+			std::set<std::string> expected_keys;
+			for (auto& key : overridable_append)
+			{
+				if (item[key].IsDefined())
+				{
+					expected_keys.insert(key);
+				}
+			}
+
+			for (auto& key : overridable_replace)
+			{
+				if (item[key].IsDefined())
+				{
+					expected_keys.insert(key);
+				}
+			}
+
+			// expected_keys is (appendable U replaceable) ^ (defined)
+			
+			if (has_overrides_append)
+			{
+				rule_loader::rule_update_info v(ctx);
+				v.name = name;
+				if (check_update_expected(expected_keys, override_append, "append", "condition", ctx))
+				{
+					decode_val(item, "condition", v.cond, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_append, "append", "exceptions", ctx))
+				{
+					read_rule_exceptions(item, v.exceptions, ctx, true);
+				}
+
+				if (check_update_expected(expected_keys, override_append, "append", "output", ctx))
+				{
+					decode_val(item, "output", v.output, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_append, "append", "desc", ctx))
+				{
+					decode_val(item, "desc", v.desc, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_append, "append", "tags", ctx))
+				{
+					decode_tags(item, v.tags, ctx);
+				}
+				
+				collector.append(cfg, v);
+			}
+
+			if (has_overrides_replace)
+			{
+				rule_loader::rule_update_info v(ctx);
+				v.name = name;
+				if (check_update_expected(expected_keys, override_replace, "replace", "condition", ctx))
+				{
+					decode_val(item, "condition", v.cond, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "exceptions", ctx))
+				{
+					read_rule_exceptions(item, v.exceptions, ctx, true);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "output", ctx))
+				{
+					decode_val(item, "output", v.output, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "desc", ctx))
+				{
+					decode_val(item, "desc", v.desc, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "tags", ctx))
+				{
+					decode_tags(item, v.tags, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "priority", ctx))
+				{
+					std::string priority;
+					decode_val(item, "priority", priority, ctx);
+					rule_loader::context prictx(item["priority"], rule_loader::context::RULE_PRIORITY, "", ctx);
+					falco_common::priority_type parsed_priority;
+					THROW(!falco_common::parse_priority(priority, parsed_priority), "Invalid priority", prictx);
+					v.priority = parsed_priority;
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "enabled", ctx))
+				{
+					decode_val(item, "enabled", v.enabled, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "warn_evttypes", ctx))
+				{
+					decode_val(item, "warn_evttypes", v.warn_evttypes, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "skip-if-unknown-filter", ctx))
+				{
+					decode_val(item, "skip-if-unknown-filter", v.skip_if_unknown_filter, ctx);
+				}
+
+				collector.selective_replace(cfg, v);
+			}
+
+			// if any expected key has not been defined throw an error
+			for (auto &key : expected_keys) {
+				rule_loader::context keyctx(item[key], rule_loader::context::OVERRIDE, key, ctx);
+				THROW(true, "Unexpected key '" + key + "': no corresponding entry under 'override' is defined.", keyctx);
+			}
+		}
+		else if(has_append_flag)
+		{
+			rule_loader::rule_update_info v(ctx);
+			v.name = name;
+
 			if(item["condition"].IsDefined())
 			{
 				v.cond_ctx = rule_loader::context(item["condition"], rule_loader::context::RULE_CONDITION, "", ctx);
+				decode_val(item, "condition", v.cond, ctx);
 			}
-			read_rule_exceptions(item, v, ctx, append);
+
+			if(item["exceptions"].IsDefined())
+			{
+				read_rule_exceptions(item, v.exceptions, ctx, true);
+			}
+
+			// TODO restore this error and update testing
+			//THROW((!v.cond.has_value() && !v.exceptions.has_value()),
+			//       "Appended rule must have exceptions or condition property",
+			//       v.ctx);
+
 			collector.append(cfg, v);
 		}
 		else
 		{
+			rule_loader::rule_info v(ctx);
+			v.name = name;
+			v.enabled = true;
+			v.warn_evttypes = true;
+			v.skip_if_unknown_filter = false;
+
 			// If the rule does *not* have any of
 			// condition/output/desc/priority, it *must*
 			// have an enabled property. Use the enabled
@@ -417,6 +694,7 @@ static void read_item(
 			    !item["priority"].IsDefined())
 			{
 				decode_val(item, "enabled", v.enabled, ctx);
+				cfg.res->add_warning(falco::load_result::LOAD_DEPRECATED_ITEM, WARNING_ENABLED, ctx);
 				collector.enable(cfg, v);
 			}
 			else
@@ -443,7 +721,7 @@ static void read_item(
 				decode_optional_val(item, "warn_evttypes", v.warn_evttypes, ctx);
 				decode_optional_val(item, "skip-if-unknown-filter", v.skip_if_unknown_filter, ctx);
 				decode_tags(item, v.tags, ctx);
-				read_rule_exceptions(item, v, ctx, append);
+				read_rule_exceptions(item, v.exceptions, ctx, has_append_flag);
 				collector.define(cfg, v);
 			}
 		}

userspace/engine/rule_loading_messages.h

@@ -0,0 +1,23 @@
+#pragma once
+
+////////////////
+// Warnings
+////////////////
+
+#define WARNING_APPEND "'append' key is deprecated. Add an 'append' entry (e.g. 'condition: append') under 'override' instead."
+
+#define WARNING_ENABLED "The standalone 'enabled' key usage is deprecated. The correct approach requires also a 'replace' entry under the 'override' key (i.e. 'enabled: replace')."
+
+////////////////
+// Errors
+////////////////
+
+#define ERROR_OVERRIDE_APPEND "Keys 'override' and 'append: true' cannot be used together. Add an 'append' entry (e.g. 'condition: append') under 'override' instead."
+
+#define ERROR_NO_PREVIOUS_MACRO "Macro uses 'append' or 'override.condition: append' but no macro by that name already exists"
+
+#define ERROR_NO_PREVIOUS_LIST "List uses 'append' or 'override.items: append' but no list by that name already exists"
+
+#define ERROR_NO_PREVIOUS_RULE_APPEND "Rule uses 'append' or 'override.<key>: append' but no rule by that name already exists"
+
+#define ERROR_NO_PREVIOUS_RULE_REPLACE "An 'override.<key>: replace' to a rule was requested but no rule by that name already exists"

userspace/engine/stats_manager.h

@@ -35,10 +35,6 @@ class stats_manager
 public:
 	stats_manager();
 	virtual ~stats_manager();
-	stats_manager(stats_manager&&) = default;
-	stats_manager& operator = (stats_manager&&) = default;
-	stats_manager(const stats_manager&) = default;
-	stats_manager& operator = (const stats_manager&) = default;
 
 	/*!
 		\brief Erases the internal state and statistics data

userspace/falco/CMakeLists.txt

@@ -36,6 +36,7 @@ set(
   app/actions/print_generated_gvisor_config.cpp
   app/actions/print_help.cpp
   app/actions/print_ignored_events.cpp
+  app/actions/print_kernel_version.cpp
   app/actions/print_plugin_info.cpp
   app/actions/print_support.cpp
   app/actions/print_syscall_events.cpp

userspace/falco/app/actions/actions.h

@@ -41,6 +41,7 @@ falco::app::run_result load_rules_files(falco::app::state& s);
 falco::app::run_result print_generated_gvisor_config(falco::app::state& s);
 falco::app::run_result print_help(falco::app::state& s);
 falco::app::run_result print_ignored_events(falco::app::state& s);
+falco::app::run_result print_kernel_version(falco::app::state& s);
 falco::app::run_result print_page_size(falco::app::state& s);
 falco::app::run_result print_plugin_info(falco::app::state& s);
 falco::app::run_result print_support(falco::app::state& s);

userspace/falco/app/actions/create_requested_paths.cpp

@@ -20,14 +20,6 @@ limitations under the License.
 #include <sys/stat.h>
 #include <filesystem>
 
-#ifndef CPPPATH_SEP
-#ifdef _MSC_VER
-#define CPPPATH_SEP "\\"
-#else
-#define CPPPATH_SEP "/"
-#endif
-#endif
-
 using namespace falco::app;
 using namespace falco::app::actions;
 

userspace/falco/app/actions/helpers.h

@@ -20,6 +20,8 @@ limitations under the License.
 #include "../state.h"
 #include "../run_result.h"
 
+#include <nlohmann/json.hpp>
+
 namespace falco {
 namespace app {
 namespace actions {
@@ -29,6 +31,8 @@ void print_enabled_event_sources(falco::app::state& s);
 void activate_interesting_kernel_tracepoints(falco::app::state& s, std::unique_ptr<sinsp>& inspector);
 void check_for_ignored_events(falco::app::state& s);
 void format_plugin_info(std::shared_ptr<sinsp_plugin> p, std::ostream& os);
+void format_described_rules_as_text(const nlohmann::json& v, std::ostream& os);
+
 falco::app::run_result open_offline_inspector(falco::app::state& s);
 falco::app::run_result open_live_inspector(
     falco::app::state& s,

userspace/falco/app/actions/helpers_generic.cpp

@@ -16,6 +16,7 @@ limitations under the License.
 */
 
 #include "helpers.h"
+#include "falco_utils.h"
 #include <plugin_manager.h>
 
 #include <unordered_set>
@@ -126,3 +127,22 @@ void falco::app::actions::format_plugin_info(std::shared_ptr<sinsp_plugin> p, st
 		os << "  - Async Events" << std::endl;
 	}
 }
+
+static void format_two_columns(std::ostream& os, const std::string& l, const std::string& r)
+{
+	static constexpr const int s_max_line_len = 4096;
+	char buf[s_max_line_len];
+	snprintf(buf, sizeof(buf) - 1, "%-50s %s", l.c_str(), r.c_str());
+	os << buf << std::endl;
+}
+
+void falco::app::actions::format_described_rules_as_text(const nlohmann::json& v, std::ostream& os)
+{
+	format_two_columns(os, "Rule", "Description");
+	format_two_columns(os,  "----", "-----------");
+	for(const auto &r : v["rules"])
+	{
+		auto str = falco::utils::wrap_text(r["info"]["description"], 51, 110) + "\n";
+		format_two_columns(os, r["info"]["name"], str);
+	}
+}

userspace/falco/app/actions/helpers_inspector.cpp

@@ -52,6 +52,11 @@ falco::app::run_result falco::app::actions::open_live_inspector(
 {
 	try
 	{
+		if((s.config->m_metrics_flags & PPM_SCAP_STATS_STATE_COUNTERS))
+		{
+			inspector->set_sinsp_stats_v2_enabled();
+		}
+
 		if (source != falco_common::syscall_source) /* Plugin engine */
 		{
 			for (const auto& p: inspector->get_plugin_manager()->plugins())

userspace/falco/app/actions/load_config.cpp

@@ -79,17 +79,17 @@ static falco::app::run_result apply_deprecated_options(falco::app::state& s)
 	if(s.config->m_syscall_drop_failed_exit != DEFAULT_DROP_FAILED_EXIT)
 	{
 		falco_logger::log(falco_logger::level::WARNING,
-				"DEPRECATION NOTICE: 'syscall_drop_failed_exit' config is deprecated and will be removed in Falco 0.38! Use `engine.<driver>.drop_failed_exit' config instead\n");
+				"DEPRECATION NOTICE: 'syscall_drop_failed_exit' config is deprecated and will be removed in Falco 0.38! Use 'engine.<driver>.drop_failed_exit' config instead\n");
 	}
 	if(s.config->m_syscall_buf_size_preset != DEFAULT_BUF_SIZE_PRESET)
 	{
 		falco_logger::log(falco_logger::level::WARNING,
-				"DEPRECATION NOTICE: 'syscall_buf_size_preset' config is deprecated and will be removed in Falco 0.38! Use `engine.<driver>.buf_size_preset' config instead\n");
+				"DEPRECATION NOTICE: 'syscall_buf_size_preset' config is deprecated and will be removed in Falco 0.38! Use 'engine.<driver>.buf_size_preset' config instead\n");
 	}
 	if(s.config->m_cpus_for_each_syscall_buffer != DEFAULT_CPUS_FOR_EACH_SYSCALL_BUFFER)
 	{
 		falco_logger::log(falco_logger::level::WARNING,
-				"DEPRECATION NOTICE: 'modern_bpf.cpus_for_each_syscall_buffer' config is deprecated and will be removed in Falco 0.38! Use `engine.modern_ebpf.cpus_for_each_buffer' config instead\n");
+				"DEPRECATION NOTICE: 'modern_bpf.cpus_for_each_syscall_buffer' config is deprecated and will be removed in Falco 0.38! Use 'engine.modern_ebpf.cpus_for_each_buffer' config instead\n");
 	}
 
 	// Replace the kmod default values in case the engine was open with the kmod.
@@ -102,7 +102,7 @@ static falco::app::run_result apply_deprecated_options(falco::app::state& s)
 	// use the requested driver.
 	if (getenv(FALCO_BPF_ENV_VARIABLE))
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the 'FALCO_BPF_PROBE' environment variable is deprecated and will be removed in Falco 0.38!\n");
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the 'FALCO_BPF_PROBE' environment variable is deprecated and will be removed in Falco 0.38! Set 'engine.kind: ebpf' and use 'engine.ebpf' config instead in falco.yaml\n");
 		s.config->m_engine_mode = engine_kind_t::EBPF;
 		s.config->m_ebpf.m_probe_path = getenv(FALCO_BPF_ENV_VARIABLE);
 		s.config->m_ebpf.m_drop_failed_exit = s.config->m_syscall_drop_failed_exit;
@@ -110,7 +110,7 @@ static falco::app::run_result apply_deprecated_options(falco::app::state& s)
 	}
 	else if (s.options.modern_bpf)
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '--modern-bpf' command line option is deprecated and will be removed in Falco 0.38!\n");
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '--modern-bpf' command line option is deprecated and will be removed in Falco 0.38! Set 'engine.kind: modern_ebpf' and use 'engine.modern_ebpf' config instead in falco.yaml\n");
 		s.config->m_engine_mode = engine_kind_t::MODERN_EBPF;
 		s.config->m_modern_ebpf.m_drop_failed_exit = s.config->m_syscall_drop_failed_exit;
 		s.config->m_modern_ebpf.m_buf_size_preset = s.config->m_syscall_buf_size_preset;
@@ -118,19 +118,19 @@ static falco::app::run_result apply_deprecated_options(falco::app::state& s)
 	}
 	if (!s.options.gvisor_config.empty())
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '-g,--gvisor-config' command line option is deprecated and will be removed in Falco 0.38!\n");
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '-g,--gvisor-config' command line option is deprecated and will be removed in Falco 0.38! Set 'engine.kind: gvisor' and use 'engine.gvisor' config instead in falco.yaml\n");
 		s.config->m_engine_mode =  engine_kind_t::GVISOR;
 		s.config->m_gvisor.m_config = s.options.gvisor_config;
 		s.config->m_gvisor.m_root = s.options.gvisor_root;
 	}
 	if (s.options.nodriver)
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '--nodriver' command line option is deprecated and will be removed in Falco 0.38!\n");
-		s.config->m_engine_mode =  engine_kind_t::NONE;
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '--nodriver' command line option is deprecated and will be removed in Falco 0.38! Set 'engine.kind: nodriver' instead in falco.yaml\n");
+		s.config->m_engine_mode =  engine_kind_t::NODRIVER;
 	}
 	if (!s.options.capture_file.empty())
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '-e' command line option is deprecated and will be removed in Falco 0.38!\n");
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '-e' command line option is deprecated and will be removed in Falco 0.38! Set 'engine.kind: replay' and use 'engine.replay' config instead in falco.yaml\n");
 		s.config->m_engine_mode = engine_kind_t::REPLAY;
 		s.config->m_replay.m_capture_file = s.options.capture_file;
 	}

userspace/falco/app/actions/load_rules_files.cpp

@@ -79,10 +79,9 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 			break;
 		}
 
-		// If verbose is true, also print any warnings
-		if(s.options.verbose && res->has_warnings())
+		if(res->has_warnings())
 		{
-			fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
+			falco_logger::log(falco_logger::level::WARNING,res->as_string(true, rc) + "\n");
 		}
 	}
 
@@ -155,17 +154,22 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 		s.engine->enable_rule_by_tag(s.options.enabled_rule_tags, true);
 	}
 
-	if (s.options.describe_all_rules)
+	// printout of `-L` option
+	if (s.options.describe_all_rules || !s.options.describe_rule.empty())
 	{
+		std::string* rptr = !s.options.describe_rule.empty() ? &(s.options.describe_rule) : nullptr;
 		const auto& plugins = s.offline_inspector->get_plugin_manager()->plugins();
-		s.engine->describe_rule(NULL, plugins, s.config->m_json_output);
-		return run_result::exit();
-	}
+		auto out = s.engine->describe_rule(rptr, plugins);
+
+		if (!s.config->m_json_output)
+		{
+			format_described_rules_as_text(out, std::cout);
+		}
+		else
+		{
+			std::cout << out.dump() << std::endl;
+		}
 
-	if (!s.options.describe_rule.empty())
-	{
-		const auto& plugins = s.offline_inspector->get_plugin_manager()->plugins();
-		s.engine->describe_rule(&(s.options.describe_rule), plugins, s.config->m_json_output);
 		return run_result::exit();
 	}
 

userspace/falco/app/actions/print_kernel_version.cpp

@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "actions.h"
+#include "helpers.h"
+#include "../app.h"
+#include <fstream>
+#include <sstream>
+#include <errno.h>
+
+using namespace falco::app;
+using namespace falco::app::actions;
+
+falco::app::run_result falco::app::actions::print_kernel_version(falco::app::state& s)
+{
+#ifdef __linux__
+	// We print this info only when a kernel driver is injected
+	if(s.is_modern_ebpf() || s.is_ebpf() || s.is_kmod())
+	{
+		std::ifstream input_file("/proc/version");
+		if(!input_file.is_open())
+		{
+			// We don't want to fail, we just need to log something
+			falco_logger::log(falco_logger::level::INFO, "Cannot read under '/proc/version' (err_message: '" + std::string(strerror(errno)) + "', err_code: " + std::to_string(errno) + "). No info provided, go on.");
+			return run_result::ok();
+		}
+
+		std::stringstream buffer;
+		buffer << input_file.rdbuf();
+		std::string contents(buffer.str());
+		falco_logger::log(falco_logger::level::INFO, "System info: " + contents);
+	}
+#endif
+	return run_result::ok();
+}

userspace/falco/app/actions/print_version.cpp

@@ -15,8 +15,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include <nlohmann/json.hpp>
-
 #include "actions.h"
 #include "../../versions_info.h"
 

userspace/falco/app/actions/process_events.cpp

@@ -45,16 +45,12 @@ class source_sync_context
 public:
 	source_sync_context(falco::semaphore& s)
 		: m_finished(false), m_joined(false), m_semaphore(s) { }
-	source_sync_context(source_sync_context&&) = default;
-	source_sync_context& operator = (source_sync_context&&) = default;
-	source_sync_context(const source_sync_context&) = delete;
-	source_sync_context& operator = (const source_sync_context&) = delete;
 
 	inline void finish()
 	{
 		bool v = false;
 		while (!m_finished.compare_exchange_weak(
-				v, true, 
+				v, true,
 				std::memory_order_seq_cst,
 				std::memory_order_seq_cst))
 		{
@@ -70,7 +66,7 @@ public:
 	{
 		bool v = false;
 		while (!m_joined.compare_exchange_weak(
-				v, true, 
+				v, true,
 				std::memory_order_seq_cst,
 				std::memory_order_seq_cst))
 		{
@@ -90,7 +86,7 @@ public:
 	{
 		return m_finished.load(std::memory_order_seq_cst);
 	}
-	
+
 private:
 	// set to true when the event processing loop finishes
 	std::atomic<bool> m_finished;
@@ -102,12 +98,6 @@ private:
 
 struct live_context
 {
-	live_context() = default;
-	live_context(live_context&&) = default;
-	live_context& operator = (live_context&&) = default;
-	live_context(const live_context&) = default;
-	live_context& operator = (const live_context&) = default;
-
 	// the name of the source of which events are processed
 	std::string source;
 	// the result of the event processing loop
@@ -269,7 +259,7 @@ static falco::app::run_result do_inspect(
 				}
 				return run_result::fatal(msg);
 			}
-	
+
 			// for capture mode, the source name can change at every event
 			stats_collector.collect(inspector, inspector->event_sources()[source_engine_idx], num_evts);
 		}
@@ -325,7 +315,7 @@ static falco::app::run_result do_inspect(
 				s.outputs->handle_event(rule_res.evt, rule_res.rule, rule_res.source, rule_res.priority_num, rule_res.format, rule_res.tags);
 			}
 		}
-		
+
 		num_evts++;
 	}
 

userspace/falco/app/actions/validate_rules_files.cpp

@@ -18,6 +18,8 @@ limitations under the License.
 #include "actions.h"
 #include "helpers.h"
 
+#include <plugin_manager.h>
+
 #include <string>
 
 using namespace falco::app;
@@ -112,24 +114,37 @@ falco::app::run_result falco::app::actions::validate_rules_files(falco::app::sta
 				// file was ok with warnings, without actually
 				// printing the warnings.
 				summary += filename + ": Ok, with warnings";
-
-				// If verbose is true, print the warnings now.
-				if(s.options.verbose)
-				{
-					fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
-				}
+				falco_logger::log(falco_logger::level::WARNING, res->as_string(true, rc) + "\n");
 			}
 		}
 
+		// printout of `-L` option
+		nlohmann::json describe_res;
+		if (successful && (s.options.describe_all_rules || !s.options.describe_rule.empty()))
+		{
+			std::string* rptr = !s.options.describe_rule.empty() ? &(s.options.describe_rule) : nullptr;
+			const auto& plugins = s.offline_inspector->get_plugin_manager()->plugins();
+			describe_res = s.engine->describe_rule(rptr, plugins);
+		}
+
 		if(s.config->m_json_output)
 		{
 			nlohmann::json res;
 			res["falco_load_results"] = results;
-			printf("%s\n", res.dump().c_str());
+			if (!describe_res.empty() && successful)
+			{
+				res["falco_describe_results"] = std::move(describe_res);
+			}
+			std::cout << res.dump() << std::endl;
 		}
 		else
 		{
-			printf("%s\n", summary.c_str());
+			std::cout << summary << std::endl;
+			if (!describe_res.empty() && successful)
+			{
+				std::cout << std::endl;
+				format_described_rules_as_text(describe_res, std::cout);
+			}
 		}
 
 		if(successful)

userspace/falco/app/app.cpp

@@ -62,6 +62,7 @@ bool falco::app::run(falco::app::state& s, bool& restart, std::string& errstr)
 	std::list<app_action> run_steps = {
 		falco::app::actions::load_config,
 		falco::app::actions::print_help,
+		falco::app::actions::print_kernel_version,
 		falco::app::actions::print_version,
 		falco::app::actions::print_page_size,
 		falco::app::actions::print_generated_gvisor_config,

userspace/falco/app/restart_handler.cpp

@@ -60,7 +60,7 @@ bool falco::app::restart_handler::start(std::string& err)
 
     for (const auto& f : m_watched_files)
     {
-        auto wd = inotify_add_watch(m_inotify_fd, f.c_str(), IN_CLOSE_WRITE);
+        auto wd = inotify_add_watch(m_inotify_fd, f.c_str(), IN_CLOSE_WRITE | IN_MOVE_SELF | IN_DELETE_SELF);
         if (wd < 0)
         {
             err = "could not watch file: " + f;
@@ -71,7 +71,7 @@ bool falco::app::restart_handler::start(std::string& err)
 
     for (const auto &f : m_watched_dirs)
     {
-        auto wd = inotify_add_watch(m_inotify_fd, f.c_str(), IN_CREATE | IN_DELETE);
+        auto wd = inotify_add_watch(m_inotify_fd, f.c_str(), IN_CREATE | IN_DELETE | IN_MOVE);
         if (wd < 0)
         {
             err = "could not watch directory: " + f;

userspace/falco/app/restart_handler.h

@@ -57,15 +57,11 @@ public:
               m_watched_dirs(watch_dirs),
               m_watched_files(watch_files) { }
     virtual ~restart_handler();
-    restart_handler(restart_handler&&) = default;
-    restart_handler& operator = (restart_handler&&) = default;
-    restart_handler(const restart_handler&) = delete;
-    restart_handler& operator = (const restart_handler&) = delete;
 
     bool start(std::string& err);
     void stop();
     void trigger();
-    
+
 private:
     void watcher_loop() noexcept;
 

userspace/falco/app/state.h

@@ -56,7 +56,7 @@ struct state
         source_info& operator = (source_info&&) = default;
         source_info(const source_info&) = default;
         source_info& operator = (const source_info&) = default;
-    
+
         // The index of the given event source in the state's falco_engine,
         // as returned by falco_engine::add_source
         std::size_t engine_idx;
@@ -93,10 +93,6 @@ struct state
     }
 
     ~state() = default;
-    state(state&&) = default;
-    state& operator = (state&&) = default;
-    state(const state&) = default;
-    state& operator = (const state&) = default;
 
     std::string cmdline;
     falco::app::options options;
@@ -145,7 +141,7 @@ struct state
     falco_webserver webserver;
 #endif
 
-    inline bool is_capture_mode() const 
+    inline bool is_capture_mode() const
     {
         return config->m_engine_mode == engine_kind_t::REPLAY;
     }
@@ -155,6 +151,11 @@ struct state
         return config->m_engine_mode == engine_kind_t::GVISOR;
     }
 
+    inline bool is_kmod() const
+    {
+        return config->m_engine_mode == engine_kind_t::KMOD;
+    }
+
     inline bool is_ebpf() const
     {
         return config->m_engine_mode == engine_kind_t::EBPF;
@@ -167,10 +168,10 @@ struct state
 
     inline bool is_nodriver() const
     {
-        return config->m_engine_mode == engine_kind_t::NONE;
+        return config->m_engine_mode == engine_kind_t::NODRIVER;
     }
 
-    inline bool is_source_enabled(const std::string& src) const 
+    inline bool is_source_enabled(const std::string& src) const
     {
         return enabled_sources.find(falco_common::syscall_source) != enabled_sources.end();
     }

userspace/falco/atomic_signal_handler.h

@@ -30,13 +30,6 @@ namespace falco
     class atomic_signal_handler
     {
     public:
-        atomic_signal_handler(): m_triggered(false), m_handled(false) { }
-        atomic_signal_handler(atomic_signal_handler&&) = default;
-        atomic_signal_handler& operator = (atomic_signal_handler&&) = default;
-        atomic_signal_handler(const atomic_signal_handler&) = delete;
-        atomic_signal_handler& operator = (const atomic_signal_handler&) = delete;
-        ~atomic_signal_handler() = default;
-        
         /**
          * @brief Returns true if the underlying atomic implementation
          * is lock-free as per C++ standard semantics.
@@ -95,7 +88,7 @@ namespace falco
          * performed. After the first handler has been performed, every
          * other invocation of handle() will be skipped and return false
          * up until the next invocation of reset().
-         * 
+         *
          * @param f The action to perform.
          * @return true If the action has been performed.
          * @return false If the action has not been performed.
@@ -134,7 +127,7 @@ namespace falco
 
     private:
         std::mutex m_mtx;
-        std::atomic<bool> m_triggered;
-        std::atomic<bool> m_handled;
+        std::atomic<bool> m_triggered{false};
+        std::atomic<bool> m_handled{false};
     };
 };

userspace/falco/configuration.cpp

@@ -69,9 +69,7 @@ falco_configuration::falco_configuration():
 	m_metrics_interval(5000),
 	m_metrics_stats_rule_enabled(false),
 	m_metrics_output_file(""),
-	m_metrics_resource_utilization_enabled(true),
-	m_metrics_kernel_event_counters_enabled(true),
-	m_metrics_libbpf_stats_enabled(true),
+	m_metrics_flags((PPM_SCAP_STATS_KERNEL_COUNTERS | PPM_SCAP_STATS_LIBBPF_STATS | PPM_SCAP_STATS_RESOURCE_UTILIZATION | PPM_SCAP_STATS_STATE_COUNTERS)),
 	m_metrics_convert_memory_to_mb(true),
 	m_metrics_include_empty_values(false)
 {
@@ -111,7 +109,7 @@ void falco_configuration::load_engine_config(const std::string& config_name, con
 		{"modern_ebpf",engine_kind_t::MODERN_EBPF},
 		{"replay",engine_kind_t::REPLAY},
 		{"gvisor",engine_kind_t::GVISOR},
-		{"none",engine_kind_t::NONE},
+		{"nodriver",engine_kind_t::NODRIVER},
 	};
 
 	auto driver_mode_str = config.get_scalar<std::string>("engine.kind", "kmod");
@@ -174,7 +172,7 @@ void falco_configuration::load_engine_config(const std::string& config_name, con
 		}
 		m_gvisor.m_root = config.get_scalar<std::string>("engine.gvisor.root", "");
 		break;
-	case engine_kind_t::NONE:
+	case engine_kind_t::NODRIVER:
 	default:
 		break;
 	}
@@ -311,6 +309,14 @@ void falco_configuration::load_yaml(const std::string& config_name, const yaml_h
 		client_key = config.get_scalar<std::string>("http_output.client_key", "/etc/ssl/certs/client.key");
 		http_output.options["client_key"] = client_key;
 
+		bool compress_uploads;
+		compress_uploads = config.get_scalar<bool>("http_output.compress_uploads", false);
+		http_output.options["compress_uploads"] = compress_uploads? std::string("true") : std::string("false");
+
+		bool keep_alive;
+		keep_alive = config.get_scalar<bool>("http_output.keep_alive", false);
+		http_output.options["keep_alive"] = keep_alive? std::string("true") : std::string("false");
+
 		m_outputs.push_back(http_output);
 	}
 
@@ -453,9 +459,29 @@ void falco_configuration::load_yaml(const std::string& config_name, const yaml_h
 	m_metrics_interval = falco::utils::parse_prometheus_interval(m_metrics_interval_str);
 	m_metrics_stats_rule_enabled = config.get_scalar<bool>("metrics.output_rule", false);
 	m_metrics_output_file = config.get_scalar<std::string>("metrics.output_file", "");
-	m_metrics_resource_utilization_enabled = config.get_scalar<bool>("metrics.resource_utilization_enabled", true);
-	m_metrics_kernel_event_counters_enabled = config.get_scalar<bool>("metrics.kernel_event_counters_enabled", true);
-	m_metrics_libbpf_stats_enabled = config.get_scalar<bool>("metrics.libbpf_stats_enabled", true);
+
+	m_metrics_flags = 0;
+	if (config.get_scalar<bool>("metrics.resource_utilization_enabled", true))
+	{
+		m_metrics_flags |= PPM_SCAP_STATS_RESOURCE_UTILIZATION;
+
+	}
+	if (config.get_scalar<bool>("metrics.state_counters_enabled", true))
+	{
+		m_metrics_flags |= PPM_SCAP_STATS_STATE_COUNTERS;
+
+	}
+	if (config.get_scalar<bool>("metrics.kernel_event_counters_enabled", true))
+	{
+		m_metrics_flags |= PPM_SCAP_STATS_KERNEL_COUNTERS;
+
+	}
+	if (config.get_scalar<bool>("metrics.libbpf_stats_enabled", true))
+	{
+		m_metrics_flags |= PPM_SCAP_STATS_LIBBPF_STATS;
+
+	}
+
 	m_metrics_convert_memory_to_mb = config.get_scalar<bool>("metrics.convert_memory_to_mb", true);
 	m_metrics_include_empty_values = config.get_scalar<bool>("metrics.include_empty_values", false);
 

userspace/falco/configuration.h

@@ -49,51 +49,44 @@ enum class engine_kind_t : uint8_t
 	MODERN_EBPF,
 	REPLAY,
 	GVISOR,
-	NONE
+	NODRIVER
 };
 
 class falco_configuration
 {
 public:
-
-	typedef struct {
-	public:
+	struct plugin_config {
 		std::string m_name;
 		std::string m_library_path;
 		std::string m_init_config;
 		std::string m_open_params;
-	} plugin_config;
+	};
 
-	typedef struct {
-	public:
+	struct kmod_config {
 		int16_t m_buf_size_preset;
 		bool m_drop_failed_exit;
-	} kmod_config;
+	};
 
-	typedef struct {
-	public:
+	struct ebpf_config {
 		std::string m_probe_path;
 		int16_t m_buf_size_preset;
 		bool m_drop_failed_exit;
-	} ebpf_config;
+	};
 
-	typedef struct {
-	public:
+	struct modern_ebpf_config {
 		uint16_t m_cpus_for_each_buffer;
 		int16_t m_buf_size_preset;
 		bool m_drop_failed_exit;
-	} modern_ebpf_config;
+	};
 
-	typedef struct {
-	public:
+	struct replay_config {
 		std::string m_capture_file;
-	} replay_config;
+	};
 
-	typedef struct {
-	public:
+	struct gvisor_config {
 		std::string m_config;
 		std::string m_root;
-	} gvisor_config;
+	};
 
 	falco_configuration();
 	virtual ~falco_configuration() = default;
@@ -158,9 +151,7 @@ public:
 	uint64_t m_metrics_interval;
 	bool m_metrics_stats_rule_enabled;
 	std::string m_metrics_output_file;
-	bool m_metrics_resource_utilization_enabled;
-	bool m_metrics_kernel_event_counters_enabled;
-	bool m_metrics_libbpf_stats_enabled;
+	uint32_t m_metrics_flags;
 	bool m_metrics_convert_memory_to_mb;
 	bool m_metrics_include_empty_values;
 	std::vector<plugin_config> m_plugins;

userspace/falco/falco_outputs.cpp

@@ -65,7 +65,7 @@ falco_outputs::falco_outputs(
 	{
 		add_output(output);
 	}
-	m_outputs_queue_num_drops = {0};
+	m_outputs_queue_num_drops = 0;
 #ifndef __EMSCRIPTEN__
 	m_queue.set_capacity(outputs_queue_capacity);
 	m_worker_thread = std::thread(&falco_outputs::worker, this);

userspace/falco/falco_semaphore.h

@@ -32,11 +32,6 @@ namespace falco
          * @brief Creates a semaphore with the given initial counter value
          */
         semaphore(int c = 0): count(c) {}
-        semaphore(semaphore&&) = default;
-        semaphore& operator = (semaphore&&) = default;
-        semaphore(const semaphore&) = delete;
-        semaphore& operator = (const semaphore&) = delete;
-        ~semaphore() = default;
 
         /**
          * @brief Increments the internal counter and unblocks acquirers

userspace/falco/outputs_http.cpp

@@ -97,6 +97,16 @@ bool falco::outputs::output_http::init(const config& oc, bool buffered, const st
 		CHECK_RES(curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, noop_write_callback));
 	}
 
+	if(m_oc.options["compress_uploads"] == std::string("true"))
+	{
+		CHECK_RES(curl_easy_setopt(m_curl, CURLOPT_TRANSFER_ENCODING, 1L));
+	}
+
+	if(m_oc.options["keep_alive"] == std::string("true"))
+	{
+		CHECK_RES(curl_easy_setopt(m_curl, CURLOPT_TCP_KEEPALIVE, 1L));
+	}
+
 	if(res != CURLE_OK)
 	{
 		err = "libcurl error: " + std::string(curl_easy_strerror(res));

userspace/falco/stats_writer.cpp

@@ -20,7 +20,6 @@ limitations under the License.
 #endif
 #include <ctime>
 #include <csignal>
-#include <nlohmann/json.hpp>
 #include <atomic>
 
 #include <nlohmann/json.hpp>
@@ -40,8 +39,8 @@ static timer_t s_timerid;
 #else
 static uint16_t s_timerid;
 #endif
-// note: Workaround for older GLIBC versions (< 2.35), where calling timer_delete() 
-// with an invalid timer ID not returned by timer_create() causes a segfault because of 
+// note: Workaround for older GLIBC versions (< 2.35), where calling timer_delete()
+// with an invalid timer ID not returned by timer_create() causes a segfault because of
 // a bug in GLIBC (https://sourceware.org/bugzilla/show_bug.cgi?id=28257).
 // Just performing a nullptr check is not enough as even after creating the timer, s_timerid
 // remains a nullptr somehow.
@@ -132,7 +131,7 @@ bool stats_writer::init_ticker(uint32_t interval_msec, std::string &err)
 	// delete any previously set timer
 	if (s_timerid_exists)
 	{
-		if (timer_delete(s_timerid) == -1) 
+		if (timer_delete(s_timerid) == -1)
 		{
 			err = std::string("Could not delete previous timer: ") + strerror(errno);
 			return false;
@@ -140,7 +139,7 @@ bool stats_writer::init_ticker(uint32_t interval_msec, std::string &err)
 		s_timerid_exists = false;
 	}
 
-	if (timer_create(CLOCK_MONOTONIC, &sev, &s_timerid) == -1) 
+	if (timer_create(CLOCK_MONOTONIC, &sev, &s_timerid) == -1)
 	{
 		err = std::string("Could not create periodic timer: ") + strerror(errno);
 		return false;
@@ -151,7 +150,7 @@ bool stats_writer::init_ticker(uint32_t interval_msec, std::string &err)
 	timer.it_value.tv_nsec = (interval_msec % 1000) * 1000 * 1000;
 	timer.it_interval = timer.it_value;
 
-	if (timer_settime(s_timerid, 0, &timer, NULL) == -1) 
+	if (timer_settime(s_timerid, 0, &timer, NULL) == -1)
 	{
 		err = std::string("Could not set up periodic timer: ") + strerror(errno);
 		return false;
@@ -265,7 +264,7 @@ void stats_writer::worker() noexcept
 		{
 			return;
 		}
-		
+
 		// this helps waiting for the first tick
 		tick = stats_writer::get_ticker();
 		if (first_tick != tick)
@@ -302,7 +301,7 @@ void stats_writer::worker() noexcept
 }
 
 stats_writer::collector::collector(const std::shared_ptr<stats_writer>& writer)
-	: m_writer(writer), m_last_tick(0), m_samples(0),
+	: m_writer(writer), m_last_tick(0),
 	  m_last_now(0), m_last_n_evts(0), m_last_n_drops(0), m_last_num_evts(0)
 {
 }
@@ -343,7 +342,7 @@ void stats_writer::collector::get_metrics_output_fields_wrapper(
 	if (m_last_num_evts != 0 && stats_snapshot_time_delta_sec > 0)
 	{
 		/* Successfully processed userspace event rate. */
-		output_fields["falco.evts_rate_sec"] = (double)((num_evts - m_last_num_evts) / (double)stats_snapshot_time_delta_sec);
+		output_fields["falco.evts_rate_sec"] = std::round((double)((num_evts - m_last_num_evts) / (double)stats_snapshot_time_delta_sec) * 10.0) / 10.0; // round to 1 decimal
 	}
 	output_fields["falco.num_evts"] = num_evts;
 	output_fields["falco.num_evts_prev"] = m_last_num_evts;
@@ -358,60 +357,87 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 	const scap_agent_info* agent_info = inspector->get_agent_info();
 
 #if !defined(MINIMAL_BUILD) and !defined(__EMSCRIPTEN__)
-	/* Resource utilization, CPU and memory usage etc. */
 	uint32_t nstats = 0;
 	int32_t rc = 0;
-	if (m_writer->m_config->m_metrics_resource_utilization_enabled)
+	uint32_t flags = m_writer->m_config->m_metrics_flags;
+
+	auto buffer = inspector->get_sinsp_stats_v2_buffer();
+	auto sinsp_stats_v2 = inspector->get_sinsp_stats_v2();
+	sinsp_thread_manager* thread_manager = inspector->m_thread_manager;
+	const scap_stats_v2* sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2, buffer, &nstats, &rc);
+
+	uint32_t base_stat = 0;
+	// todo @incertum this needs to become better with the next proper stats refactor in libs 0.15.0
+	if ((flags & PPM_SCAP_STATS_STATE_COUNTERS) && !(flags & PPM_SCAP_STATS_RESOURCE_UTILIZATION))
 	{
-		const scap_stats_v2* utilization;
-		auto buffer = inspector->get_sinsp_stats_v2_buffer();
-		utilization = libsinsp::resource_utilization::get_resource_utilization(agent_info, buffer, &nstats, &rc);
-		if (utilization && rc == 0 && nstats > 0)
+		base_stat = SINSP_STATS_V2_N_THREADS;
+	}
+
+	if (sinsp_stats_v2_snapshot && rc == 0 && nstats > 0)
+	{
+		for(uint32_t stat = base_stat; stat < nstats; stat++)
 		{
-			for(uint32_t stat = 0; stat < nstats; stat++)
+			if (sinsp_stats_v2_snapshot[stat].name[0] == '\0')
 			{
-				char metric_name[STATS_NAME_MAX] = "falco.";
-				strlcat(metric_name, utilization[stat].name, sizeof(metric_name));
-				switch(utilization[stat].type)
+				break;
+			}
+			char metric_name[STATS_NAME_MAX] = "falco.";
+			strlcat(metric_name, sinsp_stats_v2_snapshot[stat].name, sizeof(metric_name));
+			// todo @incertum temporary fix for n_fds and n_threads, type assignment was missed in libs, will be fixed in libs 0.15.0
+			if (strncmp(sinsp_stats_v2_snapshot[stat].name, "n_fds", 6) == 0 || strncmp(sinsp_stats_v2_snapshot[stat].name, "n_threads", 10) == 0)
+			{
+				output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.u64;
+			}
+
+			switch(sinsp_stats_v2_snapshot[stat].type)
+			{
+			case STATS_VALUE_TYPE_U64:
+				if (sinsp_stats_v2_snapshot[stat].value.u64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
 				{
-				case STATS_VALUE_TYPE_U64:
-					if (utilization[stat].value.u64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
-					{
-						break;
-					}
-					if (m_writer->m_config->m_metrics_convert_memory_to_mb && strncmp(utilization[stat].name, "container_memory_used", 22) == 0) // exact str match
-					{
-						output_fields[metric_name] = (uint64_t)(utilization[stat].value.u64 / (double)1024 / (double)1024);
-					}
-					else
-					{
-						output_fields[metric_name] = utilization[stat].value.u64;
-					}
-					break;
-				case STATS_VALUE_TYPE_U32:
-					if (utilization[stat].value.u32 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
-					{
-						break;
-					}
-					if (m_writer->m_config->m_metrics_convert_memory_to_mb && strncmp(utilization[stat].name, "memory_", 7) == 0) // prefix match
-					{
-						output_fields[metric_name] = (uint32_t)(utilization[stat].value.u32 / (double)1024);
-					}
-					else
-					{
-						output_fields[metric_name] = utilization[stat].value.u32;
-					}
-					break;
-				case STATS_VALUE_TYPE_D:
-					if (utilization[stat].value.d == 0 && !m_writer->m_config->m_metrics_include_empty_values)
-					{
-						break;
-					}
-					output_fields[metric_name] = utilization[stat].value.d;
-					break;
-				default:
 					break;
 				}
+				if (m_writer->m_config->m_metrics_convert_memory_to_mb)
+				{
+					if (strncmp(sinsp_stats_v2_snapshot[stat].name, "container_memory_used", 22) == 0) // exact str match
+					{
+						output_fields[metric_name] = (uint64_t)(sinsp_stats_v2_snapshot[stat].value.u64 / (double)1024 / (double)1024);
+
+					} else if (strncmp(sinsp_stats_v2_snapshot[stat].name, "memory_", 7) == 0) // prefix match
+					{
+						output_fields[metric_name] = (uint64_t)(sinsp_stats_v2_snapshot[stat].value.u64 / (double)1024);
+					} else
+					{
+						output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.u64;
+					}
+				}
+				else
+				{
+					output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.u64;
+				}
+				break;
+			case STATS_VALUE_TYPE_U32:
+				if (sinsp_stats_v2_snapshot[stat].value.u32 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				if (m_writer->m_config->m_metrics_convert_memory_to_mb && strncmp(sinsp_stats_v2_snapshot[stat].name, "memory_", 7) == 0) // prefix match
+				{
+					output_fields[metric_name] = (uint32_t)(sinsp_stats_v2_snapshot[stat].value.u32 / (double)1024);
+				}
+				else
+				{
+					output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.u32;
+				}
+				break;
+			case STATS_VALUE_TYPE_D:
+				if (sinsp_stats_v2_snapshot[stat].value.d == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.d;
+				break;
+			default:
+				break;
 			}
 		}
 	}
@@ -424,18 +450,16 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 	/* Kernel side stats counters and libbpf stats if applicable. */
 	nstats = 0;
 	rc = 0;
-	uint32_t flags = 0;
+	if (!(inspector->check_current_engine(BPF_ENGINE) || inspector->check_current_engine(MODERN_BPF_ENGINE)))
+	{
+		flags &= ~PPM_SCAP_STATS_LIBBPF_STATS;
+	}
 
-	if (m_writer->m_config->m_metrics_kernel_event_counters_enabled)
-	{
-		flags |= PPM_SCAP_STATS_KERNEL_COUNTERS;
-	}
-	if (m_writer->m_config->m_metrics_libbpf_stats_enabled && (inspector->check_current_engine(BPF_ENGINE) || inspector->check_current_engine(MODERN_BPF_ENGINE)))
-	{
-		flags |= PPM_SCAP_STATS_LIBBPF_STATS;
-	}
-	const scap_stats_v2* stats_v2 = inspector->get_capture_stats_v2(flags, &nstats, &rc);
-	if (stats_v2 && nstats > 0 && rc == 0)
+	// Note: ENGINE_FLAG_BPF_STATS_ENABLED check has been moved to libs, that is, when libbpf stats is not enabled
+	// in the kernel settings we won't collect them even if the end user enabled the libbpf stats option
+
+	const scap_stats_v2* scap_stats_v2_snapshot = inspector->get_capture_stats_v2(flags, &nstats, &rc);
+	if (scap_stats_v2_snapshot && nstats > 0 && rc == 0)
 	{
 		/* Cache n_evts and n_drops to derive n_drops_perc. */
 		uint64_t n_evts = 0;
@@ -444,24 +468,28 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 		uint64_t n_drops_delta = 0;
 		for(uint32_t stat = 0; stat < nstats; stat++)
 		{
+			if (scap_stats_v2_snapshot[stat].name[0] == '\0')
+			{
+				break;
+			}
 			// todo: as we expand scap_stats_v2 prefix may be pushed to scap or we may need to expand
 			// functionality here for example if we add userspace syscall counters that should be prefixed w/ `falco.`
 			char metric_name[STATS_NAME_MAX] = "scap.";
-			strlcat(metric_name, stats_v2[stat].name, sizeof(metric_name));
-			switch(stats_v2[stat].type)
+			strlcat(metric_name, scap_stats_v2_snapshot[stat].name, sizeof(metric_name));
+			switch(scap_stats_v2_snapshot[stat].type)
 			{
 			case STATS_VALUE_TYPE_U64:
 				/* Always send high level n_evts related fields, even if zero. */
-				if (strncmp(stats_v2[stat].name, "n_evts", 7) == 0) // exact not prefix match here
+				if (strncmp(scap_stats_v2_snapshot[stat].name, "n_evts", 7) == 0) // exact not prefix match here
 				{
-					n_evts = stats_v2[stat].value.u64;
+					n_evts = scap_stats_v2_snapshot[stat].value.u64;
 					output_fields[metric_name] = n_evts;
 					output_fields["scap.n_evts_prev"] = m_last_n_evts;
 					n_evts_delta = n_evts - m_last_n_evts;
 					if (n_evts_delta != 0 && stats_snapshot_time_delta_sec > 0)
 					{
 						/* n_evts is total number of kernel side events. */
-						output_fields["scap.evts_rate_sec"] = (double)(n_evts_delta / stats_snapshot_time_delta_sec);
+						output_fields["scap.evts_rate_sec"] = std::round((double)(n_evts_delta / stats_snapshot_time_delta_sec) * 10.0) / 10.0; // round to 1 decimal
 					}
 					else
 					{
@@ -470,16 +498,16 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 					m_last_n_evts = n_evts;
 				}
 				/* Always send high level n_drops related fields, even if zero. */
-				else if (strncmp(stats_v2[stat].name, "n_drops", 8) == 0) // exact not prefix match here
+				else if (strncmp(scap_stats_v2_snapshot[stat].name, "n_drops", 8) == 0) // exact not prefix match here
 				{
-					n_drops = stats_v2[stat].value.u64;
+					n_drops = scap_stats_v2_snapshot[stat].value.u64;
 					output_fields[metric_name] = n_drops;
 					output_fields["scap.n_drops_prev"] = m_last_n_drops;
 					n_drops_delta = n_drops - m_last_n_drops;
 					if (n_drops_delta != 0 && stats_snapshot_time_delta_sec > 0)
 					{
 						/* n_drops is total number of kernel side event drops. */
-						output_fields["scap.evts_drop_rate_sec"] = (double)(n_drops_delta / stats_snapshot_time_delta_sec);
+						output_fields["scap.evts_drop_rate_sec"] = std::round((double)(n_drops_delta / stats_snapshot_time_delta_sec) * 10.0) / 10.0; // round to 1 decimal
 					}
 					else
 					{
@@ -487,11 +515,11 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 					}
 					m_last_n_drops = n_drops;
 				}
-				if (stats_v2[stat].value.u64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				if (scap_stats_v2_snapshot[stat].value.u64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
 				{
 					break;
 				}
-				output_fields[metric_name] = stats_v2[stat].value.u64;
+				output_fields[metric_name] = scap_stats_v2_snapshot[stat].value.u64;
 				break;
 			default:
 				break;

userspace/falco/stats_writer.h

@@ -50,7 +50,7 @@ public:
 		This class is not thread-safe.
 	*/
 	class collector
-	{	
+	{
 	public:
 		/*!
 			\brief Initializes the collector with the given writer
@@ -74,11 +74,8 @@ public:
 		*/
 		void get_metrics_output_fields_additional(nlohmann::json& output_fields, const std::shared_ptr<sinsp>& inspector, double stats_snapshot_time_delta_sec, const std::string& src);
 
-	
 		std::shared_ptr<stats_writer> m_writer;
 		stats_writer::ticker_t m_last_tick;
-		uint64_t m_samples;
-		scap_stats m_last_stats;
 		uint64_t m_last_now;
 		uint64_t m_last_n_evts;
 		uint64_t m_last_n_drops;

userspace/falco/yaml_helper.h

@@ -37,6 +37,40 @@ limitations under the License.
 #include "event_drops.h"
 #include "falco_outputs.h"
 
+class yaml_helper;
+
+class yaml_visitor {
+private:
+	using Callback = std::function<void(YAML::Node&)>;
+	yaml_visitor(Callback cb): seen(), cb(std::move(cb)) {}
+
+	void operator()(YAML::Node &cur) {
+		seen.push_back(cur);
+		if (cur.IsMap()) {
+			for (YAML::detail::iterator_value pair : cur) {
+				descend(pair.second);
+			}
+		} else if (cur.IsSequence()) {
+			for (YAML::detail::iterator_value child : cur) {
+				descend(child);
+			}
+		} else if (cur.IsScalar()) {
+			cb(cur);
+		}
+	}
+
+	void descend(YAML::Node &target) {
+		if (std::find(seen.begin(), seen.end(), target) == seen.end()) {
+			(*this)(target);
+		}
+	}
+
+	std::vector<YAML::Node> seen;
+	Callback cb;
+
+	friend class yaml_helper;
+};
+
 /**
  * @brief An helper class for reading and editing YAML documents
  */
@@ -49,6 +83,7 @@ public:
 	void load_from_string(const std::string& input)
 	{
 		m_root = YAML::Load(input);
+		pre_process_env_vars();
 	}
 
 	/**
@@ -57,6 +92,7 @@ public:
 	void load_from_file(const std::string& path)
 	{
 		m_root = YAML::LoadFile(path);
+		pre_process_env_vars();
 	}
 
 	/**
@@ -77,44 +113,8 @@ public:
 		get_node(node, key);
 		if(node.IsDefined())
 		{
-			std::string value = node.as<std::string>();
-
-			// Helper function to convert string to the desired type T
-			auto convert_str_to_t = [&default_value](const std::string& str) -> T {
-				std::stringstream ss(str);
-				T result;
-				if (ss >> result) return result;
-				return default_value;
-			};
-
-			// If the value starts with `$$`, check for a subsequent `{...}`
-			if (value.size() >= 3 && value[0] == '$' && value[1] == '$')
-			{
-				// If after stripping the first `$`, the string format is like `${VAR}`, treat it as a plain string and don't resolve.
-				if (value[2] == '{' && value[value.size() - 1] == '}')
-				{
-					value = value.substr(1);
-					return convert_str_to_t(value);
-				}
-				else return convert_str_to_t(value);
-			}
-
-			// Check if the value is an environment variable reference
-			if(value.size() >= 2 && value[0] == '$' && value[1] == '{' && value[value.size() - 1] == '}')
-			{
-				// Format: ${ENV_VAR_NAME}
-				std::string env_var = value.substr(2, value.size() - 3);
-
-				const char* env_value = std::getenv(env_var.c_str()); // Get the environment variable value
-				if(env_value) return convert_str_to_t(env_value);
-
-				return default_value;
-			}
-
-			// If it's not an environment variable reference, return the value as is
-			return node.as<T>();
+			return node.as<T>(default_value);
 		}
-
 		return default_value;
 	}
 
@@ -153,6 +153,71 @@ public:
 private:
 	YAML::Node m_root;
 
+	/*
+	 * When loading a yaml file,
+	 * we immediately pre process all scalar values through a visitor private API,
+	 * and resolve any "${env_var}" to its value;
+	 * moreover, any "$${str}" is resolved to simply "${str}".
+	 */
+	void pre_process_env_vars()
+	{
+		yaml_visitor([](YAML::Node &scalar) {
+				auto value = scalar.as<std::string>();
+				auto start_pos = value.find('$');
+				while (start_pos != std::string::npos)
+				{
+					auto substr = value.substr(start_pos);
+					// Case 1 -> ${}
+					if (substr.rfind("${", 0) == 0)
+					{
+						auto end_pos = substr.find('}');
+						if (end_pos != std::string::npos)
+						{
+							// Eat "${" and "}" when getting the env var name
+							auto env_str = substr.substr(2, end_pos - 2);
+							const char* env_value = std::getenv(env_str.c_str()); // Get the environment variable value
+							if(env_value)
+							{
+								// env variable name + "${}"
+								value.replace(start_pos, env_str.length() + 3, env_value);
+							}
+							else
+							{
+								value.erase(start_pos, env_str.length() + 3);
+							}
+						}
+						else
+						{
+							// There are no "}" chars anymore; just break leaving rest of value untouched.
+							break;
+						}
+					}
+					// Case 2 -> $${}
+					else if (substr.rfind("$${", 0) == 0)
+					{
+						auto end_pos = substr.find('}');
+						if (end_pos != std::string::npos)
+						{
+							// Consume first "$" token
+							value.erase(start_pos, 1);
+						}
+						else
+						{
+							// There are no "}" chars anymore; just break leaving rest of value untouched.
+							break;
+						}
+						start_pos++; // consume the second '$' token
+					}
+					else
+					{
+						start_pos += substr.length();
+					}
+					start_pos = value.find("$", start_pos);
+				}
+				scalar = value;
+			})(m_root);
+	}
+
 	/**
 	 * Key is a string representing a node in the YAML document.
 	 * The provided key string can navigate the document in its