update(cmake): bump libs and driver to master

Signed-off-by: Federico Di Pierro <nierro92@gmail.com>

.github/workflows/ci.yml

@@ -12,6 +12,9 @@ concurrency:
   group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true  
 
+permissions:  
+  contents: read
+  
 jobs:
   fetch-version:
     uses: ./.github/workflows/reusable_fetch_version.yaml

.github/workflows/codeql.yaml

@@ -18,6 +18,9 @@ on:
     # The branches below must be a subset of the branches above
     branches: [ "master" ]
 
+permissions:  
+  contents: read
+
 jobs:
   analyze:
     name: Analyze

.github/workflows/codespell.yml

@@ -1,6 +1,10 @@
 name: Codespell
 on:
   pull_request:
+    
+permissions:  
+  contents: read
+
 jobs:
   codespell:
     runs-on: ubuntu-latest

.github/workflows/engine-version-weakcheck.yaml

@@ -9,6 +9,9 @@ on:
       - 'userspace/engine/*.cpp'
       - 'userspace/engine/*.h'
 
+permissions:  
+  contents: read
+
 jobs:
   paths-filter:
     runs-on: ubuntu-latest

.github/workflows/insecure-api.yaml

@@ -6,12 +6,15 @@ on:
       - 'release/**'
       - 'maintainers/**'
 
+permissions:  
+  contents: read
+
 jobs:
   insecure-api:
     name: check-insecure-api
     runs-on: ubuntu-latest
     container:
-      image: returntocorp/semgrep:1.41.0@sha256:85956fbe795a0e8a3825d5252f175887c0e0c6ce7a766a07062c0fb68415cd67
+      image: semgrep/semgrep:1.85.0@sha256:b4c2272e0a2e59ca551ff96d3bbae657bd2b7356e339af557b27a96d9e751544
     steps:
       - name: Checkout Falco ⤵️
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0

.github/workflows/master.yaml

@@ -8,6 +8,9 @@ concurrency:
   group: ci-master
   cancel-in-progress: true  
 
+permissions:  
+  contents: read
+
 jobs:
   fetch-version:
     uses: ./.github/workflows/reusable_fetch_version.yaml 

.github/workflows/release.yaml

@@ -8,6 +8,9 @@ concurrency:
   group: ci-release
   cancel-in-progress: true  
 
+permissions:  
+  contents: read
+  
 jobs:
   release-settings:
     runs-on: ubuntu-latest
@@ -16,7 +19,7 @@ jobs:
       bucket_suffix: ${{ steps.get_settings.outputs.bucket_suffix }}
     steps:
       - name: Get latest release
-        uses: rez0n/actions-github-release@v2.0
+        uses: rez0n/actions-github-release@27a57820ee808f8fd940c8a9d1f7188f854aa2b5 # v2.0
         id: latest_release
         env:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/reusable_build_dev.yaml

@@ -33,6 +33,9 @@ on:
         default: ''
         type: string
 
+permissions: 
+  contents: read
+
 jobs:
   build-and-test:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936

.github/workflows/reusable_build_docker.yaml

@@ -24,6 +24,10 @@ on:
 # then we upload all the tarballs to be later downloaded by reusable_publish_docker workflow.
 # In this way, we don't need to publish any arch specific image, 
 # and this "build" workflow is actually only building images.
+
+permissions:  
+  contents: read
+
 jobs:
   build-docker:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936

.github/workflows/reusable_build_packages.yaml

@@ -21,6 +21,9 @@ on:
         type: boolean
         default: false
 
+permissions:  
+  contents: read
+
 jobs:
   build-modern-bpf-skeleton:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
@@ -49,15 +52,37 @@ jobs:
           retention-days: 1
 
   build-packages:
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
     runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
     needs: [build-modern-bpf-skeleton]
     container: centos:7
     steps:
       # Always install deps before invoking checkout action, to properly perform a full clone.
-      - name: Install build dependencies
+      - name: Fix mirrors to use vault.centos.org
+        run: |
+          sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
+          sed -i s/^#.*baseurl=http/baseurl=https/g /etc/yum.repos.d/*.repo
+          sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo
+
+      - name: Install scl repos
         run: |
           yum -y install centos-release-scl
+
+      - name: Fix new mirrors to use vault.centos.org
+        run: |
+          sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
+          sed -i s/^#.*baseurl=http/baseurl=https/g /etc/yum.repos.d/*.repo
+          sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo
+
+      - name: Fix arm64 scl repos to use correct mirror
+        if: inputs.arch == 'aarch64'
+        run: |
+          sed -i 's/vault.centos.org\/centos/vault.centos.org\/altarch/g' /etc/yum.repos.d/CentOS-SCLo-scl*.repo
+
+      - name: Install build deps
+        run: |
           yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++
           source /opt/rh/devtoolset-9/enable
           yum install -y wget git make m4 rpm-build elfutils-libelf-devel perl-IPC-Cmd devtoolset-9-libasan-devel devtoolset-9-libubsan-devel

.github/workflows/reusable_fetch_version.yaml

@@ -6,6 +6,9 @@ on:
         description: "Falco version"
         value: ${{ jobs.fetch-version.outputs.version }}
 
+permissions:  
+  contents: read
+
 jobs:
   # We need to use an ubuntu-latest to fetch Falco version because
   # Falco version is computed by some cmake scripts that do git sorceries

.github/workflows/reusable_publish_docker.yaml

@@ -152,9 +152,7 @@ jobs:
 
       - name: Setup Cosign
         if: inputs.sign
-        uses: sigstore/cosign-installer@11086d25041f77fe8fe7b9ea4e48e3b9192b8f19 # v3.1.2
-        with:
-          cosign-release: v2.0.2
+        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 # v3.5.0
 
       - name: Sign images with cosign
         if: inputs.sign

.github/workflows/reusable_test_packages.yaml

@@ -21,6 +21,9 @@ on:
         default: false
         type: boolean
 
+permissions:  
+  contents: read
+  
 jobs:
   test-packages:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
@@ -54,11 +57,12 @@ jobs:
       - name: Run tests
         env:
           LSAN_OPTIONS: "intercept_tls_get_addr=0"
-        uses: falcosecurity/testing@main    
+        uses: falcosecurity/testing@main 
         with:
           test-falco: 'true'
           test-falcoctl: 'true'
           test-k8saudit: 'true'
+          test-dummy: 'true'
           static: ${{ inputs.static && 'true' || 'false' }}
           test-drivers: ${{ inputs.arch == 'x86_64' && 'true' || 'false' }}
           show-all: 'true'

.github/workflows/staticanalysis.yaml

@@ -1,6 +1,9 @@
 name: StaticAnalysis
 on:
   pull_request:
+permissions:  
+  contents: read
+
 jobs:
   staticanalysis:
     runs-on: ubuntu-22.04

ADOPTERS.md

@@ -74,6 +74,8 @@ This is a list of production adopters of Falco (in alphabetical order):
 
 * [Thought Machine](https://www.thoughtmachine.net) Thought Machine builds Vault Core and Vault Payments: cloud-native core and payments technology enabling banks and fintechs to remain competitive and flourish into the future. Vault Core and Vault Payments are the foundation layer of a bank's technology stack. They can run any bank, any product, and any payment set. Thought Machine uses Falco to perform cloud agnostic real time detections of suspicious container behaviour.
 
+* [Tulip Retail](https://tulip.com) Tulip Retail uses Falco to monitor container activity in our environments. It's numerous integration points, easy deployment and easily customizable rules were the main reasons we chose Falco.
+
 * [Vinted](https://vinted.com/) Vinted uses Falco to continuously monitor container activities, identifying security threats, and ensuring compliance. The container-native approach, rule-based real-time threat detection, community support, extensibility, and compliance capabilities are the main factors why we chose it to enhance Vinted Kubernetes security. Falco Sidekick is used to send critical and warning severity alerts to our incident management solution (RTIR).
 
 * [Xenit AB](https://xenit.se/contact/) Xenit is a growth company with services within cloud and digital transformation. We provide an open-source Kubernetes framework that we leverage to help our customers get their applications to production as quickly and as securely as possible. We use Falco's detection capabilities to identify anomalous behaviour within our clusters in both Azure and AWS.

CHANGELOG.md

@@ -1,5 +1,181 @@
 # Change Log
 
+## v0.38.2
+
+Released on 2024-08-19
+
+### Bug Fixes
+
+* fix(engine): fix metrics names to better adhere to best practices [[#3272](https://github.com/falcosecurity/falco/pull/3272)] - [@incertum](https://github.com/incertum)
+* fix(ci): use vault.centos.org for centos:7 CI build. [[#3274](https://github.com/falcosecurity/falco/pull/3274)] - [@FedeDP](https://github.com/FedeDP)
+
+## v0.38.1
+
+Released on 2024-06-19
+
+### Major Changes
+
+* new(metrics): enable plugins metrics [[#3228](https://github.com/falcosecurity/falco/pull/3228)] - [@mrgian](https://github.com/mrgian)
+
+
+### Minor Changes
+
+* cleanup(falco): clarify that --print variants only affect syscalls [[#3238](https://github.com/falcosecurity/falco/pull/3238)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* update(engine): enable -p option for all sources, -pk, -pc etc only for syscall sources [[#3239](https://github.com/falcosecurity/falco/pull/3239)] - [@LucaGuerra](https://github.com/LucaGuerra)
+
+
+### Bug Fixes
+
+* fix(engine): enable output substitution only for syscall rules, prevent engine from exiting with validation errors when a plugin is loaded and -pc/pk is specified [[#3236](https://github.com/falcosecurity/falco/pull/3236)] - [@mrgian](https://github.com/mrgian)
+* fix(metrics): allow each metric output channel to be selected independently [[#3232](https://github.com/falcosecurity/falco/pull/3232)] - [@incertum](https://github.com/incertum)
+* fix(userspace/falco): fixed `falco_metrics::to_text` implementation when running with plugins [[#3230](https://github.com/falcosecurity/falco/pull/3230)] - [@FedeDP](https://github.com/FedeDP)
+
+
+
+### Statistics
+
+|   MERGED PRS    | NUMBER |
+|-----------------|--------|
+| Not user-facing |      0 |
+| Release note    |      6 |
+| Total           |      6 |
+
+## v0.38.0
+
+Released on 2024-05-30
+
+### Breaking Changes :warning:
+
+* new(scripts,docker)!: enable automatic driver selection logic in packages and docker images. Modern eBPF is now also the default driver and the highest priority one in the new driver selection logic. [[#3154](https://github.com/falcosecurity/falco/pull/3154)] - [@FedeDP](https://github.com/FedeDP)
+* cleanup(falco.yaml)!: remove some deprecated configs [[#3087](https://github.com/falcosecurity/falco/pull/3087)] - [@Andreagit97](https://github.com/Andreagit97)
+* cleanup(docker)!: remove unused builder dockerfile [[#3088](https://github.com/falcosecurity/falco/pull/3088)] - [@Andreagit97](https://github.com/Andreagit97)
+
+More details: https://falco.org/blog/falco-0-38-0/#breaking-changes-and-deprecations
+
+### Major Changes
+
+* new(webserver): a metrics endpoint has been added providing prometheus metrics. It can be optionally enabled using the new `metrics.prometheus_enabled` configuration option. It will only be activated if the `metrics.enabled` is true as well. [[#3140](https://github.com/falcosecurity/falco/pull/3140)] - [@sgaist](https://github.com/sgaist)
+* new(metrics): add `rules_counters_enabled` option [[#3192](https://github.com/falcosecurity/falco/pull/3192)] - [@incertum](https://github.com/incertum)
+* new(build): provide signatures for .tar.gz packages [[#3201](https://github.com/falcosecurity/falco/pull/3201)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* new(engine): add print_enabled_rules_falco_logger when log_level debug [[#3189](https://github.com/falcosecurity/falco/pull/3189)] - [@incertum](https://github.com/incertum)
+* new(falco): allow selecting which rules to load from the configuration file or command line [[#3178](https://github.com/falcosecurity/falco/pull/3178)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* new(metrics): add file sha256sum metrics for loaded config and rules files [[#3187](https://github.com/falcosecurity/falco/pull/3187)] - [@incertum](https://github.com/incertum)
+* new(engine): throw an error when an invalid macro/list name is used [[#3116](https://github.com/falcosecurity/falco/pull/3116)] - [@mrgian](https://github.com/mrgian)
+* new(engine): raise warning instead of error on invalid macro/list name [[#3167](https://github.com/falcosecurity/falco/pull/3167)] - [@mrgian](https://github.com/mrgian)
+* new(userspace): support split config files [[#3024](https://github.com/falcosecurity/falco/pull/3024)] - [@FedeDP](https://github.com/FedeDP)
+* new(engine): enforce unique exceptions names [[#3134](https://github.com/falcosecurity/falco/pull/3134)] - [@mrgian](https://github.com/mrgian)
+* new(engine): add warning when appending an exception with no values [[#3133](https://github.com/falcosecurity/falco/pull/3133)] - [@mrgian](https://github.com/mrgian)
+* feat(metrics): coherent metrics stats model  including few metrics naming changes [[#3129](https://github.com/falcosecurity/falco/pull/3129)] - [@incertum](https://github.com/incertum)
+* new(config): add `falco_libs.thread_table_size` [[#3071](https://github.com/falcosecurity/falco/pull/3071)] - [@incertum](https://github.com/incertum)
+* new(proposals): introduce on host anomaly detection framework [[#2655](https://github.com/falcosecurity/falco/pull/2655)] - [@incertum](https://github.com/incertum)
+
+
+### Minor Changes
+
+* update(cmake): bump falcoctl to v0.8.0. [[#3219](https://github.com/falcosecurity/falco/pull/3219)] - [@FedeDP](https://github.com/FedeDP)
+* update(rules): update falco-rules to 3.1.0 [[#3217](https://github.com/falcosecurity/falco/pull/3217)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* refactor(userspace): move falco logger under falco engine [[#3208](https://github.com/falcosecurity/falco/pull/3208)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* chore(docs): apply features adoption and deprecation proposal to config file keys [[#3206](https://github.com/falcosecurity/falco/pull/3206)] - [@FedeDP](https://github.com/FedeDP)
+* cleanup(metrics): add original rule name as label [[#3205](https://github.com/falcosecurity/falco/pull/3205)] - [@incertum](https://github.com/incertum)
+* update(falco): deprecate options -T, -t and -D [[#3193](https://github.com/falcosecurity/falco/pull/3193)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* refactor: bump libs and driver, support field modifiers [[#3186](https://github.com/falcosecurity/falco/pull/3186)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* chore(userspace/falco): deprecated old 'rules_file' config key [[#3162](https://github.com/falcosecurity/falco/pull/3162)] - [@FedeDP](https://github.com/FedeDP)
+* chore(falco): update falco libs and driver to master (Apr 8th 2024) [[#3158](https://github.com/falcosecurity/falco/pull/3158)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* update(build): update libs to 026ffe1d8f1b25c6ccdc09afa2c02afdd3e3f672 [[#3151](https://github.com/falcosecurity/falco/pull/3151)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* cleanup: minor adjustments to readme, add new testing section [[#3072](https://github.com/falcosecurity/falco/pull/3072)] - [@incertum](https://github.com/incertum)
+* refactor(userspace/engine): reduce allocations during rules loading [[#3065](https://github.com/falcosecurity/falco/pull/3065)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* update(CI): publish wasm package as dev-wasm [[#3017](https://github.com/falcosecurity/falco/pull/3017)] - [@Rohith-Raju](https://github.com/Rohith-Raju)
+
+
+### Bug Fixes
+
+* fix(userspace/falco): fix state initialization avoid a crash during hot reload [[#3190](https://github.com/falcosecurity/falco/pull/3190)] - [@FedeDP](https://github.com/FedeDP)
+* fix(userspace/engine): make sure exception fields are not optional in replace mode [[#3108](https://github.com/falcosecurity/falco/pull/3108)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* fix(docker): added zstd to driver loader images [[#3203](https://github.com/falcosecurity/falco/pull/3203)] - [@FedeDP](https://github.com/FedeDP)
+* fix(engine): raise warning instead of error on not-unique exceptions names [[#3159](https://github.com/falcosecurity/falco/pull/3159)] - [@mrgian](https://github.com/mrgian)
+* fix(engine): apply output substitutions for all sources [[#3135](https://github.com/falcosecurity/falco/pull/3135)] - [@mrgian](https://github.com/mrgian)
+* fix(userspace/configuration): make sure that folders that would trigger permission denied are not traversed [[#3127](https://github.com/falcosecurity/falco/pull/3127)] - [@sgaist](https://github.com/sgaist)
+* fix(engine): logical issue in exceptions condition [[#3115](https://github.com/falcosecurity/falco/pull/3115)] - [@mrgian](https://github.com/mrgian)
+* fix(cmake): properly let falcoctl cmake module create /usr/share/falco/plugins/ folder. [[#3105](https://github.com/falcosecurity/falco/pull/3105)] - [@FedeDP](https://github.com/FedeDP)
+
+
+
+### Non user-facing changes
+
+* update(scripts/falcoctl): bump falco-rules version to 3 [[#3128](https://github.com/falcosecurity/falco/pull/3128)] - [@alacuku](https://github.com/alacuku)
+* build(deps): Bump submodules/falcosecurity-rules from `59bf03b` to `9e56293` [[#3212](https://github.com/falcosecurity/falco/pull/3212)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* chore(gha): update cosign to v3.5.0 [[#3209](https://github.com/falcosecurity/falco/pull/3209)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* build(deps): Bump submodules/falcosecurity-rules from `29c41c4` to `59bf03b` [[#3207](https://github.com/falcosecurity/falco/pull/3207)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* update(cmake): bumped libs to 0.17.0-rc1 and falcoctl to v0.8.0-rc6. [[#3204](https://github.com/falcosecurity/falco/pull/3204)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `3f668d0` to `3cac61c` [[#3044](https://github.com/falcosecurity/falco/pull/3044)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-testing from `ae3950a` to `7abf76f` [[#3094](https://github.com/falcosecurity/falco/pull/3094)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* fix(ci): enforce bundled deps OFF in build-dev CI [[#3118](https://github.com/falcosecurity/falco/pull/3118)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `88a40c8` to `869c9a7` [[#3156](https://github.com/falcosecurity/falco/pull/3156)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* update(cmake): bumped falcoctl to v0.8.0-rc5. [[#3199](https://github.com/falcosecurity/falco/pull/3199)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `4f153f5` to `29c41c4` [[#3198](https://github.com/falcosecurity/falco/pull/3198)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* update(cmake): bump falcoctl to v0.8.0-rc4 [[#3191](https://github.com/falcosecurity/falco/pull/3191)] - [@FedeDP](https://github.com/FedeDP)
+* refactor: smart pointer usage [[#3184](https://github.com/falcosecurity/falco/pull/3184)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* build(deps): Bump submodules/falcosecurity-rules from `ec255e6` to `4f153f5` [[#3182](https://github.com/falcosecurity/falco/pull/3182)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* update(cmake): bumped libs and driver to latest master. [[#3177](https://github.com/falcosecurity/falco/pull/3177)] - [@FedeDP](https://github.com/FedeDP)
+* chore(cmake): enable modern bpf build by default. [[#3180](https://github.com/falcosecurity/falco/pull/3180)] - [@FedeDP](https://github.com/FedeDP)
+* cleanup(docs): fix typo in license blocks [[#3175](https://github.com/falcosecurity/falco/pull/3175)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(docker,scripts): set old eBPF probe as lowest priority driver. [[#3173](https://github.com/falcosecurity/falco/pull/3173)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `869c9a7` to `ec255e6` [[#3170](https://github.com/falcosecurity/falco/pull/3170)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* update(app): close inspectors at teardown time [[#3169](https://github.com/falcosecurity/falco/pull/3169)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* fix(docker): fixed docker entrypoints for driver loading. [[#3168](https://github.com/falcosecurity/falco/pull/3168)] - [@FedeDP](https://github.com/FedeDP)
+* fix(docker,scripts): do not load falcoctl driver loader when installing Falco deb package in docker images [[#3166](https://github.com/falcosecurity/falco/pull/3166)] - [@FedeDP](https://github.com/FedeDP)
+* update(ci): build both release and debug versions [[#3161](https://github.com/falcosecurity/falco/pull/3161)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* chore(userspace/falco): watch all configs files. [[#3160](https://github.com/falcosecurity/falco/pull/3160)] - [@FedeDP](https://github.com/FedeDP)
+* fix(ci): update scorecard-action to v2.3.1 [[#3153](https://github.com/falcosecurity/falco/pull/3153)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* cleanup(falco): consolidate falco::grpc::server in one class [[#3150](https://github.com/falcosecurity/falco/pull/3150)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* new(build): enable ASan and UBSan builds with options and in CI [[#3147](https://github.com/falcosecurity/falco/pull/3147)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* fix(userspace): variable / function shadowing [[#3123](https://github.com/falcosecurity/falco/pull/3123)] - [@sgaist](https://github.com/sgaist)
+* build(deps): Bump submodules/falcosecurity-rules from `fbf0a4e` to `88a40c8` [[#3145](https://github.com/falcosecurity/falco/pull/3145)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* fix(cmake): fix USE_BUNDLED_DEPS=ON and BUILD_FALCO_UNIT_TESTS=ON [[#3146](https://github.com/falcosecurity/falco/pull/3146)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* Add --kernelversion and --kernelrelease options to falco driver loader entrypoint [[#3143](https://github.com/falcosecurity/falco/pull/3143)] - [@Sryther](https://github.com/Sryther)
+* build(deps): Bump submodules/falcosecurity-rules from `44addef` to `fbf0a4e` [[#3139](https://github.com/falcosecurity/falco/pull/3139)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* chore: bump to latest libs commit [[#3137](https://github.com/falcosecurity/falco/pull/3137)] - [@Andreagit97](https://github.com/Andreagit97)
+* refactor: Use FetchContent for integrating three bundled libs [[#3107](https://github.com/falcosecurity/falco/pull/3107)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* build(deps): Bump submodules/falcosecurity-rules from `dc7970d` to `44addef` [[#3136](https://github.com/falcosecurity/falco/pull/3136)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `f88b991` to `dc7970d` [[#3126](https://github.com/falcosecurity/falco/pull/3126)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* refactor(ci): Avoid using command make directly [[#3101](https://github.com/falcosecurity/falco/pull/3101)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* docs(proposal): 20231220-features-adoption-and-deprecation.md [[#2986](https://github.com/falcosecurity/falco/pull/2986)] - [@leogr](https://github.com/leogr)
+* build(deps): Bump submodules/falcosecurity-rules from `b499a1d` to `f88b991` [[#3125](https://github.com/falcosecurity/falco/pull/3125)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* docs(README.md): Falco Graduates within the CNCF [[#3124](https://github.com/falcosecurity/falco/pull/3124)] - [@leogr](https://github.com/leogr)
+* build(deps): Bump submodules/falcosecurity-rules from `497e011` to `b499a1d` [[#3111](https://github.com/falcosecurity/falco/pull/3111)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* chore(ci): bumped codeql actions. [[#3114](https://github.com/falcosecurity/falco/pull/3114)] - [@FedeDP](https://github.com/FedeDP)
+* Cleanup warnings and smart ptrs [[#3112](https://github.com/falcosecurity/falco/pull/3112)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* new(build): add options to use bundled dependencies [[#3092](https://github.com/falcosecurity/falco/pull/3092)] - [@mrgian](https://github.com/mrgian)
+* fix(ci): test-dev-packages-arm64 needs build-dev-packages-arm64. [[#3110](https://github.com/falcosecurity/falco/pull/3110)] - [@FedeDP](https://github.com/FedeDP)
+* refactor: bump libs and driver, and adopt unique pointers wherever possible [[#3109](https://github.com/falcosecurity/falco/pull/3109)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* cleanup: falco_engine test fixture [[#3099](https://github.com/falcosecurity/falco/pull/3099)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* refactor: test AtomicSignalHandler.handle_once_wait_consistency [[#3100](https://github.com/falcosecurity/falco/pull/3100)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* Cleanup variable use [[#3097](https://github.com/falcosecurity/falco/pull/3097)] - [@sgaist](https://github.com/sgaist)
+* cleanup(submodules): dropped testing submodule. [[#3098](https://github.com/falcosecurity/falco/pull/3098)] - [@FedeDP](https://github.com/FedeDP)
+* cleanup(ci): make use of falcosecurity/testing provided composite action [[#3093](https://github.com/falcosecurity/falco/pull/3093)] - [@FedeDP](https://github.com/FedeDP)
+* Improve const correctness [[#3083](https://github.com/falcosecurity/falco/pull/3083)] - [@sgaist](https://github.com/sgaist)
+* Improve exception throwing [[#3085](https://github.com/falcosecurity/falco/pull/3085)] - [@sgaist](https://github.com/sgaist)
+* fix(ci): update sync in deb and rpm scripts with acl [[#3062](https://github.com/falcosecurity/falco/pull/3062)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* cleanup(tests): consolidate Falco engine and rule loader tests [[#3066](https://github.com/falcosecurity/falco/pull/3066)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* cleanup: falco_engine deps and include paths [[#3090](https://github.com/falcosecurity/falco/pull/3090)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* fix: Some compiler warnings [[#3089](https://github.com/falcosecurity/falco/pull/3089)] - [@federico-sysdig](https://github.com/federico-sysdig)
+* build(deps): Bump submodules/falcosecurity-rules from `0f60976` to `497e011` [[#3081](https://github.com/falcosecurity/falco/pull/3081)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* fix(c++): add missing explicit to single argument constructors [[#3069](https://github.com/falcosecurity/falco/pull/3069)] - [@sgaist](https://github.com/sgaist)
+* Improve class initialization [[#3074](https://github.com/falcosecurity/falco/pull/3074)] - [@sgaist](https://github.com/sgaist)
+* build(deps): Bump submodules/falcosecurity-rules from `6ed2036` to `0f60976` [[#3078](https://github.com/falcosecurity/falco/pull/3078)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* build(deps): Bump submodules/falcosecurity-rules from `1053b2d` to `6ed2036` [[#3067](https://github.com/falcosecurity/falco/pull/3067)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* fix(c++): add missing overrides [[#3064](https://github.com/falcosecurity/falco/pull/3064)] - [@sgaist](https://github.com/sgaist)
+* new(build): prune deb-dev and rpm-dev directories [[#3056](https://github.com/falcosecurity/falco/pull/3056)] - [@LucaGuerra](https://github.com/LucaGuerra)
+* refactor(userspace): align falco to gen-event class family deprecation [[#3051](https://github.com/falcosecurity/falco/pull/3051)] - [@jasondellaluce](https://github.com/jasondellaluce)
+* build(deps): Bump submodules/falcosecurity-rules from `3cac61c` to `1053b2d` [[#3047](https://github.com/falcosecurity/falco/pull/3047)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+* fix: adopt new libsinsp logger [[#3026](https://github.com/falcosecurity/falco/pull/3026)] - [@therealbobo](https://github.com/therealbobo)
+* refactor: cleanup libs relative include paths [[#2936](https://github.com/falcosecurity/falco/pull/2936)] - [@therealbobo](https://github.com/therealbobo)
+* chore(ci): bumped rn2md to latest master. [[#3046](https://github.com/falcosecurity/falco/pull/3046)] - [@FedeDP](https://github.com/FedeDP)
+* Support alternate rules loader [[#3008](https://github.com/falcosecurity/falco/pull/3008)] - [@mstemm](https://github.com/mstemm)
+* fix(ci): fixed release body driver version. [[#3042](https://github.com/falcosecurity/falco/pull/3042)] - [@FedeDP](https://github.com/FedeDP)
+* build(deps): Bump submodules/falcosecurity-rules from `c39d31a` to `3f668d0` [[#3039](https://github.com/falcosecurity/falco/pull/3039)] - [@dependabot[bot]](https://github.com/apps/dependabot)
+
+
 ## v0.37.1
 
 Released on 2024-02-13

CMakeLists.txt

@@ -151,6 +151,7 @@ if (NOT EMSCRIPTEN)
 endif()
 
 include(zlib)
+include(valijson)
 if (NOT MINIMAL_BUILD)
   if (NOT WIN32 AND NOT APPLE AND NOT EMSCRIPTEN)
     include(cares)

OWNERS

@@ -6,6 +6,7 @@ approvers:
   - andreagit97
   - incertum
   - LucaGuerra
+  - sgaist
 reviewers:
   - kaizhe
 emeritus_approvers:

README.md

@@ -2,7 +2,7 @@
 
 [![Latest release](https://img.shields.io/github/v/release/falcosecurity/falco?style=for-the-badge)](https://github.com/falcosecurity/falco/releases/latest) [![Supported Architectures](https://img.shields.io/badge/ARCHS-x86__64%7Caarch64-blueviolet?style=for-the-badge)](https://github.com/falcosecurity/falco/releases/latest) [![License](https://img.shields.io/github/license/falcosecurity/falco?style=for-the-badge)](COPYING) [![Docs](https://img.shields.io/badge/docs-latest-green.svg?style=for-the-badge)](https://falco.org/docs)
 
-[![Falco Core Repository](https://github.com/falcosecurity/evolution/blob/main/repos/badges/falco-core-blue.svg)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#core-scope) [![Stable](https://img.shields.io/badge/status-stable-brightgreen?style=for-the-badge)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#stable)  [![OpenSSF Best Practices](https://img.shields.io/cii/summary/2317?label=OpenSSF%20Best%20Practices&style=for-the-badge)](https://bestpractices.coreinfrastructure.org/projects/2317) <a href="https://actuated.dev/"><img alt="Arm CI sponsored by Actuated" src="https://docs.actuated.dev/images/actuated-badge.png" width="120px"></img></a>
+[![Falco Core Repository](https://github.com/falcosecurity/evolution/blob/main/repos/badges/falco-core-blue.svg)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#core-scope) [![Stable](https://img.shields.io/badge/status-stable-brightgreen?style=for-the-badge)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#stable)  [![OpenSSF Scorecard](https://img.shields.io/ossf-scorecard/github.com/falcosecurity/falco?label=openssf%20scorecard&style=for-the-badge)](https://scorecard.dev/viewer/?uri=github.com/falcosecurity/falco)  [![OpenSSF Best Practices](https://img.shields.io/cii/summary/2317?label=OpenSSF%20Best%20Practices&style=for-the-badge)](https://bestpractices.coreinfrastructure.org/projects/2317) <a href="https://actuated.dev/"><img alt="Arm CI sponsored by Actuated" src="https://docs.actuated.dev/images/actuated-badge.png" width="120px"></img></a>
 
 [![Falco](https://falco.org/img/brand/falco-horizontal-color.svg)](https://falco.org)
 
@@ -43,6 +43,9 @@ Considerations and guidance for Falco adopters:
 
 5. Integrate with output destinations: Integrate Falco with SIEM, data lake systems, or other preferred output destinations to establish a robust foundation for comprehensive data analysis and enable effective incident response workflows.
 
+### Demo Environment
+
+A demo environment is provided via a docker-compose file that can be started on a docker host which includes falco, falcosidekick, falcosidekick-ui and its required redis database. For more information see the [docker-compose section](docker/docker-compose/)
 
 ## How to Contribute
 

cmake/modules/CompilerFlags.cmake

@@ -88,15 +88,17 @@ else() # MSVC
 	set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 
 	# The WIN32_LEAN_AND_MEAN define avoids possible macro pollution
-	# when a libsinsp consumer includes the windows.h header.
-	# See: https://stackoverflow.com/a/28380820
-
+	# when a libsinsp consumer includes the windows.h header:
+	# https://stackoverflow.com/a/28380820
+	# Same goes for NOMINMAX:
+	# https://stackoverflow.com/questions/5004858/why-is-stdmin-failing-when-windows-h-is-included
 	add_compile_definitions(
 		_HAS_STD_BYTE=0
 		_CRT_SECURE_NO_WARNINGS
 		WIN32
 		MINIMAL_BUILD
 		WIN32_LEAN_AND_MEAN
+		NOMINMAX
 	)
 
 	set(FALCOSECURITY_LIBS_COMMON_FLAGS "/EHsc /W3 /Zi /std:c++17")

cmake/modules/driver.cmake

@@ -34,8 +34,8 @@ else()
   # In case you want to test against another driver version (or branch, or commit) just pass the variable -
   # ie., `cmake -DDRIVER_VERSION=dev ..`
   if(NOT DRIVER_VERSION)
-    set(DRIVER_VERSION "0.17.0-rc1")
-    set(DRIVER_CHECKSUM "SHA256=63809beb5e448911f153b8c25f814075238a55f301923aeb3d2374be6309460b")
+    set(DRIVER_VERSION "dd165076cb71c5640f705c2e67acb604b2d3bdc9")
+    #set(DRIVER_CHECKSUM "SHA256=aedc4120d335ee2ebf7f3c2e6346fc985ca09a49d14ff7c6249a033e47d160bf")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

cmake/modules/falcoctl.cmake

@@ -14,16 +14,21 @@
 
 include(ExternalProject)
 
+option(ADD_FALCOCTL_DEPENDENCY "Add falcoctl dependency while building falco" ON)
+
+if(ADD_FALCOCTL_DEPENDENCY)
 string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} FALCOCTL_SYSTEM_NAME)
 
-set(FALCOCTL_VERSION "0.8.0-rc6")
+set(FALCOCTL_VERSION "0.10.0-rc1")
+
+message(STATUS "Building with falcoctl: ${FALCOCTL_VERSION}")
 
 if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
     set(FALCOCTL_SYSTEM_PROC_GO "amd64")
-    set(FALCOCTL_HASH "af49a15f28281aff37aa57808211cdd0772966a694da3b5a256d0e58e27bd16b")
+    set(FALCOCTL_HASH "efef3b6912837622ec1822941643d67d7b8ba704ba759751a17491b7f9fd05c3")
 else() # aarch64
     set(FALCOCTL_SYSTEM_PROC_GO "arm64")
-    set(FALCOCTL_HASH "262189f954be20372ff79c5e984b64e530cdfeecc6df74be3b8846fb52ee2bdf")
+    set(FALCOCTL_HASH "0429782ae5dc46ff8611a5f6ab2c275d8dd0e0721937ace0e8f6abc23de9dd57")
 endif()
 
 ExternalProject_Add(
@@ -36,3 +41,6 @@ ExternalProject_Add(
 
 install(PROGRAMS "${PROJECT_BINARY_DIR}/falcoctl-prefix/src/falcoctl/falcoctl" DESTINATION "${FALCO_BIN_DIR}" COMPONENT "${FALCO_COMPONENT_NAME}")
 install(DIRECTORY DESTINATION "${FALCO_ABSOLUTE_SHARE_DIR}/plugins" COMPONENT "${FALCO_COMPONENT_NAME}")
+else()
+    message(STATUS "Won't build with falcoctl")
+endif()

cmake/modules/falcosecurity-libs.cmake

@@ -35,8 +35,8 @@ else()
   # In case you want to test against another falcosecurity/libs version (or branch, or commit) just pass the variable -
   # ie., `cmake -DFALCOSECURITY_LIBS_VERSION=dev ..`
   if(NOT FALCOSECURITY_LIBS_VERSION)
-    set(FALCOSECURITY_LIBS_VERSION "0.17.0-rc1")
-    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=63809beb5e448911f153b8c25f814075238a55f301923aeb3d2374be6309460b")
+    set(FALCOSECURITY_LIBS_VERSION "dd165076cb71c5640f705c2e67acb604b2d3bdc9")
+#    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=aedc4120d335ee2ebf7f3c2e6346fc985ca09a49d14ff7c6249a033e47d160bf")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

cmake/modules/rules.cmake

@@ -15,9 +15,10 @@
 include(GNUInstallDirs)
 include(ExternalProject)
 
+if(NOT DEFINED FALCOSECURITY_RULES_FALCO_PATH)
 # falco_rules.yaml
-set(FALCOSECURITY_RULES_FALCO_VERSION "falco-rules-3.0.0")
-set(FALCOSECURITY_RULES_FALCO_CHECKSUM "SHA256=2e91799fee49c2daf58fb482e47410a21433eb116e02cde18206f7af87449ddb")
+set(FALCOSECURITY_RULES_FALCO_VERSION "falco-rules-3.1.0")
+set(FALCOSECURITY_RULES_FALCO_CHECKSUM "SHA256=3b617920c0b66128627613e591a954eb9572747a4c287bc13b53b38786250162")
 set(FALCOSECURITY_RULES_FALCO_PATH "${PROJECT_BINARY_DIR}/falcosecurity-rules-falco-prefix/src/falcosecurity-rules-falco/falco_rules.yaml")
 ExternalProject_Add(
   falcosecurity-rules-falco
@@ -28,10 +29,13 @@ ExternalProject_Add(
   INSTALL_COMMAND ""
   TEST_COMMAND ""
 )
+endif()
 
+if(NOT DEFINED FALCOSECURITY_RULES_LOCAL_PATH)
 # falco_rules.local.yaml
 set(FALCOSECURITY_RULES_LOCAL_PATH "${PROJECT_BINARY_DIR}/falcosecurity-rules-local-prefix/falco_rules.local.yaml")
 file(WRITE "${FALCOSECURITY_RULES_LOCAL_PATH}" "# Your custom rules!\n")
+endif()
 
 if(NOT DEFINED FALCO_ETC_DIR)
 	set(FALCO_ETC_DIR "${CMAKE_INSTALL_FULL_SYSCONFDIR}/falco")

docker/docker-compose/README.md

@@ -0,0 +1,17 @@
+# Warning
+
+This environment is provided for demonstration purposes only and does not represent a production ready deployment of Falco.
+
+# Components
+The components that this docker-compose file spins up are [Falco](https://falco.org/), [falcosidekick](https://github.com/falcosecurity/falcosidekick), [falcosidekick-ui](https://github.com/falcosecurity/falcosidekick-ui) and a [redis](https://redis.io/) database.
+
+# Running
+To start this environment run `docker-compose up`.
+Note: You may need to use sudo for Falco to start correctly.
+
+# Cleaning up
+
+To clean up run `docker-compose rm`.
+
+# Generating events
+If you'd like to generate events that will trigger rules and show up in the UI you can run `docker run -it --rm falcosecurity/event-generator run syscall --loop`

docker/docker-compose/config/http_output.yml

@@ -0,0 +1,11 @@
+# [Stable] `http_output`
+#
+# Send logs to an HTTP endpoint or webhook.
+#
+# When using falcosidekick, it is necessary to set `json_output` to true.
+json_output: true
+json_include_output_property: true
+http_output:
+  enabled: true
+  url: "http://falco-sidekick:2801/"
+  

docker/docker-compose/docker-compose.yaml

@@ -0,0 +1,34 @@
+version: "3"
+services:
+  falco:
+    container_name: falco
+    cap_drop:
+      - all
+    cap_add:
+      - sys_admin
+      - sys_resource
+      - sys_ptrace
+    volumes:
+      - /var/run/docker.sock:/host/var/run/docker.sock
+      - /proc:/host/proc:ro
+      - /etc:/host/etc:ro
+      - ./config/http_output.yml:/etc/falco/config.d/http_output.yml
+    image: falcosecurity/falco-no-driver:latest
+
+  sidekick:
+    container_name: falco-sidekick
+    image: falcosecurity/falcosidekick
+    environment:
+      WEBUI_URL: http://falco-webui:2802
+
+  webui:
+    container_name: falco-webui
+    image: falcosecurity/falcosidekick-ui:2.2.0
+    ports:
+      - 2802:2802
+    depends_on:
+      - redis
+    command: ['-r', 'redis:6379', '-d']
+
+  redis:
+    image: redis/redis-stack:7.2.0-v11

falco.yaml

@@ -468,7 +468,7 @@ load_plugins: []
 plugins:
   - name: k8saudit
     library_path: libk8saudit.so
-    init_config:
+    init_config: ""
     #   maxEventSize: 262144
     #   webhookMaxBatchSize: 12582912
     #   sslCertificate: /etc/falco/falco.pem
@@ -576,6 +576,42 @@ rule_matching: first
 outputs_queue:
   capacity: 0
 
+# [Sandbox] `append_output`
+#
+# Add information to the Falco output.
+# With this setting you can add more information to the Falco output message, customizable by
+# rule, tag or source.
+# You can also add additional data that will appear in the output_fields property
+# of JSON formatted messages or gRPC output but will not be part of the regular output message.
+# This allows you to add custom fields that can help you filter your Falco events without
+# polluting the message text.
+#
+# Each append_output entry has optional fields (ANDed together) to filter events:
+#   `rule`: append output only to a specific rule
+#   `source`: append output only to a specific source
+#   `tag`: append output only to a specific tag
+# If none of the above are specified output is appended to all events, if more than one is
+# specified output will be appended to events that match all conditions.
+# And several options to add output:
+#   `format`: add output to the Falco message
+#   `fields`: add new fields to the JSON output and structured output, which will not
+#             affect the regular Falco message in any way. These can be specified as a
+#             custom name with a custom format or as any supported field
+#             (see: https://falco.org/docs/reference/rules/supported-fields/)
+#
+# Example:
+#
+#   - source: syscall
+#     format: "on CPU %evt.cpu"
+#     fields:
+#       - home_directory: "${HOME}"
+#       - evt.hostname
+#
+# In the example above every event coming from the syscall source will get an extra message
+# at the end telling the CPU number. In addition, if `json_output` is true, in the "output_fields"
+# property you will find three new ones: "evt.cpu", "home_directory" which will contain the value of the
+# environment variable $HOME, and "evt.hostname" which will contain the hostname.
+
 
 ##########################
 # Falco outputs channels #
@@ -1037,6 +1073,11 @@ syscall_event_drops:
 # beneficial for exploring the data schema and ensuring that fields with empty
 # values are included in the output.
 #
+# `plugins_metrics_enabled`: Falco can now expose your custom plugins' 
+# metrics. Please note that if the respective plugin has no metrics implemented, 
+# there will be no metrics available. In other words, there are no default or 
+# generic plugin metrics at this time. This may be subject to change.
+#
 # If metrics are enabled, the web server can be configured to activate the
 # corresponding Prometheus endpoint using `webserver.prometheus_metrics_enabled`.
 # Prometheus output can be used in combination with the other output options.
@@ -1055,6 +1096,7 @@ metrics:
   state_counters_enabled: true
   kernel_event_counters_enabled: true
   libbpf_stats_enabled: true
+  plugins_metrics_enabled: true
   convert_memory_to_mb: true
   include_empty_values: false
 
@@ -1200,33 +1242,43 @@ base_syscalls:
 falco_libs:
   thread_table_size: 262144
 
-# [Stable] Guidance for Kubernetes container engine command-line args settings
+# [Incubating] `container_engines`
 #
-# Modern cloud environments, particularly Kubernetes, heavily rely on
-# containerized workload deployments. When capturing events with Falco, it
-# becomes essential to identify the owner of the workload for which events are
-# being captured, such as syscall events. Falco integrates with the container
-# runtime to enrich its events with container information, including fields like
-# `container.image.repository`, `container.image.tag`, ... , `k8s.ns.name`,
-# `k8s.pod.name`, `k8s.pod.*` in the Falco output (Falco retrieves Kubernetes
-# namespace and pod name directly from the container runtime, see
-# https://falco.org/docs/reference/rules/supported-fields/#field-class-container).
+# This option allows you to explicitly enable or disable API lookups against container 
+# runtime sockets for each supported container runtime. 
+# Access to these sockets enables Falco to retrieve container and Kubernetes fields, 
+# helping identify workload owners in modern containerized environments. 
+# Refer to the fields docs:
+# 
+# - [Kubernetes fields](https://falco.org/docs/reference/rules/supported-fields/#field-class-k8s)
+# - [Container fields](https://falco.org/docs/reference/rules/supported-fields/#container)
+# 
+# Additionally, Falco can use container events as a data source for alerting (evt.type = container).
+# 
+# For most container engines, you can solely enable or disable them, and Falco will search the 
+# default (hard-coded) container runtime socket paths, such as `/var/run/docker.sock` for Docker.
 #
-# Furthermore, Falco exposes container events themselves as a data source for
-# alerting. To achieve this integration with the container runtime, Falco
-# requires access to the runtime socket. By default, for Kubernetes, Falco
-# attempts to connect to the following sockets:
-# "/run/containerd/containerd.sock", "/run/crio/crio.sock",
-# "/run/k3s/containerd/containerd.sock". If you have a custom path, you can use
-# the `--cri` option to specify the correct location.
-#
-# In some cases, you may encounter empty fields for container metadata. To
-# address this, you can explore the `--disable-cri-async` option, which disables
-# asynchronous fetching if the fetch operation is not completing quickly enough.
-#
-# To get more information on these command-line arguments, you can run `falco
-# --help` in your terminal to view their current descriptions.
-#
-# !!! The options mentioned here are not available in the falco.yaml
-# configuration file. Instead, they can can be used as a command-line argument
-# when running the Falco command.
+# However, for Kubernetes settings, you can customize the CRI socket paths:
+# 
+# - `container_engines.cri.sockets`: Pass a list of container runtime sockets.
+# - `container_engines.cri.disable_async`: Since API lookups may not always be quick or 
+# perfect, resulting in empty fields for container metadata, you can use this option option 
+# to disable asynchronous fetching. Note that missing fields may still occasionally occur.
+# 
+# The equivalent (stable) CLI args are `--cri` or `--disable-cri-async`.
+
+container_engines:
+  docker:
+    enabled: true
+  cri:
+    enabled: true
+    sockets: ["/run/containerd/containerd.sock", "/run/crio/crio.sock", "/run/k3s/containerd/containerd.sock"]
+    disable_async: false
+  podman:
+    enabled: true
+  lxc:
+    enabled: true
+  libvirt_lxc:
+    enabled: true
+  bpm:
+    enabled: true

proposals/20210501-plugin-system.md

@@ -335,7 +335,7 @@ typedef struct
 	// the type of the value they return (string, integer...).
 	// Required: no
 	// Arguments:
-	// - evtnum: the number of the event that is bein processed
+	// - evtnum: the number of the event that is being processed
 	// - id: the numeric identifier of the field to extract. It corresponds to the
 	//   position of the field in the array returned by get_fields().
 	// - arg: the field argument, if an argument has been specified for the field,

unit_tests/CMakeLists.txt

@@ -36,6 +36,7 @@ add_executable(falco_unit_tests
     engine/test_add_source.cpp
     engine/test_alt_rule_loader.cpp
     engine/test_enable_rule.cpp
+    engine/test_extra_output.cpp
     engine/test_falco_utils.cpp
     engine/test_filter_details_resolver.cpp
     engine/test_filter_macro_resolver.cpp
@@ -45,6 +46,10 @@ add_executable(falco_unit_tests
     engine/test_rulesets.cpp
     falco/test_configuration.cpp
     falco/test_configuration_rule_selection.cpp
+    falco/test_configuration_config_files.cpp
+    falco/test_configuration_env_vars.cpp
+    falco/test_configuration_output_options.cpp
+    falco/test_configuration_schema.cpp
     falco/app/actions/test_select_event_sources.cpp
     falco/app/actions/test_load_config.cpp
 )

unit_tests/engine/test_extra_output.cpp

@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+
+#include "../test_falco_engine.h"
+
+TEST_F(test_falco_engine, extra_format_all)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+)END";
+
+	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "", false);
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"user=%user.name command=%proc.cmdline file=%fd.name evt.type=%evt.type");
+}
+
+TEST_F(test_falco_engine, extra_format_by_rule)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1
+  priority: INFO
+
+- rule: another_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 2
+  priority: INFO
+)END";
+
+	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "legit_rule", false);
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 evt.type=%evt.type");
+	EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2");
+}
+
+TEST_F(test_falco_engine, extra_format_by_tag_rule)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1
+  priority: INFO
+  tags: [tag1]
+
+- rule: another_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 2
+  priority: INFO
+  tags: [tag1]
+)END";
+
+	m_engine->add_extra_output_format("extra 1", "", "tag1", "", false);
+	m_engine->add_extra_output_format("extra 2", "", "", "another_rule", false);
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 extra 1");
+	EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2 extra 1 extra 2");
+}
+
+TEST_F(test_falco_engine, extra_format_replace_container_info)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1 (%container.info)
+  priority: INFO
+  tags: [tag1]
+
+- rule: another_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 2
+  priority: INFO
+  tags: [tag1]
+)END";
+
+	m_engine->add_extra_output_format("extra 1", "", "", "", true);
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"), "out 1 (extra 1)");
+	EXPECT_EQ(get_compiled_rule_output("another_rule"), "out 2 extra 1");
+}
+
+TEST_F(test_falco_engine, extra_format_do_not_replace_container_info)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1 (%container.info)
+  priority: INFO
+  tags: [tag1]
+)END";
+	
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	auto output = get_compiled_rule_output("legit_rule");
+	EXPECT_TRUE(output.find("%container.info") == output.npos);
+}
+
+TEST_F(test_falco_engine, extra_fields_all)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+)END";
+
+	std::unordered_map<std::string, std::string> extra_formatted_fields = {{"my_field", "hello %evt.num"}};
+  for (auto const& f : extra_formatted_fields)
+  {
+    m_engine->add_extra_output_formatted_field(f.first, f.second, "", "", "");
+  }
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_formatted_fields("legit_rule"), extra_formatted_fields);
+}

unit_tests/engine/test_rule_loader.cpp

@@ -1034,7 +1034,7 @@ TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = proc.pname)");
-  EXPECT_TRUE(check_warning_message("string 'proc.pname' may be a valid field wrongly interpreted as a string value"));
+  EXPECT_TRUE(check_warning_message("'proc.pname' may be a valid field misused as a const string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous_quoted)
@@ -1050,7 +1050,7 @@ TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous_quoted)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = proc.pname)");
-  EXPECT_TRUE(check_warning_message("string 'proc.pname' may be a valid field wrongly interpreted as a string value"));
+  EXPECT_TRUE(check_warning_message("'proc.pname' may be a valid field misused as a const string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous_space_quoted)
@@ -1066,7 +1066,7 @@ TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous_space_quoted)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = \"proc.pname \")");
-  EXPECT_TRUE(check_warning_message("string 'proc.pname ' may be a valid field wrongly interpreted as a string value"));
+  EXPECT_TRUE(check_warning_message("'proc.pname ' may be a valid field misused as a const string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_values_rhs_transformer)
@@ -1112,7 +1112,7 @@ TEST_F(test_falco_engine, exceptions_values_transformer_space)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = \"toupper( proc.pname)\")");
-  EXPECT_TRUE(check_warning_message("string 'toupper( proc.pname)' may be a valid field transformer wrongly interpreted as a string value"));
+  EXPECT_TRUE(check_warning_message("'toupper( proc.pname)' may be a valid field transformer misused as a const string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_values_transformer_space_quoted)
@@ -1128,7 +1128,7 @@ TEST_F(test_falco_engine, exceptions_values_transformer_space_quoted)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = \"toupper( proc.pname)\")");
-  EXPECT_TRUE(check_warning_message("string 'toupper( proc.pname)' may be a valid field transformer wrongly interpreted as a string value"));
+  EXPECT_TRUE(check_warning_message("'toupper( proc.pname)' may be a valid field transformer misused as a const string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_fields_transformer)

unit_tests/falco/test_atomic_signal_handler.cpp

@@ -16,7 +16,7 @@ limitations under the License.
 */
 
 #include <falco/atomic_signal_handler.h>
-#include <falco/logger.h>
+#include <engine/logger.h>
 
 #include <gtest/gtest.h>
 

unit_tests/falco/test_configuration.cpp

@@ -18,12 +18,6 @@ limitations under the License.
 #include <gtest/gtest.h>
 #include <falco/configuration.h>
 
-#ifdef _WIN32
-#define SET_ENV_VAR(env_var_name, env_var_value) _putenv_s(env_var_name, env_var_value)
-#else
-#define SET_ENV_VAR(env_var_name, env_var_value) setenv(env_var_name, env_var_value, 1)
-#endif
-
 static std::string sample_yaml =
 	"base_value:\n"
 	"    id: 1\n"
@@ -109,657 +103,6 @@ TEST(Configuration, modify_yaml_fields)
 	ASSERT_EQ(conf.get_scalar<bool>(key, false), true);
 }
 
-TEST(Configuration, configuration_config_files_secondary_fail)
-{
-	/* Test that a secondary config file is not able to include anything, triggering an exception. */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_2.yaml\n"
-					   "  - conf_3.yaml\n"
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_4.yaml\n"
-					   "foo2: bar2\n"
-					   "base_value_2:\n"
-					   "    id: 2\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_ANY_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-}
-
-TEST(Configuration, configuration_config_files_ok)
-{
-	/* Test that every included config file was correctly parsed */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_2.yaml\n"
-					   "  - conf_3.yaml\n"
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-	const std::string conf_yaml_3 =
-		"foo3: bar3\n"
-		"base_value_3:\n"
-		"    id: 3\n"
-		"    name: foo3\n";
-	const std::string conf_yaml_4 =
-		"base_value_4:\n"
-		"    id: 4\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	outfile.open("conf_3.yaml");
-	outfile << conf_yaml_3;
-	outfile.close();
-
-	outfile.open("conf_4.yaml");
-	outfile << conf_yaml_4;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
-
-	// main + conf_2 + conf_3
-	ASSERT_EQ(loaded_conf_files.size(), 3);
-
-	ASSERT_TRUE(falco_config.config.is_defined("foo"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
-	ASSERT_TRUE(falco_config.config.is_defined("foo3"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo3", ""), "bar3");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_3.id", 0), 3);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.name"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value_3.name", ""), "foo3");
-	ASSERT_FALSE(falco_config.config.is_defined("base_value_4.id")); // conf_4 is not included
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-	std::filesystem::remove("conf_3.yaml");
-	std::filesystem::remove("conf_4.yaml");
-}
-
-
-TEST(Configuration, configuration_config_files_relative_main)
-{
-	/*
-	 * Test that relative path are treated as relative to cwd and not to main config folder,
-	 * and that absolute includes are ok too.
-	 */
-	const auto temp_main = std::filesystem::temp_directory_path() / "main.yaml";
-	// So, conf_2 will be looked up in the same folder as main config file,
-	// while conf_3, since is absolute, will be looked up in the absolute path (and found!).
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-		"  - conf_2.yaml\n"
-		"  - " +
-		std::filesystem::current_path().string() + "/conf_3.yaml\n"
-							   "foo: bar\n"
-							   "base_value:\n"
-							   "    id: 1\n"
-							   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-	const std::string conf_yaml_3 =
-		"foo3: bar3\n"
-		"base_value_3:\n"
-		"    id: 3\n"
-		"    name: foo3\n";
-
-	std::ofstream outfile(temp_main.string());
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	outfile.open("conf_3.yaml");
-	outfile << conf_yaml_3;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_file(temp_main.string(), loaded_conf_files, cmdline_config_options));
-
-	// main + conf_2 + conf_3
-	ASSERT_EQ(loaded_conf_files.size(), 3);
-
-	ASSERT_TRUE(falco_config.config.is_defined("foo"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_2"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_3.id", 0), 3);
-
-	std::filesystem::remove(temp_main.string());
-	std::filesystem::remove("conf_2.yaml");
-	std::filesystem::remove("conf_3.yaml");
-}
-
-TEST(Configuration, configuration_config_files_override)
-{
-	/* Test that included config files are able to override configs from main file */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_2.yaml\n"
-					   "  - conf_3.yaml\n"
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-	const std::string conf_yaml_3 =
-		"base_value:\n"
-		"    id: 3\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	outfile.open("conf_3.yaml");
-	outfile << conf_yaml_3;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
-
-	// main + conf_2 + conf_3
-	ASSERT_EQ(loaded_conf_files.size(), 3);
-
-	ASSERT_TRUE(falco_config.config.is_defined("foo"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 3); // overridden!
-	ASSERT_FALSE(falco_config.config.is_defined("base_value.name"));	// no more present since entire `base_value` block was overridden
-	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
-	ASSERT_FALSE(falco_config.config.is_defined("base_value_3.id")); // not defined
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-	std::filesystem::remove("conf_3.yaml");
-}
-
-TEST(Configuration, configuration_config_files_unexistent)
-{
-	/* Test that including an unexistent file just skips it */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-		"  - conf_5.yaml\n"
-		"base_value:\n"
-		"    id: 1\n"
-		"    name: foo\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
-
-	// main
-	ASSERT_EQ(loaded_conf_files.size(), 1);
-
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
-
-	std::filesystem::remove("main.yaml");
-}
-
-TEST(Configuration, configuration_config_files_scalar_config_files)
-{
-	/* Test that a single file can be included as a scalar (thanks to get_sequence_from_node magic) */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ": conf_2.yaml\n"
-		"foo: bar\n"
-		"base_value:\n"
-		"    id: 1\n"
-		"    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
-
-	// main + conf_2
-	ASSERT_EQ(loaded_conf_files.size(), 2);
-
-	ASSERT_TRUE(falco_config.config.is_defined("foo"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-}
-
-TEST(Configuration, configuration_config_files_empty_config_files)
-{
-	/* Test that empty includes list is accepted */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-		"foo: bar\n"
-		"base_value:\n"
-		"    id: 1\n"
-		"    name: foo\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
-
-	// main
-	ASSERT_EQ(loaded_conf_files.size(), 1);
-
-	ASSERT_TRUE(falco_config.config.is_defined("foo"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
-
-	std::filesystem::remove("main.yaml");
-}
-
-TEST(Configuration, configuration_config_files_self)
-{
-	/* Test that main config file cannot include itself */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ": main.yaml\n"
-		"foo: bar\n"
-		"base_value:\n"
-		"    id: 1\n"
-		"    name: foo\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_ANY_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
-
-	std::filesystem::remove("main.yaml");
-}
-
-TEST(Configuration, configuration_config_files_directory)
-{
-	/*
-	 * Test that when main config file includes a config directory,
-	 * the config directory is parsed in lexicographic order,
-	 * and only regular files are parsed.
-	 */
-	// Main config includes whole temp directory
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ": " + std::filesystem::temp_directory_path().string() + "/test\n"
-		   "foo: bar\n"
-		   "base_value:\n"
-		   "    id: 1\n"
-		   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-	const std::string conf_yaml_3 =
-		"foo2: bar3\n"
-		"base_value_3:\n"
-		"    id: 3\n"
-		"    name: foo3\n";
-	const std::string conf_yaml_4 =
-		"foo4: bar4\n";
-
-	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test");
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open(std::filesystem::temp_directory_path()/"test/conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	outfile.open(std::filesystem::temp_directory_path()/"test/conf_3.yaml");
-	outfile << conf_yaml_3;
-	outfile.close();
-
-	// Create a directory and create a config inside it. We will later check that it was not parsed
-	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test" / "foo");
-	outfile.open(std::filesystem::temp_directory_path()/"test/foo/conf_4.yaml");
-	outfile << conf_yaml_4;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
-
-	// main + conf_2 + conf_3.
-	// test/foo is not parsed.
-	ASSERT_EQ(loaded_conf_files.size(), 3);
-
-	ASSERT_TRUE(falco_config.config.is_defined("foo"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_2"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_3.id", 0), 3);
-	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar3");
-	ASSERT_FALSE(falco_config.config.is_defined("foo4"));
-
-	std::filesystem::remove("main");
-	std::filesystem::remove_all(std::filesystem::temp_directory_path()/"test");
-}
-
-TEST(Configuration, configuration_config_files_cmdline)
-{
-	/* Test that we support including configs files from cmdline option */
-	const std::string main_conf_yaml =
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	// Pass "config_files=..." cmdline option
-	std::vector<std::string> cmdline_config_options;
-	cmdline_config_options.push_back((yaml_helper::configs_key+"=conf_2.yaml"));
-
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
-
-	// main + conf_2
-	ASSERT_EQ(loaded_conf_files.size(), 2);
-
-	ASSERT_TRUE(falco_config.config.is_defined("foo"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
-	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-}
-
-TEST(Configuration, configuration_environment_variables)
-{
-    // Set an environment variable for testing purposes
-    std::string env_var_value = "envVarValue";
-    std::string env_var_name = "ENV_VAR";
-    SET_ENV_VAR(env_var_name.c_str(), env_var_value.c_str());
-
-    std::string embedded_env_var_value = "${ENV_VAR}";
-    std::string embedded_env_var_name = "ENV_VAR_EMBEDDED";
-    SET_ENV_VAR(embedded_env_var_name.c_str(), embedded_env_var_value.c_str());
-
-    std::string bool_env_var_value = "true";
-    std::string bool_env_var_name = "ENV_VAR_BOOL";
-    SET_ENV_VAR(bool_env_var_name.c_str(), bool_env_var_value.c_str());
-
-    std::string int_env_var_value = "12";
-    std::string int_env_var_name = "ENV_VAR_INT";
-    SET_ENV_VAR(int_env_var_name.c_str(), int_env_var_value.c_str());
-
-    std::string empty_env_var_value = "";
-    std::string empty_env_var_name = "ENV_VAR_EMPTY";
-    SET_ENV_VAR(empty_env_var_name.c_str(), empty_env_var_value.c_str());
-
-    std::string default_value = "default";
-    std::string env_var_sample_yaml =
-        "base_value:\n"
-        "    id: $ENV_VAR\n"
-        "    name: '${ENV_VAR}'\n"
-        "    string: my_string\n"
-        "    invalid: $${ENV_VAR}\n"
-	"    invalid_env: $$ENV_VAR\n"
-	"    invalid_double_env: $${ENV_VAR}$${ENV_VAR}\n"
-	"    invalid_embedded_env: $${${ENV_VAR}}\n"
-	"    invalid_valid_env: $${ENV_VAR}${ENV_VAR}\n"
-        "    escaped: \"${ENV_VAR}\"\n"
-        "    subvalue:\n"
-        "        subvalue2:\n"
-        "            boolean: ${UNSED_XX_X_X_VAR}\n"
-        "base_value_2:\n"
-        "    sample_list:\n"
-        "        - ${ENV_VAR}\n"
-        "        - ' ${ENV_VAR}'\n"
-	"        - '${ENV_VAR} '\n"
-        "        - $UNSED_XX_X_X_VAR\n"
-        "paths:\n"
-	"    - ${ENV_VAR}/foo\n"
-	"    - $ENV_VAR/foo\n"
-	"    - /foo/${ENV_VAR}/\n"
-	"    - /${ENV_VAR}/${ENV_VAR}${ENV_VAR}/foo\n"
-	"    - ${ENV_VAR_EMBEDDED}/foo\n"
-	"is_test: ${ENV_VAR_BOOL}\n"
-	"num_test: ${ENV_VAR_INT}\n"
-	"empty_test: ${ENV_VAR_EMPTY}\n"
-	"plugins:\n"
-	"  - name: k8saudit\n"
-	"    library_path: /foo/${ENV_VAR}/libk8saudit.so\n"
-	"    open_params: ${ENV_VAR_INT}\n";
-
-    yaml_helper conf;
-    conf.load_from_string(env_var_sample_yaml);
-
-    /* Check if the base values are defined */
-    ASSERT_TRUE(conf.is_defined("base_value"));
-    ASSERT_TRUE(conf.is_defined("base_value_2"));
-    ASSERT_TRUE(conf.is_defined("paths"));
-    ASSERT_FALSE(conf.is_defined("unknown_base_value"));
-
-    /* Test fetching of a regular string without any environment variable */
-    auto base_value_string = conf.get_scalar<std::string>("base_value.string", default_value);
-    ASSERT_EQ(base_value_string, "my_string");
-
-    /* Test fetching of escaped environment variable format. Should return the string as-is after stripping the leading `$` */
-    auto base_value_invalid = conf.get_scalar<std::string>("base_value.invalid", default_value);
-    ASSERT_EQ(base_value_invalid, "${ENV_VAR}");
-
-    /* Test fetching of invalid escaped environment variable format. Should return the string as-is */
-    auto base_value_invalid_env = conf.get_scalar<std::string>("base_value.invalid_env", default_value);
-    ASSERT_EQ(base_value_invalid_env, "$$ENV_VAR");
-
-    /* Test fetching of 2 escaped environment variables side by side. Should return the string as-is after stripping the leading `$` */
-    auto base_value_double_invalid = conf.get_scalar<std::string>("base_value.invalid_double_env", default_value);
-    ASSERT_EQ(base_value_double_invalid, "${ENV_VAR}${ENV_VAR}");
-
-    /*
-     * Test fetching of escaped environment variable format with inside an env variable.
-     * Should return the string as-is after stripping the leading `$` with the resolved env variable within
-     */
-    auto base_value_embedded_invalid = conf.get_scalar<std::string>("base_value.invalid_embedded_env", default_value);
-    ASSERT_EQ(base_value_embedded_invalid, "${" + env_var_value + "}");
-
-    /*
-     * Test fetching of an escaped env variable plus an env variable side by side.
-     * Should return the escaped one trimming the leading `$` plus the second one resolved.
-     */
-    auto base_value_valid_invalid = conf.get_scalar<std::string>("base_value.invalid_valid_env", default_value);
-    ASSERT_EQ(base_value_valid_invalid, "${ENV_VAR}" + env_var_value);
-
-    /* Test fetching of strings that contain environment variables */
-    auto base_value_id = conf.get_scalar<std::string>("base_value.id", default_value);
-    ASSERT_EQ(base_value_id, "$ENV_VAR"); // Does not follow the `${VAR}` format, so it should be treated as a regular string
-
-    auto base_value_name = conf.get_scalar<std::string>("base_value.name", default_value);
-    ASSERT_EQ(base_value_name, env_var_value); // Proper environment variable format
-
-    auto base_value_escaped = conf.get_scalar<std::string>("base_value.escaped", default_value);
-    ASSERT_EQ(base_value_escaped, env_var_value); // Environment variable within quotes
-
-    /* Test fetching of an undefined environment variable. Resolves to empty string. */
-    auto unknown_boolean = conf.get_scalar<std::string>("base_value.subvalue.subvalue2.boolean", default_value);
-    ASSERT_EQ(unknown_boolean, "");
-
-    /* Test fetching of environment variables from a list */
-    auto base_value_2_list_0 = conf.get_scalar<std::string>("base_value_2.sample_list[0]", default_value);
-    ASSERT_EQ(base_value_2_list_0, env_var_value); // Proper environment variable format
-
-    auto base_value_2_list_1 = conf.get_scalar<std::string>("base_value_2.sample_list[1]", default_value);
-    ASSERT_EQ(base_value_2_list_1, " " + env_var_value); // Environment variable preceded by a space, still extracted env var with leading space
-
-    auto base_value_2_list_2 = conf.get_scalar<std::string>("base_value_2.sample_list[2]", default_value);
-    ASSERT_EQ(base_value_2_list_2, env_var_value + " "); // Environment variable followed by a space, still extracted env var with trailing space
-
-    auto base_value_2_list_3 = conf.get_scalar<std::string>("base_value_2.sample_list[3]", default_value);
-    ASSERT_EQ(base_value_2_list_3, "$UNSED_XX_X_X_VAR"); // Does not follow the `${VAR}` format, so should be treated as a regular string
-
-    /* Test expansion of environment variables within strings */
-    auto path_list_0 = conf.get_scalar<std::string>("paths[0]", default_value);
-    ASSERT_EQ(path_list_0, env_var_value + "/foo"); // Even if env var is part of bigger string, it gets expanded
-
-    auto path_list_1 = conf.get_scalar<std::string>("paths[1]", default_value);
-    ASSERT_EQ(path_list_1, "$ENV_VAR/foo"); // Does not follow the `${VAR}` format, so should be treated as a regular string
-
-    auto path_list_2 = conf.get_scalar<std::string>("paths[2]", default_value);
-    ASSERT_EQ(path_list_2, "/foo/" + env_var_value + "/"); // Even when env var is in the middle of a string. it gets expanded
-
-    auto path_list_3 = conf.get_scalar<std::string>("paths[3]", default_value);
-    ASSERT_EQ(path_list_3, "/" + env_var_value + "/" + env_var_value + env_var_value + "/foo"); // Even when the string contains multiple env vars they are correctly expanded
-
-    auto path_list_4 = conf.get_scalar<std::string>("paths[4]", default_value);
-    ASSERT_EQ(path_list_4, env_var_value + "/foo"); // Even when the env var contains another env var, it gets correctly double-expanded
-
-    /* Check that variable expansion is type-aware */
-    auto boolean = conf.get_scalar<bool>("is_test", false);
-    ASSERT_EQ(boolean, true); // `true` can be parsed to bool.
-
-    auto boolean_as_str = conf.get_scalar<std::string>("is_test", "false");
-    ASSERT_EQ(boolean_as_str, "true"); // `true` can be parsed to string.
-
-    auto boolean_as_int = conf.get_scalar<int32_t>("is_test", 0);
-    ASSERT_EQ(boolean_as_int, 0); // `true` cannot be parsed to integer.
-
-    auto integer = conf.get_scalar<int32_t>("num_test", -1);
-    ASSERT_EQ(integer, 12);
-
-    // An env var that resolves to an empty string returns ""
-    auto empty_default_str = conf.get_scalar<std::string>("empty_test", default_value);
-    ASSERT_EQ(empty_default_str, "");
-
-    std::list<falco_configuration::plugin_config> plugins;
-    conf.get_sequence<std::list<falco_configuration::plugin_config>>(plugins, std::string("plugins"));
-    std::vector<falco_configuration::plugin_config> m_plugins{ std::make_move_iterator(std::begin(plugins)),
-							      std::make_move_iterator(std::end(plugins)) };
-    ASSERT_EQ(m_plugins[0].m_name, "k8saudit");
-    ASSERT_EQ(m_plugins[0].m_library_path, "/foo/" + env_var_value + "/libk8saudit.so");
-    ASSERT_EQ(m_plugins[0].m_open_params, "12");
-
-    /* Clear the set environment variables after testing */
-    SET_ENV_VAR(env_var_name.c_str(), "");
-    SET_ENV_VAR(embedded_env_var_name.c_str(), "");
-    SET_ENV_VAR(bool_env_var_name.c_str(), "");
-    SET_ENV_VAR(int_env_var_name.c_str(), "");
-    SET_ENV_VAR(empty_env_var_name.c_str(), "");
-}
-
 TEST(Configuration, configuration_webserver_ip)
 {
     falco_configuration falco_config;

unit_tests/falco/test_configuration_config_files.cpp

@@ -0,0 +1,496 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+#include <falco/configuration.h>
+
+TEST(Configuration, configuration_config_files_secondary_fail)
+{
+	/* Test that a secondary config file is not able to include anything, triggering an exception. */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - conf_3.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_4.yaml\n"
+					   "foo2: bar2\n"
+					   "base_value_2:\n"
+					   "    id: 2\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	ASSERT_ANY_THROW(falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+}
+
+TEST(Configuration, configuration_config_files_ok)
+{
+	/* Test that every included config file was correctly parsed */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - conf_3.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo3: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+	const std::string conf_yaml_4 =
+		"base_value_4:\n"
+		"    id: 4\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	outfile.open("conf_4.yaml");
+	outfile << conf_yaml_4;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(res.size(), 3);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo3"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo3", ""), "bar3");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_3.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_3.id", 0), 3);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_3.name"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value_3.name", ""), "foo3");
+	ASSERT_FALSE(falco_config.m_config.is_defined("base_value_4.id")); // conf_4 is not included
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+	std::filesystem::remove("conf_4.yaml");
+}
+
+
+TEST(Configuration, configuration_config_files_relative_main)
+{
+	/*
+	 * Test that relative path are treated as relative to cwd and not to main config folder,
+	 * and that absolute includes are ok too.
+	 */
+	const auto temp_main = std::filesystem::temp_directory_path() / "main.yaml";
+	// So, conf_2 will be looked up in the same folder as main config file,
+	// while conf_3, since is absolute, will be looked up in the absolute path (and found!).
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - " +
+		std::filesystem::current_path().string() + "/conf_3.yaml\n"
+							   "foo: bar\n"
+							   "base_value:\n"
+							   "    id: 1\n"
+							   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo3: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+
+	std::ofstream outfile(temp_main.string());
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file(temp_main.string(), cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(res.size(), 3);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_3.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_3.id", 0), 3);
+
+	std::filesystem::remove(temp_main.string());
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+}
+
+TEST(Configuration, configuration_config_files_override)
+{
+	/* Test that included config files are able to override configs from main file */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - conf_3.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"base_value:\n"
+		"    id: 3\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(res.size(), 3);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 3); // overridden!
+	ASSERT_FALSE(falco_config.m_config.is_defined("base_value.name"));	// no more present since entire `base_value` block was overridden
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_FALSE(falco_config.m_config.is_defined("base_value_3.id")); // not defined
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+}
+
+TEST(Configuration, configuration_config_files_unexistent)
+{
+	/* Test that including an unexistent file just skips it */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_5.yaml\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main
+	ASSERT_EQ(res.size(), 1);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
+
+	std::filesystem::remove("main.yaml");
+}
+
+TEST(Configuration, configuration_config_files_scalar_config_files)
+{
+	/* Test that a single file can be included as a scalar (thanks to get_sequence_from_node magic) */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ": conf_2.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main + conf_2
+	ASSERT_EQ(res.size(), 2);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+}
+
+TEST(Configuration, configuration_config_files_empty_config_files)
+{
+	/* Test that empty includes list is accepted */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main
+	ASSERT_EQ(res.size(), 1);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
+
+	std::filesystem::remove("main.yaml");
+}
+
+TEST(Configuration, configuration_config_files_self)
+{
+	/* Test that main config file cannot include itself */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ": main.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	ASSERT_ANY_THROW(falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	std::filesystem::remove("main.yaml");
+}
+
+TEST(Configuration, configuration_config_files_directory)
+{
+	/*
+	 * Test that when main config file includes a config directory,
+	 * the config directory is parsed in lexicographic order,
+	 * and only regular files are parsed.
+	 */
+	// Main config includes whole temp directory
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ": " + std::filesystem::temp_directory_path().string() + "/test\n"
+												    "foo: bar\n"
+												    "base_value:\n"
+												    "    id: 1\n"
+												    "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo2: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+	const std::string conf_yaml_4 =
+		"foo4: bar4\n";
+
+	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test");
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open(std::filesystem::temp_directory_path()/"test/conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open(std::filesystem::temp_directory_path()/"test/conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	// Create a directory and create a config inside it. We will later check that it was not parsed
+	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test" / "foo");
+	outfile.open(std::filesystem::temp_directory_path()/"test/foo/conf_4.yaml");
+	outfile << conf_yaml_4;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main + conf_2 + conf_3.
+	// test/foo is not parsed.
+	ASSERT_EQ(res.size(), 3);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_3.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_3.id", 0), 3);
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar3");
+	ASSERT_FALSE(falco_config.m_config.is_defined("foo4"));
+
+	std::filesystem::remove("main");
+	std::filesystem::remove_all(std::filesystem::temp_directory_path()/"test");
+}
+
+TEST(Configuration, configuration_config_files_cmdline)
+{
+	/* Test that we support including configs files from cmdline option */
+	const std::string main_conf_yaml =
+		"foo: bar\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	// Pass "config_files=..." cmdline option
+	std::vector<std::string> cmdline_config_options;
+	cmdline_config_options.push_back((yaml_helper::configs_key+"=conf_2.yaml"));
+
+	falco_configuration falco_config;
+	config_loaded_res res;
+	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
+
+	// main + conf_2
+	ASSERT_EQ(res.size(), 2);
+
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+}

unit_tests/falco/test_configuration_env_vars.cpp

@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+#include <falco/configuration.h>
+
+#ifdef _WIN32
+#define SET_ENV_VAR(env_var_name, env_var_value) _putenv_s(env_var_name, env_var_value)
+#else
+#define SET_ENV_VAR(env_var_name, env_var_value) setenv(env_var_name, env_var_value, 1)
+#endif
+
+TEST(Configuration, configuration_environment_variables)
+{
+	// Set an environment variable for testing purposes
+	std::string env_var_value = "envVarValue";
+	std::string env_var_name = "ENV_VAR";
+	SET_ENV_VAR(env_var_name.c_str(), env_var_value.c_str());
+
+	std::string embedded_env_var_value = "${ENV_VAR}";
+	std::string embedded_env_var_name = "ENV_VAR_EMBEDDED";
+	SET_ENV_VAR(embedded_env_var_name.c_str(), embedded_env_var_value.c_str());
+
+	std::string bool_env_var_value = "true";
+	std::string bool_env_var_name = "ENV_VAR_BOOL";
+	SET_ENV_VAR(bool_env_var_name.c_str(), bool_env_var_value.c_str());
+
+	std::string int_env_var_value = "12";
+	std::string int_env_var_name = "ENV_VAR_INT";
+	SET_ENV_VAR(int_env_var_name.c_str(), int_env_var_value.c_str());
+
+	std::string empty_env_var_value = "";
+	std::string empty_env_var_name = "ENV_VAR_EMPTY";
+	SET_ENV_VAR(empty_env_var_name.c_str(), empty_env_var_value.c_str());
+
+	std::string default_value = "default";
+	std::string env_var_sample_yaml =
+		"base_value:\n"
+		"    id: $ENV_VAR\n"
+		"    name: '${ENV_VAR}'\n"
+		"    string: my_string\n"
+		"    invalid: $${ENV_VAR}\n"
+		"    invalid_env: $$ENV_VAR\n"
+		"    invalid_double_env: $${ENV_VAR}$${ENV_VAR}\n"
+		"    invalid_embedded_env: $${${ENV_VAR}}\n"
+		"    invalid_valid_env: $${ENV_VAR}${ENV_VAR}\n"
+		"    escaped: \"${ENV_VAR}\"\n"
+		"    subvalue:\n"
+		"        subvalue2:\n"
+		"            boolean: ${UNSED_XX_X_X_VAR}\n"
+		"base_value_2:\n"
+		"    sample_list:\n"
+		"        - ${ENV_VAR}\n"
+		"        - ' ${ENV_VAR}'\n"
+		"        - '${ENV_VAR} '\n"
+		"        - $UNSED_XX_X_X_VAR\n"
+		"paths:\n"
+		"    - ${ENV_VAR}/foo\n"
+		"    - $ENV_VAR/foo\n"
+		"    - /foo/${ENV_VAR}/\n"
+		"    - /${ENV_VAR}/${ENV_VAR}${ENV_VAR}/foo\n"
+		"    - ${ENV_VAR_EMBEDDED}/foo\n"
+		"is_test: ${ENV_VAR_BOOL}\n"
+		"num_test: ${ENV_VAR_INT}\n"
+		"empty_test: ${ENV_VAR_EMPTY}\n"
+		"plugins:\n"
+		"  - name: k8saudit\n"
+		"    library_path: /foo/${ENV_VAR}/libk8saudit.so\n"
+		"    open_params: ${ENV_VAR_INT}\n";
+
+	yaml_helper conf;
+	conf.load_from_string(env_var_sample_yaml);
+
+	/* Check if the base values are defined */
+	ASSERT_TRUE(conf.is_defined("base_value"));
+	ASSERT_TRUE(conf.is_defined("base_value_2"));
+	ASSERT_TRUE(conf.is_defined("paths"));
+	ASSERT_FALSE(conf.is_defined("unknown_base_value"));
+
+	/* Test fetching of a regular string without any environment variable */
+	auto base_value_string = conf.get_scalar<std::string>("base_value.string", default_value);
+	ASSERT_EQ(base_value_string, "my_string");
+
+	/* Test fetching of escaped environment variable format. Should return the string as-is after stripping the leading `$` */
+	auto base_value_invalid = conf.get_scalar<std::string>("base_value.invalid", default_value);
+	ASSERT_EQ(base_value_invalid, "${ENV_VAR}");
+
+	/* Test fetching of invalid escaped environment variable format. Should return the string as-is */
+	auto base_value_invalid_env = conf.get_scalar<std::string>("base_value.invalid_env", default_value);
+	ASSERT_EQ(base_value_invalid_env, "$$ENV_VAR");
+
+	/* Test fetching of 2 escaped environment variables side by side. Should return the string as-is after stripping the leading `$` */
+	auto base_value_double_invalid = conf.get_scalar<std::string>("base_value.invalid_double_env", default_value);
+	ASSERT_EQ(base_value_double_invalid, "${ENV_VAR}${ENV_VAR}");
+
+	/*
+     * Test fetching of escaped environment variable format with inside an env variable.
+     * Should return the string as-is after stripping the leading `$` with the resolved env variable within
+	 */
+	auto base_value_embedded_invalid = conf.get_scalar<std::string>("base_value.invalid_embedded_env", default_value);
+	ASSERT_EQ(base_value_embedded_invalid, "${" + env_var_value + "}");
+
+	/*
+     * Test fetching of an escaped env variable plus an env variable side by side.
+     * Should return the escaped one trimming the leading `$` plus the second one resolved.
+	 */
+	auto base_value_valid_invalid = conf.get_scalar<std::string>("base_value.invalid_valid_env", default_value);
+	ASSERT_EQ(base_value_valid_invalid, "${ENV_VAR}" + env_var_value);
+
+	/* Test fetching of strings that contain environment variables */
+	auto base_value_id = conf.get_scalar<std::string>("base_value.id", default_value);
+	ASSERT_EQ(base_value_id, "$ENV_VAR"); // Does not follow the `${VAR}` format, so it should be treated as a regular string
+
+	auto base_value_name = conf.get_scalar<std::string>("base_value.name", default_value);
+	ASSERT_EQ(base_value_name, env_var_value); // Proper environment variable format
+
+	auto base_value_escaped = conf.get_scalar<std::string>("base_value.escaped", default_value);
+	ASSERT_EQ(base_value_escaped, env_var_value); // Environment variable within quotes
+
+	/* Test fetching of an undefined environment variable. Resolves to empty string. */
+	auto unknown_boolean = conf.get_scalar<std::string>("base_value.subvalue.subvalue2.boolean", default_value);
+	ASSERT_EQ(unknown_boolean, "");
+
+	/* Test fetching of environment variables from a list */
+	auto base_value_2_list_0 = conf.get_scalar<std::string>("base_value_2.sample_list[0]", default_value);
+	ASSERT_EQ(base_value_2_list_0, env_var_value); // Proper environment variable format
+
+	auto base_value_2_list_1 = conf.get_scalar<std::string>("base_value_2.sample_list[1]", default_value);
+	ASSERT_EQ(base_value_2_list_1, " " + env_var_value); // Environment variable preceded by a space, still extracted env var with leading space
+
+	auto base_value_2_list_2 = conf.get_scalar<std::string>("base_value_2.sample_list[2]", default_value);
+	ASSERT_EQ(base_value_2_list_2, env_var_value + " "); // Environment variable followed by a space, still extracted env var with trailing space
+
+	auto base_value_2_list_3 = conf.get_scalar<std::string>("base_value_2.sample_list[3]", default_value);
+	ASSERT_EQ(base_value_2_list_3, "$UNSED_XX_X_X_VAR"); // Does not follow the `${VAR}` format, so should be treated as a regular string
+
+	/* Test expansion of environment variables within strings */
+	auto path_list_0 = conf.get_scalar<std::string>("paths[0]", default_value);
+	ASSERT_EQ(path_list_0, env_var_value + "/foo"); // Even if env var is part of bigger string, it gets expanded
+
+	auto path_list_1 = conf.get_scalar<std::string>("paths[1]", default_value);
+	ASSERT_EQ(path_list_1, "$ENV_VAR/foo"); // Does not follow the `${VAR}` format, so should be treated as a regular string
+
+	auto path_list_2 = conf.get_scalar<std::string>("paths[2]", default_value);
+	ASSERT_EQ(path_list_2, "/foo/" + env_var_value + "/"); // Even when env var is in the middle of a string. it gets expanded
+
+	auto path_list_3 = conf.get_scalar<std::string>("paths[3]", default_value);
+	ASSERT_EQ(path_list_3, "/" + env_var_value + "/" + env_var_value + env_var_value + "/foo"); // Even when the string contains multiple env vars they are correctly expanded
+
+	auto path_list_4 = conf.get_scalar<std::string>("paths[4]", default_value);
+	ASSERT_EQ(path_list_4, env_var_value + "/foo"); // Even when the env var contains another env var, it gets correctly double-expanded
+
+	/* Check that variable expansion is type-aware */
+	auto boolean = conf.get_scalar<bool>("is_test", false);
+	ASSERT_EQ(boolean, true); // `true` can be parsed to bool.
+
+	auto boolean_as_str = conf.get_scalar<std::string>("is_test", "false");
+	ASSERT_EQ(boolean_as_str, "true"); // `true` can be parsed to string.
+
+	auto boolean_as_int = conf.get_scalar<int32_t>("is_test", 0);
+	ASSERT_EQ(boolean_as_int, 0); // `true` cannot be parsed to integer.
+
+	auto integer = conf.get_scalar<int32_t>("num_test", -1);
+	ASSERT_EQ(integer, 12);
+
+	// An env var that resolves to an empty string returns ""
+	auto empty_default_str = conf.get_scalar<std::string>("empty_test", default_value);
+	ASSERT_EQ(empty_default_str, "");
+
+	std::list<falco_configuration::plugin_config> plugins;
+	conf.get_sequence<std::list<falco_configuration::plugin_config>>(plugins, std::string("plugins"));
+	std::vector<falco_configuration::plugin_config> m_plugins{ std::make_move_iterator(std::begin(plugins)),
+								  std::make_move_iterator(std::end(plugins)) };
+	ASSERT_EQ(m_plugins[0].m_name, "k8saudit");
+	ASSERT_EQ(m_plugins[0].m_library_path, "/foo/" + env_var_value + "/libk8saudit.so");
+	ASSERT_EQ(m_plugins[0].m_open_params, "12");
+
+	/* Clear the set environment variables after testing */
+	SET_ENV_VAR(env_var_name.c_str(), "");
+	SET_ENV_VAR(embedded_env_var_name.c_str(), "");
+	SET_ENV_VAR(bool_env_var_name.c_str(), "");
+	SET_ENV_VAR(int_env_var_name.c_str(), "");
+	SET_ENV_VAR(empty_env_var_name.c_str(), "");
+}

unit_tests/falco/test_configuration_output_options.cpp

@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+#include <falco/configuration.h>
+
+TEST(ConfigurationRuleOutputOptions, parse_yaml)
+{
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_content(R"(
+append_output:
+  - source: syscall
+    tag: persistence
+    rule: some rule name
+    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
+
+  - tag: persistence
+    fields:
+      - proc.aname[2]: "%proc.aname[2]"
+      - proc.aname[3]: "%proc.aname[3]"
+      - proc.aname[4]: "%proc.aname[4]"
+    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
+
+  - source: k8s_audit
+    fields:
+      - ka.verb
+      - static_field: "static content"
+
+	)", {}));
+
+	EXPECT_EQ(falco_config.m_append_output.size(), 3);
+
+	EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall");
+	EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name");
+	EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0);
+	EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3);
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[2]"], "%proc.aname[2]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[3]"], "%proc.aname[3]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[4]"], "%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_source, "k8s_audit");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields["static_field"], "static content");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1);
+}
+
+TEST(ConfigurationRuleOutputOptions, cli_options)
+{
+	falco_configuration falco_config;
+
+	ASSERT_NO_THROW(falco_config.init_from_content("", 
+		std::vector<std::string>{
+			R"(append_output[]={"source": "syscall", "tag": "persistence", "rule": "some rule name", "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
+			R"(append_output[]={"tag": "persistence", "fields": [{"proc.aname[2]": "%proc.aname[2]"}, {"proc.aname[3]": "%proc.aname[3]"}, {"proc.aname[4]": "%proc.aname[4]"}], "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
+			R"(append_output[]={"source": "k8s_audit", "fields": ["ka.verb", {"static_field": "static content"}]})"}));
+
+	EXPECT_EQ(falco_config.m_append_output.size(), 3);
+
+	EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall");
+	EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name");
+	EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0);
+	EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3);
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[2]"], "%proc.aname[2]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[3]"], "%proc.aname[3]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[4]"], "%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_source, "k8s_audit");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields["static_field"], "static content");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1);
+}

unit_tests/falco/test_configuration_rule_selection.cpp

@@ -21,7 +21,7 @@ limitations under the License.
 TEST(ConfigurationRuleSelection, parse_yaml)
 {
 	falco_configuration falco_config;
-	EXPECT_NO_THROW(falco_config.init_from_content(R"(
+	ASSERT_NO_THROW(falco_config.init_from_content(R"(
 rules:
   - enable:
       rule: 'Terminal Shell in Container'
@@ -33,28 +33,42 @@ rules:
       rule: 'hello*'
 	)", {}));
 
-	ASSERT_EQ(falco_config.m_rules_selection.size(), 3);
+	EXPECT_EQ(falco_config.m_rules_selection.size(), 3);
 
-	ASSERT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::enable);
-	ASSERT_EQ(falco_config.m_rules_selection[0].m_rule, "Terminal Shell in Container");
+	EXPECT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::enable);
+	EXPECT_EQ(falco_config.m_rules_selection[0].m_rule, "Terminal Shell in Container");
 
-	ASSERT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::disable);
-	ASSERT_EQ(falco_config.m_rules_selection[1].m_tag, "experimental");
+	EXPECT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::disable);
+	EXPECT_EQ(falco_config.m_rules_selection[1].m_tag, "experimental");
 
-	ASSERT_EQ(falco_config.m_rules_selection[2].m_op, falco_configuration::rule_selection_operation::enable);
-	ASSERT_EQ(falco_config.m_rules_selection[2].m_rule, "hello*");
+	EXPECT_EQ(falco_config.m_rules_selection[2].m_op, falco_configuration::rule_selection_operation::enable);
+	EXPECT_EQ(falco_config.m_rules_selection[2].m_rule, "hello*");
 }
 
 TEST(ConfigurationRuleSelection, cli_options)
 {
 	falco_configuration falco_config;
-	EXPECT_NO_THROW(falco_config.init_from_content("", std::vector<std::string>{"rules[].disable.tag=maturity_incubating", "rules[].enable.rule=Adding ssh keys to authorized_keys"}));
+	ASSERT_NO_THROW(falco_config.init_from_content("", std::vector<std::string>{"rules[].disable.tag=maturity_incubating", "rules[].enable.rule=Adding ssh keys to authorized_keys"}));
 
-	ASSERT_EQ(falco_config.m_rules_selection.size(), 2);
+	EXPECT_EQ(falco_config.m_rules_selection.size(), 2);
 
-	ASSERT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::disable);
-	ASSERT_EQ(falco_config.m_rules_selection[0].m_tag, "maturity_incubating");
+	EXPECT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::disable);
+	EXPECT_EQ(falco_config.m_rules_selection[0].m_tag, "maturity_incubating");
 
-	ASSERT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::enable);
-	ASSERT_EQ(falco_config.m_rules_selection[1].m_rule, "Adding ssh keys to authorized_keys");
+	EXPECT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::enable);
+	EXPECT_EQ(falco_config.m_rules_selection[1].m_rule, "Adding ssh keys to authorized_keys");
+}
+
+TEST(ConfigurationRuleSelection, cli_options_object)
+{
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_content("", std::vector<std::string>{R"(rules[]={"disable": {"tag": "maturity_incubating"}})", R"(rules[]={"enable": {"rule": "Adding ssh keys to authorized_keys"}})"}));
+
+	EXPECT_EQ(falco_config.m_rules_selection.size(), 2);
+
+	EXPECT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::disable);
+	EXPECT_EQ(falco_config.m_rules_selection[0].m_tag, "maturity_incubating");
+
+	EXPECT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::enable);
+	EXPECT_EQ(falco_config.m_rules_selection[1].m_rule, "Adding ssh keys to authorized_keys");
 }

unit_tests/falco/test_configuration_schema.cpp

@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+#include <falco/configuration.h>
+#include <falco_test_var.h>
+
+#define EXPECT_VALIDATION_STATUS(res, status) \
+        do { \
+		for(const auto& pair : res) { \
+			auto validation_status = pair.second; \
+			EXPECT_TRUE(sinsp_utils::startswith(validation_status, status)) << validation_status; \
+		}                                           \
+	} \
+	while (0)
+
+// Read Falco config from current repo-path
+TEST(Configuration, schema_validate_config)
+{
+	falco_configuration falco_config;
+	config_loaded_res res;
+
+	if (!std::filesystem::exists(TEST_FALCO_CONFIG))
+	{
+		GTEST_SKIP() << "Falco config not present under " << TEST_FALCO_CONFIG;
+	}
+	EXPECT_NO_THROW(res = falco_config.init_from_file(TEST_FALCO_CONFIG, {}));
+	EXPECT_VALIDATION_STATUS(res, yaml_helper::validation_ok);
+}
+
+TEST(Configuration, schema_ok)
+{
+	falco_configuration falco_config;
+	config_loaded_res res;
+
+	/* OK YAML */
+	std::string config =
+		"falco_libs:\n"
+		"    thread_table_size: 50\n";
+
+	EXPECT_NO_THROW(res = falco_config.init_from_content(config, {}));
+	EXPECT_VALIDATION_STATUS(res, yaml_helper::validation_ok);
+}
+
+TEST(Configuration, schema_wrong_key)
+{
+	falco_configuration falco_config;
+	config_loaded_res res;
+
+	/* Miss-typed key YAML */
+	std::string config =
+		"falco_libss:\n"
+		"    thread_table_size: 50\n";
+
+	EXPECT_NO_THROW(res = falco_config.init_from_content(config, {}));
+	EXPECT_VALIDATION_STATUS(res, yaml_helper::validation_failed);
+}
+
+TEST(Configuration, schema_wrong_type)
+{
+	falco_configuration falco_config;
+
+	/* Wrong value type YAML */
+	std::string config =
+		"falco_libs: 512\n";
+
+	// We expect an exception since `falco_configuration::load_yaml()`
+	// will fail to parse `falco_libs` node.
+	ASSERT_ANY_THROW(falco_config.init_from_content(config, {}));
+}
+
+TEST(Configuration, schema_wrong_embedded_key)
+{
+	falco_configuration falco_config;
+	config_loaded_res res;
+
+	/* Miss-typed sub-key YAML */
+	std::string config =
+		"falco_libs:\n"
+		"    thread_table_sizeee: 50\n";
+
+	EXPECT_NO_THROW(res = falco_config.init_from_content(config, {}));
+	EXPECT_VALIDATION_STATUS(res, yaml_helper::validation_failed);
+}
+
+TEST(Configuration, schema_yaml_helper_validator)
+{
+	yaml_helper conf;
+	falco_configuration falco_config;
+
+	/* Broken YAML */
+	std::string sample_yaml =
+		"falco_libs:\n"
+		"    thread_table_size: 50\n";
+
+	// Ok, we don't ask for any validation
+	EXPECT_NO_THROW(conf.load_from_string(sample_yaml));
+
+	// We pass a string variable but not a schema
+	std::string validation;
+	EXPECT_NO_THROW(conf.load_from_string(sample_yaml, nlohmann::json{}, &validation));
+	EXPECT_EQ(validation, yaml_helper::validation_none);
+
+	// We pass a schema but not a string storage for the validation; no validation takes place
+	EXPECT_NO_THROW(conf.load_from_string(sample_yaml, falco_config.m_config_schema, nullptr));
+
+	// We pass everything
+	EXPECT_NO_THROW(conf.load_from_string(sample_yaml, falco_config.m_config_schema, &validation));
+	EXPECT_EQ(validation, yaml_helper::validation_ok);
+}

unit_tests/falco_test_var.h.in

@@ -2,3 +2,4 @@
 
 #define TEST_ENGINE_KMOD_CONFIG "${CMAKE_SOURCE_DIR}/unit_tests/falco/test_configs/engine_kmod_config.yaml"
 #define TEST_ENGINE_MODERN_CONFIG "${CMAKE_SOURCE_DIR}/unit_tests/falco/test_configs/engine_modern_config.yaml"
+#define TEST_FALCO_CONFIG "${CMAKE_SOURCE_DIR}/falco.yaml"

unit_tests/test_falco_engine.cpp

@@ -85,3 +85,15 @@ std::string test_falco_engine::get_compiled_rule_condition(std::string rule_name
 	auto rule_description = m_engine->describe_rule(&rule_name, {});
 	return rule_description["rules"][0]["details"]["condition_compiled"].template get<std::string>();
 }
+
+std::string test_falco_engine::get_compiled_rule_output(std::string rule_name) const
+{
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	return rule_description["rules"][0]["details"]["output_compiled"].template get<std::string>();
+}
+
+std::unordered_map<std::string, std::string> test_falco_engine::get_compiled_rule_formatted_fields(std::string rule_name) const
+{
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	return rule_description["rules"][0]["details"]["extra_output_formatted_fields"].template get<std::unordered_map<std::string, std::string>>();
+}

unit_tests/test_falco_engine.h

@@ -6,6 +6,7 @@
 #include "rule_loading_messages.h"
 
 #include <gtest/gtest.h>
+#include <unordered_map>
 
 class test_falco_engine : public testing::Test
 {
@@ -19,6 +20,8 @@ protected:
 	bool check_warning_message(const std::string& warning_msg) const;
 	bool check_error_message(const std::string& error_msg) const;
 	std::string get_compiled_rule_condition(std::string rule_name = "") const;
+	std::string get_compiled_rule_output(std::string rule_name = "") const;
+	std::unordered_map<std::string, std::string> get_compiled_rule_formatted_fields(std::string rule_name) const;
 
 	std::string m_sample_ruleset = "sample-ruleset";
 	std::string m_sample_source = falco_common::syscall_source;

userspace/engine/CMakeLists.txt

@@ -22,6 +22,7 @@ add_library(falco_engine STATIC
     filter_details_resolver.cpp
     filter_macro_resolver.cpp
     filter_warning_resolver.cpp
+    logger.cpp
     stats_manager.cpp
     rule_loader.cpp
     rule_loader_reader.cpp

userspace/engine/evttype_index_ruleset.cpp

@@ -17,14 +17,13 @@ limitations under the License.
 
 #include "evttype_index_ruleset.h"
 
-#include "falco_utils.h"
-
-#include "../falco/logger.h"
+#include "logger.h"
 
 #include <algorithm>
 
 evttype_index_ruleset::evttype_index_ruleset(
-	std::shared_ptr<sinsp_filter_factory> f): m_filter_factory(f)
+	std::shared_ptr<sinsp_filter_factory> f):
+	m_filter_factory(f)
 {
 }
 
@@ -32,170 +31,6 @@ evttype_index_ruleset::~evttype_index_ruleset()
 {
 }
 
-evttype_index_ruleset::ruleset_filters::ruleset_filters()
-{
-}
-
-evttype_index_ruleset::ruleset_filters::~ruleset_filters()
-{
-}
-
-void evttype_index_ruleset::ruleset_filters::add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
-{
-	// This is O(n) but it's also uncommon
-	// (when loading rules only).
-	auto pos = std::find(wrappers.begin(),
-			     wrappers.end(),
-			     wrap);
-
-	if(pos == wrappers.end())
-	{
-		wrappers.push_back(wrap);
-	}
-}
-
-void evttype_index_ruleset::ruleset_filters::remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
-{
-	// This is O(n) but it's also uncommon
-	// (when loading rules only).
-	auto pos = std::find(wrappers.begin(),
-			     wrappers.end(),
-			     wrap);
-	if(pos != wrappers.end())
-	{
-		wrappers.erase(pos);
-	}
-}
-
-void evttype_index_ruleset::ruleset_filters::add_filter(std::shared_ptr<filter_wrapper> wrap)
-{
-	if(wrap->event_codes.empty())
-	{
-		// Should run for all event types
-		add_wrapper_to_list(m_filter_all_event_types, wrap);
-	}
-	else
-	{
-		for(auto &etype : wrap->event_codes)
-		{
-			if(m_filter_by_event_type.size() <= etype)
-			{
-				m_filter_by_event_type.resize(etype + 1);
-			}
-
-			add_wrapper_to_list(m_filter_by_event_type[etype], wrap);
-		}
-	}
-
-	m_filters.insert(wrap);
-}
-
-void evttype_index_ruleset::ruleset_filters::remove_filter(std::shared_ptr<filter_wrapper> wrap)
-{
-	if(wrap->event_codes.empty())
-	{
-		remove_wrapper_from_list(m_filter_all_event_types, wrap);
-	}
-	else
-	{
-		for(auto &etype : wrap->event_codes)
-		{
-			if( etype < m_filter_by_event_type.size() )
-			{
-				remove_wrapper_from_list(m_filter_by_event_type[etype], wrap);
-			}
-		}
-	}
-
-	m_filters.erase(wrap);
-}
-
-uint64_t evttype_index_ruleset::ruleset_filters::num_filters()
-{
-	return m_filters.size();
-}
-
-bool evttype_index_ruleset::ruleset_filters::run(sinsp_evt *evt, falco_rule& match)
-{
-    if(evt->get_type() < m_filter_by_event_type.size())
-    {
-        for(const auto &wrap : m_filter_by_event_type[evt->get_type()])
-        {
-            if(wrap->filter->run(evt))
-            {
-				match = wrap->rule;
-                return true;
-            }
-        }
-    }
-
-	// Finally, try filters that are not specific to an event type.
-	for(const auto &wrap : m_filter_all_event_types)
-	{
-		if(wrap->filter->run(evt))
-		{
-			match = wrap->rule;
-			return true;
-		}
-	}
-
-	return false;
-}
-
-bool evttype_index_ruleset::ruleset_filters::run(sinsp_evt *evt, std::vector<falco_rule>& matches)
-{
-	bool match_found = false;
-
-	if(evt->get_type() < m_filter_by_event_type.size())
-	{
-		for(const auto &wrap : m_filter_by_event_type[evt->get_type()])
-		{
-			if(wrap->filter->run(evt))
-			{
-				matches.push_back(wrap->rule);
-				match_found = true;
-			}
-		}
-	}
-
-	if(match_found)
-	{
-		return true;
-	}
-
-	// Finally, try filters that are not specific to an event type.
-	for(const auto &wrap : m_filter_all_event_types)
-	{
-		if(wrap->filter->run(evt))
-		{
-			matches.push_back(wrap->rule);
-			match_found = true;
-		}
-	}
-
-	return match_found;
-}
-
-libsinsp::events::set<ppm_sc_code> evttype_index_ruleset::ruleset_filters::sc_codes()
-{
-	libsinsp::events::set<ppm_sc_code> res;
-	for(const auto &wrap : m_filters)
-	{
-		res.insert(wrap->sc_codes.begin(), wrap->sc_codes.end());
-	}
-	return res;
-}
-
-libsinsp::events::set<ppm_event_code> evttype_index_ruleset::ruleset_filters::event_codes()
-{
-	libsinsp::events::set<ppm_event_code> res;
-	for(const auto &wrap : m_filters)
-	{
-		res.insert(wrap->event_codes.begin(), wrap->event_codes.end());
-	}
-	return res;
-}
-
 void evttype_index_ruleset::add(
 		const falco_rule& rule,
 		std::shared_ptr<sinsp_filter> filter,
@@ -203,21 +38,22 @@ void evttype_index_ruleset::add(
 {
 	try
 	{
-		auto wrap = std::make_shared<filter_wrapper>();
-		wrap->rule = rule;
-		wrap->filter = filter;
+		auto wrap = std::make_shared<evttype_index_wrapper>();
+		wrap->m_rule = rule;
+		wrap->m_filter = filter;
 		if(rule.source == falco_common::syscall_source)
 		{
-			wrap->sc_codes = libsinsp::filter::ast::ppm_sc_codes(condition.get());
-			wrap->event_codes = libsinsp::filter::ast::ppm_event_codes(condition.get());
+			wrap->m_sc_codes = libsinsp::filter::ast::ppm_sc_codes(condition.get());
+			wrap->m_event_codes = libsinsp::filter::ast::ppm_event_codes(condition.get());
 		}
 		else
 		{
-			wrap->sc_codes = { };
-			wrap->event_codes = { ppm_event_code::PPME_PLUGINEVENT_E };
+			wrap->m_sc_codes = {};
+			wrap->m_event_codes = {ppm_event_code::PPME_PLUGINEVENT_E};
 		}
-		wrap->event_codes.insert(ppm_event_code::PPME_ASYNCEVENT_E);
-		m_filters.insert(wrap);
+		wrap->m_event_codes.insert(ppm_event_code::PPME_ASYNCEVENT_E);
+
+		add_wrapper(wrap);
 	}
 	catch (const sinsp_exception& e)
 	{
@@ -230,180 +66,46 @@ void evttype_index_ruleset::on_loading_complete()
 	print_enabled_rules_falco_logger();
 }
 
+bool evttype_index_ruleset::run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, falco_rule &match)
+{
+	for(auto &wrap : wrappers)
+	{
+		if(wrap->m_filter->run(evt))
+		{
+			match = wrap->m_rule;
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool evttype_index_ruleset::run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, std::vector<falco_rule> &matches)
+{
+	bool match_found = false;
+
+	for(auto &wrap : wrappers)
+	{
+		if(wrap->m_filter->run(evt))
+		{
+			matches.push_back(wrap->m_rule);
+			match_found = true;
+		}
+	}
+
+	return match_found;
+}
+
 void evttype_index_ruleset::print_enabled_rules_falco_logger()
 {
 	falco_logger::log(falco_logger::level::DEBUG, "Enabled rules:\n");
-	int n = 0;
-	for (const auto& ruleset_ptr : m_rulesets)
+
+	auto logger = [](std::shared_ptr<evttype_index_wrapper> wrap)
 	{
-		if (ruleset_ptr)
-		{
-			for (const auto& wrap : ruleset_ptr->get_filters())
-			{
-				n++;
-				falco_logger::log(falco_logger::level::DEBUG, std::string("   ") + wrap->rule.name + "\n");
-			}
-		}
-	}
-	falco_logger::log(falco_logger::level::DEBUG, "(" + std::to_string(n) + ") enabled rules in total\n");
-}
-
-void evttype_index_ruleset::clear()
-{
-	for (size_t i = 0; i < m_rulesets.size(); i++)
-	{
-		m_rulesets[i] = std::make_shared<ruleset_filters>();
-	}
-	m_filters.clear();
-}
-
-void evttype_index_ruleset::enable(const std::string &pattern, match_type match, uint16_t ruleset_id)
-{
-	enable_disable(pattern, match, true, ruleset_id);
-}
-
-void evttype_index_ruleset::disable(const std::string &pattern, match_type match, uint16_t ruleset_id)
-{
-	enable_disable(pattern, match, false, ruleset_id);
-}
-
-void evttype_index_ruleset::enable_disable(const std::string &pattern, match_type match, bool enabled, uint16_t ruleset_id)
-{
-	while(m_rulesets.size() < (size_t)ruleset_id + 1)
-	{
-		m_rulesets.emplace_back(std::make_shared<ruleset_filters>());
-	}
-
-	for(const auto &wrap : m_filters)
-	{
-		bool matches;
-		std::string::size_type pos;
-
-		switch(match)
-		{
-		case match_type::exact:
-			pos = wrap->rule.name.find(pattern);
-
-			matches = (pattern == "" || (pos == 0 &&
-						       pattern.size() == wrap->rule.name.size()));
-			break;
-		case match_type::substring:
-			matches = (pattern == "" || (wrap->rule.name.find(pattern) != std::string::npos));
-			break;
-		case match_type::wildcard:
-			matches = falco::utils::matches_wildcard(pattern, wrap->rule.name);
-			break;
-		default:
-			// should never happen
-			matches = false;
-		}
-
-		if(matches)
-		{
-			if(enabled)
-			{
-				m_rulesets[ruleset_id]->add_filter(wrap);
-			}
-			else
-			{
-				m_rulesets[ruleset_id]->remove_filter(wrap);
-			}
-		}
-	}
-}
-
-void evttype_index_ruleset::enable_tags(const std::set<std::string> &tags, uint16_t ruleset_id)
-{
-	enable_disable_tags(tags, true, ruleset_id);
-}
-
-void evttype_index_ruleset::disable_tags(const std::set<std::string> &tags, uint16_t ruleset_id)
-{
-	enable_disable_tags(tags, false, ruleset_id);
-}
-
-void evttype_index_ruleset::enable_disable_tags(const std::set<std::string> &tags, bool enabled, uint16_t ruleset_id)
-{
-	while(m_rulesets.size() < (size_t)ruleset_id + 1)
-	{
-		m_rulesets.emplace_back(std::make_shared<ruleset_filters>());
-	}
-
-	for(const auto &wrap : m_filters)
-	{
-		std::set<std::string> intersect;
-
-		set_intersection(tags.begin(), tags.end(),
-				 wrap->rule.tags.begin(), wrap->rule.tags.end(),
-				 inserter(intersect, intersect.begin()));
-
-		if(!intersect.empty())
-		{
-			if(enabled)
-			{
-				m_rulesets[ruleset_id]->add_filter(wrap);
-			}
-			else
-			{
-				m_rulesets[ruleset_id]->remove_filter(wrap);
-			}
-		}
-	}
-}
-
-uint64_t evttype_index_ruleset::enabled_count(uint16_t ruleset_id)
-{
-	while(m_rulesets.size() < (size_t)ruleset_id + 1)
-	{
-		m_rulesets.emplace_back(std::make_shared<ruleset_filters>());
-	}
-
-	return m_rulesets[ruleset_id]->num_filters();
-}
-
-bool evttype_index_ruleset::run(sinsp_evt *evt, falco_rule& match, uint16_t ruleset_id)
-{
-	if(m_rulesets.size() < (size_t)ruleset_id + 1)
-	{
-		return false;
-	}
-
-	return m_rulesets[ruleset_id]->run(evt, match);
-}
-
-bool evttype_index_ruleset::run(sinsp_evt *evt, std::vector<falco_rule>& matches, uint16_t ruleset_id)
-{
-	if(m_rulesets.size() < (size_t)ruleset_id + 1)
-	{
-		return false;
-	}
-
-	return m_rulesets[ruleset_id]->run(evt, matches);
-}
-
-void evttype_index_ruleset::enabled_evttypes(std::set<uint16_t> &evttypes, uint16_t ruleset_id)
-{
-	evttypes.clear();
-	for (const auto& e : enabled_event_codes(ruleset_id))
-	{
-		evttypes.insert((uint16_t) e);
-	}
-}
-
-libsinsp::events::set<ppm_sc_code> evttype_index_ruleset::enabled_sc_codes(uint16_t ruleset)
-{
-	if(m_rulesets.size() < (size_t)ruleset + 1)
-	{
-		return {};
-	}
-	return m_rulesets[ruleset]->sc_codes();
-}
-
-libsinsp::events::set<ppm_event_code> evttype_index_ruleset::enabled_event_codes(uint16_t ruleset)
-{
-	if(m_rulesets.size() < (size_t)ruleset + 1)
-	{
-		return {};
-	}
-	return m_rulesets[ruleset]->event_codes();
+		falco_logger::log(falco_logger::level::DEBUG, std::string("   ") + wrap->name() + "\n");
+	};
+
+	uint64_t num_filters = iterate(logger);
+
+	falco_logger::log(falco_logger::level::DEBUG, "(" + std::to_string(num_filters) + ") enabled rules in total\n");
 }

userspace/engine/evttype_index_ruleset.h

@@ -17,151 +17,54 @@ limitations under the License.
 
 #pragma once
 
+#include "indexable_ruleset.h"
+
 #include <string>
 #include <set>
 #include <vector>
-#include <list>
-#include <map>
-
-#include "filter_ruleset.h"
-#include <libsinsp/sinsp.h>
-#include <libsinsp/filter.h>
-#include <libsinsp/event.h>
 
 /*!
 	\brief A filter_ruleset that indexes enabled rules by event type,
 	and performs linear search on each event type bucket
 */
-class evttype_index_ruleset: public filter_ruleset
+
+struct evttype_index_wrapper
+{
+	const std::string &name() { return m_rule.name; }
+	const std::set<std::string> &tags() { return m_rule.tags; }
+	const libsinsp::events::set<ppm_sc_code> &sc_codes() { return m_sc_codes; }
+	const libsinsp::events::set<ppm_event_code> &event_codes() { return m_event_codes; }
+
+	falco_rule m_rule;
+	libsinsp::events::set<ppm_sc_code> m_sc_codes;
+	libsinsp::events::set<ppm_event_code> m_event_codes;
+	std::shared_ptr<sinsp_filter> m_filter;
+};
+
+class evttype_index_ruleset : public indexable_ruleset<evttype_index_wrapper>
 {
 public:
 	explicit evttype_index_ruleset(std::shared_ptr<sinsp_filter_factory> factory);
 	virtual ~evttype_index_ruleset();
 
+	// From filter_ruleset
 	void add(
 		const falco_rule& rule,
 		std::shared_ptr<sinsp_filter> filter,
 		std::shared_ptr<libsinsp::filter::ast::expr> condition) override;
 
-	void clear() override;
-
-	bool run(sinsp_evt *evt, falco_rule& match, uint16_t ruleset_id) override;
-	bool run(sinsp_evt *evt, std::vector<falco_rule>&matches, uint16_t ruleset_id) override;
-
-	uint64_t enabled_count(uint16_t ruleset_id) override;
-
 	void on_loading_complete() override;
 
+	// From indexable_ruleset
+	bool run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, falco_rule &match) override;
+	bool run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, std::vector<falco_rule> &matches) override;
+
 	// Print each enabled rule when running Falco with falco logger
 	// log_level=debug; invoked within on_loading_complete()
 	void print_enabled_rules_falco_logger();
 
-	void enable(
-		const std::string &pattern,
-		match_type match,
-		uint16_t rulset_id) override;
-
-	void disable(
-		const std::string &pattern,
-		match_type match,
-		uint16_t rulset_id) override;
-
-	void enable_tags(
-		const std::set<std::string> &tags,
-		uint16_t rulset_id) override;
-
-	void disable_tags(
-		const std::set<std::string> &tags,
-		uint16_t rulset_id) override;
-
-	// note(jasondellaluce): this is deprecated, must use the new
-	// typing-improved `enabled_event_codes` and `enabled_sc_codes` instead
-	// todo(jasondellaluce): remove this in future code refactors
-	void enabled_evttypes(
-		std::set<uint16_t> &evttypes,
-		uint16_t ruleset) override;
-
-	libsinsp::events::set<ppm_sc_code> enabled_sc_codes(uint16_t ruleset) override;
-
-	libsinsp::events::set<ppm_event_code> enabled_event_codes(uint16_t ruleset) override;
-
 private:
-
-	// Helper used by enable()/disable()
-	void enable_disable(
-		const std::string &pattern,
-		match_type match,
-		bool enabled,
-		uint16_t rulset_id);
-
-	// Helper used by enable_tags()/disable_tags()
-	void enable_disable_tags(
-		const std::set<std::string> &tags,
-		bool enabled,
-		uint16_t rulset_id);
-
-	struct filter_wrapper
-	{
-		falco_rule rule;
-		libsinsp::events::set<ppm_sc_code> sc_codes;
-		libsinsp::events::set<ppm_event_code> event_codes;
-		std::shared_ptr<sinsp_filter> filter;
-	};
-
-	typedef std::list<std::shared_ptr<filter_wrapper>> filter_wrapper_list;
-
-	// A group of filters all having the same ruleset
-	class ruleset_filters {
-	public:
-		ruleset_filters();
-
-		virtual ~ruleset_filters();
-
-		void add_filter(std::shared_ptr<filter_wrapper> wrap);
-		void remove_filter(std::shared_ptr<filter_wrapper> wrap);
-
-		uint64_t num_filters();
-
-		inline const std::set<std::shared_ptr<filter_wrapper>>& get_filters() const
-		{
-			return m_filters;
-		}
-
-		// Evaluate an event against the ruleset and return the first rule
-		// that matched.
-		bool run(sinsp_evt *evt, falco_rule& match);
-
-		//  Evaluate an event against the ruleset and return all the
-		//	matching rules.
-		bool run(sinsp_evt *evt, std::vector<falco_rule>& matches);
-
-		libsinsp::events::set<ppm_sc_code> sc_codes();
-
-		libsinsp::events::set<ppm_event_code> event_codes();
-
-	private:
-		void add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap);
-		void remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap);
-
-		// Vector indexes from event type to a set of filters. There can
-		// be multiple filters for a given event type.
-		// NOTE: This is used only when the event sub-type is 0.
-		std::vector<filter_wrapper_list> m_filter_by_event_type;
-
-		filter_wrapper_list m_filter_all_event_types;
-
-		// All filters added. Used to make num_filters() fast.
-		std::set<std::shared_ptr<filter_wrapper>> m_filters;
-	};
-
-	// Vector indexes from ruleset id to set of rules.
-	std::vector<std::shared_ptr<ruleset_filters>> m_rulesets;
-
-	// All filters added. The set of enabled filters is held in m_rulesets
-	std::set<std::shared_ptr<filter_wrapper>> m_filters;
-
 	std::shared_ptr<sinsp_filter_factory> m_filter_factory;
-	std::vector<std::string> m_ruleset_names;
 };
 
 class evttype_index_ruleset_factory: public filter_ruleset_factory

userspace/engine/falco_common.h

@@ -72,3 +72,5 @@ namespace falco_common
 
 	bool parse_rule_matching(const std::string& v, rule_matching& out);
 };
+
+typedef std::unordered_map<std::string, std::pair<std::string, bool>> extra_output_field_t;

userspace/engine/falco_engine.cpp

@@ -57,8 +57,7 @@ falco_engine::falco_engine(bool seed_rng)
 	  m_rule_compiler(std::make_shared<rule_loader::compiler>()),
 	  m_next_ruleset_id(0),
 	  m_min_priority(falco_common::PRIORITY_DEBUG),
-	  m_sampling_ratio(1), m_sampling_multiplier(0),
-	  m_replace_container_info(false)
+	  m_sampling_ratio(1), m_sampling_multiplier(0)
 {
 	if(seed_rng)
 	{
@@ -76,6 +75,7 @@ falco_engine::~falco_engine()
 	m_rule_collector->clear();
 	m_rule_stats_manager.clear();
 	m_sources.clear();
+	m_extra_output_format.clear();
 }
 
 sinsp_version falco_engine::engine_version()
@@ -194,8 +194,8 @@ void falco_engine::list_fields(const std::string &source, bool verbose, bool nam
 std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_content, const std::string &name)
 {
 	rule_loader::configuration cfg(rules_content, m_sources, name);
-	cfg.output_extra = m_extra;
-	cfg.replace_output_container_info = m_replace_container_info;
+	cfg.extra_output_format = m_extra_output_format;
+	cfg.extra_output_fields = m_extra_output_fields;
 
 	// read rules YAML file and collect its definitions
 	if(m_rule_reader->read(cfg, *m_rule_collector))
@@ -204,6 +204,11 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 		m_last_compile_output = m_rule_compiler->new_compile_output();
 		m_rule_compiler->compile(cfg, *m_rule_collector, *m_last_compile_output);
 
+		if(!cfg.res->successful())
+		{
+			return std::move(cfg.res);
+		}
+
 		// clear the rules known by the engine and each ruleset
 		m_rules.clear();
 		for (auto &src : m_sources)
@@ -450,6 +455,7 @@ std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_ev
 		rule_result.priority_num = rule.priority;
 		rule_result.tags = rule.tags;
 		rule_result.exception_fields = rule.exception_fields;
+		rule_result.extra_output_fields = rule.extra_output_fields;
 		m_rule_stats_manager.on_event(rule);
 		res->push_back(rule_result);
 	}
@@ -641,9 +647,22 @@ void falco_engine::get_json_details(
 	out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators);
 	out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields);
 
+	// Get extra requested fields
+	std::vector<std::string> out_fields;
+
+	for(auto const& f : r.extra_output_fields)
+	{
+		// add all the field keys
+		out_fields.emplace_back(f.second.first);
+
+		if (!f.second.second) // formatted field
+		{
+			out["details"]["extra_output_formatted_fields"][f.first] = f.second.first;
+		}
+	}
+
 	// Get fields from output string
 	auto fmt = create_formatter(r.source, r.output);
-	std::vector<std::string> out_fields;
 	fmt->get_field_names(out_fields);
 	out["details"]["output_fields"] = sequence_to_json_array(out_fields);
 
@@ -1077,10 +1096,37 @@ void falco_engine::set_sampling_multiplier(double sampling_multiplier)
 	m_sampling_multiplier = sampling_multiplier;
 }
 
-void falco_engine::set_extra(const std::string &extra, bool replace_container_info)
+void falco_engine::add_extra_output_format(
+	const std::string &format,
+	const std::string &source,
+	const std::string &tag,
+	const std::string &rule,
+	bool replace_container_info
+)
 {
-	m_extra = extra;
-	m_replace_container_info = replace_container_info;
+	m_extra_output_format.push_back({format, source, tag, rule, replace_container_info});
+}
+
+void falco_engine::add_extra_output_formatted_field(
+	const std::string &key,
+	const std::string &format,
+	const std::string &source,
+	const std::string &tag,
+	const std::string &rule
+)
+{
+	m_extra_output_fields.push_back({key, format, source, tag, rule, false});
+}
+
+void falco_engine::add_extra_output_raw_field(
+	const std::string &key,
+	const std::string &source,
+	const std::string &tag,
+	const std::string &rule
+)
+{
+	std::string format = "%" + key;
+	m_extra_output_fields.push_back({key, format, source, tag, rule, true});
 }
 
 inline bool falco_engine::should_drop_evt() const

userspace/engine/falco_engine.h

@@ -176,15 +176,40 @@ public:
 	//
 	void set_sampling_multiplier(double sampling_multiplier);
 
-	//
-	// You can optionally add "extra" formatting fields to the end
+	// You can optionally add "extra" output to the end
 	// of all output expressions. You can also choose to replace
 	// %container.info with the extra information or add it to the
 	// end of the expression. This is used in open source falco to
 	// add k8s/container information to outputs when
 	// available.
 	//
-	void set_extra(const std::string &extra, bool replace_container_info);
+	void add_extra_output_format(
+		const std::string &format,
+		const std::string &source,
+		const std::string &tag,
+		const std::string &rule,
+		bool replace_container_info
+	);
+
+	// You can optionally add fields that will only show up in the object
+	// output (e.g. json, gRPC) alongside other output_fields
+	// and not in the text message output.
+	// You can add two types of fields: formatted which will act like
+	// an additional output format that appears in the output field
+	void add_extra_output_formatted_field(
+		const std::string &key,
+		const std::string &format,
+		const std::string &source,
+		const std::string &tag,
+		const std::string &rule
+	);
+
+	void add_extra_output_raw_field(
+		const std::string &key,
+		const std::string &source,
+		const std::string &tag,
+		const std::string &rule
+	);
 
 	// Represents the result of matching an event against a set of
 	// rules.
@@ -196,6 +221,7 @@ public:
 		std::string format;
 		std::set<std::string> exception_fields;
 		std::set<std::string> tags;
+		extra_output_field_t extra_output_fields;
 	};
 
 	//
@@ -461,6 +487,6 @@ private:
 	static const std::string s_default_ruleset;
 	uint32_t m_default_ruleset_id;
 
-	std::string m_extra;
-	bool m_replace_container_info;
+	std::vector<rule_loader::extra_output_format_conf> m_extra_output_format;
+	std::vector<rule_loader::extra_output_field_conf> m_extra_output_fields;
 };

userspace/engine/falco_engine_version.h

@@ -20,7 +20,7 @@ limitations under the License.
 
 // The version of this Falco engine
 #define FALCO_ENGINE_VERSION_MAJOR 0
-#define FALCO_ENGINE_VERSION_MINOR 40
+#define FALCO_ENGINE_VERSION_MINOR 42
 #define FALCO_ENGINE_VERSION_PATCH 0
 
 #define FALCO_ENGINE_VERSION \
@@ -34,4 +34,4 @@ limitations under the License.
 // It represents the fields supported by this version of Falco,
 // the event types, and the underlying driverevent schema. It's used to
 // detetect changes in engine version in our CI jobs.
-#define FALCO_ENGINE_CHECKSUM "bc9d0d94ae70ef26b7cf814f62273a48b2bb4133dff0baff5f194f6f1711875a"
+#define FALCO_ENGINE_CHECKSUM "786745197f42d685b536581ddbb5eab8500939a3f6ae67cfa23a212da10e423c"

userspace/engine/falco_rule.h

@@ -79,6 +79,7 @@ struct falco_rule
 	std::string name;
 	std::string description;
 	std::string output;
+	extra_output_field_t extra_output_fields;
 	std::set<std::string> tags;
 	std::set<std::string> exception_fields;
 	falco_common::priority_type priority;

userspace/engine/formats.cpp

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include <json/json.h>
+#include <nlohmann/json.hpp>
 
 #include "formats.h"
 #include "falco_engine.h"
@@ -35,7 +35,7 @@ falco_formats::~falco_formats()
 
 std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
 				   const std::string &level, const std::string &format, const std::set<std::string> &tags,
-				   const std::string &hostname) const
+				   const std::string &hostname, const extra_output_field_t &extra_fields) const
 {
 	std::string line;
 
@@ -48,27 +48,17 @@ std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule,
 
 	if(formatter->get_output_format() == sinsp_evt_formatter::OF_JSON)
 	{
-		std::string json_line;
+		std::string json_fields;
 
 		// Format the event into a json object with all fields resolved
-		formatter->tostring(evt, json_line);
-
-		// The formatted string might have a leading newline. If it does, remove it.
-		if(json_line[0] == '\n')
-		{
-			json_line.erase(0, 1);
-		}
+		formatter->tostring(evt, json_fields);
 
 		// For JSON output, the formatter returned a json-as-text
 		// object containing all the fields in the original format
 		// message as well as the event time in ns. Use this to build
 		// a more detailed object containing the event time, rule,
 		// severity, full output, and fields.
-		Json::Value event;
-		Json::Value rule_tags;
-		Json::FastWriter writer;
-		std::string full_line;
-		unsigned int rule_tags_idx = 0;
+		nlohmann::json event;
 
 		// Convert the time-as-nanoseconds to a more json-friendly ISO8601.
 		time_t evttime = evt->get_ts() / 1000000000;
@@ -94,43 +84,54 @@ std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule,
 
 		if(m_json_include_tags_property)
 		{
-			if (tags.size() == 0)
-			{
-				// This sets an empty array
-				rule_tags = Json::arrayValue;
-			}
-			else
-			{
-				for (const auto &tag : tags)
-				{
-					rule_tags[rule_tags_idx++] = tag;
-				}
-			}
-			event["tags"] = rule_tags;
+			event["tags"] = tags;
 		}
 
-		full_line = writer.write(event);
+		event["output_fields"] = nlohmann::json::parse(json_fields);
 
-		// Json::FastWriter may add a trailing newline. If it
-		// does, remove it.
-		if(full_line[full_line.length() - 1] == '\n')
+		for (auto const& ef : extra_fields)
 		{
-			full_line.resize(full_line.length() - 1);
+			std::string fformat = ef.second.first;
+			if (fformat.size() == 0)
+			{
+				continue;
+			}
+
+			if (!(fformat[0] == '*'))
+			{
+				fformat = "*" + fformat;
+			}
+
+			if(ef.second.second) // raw field
+			{
+				std::string json_field_map;
+				formatter = m_falco_engine->create_formatter(source, fformat);
+				formatter->tostring_withformat(evt, json_field_map, sinsp_evt_formatter::OF_JSON);
+				auto json_obj = nlohmann::json::parse(json_field_map);
+				event["output_fields"][ef.first] = json_obj[ef.first];
+			} else
+			{
+				event["output_fields"][ef.first] = format_string(evt, fformat, source);
+			}
 		}
 
-		// Cheat-graft the output from the formatter into this
-		// string. Avoids an unnecessary json parse just to
-		// merge the formatted fields at the object level.
-		full_line.pop_back();
-		full_line.append(", \"output_fields\": ");
-		full_line.append(json_line);
-		full_line.append("}");
-		line = full_line;
+		line = event.dump();
 	}
 
 	return line;
 }
 
+std::string falco_formats::format_string(sinsp_evt *evt, const std::string &format, const std::string &source) const
+{
+	std::string line;
+	std::shared_ptr<sinsp_evt_formatter> formatter;
+
+	formatter = m_falco_engine->create_formatter(source, format);
+	formatter->tostring_withformat(evt, line, sinsp_evt_formatter::OF_NORMAL);
+
+	return line;
+}
+
 std::map<std::string, std::string> falco_formats::get_field_values(sinsp_evt *evt, const std::string &source,
 						    const std::string &format) const
 {

userspace/engine/formats.h

@@ -31,7 +31,9 @@ public:
 
 	std::string format_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
 				 const std::string &level, const std::string &format, const std::set<std::string> &tags,
-				 const std::string &hostname) const;
+				 const std::string &hostname, const extra_output_field_t &extra_fields) const;
+
+	std::string format_string(sinsp_evt *evt, const std::string &format, const std::string &source) const;
 
 	std::map<std::string, std::string> get_field_values(sinsp_evt *evt, const std::string &source,
 					     const std::string &format) const ;

userspace/engine/indexable_ruleset.h

@@ -0,0 +1,454 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* This describes the interface for an "indexable" ruleset, that is, a
+ * ruleset that can enable/disable abstract filters for various
+ * ruleset ids.
+ *
+ * It's used by evttype_index_ruleset as well as other rulesets that
+ * need the same functionality but don't want to copy the same code.
+ */
+
+#pragma once
+
+#include "falco_utils.h"
+#include "filter_ruleset.h"
+
+#include <libsinsp/sinsp.h>
+#include <libsinsp/filter.h>
+#include <libsinsp/event.h>
+
+#include <functional>
+#include <memory>
+#include <string>
+
+// A filter_wrapper should implement these methods:
+//   const std::string &filter_wrapper::name();
+//   const std::set<std::string> &filter_wrapper::tags();
+//   const libsinsp::events::set<ppm_sc_code> &filter_wrapper::sc_codes();
+//   const libsinsp::events::set<ppm_event_code> &filter_wrapper::event_codes();
+
+template<class filter_wrapper>
+class indexable_ruleset : public filter_ruleset
+{
+public:
+	indexable_ruleset() = default;
+	virtual ~indexable_ruleset() = default;
+
+	// Required to implement filter_ruleset
+	void clear() override
+	{
+		for(size_t i = 0; i < m_rulesets.size(); i++)
+		{
+			m_rulesets[i] = std::make_shared<ruleset_filters>(i);
+		}
+		m_filters.clear();
+	}
+
+	uint64_t enabled_count(uint16_t ruleset_id) override
+	{
+		while(m_rulesets.size() < (size_t)ruleset_id + 1)
+		{
+			m_rulesets.emplace_back(std::make_shared<ruleset_filters>(m_rulesets.size()));
+		}
+
+		return m_rulesets[ruleset_id]->num_filters();
+	}
+
+	void enabled_evttypes(
+		std::set<uint16_t> &evttypes,
+		uint16_t ruleset_id) override
+	{
+		evttypes.clear();
+		for(const auto &e : enabled_event_codes(ruleset_id))
+		{
+			evttypes.insert((uint16_t)e);
+		}
+	}
+
+	libsinsp::events::set<ppm_sc_code> enabled_sc_codes(
+		uint16_t ruleset_id) override
+	{
+		if(m_rulesets.size() < (size_t)ruleset_id + 1)
+		{
+			return {};
+		}
+		return m_rulesets[ruleset_id]->sc_codes();
+	}
+
+	libsinsp::events::set<ppm_event_code> enabled_event_codes(
+		uint16_t ruleset_id) override
+	{
+		if(m_rulesets.size() < (size_t)ruleset_id + 1)
+		{
+			return {};
+		}
+		return m_rulesets[ruleset_id]->event_codes();
+	}
+
+	void enable(
+		const std::string &pattern,
+		match_type match,
+		uint16_t ruleset_id) override
+	{
+		enable_disable(pattern, match, true, ruleset_id);
+	}
+
+	void disable(
+		const std::string &pattern,
+		match_type match,
+		uint16_t ruleset_id) override
+	{
+		enable_disable(pattern, match, false, ruleset_id);
+	}
+
+	void enable_tags(
+		const std::set<std::string> &tags,
+		uint16_t ruleset_id) override
+	{
+		enable_disable_tags(tags, true, ruleset_id);
+	}
+
+	void disable_tags(
+		const std::set<std::string> &tags,
+		uint16_t ruleset_id) override
+	{
+		enable_disable_tags(tags, false, ruleset_id);
+	}
+
+	// Note that subclasses do *not* implement run. Instead, they
+	// implement run_wrappers.
+	bool run(sinsp_evt *evt, falco_rule &match, uint16_t ruleset_id) override
+	{
+		if(m_rulesets.size() < (size_t)ruleset_id + 1)
+		{
+			return false;
+		}
+
+		return m_rulesets[ruleset_id]->run(*this, evt, match);
+	}
+
+	bool run(sinsp_evt *evt, std::vector<falco_rule> &matches, uint16_t ruleset_id) override
+	{
+		if(m_rulesets.size() < (size_t)ruleset_id + 1)
+		{
+			return false;
+		}
+
+		return m_rulesets[ruleset_id]->run(*this, evt, matches);
+	}
+
+	typedef std::list<std::shared_ptr<filter_wrapper>>
+		filter_wrapper_list;
+
+	// Subclasses should call add_wrapper (most likely from
+	// filter_ruleset::add or ::add_compile_output) to add filters.
+	void add_wrapper(std::shared_ptr<filter_wrapper> wrap)
+	{
+		m_filters.insert(wrap);
+	}
+
+	// If a subclass needs to iterate over all filters, they can
+	// call iterate with this function, which will be called for
+	// all filters.
+	typedef std::function<void(const std::shared_ptr<filter_wrapper> &wrap)> filter_wrapper_func;
+	uint64_t iterate(filter_wrapper_func func)
+	{
+		uint64_t num_filters = 0;
+
+		for(const auto &ruleset_ptr : m_rulesets)
+		{
+			if(ruleset_ptr)
+			{
+				for(const auto &wrap : ruleset_ptr->get_filters())
+				{
+					num_filters++;
+					func(wrap);
+				}
+			}
+		}
+
+		return num_filters;
+	}
+
+	// A subclass must implement these methods. They are analogous
+	// to run() but take care of selecting filters that match a
+	// ruleset and possibly an event type.
+	virtual bool run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, std::vector<falco_rule> &matches) = 0;
+	virtual bool run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, falco_rule &match) = 0;
+
+private:
+	// Helper used by enable()/disable()
+	void enable_disable(
+		const std::string &pattern,
+		match_type match,
+		bool enabled,
+		uint16_t ruleset_id)
+	{
+		while(m_rulesets.size() < (size_t)ruleset_id + 1)
+		{
+			m_rulesets.emplace_back(std::make_shared<ruleset_filters>(m_rulesets.size()));
+		}
+
+		for(const auto &wrap : m_filters)
+		{
+			bool matches;
+			std::string::size_type pos;
+
+			switch(match)
+			{
+			case match_type::exact:
+				pos = wrap->name().find(pattern);
+
+				matches = (pattern == "" || (pos == 0 &&
+							     pattern.size() == wrap->name().size()));
+				break;
+			case match_type::substring:
+				matches = (pattern == "" || (wrap->name().find(pattern) != std::string::npos));
+				break;
+			case match_type::wildcard:
+				matches = falco::utils::matches_wildcard(pattern, wrap->name());
+				break;
+			default:
+				// should never happen
+				matches = false;
+			}
+
+			if(matches)
+			{
+				if(enabled)
+				{
+					m_rulesets[ruleset_id]->add_filter(wrap);
+				}
+				else
+				{
+					m_rulesets[ruleset_id]->remove_filter(wrap);
+				}
+			}
+		}
+	}
+
+	// Helper used by enable_tags()/disable_tags()
+	void enable_disable_tags(
+		const std::set<std::string> &tags,
+		bool enabled,
+		uint16_t ruleset_id)
+	{
+		while(m_rulesets.size() < (size_t)ruleset_id + 1)
+		{
+			m_rulesets.emplace_back(std::make_shared<ruleset_filters>(m_rulesets.size()));
+		}
+
+		for(const auto &wrap : m_filters)
+		{
+			std::set<std::string> intersect;
+
+			set_intersection(tags.begin(), tags.end(),
+					 wrap->tags().begin(), wrap->tags().end(),
+					 inserter(intersect, intersect.begin()));
+
+			if(!intersect.empty())
+			{
+				if(enabled)
+				{
+					m_rulesets[ruleset_id]->add_filter(wrap);
+				}
+				else
+				{
+					m_rulesets[ruleset_id]->remove_filter(wrap);
+				}
+			}
+		}
+	}
+
+	// A group of filters all having the same ruleset
+	class ruleset_filters
+	{
+	public:
+		ruleset_filters(uint16_t ruleset_id):
+			m_ruleset_id(ruleset_id) {}
+
+		virtual ~ruleset_filters(){};
+
+		void add_filter(std::shared_ptr<filter_wrapper> wrap)
+		{
+			if(wrap->event_codes().empty())
+			{
+				// Should run for all event types
+				add_wrapper_to_list(m_filter_all_event_types, wrap);
+			}
+			else
+			{
+				for(auto &etype : wrap->event_codes())
+				{
+					if(m_filter_by_event_type.size() <= etype)
+					{
+						m_filter_by_event_type.resize(etype + 1);
+					}
+
+					add_wrapper_to_list(m_filter_by_event_type[etype], wrap);
+				}
+			}
+
+			m_filters.insert(wrap);
+		}
+
+		void remove_filter(std::shared_ptr<filter_wrapper> wrap)
+		{
+			if(wrap->event_codes().empty())
+			{
+				remove_wrapper_from_list(m_filter_all_event_types, wrap);
+			}
+			else
+			{
+				for(auto &etype : wrap->event_codes())
+				{
+					if(etype < m_filter_by_event_type.size())
+					{
+						remove_wrapper_from_list(m_filter_by_event_type[etype], wrap);
+					}
+				}
+			}
+
+			m_filters.erase(wrap);
+		}
+
+		uint64_t num_filters()
+		{
+			return m_filters.size();
+		}
+
+		inline const std::set<std::shared_ptr<filter_wrapper>> &get_filters() const
+		{
+			return m_filters;
+		}
+
+		// Evaluate an event against the ruleset and return the first rule
+		// that matched.
+		bool run(indexable_ruleset &ruleset, sinsp_evt *evt, falco_rule &match)
+		{
+			if(evt->get_type() < m_filter_by_event_type.size() &&
+			   m_filter_by_event_type[evt->get_type()].size() > 0)
+			{
+				if(ruleset.run_wrappers(evt, m_filter_by_event_type[evt->get_type()], m_ruleset_id, match))
+				{
+					return true;
+				}
+			}
+
+			// Finally, try filters that are not specific to an event type.
+			if(m_filter_all_event_types.size() > 0)
+			{
+				if(ruleset.run_wrappers(evt, m_filter_all_event_types, m_ruleset_id, match))
+				{
+					return true;
+				}
+			}
+
+			return false;
+		}
+
+		//  Evaluate an event against the ruleset and return all the
+		//  matching rules.
+		bool run(indexable_ruleset &ruleset, sinsp_evt *evt, std::vector<falco_rule> &matches)
+		{
+			if(evt->get_type() < m_filter_by_event_type.size() &&
+			   m_filter_by_event_type[evt->get_type()].size() > 0)
+			{
+				if(ruleset.run_wrappers(evt, m_filter_by_event_type[evt->get_type()], m_ruleset_id, matches))
+				{
+					return true;
+				}
+			}
+
+			// Finally, try filters that are not specific to an event type.
+			if(m_filter_all_event_types.size() > 0)
+			{
+				if(ruleset.run_wrappers(evt, m_filter_all_event_types, m_ruleset_id, matches))
+				{
+					return true;
+				}
+			}
+
+			return false;
+		}
+
+		libsinsp::events::set<ppm_sc_code> sc_codes()
+		{
+			libsinsp::events::set<ppm_sc_code> res;
+			for(const auto &wrap : m_filters)
+			{
+				res.insert(wrap->sc_codes().begin(), wrap->sc_codes().end());
+			}
+			return res;
+		}
+		libsinsp::events::set<ppm_event_code> event_codes()
+		{
+			libsinsp::events::set<ppm_event_code> res;
+			for(const auto &wrap : m_filters)
+			{
+				res.insert(wrap->event_codes().begin(), wrap->event_codes().end());
+			}
+			return res;
+		}
+
+	private:
+		void add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
+		{
+			// This is O(n) but it's also uncommon
+			// (when loading rules only).
+			auto pos = std::find(wrappers.begin(),
+					     wrappers.end(),
+					     wrap);
+
+			if(pos == wrappers.end())
+			{
+				wrappers.push_back(wrap);
+			}
+		}
+
+		void remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
+		{
+			// This is O(n) but it's also uncommon
+			// (when loading rules only).
+			auto pos = std::find(wrappers.begin(),
+					     wrappers.end(),
+					     wrap);
+			if(pos != wrappers.end())
+			{
+				wrappers.erase(pos);
+			}
+		}
+		uint16_t m_ruleset_id;
+
+		// Vector indexes from event type to a set of filters. There can
+		// be multiple filters for a given event type.
+		// NOTE: This is used only when the event sub-type is 0.
+		std::vector<filter_wrapper_list> m_filter_by_event_type;
+
+		filter_wrapper_list m_filter_all_event_types;
+
+		// All filters added. Used to make num_filters() fast.
+		std::set<std::shared_ptr<filter_wrapper>> m_filters;
+	};
+
+	// Vector indexes from ruleset id to set of rules.
+	std::vector<std::shared_ptr<ruleset_filters>> m_rulesets;
+
+	// All filters added. The set of enabled filters is held in m_rulesets
+	std::set<std::shared_ptr<filter_wrapper>> m_filters;
+};

userspace/engine/rule_loader.h

@@ -20,6 +20,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 #include <optional>
+#include <unordered_map>
 #include <yaml-cpp/yaml.h>
 #include <nlohmann/json.hpp>
 #include "falco_source.h"
@@ -261,6 +262,25 @@ namespace rule_loader
 		nlohmann::json res_json;
 	};
 
+	struct extra_output_format_conf
+	{
+		std::string m_format;
+		std::string m_source;
+		std::string m_tag;
+		std::string m_rule;
+		bool m_replace_container_info;
+	};
+
+	struct extra_output_field_conf
+	{
+		std::string m_key;
+		std::string m_format;
+		std::string m_source;
+		std::string m_tag;
+		std::string m_rule;
+		bool m_raw;
+	};
+
 	/*!
 		\brief Contains the info required to load rule definitions
 	*/
@@ -278,8 +298,9 @@ namespace rule_loader
 		const std::string& content;
 		const indexed_vector<falco_source>& sources;
 		std::string name;
-		std::string output_extra;
-		bool replace_output_container_info = false;
+
+		std::vector<extra_output_format_conf> extra_output_format;
+		std::vector<extra_output_field_conf> extra_output_fields;
 
 		// outputs
 		std::unique_ptr<result> res;

userspace/engine/rule_loader_compiler.cpp

@@ -322,22 +322,6 @@ static std::shared_ptr<ast::expr> parse_condition(
 	}
 }
 
-static void apply_output_substitutions(
-	rule_loader::configuration& cfg,
-	std::string& out)
-{
-	if (out.find(s_container_info_fmt) != std::string::npos)
-	{
-		if (cfg.replace_output_container_info)
-		{
-			out = replace(out, s_container_info_fmt, cfg.output_extra);
-			return;
-		}
-		out = replace(out, s_container_info_fmt, s_default_extra_fmt);
-	}
-	out += cfg.output_extra.empty() ? "" : " " + cfg.output_extra;
-}
-
 void rule_loader::compiler::compile_list_infos(
 		configuration& cfg,
 		const collector& col,
@@ -509,7 +493,66 @@ void rule_loader::compiler::compile_rule_infos(
 
 		// build rule output message
 		rule.output = r.output;
-		apply_output_substitutions(cfg, rule.output);
+
+		for (auto& extra : cfg.extra_output_format)
+		{
+			if (extra.m_source != "" && r.source != extra.m_source)
+			{
+				continue;
+			}
+
+			if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0)
+			{
+				continue;
+			}
+
+			if (extra.m_rule != "" && r.name != extra.m_rule)
+			{
+				continue;
+			}
+
+			if (extra.m_replace_container_info)
+			{
+				if (rule.output.find(s_container_info_fmt) != std::string::npos)
+				{
+					rule.output = replace(rule.output, s_container_info_fmt, extra.m_format);
+				}
+				else
+				{
+					rule.output = rule.output + " " + extra.m_format;
+				}
+			} else
+			{
+				rule.output = rule.output + " " + extra.m_format;
+			}
+		}
+
+		if (rule.output.find(s_container_info_fmt) != std::string::npos)
+		{
+			rule.output = replace(rule.output, s_container_info_fmt, s_default_extra_fmt);
+		}
+
+		// build extra output fields if required
+
+		for (auto const& extra : cfg.extra_output_fields)
+		{
+			if (extra.m_source != "" && r.source != extra.m_source)
+			{
+				continue;
+			}
+
+			if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0)
+			{
+				continue;
+			}
+
+			if (extra.m_rule != "" && r.name != extra.m_rule)
+			{
+				continue;
+			}
+
+			rule.extra_output_fields[extra.m_key] = {extra.m_format, extra.m_raw};
+		}
 
 		// validate the rule's output
 		if(!is_format_valid(*cfg.sources.at(r.source), rule.output, err))
@@ -530,6 +573,18 @@ void rule_loader::compiler::compile_rule_infos(
 				r.output_ctx);
 		}
 
+		// validate the rule's extra fields if any
+		for (auto const& ef : rule.extra_output_fields)
+		{
+			if(!is_format_valid(*cfg.sources.at(r.source), ef.second.first, err))
+			{
+				throw rule_load_exception(
+					falco::load_result::load_result::LOAD_ERR_COMPILE_OUTPUT,
+					err,
+					r.output_ctx);
+			}
+		}
+
 		if (!compile_condition(cfg,
 				  macro_resolver,
 				  lists,

userspace/falco/CMakeLists.txt

@@ -49,8 +49,8 @@ add_library(falco_application STATIC
   app/actions/validate_rules_files.cpp
   app/actions/create_requested_paths.cpp
   app/actions/close_inspectors.cpp
+  app/actions/print_config_schema.cpp
   configuration.cpp
-  logger.cpp
   falco_outputs.cpp
   outputs_file.cpp
   outputs_stdout.cpp

userspace/falco/app/actions/actions.h

@@ -38,6 +38,7 @@ falco::app::run_result list_plugins(const falco::app::state& s);
 falco::app::run_result load_config(const falco::app::state& s);
 falco::app::run_result load_plugins(falco::app::state& s);
 falco::app::run_result load_rules_files(falco::app::state& s);
+falco::app::run_result print_config_schema(falco::app::state& s);
 falco::app::run_result print_generated_gvisor_config(falco::app::state& s);
 falco::app::run_result print_help(falco::app::state& s);
 falco::app::run_result print_ignored_events(const falco::app::state& s);

userspace/falco/app/actions/helpers_inspector.cpp

@@ -48,11 +48,6 @@ falco::app::run_result falco::app::actions::open_live_inspector(
 {
 	try
 	{
-		if((s.config->m_metrics_flags & METRICS_V2_STATE_COUNTERS))
-		{
-			inspector->set_sinsp_stats_v2_enabled();
-		}
-
 		if(s.config->m_falco_libs_thread_table_size > 0)
 		{
 			// Default value is set in libs as part of the sinsp_thread_manager setup
@@ -70,7 +65,7 @@ falco::app::run_result falco::app::actions::open_live_inspector(
 				{
 					auto cfg = s.plugin_configs.at(p->name());
 					falco_logger::log(falco_logger::level::INFO, "Opening '" + source + "' source with plugin '" + cfg->m_name + "'");
-					inspector->open_plugin(cfg->m_name, cfg->m_open_params);
+					inspector->open_plugin(cfg->m_name, cfg->m_open_params, sinsp_plugin_platform::SINSP_PLATFORM_HOSTINFO);
 					return run_result::ok();
 				}
 			}
@@ -88,7 +83,7 @@ falco::app::run_result falco::app::actions::open_live_inspector(
 				{
 					auto cfg = s.plugin_configs.at(p->name());
 					falco_logger::log(falco_logger::level::INFO, "Opening '" + source + "' source with plugin '" + cfg->m_name + "'");
-					inspector->open_plugin(cfg->m_name, cfg->m_open_params);
+					inspector->open_plugin(cfg->m_name, cfg->m_open_params, sinsp_plugin_platform::SINSP_PLATFORM_FULL);
 					return run_result::ok();
 				}
 			}

userspace/falco/app/actions/init_falco_engine.cpp

@@ -17,49 +17,55 @@ limitations under the License.
 
 #include "actions.h"
 #include <libsinsp/plugin_manager.h>
+#include <falco_common.h>
 
 using namespace falco::app;
 using namespace falco::app::actions;
 
 void configure_output_format(falco::app::state& s)
 {
+	for (auto& eo : s.config->m_append_output)
+	{
+		if (eo.m_format != "")
+		{
+			s.engine->add_extra_output_format(eo.m_format, eo.m_source, eo.m_tag, eo.m_rule, false);
+		}
+
+		for (auto const& ff : eo.m_formatted_fields)
+		{
+			s.engine->add_extra_output_formatted_field(ff.first, ff.second, eo.m_source, eo.m_tag, eo.m_rule);
+		}
+
+		for (auto const& rf : eo.m_raw_fields)
+		{
+			s.engine->add_extra_output_raw_field(rf, eo.m_source, eo.m_tag, eo.m_rule);
+		}
+	}
+
 	// See https://falco.org/docs/rules/style-guide/
 	const std::string container_info = "container_id=%container.id container_image=%container.image.repository container_image_tag=%container.image.tag container_name=%container.name";
 	const std::string k8s_info = "k8s_ns=%k8s.ns.name k8s_pod_name=%k8s.pod.name";
 	const std::string gvisor_info = "vpid=%proc.vpid vtid=%thread.vtid";
 
-	std::string output_format;
-	bool replace_container_info = false;
-
 	if(s.options.print_additional == "c" || s.options.print_additional == "container")
 	{
-		output_format = container_info;
-		replace_container_info = true;
+		s.engine->add_extra_output_format(container_info, falco_common::syscall_source, "", "", true);
 	}
 	else if(s.options.print_additional == "cg" || s.options.print_additional == "container-gvisor")
 	{
-		output_format = gvisor_info + " " + container_info;
-		replace_container_info = true;
+		s.engine->add_extra_output_format(gvisor_info + " " + container_info, falco_common::syscall_source, "", "", true);
 	}
 	else if(s.options.print_additional == "k" || s.options.print_additional == "kubernetes")
 	{
-		output_format = container_info + " " + k8s_info;
-		replace_container_info = true;
+		s.engine->add_extra_output_format(container_info + " " + k8s_info, falco_common::syscall_source, "", "", true);
 	}
 	else if(s.options.print_additional == "kg" || s.options.print_additional == "kubernetes-gvisor")
 	{
-		output_format = gvisor_info + " " + container_info + " " + k8s_info;
-		replace_container_info = true;
+		s.engine->add_extra_output_format(gvisor_info + " " + container_info + " " + k8s_info, falco_common::syscall_source, "", "", true);
 	}
 	else if(!s.options.print_additional.empty())
 	{
-		output_format = s.options.print_additional;
-		replace_container_info = false;
-	}
-
-	if(!output_format.empty())
-	{
-		s.engine->set_extra(output_format, replace_container_info);
+		s.engine->add_extra_output_format(s.options.print_additional, "", "", "", false);
 	}
 }
 

userspace/falco/app/actions/init_inspectors.cpp

@@ -29,18 +29,72 @@ static void init_syscall_inspector(falco::app::state& s, std::shared_ptr<sinsp>
 {
 	inspector->set_buffer_format(s.options.event_buffer_format);
 
+	//
+	// Container engines
+	//
+
+	// Debug log messages
+	if(s.config->m_container_engines_mask & (1 << CT_DOCKER))
+	{
+		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'docker'");
+	}
+
+	if(s.config->m_container_engines_mask & (1 << CT_PODMAN))
+	{
+		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'podman'");
+	}
+
+	if(s.config->m_container_engines_mask & ((1 << CT_CRI) | (1 << CT_CRIO) | (1 << CT_CONTAINERD)))
+	{
+		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'CRI'");
+	}
+
+	if(s.config->m_container_engines_mask & (1 << CT_LXC))
+	{
+		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'lxc'");
+	}
+
+	if(s.config->m_container_engines_mask & (1 << CT_LIBVIRT_LXC))
+	{
+		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'libvirt_lxc'");
+	}
+
+	if(s.config->m_container_engines_mask & (1 << CT_BPM))
+	{
+		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'bpm'");
+	}
+
+	// Container engines configs via falco.yaml
+	inspector->set_container_engine_mask(s.config->m_container_engines_mask);
+	for (auto &p : s.config->m_container_engines_cri_socket_paths)
+	{
+		if (!p.empty())
+		{
+			inspector->add_cri_socket_path(p);
+			falco_logger::log(falco_logger::level::DEBUG, "Enabled container runtime socket at '" + p + "' via config file");
+		}
+	}
+	inspector->set_cri_async(!s.config->m_container_engines_disable_cri_async);
+
+	// Container engines configs via CLI args
 	// If required, set the CRI paths
 	for (auto &p : s.options.cri_socket_paths)
 	{
 		if (!p.empty())
 		{
 			inspector->add_cri_socket_path(p);
+			falco_logger::log(falco_logger::level::DEBUG, "Enabled container runtime socket at '" + p + "' via CLI args");
 		}
 	}
 
 	// Decide whether to do sync or async for CRI metadata fetch
 	inspector->set_cri_async(!s.options.disable_cri_async);
 
+	if(s.options.disable_cri_async || s.config->m_container_engines_disable_cri_async)
+	{
+		falco_logger::log(falco_logger::level::DEBUG, "Disabling async lookups for 'CRI'");
+	}
+
 	//
 	// If required, set the snaplen
 	//
@@ -109,15 +163,25 @@ falco::app::run_result falco::app::actions::init_inspectors(falco::app::state& s
 	std::unordered_set<std::string> used_plugins;
 	const auto& all_plugins = s.offline_inspector->get_plugin_manager()->plugins();
 
+	if((s.config->m_metrics_flags & METRICS_V2_STATE_COUNTERS))
+	{
+
+	}
+
 	for (const auto &src : s.loaded_sources)
 	{
 		auto src_info = s.source_infos.at(src);
 
 		// in capture mode, every event source uses the offline inspector.
 		// in live mode, we create a new inspector for each event source
-		src_info->inspector = s.is_capture_mode()
-			? s.offline_inspector
-			: std::make_shared<sinsp>();
+		if (s.is_capture_mode())
+		{
+			src_info->inspector = s.offline_inspector;
+		}
+		else
+		{
+			src_info->inspector = std::make_shared<sinsp>(s.config->m_metrics_flags & METRICS_V2_STATE_COUNTERS);
+		}
 
 		// do extra preparation for the syscall source
 		if (src == falco_common::syscall_source)

userspace/falco/app/actions/load_config.cpp

@@ -31,19 +31,19 @@ falco::app::run_result falco::app::actions::load_config(const falco::app::state&
 {
 	// List of loaded conf files, ie: s.options.conf_filename
 	// plus all the `config_files` expanded list of configs.
-	std::vector<std::string> loaded_conf_files;
+	config_loaded_res res;
 	try
 	{
 		if (!s.options.conf_filename.empty())
 		{
-			s.config->init_from_file(s.options.conf_filename, loaded_conf_files, s.options.cmdline_config_options);
+			res = s.config->init_from_file(s.options.conf_filename, s.options.cmdline_config_options);
 		}
 		else
 		{
 			// Is possible to have an empty config file when we want to use some command line
 			// options like `--help`, `--version`, ...
 			// The configs used in `load_yaml` will be initialized to the default values.
-			s.config->init_from_content("", s.options.cmdline_config_options);
+			res = s.config->init_from_content("", s.options.cmdline_config_options);
 		}
 	}
 	catch (std::exception& e)
@@ -61,9 +61,12 @@ falco::app::run_result falco::app::actions::load_config(const falco::app::state&
 	if (!s.options.conf_filename.empty())
 	{
 		falco_logger::log(falco_logger::level::INFO, "Falco initialized with configuration files:\n");
-		for (const auto& path : loaded_conf_files)
+		for (const auto& pair : res)
 		{
-			falco_logger::log(falco_logger::level::INFO, std::string("   ") + path + "\n");
+			auto config_path = pair.first;
+			auto validation = pair.second;
+			auto priority = validation == yaml_helper::validation_ok ? falco_logger::level::INFO : falco_logger::level::WARNING;
+			falco_logger::log(priority, std::string("   ") + config_path + " | validation: " + validation + "\n");
 		}
 	}
 
@@ -85,4 +88,4 @@ falco::app::run_result falco::app::actions::require_config_file(const falco::app
 	}
 #endif // __EMSCRIPTEN__
 	return run_result::ok();
-}
+}

userspace/falco/app/actions/load_rules_files.cpp

@@ -131,39 +131,6 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 		return run_result::fatal(err);
 	}
 
-	if((!s.options.disabled_rule_substrings.empty() || !s.options.disabled_rule_tags.empty() || !s.options.enabled_rule_tags.empty()) &&
-		!s.config->m_rules_selection.empty())
-	{
-		return run_result::fatal("Specifying -D, -t, -T command line options together with \"rules:\" configuration or -o \"rules...\" is not supported.");
-	}
-
-	for (const auto& substring : s.options.disabled_rule_substrings)
-	{
-		falco_logger::log(falco_logger::level::INFO, "Disabling rules matching substring: " + substring + "\n");
-		s.engine->enable_rule(substring, false);
-	}
-
-	if(!s.options.disabled_rule_tags.empty())
-	{
-		for(const auto &tag : s.options.disabled_rule_tags)
-		{
-			falco_logger::log(falco_logger::level::INFO, "Disabling rules with tag: " + tag + "\n");
-		}
-		s.engine->enable_rule_by_tag(s.options.disabled_rule_tags, false);
-	}
-
-	if(!s.options.enabled_rule_tags.empty())
-	{
-		// Since we only want to enable specific
-		// rules, first disable all rules.
-		s.engine->enable_rule(all_rules, false);
-		for(const auto &tag : s.options.enabled_rule_tags)
-		{
-			falco_logger::log(falco_logger::level::INFO, "Enabling rules with tag: " + tag + "\n");
-		}
-		s.engine->enable_rule_by_tag(s.options.enabled_rule_tags, true);
-	}
-
 	for(const auto& sel : s.config->m_rules_selection)
 	{
 		bool enable = sel.m_op == falco_configuration::rule_selection_operation::enable;

userspace/falco/app/actions/print_config_schema.cpp

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include "actions.h"
+
+using namespace falco::app;
+using namespace falco::app::actions;
+
+falco::app::run_result falco::app::actions::print_config_schema(falco::app::state &s)
+{
+	if(s.options.print_config_schema)
+	{
+		printf("%s", s.config->m_config_schema.dump(2).c_str());
+		return run_result::exit();
+	}
+	return run_result::ok();
+}

userspace/falco/app/actions/process_events.cpp

@@ -312,7 +312,9 @@ static falco::app::run_result do_inspect(
 		{
 			for(auto& rule_res : *res)
 			{
-				s.outputs->handle_event(rule_res.evt, rule_res.rule, rule_res.source, rule_res.priority_num, rule_res.format, rule_res.tags);
+				s.outputs->handle_event(
+					rule_res.evt, rule_res.rule, rule_res.source, rule_res.priority_num,
+					rule_res.format, rule_res.tags, rule_res.extra_output_fields);
 			}
 		}
 
@@ -413,9 +415,9 @@ static falco::app::run_result init_stats_writer(
 		return falco::app::run_result::fatal("Metrics interval was passed as numeric value without Prometheus time unit. Please specify a time unit");
 	}
 
-	if (config->m_metrics_enabled && !sw->has_output())
+	if (config->m_metrics_enabled && !(sw->has_output() || config->m_webserver_config.m_prometheus_metrics_enabled))
 	{
-		return falco::app::run_result::fatal("Metrics are enabled with no output configured. Please enable at least one output channel");
+		return falco::app::run_result::fatal("Metrics are enabled with no output configured. Please enable at least one output channel ('metrics.output_rule', 'metrics.output_file' or 'webserver.prometheus_metrics_enabled')");
 	}
 
 	falco_logger::log(falco_logger::level::INFO, "Setting metrics interval to " + config->m_metrics_interval_str + ", equivalent to " + std::to_string(config->m_metrics_interval) + " (ms)\n");

userspace/falco/app/app.cpp

@@ -60,6 +60,7 @@ bool falco::app::run(falco::app::state& s, bool& restart, std::string& errstr)
 	// dependencies are honored (e.g. don't process events before
 	// loading plugins, opening inspector, etc.).
 	std::list<app_action> run_steps = {
+		falco::app::actions::print_config_schema,
 		falco::app::actions::load_config,
 		falco::app::actions::print_help,
 		falco::app::actions::print_kernel_version,

userspace/falco/app/options.cpp

@@ -95,40 +95,6 @@ bool options::parse(int argc, char **argv, std::string &errstr)
 		}
 	}
 
-	// Convert the vectors of enabled/disabled tags into sets to match falco engine API
-	if(m_cmdline_parsed.count("T") > 0)
-	{
-		falco_logger::log(falco_logger::level::WARNING, "The -T option is deprecated and will be removed in Falco 0.39.0. Use -o rules[].disable.tag=<tag> instead.");
-		for(auto &tag : m_cmdline_parsed["T"].as<std::vector<std::string>>())
-		{
-			disabled_rule_tags.insert(tag);
-		}
-	}
-
-	if(m_cmdline_parsed.count("t") > 0)
-	{
-		falco_logger::log(falco_logger::level::WARNING, "The -t option is deprecated and will be removed in Falco 0.39.0. Use -o rules[].disable.rule=* -o rules[].enable.tag=<tag> instead.");
-		for(auto &tag : m_cmdline_parsed["t"].as<std::vector<std::string>>())
-		{
-			enabled_rule_tags.insert(tag);
-		}
-	}
-
-	if(disabled_rule_substrings.size() > 0)
-	{
-		falco_logger::log(falco_logger::level::WARNING, "The -D option is deprecated and will be removed in Falco 0.39.0. Use -o rules[].disable.rule=<wildcard-pattern> instead.");
-	}
-
-	// Some combinations of arguments are not allowed.
-
-	// You can't both disable and enable rules
-	if((disabled_rule_substrings.size() + disabled_rule_tags.size() > 0) &&
-	   !enabled_rule_tags.empty())
-	{
-		errstr = std::string("You can not specify both disabled (-D/-T) and enabled (-t) rules");
-		return false;
-	}
-
 	list_fields = m_cmdline_parsed.count("list") > 0;
 
 	return true;
@@ -144,10 +110,11 @@ void options::define(cxxopts::Options& opts)
 	opts.add_options()
 		("h,help",                        "Print this help list and exit.", cxxopts::value(help)->default_value("false"))
 #ifdef BUILD_TYPE_RELEASE
-		("c",                             "Configuration file. If not specified uses " FALCO_INSTALL_CONF_FILE ".", cxxopts::value(conf_filename), "<path>")
+		("c",                             	"Configuration file. If not specified uses " FALCO_INSTALL_CONF_FILE ".", cxxopts::value(conf_filename), "<path>")
 #else
 		("c",                             "Configuration file. If not specified tries " FALCO_SOURCE_CONF_FILE ", " FALCO_INSTALL_CONF_FILE ".", cxxopts::value(conf_filename), "<path>")
 #endif
+		("config-schema",                 "Print the config json schema and exit.", cxxopts::value(print_config_schema)->default_value("false"))
 		("A",                             "Monitor all events supported by Falco and defined in rules and configs. Some events are ignored by default when -A is not specified (the -i option lists these events ignored). Using -A can impact performance. This option has no effect when reproducing events from a capture file.", cxxopts::value(all_events)->default_value("false"))
 		("b,print-base64",                "Print data buffers in base64. This is useful for encoding binary data that needs to be used over media designed to consume this format.")
 #if !defined(_WIN32) && !defined(__EMSCRIPTEN__) && !defined(MINIMAL_BUILD)
@@ -156,7 +123,6 @@ void options::define(cxxopts::Options& opts)
 #endif
 		("disable-source",                "Turn off a specific <event_source>. By default, all loaded sources get enabled. Available sources are 'syscall' plus all sources defined by loaded plugins supporting the event sourcing capability. This option can be passed multiple times, but turning off all event sources simultaneously is not permitted. This option can not be mixed with --enable-source. This option has no effect when reproducing events from a capture file.", cxxopts::value(disable_sources), "<event_source>")
 		("dry-run",                       "Run Falco without processing events. It can help check that the configuration and rules do not have any errors.", cxxopts::value(dry_run)->default_value("false"))
-		("D",                             "DEPRECATED: use -o rules[].disable.rule=<wildcard-pattern> instead. Turn off any rules with names having the substring <substring>. This option can be passed multiple times. It cannot be mixed with -t.", cxxopts::value(disabled_rule_substrings), "<substring>")
 		("enable-source",                 "Enable a specific <event_source>. By default, all loaded sources get enabled. Available sources are 'syscall' plus all sources defined by loaded plugins supporting the event sourcing capability. This option can be passed multiple times. When using this option, only the event sources specified by it will be enabled. This option can not be mixed with --disable-source. This option has no effect when reproducing events from a capture file.", cxxopts::value(enable_sources), "<event_source>")
 #ifdef HAS_GVISOR
 		("gvisor-generate-config",		  "Generate a configuration file that can be used for gVisor and exit. See --gvisor-config for more details.", cxxopts::value<std::string>(gvisor_generate_config_with_socket)->implicit_value("/run/falco/gvisor.sock"), "<socket_path>")
@@ -172,13 +138,11 @@ void options::define(cxxopts::Options& opts)
 		("N",                             "Only print field names when used in conjunction with the --list option. It has no effect when used with other options.", cxxopts::value(names_only)->default_value("false"))
 		("o,option",                      "Set the value of option <opt> to <val>. Overrides values in the configuration file. <opt> can be identified using its location in the configuration file using dot notation. Elements of list entries can be accessed via square brackets [].\n    E.g. base.id = val\n         base.subvalue.subvalue2 = val\n         base.list[1]=val", cxxopts::value(cmdline_config_options), "<opt>=<val>")
 		("plugin-info",                   "Print info for the plugin specified by <plugin_name> and exit.\nThis includes all descriptive information like name and author, along with the\nschema format for the init configuration and a list of suggested open parameters.\n<plugin_name> can be the plugin's name or its configured 'library_path'.", cxxopts::value(print_plugin_info), "<plugin_name>")
-		("p,print",                       "Print (or replace) additional information in the rule's output.\nUse -pc or -pcontainer to append container details.\nUse -pk or -pkubernetes to add both container and Kubernetes details.\nIf using gVisor, choose -pcg or -pkg variants (or -pcontainer-gvisor and -pkubernetes-gvisor, respectively).\nIf a rule's output contains %container.info, it will be replaced with the corresponding details. Otherwise, these details will be directly appended to the rule's output.\nAlternatively, use -p <output_format> for a custom format. In this case, the given <output_format> will be appended to the rule's output without any replacement.", cxxopts::value(print_additional), "<output_format>")
+		("p,print",                       "Print (or replace) additional information in the rule's output.\nUse -pc or -pcontainer to append container details to syscall events.\nUse -pk or -pkubernetes to add both container and Kubernetes details to syscall events.\nIf using gVisor, choose -pcg or -pkg variants (or -pcontainer-gvisor and -pkubernetes-gvisor, respectively).\nIf a syscall rule's output contains %container.info, it will be replaced with the corresponding details. Otherwise, these details will be directly appended to the rule's output.\nAlternatively, use -p <output_format> for a custom format. In this case, the given <output_format> will be appended to the rule's output without any replacement to all events, including plugin events.", cxxopts::value(print_additional), "<output_format>")
 		("P,pidfile",                     "Write PID to specified <pid_file> path. By default, no PID file is created.", cxxopts::value(pidfilename)->default_value(""), "<pid_file>")
 		("r",                             "Rules file or directory to be loaded. This option can be passed multiple times. Falco defaults to the values in the configuration file when this option is not specified.", cxxopts::value<std::vector<std::string>>(), "<rules_file>")
 		("S,snaplen",                     "Collect only the first <len> bytes of each I/O buffer for 'syscall' events. By default, the first 80 bytes are collected by the driver and sent to the user space for processing. Use this option with caution since it can have a strong performance impact.", cxxopts::value(snaplen)->default_value("0"), "<len>")
 		("support",                       "Print support information, including version, rules files used, loaded configuration, etc., and exit. The output is in JSON format.", cxxopts::value(print_support)->default_value("false"))
-		("T",                             "DEPRECATED: use -o rules[].disable.tag=<tag> instead. Turn off any rules with a tag=<tag>. This option can be passed multiple times. This option can not be mixed with -t.", cxxopts::value<std::vector<std::string>>(), "<tag>")
-		("t",                             "DEPRECATED: use -o rules[].disable.rule=* -o rules[].enable.tag=<tag> instead. Only enable those rules with a tag=<tag>. This option can be passed multiple times. This option can not be mixed with -T/-D.", cxxopts::value<std::vector<std::string>>(), "<tag>")
 		("U,unbuffered",                  "Turn off output buffering for configured outputs. This causes every single line emitted by Falco to be flushed, which generates higher CPU usage but is useful when piping those outputs into another process or a script.", cxxopts::value(unbuffered_outputs)->default_value("false"))
 		("V,validate",                    "Read the contents of the specified <rules_file> file(s), validate the loaded rules, and exit. This option can be passed multiple times to validate multiple files.", cxxopts::value(validate_rules_filenames), "<rules_file>")
 		("v",                             "Enable verbose output.", cxxopts::value(verbose)->default_value("false"))

userspace/falco/app/options.h

@@ -40,13 +40,13 @@ public:
 
 	// Each of these maps directly to a command line option.
 	bool help = false;
+	bool print_config_schema = false;
 	std::string conf_filename;
 	bool all_events = false;
 	sinsp_evt::param_fmt event_buffer_format = sinsp_evt::PF_NORMAL;
 	std::vector<std::string> cri_socket_paths;
 	bool disable_cri_async = false;
 	std::vector<std::string> disable_sources;
-	std::vector<std::string> disabled_rule_substrings;
 	std::vector<std::string> enable_sources;
 	std::string gvisor_generate_config_with_socket;
 	bool describe_all_rules = false;
@@ -67,8 +67,6 @@ public:
 	std::list<std::string> rules_filenames;
 	uint64_t snaplen = 0;
 	bool print_support = false;
-	std::set<std::string> disabled_rule_tags;
-	std::set<std::string> enabled_rule_tags;
 	bool unbuffered_outputs = false;
 	std::vector<std::string> validate_rules_filenames;
 	bool verbose = false;

userspace/falco/app/restart_handler.cpp

@@ -17,7 +17,7 @@ limitations under the License.
 
 #include "restart_handler.h"
 #include "signals.h"
-#include "../logger.h"
+#include "logger.h"
 
 #include <string.h>
 #include <fcntl.h>

userspace/falco/configuration.h

@@ -47,6 +47,9 @@ enum class engine_kind_t : uint8_t
 	NODRIVER
 };
 
+// Map that holds { config filename | validation status } for each loaded config file.
+typedef std::map<std::string, std::string> config_loaded_res;
+
 class falco_configuration
 {
 public:
@@ -104,11 +107,20 @@ public:
 		std::string m_rule;
 	};
 
+	struct append_output_config {
+		std::string m_source;
+		std::string m_tag;
+		std::string m_rule;
+		std::string m_format;
+		std::unordered_map<std::string, std::string> m_formatted_fields;
+		std::set<std::string> m_raw_fields;
+	};
+
 	falco_configuration();
 	virtual ~falco_configuration() = default;
 
-	void init_from_file(const std::string& conf_filename, std::vector<std::string>& loaded_conf_files, const std::vector<std::string>& cmdline_options);
-	void init_from_content(const std::string& config_content, const std::vector<std::string>& cmdline_options, const std::string& filename="default");
+	config_loaded_res init_from_file(const std::string& conf_filename, const std::vector<std::string>& cmdline_options);
+	config_loaded_res init_from_content(const std::string& config_content, const std::vector<std::string>& cmdline_options, const std::string& filename="default");
 
 	std::string dump();
 
@@ -131,6 +143,8 @@ public:
 	std::list<std::string> m_loaded_rules_folders;
 	// Rule selection options passed by the user
 	std::vector<rule_selection_config> m_rules_selection;
+	// Append output configuration passed by the user
+	std::vector<append_output_config> m_append_output;
 
 	bool m_json_output;
 	bool m_json_include_output_property;
@@ -183,6 +197,11 @@ public:
 	bool m_metrics_include_empty_values;
 	std::vector<plugin_config> m_plugins;
 
+	// container engines
+	uint64_t m_container_engines_mask;
+	uint64_t m_container_engines_disable_cri_async;
+	std::vector<std::string> m_container_engines_cri_socket_paths;
+
 	// Falco engine
 	engine_kind_t m_engine_mode = engine_kind_t::KMOD;
 	kmod_config m_kmod = {};
@@ -192,10 +211,11 @@ public:
 	gvisor_config m_gvisor = {};
 
 	// Needed by tests
-	yaml_helper config;
+	yaml_helper m_config;
+	nlohmann::json m_config_schema;
 
 private:
-	void merge_config_files(const std::string& config_name, std::vector<std::string>& loaded_config_files);
+	void merge_config_files(const std::string& config_name, config_loaded_res &res);
 	void load_yaml(const std::string& config_name);
 	void init_logger();
 	void load_engine_config(const std::string& config_name);
@@ -210,6 +230,114 @@ private:
 };
 
 namespace YAML {
+	template<>
+	struct convert<falco_configuration::append_output_config> {
+		static Node encode(const falco_configuration::append_output_config & rhs) {
+			Node node;
+
+			if(rhs.m_source != "")
+			{
+				node["source"] = rhs.m_source;
+			}
+
+			if(rhs.m_rule != "")
+			{
+				node["rule"] = rhs.m_rule;
+			}
+
+			if(rhs.m_tag != "")
+			{
+				node["tag"] = rhs.m_tag;
+			}
+
+			if(rhs.m_format != "")
+			{
+				node["format"] = rhs.m_format;
+			}
+
+			for(auto const& field : rhs.m_formatted_fields)
+			{
+				YAML::Node field_node;
+				field_node[field.first] = field.second;
+				node["fields"].push_back(field_node);
+			}
+
+			for(auto const& field : rhs.m_raw_fields)
+			{
+				node["fields"].push_back(field);
+			}
+
+			return node;
+		}
+
+		static bool decode(const Node& node, falco_configuration::append_output_config & rhs) {
+			if(!node.IsMap())
+			{
+				return false;
+			}
+
+			if(node["source"])
+			{
+				rhs.m_source = node["source"].as<std::string>();
+			}
+
+			if(node["tag"])
+			{
+				rhs.m_tag = node["tag"].as<std::string>();
+			}
+
+			if(node["rule"])
+			{
+				rhs.m_rule = node["rule"].as<std::string>();
+			}
+
+			if(node["format"])
+			{
+				rhs.m_format = node["format"].as<std::string>();
+			}
+
+			if(node["fields"])
+			{
+				if(!node["fields"].IsSequence())
+				{
+					return false;
+				}
+
+				for(auto& field_definition : node["fields"])
+				{
+					if(field_definition.IsMap() && field_definition.size() == 1)
+					{
+						YAML::const_iterator def = field_definition.begin();
+						std::string key = def->first.as<std::string>();
+
+						// it is an error to redefine an existing key
+						if (rhs.m_formatted_fields.count(key) != 0 || rhs.m_raw_fields.count(key) != 0)
+						{
+							return false;
+						}
+
+						rhs.m_formatted_fields[key] = def->second.as<std::string>();
+					} else if (field_definition.IsScalar())
+					{
+						std::string key = field_definition.as<std::string>();
+
+						// it is an error to redefine an existing key
+						if (rhs.m_formatted_fields.count(key) != 0)
+						{
+							return false;
+						}
+
+						rhs.m_raw_fields.insert(key);
+					} else {
+						return false;
+					}
+				}
+			}
+
+			return true;
+		}
+	};
+
 	template<>
 	struct convert<falco_configuration::rule_selection_config> {
 		static Node encode(const falco_configuration::rule_selection_config & rhs) {

userspace/falco/falco_metrics.cpp

@@ -53,20 +53,20 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		BPF_ENGINE, KMOD_ENGINE, MODERN_BPF_ENGINE,
 		SOURCE_PLUGIN_ENGINE, NODRIVER_ENGINE, GVISOR_ENGINE };
 
-	std::vector<sinsp*> inspectors;
+	std::vector<std::shared_ptr<sinsp>> inspectors;
 	std::vector<libs::metrics::libs_metrics_collector> metrics_collectors;
 
-	for (const auto& source_info: state.source_infos)
+	for (const auto& source: state.enabled_sources)
 	{
-		sinsp *source_inspector = source_info.inspector.get();
+		auto source_info = state.source_infos.at(source);
+		auto source_inspector = source_info->inspector;
 		inspectors.emplace_back(source_inspector);
-		metrics_collectors.emplace_back(libs::metrics::libs_metrics_collector(source_inspector, state.config->m_metrics_flags));
+		metrics_collectors.emplace_back(libs::metrics::libs_metrics_collector(source_inspector.get(), state.config->m_metrics_flags));
 	}
-
 	libs::metrics::prometheus_metrics_converter prometheus_metrics_converter;
 	std::string prometheus_text;
 
-	for (auto* inspector: inspectors)
+	for (auto inspector: inspectors)
 	{
 		// Falco wrapper metrics
 		//
@@ -79,31 +79,69 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 			}
 		}
 
+
 		const scap_agent_info* agent_info = inspector->get_agent_info();
 		const scap_machine_info* machine_info = inspector->get_machine_info();
-
-		libs::metrics::libs_metrics_collector libs_metrics_collector(inspector, 0);
-
+		libs::metrics::libs_metrics_collector libs_metrics_collector(inspector.get(), 0);
 		prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("version", "falcosecurity", "falco", {{"version", FALCO_VERSION}});
-		prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("kernel_release", "falcosecurity", "falco", {{"kernel_release", agent_info->uname_r}});
-		prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("hostname", "falcosecurity", "evt", {{"hostname", machine_info->hostname}});
+
+		// Not all scap engines report agent and machine infos.
+		if (agent_info)
+		{
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("kernel_release", "falcosecurity", "falco", {{"kernel_release", agent_info->uname_r}});
+		}
+		if (machine_info)
+		{
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("hostname", "falcosecurity", "evt", {{"hostname", machine_info->hostname}});
+		}
 
 #if defined(__linux__) and !defined(MINIMAL_BUILD) and !defined(__EMSCRIPTEN__)
+		// Distinguish between config and rules files using labels, following Prometheus best practices: https://prometheus.io/docs/practices/naming/#labels
 		for (const auto& item : state.config.get()->m_loaded_rules_filenames_sha256sum)
 		{
 			fs::path fs_path = item.first;
-			std::string metric_name_file_sha256 = fs_path.filename().stem();
-			metric_name_file_sha256 = "falco.sha256_rules_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
-			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric_name_file_sha256, "falcosecurity", "falco", {{metric_name_file_sha256, item.second}});
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("falco_sha256_rules_files", "falcosecurity", "falco", {{"file_name", fs_path.filename().stem()}, {"sha256", item.second}});
 		}
 
 		for (const auto& item : state.config.get()->m_loaded_configs_filenames_sha256sum)
 		{
 			fs::path fs_path = item.first;
-			std::string metric_name_file_sha256 = fs_path.filename().stem();
-			metric_name_file_sha256 = "falco.sha256_config_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
-			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric_name_file_sha256, "falcosecurity", "falco", {{metric_name_file_sha256, item.second}});
+			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("falco_sha256_config_files", "falcosecurity", "falco", {{"file_name", fs_path.filename().stem()}, {"sha256", item.second}});
 		}
+
+		static std::string ifinfo_json_escaped;
+		auto ipv4list = inspector->get_ifaddr_list().get_ipv4_list();
+		auto ipv6list = inspector->get_ifaddr_list().get_ipv6_list();
+		nlohmann::json ipv4_json;
+		nlohmann::json ipv6_json;
+		if(ipv4list)
+		{
+			for (const auto& item : *ipv4list)
+			{
+				if(item.m_name == "lo")
+				{
+					continue;
+				}
+				ipv4_json[item.m_name] = item.addr_to_string();
+			}
+		}
+
+		if(ipv6list)
+		{
+			for (const auto& item : *ipv6list)
+			{
+				if(item.m_name == "lo")
+				{
+					continue;
+				}
+				ipv6_json[item.m_name] = item.addr_to_string();
+			}
+		}
+		nlohmann::json ifinfo_json;
+		ifinfo_json["ipv4"] = ipv4_json;
+		ifinfo_json["ipv6"] = ipv6_json;
+		ifinfo_json_escaped = ifinfo_json.dump();
+		prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("host_ifinfo_json", "falcosecurity", "falco", {{"host_ifinfo_json", ifinfo_json_escaped}});
 #endif
 
 		for (const std::string& source: inspector->event_sources())
@@ -112,39 +150,47 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		}
 		std::vector<metrics_v2> additional_wrapper_metrics;
 
-		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("start_ts",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
-																	METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
-																	agent_info->start_ts_epoch));
-		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_boot_ts",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
-																	METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
-																	machine_info->boot_ts_epoch));
-		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_num_cpus",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U32,
-																	METRIC_VALUE_UNIT_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
-																	machine_info->num_cpus));
-		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("outputs_queue_num_drops",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
-																	state.outputs->get_outputs_queue_num_drops()));
+		if (agent_info)
+		{
+			additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("start_ts",
+												  METRICS_V2_MISC,
+												  METRIC_VALUE_TYPE_U64,
+												  METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
+												  METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
+												  agent_info->start_ts_epoch));
+		}
+		if (machine_info)
+		{
+			additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("host_boot_ts",
+												  METRICS_V2_MISC,
+												  METRIC_VALUE_TYPE_U64,
+												  METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
+												  METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
+												  machine_info->boot_ts_epoch));
+			additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("host_num_cpus",
+												  METRICS_V2_MISC,
+												  METRIC_VALUE_TYPE_U32,
+												  METRIC_VALUE_UNIT_COUNT,
+												  METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
+												  machine_info->num_cpus));
+		}
+		additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("outputs_queue_num_drops",
+												METRICS_V2_MISC,
+												METRIC_VALUE_TYPE_U64,
+												METRIC_VALUE_UNIT_COUNT,
+												METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+												state.outputs->get_outputs_queue_num_drops()));
 
-		auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
-
-		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("duration_sec",
-																	METRICS_V2_MISC,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_TIME_S_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
-																	(uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS)));
+		if (agent_info)
+		{
+			auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+			additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("duration_sec",
+												  METRICS_V2_MISC,
+												  METRIC_VALUE_TYPE_U64,
+												  METRIC_VALUE_UNIT_TIME_S_COUNT,
+												  METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+												  (uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS)));
+		}
 
 		for (auto metric: additional_wrapper_metrics)
 		{
@@ -159,35 +205,29 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		{
 			const stats_manager& rule_stats_manager = state.engine->get_rule_stats_manager();
 			const indexed_vector<falco_rule>& rules = state.engine->get_rules();
-			auto metric = libs_metrics_collector.new_metric("rules.matches_total",
-																	METRICS_V2_RULE_COUNTERS,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
-																	rule_stats_manager.get_total().load());
-
-			prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
-			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", "falco");
 			const std::vector<std::unique_ptr<std::atomic<uint64_t>>>& rules_by_id = rule_stats_manager.get_by_rule_id();
+			// Distinguish between rules counters using labels, following Prometheus best practices: https://prometheus.io/docs/practices/naming/#labels
 			for (size_t i = 0; i < rules_by_id.size(); i++)
 			{
 				auto rule = rules.at(i);
-				std::string rules_metric_name = "rules." + falco::utils::sanitize_metric_name(rule->name);
-				// Separate processing of rules counter metrics given we add extra tags
-				auto metric = libs_metrics_collector.new_metric(rules_metric_name.c_str(),
-																	METRICS_V2_RULE_COUNTERS,
-																	METRIC_VALUE_TYPE_U64,
-																	METRIC_VALUE_UNIT_COUNT,
-																	METRIC_VALUE_METRIC_TYPE_MONOTONIC,
-																	rules_by_id[i]->load());
-				prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
-				const std::map<std::string, std::string>& const_labels = {
-					{"rule", rule->name},
-					{"priority", std::to_string(rule->priority)},
-					{"source", rule->source},
-					{"tags", concat_set_in_order(rule->tags)}
-				};
-				prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", "falco", const_labels);
+				auto count = rules_by_id[i]->load();
+				if (count > 0)
+				{
+					auto metric = libs::metrics::libsinsp_metrics::new_metric("rules_counters",
+											METRICS_V2_RULE_COUNTERS,
+											METRIC_VALUE_TYPE_U64,
+											METRIC_VALUE_UNIT_COUNT,
+											METRIC_VALUE_METRIC_TYPE_MONOTONIC,
+											rules_by_id[i]->load());
+					prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
+					const std::map<std::string, std::string>& const_labels = {
+						{"rule_name", rule->name},
+						{"priority", std::to_string(rule->priority)},
+						{"source", rule->source},
+						{"tags", concat_set_in_order(rule->tags)}
+					};
+					prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", "falco", const_labels);
+				}
 			}
 		}
 	}
@@ -207,10 +247,17 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		{
 			prometheus_metrics_converter.convert_metric_to_unit_convention(metric);
 			std::string namespace_name = "scap";
+			
 			if (metric.flags & METRICS_V2_RESOURCE_UTILIZATION || metric.flags & METRICS_V2_KERNEL_COUNTERS)
 			{
 				namespace_name = "falco";
 			}
+
+			if (metric.flags & METRICS_V2_PLUGINS)
+			{
+				namespace_name = "plugins";
+			}
+
 			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus(metric, "falcosecurity", namespace_name);
 		}
 

userspace/falco/falco_outputs.cpp

@@ -127,7 +127,8 @@ void falco_outputs::add_output(const falco::outputs::config &oc)
 }
 
 void falco_outputs::handle_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
-				 falco_common::priority_type priority, const std::string &format, std::set<std::string> &tags)
+				 falco_common::priority_type priority, const std::string &format, std::set<std::string> &tags,
+				 extra_output_field_t &extra_fields)
 {
 	falco_outputs::ctrl_msg cmsg = {};
 	cmsg.ts = evt->get_ts();
@@ -157,9 +158,30 @@ void falco_outputs::handle_event(sinsp_evt *evt, const std::string &rule, const
 	}
 
 	cmsg.msg = m_formats->format_event(
-		evt, rule, source, falco_common::format_priority(priority), sformat, tags, m_hostname
+		evt, rule, source, falco_common::format_priority(priority), sformat, tags, m_hostname, extra_fields
 	);
-	cmsg.fields = m_formats->get_field_values(evt, source, sformat);
+
+	auto fields = m_formats->get_field_values(evt, source, sformat);
+	for (auto const& ef : extra_fields)
+	{
+		// when formatting for the control message we always want strings,
+		// so we can simply format raw fields as string
+		std::string fformat = ef.second.first;
+		if (fformat.size() == 0)
+		{
+			continue;
+		}
+
+		if (!(fformat[0] == '*'))
+		{
+			fformat = "*" + fformat;
+		}
+
+		fields[ef.first] = m_formats->format_string(evt, fformat, source);
+	}
+
+	cmsg.fields = fields;
+
 	cmsg.tags.insert(tags.begin(), tags.end());
 
 	cmsg.type = ctrl_msg_type::CTRL_MSG_OUTPUT;

userspace/falco/falco_outputs.h

@@ -59,7 +59,8 @@ public:
 		is an event that has matched some rule).
 	*/
 	void handle_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
-			  falco_common::priority_type priority, const std::string &format, std::set<std::string> &tags);
+			falco_common::priority_type priority, const std::string &format, std::set<std::string> &tags,
+			extra_output_field_t &extra_fields);
 
 	/*!
 		\brief Format then send a generic message to all outputs.

userspace/falco/stats_writer.cpp

@@ -327,12 +327,18 @@ void stats_writer::collector::get_metrics_output_fields_wrapper(
 	/* Wrapper fields useful for statistical analyses and attributions. Always enabled. */
 	output_fields["evt.time"] = now; /* Some ETLs may prefer a consistent timestamp within output_fields. */
 	output_fields["falco.version"] = FALCO_VERSION;
-	output_fields["falco.start_ts"] = agent_info->start_ts_epoch;
-	output_fields["falco.duration_sec"] = (uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS);
-	output_fields["falco.kernel_release"] = agent_info->uname_r;
-	output_fields["evt.hostname"] = machine_info->hostname; /* Explicitly add hostname to log msg in case hostname rule output field is disabled. */
-	output_fields["falco.host_boot_ts"] = machine_info->boot_ts_epoch;
-	output_fields["falco.host_num_cpus"] = machine_info->num_cpus;
+	if (agent_info)
+	{
+		output_fields["falco.start_ts"] = agent_info->start_ts_epoch;
+		output_fields["falco.duration_sec"] = (uint64_t)((now - agent_info->start_ts_epoch) / ONE_SECOND_IN_NS);
+		output_fields["falco.kernel_release"] = agent_info->uname_r;
+	}
+	if (machine_info)
+	{
+		output_fields["evt.hostname"] = machine_info->hostname; /* Explicitly add hostname to log msg in case hostname rule output field is disabled. */
+		output_fields["falco.host_boot_ts"] = machine_info->boot_ts_epoch;
+		output_fields["falco.host_num_cpus"] = machine_info->num_cpus;
+	}
 	output_fields["falco.outputs_queue_num_drops"] = m_writer->m_outputs->get_outputs_queue_num_drops();
 
 #if defined(__linux__) and !defined(MINIMAL_BUILD) and !defined(__EMSCRIPTEN__)
@@ -351,6 +357,40 @@ void stats_writer::collector::get_metrics_output_fields_wrapper(
 		metric_name_file_sha256 = "falco.sha256_config_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
 		output_fields[metric_name_file_sha256] = item.second;
 	}
+
+	auto ipv4list = inspector->get_ifaddr_list().get_ipv4_list();
+	auto ipv6list = inspector->get_ifaddr_list().get_ipv6_list();
+	nlohmann::json ipv4_json;
+	nlohmann::json ipv6_json;
+	if(ipv4list)
+	{
+		for (const auto& item : *ipv4list)
+		{
+			if(item.m_name == "lo")
+			{
+				continue;
+			}
+			ipv4_json[item.m_name] = item.addr_to_string();
+		}
+	}
+
+	if(ipv6list)
+	{
+		for (const auto& item : *ipv6list)
+		{
+			if(item.m_name == "lo")
+			{
+				continue;
+			}
+			ipv6_json[item.m_name] = item.addr_to_string();
+		}
+	}
+	nlohmann::json ifinfo_json;
+	ifinfo_json["ipv4"] = ipv4_json;
+	ifinfo_json["ipv6"] = ipv6_json;
+	m_ifinfo_json_escaped = ifinfo_json.dump();
+	output_fields["falco.host_ifinfo_json"] = m_ifinfo_json_escaped;
+
 #endif
 	output_fields["evt.source"] = src;
 	for (size_t i = 0; i < sizeof(all_driver_engines) / sizeof(const char*); i++)
@@ -434,6 +474,10 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 			{
 				strlcpy(metric_name, "scap.", sizeof(metric_name));
 			}
+			if(metric.flags & METRICS_V2_PLUGINS)
+			{
+				strlcpy(metric_name, "plugins.", sizeof(metric_name));
+			}
 			strlcat(metric_name, metric.name, sizeof(metric_name));
 
 			switch (metric.type)
@@ -445,6 +489,13 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 				}
 				output_fields[metric_name] = metric.value.u32;
 				break;
+			case METRIC_VALUE_TYPE_S32:
+				if (metric.value.s32 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				output_fields[metric_name] = metric.value.s32;
+				break;
 			case METRIC_VALUE_TYPE_U64:
 				if (strncmp(metric.name, "n_evts", 7) == 0)
 				{
@@ -486,6 +537,13 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 				}
 				output_fields[metric_name] = metric.value.u64;
 				break;
+			case METRIC_VALUE_TYPE_S64:
+				if (metric.value.s64 == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				output_fields[metric_name] = metric.value.s64;
+				break;
 			case METRIC_VALUE_TYPE_D:
 				if (metric.value.d == 0 && !m_writer->m_config->m_metrics_include_empty_values)
 				{
@@ -493,6 +551,20 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 				}
 				output_fields[metric_name] = metric.value.d;
 				break;
+			case METRIC_VALUE_TYPE_F:
+				if (metric.value.f == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				output_fields[metric_name] = metric.value.f;
+				break;
+			case METRIC_VALUE_TYPE_I:
+				if (metric.value.i == 0 && !m_writer->m_config->m_metrics_include_empty_values)
+				{
+					break;
+				}
+				output_fields[metric_name] = metric.value.i;
+				break;
 			default:
 				break;
 			}

userspace/falco/stats_writer.h

@@ -80,6 +80,7 @@ public:
 		uint64_t m_last_n_evts = 0;
 		uint64_t m_last_n_drops = 0;
 		uint64_t m_last_num_evts = 0;
+		std::string m_ifinfo_json_escaped;
 	};
 
 	stats_writer(const stats_writer&) = delete;

userspace/falco/yaml_helper.h

@@ -32,6 +32,14 @@ limitations under the License.
 #include <iostream>
 #include <fstream>
 #include <filesystem>
+#include <numeric>
+
+#include <nlohmann/json.hpp>
+#include <valijson/adapters/nlohmann_json_adapter.hpp>
+#include <valijson/adapters/yaml_cpp_adapter.hpp>
+#include <valijson/schema.hpp>
+#include <valijson/schema_parser.hpp>
+#include <valijson/validator.hpp>
 
 #include "config_falco.h"
 
@@ -79,27 +87,42 @@ class yaml_helper
 {
 public:
 	inline static const std::string configs_key = "config_files";
+	inline static const std::string validation_ok = "ok";
+	inline static const std::string validation_failed = "failed";
+	inline static const std::string validation_none = "schema not provided";
 
 	/**
 	* Load the YAML document represented by the input string.
 	*/
-	void load_from_string(const std::string& input)
+	void load_from_string(const std::string& input, const nlohmann::json& schema={}, std::string *validation=nullptr)
 	{
 		m_root = YAML::Load(input);
 		pre_process_env_vars(m_root);
+
+		if (validation)
+		{
+			if(!schema.empty())
+			{
+				*validation = validate_node(m_root, schema);
+			}
+			else
+			{
+				*validation = validation_none;
+			}
+		}
 	}
 
 	/**
 	* Load the YAML document from the given file path.
 	*/
-	void load_from_file(const std::string& path)
+	void load_from_file(const std::string& path, const nlohmann::json& schema={}, std::string *validation=nullptr)
 	{
-		m_root = load_from_file_int(path);
+		m_root = load_from_file_int(path, schema, validation);
 	}
 
-	void include_config_file(const std::string& include_file_path)
+	void include_config_file(const std::string& include_file_path, const nlohmann::json& schema={}, std::string *validation=nullptr)
 	{
-		auto loaded_nodes = load_from_file_int(include_file_path);
+		auto loaded_nodes = load_from_file_int(include_file_path, schema, validation);
 		for(auto n : loaded_nodes)
 		{
 			/*
@@ -154,6 +177,16 @@ public:
 		node = value;
 	}
 
+	/**
+	 * Set the node identified by key to an object value
+	 */
+	void set_object(const std::string& key, const YAML::Node& value)
+	{
+		YAML::Node node;
+		get_node(node, key, true);
+		node = value;
+	}
+
 	/**
 	* Get the sequence value from the node identified by key.
 	*/
@@ -185,13 +218,52 @@ public:
 private:
 	YAML::Node m_root;
 
-	YAML::Node load_from_file_int(const std::string& path)
+	YAML::Node load_from_file_int(const std::string& path, const nlohmann::json& schema={}, std::string *validation=nullptr)
 	{
 		auto root = YAML::LoadFile(path);
 		pre_process_env_vars(root);
+
+		if (validation)
+		{
+			if(!schema.empty())
+			{
+				*validation = validate_node(root, schema);
+			}
+			else
+			{
+				*validation = validation_none;
+			}
+		}
 		return root;
 	}
 
+	std::string validate_node(const YAML::Node &node, const nlohmann::json& schema={})
+	{
+		// Validate the yaml against our json schema
+		valijson::Schema schemaDef;
+		valijson::SchemaParser schemaParser;
+		valijson::Validator validator(valijson::Validator::kWeakTypes);
+		valijson::ValidationResults validationResults;
+		valijson::adapters::YamlCppAdapter configAdapter(node);
+		valijson::adapters::NlohmannJsonAdapter schemaAdapter(schema);
+		schemaParser.populateSchema(schemaAdapter, schemaDef);
+
+		if (!validator.validate(schemaDef, configAdapter, &validationResults))
+		{
+			valijson::ValidationResults::Error error;
+			// report only the top-most error
+			if (validationResults.popError(error))
+			{
+				return std::string(validation_failed + " for ")
+				       + std::accumulate(error.context.begin(), error.context.end(), std::string(""))
+				       + ": "
+				       + error.description;
+			}
+			return validation_failed;
+		}
+		return validation_ok;
+	}
+
 	/*
 	 * When loading a yaml file,
 	 * we immediately pre process all scalar values through a visitor private API,
@@ -420,5 +492,6 @@ namespace YAML {
 
 			return true;
 		}
+		// The "encode" function is not needed here, in fact you can simply YAML::load any json string.
 	};
 }