new(docs): add changelog for 0.38.2

Signed-off-by: Luca Guerra <luca@guerra.sh>

.github/workflows/ci.yml

@@ -12,9 +12,6 @@ concurrency:
   group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true  
 
-permissions:  
-  contents: read
-  
 jobs:
   fetch-version:
     uses: ./.github/workflows/reusable_fetch_version.yaml

.github/workflows/codeql.yaml

@@ -18,9 +18,6 @@ on:
     # The branches below must be a subset of the branches above
     branches: [ "master" ]
 
-permissions:  
-  contents: read
-
 jobs:
   analyze:
     name: Analyze

.github/workflows/codespell.yml

@@ -1,10 +1,6 @@
 name: Codespell
 on:
   pull_request:
-    
-permissions:  
-  contents: read
-
 jobs:
   codespell:
     runs-on: ubuntu-latest

.github/workflows/engine-version-weakcheck.yaml

@@ -9,9 +9,6 @@ on:
       - 'userspace/engine/*.cpp'
       - 'userspace/engine/*.h'
 
-permissions:  
-  contents: read
-
 jobs:
   paths-filter:
     runs-on: ubuntu-latest

.github/workflows/insecure-api.yaml

@@ -6,15 +6,12 @@ on:
       - 'release/**'
       - 'maintainers/**'
 
-permissions:  
-  contents: read
-
 jobs:
   insecure-api:
     name: check-insecure-api
     runs-on: ubuntu-latest
     container:
-      image: semgrep/semgrep:1.85.0@sha256:b4c2272e0a2e59ca551ff96d3bbae657bd2b7356e339af557b27a96d9e751544
+      image: returntocorp/semgrep:1.41.0@sha256:85956fbe795a0e8a3825d5252f175887c0e0c6ce7a766a07062c0fb68415cd67
     steps:
       - name: Checkout Falco ⤵️
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0

.github/workflows/master.yaml

@@ -8,9 +8,6 @@ concurrency:
   group: ci-master
   cancel-in-progress: true  
 
-permissions:  
-  contents: read
-
 jobs:
   fetch-version:
     uses: ./.github/workflows/reusable_fetch_version.yaml 

.github/workflows/release.yaml

@@ -8,9 +8,6 @@ concurrency:
   group: ci-release
   cancel-in-progress: true  
 
-permissions:  
-  contents: read
-  
 jobs:
   release-settings:
     runs-on: ubuntu-latest
@@ -19,7 +16,7 @@ jobs:
       bucket_suffix: ${{ steps.get_settings.outputs.bucket_suffix }}
     steps:
       - name: Get latest release
-        uses: rez0n/actions-github-release@27a57820ee808f8fd940c8a9d1f7188f854aa2b5 # v2.0
+        uses: rez0n/actions-github-release@v2.0
         id: latest_release
         env:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/reusable_build_dev.yaml

@@ -33,9 +33,6 @@ on:
         default: ''
         type: string
 
-permissions: 
-  contents: read
-
 jobs:
   build-and-test:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936

.github/workflows/reusable_build_docker.yaml

@@ -24,10 +24,6 @@ on:
 # then we upload all the tarballs to be later downloaded by reusable_publish_docker workflow.
 # In this way, we don't need to publish any arch specific image, 
 # and this "build" workflow is actually only building images.
-
-permissions:  
-  contents: read
-
 jobs:
   build-docker:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936

.github/workflows/reusable_build_packages.yaml

@@ -21,9 +21,6 @@ on:
         type: boolean
         default: false
 
-permissions:  
-  contents: read
-
 jobs:
   build-modern-bpf-skeleton:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936

.github/workflows/reusable_fetch_version.yaml

@@ -6,9 +6,6 @@ on:
         description: "Falco version"
         value: ${{ jobs.fetch-version.outputs.version }}
 
-permissions:  
-  contents: read
-
 jobs:
   # We need to use an ubuntu-latest to fetch Falco version because
   # Falco version is computed by some cmake scripts that do git sorceries

.github/workflows/reusable_test_packages.yaml

@@ -21,9 +21,6 @@ on:
         default: false
         type: boolean
 
-permissions:  
-  contents: read
-  
 jobs:
   test-packages:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
@@ -57,12 +54,11 @@ jobs:
       - name: Run tests
         env:
           LSAN_OPTIONS: "intercept_tls_get_addr=0"
-        uses: falcosecurity/testing@main 
+        uses: falcosecurity/testing@main    
         with:
           test-falco: 'true'
           test-falcoctl: 'true'
           test-k8saudit: 'true'
-          test-dummy: 'true'
           static: ${{ inputs.static && 'true' || 'false' }}
           test-drivers: ${{ inputs.arch == 'x86_64' && 'true' || 'false' }}
           show-all: 'true'

.github/workflows/staticanalysis.yaml

@@ -1,9 +1,6 @@
 name: StaticAnalysis
 on:
   pull_request:
-permissions:  
-  contents: read
-
 jobs:
   staticanalysis:
     runs-on: ubuntu-22.04

ADOPTERS.md

@@ -74,8 +74,6 @@ This is a list of production adopters of Falco (in alphabetical order):
 
 * [Thought Machine](https://www.thoughtmachine.net) Thought Machine builds Vault Core and Vault Payments: cloud-native core and payments technology enabling banks and fintechs to remain competitive and flourish into the future. Vault Core and Vault Payments are the foundation layer of a bank's technology stack. They can run any bank, any product, and any payment set. Thought Machine uses Falco to perform cloud agnostic real time detections of suspicious container behaviour.
 
-* [Tulip Retail](https://tulip.com) Tulip Retail uses Falco to monitor container activity in our environments. It's numerous integration points, easy deployment and easily customizable rules were the main reasons we chose Falco.
-
 * [Vinted](https://vinted.com/) Vinted uses Falco to continuously monitor container activities, identifying security threats, and ensuring compliance. The container-native approach, rule-based real-time threat detection, community support, extensibility, and compliance capabilities are the main factors why we chose it to enhance Vinted Kubernetes security. Falco Sidekick is used to send critical and warning severity alerts to our incident management solution (RTIR).
 
 * [Xenit AB](https://xenit.se/contact/) Xenit is a growth company with services within cloud and digital transformation. We provide an open-source Kubernetes framework that we leverage to help our customers get their applications to production as quickly and as securely as possible. We use Falco's detection capabilities to identify anomalous behaviour within our clusters in both Azure and AWS.

CMakeLists.txt

@@ -151,7 +151,6 @@ if (NOT EMSCRIPTEN)
 endif()
 
 include(zlib)
-include(valijson)
 if (NOT MINIMAL_BUILD)
   if (NOT WIN32 AND NOT APPLE AND NOT EMSCRIPTEN)
     include(cares)

OWNERS

@@ -6,7 +6,6 @@ approvers:
   - andreagit97
   - incertum
   - LucaGuerra
-  - sgaist
 reviewers:
   - kaizhe
 emeritus_approvers:

README.md

@@ -2,7 +2,7 @@
 
 [![Latest release](https://img.shields.io/github/v/release/falcosecurity/falco?style=for-the-badge)](https://github.com/falcosecurity/falco/releases/latest) [![Supported Architectures](https://img.shields.io/badge/ARCHS-x86__64%7Caarch64-blueviolet?style=for-the-badge)](https://github.com/falcosecurity/falco/releases/latest) [![License](https://img.shields.io/github/license/falcosecurity/falco?style=for-the-badge)](COPYING) [![Docs](https://img.shields.io/badge/docs-latest-green.svg?style=for-the-badge)](https://falco.org/docs)
 
-[![Falco Core Repository](https://github.com/falcosecurity/evolution/blob/main/repos/badges/falco-core-blue.svg)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#core-scope) [![Stable](https://img.shields.io/badge/status-stable-brightgreen?style=for-the-badge)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#stable)  [![OpenSSF Scorecard](https://img.shields.io/ossf-scorecard/github.com/falcosecurity/falco?label=openssf%20scorecard&style=for-the-badge)](https://scorecard.dev/viewer/?uri=github.com/falcosecurity/falco)  [![OpenSSF Best Practices](https://img.shields.io/cii/summary/2317?label=OpenSSF%20Best%20Practices&style=for-the-badge)](https://bestpractices.coreinfrastructure.org/projects/2317) <a href="https://actuated.dev/"><img alt="Arm CI sponsored by Actuated" src="https://docs.actuated.dev/images/actuated-badge.png" width="120px"></img></a>
+[![Falco Core Repository](https://github.com/falcosecurity/evolution/blob/main/repos/badges/falco-core-blue.svg)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#core-scope) [![Stable](https://img.shields.io/badge/status-stable-brightgreen?style=for-the-badge)](https://github.com/falcosecurity/evolution/blob/main/REPOSITORIES.md#stable)  [![OpenSSF Best Practices](https://img.shields.io/cii/summary/2317?label=OpenSSF%20Best%20Practices&style=for-the-badge)](https://bestpractices.coreinfrastructure.org/projects/2317) <a href="https://actuated.dev/"><img alt="Arm CI sponsored by Actuated" src="https://docs.actuated.dev/images/actuated-badge.png" width="120px"></img></a>
 
 [![Falco](https://falco.org/img/brand/falco-horizontal-color.svg)](https://falco.org)
 
@@ -43,9 +43,6 @@ Considerations and guidance for Falco adopters:
 
 5. Integrate with output destinations: Integrate Falco with SIEM, data lake systems, or other preferred output destinations to establish a robust foundation for comprehensive data analysis and enable effective incident response workflows.
 
-### Demo Environment
-
-A demo environment is provided via a docker-compose file that can be started on a docker host which includes falco, falcosidekick, falcosidekick-ui and its required redis database. For more information see the [docker-compose section](docker/docker-compose/)
 
 ## How to Contribute
 

cmake/modules/CompilerFlags.cmake

@@ -88,17 +88,15 @@ else() # MSVC
 	set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 
 	# The WIN32_LEAN_AND_MEAN define avoids possible macro pollution
-	# when a libsinsp consumer includes the windows.h header:
-	# https://stackoverflow.com/a/28380820
-	# Same goes for NOMINMAX:
-	# https://stackoverflow.com/questions/5004858/why-is-stdmin-failing-when-windows-h-is-included
+	# when a libsinsp consumer includes the windows.h header.
+	# See: https://stackoverflow.com/a/28380820
+
 	add_compile_definitions(
 		_HAS_STD_BYTE=0
 		_CRT_SECURE_NO_WARNINGS
 		WIN32
 		MINIMAL_BUILD
 		WIN32_LEAN_AND_MEAN
-		NOMINMAX
 	)
 
 	set(FALCOSECURITY_LIBS_COMMON_FLAGS "/EHsc /W3 /Zi /std:c++17")

cmake/modules/driver.cmake

@@ -34,8 +34,8 @@ else()
   # In case you want to test against another driver version (or branch, or commit) just pass the variable -
   # ie., `cmake -DDRIVER_VERSION=dev ..`
   if(NOT DRIVER_VERSION)
-    set(DRIVER_VERSION "dd165076cb71c5640f705c2e67acb604b2d3bdc9")
-    #set(DRIVER_CHECKSUM "SHA256=aedc4120d335ee2ebf7f3c2e6346fc985ca09a49d14ff7c6249a033e47d160bf")
+    set(DRIVER_VERSION "7.2.1+driver")
+    set(DRIVER_CHECKSUM "SHA256=0ae749718557812dc008bdfd8eaa81355094a0975380df1021b1e2bf2ee91457")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

cmake/modules/falcoctl.cmake

@@ -14,21 +14,16 @@
 
 include(ExternalProject)
 
-option(ADD_FALCOCTL_DEPENDENCY "Add falcoctl dependency while building falco" ON)
-
-if(ADD_FALCOCTL_DEPENDENCY)
 string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} FALCOCTL_SYSTEM_NAME)
 
-set(FALCOCTL_VERSION "0.10.0-rc1")
-
-message(STATUS "Building with falcoctl: ${FALCOCTL_VERSION}")
+set(FALCOCTL_VERSION "0.9.0")
 
 if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
     set(FALCOCTL_SYSTEM_PROC_GO "amd64")
-    set(FALCOCTL_HASH "efef3b6912837622ec1822941643d67d7b8ba704ba759751a17491b7f9fd05c3")
+    set(FALCOCTL_HASH "04a689cca5b18c82427fe0cdc15c37b35f3f4696f6bc13d92aa903183b25b2c5")
 else() # aarch64
     set(FALCOCTL_SYSTEM_PROC_GO "arm64")
-    set(FALCOCTL_HASH "0429782ae5dc46ff8611a5f6ab2c275d8dd0e0721937ace0e8f6abc23de9dd57")
+    set(FALCOCTL_HASH "cd37537a7d1a81e5e372760e14b3a945c650f845e98649fc15e560b0ba7a6597")
 endif()
 
 ExternalProject_Add(
@@ -41,6 +36,3 @@ ExternalProject_Add(
 
 install(PROGRAMS "${PROJECT_BINARY_DIR}/falcoctl-prefix/src/falcoctl/falcoctl" DESTINATION "${FALCO_BIN_DIR}" COMPONENT "${FALCO_COMPONENT_NAME}")
 install(DIRECTORY DESTINATION "${FALCO_ABSOLUTE_SHARE_DIR}/plugins" COMPONENT "${FALCO_COMPONENT_NAME}")
-else()
-    message(STATUS "Won't build with falcoctl")
-endif()

cmake/modules/falcosecurity-libs.cmake

@@ -35,8 +35,8 @@ else()
   # In case you want to test against another falcosecurity/libs version (or branch, or commit) just pass the variable -
   # ie., `cmake -DFALCOSECURITY_LIBS_VERSION=dev ..`
   if(NOT FALCOSECURITY_LIBS_VERSION)
-    set(FALCOSECURITY_LIBS_VERSION "dd165076cb71c5640f705c2e67acb604b2d3bdc9")
-#    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=aedc4120d335ee2ebf7f3c2e6346fc985ca09a49d14ff7c6249a033e47d160bf")
+    set(FALCOSECURITY_LIBS_VERSION "0.17.3")
+    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=6ff90672fe35d725e79dcb1d940c1518154daef28a3eb1cd127432c503cab079")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

cmake/modules/rules.cmake

@@ -15,7 +15,6 @@
 include(GNUInstallDirs)
 include(ExternalProject)
 
-if(NOT DEFINED FALCOSECURITY_RULES_FALCO_PATH)
 # falco_rules.yaml
 set(FALCOSECURITY_RULES_FALCO_VERSION "falco-rules-3.1.0")
 set(FALCOSECURITY_RULES_FALCO_CHECKSUM "SHA256=3b617920c0b66128627613e591a954eb9572747a4c287bc13b53b38786250162")
@@ -29,13 +28,10 @@ ExternalProject_Add(
   INSTALL_COMMAND ""
   TEST_COMMAND ""
 )
-endif()
 
-if(NOT DEFINED FALCOSECURITY_RULES_LOCAL_PATH)
 # falco_rules.local.yaml
 set(FALCOSECURITY_RULES_LOCAL_PATH "${PROJECT_BINARY_DIR}/falcosecurity-rules-local-prefix/falco_rules.local.yaml")
 file(WRITE "${FALCOSECURITY_RULES_LOCAL_PATH}" "# Your custom rules!\n")
-endif()
 
 if(NOT DEFINED FALCO_ETC_DIR)
 	set(FALCO_ETC_DIR "${CMAKE_INSTALL_FULL_SYSCONFDIR}/falco")

docker/docker-compose/README.md

@@ -1,17 +0,0 @@
-# Warning
-
-This environment is provided for demonstration purposes only and does not represent a production ready deployment of Falco.
-
-# Components
-The components that this docker-compose file spins up are [Falco](https://falco.org/), [falcosidekick](https://github.com/falcosecurity/falcosidekick), [falcosidekick-ui](https://github.com/falcosecurity/falcosidekick-ui) and a [redis](https://redis.io/) database.
-
-# Running
-To start this environment run `docker-compose up`.
-Note: You may need to use sudo for Falco to start correctly.
-
-# Cleaning up
-
-To clean up run `docker-compose rm`.
-
-# Generating events
-If you'd like to generate events that will trigger rules and show up in the UI you can run `docker run -it --rm falcosecurity/event-generator run syscall --loop`

docker/docker-compose/config/http_output.yml

@@ -1,11 +0,0 @@
-# [Stable] `http_output`
-#
-# Send logs to an HTTP endpoint or webhook.
-#
-# When using falcosidekick, it is necessary to set `json_output` to true.
-json_output: true
-json_include_output_property: true
-http_output:
-  enabled: true
-  url: "http://falco-sidekick:2801/"
-  

docker/docker-compose/docker-compose.yaml

@@ -1,34 +0,0 @@
-version: "3"
-services:
-  falco:
-    container_name: falco
-    cap_drop:
-      - all
-    cap_add:
-      - sys_admin
-      - sys_resource
-      - sys_ptrace
-    volumes:
-      - /var/run/docker.sock:/host/var/run/docker.sock
-      - /proc:/host/proc:ro
-      - /etc:/host/etc:ro
-      - ./config/http_output.yml:/etc/falco/config.d/http_output.yml
-    image: falcosecurity/falco-no-driver:latest
-
-  sidekick:
-    container_name: falco-sidekick
-    image: falcosecurity/falcosidekick
-    environment:
-      WEBUI_URL: http://falco-webui:2802
-
-  webui:
-    container_name: falco-webui
-    image: falcosecurity/falcosidekick-ui:2.2.0
-    ports:
-      - 2802:2802
-    depends_on:
-      - redis
-    command: ['-r', 'redis:6379', '-d']
-
-  redis:
-    image: redis/redis-stack:7.2.0-v11

falco.yaml

@@ -468,7 +468,7 @@ load_plugins: []
 plugins:
   - name: k8saudit
     library_path: libk8saudit.so
-    init_config: ""
+    init_config:
     #   maxEventSize: 262144
     #   webhookMaxBatchSize: 12582912
     #   sslCertificate: /etc/falco/falco.pem
@@ -576,42 +576,6 @@ rule_matching: first
 outputs_queue:
   capacity: 0
 
-# [Sandbox] `append_output`
-#
-# Add information to the Falco output.
-# With this setting you can add more information to the Falco output message, customizable by
-# rule, tag or source.
-# You can also add additional data that will appear in the output_fields property
-# of JSON formatted messages or gRPC output but will not be part of the regular output message.
-# This allows you to add custom fields that can help you filter your Falco events without
-# polluting the message text.
-#
-# Each append_output entry has optional fields (ANDed together) to filter events:
-#   `rule`: append output only to a specific rule
-#   `source`: append output only to a specific source
-#   `tag`: append output only to a specific tag
-# If none of the above are specified output is appended to all events, if more than one is
-# specified output will be appended to events that match all conditions.
-# And several options to add output:
-#   `format`: add output to the Falco message
-#   `fields`: add new fields to the JSON output and structured output, which will not
-#             affect the regular Falco message in any way. These can be specified as a
-#             custom name with a custom format or as any supported field
-#             (see: https://falco.org/docs/reference/rules/supported-fields/)
-#
-# Example:
-#
-#   - source: syscall
-#     format: "on CPU %evt.cpu"
-#     fields:
-#       - home_directory: "${HOME}"
-#       - evt.hostname
-#
-# In the example above every event coming from the syscall source will get an extra message
-# at the end telling the CPU number. In addition, if `json_output` is true, in the "output_fields"
-# property you will find three new ones: "evt.cpu", "home_directory" which will contain the value of the
-# environment variable $HOME, and "evt.hostname" which will contain the hostname.
-
 
 ##########################
 # Falco outputs channels #
@@ -1242,43 +1206,33 @@ base_syscalls:
 falco_libs:
   thread_table_size: 262144
 
-# [Incubating] `container_engines`
+# [Stable] Guidance for Kubernetes container engine command-line args settings
 #
-# This option allows you to explicitly enable or disable API lookups against container 
-# runtime sockets for each supported container runtime. 
-# Access to these sockets enables Falco to retrieve container and Kubernetes fields, 
-# helping identify workload owners in modern containerized environments. 
-# Refer to the fields docs:
-# 
-# - [Kubernetes fields](https://falco.org/docs/reference/rules/supported-fields/#field-class-k8s)
-# - [Container fields](https://falco.org/docs/reference/rules/supported-fields/#container)
-# 
-# Additionally, Falco can use container events as a data source for alerting (evt.type = container).
-# 
-# For most container engines, you can solely enable or disable them, and Falco will search the 
-# default (hard-coded) container runtime socket paths, such as `/var/run/docker.sock` for Docker.
+# Modern cloud environments, particularly Kubernetes, heavily rely on
+# containerized workload deployments. When capturing events with Falco, it
+# becomes essential to identify the owner of the workload for which events are
+# being captured, such as syscall events. Falco integrates with the container
+# runtime to enrich its events with container information, including fields like
+# `container.image.repository`, `container.image.tag`, ... , `k8s.ns.name`,
+# `k8s.pod.name`, `k8s.pod.*` in the Falco output (Falco retrieves Kubernetes
+# namespace and pod name directly from the container runtime, see
+# https://falco.org/docs/reference/rules/supported-fields/#field-class-container).
 #
-# However, for Kubernetes settings, you can customize the CRI socket paths:
-# 
-# - `container_engines.cri.sockets`: Pass a list of container runtime sockets.
-# - `container_engines.cri.disable_async`: Since API lookups may not always be quick or 
-# perfect, resulting in empty fields for container metadata, you can use this option option 
-# to disable asynchronous fetching. Note that missing fields may still occasionally occur.
-# 
-# The equivalent (stable) CLI args are `--cri` or `--disable-cri-async`.
-
-container_engines:
-  docker:
-    enabled: true
-  cri:
-    enabled: true
-    sockets: ["/run/containerd/containerd.sock", "/run/crio/crio.sock", "/run/k3s/containerd/containerd.sock"]
-    disable_async: false
-  podman:
-    enabled: true
-  lxc:
-    enabled: true
-  libvirt_lxc:
-    enabled: true
-  bpm:
-    enabled: true
+# Furthermore, Falco exposes container events themselves as a data source for
+# alerting. To achieve this integration with the container runtime, Falco
+# requires access to the runtime socket. By default, for Kubernetes, Falco
+# attempts to connect to the following sockets:
+# "/run/containerd/containerd.sock", "/run/crio/crio.sock",
+# "/run/k3s/containerd/containerd.sock". If you have a custom path, you can use
+# the `--cri` option to specify the correct location.
+#
+# In some cases, you may encounter empty fields for container metadata. To
+# address this, you can explore the `--disable-cri-async` option, which disables
+# asynchronous fetching if the fetch operation is not completing quickly enough.
+#
+# To get more information on these command-line arguments, you can run `falco
+# --help` in your terminal to view their current descriptions.
+#
+# !!! The options mentioned here are not available in the falco.yaml
+# configuration file. Instead, they can can be used as a command-line argument
+# when running the Falco command.

unit_tests/CMakeLists.txt

@@ -36,7 +36,6 @@ add_executable(falco_unit_tests
     engine/test_add_source.cpp
     engine/test_alt_rule_loader.cpp
     engine/test_enable_rule.cpp
-    engine/test_extra_output.cpp
     engine/test_falco_utils.cpp
     engine/test_filter_details_resolver.cpp
     engine/test_filter_macro_resolver.cpp
@@ -46,10 +45,6 @@ add_executable(falco_unit_tests
     engine/test_rulesets.cpp
     falco/test_configuration.cpp
     falco/test_configuration_rule_selection.cpp
-    falco/test_configuration_config_files.cpp
-    falco/test_configuration_env_vars.cpp
-    falco/test_configuration_output_options.cpp
-    falco/test_configuration_schema.cpp
     falco/app/actions/test_select_event_sources.cpp
     falco/app/actions/test_load_config.cpp
 )

unit_tests/engine/test_extra_output.cpp

@@ -1,150 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-/*
-Copyright (C) 2024 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include <gtest/gtest.h>
-
-#include "../test_falco_engine.h"
-
-TEST_F(test_falco_engine, extra_format_all)
-{
-	std::string rules_content = R"END(
-- rule: legit_rule
-  desc: legit rule description
-  condition: evt.type=open
-  output: user=%user.name command=%proc.cmdline file=%fd.name
-  priority: INFO
-)END";
-
-	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "", false);
-	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
-
-	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"user=%user.name command=%proc.cmdline file=%fd.name evt.type=%evt.type");
-}
-
-TEST_F(test_falco_engine, extra_format_by_rule)
-{
-	std::string rules_content = R"END(
-- rule: legit_rule
-  desc: legit rule description
-  condition: evt.type=open
-  output: out 1
-  priority: INFO
-
-- rule: another_rule
-  desc: legit rule description
-  condition: evt.type=open
-  output: out 2
-  priority: INFO
-)END";
-
-	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "legit_rule", false);
-	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
-
-	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 evt.type=%evt.type");
-	EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2");
-}
-
-TEST_F(test_falco_engine, extra_format_by_tag_rule)
-{
-	std::string rules_content = R"END(
-- rule: legit_rule
-  desc: legit rule description
-  condition: evt.type=open
-  output: out 1
-  priority: INFO
-  tags: [tag1]
-
-- rule: another_rule
-  desc: legit rule description
-  condition: evt.type=open
-  output: out 2
-  priority: INFO
-  tags: [tag1]
-)END";
-
-	m_engine->add_extra_output_format("extra 1", "", "tag1", "", false);
-	m_engine->add_extra_output_format("extra 2", "", "", "another_rule", false);
-
-	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
-
-	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 extra 1");
-	EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2 extra 1 extra 2");
-}
-
-TEST_F(test_falco_engine, extra_format_replace_container_info)
-{
-	std::string rules_content = R"END(
-- rule: legit_rule
-  desc: legit rule description
-  condition: evt.type=open
-  output: out 1 (%container.info)
-  priority: INFO
-  tags: [tag1]
-
-- rule: another_rule
-  desc: legit rule description
-  condition: evt.type=open
-  output: out 2
-  priority: INFO
-  tags: [tag1]
-)END";
-
-	m_engine->add_extra_output_format("extra 1", "", "", "", true);
-
-	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
-
-	EXPECT_EQ(get_compiled_rule_output("legit_rule"), "out 1 (extra 1)");
-	EXPECT_EQ(get_compiled_rule_output("another_rule"), "out 2 extra 1");
-}
-
-TEST_F(test_falco_engine, extra_format_do_not_replace_container_info)
-{
-	std::string rules_content = R"END(
-- rule: legit_rule
-  desc: legit rule description
-  condition: evt.type=open
-  output: out 1 (%container.info)
-  priority: INFO
-  tags: [tag1]
-)END";
-	
-	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
-
-	auto output = get_compiled_rule_output("legit_rule");
-	EXPECT_TRUE(output.find("%container.info") == output.npos);
-}
-
-TEST_F(test_falco_engine, extra_fields_all)
-{
-	std::string rules_content = R"END(
-- rule: legit_rule
-  desc: legit rule description
-  condition: evt.type=open
-  output: user=%user.name command=%proc.cmdline file=%fd.name
-  priority: INFO
-)END";
-
-	std::unordered_map<std::string, std::string> extra_formatted_fields = {{"my_field", "hello %evt.num"}};
-  for (auto const& f : extra_formatted_fields)
-  {
-    m_engine->add_extra_output_formatted_field(f.first, f.second, "", "", "");
-  }
-
-	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
-
-	EXPECT_EQ(get_compiled_rule_formatted_fields("legit_rule"), extra_formatted_fields);
-}

unit_tests/engine/test_rule_loader.cpp

@@ -1034,7 +1034,7 @@ TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = proc.pname)");
-  EXPECT_TRUE(check_warning_message("'proc.pname' may be a valid field misused as a const string value"));
+  EXPECT_TRUE(check_warning_message("string 'proc.pname' may be a valid field wrongly interpreted as a string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous_quoted)
@@ -1050,7 +1050,7 @@ TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous_quoted)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = proc.pname)");
-  EXPECT_TRUE(check_warning_message("'proc.pname' may be a valid field misused as a const string value"));
+  EXPECT_TRUE(check_warning_message("string 'proc.pname' may be a valid field wrongly interpreted as a string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous_space_quoted)
@@ -1066,7 +1066,7 @@ TEST_F(test_falco_engine, exceptions_values_rhs_field_ambiguous_space_quoted)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = \"proc.pname \")");
-  EXPECT_TRUE(check_warning_message("'proc.pname ' may be a valid field misused as a const string value"));
+  EXPECT_TRUE(check_warning_message("string 'proc.pname ' may be a valid field wrongly interpreted as a string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_values_rhs_transformer)
@@ -1112,7 +1112,7 @@ TEST_F(test_falco_engine, exceptions_values_transformer_space)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = \"toupper( proc.pname)\")");
-  EXPECT_TRUE(check_warning_message("'toupper( proc.pname)' may be a valid field transformer misused as a const string value"));
+  EXPECT_TRUE(check_warning_message("string 'toupper( proc.pname)' may be a valid field transformer wrongly interpreted as a string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_values_transformer_space_quoted)
@@ -1128,7 +1128,7 @@ TEST_F(test_falco_engine, exceptions_values_transformer_space_quoted)
 
   ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
   EXPECT_EQ(get_compiled_rule_condition("test_rule"), "(evt.type = open and not proc.name = \"toupper( proc.pname)\")");
-  EXPECT_TRUE(check_warning_message("'toupper( proc.pname)' may be a valid field transformer misused as a const string value"));
+  EXPECT_TRUE(check_warning_message("string 'toupper( proc.pname)' may be a valid field transformer wrongly interpreted as a string value"));
 }
 
 TEST_F(test_falco_engine, exceptions_fields_transformer)

unit_tests/falco/test_configuration.cpp

@@ -18,6 +18,12 @@ limitations under the License.
 #include <gtest/gtest.h>
 #include <falco/configuration.h>
 
+#ifdef _WIN32
+#define SET_ENV_VAR(env_var_name, env_var_value) _putenv_s(env_var_name, env_var_value)
+#else
+#define SET_ENV_VAR(env_var_name, env_var_value) setenv(env_var_name, env_var_value, 1)
+#endif
+
 static std::string sample_yaml =
 	"base_value:\n"
 	"    id: 1\n"
@@ -103,6 +109,657 @@ TEST(Configuration, modify_yaml_fields)
 	ASSERT_EQ(conf.get_scalar<bool>(key, false), true);
 }
 
+TEST(Configuration, configuration_config_files_secondary_fail)
+{
+	/* Test that a secondary config file is not able to include anything, triggering an exception. */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - conf_3.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_4.yaml\n"
+					   "foo2: bar2\n"
+					   "base_value_2:\n"
+					   "    id: 2\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_ANY_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+}
+
+TEST(Configuration, configuration_config_files_ok)
+{
+	/* Test that every included config file was correctly parsed */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - conf_3.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo3: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+	const std::string conf_yaml_4 =
+		"base_value_4:\n"
+		"    id: 4\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	outfile.open("conf_4.yaml");
+	outfile << conf_yaml_4;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(loaded_conf_files.size(), 3);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(falco_config.config.is_defined("foo3"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo3", ""), "bar3");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_3.id", 0), 3);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value_3.name", ""), "foo3");
+	ASSERT_FALSE(falco_config.config.is_defined("base_value_4.id")); // conf_4 is not included
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+	std::filesystem::remove("conf_4.yaml");
+}
+
+
+TEST(Configuration, configuration_config_files_relative_main)
+{
+	/*
+	 * Test that relative path are treated as relative to cwd and not to main config folder,
+	 * and that absolute includes are ok too.
+	 */
+	const auto temp_main = std::filesystem::temp_directory_path() / "main.yaml";
+	// So, conf_2 will be looked up in the same folder as main config file,
+	// while conf_3, since is absolute, will be looked up in the absolute path (and found!).
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+		"  - conf_2.yaml\n"
+		"  - " +
+		std::filesystem::current_path().string() + "/conf_3.yaml\n"
+							   "foo: bar\n"
+							   "base_value:\n"
+							   "    id: 1\n"
+							   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo3: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+
+	std::ofstream outfile(temp_main.string());
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_file(temp_main.string(), loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(loaded_conf_files.size(), 3);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_3.id", 0), 3);
+
+	std::filesystem::remove(temp_main.string());
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+}
+
+TEST(Configuration, configuration_config_files_override)
+{
+	/* Test that included config files are able to override configs from main file */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+					   "  - conf_2.yaml\n"
+					   "  - conf_3.yaml\n"
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"base_value:\n"
+		"    id: 3\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open("conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2 + conf_3
+	ASSERT_EQ(loaded_conf_files.size(), 3);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 3); // overridden!
+	ASSERT_FALSE(falco_config.config.is_defined("base_value.name"));	// no more present since entire `base_value` block was overridden
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_FALSE(falco_config.config.is_defined("base_value_3.id")); // not defined
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+	std::filesystem::remove("conf_3.yaml");
+}
+
+TEST(Configuration, configuration_config_files_unexistent)
+{
+	/* Test that including an unexistent file just skips it */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+		"  - conf_5.yaml\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main
+	ASSERT_EQ(loaded_conf_files.size(), 1);
+
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+
+	std::filesystem::remove("main.yaml");
+}
+
+TEST(Configuration, configuration_config_files_scalar_config_files)
+{
+	/* Test that a single file can be included as a scalar (thanks to get_sequence_from_node magic) */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ": conf_2.yaml\n"
+		"foo: bar\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2
+	ASSERT_EQ(loaded_conf_files.size(), 2);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+}
+
+TEST(Configuration, configuration_config_files_empty_config_files)
+{
+	/* Test that empty includes list is accepted */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ":\n"
+		"foo: bar\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main
+	ASSERT_EQ(loaded_conf_files.size(), 1);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+
+	std::filesystem::remove("main.yaml");
+}
+
+TEST(Configuration, configuration_config_files_self)
+{
+	/* Test that main config file cannot include itself */
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ": main.yaml\n"
+		"foo: bar\n"
+		"base_value:\n"
+		"    id: 1\n"
+		"    name: foo\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_ANY_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	std::filesystem::remove("main.yaml");
+}
+
+TEST(Configuration, configuration_config_files_directory)
+{
+	/*
+	 * Test that when main config file includes a config directory,
+	 * the config directory is parsed in lexicographic order,
+	 * and only regular files are parsed.
+	 */
+	// Main config includes whole temp directory
+	const std::string main_conf_yaml =
+		yaml_helper::configs_key + ": " + std::filesystem::temp_directory_path().string() + "/test\n"
+		   "foo: bar\n"
+		   "base_value:\n"
+		   "    id: 1\n"
+		   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+	const std::string conf_yaml_3 =
+		"foo2: bar3\n"
+		"base_value_3:\n"
+		"    id: 3\n"
+		"    name: foo3\n";
+	const std::string conf_yaml_4 =
+		"foo4: bar4\n";
+
+	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test");
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open(std::filesystem::temp_directory_path()/"test/conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	outfile.open(std::filesystem::temp_directory_path()/"test/conf_3.yaml");
+	outfile << conf_yaml_3;
+	outfile.close();
+
+	// Create a directory and create a config inside it. We will later check that it was not parsed
+	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test" / "foo");
+	outfile.open(std::filesystem::temp_directory_path()/"test/foo/conf_4.yaml");
+	outfile << conf_yaml_4;
+	outfile.close();
+
+	std::vector<std::string> cmdline_config_options;
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2 + conf_3.
+	// test/foo is not parsed.
+	ASSERT_EQ(loaded_conf_files.size(), 3);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_3.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_3.id", 0), 3);
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar3");
+	ASSERT_FALSE(falco_config.config.is_defined("foo4"));
+
+	std::filesystem::remove("main");
+	std::filesystem::remove_all(std::filesystem::temp_directory_path()/"test");
+}
+
+TEST(Configuration, configuration_config_files_cmdline)
+{
+	/* Test that we support including configs files from cmdline option */
+	const std::string main_conf_yaml =
+					   "foo: bar\n"
+					   "base_value:\n"
+					   "    id: 1\n"
+					   "    name: foo\n";
+	const std::string conf_yaml_2 =
+		"foo2: bar2\n"
+		"base_value_2:\n"
+		"    id: 2\n";
+
+	std::ofstream outfile("main.yaml");
+	outfile << main_conf_yaml;
+	outfile.close();
+
+	outfile.open("conf_2.yaml");
+	outfile << conf_yaml_2;
+	outfile.close();
+
+	// Pass "config_files=..." cmdline option
+	std::vector<std::string> cmdline_config_options;
+	cmdline_config_options.push_back((yaml_helper::configs_key+"=conf_2.yaml"));
+
+	std::vector<std::string> loaded_conf_files;
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_file("main.yaml", loaded_conf_files, cmdline_config_options));
+
+	// main + conf_2
+	ASSERT_EQ(loaded_conf_files.size(), 2);
+
+	ASSERT_TRUE(falco_config.config.is_defined("foo"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo", ""), "bar");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value.id", 0), 1);
+	ASSERT_TRUE(falco_config.config.is_defined("base_value.name"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("base_value.name", ""), "foo");
+	ASSERT_TRUE(falco_config.config.is_defined("foo2"));
+	ASSERT_EQ(falco_config.config.get_scalar<std::string>("foo2", ""), "bar2");
+	ASSERT_TRUE(falco_config.config.is_defined("base_value_2.id"));
+	ASSERT_EQ(falco_config.config.get_scalar<int>("base_value_2.id", 0), 2);
+
+	std::filesystem::remove("main.yaml");
+	std::filesystem::remove("conf_2.yaml");
+}
+
+TEST(Configuration, configuration_environment_variables)
+{
+    // Set an environment variable for testing purposes
+    std::string env_var_value = "envVarValue";
+    std::string env_var_name = "ENV_VAR";
+    SET_ENV_VAR(env_var_name.c_str(), env_var_value.c_str());
+
+    std::string embedded_env_var_value = "${ENV_VAR}";
+    std::string embedded_env_var_name = "ENV_VAR_EMBEDDED";
+    SET_ENV_VAR(embedded_env_var_name.c_str(), embedded_env_var_value.c_str());
+
+    std::string bool_env_var_value = "true";
+    std::string bool_env_var_name = "ENV_VAR_BOOL";
+    SET_ENV_VAR(bool_env_var_name.c_str(), bool_env_var_value.c_str());
+
+    std::string int_env_var_value = "12";
+    std::string int_env_var_name = "ENV_VAR_INT";
+    SET_ENV_VAR(int_env_var_name.c_str(), int_env_var_value.c_str());
+
+    std::string empty_env_var_value = "";
+    std::string empty_env_var_name = "ENV_VAR_EMPTY";
+    SET_ENV_VAR(empty_env_var_name.c_str(), empty_env_var_value.c_str());
+
+    std::string default_value = "default";
+    std::string env_var_sample_yaml =
+        "base_value:\n"
+        "    id: $ENV_VAR\n"
+        "    name: '${ENV_VAR}'\n"
+        "    string: my_string\n"
+        "    invalid: $${ENV_VAR}\n"
+	"    invalid_env: $$ENV_VAR\n"
+	"    invalid_double_env: $${ENV_VAR}$${ENV_VAR}\n"
+	"    invalid_embedded_env: $${${ENV_VAR}}\n"
+	"    invalid_valid_env: $${ENV_VAR}${ENV_VAR}\n"
+        "    escaped: \"${ENV_VAR}\"\n"
+        "    subvalue:\n"
+        "        subvalue2:\n"
+        "            boolean: ${UNSED_XX_X_X_VAR}\n"
+        "base_value_2:\n"
+        "    sample_list:\n"
+        "        - ${ENV_VAR}\n"
+        "        - ' ${ENV_VAR}'\n"
+	"        - '${ENV_VAR} '\n"
+        "        - $UNSED_XX_X_X_VAR\n"
+        "paths:\n"
+	"    - ${ENV_VAR}/foo\n"
+	"    - $ENV_VAR/foo\n"
+	"    - /foo/${ENV_VAR}/\n"
+	"    - /${ENV_VAR}/${ENV_VAR}${ENV_VAR}/foo\n"
+	"    - ${ENV_VAR_EMBEDDED}/foo\n"
+	"is_test: ${ENV_VAR_BOOL}\n"
+	"num_test: ${ENV_VAR_INT}\n"
+	"empty_test: ${ENV_VAR_EMPTY}\n"
+	"plugins:\n"
+	"  - name: k8saudit\n"
+	"    library_path: /foo/${ENV_VAR}/libk8saudit.so\n"
+	"    open_params: ${ENV_VAR_INT}\n";
+
+    yaml_helper conf;
+    conf.load_from_string(env_var_sample_yaml);
+
+    /* Check if the base values are defined */
+    ASSERT_TRUE(conf.is_defined("base_value"));
+    ASSERT_TRUE(conf.is_defined("base_value_2"));
+    ASSERT_TRUE(conf.is_defined("paths"));
+    ASSERT_FALSE(conf.is_defined("unknown_base_value"));
+
+    /* Test fetching of a regular string without any environment variable */
+    auto base_value_string = conf.get_scalar<std::string>("base_value.string", default_value);
+    ASSERT_EQ(base_value_string, "my_string");
+
+    /* Test fetching of escaped environment variable format. Should return the string as-is after stripping the leading `$` */
+    auto base_value_invalid = conf.get_scalar<std::string>("base_value.invalid", default_value);
+    ASSERT_EQ(base_value_invalid, "${ENV_VAR}");
+
+    /* Test fetching of invalid escaped environment variable format. Should return the string as-is */
+    auto base_value_invalid_env = conf.get_scalar<std::string>("base_value.invalid_env", default_value);
+    ASSERT_EQ(base_value_invalid_env, "$$ENV_VAR");
+
+    /* Test fetching of 2 escaped environment variables side by side. Should return the string as-is after stripping the leading `$` */
+    auto base_value_double_invalid = conf.get_scalar<std::string>("base_value.invalid_double_env", default_value);
+    ASSERT_EQ(base_value_double_invalid, "${ENV_VAR}${ENV_VAR}");
+
+    /*
+     * Test fetching of escaped environment variable format with inside an env variable.
+     * Should return the string as-is after stripping the leading `$` with the resolved env variable within
+     */
+    auto base_value_embedded_invalid = conf.get_scalar<std::string>("base_value.invalid_embedded_env", default_value);
+    ASSERT_EQ(base_value_embedded_invalid, "${" + env_var_value + "}");
+
+    /*
+     * Test fetching of an escaped env variable plus an env variable side by side.
+     * Should return the escaped one trimming the leading `$` plus the second one resolved.
+     */
+    auto base_value_valid_invalid = conf.get_scalar<std::string>("base_value.invalid_valid_env", default_value);
+    ASSERT_EQ(base_value_valid_invalid, "${ENV_VAR}" + env_var_value);
+
+    /* Test fetching of strings that contain environment variables */
+    auto base_value_id = conf.get_scalar<std::string>("base_value.id", default_value);
+    ASSERT_EQ(base_value_id, "$ENV_VAR"); // Does not follow the `${VAR}` format, so it should be treated as a regular string
+
+    auto base_value_name = conf.get_scalar<std::string>("base_value.name", default_value);
+    ASSERT_EQ(base_value_name, env_var_value); // Proper environment variable format
+
+    auto base_value_escaped = conf.get_scalar<std::string>("base_value.escaped", default_value);
+    ASSERT_EQ(base_value_escaped, env_var_value); // Environment variable within quotes
+
+    /* Test fetching of an undefined environment variable. Resolves to empty string. */
+    auto unknown_boolean = conf.get_scalar<std::string>("base_value.subvalue.subvalue2.boolean", default_value);
+    ASSERT_EQ(unknown_boolean, "");
+
+    /* Test fetching of environment variables from a list */
+    auto base_value_2_list_0 = conf.get_scalar<std::string>("base_value_2.sample_list[0]", default_value);
+    ASSERT_EQ(base_value_2_list_0, env_var_value); // Proper environment variable format
+
+    auto base_value_2_list_1 = conf.get_scalar<std::string>("base_value_2.sample_list[1]", default_value);
+    ASSERT_EQ(base_value_2_list_1, " " + env_var_value); // Environment variable preceded by a space, still extracted env var with leading space
+
+    auto base_value_2_list_2 = conf.get_scalar<std::string>("base_value_2.sample_list[2]", default_value);
+    ASSERT_EQ(base_value_2_list_2, env_var_value + " "); // Environment variable followed by a space, still extracted env var with trailing space
+
+    auto base_value_2_list_3 = conf.get_scalar<std::string>("base_value_2.sample_list[3]", default_value);
+    ASSERT_EQ(base_value_2_list_3, "$UNSED_XX_X_X_VAR"); // Does not follow the `${VAR}` format, so should be treated as a regular string
+
+    /* Test expansion of environment variables within strings */
+    auto path_list_0 = conf.get_scalar<std::string>("paths[0]", default_value);
+    ASSERT_EQ(path_list_0, env_var_value + "/foo"); // Even if env var is part of bigger string, it gets expanded
+
+    auto path_list_1 = conf.get_scalar<std::string>("paths[1]", default_value);
+    ASSERT_EQ(path_list_1, "$ENV_VAR/foo"); // Does not follow the `${VAR}` format, so should be treated as a regular string
+
+    auto path_list_2 = conf.get_scalar<std::string>("paths[2]", default_value);
+    ASSERT_EQ(path_list_2, "/foo/" + env_var_value + "/"); // Even when env var is in the middle of a string. it gets expanded
+
+    auto path_list_3 = conf.get_scalar<std::string>("paths[3]", default_value);
+    ASSERT_EQ(path_list_3, "/" + env_var_value + "/" + env_var_value + env_var_value + "/foo"); // Even when the string contains multiple env vars they are correctly expanded
+
+    auto path_list_4 = conf.get_scalar<std::string>("paths[4]", default_value);
+    ASSERT_EQ(path_list_4, env_var_value + "/foo"); // Even when the env var contains another env var, it gets correctly double-expanded
+
+    /* Check that variable expansion is type-aware */
+    auto boolean = conf.get_scalar<bool>("is_test", false);
+    ASSERT_EQ(boolean, true); // `true` can be parsed to bool.
+
+    auto boolean_as_str = conf.get_scalar<std::string>("is_test", "false");
+    ASSERT_EQ(boolean_as_str, "true"); // `true` can be parsed to string.
+
+    auto boolean_as_int = conf.get_scalar<int32_t>("is_test", 0);
+    ASSERT_EQ(boolean_as_int, 0); // `true` cannot be parsed to integer.
+
+    auto integer = conf.get_scalar<int32_t>("num_test", -1);
+    ASSERT_EQ(integer, 12);
+
+    // An env var that resolves to an empty string returns ""
+    auto empty_default_str = conf.get_scalar<std::string>("empty_test", default_value);
+    ASSERT_EQ(empty_default_str, "");
+
+    std::list<falco_configuration::plugin_config> plugins;
+    conf.get_sequence<std::list<falco_configuration::plugin_config>>(plugins, std::string("plugins"));
+    std::vector<falco_configuration::plugin_config> m_plugins{ std::make_move_iterator(std::begin(plugins)),
+							      std::make_move_iterator(std::end(plugins)) };
+    ASSERT_EQ(m_plugins[0].m_name, "k8saudit");
+    ASSERT_EQ(m_plugins[0].m_library_path, "/foo/" + env_var_value + "/libk8saudit.so");
+    ASSERT_EQ(m_plugins[0].m_open_params, "12");
+
+    /* Clear the set environment variables after testing */
+    SET_ENV_VAR(env_var_name.c_str(), "");
+    SET_ENV_VAR(embedded_env_var_name.c_str(), "");
+    SET_ENV_VAR(bool_env_var_name.c_str(), "");
+    SET_ENV_VAR(int_env_var_name.c_str(), "");
+    SET_ENV_VAR(empty_env_var_name.c_str(), "");
+}
+
 TEST(Configuration, configuration_webserver_ip)
 {
     falco_configuration falco_config;

unit_tests/falco/test_configuration_config_files.cpp

@@ -1,496 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-/*
-Copyright (C) 2023 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include <gtest/gtest.h>
-#include <falco/configuration.h>
-
-TEST(Configuration, configuration_config_files_secondary_fail)
-{
-	/* Test that a secondary config file is not able to include anything, triggering an exception. */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_2.yaml\n"
-					   "  - conf_3.yaml\n"
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_4.yaml\n"
-					   "foo2: bar2\n"
-					   "base_value_2:\n"
-					   "    id: 2\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	falco_configuration falco_config;
-	ASSERT_ANY_THROW(falco_config.init_from_file("main.yaml", cmdline_config_options));
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-}
-
-TEST(Configuration, configuration_config_files_ok)
-{
-	/* Test that every included config file was correctly parsed */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_2.yaml\n"
-					   "  - conf_3.yaml\n"
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-	const std::string conf_yaml_3 =
-		"foo3: bar3\n"
-		"base_value_3:\n"
-		"    id: 3\n"
-		"    name: foo3\n";
-	const std::string conf_yaml_4 =
-		"base_value_4:\n"
-		"    id: 4\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	outfile.open("conf_3.yaml");
-	outfile << conf_yaml_3;
-	outfile.close();
-
-	outfile.open("conf_4.yaml");
-	outfile << conf_yaml_4;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	std::vector<std::string> loaded_conf_files;
-	falco_configuration falco_config;
-	config_loaded_res res;
-	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
-
-	// main + conf_2 + conf_3
-	ASSERT_EQ(res.size(), 3);
-
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo3"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo3", ""), "bar3");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_3.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_3.id", 0), 3);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_3.name"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value_3.name", ""), "foo3");
-	ASSERT_FALSE(falco_config.m_config.is_defined("base_value_4.id")); // conf_4 is not included
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-	std::filesystem::remove("conf_3.yaml");
-	std::filesystem::remove("conf_4.yaml");
-}
-
-
-TEST(Configuration, configuration_config_files_relative_main)
-{
-	/*
-	 * Test that relative path are treated as relative to cwd and not to main config folder,
-	 * and that absolute includes are ok too.
-	 */
-	const auto temp_main = std::filesystem::temp_directory_path() / "main.yaml";
-	// So, conf_2 will be looked up in the same folder as main config file,
-	// while conf_3, since is absolute, will be looked up in the absolute path (and found!).
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_2.yaml\n"
-					   "  - " +
-		std::filesystem::current_path().string() + "/conf_3.yaml\n"
-							   "foo: bar\n"
-							   "base_value:\n"
-							   "    id: 1\n"
-							   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-	const std::string conf_yaml_3 =
-		"foo3: bar3\n"
-		"base_value_3:\n"
-		"    id: 3\n"
-		"    name: foo3\n";
-
-	std::ofstream outfile(temp_main.string());
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	outfile.open("conf_3.yaml");
-	outfile << conf_yaml_3;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	falco_configuration falco_config;
-	config_loaded_res res;
-	ASSERT_NO_THROW(res = falco_config.init_from_file(temp_main.string(), cmdline_config_options));
-
-	// main + conf_2 + conf_3
-	ASSERT_EQ(res.size(), 3);
-
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_3.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_3.id", 0), 3);
-
-	std::filesystem::remove(temp_main.string());
-	std::filesystem::remove("conf_2.yaml");
-	std::filesystem::remove("conf_3.yaml");
-}
-
-TEST(Configuration, configuration_config_files_override)
-{
-	/* Test that included config files are able to override configs from main file */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_2.yaml\n"
-					   "  - conf_3.yaml\n"
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-	const std::string conf_yaml_3 =
-		"base_value:\n"
-		"    id: 3\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	outfile.open("conf_3.yaml");
-	outfile << conf_yaml_3;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	falco_configuration falco_config;
-	config_loaded_res res;
-	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
-
-	// main + conf_2 + conf_3
-	ASSERT_EQ(res.size(), 3);
-
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 3); // overridden!
-	ASSERT_FALSE(falco_config.m_config.is_defined("base_value.name"));	// no more present since entire `base_value` block was overridden
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
-	ASSERT_FALSE(falco_config.m_config.is_defined("base_value_3.id")); // not defined
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-	std::filesystem::remove("conf_3.yaml");
-}
-
-TEST(Configuration, configuration_config_files_unexistent)
-{
-	/* Test that including an unexistent file just skips it */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-					   "  - conf_5.yaml\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	falco_configuration falco_config;
-	config_loaded_res res;
-	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
-
-	// main
-	ASSERT_EQ(res.size(), 1);
-
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
-
-	std::filesystem::remove("main.yaml");
-}
-
-TEST(Configuration, configuration_config_files_scalar_config_files)
-{
-	/* Test that a single file can be included as a scalar (thanks to get_sequence_from_node magic) */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ": conf_2.yaml\n"
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	falco_configuration falco_config;
-	config_loaded_res res;
-	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
-
-	// main + conf_2
-	ASSERT_EQ(res.size(), 2);
-
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-}
-
-TEST(Configuration, configuration_config_files_empty_config_files)
-{
-	/* Test that empty includes list is accepted */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ":\n"
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	falco_configuration falco_config;
-	config_loaded_res res;
-	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
-
-	// main
-	ASSERT_EQ(res.size(), 1);
-
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
-
-	std::filesystem::remove("main.yaml");
-}
-
-TEST(Configuration, configuration_config_files_self)
-{
-	/* Test that main config file cannot include itself */
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ": main.yaml\n"
-					   "foo: bar\n"
-					   "base_value:\n"
-					   "    id: 1\n"
-					   "    name: foo\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	falco_configuration falco_config;
-	ASSERT_ANY_THROW(falco_config.init_from_file("main.yaml", cmdline_config_options));
-
-	std::filesystem::remove("main.yaml");
-}
-
-TEST(Configuration, configuration_config_files_directory)
-{
-	/*
-	 * Test that when main config file includes a config directory,
-	 * the config directory is parsed in lexicographic order,
-	 * and only regular files are parsed.
-	 */
-	// Main config includes whole temp directory
-	const std::string main_conf_yaml =
-		yaml_helper::configs_key + ": " + std::filesystem::temp_directory_path().string() + "/test\n"
-												    "foo: bar\n"
-												    "base_value:\n"
-												    "    id: 1\n"
-												    "    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-	const std::string conf_yaml_3 =
-		"foo2: bar3\n"
-		"base_value_3:\n"
-		"    id: 3\n"
-		"    name: foo3\n";
-	const std::string conf_yaml_4 =
-		"foo4: bar4\n";
-
-	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test");
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open(std::filesystem::temp_directory_path()/"test/conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	outfile.open(std::filesystem::temp_directory_path()/"test/conf_3.yaml");
-	outfile << conf_yaml_3;
-	outfile.close();
-
-	// Create a directory and create a config inside it. We will later check that it was not parsed
-	std::filesystem::create_directory(std::filesystem::temp_directory_path() / "test" / "foo");
-	outfile.open(std::filesystem::temp_directory_path()/"test/foo/conf_4.yaml");
-	outfile << conf_yaml_4;
-	outfile.close();
-
-	std::vector<std::string> cmdline_config_options;
-	falco_configuration falco_config;
-	config_loaded_res res;
-	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
-
-	// main + conf_2 + conf_3.
-	// test/foo is not parsed.
-	ASSERT_EQ(res.size(), 3);
-
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_3.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_3.id", 0), 3);
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar3");
-	ASSERT_FALSE(falco_config.m_config.is_defined("foo4"));
-
-	std::filesystem::remove("main");
-	std::filesystem::remove_all(std::filesystem::temp_directory_path()/"test");
-}
-
-TEST(Configuration, configuration_config_files_cmdline)
-{
-	/* Test that we support including configs files from cmdline option */
-	const std::string main_conf_yaml =
-		"foo: bar\n"
-		"base_value:\n"
-		"    id: 1\n"
-		"    name: foo\n";
-	const std::string conf_yaml_2 =
-		"foo2: bar2\n"
-		"base_value_2:\n"
-		"    id: 2\n";
-
-	std::ofstream outfile("main.yaml");
-	outfile << main_conf_yaml;
-	outfile.close();
-
-	outfile.open("conf_2.yaml");
-	outfile << conf_yaml_2;
-	outfile.close();
-
-	// Pass "config_files=..." cmdline option
-	std::vector<std::string> cmdline_config_options;
-	cmdline_config_options.push_back((yaml_helper::configs_key+"=conf_2.yaml"));
-
-	falco_configuration falco_config;
-	config_loaded_res res;
-	ASSERT_NO_THROW(res = falco_config.init_from_file("main.yaml", cmdline_config_options));
-
-	// main + conf_2
-	ASSERT_EQ(res.size(), 2);
-
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo", ""), "bar");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value.id", 0), 1);
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value.name"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("base_value.name", ""), "foo");
-	ASSERT_TRUE(falco_config.m_config.is_defined("foo2"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<std::string>("foo2", ""), "bar2");
-	ASSERT_TRUE(falco_config.m_config.is_defined("base_value_2.id"));
-	ASSERT_EQ(falco_config.m_config.get_scalar<int>("base_value_2.id", 0), 2);
-
-	std::filesystem::remove("main.yaml");
-	std::filesystem::remove("conf_2.yaml");
-}

unit_tests/falco/test_configuration_env_vars.cpp

@@ -1,198 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-/*
-Copyright (C) 2023 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include <gtest/gtest.h>
-#include <falco/configuration.h>
-
-#ifdef _WIN32
-#define SET_ENV_VAR(env_var_name, env_var_value) _putenv_s(env_var_name, env_var_value)
-#else
-#define SET_ENV_VAR(env_var_name, env_var_value) setenv(env_var_name, env_var_value, 1)
-#endif
-
-TEST(Configuration, configuration_environment_variables)
-{
-	// Set an environment variable for testing purposes
-	std::string env_var_value = "envVarValue";
-	std::string env_var_name = "ENV_VAR";
-	SET_ENV_VAR(env_var_name.c_str(), env_var_value.c_str());
-
-	std::string embedded_env_var_value = "${ENV_VAR}";
-	std::string embedded_env_var_name = "ENV_VAR_EMBEDDED";
-	SET_ENV_VAR(embedded_env_var_name.c_str(), embedded_env_var_value.c_str());
-
-	std::string bool_env_var_value = "true";
-	std::string bool_env_var_name = "ENV_VAR_BOOL";
-	SET_ENV_VAR(bool_env_var_name.c_str(), bool_env_var_value.c_str());
-
-	std::string int_env_var_value = "12";
-	std::string int_env_var_name = "ENV_VAR_INT";
-	SET_ENV_VAR(int_env_var_name.c_str(), int_env_var_value.c_str());
-
-	std::string empty_env_var_value = "";
-	std::string empty_env_var_name = "ENV_VAR_EMPTY";
-	SET_ENV_VAR(empty_env_var_name.c_str(), empty_env_var_value.c_str());
-
-	std::string default_value = "default";
-	std::string env_var_sample_yaml =
-		"base_value:\n"
-		"    id: $ENV_VAR\n"
-		"    name: '${ENV_VAR}'\n"
-		"    string: my_string\n"
-		"    invalid: $${ENV_VAR}\n"
-		"    invalid_env: $$ENV_VAR\n"
-		"    invalid_double_env: $${ENV_VAR}$${ENV_VAR}\n"
-		"    invalid_embedded_env: $${${ENV_VAR}}\n"
-		"    invalid_valid_env: $${ENV_VAR}${ENV_VAR}\n"
-		"    escaped: \"${ENV_VAR}\"\n"
-		"    subvalue:\n"
-		"        subvalue2:\n"
-		"            boolean: ${UNSED_XX_X_X_VAR}\n"
-		"base_value_2:\n"
-		"    sample_list:\n"
-		"        - ${ENV_VAR}\n"
-		"        - ' ${ENV_VAR}'\n"
-		"        - '${ENV_VAR} '\n"
-		"        - $UNSED_XX_X_X_VAR\n"
-		"paths:\n"
-		"    - ${ENV_VAR}/foo\n"
-		"    - $ENV_VAR/foo\n"
-		"    - /foo/${ENV_VAR}/\n"
-		"    - /${ENV_VAR}/${ENV_VAR}${ENV_VAR}/foo\n"
-		"    - ${ENV_VAR_EMBEDDED}/foo\n"
-		"is_test: ${ENV_VAR_BOOL}\n"
-		"num_test: ${ENV_VAR_INT}\n"
-		"empty_test: ${ENV_VAR_EMPTY}\n"
-		"plugins:\n"
-		"  - name: k8saudit\n"
-		"    library_path: /foo/${ENV_VAR}/libk8saudit.so\n"
-		"    open_params: ${ENV_VAR_INT}\n";
-
-	yaml_helper conf;
-	conf.load_from_string(env_var_sample_yaml);
-
-	/* Check if the base values are defined */
-	ASSERT_TRUE(conf.is_defined("base_value"));
-	ASSERT_TRUE(conf.is_defined("base_value_2"));
-	ASSERT_TRUE(conf.is_defined("paths"));
-	ASSERT_FALSE(conf.is_defined("unknown_base_value"));
-
-	/* Test fetching of a regular string without any environment variable */
-	auto base_value_string = conf.get_scalar<std::string>("base_value.string", default_value);
-	ASSERT_EQ(base_value_string, "my_string");
-
-	/* Test fetching of escaped environment variable format. Should return the string as-is after stripping the leading `$` */
-	auto base_value_invalid = conf.get_scalar<std::string>("base_value.invalid", default_value);
-	ASSERT_EQ(base_value_invalid, "${ENV_VAR}");
-
-	/* Test fetching of invalid escaped environment variable format. Should return the string as-is */
-	auto base_value_invalid_env = conf.get_scalar<std::string>("base_value.invalid_env", default_value);
-	ASSERT_EQ(base_value_invalid_env, "$$ENV_VAR");
-
-	/* Test fetching of 2 escaped environment variables side by side. Should return the string as-is after stripping the leading `$` */
-	auto base_value_double_invalid = conf.get_scalar<std::string>("base_value.invalid_double_env", default_value);
-	ASSERT_EQ(base_value_double_invalid, "${ENV_VAR}${ENV_VAR}");
-
-	/*
-     * Test fetching of escaped environment variable format with inside an env variable.
-     * Should return the string as-is after stripping the leading `$` with the resolved env variable within
-	 */
-	auto base_value_embedded_invalid = conf.get_scalar<std::string>("base_value.invalid_embedded_env", default_value);
-	ASSERT_EQ(base_value_embedded_invalid, "${" + env_var_value + "}");
-
-	/*
-     * Test fetching of an escaped env variable plus an env variable side by side.
-     * Should return the escaped one trimming the leading `$` plus the second one resolved.
-	 */
-	auto base_value_valid_invalid = conf.get_scalar<std::string>("base_value.invalid_valid_env", default_value);
-	ASSERT_EQ(base_value_valid_invalid, "${ENV_VAR}" + env_var_value);
-
-	/* Test fetching of strings that contain environment variables */
-	auto base_value_id = conf.get_scalar<std::string>("base_value.id", default_value);
-	ASSERT_EQ(base_value_id, "$ENV_VAR"); // Does not follow the `${VAR}` format, so it should be treated as a regular string
-
-	auto base_value_name = conf.get_scalar<std::string>("base_value.name", default_value);
-	ASSERT_EQ(base_value_name, env_var_value); // Proper environment variable format
-
-	auto base_value_escaped = conf.get_scalar<std::string>("base_value.escaped", default_value);
-	ASSERT_EQ(base_value_escaped, env_var_value); // Environment variable within quotes
-
-	/* Test fetching of an undefined environment variable. Resolves to empty string. */
-	auto unknown_boolean = conf.get_scalar<std::string>("base_value.subvalue.subvalue2.boolean", default_value);
-	ASSERT_EQ(unknown_boolean, "");
-
-	/* Test fetching of environment variables from a list */
-	auto base_value_2_list_0 = conf.get_scalar<std::string>("base_value_2.sample_list[0]", default_value);
-	ASSERT_EQ(base_value_2_list_0, env_var_value); // Proper environment variable format
-
-	auto base_value_2_list_1 = conf.get_scalar<std::string>("base_value_2.sample_list[1]", default_value);
-	ASSERT_EQ(base_value_2_list_1, " " + env_var_value); // Environment variable preceded by a space, still extracted env var with leading space
-
-	auto base_value_2_list_2 = conf.get_scalar<std::string>("base_value_2.sample_list[2]", default_value);
-	ASSERT_EQ(base_value_2_list_2, env_var_value + " "); // Environment variable followed by a space, still extracted env var with trailing space
-
-	auto base_value_2_list_3 = conf.get_scalar<std::string>("base_value_2.sample_list[3]", default_value);
-	ASSERT_EQ(base_value_2_list_3, "$UNSED_XX_X_X_VAR"); // Does not follow the `${VAR}` format, so should be treated as a regular string
-
-	/* Test expansion of environment variables within strings */
-	auto path_list_0 = conf.get_scalar<std::string>("paths[0]", default_value);
-	ASSERT_EQ(path_list_0, env_var_value + "/foo"); // Even if env var is part of bigger string, it gets expanded
-
-	auto path_list_1 = conf.get_scalar<std::string>("paths[1]", default_value);
-	ASSERT_EQ(path_list_1, "$ENV_VAR/foo"); // Does not follow the `${VAR}` format, so should be treated as a regular string
-
-	auto path_list_2 = conf.get_scalar<std::string>("paths[2]", default_value);
-	ASSERT_EQ(path_list_2, "/foo/" + env_var_value + "/"); // Even when env var is in the middle of a string. it gets expanded
-
-	auto path_list_3 = conf.get_scalar<std::string>("paths[3]", default_value);
-	ASSERT_EQ(path_list_3, "/" + env_var_value + "/" + env_var_value + env_var_value + "/foo"); // Even when the string contains multiple env vars they are correctly expanded
-
-	auto path_list_4 = conf.get_scalar<std::string>("paths[4]", default_value);
-	ASSERT_EQ(path_list_4, env_var_value + "/foo"); // Even when the env var contains another env var, it gets correctly double-expanded
-
-	/* Check that variable expansion is type-aware */
-	auto boolean = conf.get_scalar<bool>("is_test", false);
-	ASSERT_EQ(boolean, true); // `true` can be parsed to bool.
-
-	auto boolean_as_str = conf.get_scalar<std::string>("is_test", "false");
-	ASSERT_EQ(boolean_as_str, "true"); // `true` can be parsed to string.
-
-	auto boolean_as_int = conf.get_scalar<int32_t>("is_test", 0);
-	ASSERT_EQ(boolean_as_int, 0); // `true` cannot be parsed to integer.
-
-	auto integer = conf.get_scalar<int32_t>("num_test", -1);
-	ASSERT_EQ(integer, 12);
-
-	// An env var that resolves to an empty string returns ""
-	auto empty_default_str = conf.get_scalar<std::string>("empty_test", default_value);
-	ASSERT_EQ(empty_default_str, "");
-
-	std::list<falco_configuration::plugin_config> plugins;
-	conf.get_sequence<std::list<falco_configuration::plugin_config>>(plugins, std::string("plugins"));
-	std::vector<falco_configuration::plugin_config> m_plugins{ std::make_move_iterator(std::begin(plugins)),
-								  std::make_move_iterator(std::end(plugins)) };
-	ASSERT_EQ(m_plugins[0].m_name, "k8saudit");
-	ASSERT_EQ(m_plugins[0].m_library_path, "/foo/" + env_var_value + "/libk8saudit.so");
-	ASSERT_EQ(m_plugins[0].m_open_params, "12");
-
-	/* Clear the set environment variables after testing */
-	SET_ENV_VAR(env_var_name.c_str(), "");
-	SET_ENV_VAR(embedded_env_var_name.c_str(), "");
-	SET_ENV_VAR(bool_env_var_name.c_str(), "");
-	SET_ENV_VAR(int_env_var_name.c_str(), "");
-	SET_ENV_VAR(empty_env_var_name.c_str(), "");
-}

unit_tests/falco/test_configuration_output_options.cpp

@@ -1,103 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-/*
-Copyright (C) 2024 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include <gtest/gtest.h>
-#include <falco/configuration.h>
-
-TEST(ConfigurationRuleOutputOptions, parse_yaml)
-{
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_content(R"(
-append_output:
-  - source: syscall
-    tag: persistence
-    rule: some rule name
-    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
-
-  - tag: persistence
-    fields:
-      - proc.aname[2]: "%proc.aname[2]"
-      - proc.aname[3]: "%proc.aname[3]"
-      - proc.aname[4]: "%proc.aname[4]"
-    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
-
-  - source: k8s_audit
-    fields:
-      - ka.verb
-      - static_field: "static content"
-
-	)", {}));
-
-	EXPECT_EQ(falco_config.m_append_output.size(), 3);
-
-	EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall");
-	EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence");
-	EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name");
-	EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0);
-	EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
-
-	EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence");
-	EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
-
-	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3);
-	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[2]"], "%proc.aname[2]");
-	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[3]"], "%proc.aname[3]");
-	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[4]"], "%proc.aname[4]");
-
-	EXPECT_EQ(falco_config.m_append_output[2].m_source, "k8s_audit");
-
-	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields.size(), 1);
-	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields["static_field"], "static content");
-
-	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1);
-	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1);
-}
-
-TEST(ConfigurationRuleOutputOptions, cli_options)
-{
-	falco_configuration falco_config;
-
-	ASSERT_NO_THROW(falco_config.init_from_content("", 
-		std::vector<std::string>{
-			R"(append_output[]={"source": "syscall", "tag": "persistence", "rule": "some rule name", "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
-			R"(append_output[]={"tag": "persistence", "fields": [{"proc.aname[2]": "%proc.aname[2]"}, {"proc.aname[3]": "%proc.aname[3]"}, {"proc.aname[4]": "%proc.aname[4]"}], "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
-			R"(append_output[]={"source": "k8s_audit", "fields": ["ka.verb", {"static_field": "static content"}]})"}));
-
-	EXPECT_EQ(falco_config.m_append_output.size(), 3);
-
-	EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall");
-	EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence");
-	EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name");
-	EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0);
-	EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
-
-	EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence");
-	EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
-
-	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3);
-	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[2]"], "%proc.aname[2]");
-	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[3]"], "%proc.aname[3]");
-	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[4]"], "%proc.aname[4]");
-
-	EXPECT_EQ(falco_config.m_append_output[2].m_source, "k8s_audit");
-
-	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields.size(), 1);
-	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields["static_field"], "static content");
-
-	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1);
-	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1);
-}

unit_tests/falco/test_configuration_rule_selection.cpp

@@ -21,7 +21,7 @@ limitations under the License.
 TEST(ConfigurationRuleSelection, parse_yaml)
 {
 	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_content(R"(
+	EXPECT_NO_THROW(falco_config.init_from_content(R"(
 rules:
   - enable:
       rule: 'Terminal Shell in Container'
@@ -33,42 +33,28 @@ rules:
       rule: 'hello*'
 	)", {}));
 
-	EXPECT_EQ(falco_config.m_rules_selection.size(), 3);
+	ASSERT_EQ(falco_config.m_rules_selection.size(), 3);
 
-	EXPECT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::enable);
-	EXPECT_EQ(falco_config.m_rules_selection[0].m_rule, "Terminal Shell in Container");
+	ASSERT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::enable);
+	ASSERT_EQ(falco_config.m_rules_selection[0].m_rule, "Terminal Shell in Container");
 
-	EXPECT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::disable);
-	EXPECT_EQ(falco_config.m_rules_selection[1].m_tag, "experimental");
+	ASSERT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::disable);
+	ASSERT_EQ(falco_config.m_rules_selection[1].m_tag, "experimental");
 
-	EXPECT_EQ(falco_config.m_rules_selection[2].m_op, falco_configuration::rule_selection_operation::enable);
-	EXPECT_EQ(falco_config.m_rules_selection[2].m_rule, "hello*");
+	ASSERT_EQ(falco_config.m_rules_selection[2].m_op, falco_configuration::rule_selection_operation::enable);
+	ASSERT_EQ(falco_config.m_rules_selection[2].m_rule, "hello*");
 }
 
 TEST(ConfigurationRuleSelection, cli_options)
 {
 	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_content("", std::vector<std::string>{"rules[].disable.tag=maturity_incubating", "rules[].enable.rule=Adding ssh keys to authorized_keys"}));
+	EXPECT_NO_THROW(falco_config.init_from_content("", std::vector<std::string>{"rules[].disable.tag=maturity_incubating", "rules[].enable.rule=Adding ssh keys to authorized_keys"}));
 
-	EXPECT_EQ(falco_config.m_rules_selection.size(), 2);
+	ASSERT_EQ(falco_config.m_rules_selection.size(), 2);
 
-	EXPECT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::disable);
-	EXPECT_EQ(falco_config.m_rules_selection[0].m_tag, "maturity_incubating");
+	ASSERT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::disable);
+	ASSERT_EQ(falco_config.m_rules_selection[0].m_tag, "maturity_incubating");
 
-	EXPECT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::enable);
-	EXPECT_EQ(falco_config.m_rules_selection[1].m_rule, "Adding ssh keys to authorized_keys");
-}
-
-TEST(ConfigurationRuleSelection, cli_options_object)
-{
-	falco_configuration falco_config;
-	ASSERT_NO_THROW(falco_config.init_from_content("", std::vector<std::string>{R"(rules[]={"disable": {"tag": "maturity_incubating"}})", R"(rules[]={"enable": {"rule": "Adding ssh keys to authorized_keys"}})"}));
-
-	EXPECT_EQ(falco_config.m_rules_selection.size(), 2);
-
-	EXPECT_EQ(falco_config.m_rules_selection[0].m_op, falco_configuration::rule_selection_operation::disable);
-	EXPECT_EQ(falco_config.m_rules_selection[0].m_tag, "maturity_incubating");
-
-	EXPECT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::enable);
-	EXPECT_EQ(falco_config.m_rules_selection[1].m_rule, "Adding ssh keys to authorized_keys");
+	ASSERT_EQ(falco_config.m_rules_selection[1].m_op, falco_configuration::rule_selection_operation::enable);
+	ASSERT_EQ(falco_config.m_rules_selection[1].m_rule, "Adding ssh keys to authorized_keys");
 }

unit_tests/falco/test_configuration_schema.cpp

@@ -1,124 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-/*
-Copyright (C) 2023 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include <gtest/gtest.h>
-#include <falco/configuration.h>
-#include <falco_test_var.h>
-
-#define EXPECT_VALIDATION_STATUS(res, status) \
-        do { \
-		for(const auto& pair : res) { \
-			auto validation_status = pair.second; \
-			EXPECT_TRUE(sinsp_utils::startswith(validation_status, status)) << validation_status; \
-		}                                           \
-	} \
-	while (0)
-
-// Read Falco config from current repo-path
-TEST(Configuration, schema_validate_config)
-{
-	falco_configuration falco_config;
-	config_loaded_res res;
-
-	if (!std::filesystem::exists(TEST_FALCO_CONFIG))
-	{
-		GTEST_SKIP() << "Falco config not present under " << TEST_FALCO_CONFIG;
-	}
-	EXPECT_NO_THROW(res = falco_config.init_from_file(TEST_FALCO_CONFIG, {}));
-	EXPECT_VALIDATION_STATUS(res, yaml_helper::validation_ok);
-}
-
-TEST(Configuration, schema_ok)
-{
-	falco_configuration falco_config;
-	config_loaded_res res;
-
-	/* OK YAML */
-	std::string config =
-		"falco_libs:\n"
-		"    thread_table_size: 50\n";
-
-	EXPECT_NO_THROW(res = falco_config.init_from_content(config, {}));
-	EXPECT_VALIDATION_STATUS(res, yaml_helper::validation_ok);
-}
-
-TEST(Configuration, schema_wrong_key)
-{
-	falco_configuration falco_config;
-	config_loaded_res res;
-
-	/* Miss-typed key YAML */
-	std::string config =
-		"falco_libss:\n"
-		"    thread_table_size: 50\n";
-
-	EXPECT_NO_THROW(res = falco_config.init_from_content(config, {}));
-	EXPECT_VALIDATION_STATUS(res, yaml_helper::validation_failed);
-}
-
-TEST(Configuration, schema_wrong_type)
-{
-	falco_configuration falco_config;
-
-	/* Wrong value type YAML */
-	std::string config =
-		"falco_libs: 512\n";
-
-	// We expect an exception since `falco_configuration::load_yaml()`
-	// will fail to parse `falco_libs` node.
-	ASSERT_ANY_THROW(falco_config.init_from_content(config, {}));
-}
-
-TEST(Configuration, schema_wrong_embedded_key)
-{
-	falco_configuration falco_config;
-	config_loaded_res res;
-
-	/* Miss-typed sub-key YAML */
-	std::string config =
-		"falco_libs:\n"
-		"    thread_table_sizeee: 50\n";
-
-	EXPECT_NO_THROW(res = falco_config.init_from_content(config, {}));
-	EXPECT_VALIDATION_STATUS(res, yaml_helper::validation_failed);
-}
-
-TEST(Configuration, schema_yaml_helper_validator)
-{
-	yaml_helper conf;
-	falco_configuration falco_config;
-
-	/* Broken YAML */
-	std::string sample_yaml =
-		"falco_libs:\n"
-		"    thread_table_size: 50\n";
-
-	// Ok, we don't ask for any validation
-	EXPECT_NO_THROW(conf.load_from_string(sample_yaml));
-
-	// We pass a string variable but not a schema
-	std::string validation;
-	EXPECT_NO_THROW(conf.load_from_string(sample_yaml, nlohmann::json{}, &validation));
-	EXPECT_EQ(validation, yaml_helper::validation_none);
-
-	// We pass a schema but not a string storage for the validation; no validation takes place
-	EXPECT_NO_THROW(conf.load_from_string(sample_yaml, falco_config.m_config_schema, nullptr));
-
-	// We pass everything
-	EXPECT_NO_THROW(conf.load_from_string(sample_yaml, falco_config.m_config_schema, &validation));
-	EXPECT_EQ(validation, yaml_helper::validation_ok);
-}

unit_tests/falco_test_var.h.in

@@ -2,4 +2,3 @@
 
 #define TEST_ENGINE_KMOD_CONFIG "${CMAKE_SOURCE_DIR}/unit_tests/falco/test_configs/engine_kmod_config.yaml"
 #define TEST_ENGINE_MODERN_CONFIG "${CMAKE_SOURCE_DIR}/unit_tests/falco/test_configs/engine_modern_config.yaml"
-#define TEST_FALCO_CONFIG "${CMAKE_SOURCE_DIR}/falco.yaml"

unit_tests/test_falco_engine.cpp

@@ -85,15 +85,3 @@ std::string test_falco_engine::get_compiled_rule_condition(std::string rule_name
 	auto rule_description = m_engine->describe_rule(&rule_name, {});
 	return rule_description["rules"][0]["details"]["condition_compiled"].template get<std::string>();
 }
-
-std::string test_falco_engine::get_compiled_rule_output(std::string rule_name) const
-{
-	auto rule_description = m_engine->describe_rule(&rule_name, {});
-	return rule_description["rules"][0]["details"]["output_compiled"].template get<std::string>();
-}
-
-std::unordered_map<std::string, std::string> test_falco_engine::get_compiled_rule_formatted_fields(std::string rule_name) const
-{
-	auto rule_description = m_engine->describe_rule(&rule_name, {});
-	return rule_description["rules"][0]["details"]["extra_output_formatted_fields"].template get<std::unordered_map<std::string, std::string>>();
-}

unit_tests/test_falco_engine.h

@@ -6,7 +6,6 @@
 #include "rule_loading_messages.h"
 
 #include <gtest/gtest.h>
-#include <unordered_map>
 
 class test_falco_engine : public testing::Test
 {
@@ -20,8 +19,6 @@ protected:
 	bool check_warning_message(const std::string& warning_msg) const;
 	bool check_error_message(const std::string& error_msg) const;
 	std::string get_compiled_rule_condition(std::string rule_name = "") const;
-	std::string get_compiled_rule_output(std::string rule_name = "") const;
-	std::unordered_map<std::string, std::string> get_compiled_rule_formatted_fields(std::string rule_name) const;
 
 	std::string m_sample_ruleset = "sample-ruleset";
 	std::string m_sample_source = falco_common::syscall_source;

userspace/engine/evttype_index_ruleset.cpp

@@ -17,13 +17,14 @@ limitations under the License.
 
 #include "evttype_index_ruleset.h"
 
+#include "falco_utils.h"
+
 #include "logger.h"
 
 #include <algorithm>
 
 evttype_index_ruleset::evttype_index_ruleset(
-	std::shared_ptr<sinsp_filter_factory> f):
-	m_filter_factory(f)
+	std::shared_ptr<sinsp_filter_factory> f): m_filter_factory(f)
 {
 }
 
@@ -31,6 +32,170 @@ evttype_index_ruleset::~evttype_index_ruleset()
 {
 }
 
+evttype_index_ruleset::ruleset_filters::ruleset_filters()
+{
+}
+
+evttype_index_ruleset::ruleset_filters::~ruleset_filters()
+{
+}
+
+void evttype_index_ruleset::ruleset_filters::add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
+{
+	// This is O(n) but it's also uncommon
+	// (when loading rules only).
+	auto pos = std::find(wrappers.begin(),
+			     wrappers.end(),
+			     wrap);
+
+	if(pos == wrappers.end())
+	{
+		wrappers.push_back(wrap);
+	}
+}
+
+void evttype_index_ruleset::ruleset_filters::remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
+{
+	// This is O(n) but it's also uncommon
+	// (when loading rules only).
+	auto pos = std::find(wrappers.begin(),
+			     wrappers.end(),
+			     wrap);
+	if(pos != wrappers.end())
+	{
+		wrappers.erase(pos);
+	}
+}
+
+void evttype_index_ruleset::ruleset_filters::add_filter(std::shared_ptr<filter_wrapper> wrap)
+{
+	if(wrap->event_codes.empty())
+	{
+		// Should run for all event types
+		add_wrapper_to_list(m_filter_all_event_types, wrap);
+	}
+	else
+	{
+		for(auto &etype : wrap->event_codes)
+		{
+			if(m_filter_by_event_type.size() <= etype)
+			{
+				m_filter_by_event_type.resize(etype + 1);
+			}
+
+			add_wrapper_to_list(m_filter_by_event_type[etype], wrap);
+		}
+	}
+
+	m_filters.insert(wrap);
+}
+
+void evttype_index_ruleset::ruleset_filters::remove_filter(std::shared_ptr<filter_wrapper> wrap)
+{
+	if(wrap->event_codes.empty())
+	{
+		remove_wrapper_from_list(m_filter_all_event_types, wrap);
+	}
+	else
+	{
+		for(auto &etype : wrap->event_codes)
+		{
+			if( etype < m_filter_by_event_type.size() )
+			{
+				remove_wrapper_from_list(m_filter_by_event_type[etype], wrap);
+			}
+		}
+	}
+
+	m_filters.erase(wrap);
+}
+
+uint64_t evttype_index_ruleset::ruleset_filters::num_filters()
+{
+	return m_filters.size();
+}
+
+bool evttype_index_ruleset::ruleset_filters::run(sinsp_evt *evt, falco_rule& match)
+{
+    if(evt->get_type() < m_filter_by_event_type.size())
+    {
+        for(const auto &wrap : m_filter_by_event_type[evt->get_type()])
+        {
+            if(wrap->filter->run(evt))
+            {
+				match = wrap->rule;
+                return true;
+            }
+        }
+    }
+
+	// Finally, try filters that are not specific to an event type.
+	for(const auto &wrap : m_filter_all_event_types)
+	{
+		if(wrap->filter->run(evt))
+		{
+			match = wrap->rule;
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool evttype_index_ruleset::ruleset_filters::run(sinsp_evt *evt, std::vector<falco_rule>& matches)
+{
+	bool match_found = false;
+
+	if(evt->get_type() < m_filter_by_event_type.size())
+	{
+		for(const auto &wrap : m_filter_by_event_type[evt->get_type()])
+		{
+			if(wrap->filter->run(evt))
+			{
+				matches.push_back(wrap->rule);
+				match_found = true;
+			}
+		}
+	}
+
+	if(match_found)
+	{
+		return true;
+	}
+
+	// Finally, try filters that are not specific to an event type.
+	for(const auto &wrap : m_filter_all_event_types)
+	{
+		if(wrap->filter->run(evt))
+		{
+			matches.push_back(wrap->rule);
+			match_found = true;
+		}
+	}
+
+	return match_found;
+}
+
+libsinsp::events::set<ppm_sc_code> evttype_index_ruleset::ruleset_filters::sc_codes()
+{
+	libsinsp::events::set<ppm_sc_code> res;
+	for(const auto &wrap : m_filters)
+	{
+		res.insert(wrap->sc_codes.begin(), wrap->sc_codes.end());
+	}
+	return res;
+}
+
+libsinsp::events::set<ppm_event_code> evttype_index_ruleset::ruleset_filters::event_codes()
+{
+	libsinsp::events::set<ppm_event_code> res;
+	for(const auto &wrap : m_filters)
+	{
+		res.insert(wrap->event_codes.begin(), wrap->event_codes.end());
+	}
+	return res;
+}
+
 void evttype_index_ruleset::add(
 		const falco_rule& rule,
 		std::shared_ptr<sinsp_filter> filter,
@@ -38,22 +203,21 @@ void evttype_index_ruleset::add(
 {
 	try
 	{
-		auto wrap = std::make_shared<evttype_index_wrapper>();
-		wrap->m_rule = rule;
-		wrap->m_filter = filter;
+		auto wrap = std::make_shared<filter_wrapper>();
+		wrap->rule = rule;
+		wrap->filter = filter;
 		if(rule.source == falco_common::syscall_source)
 		{
-			wrap->m_sc_codes = libsinsp::filter::ast::ppm_sc_codes(condition.get());
-			wrap->m_event_codes = libsinsp::filter::ast::ppm_event_codes(condition.get());
+			wrap->sc_codes = libsinsp::filter::ast::ppm_sc_codes(condition.get());
+			wrap->event_codes = libsinsp::filter::ast::ppm_event_codes(condition.get());
 		}
 		else
 		{
-			wrap->m_sc_codes = {};
-			wrap->m_event_codes = {ppm_event_code::PPME_PLUGINEVENT_E};
+			wrap->sc_codes = { };
+			wrap->event_codes = { ppm_event_code::PPME_PLUGINEVENT_E };
 		}
-		wrap->m_event_codes.insert(ppm_event_code::PPME_ASYNCEVENT_E);
-
-		add_wrapper(wrap);
+		wrap->event_codes.insert(ppm_event_code::PPME_ASYNCEVENT_E);
+		m_filters.insert(wrap);
 	}
 	catch (const sinsp_exception& e)
 	{
@@ -66,46 +230,180 @@ void evttype_index_ruleset::on_loading_complete()
 	print_enabled_rules_falco_logger();
 }
 
-bool evttype_index_ruleset::run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, falco_rule &match)
-{
-	for(auto &wrap : wrappers)
-	{
-		if(wrap->m_filter->run(evt))
-		{
-			match = wrap->m_rule;
-			return true;
-		}
-	}
-
-	return false;
-}
-
-bool evttype_index_ruleset::run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, std::vector<falco_rule> &matches)
-{
-	bool match_found = false;
-
-	for(auto &wrap : wrappers)
-	{
-		if(wrap->m_filter->run(evt))
-		{
-			matches.push_back(wrap->m_rule);
-			match_found = true;
-		}
-	}
-
-	return match_found;
-}
-
 void evttype_index_ruleset::print_enabled_rules_falco_logger()
 {
 	falco_logger::log(falco_logger::level::DEBUG, "Enabled rules:\n");
-
-	auto logger = [](std::shared_ptr<evttype_index_wrapper> wrap)
+	int n = 0;
+	for (const auto& ruleset_ptr : m_rulesets)
 	{
-		falco_logger::log(falco_logger::level::DEBUG, std::string("   ") + wrap->name() + "\n");
-	};
-
-	uint64_t num_filters = iterate(logger);
-
-	falco_logger::log(falco_logger::level::DEBUG, "(" + std::to_string(num_filters) + ") enabled rules in total\n");
+		if (ruleset_ptr)
+		{
+			for (const auto& wrap : ruleset_ptr->get_filters())
+			{
+				n++;
+				falco_logger::log(falco_logger::level::DEBUG, std::string("   ") + wrap->rule.name + "\n");
+			}
+		}
+	}
+	falco_logger::log(falco_logger::level::DEBUG, "(" + std::to_string(n) + ") enabled rules in total\n");
+}
+
+void evttype_index_ruleset::clear()
+{
+	for (size_t i = 0; i < m_rulesets.size(); i++)
+	{
+		m_rulesets[i] = std::make_shared<ruleset_filters>();
+	}
+	m_filters.clear();
+}
+
+void evttype_index_ruleset::enable(const std::string &pattern, match_type match, uint16_t ruleset_id)
+{
+	enable_disable(pattern, match, true, ruleset_id);
+}
+
+void evttype_index_ruleset::disable(const std::string &pattern, match_type match, uint16_t ruleset_id)
+{
+	enable_disable(pattern, match, false, ruleset_id);
+}
+
+void evttype_index_ruleset::enable_disable(const std::string &pattern, match_type match, bool enabled, uint16_t ruleset_id)
+{
+	while(m_rulesets.size() < (size_t)ruleset_id + 1)
+	{
+		m_rulesets.emplace_back(std::make_shared<ruleset_filters>());
+	}
+
+	for(const auto &wrap : m_filters)
+	{
+		bool matches;
+		std::string::size_type pos;
+
+		switch(match)
+		{
+		case match_type::exact:
+			pos = wrap->rule.name.find(pattern);
+
+			matches = (pattern == "" || (pos == 0 &&
+						       pattern.size() == wrap->rule.name.size()));
+			break;
+		case match_type::substring:
+			matches = (pattern == "" || (wrap->rule.name.find(pattern) != std::string::npos));
+			break;
+		case match_type::wildcard:
+			matches = falco::utils::matches_wildcard(pattern, wrap->rule.name);
+			break;
+		default:
+			// should never happen
+			matches = false;
+		}
+
+		if(matches)
+		{
+			if(enabled)
+			{
+				m_rulesets[ruleset_id]->add_filter(wrap);
+			}
+			else
+			{
+				m_rulesets[ruleset_id]->remove_filter(wrap);
+			}
+		}
+	}
+}
+
+void evttype_index_ruleset::enable_tags(const std::set<std::string> &tags, uint16_t ruleset_id)
+{
+	enable_disable_tags(tags, true, ruleset_id);
+}
+
+void evttype_index_ruleset::disable_tags(const std::set<std::string> &tags, uint16_t ruleset_id)
+{
+	enable_disable_tags(tags, false, ruleset_id);
+}
+
+void evttype_index_ruleset::enable_disable_tags(const std::set<std::string> &tags, bool enabled, uint16_t ruleset_id)
+{
+	while(m_rulesets.size() < (size_t)ruleset_id + 1)
+	{
+		m_rulesets.emplace_back(std::make_shared<ruleset_filters>());
+	}
+
+	for(const auto &wrap : m_filters)
+	{
+		std::set<std::string> intersect;
+
+		set_intersection(tags.begin(), tags.end(),
+				 wrap->rule.tags.begin(), wrap->rule.tags.end(),
+				 inserter(intersect, intersect.begin()));
+
+		if(!intersect.empty())
+		{
+			if(enabled)
+			{
+				m_rulesets[ruleset_id]->add_filter(wrap);
+			}
+			else
+			{
+				m_rulesets[ruleset_id]->remove_filter(wrap);
+			}
+		}
+	}
+}
+
+uint64_t evttype_index_ruleset::enabled_count(uint16_t ruleset_id)
+{
+	while(m_rulesets.size() < (size_t)ruleset_id + 1)
+	{
+		m_rulesets.emplace_back(std::make_shared<ruleset_filters>());
+	}
+
+	return m_rulesets[ruleset_id]->num_filters();
+}
+
+bool evttype_index_ruleset::run(sinsp_evt *evt, falco_rule& match, uint16_t ruleset_id)
+{
+	if(m_rulesets.size() < (size_t)ruleset_id + 1)
+	{
+		return false;
+	}
+
+	return m_rulesets[ruleset_id]->run(evt, match);
+}
+
+bool evttype_index_ruleset::run(sinsp_evt *evt, std::vector<falco_rule>& matches, uint16_t ruleset_id)
+{
+	if(m_rulesets.size() < (size_t)ruleset_id + 1)
+	{
+		return false;
+	}
+
+	return m_rulesets[ruleset_id]->run(evt, matches);
+}
+
+void evttype_index_ruleset::enabled_evttypes(std::set<uint16_t> &evttypes, uint16_t ruleset_id)
+{
+	evttypes.clear();
+	for (const auto& e : enabled_event_codes(ruleset_id))
+	{
+		evttypes.insert((uint16_t) e);
+	}
+}
+
+libsinsp::events::set<ppm_sc_code> evttype_index_ruleset::enabled_sc_codes(uint16_t ruleset)
+{
+	if(m_rulesets.size() < (size_t)ruleset + 1)
+	{
+		return {};
+	}
+	return m_rulesets[ruleset]->sc_codes();
+}
+
+libsinsp::events::set<ppm_event_code> evttype_index_ruleset::enabled_event_codes(uint16_t ruleset)
+{
+	if(m_rulesets.size() < (size_t)ruleset + 1)
+	{
+		return {};
+	}
+	return m_rulesets[ruleset]->event_codes();
 }

userspace/engine/evttype_index_ruleset.h

@@ -17,54 +17,151 @@ limitations under the License.
 
 #pragma once
 
-#include "indexable_ruleset.h"
-
 #include <string>
 #include <set>
 #include <vector>
+#include <list>
+#include <map>
+
+#include "filter_ruleset.h"
+#include <libsinsp/sinsp.h>
+#include <libsinsp/filter.h>
+#include <libsinsp/event.h>
 
 /*!
 	\brief A filter_ruleset that indexes enabled rules by event type,
 	and performs linear search on each event type bucket
 */
-
-struct evttype_index_wrapper
-{
-	const std::string &name() { return m_rule.name; }
-	const std::set<std::string> &tags() { return m_rule.tags; }
-	const libsinsp::events::set<ppm_sc_code> &sc_codes() { return m_sc_codes; }
-	const libsinsp::events::set<ppm_event_code> &event_codes() { return m_event_codes; }
-
-	falco_rule m_rule;
-	libsinsp::events::set<ppm_sc_code> m_sc_codes;
-	libsinsp::events::set<ppm_event_code> m_event_codes;
-	std::shared_ptr<sinsp_filter> m_filter;
-};
-
-class evttype_index_ruleset : public indexable_ruleset<evttype_index_wrapper>
+class evttype_index_ruleset: public filter_ruleset
 {
 public:
 	explicit evttype_index_ruleset(std::shared_ptr<sinsp_filter_factory> factory);
 	virtual ~evttype_index_ruleset();
 
-	// From filter_ruleset
 	void add(
 		const falco_rule& rule,
 		std::shared_ptr<sinsp_filter> filter,
 		std::shared_ptr<libsinsp::filter::ast::expr> condition) override;
 
-	void on_loading_complete() override;
+	void clear() override;
 
-	// From indexable_ruleset
-	bool run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, falco_rule &match) override;
-	bool run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, std::vector<falco_rule> &matches) override;
+	bool run(sinsp_evt *evt, falco_rule& match, uint16_t ruleset_id) override;
+	bool run(sinsp_evt *evt, std::vector<falco_rule>&matches, uint16_t ruleset_id) override;
+
+	uint64_t enabled_count(uint16_t ruleset_id) override;
+
+	void on_loading_complete() override;
 
 	// Print each enabled rule when running Falco with falco logger
 	// log_level=debug; invoked within on_loading_complete()
 	void print_enabled_rules_falco_logger();
 
+	void enable(
+		const std::string &pattern,
+		match_type match,
+		uint16_t rulset_id) override;
+
+	void disable(
+		const std::string &pattern,
+		match_type match,
+		uint16_t rulset_id) override;
+
+	void enable_tags(
+		const std::set<std::string> &tags,
+		uint16_t rulset_id) override;
+
+	void disable_tags(
+		const std::set<std::string> &tags,
+		uint16_t rulset_id) override;
+
+	// note(jasondellaluce): this is deprecated, must use the new
+	// typing-improved `enabled_event_codes` and `enabled_sc_codes` instead
+	// todo(jasondellaluce): remove this in future code refactors
+	void enabled_evttypes(
+		std::set<uint16_t> &evttypes,
+		uint16_t ruleset) override;
+
+	libsinsp::events::set<ppm_sc_code> enabled_sc_codes(uint16_t ruleset) override;
+
+	libsinsp::events::set<ppm_event_code> enabled_event_codes(uint16_t ruleset) override;
+
 private:
+
+	// Helper used by enable()/disable()
+	void enable_disable(
+		const std::string &pattern,
+		match_type match,
+		bool enabled,
+		uint16_t rulset_id);
+
+	// Helper used by enable_tags()/disable_tags()
+	void enable_disable_tags(
+		const std::set<std::string> &tags,
+		bool enabled,
+		uint16_t rulset_id);
+
+	struct filter_wrapper
+	{
+		falco_rule rule;
+		libsinsp::events::set<ppm_sc_code> sc_codes;
+		libsinsp::events::set<ppm_event_code> event_codes;
+		std::shared_ptr<sinsp_filter> filter;
+	};
+
+	typedef std::list<std::shared_ptr<filter_wrapper>> filter_wrapper_list;
+
+	// A group of filters all having the same ruleset
+	class ruleset_filters {
+	public:
+		ruleset_filters();
+
+		virtual ~ruleset_filters();
+
+		void add_filter(std::shared_ptr<filter_wrapper> wrap);
+		void remove_filter(std::shared_ptr<filter_wrapper> wrap);
+
+		uint64_t num_filters();
+
+		inline const std::set<std::shared_ptr<filter_wrapper>>& get_filters() const
+		{
+			return m_filters;
+		}
+
+		// Evaluate an event against the ruleset and return the first rule
+		// that matched.
+		bool run(sinsp_evt *evt, falco_rule& match);
+
+		//  Evaluate an event against the ruleset and return all the
+		//	matching rules.
+		bool run(sinsp_evt *evt, std::vector<falco_rule>& matches);
+
+		libsinsp::events::set<ppm_sc_code> sc_codes();
+
+		libsinsp::events::set<ppm_event_code> event_codes();
+
+	private:
+		void add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap);
+		void remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap);
+
+		// Vector indexes from event type to a set of filters. There can
+		// be multiple filters for a given event type.
+		// NOTE: This is used only when the event sub-type is 0.
+		std::vector<filter_wrapper_list> m_filter_by_event_type;
+
+		filter_wrapper_list m_filter_all_event_types;
+
+		// All filters added. Used to make num_filters() fast.
+		std::set<std::shared_ptr<filter_wrapper>> m_filters;
+	};
+
+	// Vector indexes from ruleset id to set of rules.
+	std::vector<std::shared_ptr<ruleset_filters>> m_rulesets;
+
+	// All filters added. The set of enabled filters is held in m_rulesets
+	std::set<std::shared_ptr<filter_wrapper>> m_filters;
+
 	std::shared_ptr<sinsp_filter_factory> m_filter_factory;
+	std::vector<std::string> m_ruleset_names;
 };
 
 class evttype_index_ruleset_factory: public filter_ruleset_factory

userspace/engine/falco_common.h

@@ -72,5 +72,3 @@ namespace falco_common
 
 	bool parse_rule_matching(const std::string& v, rule_matching& out);
 };
-
-typedef std::unordered_map<std::string, std::pair<std::string, bool>> extra_output_field_t;

userspace/engine/falco_engine.cpp

@@ -57,7 +57,8 @@ falco_engine::falco_engine(bool seed_rng)
 	  m_rule_compiler(std::make_shared<rule_loader::compiler>()),
 	  m_next_ruleset_id(0),
 	  m_min_priority(falco_common::PRIORITY_DEBUG),
-	  m_sampling_ratio(1), m_sampling_multiplier(0)
+	  m_sampling_ratio(1), m_sampling_multiplier(0),
+	  m_replace_container_info(false)
 {
 	if(seed_rng)
 	{
@@ -75,7 +76,6 @@ falco_engine::~falco_engine()
 	m_rule_collector->clear();
 	m_rule_stats_manager.clear();
 	m_sources.clear();
-	m_extra_output_format.clear();
 }
 
 sinsp_version falco_engine::engine_version()
@@ -194,8 +194,8 @@ void falco_engine::list_fields(const std::string &source, bool verbose, bool nam
 std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_content, const std::string &name)
 {
 	rule_loader::configuration cfg(rules_content, m_sources, name);
-	cfg.extra_output_format = m_extra_output_format;
-	cfg.extra_output_fields = m_extra_output_fields;
+	cfg.output_extra = m_extra;
+	cfg.replace_output_container_info = m_replace_container_info;
 
 	// read rules YAML file and collect its definitions
 	if(m_rule_reader->read(cfg, *m_rule_collector))
@@ -204,11 +204,6 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 		m_last_compile_output = m_rule_compiler->new_compile_output();
 		m_rule_compiler->compile(cfg, *m_rule_collector, *m_last_compile_output);
 
-		if(!cfg.res->successful())
-		{
-			return std::move(cfg.res);
-		}
-
 		// clear the rules known by the engine and each ruleset
 		m_rules.clear();
 		for (auto &src : m_sources)
@@ -455,7 +450,6 @@ std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_ev
 		rule_result.priority_num = rule.priority;
 		rule_result.tags = rule.tags;
 		rule_result.exception_fields = rule.exception_fields;
-		rule_result.extra_output_fields = rule.extra_output_fields;
 		m_rule_stats_manager.on_event(rule);
 		res->push_back(rule_result);
 	}
@@ -647,22 +641,9 @@ void falco_engine::get_json_details(
 	out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators);
 	out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields);
 
-	// Get extra requested fields
-	std::vector<std::string> out_fields;
-
-	for(auto const& f : r.extra_output_fields)
-	{
-		// add all the field keys
-		out_fields.emplace_back(f.second.first);
-
-		if (!f.second.second) // formatted field
-		{
-			out["details"]["extra_output_formatted_fields"][f.first] = f.second.first;
-		}
-	}
-
 	// Get fields from output string
 	auto fmt = create_formatter(r.source, r.output);
+	std::vector<std::string> out_fields;
 	fmt->get_field_names(out_fields);
 	out["details"]["output_fields"] = sequence_to_json_array(out_fields);
 
@@ -1096,37 +1077,10 @@ void falco_engine::set_sampling_multiplier(double sampling_multiplier)
 	m_sampling_multiplier = sampling_multiplier;
 }
 
-void falco_engine::add_extra_output_format(
-	const std::string &format,
-	const std::string &source,
-	const std::string &tag,
-	const std::string &rule,
-	bool replace_container_info
-)
+void falco_engine::set_extra(const std::string &extra, bool replace_container_info)
 {
-	m_extra_output_format.push_back({format, source, tag, rule, replace_container_info});
-}
-
-void falco_engine::add_extra_output_formatted_field(
-	const std::string &key,
-	const std::string &format,
-	const std::string &source,
-	const std::string &tag,
-	const std::string &rule
-)
-{
-	m_extra_output_fields.push_back({key, format, source, tag, rule, false});
-}
-
-void falco_engine::add_extra_output_raw_field(
-	const std::string &key,
-	const std::string &source,
-	const std::string &tag,
-	const std::string &rule
-)
-{
-	std::string format = "%" + key;
-	m_extra_output_fields.push_back({key, format, source, tag, rule, true});
+	m_extra = extra;
+	m_replace_container_info = replace_container_info;
 }
 
 inline bool falco_engine::should_drop_evt() const

userspace/engine/falco_engine.h

@@ -176,40 +176,15 @@ public:
 	//
 	void set_sampling_multiplier(double sampling_multiplier);
 
-	// You can optionally add "extra" output to the end
+	//
+	// You can optionally add "extra" formatting fields to the end
 	// of all output expressions. You can also choose to replace
 	// %container.info with the extra information or add it to the
 	// end of the expression. This is used in open source falco to
 	// add k8s/container information to outputs when
 	// available.
 	//
-	void add_extra_output_format(
-		const std::string &format,
-		const std::string &source,
-		const std::string &tag,
-		const std::string &rule,
-		bool replace_container_info
-	);
-
-	// You can optionally add fields that will only show up in the object
-	// output (e.g. json, gRPC) alongside other output_fields
-	// and not in the text message output.
-	// You can add two types of fields: formatted which will act like
-	// an additional output format that appears in the output field
-	void add_extra_output_formatted_field(
-		const std::string &key,
-		const std::string &format,
-		const std::string &source,
-		const std::string &tag,
-		const std::string &rule
-	);
-
-	void add_extra_output_raw_field(
-		const std::string &key,
-		const std::string &source,
-		const std::string &tag,
-		const std::string &rule
-	);
+	void set_extra(const std::string &extra, bool replace_container_info);
 
 	// Represents the result of matching an event against a set of
 	// rules.
@@ -221,7 +196,6 @@ public:
 		std::string format;
 		std::set<std::string> exception_fields;
 		std::set<std::string> tags;
-		extra_output_field_t extra_output_fields;
 	};
 
 	//
@@ -487,6 +461,6 @@ private:
 	static const std::string s_default_ruleset;
 	uint32_t m_default_ruleset_id;
 
-	std::vector<rule_loader::extra_output_format_conf> m_extra_output_format;
-	std::vector<rule_loader::extra_output_field_conf> m_extra_output_fields;
+	std::string m_extra;
+	bool m_replace_container_info;
 };

userspace/engine/falco_engine_version.h

@@ -20,7 +20,7 @@ limitations under the License.
 
 // The version of this Falco engine
 #define FALCO_ENGINE_VERSION_MAJOR 0
-#define FALCO_ENGINE_VERSION_MINOR 42
+#define FALCO_ENGINE_VERSION_MINOR 40
 #define FALCO_ENGINE_VERSION_PATCH 0
 
 #define FALCO_ENGINE_VERSION \
@@ -34,4 +34,4 @@ limitations under the License.
 // It represents the fields supported by this version of Falco,
 // the event types, and the underlying driverevent schema. It's used to
 // detetect changes in engine version in our CI jobs.
-#define FALCO_ENGINE_CHECKSUM "786745197f42d685b536581ddbb5eab8500939a3f6ae67cfa23a212da10e423c"
+#define FALCO_ENGINE_CHECKSUM "bc9d0d94ae70ef26b7cf814f62273a48b2bb4133dff0baff5f194f6f1711875a"

userspace/engine/falco_rule.h

@@ -79,7 +79,6 @@ struct falco_rule
 	std::string name;
 	std::string description;
 	std::string output;
-	extra_output_field_t extra_output_fields;
 	std::set<std::string> tags;
 	std::set<std::string> exception_fields;
 	falco_common::priority_type priority;

userspace/engine/formats.cpp

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-#include <nlohmann/json.hpp>
+#include <json/json.h>
 
 #include "formats.h"
 #include "falco_engine.h"
@@ -35,7 +35,7 @@ falco_formats::~falco_formats()
 
 std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
 				   const std::string &level, const std::string &format, const std::set<std::string> &tags,
-				   const std::string &hostname, const extra_output_field_t &extra_fields) const
+				   const std::string &hostname) const
 {
 	std::string line;
 
@@ -48,17 +48,27 @@ std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule,
 
 	if(formatter->get_output_format() == sinsp_evt_formatter::OF_JSON)
 	{
-		std::string json_fields;
+		std::string json_line;
 
 		// Format the event into a json object with all fields resolved
-		formatter->tostring(evt, json_fields);
+		formatter->tostring(evt, json_line);
+
+		// The formatted string might have a leading newline. If it does, remove it.
+		if(json_line[0] == '\n')
+		{
+			json_line.erase(0, 1);
+		}
 
 		// For JSON output, the formatter returned a json-as-text
 		// object containing all the fields in the original format
 		// message as well as the event time in ns. Use this to build
 		// a more detailed object containing the event time, rule,
 		// severity, full output, and fields.
-		nlohmann::json event;
+		Json::Value event;
+		Json::Value rule_tags;
+		Json::FastWriter writer;
+		std::string full_line;
+		unsigned int rule_tags_idx = 0;
 
 		// Convert the time-as-nanoseconds to a more json-friendly ISO8601.
 		time_t evttime = evt->get_ts() / 1000000000;
@@ -84,54 +94,43 @@ std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule,
 
 		if(m_json_include_tags_property)
 		{
-			event["tags"] = tags;
+			if (tags.size() == 0)
+			{
+				// This sets an empty array
+				rule_tags = Json::arrayValue;
+			}
+			else
+			{
+				for (const auto &tag : tags)
+				{
+					rule_tags[rule_tags_idx++] = tag;
+				}
+			}
+			event["tags"] = rule_tags;
 		}
 
-		event["output_fields"] = nlohmann::json::parse(json_fields);
+		full_line = writer.write(event);
 
-		for (auto const& ef : extra_fields)
+		// Json::FastWriter may add a trailing newline. If it
+		// does, remove it.
+		if(full_line[full_line.length() - 1] == '\n')
 		{
-			std::string fformat = ef.second.first;
-			if (fformat.size() == 0)
-			{
-				continue;
-			}
-
-			if (!(fformat[0] == '*'))
-			{
-				fformat = "*" + fformat;
-			}
-
-			if(ef.second.second) // raw field
-			{
-				std::string json_field_map;
-				formatter = m_falco_engine->create_formatter(source, fformat);
-				formatter->tostring_withformat(evt, json_field_map, sinsp_evt_formatter::OF_JSON);
-				auto json_obj = nlohmann::json::parse(json_field_map);
-				event["output_fields"][ef.first] = json_obj[ef.first];
-			} else
-			{
-				event["output_fields"][ef.first] = format_string(evt, fformat, source);
-			}
+			full_line.resize(full_line.length() - 1);
 		}
 
-		line = event.dump();
+		// Cheat-graft the output from the formatter into this
+		// string. Avoids an unnecessary json parse just to
+		// merge the formatted fields at the object level.
+		full_line.pop_back();
+		full_line.append(", \"output_fields\": ");
+		full_line.append(json_line);
+		full_line.append("}");
+		line = full_line;
 	}
 
 	return line;
 }
 
-std::string falco_formats::format_string(sinsp_evt *evt, const std::string &format, const std::string &source) const
-{
-	std::string line;
-	std::shared_ptr<sinsp_evt_formatter> formatter;
-
-	formatter = m_falco_engine->create_formatter(source, format);
-	formatter->tostring_withformat(evt, line, sinsp_evt_formatter::OF_NORMAL);
-
-	return line;
-}
-
 std::map<std::string, std::string> falco_formats::get_field_values(sinsp_evt *evt, const std::string &source,
 						    const std::string &format) const
 {

userspace/engine/formats.h

@@ -31,9 +31,7 @@ public:
 
 	std::string format_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
 				 const std::string &level, const std::string &format, const std::set<std::string> &tags,
-				 const std::string &hostname, const extra_output_field_t &extra_fields) const;
-
-	std::string format_string(sinsp_evt *evt, const std::string &format, const std::string &source) const;
+				 const std::string &hostname) const;
 
 	std::map<std::string, std::string> get_field_values(sinsp_evt *evt, const std::string &source,
 					     const std::string &format) const ;

userspace/engine/indexable_ruleset.h

@@ -1,454 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-/*
-Copyright (C) 2024 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/* This describes the interface for an "indexable" ruleset, that is, a
- * ruleset that can enable/disable abstract filters for various
- * ruleset ids.
- *
- * It's used by evttype_index_ruleset as well as other rulesets that
- * need the same functionality but don't want to copy the same code.
- */
-
-#pragma once
-
-#include "falco_utils.h"
-#include "filter_ruleset.h"
-
-#include <libsinsp/sinsp.h>
-#include <libsinsp/filter.h>
-#include <libsinsp/event.h>
-
-#include <functional>
-#include <memory>
-#include <string>
-
-// A filter_wrapper should implement these methods:
-//   const std::string &filter_wrapper::name();
-//   const std::set<std::string> &filter_wrapper::tags();
-//   const libsinsp::events::set<ppm_sc_code> &filter_wrapper::sc_codes();
-//   const libsinsp::events::set<ppm_event_code> &filter_wrapper::event_codes();
-
-template<class filter_wrapper>
-class indexable_ruleset : public filter_ruleset
-{
-public:
-	indexable_ruleset() = default;
-	virtual ~indexable_ruleset() = default;
-
-	// Required to implement filter_ruleset
-	void clear() override
-	{
-		for(size_t i = 0; i < m_rulesets.size(); i++)
-		{
-			m_rulesets[i] = std::make_shared<ruleset_filters>(i);
-		}
-		m_filters.clear();
-	}
-
-	uint64_t enabled_count(uint16_t ruleset_id) override
-	{
-		while(m_rulesets.size() < (size_t)ruleset_id + 1)
-		{
-			m_rulesets.emplace_back(std::make_shared<ruleset_filters>(m_rulesets.size()));
-		}
-
-		return m_rulesets[ruleset_id]->num_filters();
-	}
-
-	void enabled_evttypes(
-		std::set<uint16_t> &evttypes,
-		uint16_t ruleset_id) override
-	{
-		evttypes.clear();
-		for(const auto &e : enabled_event_codes(ruleset_id))
-		{
-			evttypes.insert((uint16_t)e);
-		}
-	}
-
-	libsinsp::events::set<ppm_sc_code> enabled_sc_codes(
-		uint16_t ruleset_id) override
-	{
-		if(m_rulesets.size() < (size_t)ruleset_id + 1)
-		{
-			return {};
-		}
-		return m_rulesets[ruleset_id]->sc_codes();
-	}
-
-	libsinsp::events::set<ppm_event_code> enabled_event_codes(
-		uint16_t ruleset_id) override
-	{
-		if(m_rulesets.size() < (size_t)ruleset_id + 1)
-		{
-			return {};
-		}
-		return m_rulesets[ruleset_id]->event_codes();
-	}
-
-	void enable(
-		const std::string &pattern,
-		match_type match,
-		uint16_t ruleset_id) override
-	{
-		enable_disable(pattern, match, true, ruleset_id);
-	}
-
-	void disable(
-		const std::string &pattern,
-		match_type match,
-		uint16_t ruleset_id) override
-	{
-		enable_disable(pattern, match, false, ruleset_id);
-	}
-
-	void enable_tags(
-		const std::set<std::string> &tags,
-		uint16_t ruleset_id) override
-	{
-		enable_disable_tags(tags, true, ruleset_id);
-	}
-
-	void disable_tags(
-		const std::set<std::string> &tags,
-		uint16_t ruleset_id) override
-	{
-		enable_disable_tags(tags, false, ruleset_id);
-	}
-
-	// Note that subclasses do *not* implement run. Instead, they
-	// implement run_wrappers.
-	bool run(sinsp_evt *evt, falco_rule &match, uint16_t ruleset_id) override
-	{
-		if(m_rulesets.size() < (size_t)ruleset_id + 1)
-		{
-			return false;
-		}
-
-		return m_rulesets[ruleset_id]->run(*this, evt, match);
-	}
-
-	bool run(sinsp_evt *evt, std::vector<falco_rule> &matches, uint16_t ruleset_id) override
-	{
-		if(m_rulesets.size() < (size_t)ruleset_id + 1)
-		{
-			return false;
-		}
-
-		return m_rulesets[ruleset_id]->run(*this, evt, matches);
-	}
-
-	typedef std::list<std::shared_ptr<filter_wrapper>>
-		filter_wrapper_list;
-
-	// Subclasses should call add_wrapper (most likely from
-	// filter_ruleset::add or ::add_compile_output) to add filters.
-	void add_wrapper(std::shared_ptr<filter_wrapper> wrap)
-	{
-		m_filters.insert(wrap);
-	}
-
-	// If a subclass needs to iterate over all filters, they can
-	// call iterate with this function, which will be called for
-	// all filters.
-	typedef std::function<void(const std::shared_ptr<filter_wrapper> &wrap)> filter_wrapper_func;
-	uint64_t iterate(filter_wrapper_func func)
-	{
-		uint64_t num_filters = 0;
-
-		for(const auto &ruleset_ptr : m_rulesets)
-		{
-			if(ruleset_ptr)
-			{
-				for(const auto &wrap : ruleset_ptr->get_filters())
-				{
-					num_filters++;
-					func(wrap);
-				}
-			}
-		}
-
-		return num_filters;
-	}
-
-	// A subclass must implement these methods. They are analogous
-	// to run() but take care of selecting filters that match a
-	// ruleset and possibly an event type.
-	virtual bool run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, std::vector<falco_rule> &matches) = 0;
-	virtual bool run_wrappers(sinsp_evt *evt, filter_wrapper_list &wrappers, uint16_t ruleset_id, falco_rule &match) = 0;
-
-private:
-	// Helper used by enable()/disable()
-	void enable_disable(
-		const std::string &pattern,
-		match_type match,
-		bool enabled,
-		uint16_t ruleset_id)
-	{
-		while(m_rulesets.size() < (size_t)ruleset_id + 1)
-		{
-			m_rulesets.emplace_back(std::make_shared<ruleset_filters>(m_rulesets.size()));
-		}
-
-		for(const auto &wrap : m_filters)
-		{
-			bool matches;
-			std::string::size_type pos;
-
-			switch(match)
-			{
-			case match_type::exact:
-				pos = wrap->name().find(pattern);
-
-				matches = (pattern == "" || (pos == 0 &&
-							     pattern.size() == wrap->name().size()));
-				break;
-			case match_type::substring:
-				matches = (pattern == "" || (wrap->name().find(pattern) != std::string::npos));
-				break;
-			case match_type::wildcard:
-				matches = falco::utils::matches_wildcard(pattern, wrap->name());
-				break;
-			default:
-				// should never happen
-				matches = false;
-			}
-
-			if(matches)
-			{
-				if(enabled)
-				{
-					m_rulesets[ruleset_id]->add_filter(wrap);
-				}
-				else
-				{
-					m_rulesets[ruleset_id]->remove_filter(wrap);
-				}
-			}
-		}
-	}
-
-	// Helper used by enable_tags()/disable_tags()
-	void enable_disable_tags(
-		const std::set<std::string> &tags,
-		bool enabled,
-		uint16_t ruleset_id)
-	{
-		while(m_rulesets.size() < (size_t)ruleset_id + 1)
-		{
-			m_rulesets.emplace_back(std::make_shared<ruleset_filters>(m_rulesets.size()));
-		}
-
-		for(const auto &wrap : m_filters)
-		{
-			std::set<std::string> intersect;
-
-			set_intersection(tags.begin(), tags.end(),
-					 wrap->tags().begin(), wrap->tags().end(),
-					 inserter(intersect, intersect.begin()));
-
-			if(!intersect.empty())
-			{
-				if(enabled)
-				{
-					m_rulesets[ruleset_id]->add_filter(wrap);
-				}
-				else
-				{
-					m_rulesets[ruleset_id]->remove_filter(wrap);
-				}
-			}
-		}
-	}
-
-	// A group of filters all having the same ruleset
-	class ruleset_filters
-	{
-	public:
-		ruleset_filters(uint16_t ruleset_id):
-			m_ruleset_id(ruleset_id) {}
-
-		virtual ~ruleset_filters(){};
-
-		void add_filter(std::shared_ptr<filter_wrapper> wrap)
-		{
-			if(wrap->event_codes().empty())
-			{
-				// Should run for all event types
-				add_wrapper_to_list(m_filter_all_event_types, wrap);
-			}
-			else
-			{
-				for(auto &etype : wrap->event_codes())
-				{
-					if(m_filter_by_event_type.size() <= etype)
-					{
-						m_filter_by_event_type.resize(etype + 1);
-					}
-
-					add_wrapper_to_list(m_filter_by_event_type[etype], wrap);
-				}
-			}
-
-			m_filters.insert(wrap);
-		}
-
-		void remove_filter(std::shared_ptr<filter_wrapper> wrap)
-		{
-			if(wrap->event_codes().empty())
-			{
-				remove_wrapper_from_list(m_filter_all_event_types, wrap);
-			}
-			else
-			{
-				for(auto &etype : wrap->event_codes())
-				{
-					if(etype < m_filter_by_event_type.size())
-					{
-						remove_wrapper_from_list(m_filter_by_event_type[etype], wrap);
-					}
-				}
-			}
-
-			m_filters.erase(wrap);
-		}
-
-		uint64_t num_filters()
-		{
-			return m_filters.size();
-		}
-
-		inline const std::set<std::shared_ptr<filter_wrapper>> &get_filters() const
-		{
-			return m_filters;
-		}
-
-		// Evaluate an event against the ruleset and return the first rule
-		// that matched.
-		bool run(indexable_ruleset &ruleset, sinsp_evt *evt, falco_rule &match)
-		{
-			if(evt->get_type() < m_filter_by_event_type.size() &&
-			   m_filter_by_event_type[evt->get_type()].size() > 0)
-			{
-				if(ruleset.run_wrappers(evt, m_filter_by_event_type[evt->get_type()], m_ruleset_id, match))
-				{
-					return true;
-				}
-			}
-
-			// Finally, try filters that are not specific to an event type.
-			if(m_filter_all_event_types.size() > 0)
-			{
-				if(ruleset.run_wrappers(evt, m_filter_all_event_types, m_ruleset_id, match))
-				{
-					return true;
-				}
-			}
-
-			return false;
-		}
-
-		//  Evaluate an event against the ruleset and return all the
-		//  matching rules.
-		bool run(indexable_ruleset &ruleset, sinsp_evt *evt, std::vector<falco_rule> &matches)
-		{
-			if(evt->get_type() < m_filter_by_event_type.size() &&
-			   m_filter_by_event_type[evt->get_type()].size() > 0)
-			{
-				if(ruleset.run_wrappers(evt, m_filter_by_event_type[evt->get_type()], m_ruleset_id, matches))
-				{
-					return true;
-				}
-			}
-
-			// Finally, try filters that are not specific to an event type.
-			if(m_filter_all_event_types.size() > 0)
-			{
-				if(ruleset.run_wrappers(evt, m_filter_all_event_types, m_ruleset_id, matches))
-				{
-					return true;
-				}
-			}
-
-			return false;
-		}
-
-		libsinsp::events::set<ppm_sc_code> sc_codes()
-		{
-			libsinsp::events::set<ppm_sc_code> res;
-			for(const auto &wrap : m_filters)
-			{
-				res.insert(wrap->sc_codes().begin(), wrap->sc_codes().end());
-			}
-			return res;
-		}
-		libsinsp::events::set<ppm_event_code> event_codes()
-		{
-			libsinsp::events::set<ppm_event_code> res;
-			for(const auto &wrap : m_filters)
-			{
-				res.insert(wrap->event_codes().begin(), wrap->event_codes().end());
-			}
-			return res;
-		}
-
-	private:
-		void add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
-		{
-			// This is O(n) but it's also uncommon
-			// (when loading rules only).
-			auto pos = std::find(wrappers.begin(),
-					     wrappers.end(),
-					     wrap);
-
-			if(pos == wrappers.end())
-			{
-				wrappers.push_back(wrap);
-			}
-		}
-
-		void remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr<filter_wrapper> wrap)
-		{
-			// This is O(n) but it's also uncommon
-			// (when loading rules only).
-			auto pos = std::find(wrappers.begin(),
-					     wrappers.end(),
-					     wrap);
-			if(pos != wrappers.end())
-			{
-				wrappers.erase(pos);
-			}
-		}
-		uint16_t m_ruleset_id;
-
-		// Vector indexes from event type to a set of filters. There can
-		// be multiple filters for a given event type.
-		// NOTE: This is used only when the event sub-type is 0.
-		std::vector<filter_wrapper_list> m_filter_by_event_type;
-
-		filter_wrapper_list m_filter_all_event_types;
-
-		// All filters added. Used to make num_filters() fast.
-		std::set<std::shared_ptr<filter_wrapper>> m_filters;
-	};
-
-	// Vector indexes from ruleset id to set of rules.
-	std::vector<std::shared_ptr<ruleset_filters>> m_rulesets;
-
-	// All filters added. The set of enabled filters is held in m_rulesets
-	std::set<std::shared_ptr<filter_wrapper>> m_filters;
-};

userspace/engine/rule_loader.h

@@ -20,7 +20,6 @@ limitations under the License.
 #include <string>
 #include <vector>
 #include <optional>
-#include <unordered_map>
 #include <yaml-cpp/yaml.h>
 #include <nlohmann/json.hpp>
 #include "falco_source.h"
@@ -262,25 +261,6 @@ namespace rule_loader
 		nlohmann::json res_json;
 	};
 
-	struct extra_output_format_conf
-	{
-		std::string m_format;
-		std::string m_source;
-		std::string m_tag;
-		std::string m_rule;
-		bool m_replace_container_info;
-	};
-
-	struct extra_output_field_conf
-	{
-		std::string m_key;
-		std::string m_format;
-		std::string m_source;
-		std::string m_tag;
-		std::string m_rule;
-		bool m_raw;
-	};
-
 	/*!
 		\brief Contains the info required to load rule definitions
 	*/
@@ -298,9 +278,8 @@ namespace rule_loader
 		const std::string& content;
 		const indexed_vector<falco_source>& sources;
 		std::string name;
-
-		std::vector<extra_output_format_conf> extra_output_format;
-		std::vector<extra_output_field_conf> extra_output_fields;
+		std::string output_extra;
+		bool replace_output_container_info = false;
 
 		// outputs
 		std::unique_ptr<result> res;

userspace/engine/rule_loader_compiler.cpp

@@ -322,6 +322,22 @@ static std::shared_ptr<ast::expr> parse_condition(
 	}
 }
 
+static void apply_output_substitutions(
+	rule_loader::configuration& cfg,
+	std::string& out)
+{
+	if (out.find(s_container_info_fmt) != std::string::npos)
+	{
+		if (cfg.replace_output_container_info)
+		{
+			out = replace(out, s_container_info_fmt, cfg.output_extra);
+			return;
+		}
+		out = replace(out, s_container_info_fmt, s_default_extra_fmt);
+	}
+	out += cfg.output_extra.empty() ? "" : " " + cfg.output_extra;
+}
+
 void rule_loader::compiler::compile_list_infos(
 		configuration& cfg,
 		const collector& col,
@@ -494,64 +510,13 @@ void rule_loader::compiler::compile_rule_infos(
 		// build rule output message
 		rule.output = r.output;
 
-		for (auto& extra : cfg.extra_output_format)
+		// plugins sources do not have any container info and so we won't apply -pk, -pc, etc.
+		// on the other hand, when using plugins you might want to append custom output based on the plugin
+		// TODO: this is not flexible enough (esp. if you mix plugin with syscalls),
+		// it would be better to add configuration options to control the output.
+		if (!cfg.replace_output_container_info || r.source == falco_common::syscall_source)
 		{
-			if (extra.m_source != "" && r.source != extra.m_source)
-			{
-				continue;
-			}
-
-			if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0)
-			{
-				continue;
-			}
-
-			if (extra.m_rule != "" && r.name != extra.m_rule)
-			{
-				continue;
-			}
-
-			if (extra.m_replace_container_info)
-			{
-				if (rule.output.find(s_container_info_fmt) != std::string::npos)
-				{
-					rule.output = replace(rule.output, s_container_info_fmt, extra.m_format);
-				}
-				else
-				{
-					rule.output = rule.output + " " + extra.m_format;
-				}
-			} else
-			{
-				rule.output = rule.output + " " + extra.m_format;
-			}
-		}
-
-		if (rule.output.find(s_container_info_fmt) != std::string::npos)
-		{
-			rule.output = replace(rule.output, s_container_info_fmt, s_default_extra_fmt);
-		}
-
-		// build extra output fields if required
-
-		for (auto const& extra : cfg.extra_output_fields)
-		{
-			if (extra.m_source != "" && r.source != extra.m_source)
-			{
-				continue;
-			}
-
-			if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0)
-			{
-				continue;
-			}
-
-			if (extra.m_rule != "" && r.name != extra.m_rule)
-			{
-				continue;
-			}
-
-			rule.extra_output_fields[extra.m_key] = {extra.m_format, extra.m_raw};
+			apply_output_substitutions(cfg, rule.output);
 		}
 
 		// validate the rule's output
@@ -573,18 +538,6 @@ void rule_loader::compiler::compile_rule_infos(
 				r.output_ctx);
 		}
 
-		// validate the rule's extra fields if any
-		for (auto const& ef : rule.extra_output_fields)
-		{
-			if(!is_format_valid(*cfg.sources.at(r.source), ef.second.first, err))
-			{
-				throw rule_load_exception(
-					falco::load_result::load_result::LOAD_ERR_COMPILE_OUTPUT,
-					err,
-					r.output_ctx);
-			}
-		}
-
 		if (!compile_condition(cfg,
 				  macro_resolver,
 				  lists,

userspace/falco/CMakeLists.txt

@@ -49,7 +49,6 @@ add_library(falco_application STATIC
   app/actions/validate_rules_files.cpp
   app/actions/create_requested_paths.cpp
   app/actions/close_inspectors.cpp
-  app/actions/print_config_schema.cpp
   configuration.cpp
   falco_outputs.cpp
   outputs_file.cpp

userspace/falco/app/actions/actions.h

@@ -38,7 +38,6 @@ falco::app::run_result list_plugins(const falco::app::state& s);
 falco::app::run_result load_config(const falco::app::state& s);
 falco::app::run_result load_plugins(falco::app::state& s);
 falco::app::run_result load_rules_files(falco::app::state& s);
-falco::app::run_result print_config_schema(falco::app::state& s);
 falco::app::run_result print_generated_gvisor_config(falco::app::state& s);
 falco::app::run_result print_help(falco::app::state& s);
 falco::app::run_result print_ignored_events(const falco::app::state& s);

userspace/falco/app/actions/helpers_inspector.cpp

@@ -65,7 +65,7 @@ falco::app::run_result falco::app::actions::open_live_inspector(
 				{
 					auto cfg = s.plugin_configs.at(p->name());
 					falco_logger::log(falco_logger::level::INFO, "Opening '" + source + "' source with plugin '" + cfg->m_name + "'");
-					inspector->open_plugin(cfg->m_name, cfg->m_open_params, sinsp_plugin_platform::SINSP_PLATFORM_HOSTINFO);
+					inspector->open_plugin(cfg->m_name, cfg->m_open_params);
 					return run_result::ok();
 				}
 			}
@@ -83,7 +83,7 @@ falco::app::run_result falco::app::actions::open_live_inspector(
 				{
 					auto cfg = s.plugin_configs.at(p->name());
 					falco_logger::log(falco_logger::level::INFO, "Opening '" + source + "' source with plugin '" + cfg->m_name + "'");
-					inspector->open_plugin(cfg->m_name, cfg->m_open_params, sinsp_plugin_platform::SINSP_PLATFORM_FULL);
+					inspector->open_plugin(cfg->m_name, cfg->m_open_params);
 					return run_result::ok();
 				}
 			}

userspace/falco/app/actions/init_falco_engine.cpp

@@ -17,55 +17,49 @@ limitations under the License.
 
 #include "actions.h"
 #include <libsinsp/plugin_manager.h>
-#include <falco_common.h>
 
 using namespace falco::app;
 using namespace falco::app::actions;
 
 void configure_output_format(falco::app::state& s)
 {
-	for (auto& eo : s.config->m_append_output)
-	{
-		if (eo.m_format != "")
-		{
-			s.engine->add_extra_output_format(eo.m_format, eo.m_source, eo.m_tag, eo.m_rule, false);
-		}
-
-		for (auto const& ff : eo.m_formatted_fields)
-		{
-			s.engine->add_extra_output_formatted_field(ff.first, ff.second, eo.m_source, eo.m_tag, eo.m_rule);
-		}
-
-		for (auto const& rf : eo.m_raw_fields)
-		{
-			s.engine->add_extra_output_raw_field(rf, eo.m_source, eo.m_tag, eo.m_rule);
-		}
-	}
-
 	// See https://falco.org/docs/rules/style-guide/
 	const std::string container_info = "container_id=%container.id container_image=%container.image.repository container_image_tag=%container.image.tag container_name=%container.name";
 	const std::string k8s_info = "k8s_ns=%k8s.ns.name k8s_pod_name=%k8s.pod.name";
 	const std::string gvisor_info = "vpid=%proc.vpid vtid=%thread.vtid";
 
+	std::string output_format;
+	bool replace_container_info = false;
+
 	if(s.options.print_additional == "c" || s.options.print_additional == "container")
 	{
-		s.engine->add_extra_output_format(container_info, falco_common::syscall_source, "", "", true);
+		output_format = container_info;
+		replace_container_info = true;
 	}
 	else if(s.options.print_additional == "cg" || s.options.print_additional == "container-gvisor")
 	{
-		s.engine->add_extra_output_format(gvisor_info + " " + container_info, falco_common::syscall_source, "", "", true);
+		output_format = gvisor_info + " " + container_info;
+		replace_container_info = true;
 	}
 	else if(s.options.print_additional == "k" || s.options.print_additional == "kubernetes")
 	{
-		s.engine->add_extra_output_format(container_info + " " + k8s_info, falco_common::syscall_source, "", "", true);
+		output_format = container_info + " " + k8s_info;
+		replace_container_info = true;
 	}
 	else if(s.options.print_additional == "kg" || s.options.print_additional == "kubernetes-gvisor")
 	{
-		s.engine->add_extra_output_format(gvisor_info + " " + container_info + " " + k8s_info, falco_common::syscall_source, "", "", true);
+		output_format = gvisor_info + " " + container_info + " " + k8s_info;
+		replace_container_info = true;
 	}
 	else if(!s.options.print_additional.empty())
 	{
-		s.engine->add_extra_output_format(s.options.print_additional, "", "", "", false);
+		output_format = s.options.print_additional;
+		replace_container_info = false;
+	}
+
+	if(!output_format.empty())
+	{
+		s.engine->set_extra(output_format, replace_container_info);
 	}
 }
 

userspace/falco/app/actions/init_inspectors.cpp

@@ -29,72 +29,18 @@ static void init_syscall_inspector(falco::app::state& s, std::shared_ptr<sinsp>
 {
 	inspector->set_buffer_format(s.options.event_buffer_format);
 
-	//
-	// Container engines
-	//
-
-	// Debug log messages
-	if(s.config->m_container_engines_mask & (1 << CT_DOCKER))
-	{
-		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'docker'");
-	}
-
-	if(s.config->m_container_engines_mask & (1 << CT_PODMAN))
-	{
-		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'podman'");
-	}
-
-	if(s.config->m_container_engines_mask & ((1 << CT_CRI) | (1 << CT_CRIO) | (1 << CT_CONTAINERD)))
-	{
-		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'CRI'");
-	}
-
-	if(s.config->m_container_engines_mask & (1 << CT_LXC))
-	{
-		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'lxc'");
-	}
-
-	if(s.config->m_container_engines_mask & (1 << CT_LIBVIRT_LXC))
-	{
-		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'libvirt_lxc'");
-	}
-
-	if(s.config->m_container_engines_mask & (1 << CT_BPM))
-	{
-		falco_logger::log(falco_logger::level::DEBUG, "Enabled container engine 'bpm'");
-	}
-
-	// Container engines configs via falco.yaml
-	inspector->set_container_engine_mask(s.config->m_container_engines_mask);
-	for (auto &p : s.config->m_container_engines_cri_socket_paths)
-	{
-		if (!p.empty())
-		{
-			inspector->add_cri_socket_path(p);
-			falco_logger::log(falco_logger::level::DEBUG, "Enabled container runtime socket at '" + p + "' via config file");
-		}
-	}
-	inspector->set_cri_async(!s.config->m_container_engines_disable_cri_async);
-
-	// Container engines configs via CLI args
 	// If required, set the CRI paths
 	for (auto &p : s.options.cri_socket_paths)
 	{
 		if (!p.empty())
 		{
 			inspector->add_cri_socket_path(p);
-			falco_logger::log(falco_logger::level::DEBUG, "Enabled container runtime socket at '" + p + "' via CLI args");
 		}
 	}
 
 	// Decide whether to do sync or async for CRI metadata fetch
 	inspector->set_cri_async(!s.options.disable_cri_async);
 
-	if(s.options.disable_cri_async || s.config->m_container_engines_disable_cri_async)
-	{
-		falco_logger::log(falco_logger::level::DEBUG, "Disabling async lookups for 'CRI'");
-	}
-
 	//
 	// If required, set the snaplen
 	//
@@ -163,11 +109,6 @@ falco::app::run_result falco::app::actions::init_inspectors(falco::app::state& s
 	std::unordered_set<std::string> used_plugins;
 	const auto& all_plugins = s.offline_inspector->get_plugin_manager()->plugins();
 
-	if((s.config->m_metrics_flags & METRICS_V2_STATE_COUNTERS))
-	{
-
-	}
-
 	for (const auto &src : s.loaded_sources)
 	{
 		auto src_info = s.source_infos.at(src);
@@ -180,9 +121,13 @@ falco::app::run_result falco::app::actions::init_inspectors(falco::app::state& s
 		}
 		else
 		{
-			src_info->inspector = std::make_shared<sinsp>(s.config->m_metrics_flags & METRICS_V2_STATE_COUNTERS);
+			src_info->inspector = std::make_shared<sinsp>(false,
+								      "",
+								      "",
+								      "",
+								      s.config->m_metrics_flags & METRICS_V2_STATE_COUNTERS);
 		}
-
+		
 		// do extra preparation for the syscall source
 		if (src == falco_common::syscall_source)
 		{

userspace/falco/app/actions/load_config.cpp

@@ -31,19 +31,19 @@ falco::app::run_result falco::app::actions::load_config(const falco::app::state&
 {
 	// List of loaded conf files, ie: s.options.conf_filename
 	// plus all the `config_files` expanded list of configs.
-	config_loaded_res res;
+	std::vector<std::string> loaded_conf_files;
 	try
 	{
 		if (!s.options.conf_filename.empty())
 		{
-			res = s.config->init_from_file(s.options.conf_filename, s.options.cmdline_config_options);
+			s.config->init_from_file(s.options.conf_filename, loaded_conf_files, s.options.cmdline_config_options);
 		}
 		else
 		{
 			// Is possible to have an empty config file when we want to use some command line
 			// options like `--help`, `--version`, ...
 			// The configs used in `load_yaml` will be initialized to the default values.
-			res = s.config->init_from_content("", s.options.cmdline_config_options);
+			s.config->init_from_content("", s.options.cmdline_config_options);
 		}
 	}
 	catch (std::exception& e)
@@ -61,12 +61,9 @@ falco::app::run_result falco::app::actions::load_config(const falco::app::state&
 	if (!s.options.conf_filename.empty())
 	{
 		falco_logger::log(falco_logger::level::INFO, "Falco initialized with configuration files:\n");
-		for (const auto& pair : res)
+		for (const auto& path : loaded_conf_files)
 		{
-			auto config_path = pair.first;
-			auto validation = pair.second;
-			auto priority = validation == yaml_helper::validation_ok ? falco_logger::level::INFO : falco_logger::level::WARNING;
-			falco_logger::log(priority, std::string("   ") + config_path + " | validation: " + validation + "\n");
+			falco_logger::log(falco_logger::level::INFO, std::string("   ") + path + "\n");
 		}
 	}
 
@@ -88,4 +85,4 @@ falco::app::run_result falco::app::actions::require_config_file(const falco::app
 	}
 #endif // __EMSCRIPTEN__
 	return run_result::ok();
-}
+}

userspace/falco/app/actions/load_rules_files.cpp

@@ -131,6 +131,39 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 		return run_result::fatal(err);
 	}
 
+	if((!s.options.disabled_rule_substrings.empty() || !s.options.disabled_rule_tags.empty() || !s.options.enabled_rule_tags.empty()) &&
+		!s.config->m_rules_selection.empty())
+	{
+		return run_result::fatal("Specifying -D, -t, -T command line options together with \"rules:\" configuration or -o \"rules...\" is not supported.");
+	}
+
+	for (const auto& substring : s.options.disabled_rule_substrings)
+	{
+		falco_logger::log(falco_logger::level::INFO, "Disabling rules matching substring: " + substring + "\n");
+		s.engine->enable_rule(substring, false);
+	}
+
+	if(!s.options.disabled_rule_tags.empty())
+	{
+		for(const auto &tag : s.options.disabled_rule_tags)
+		{
+			falco_logger::log(falco_logger::level::INFO, "Disabling rules with tag: " + tag + "\n");
+		}
+		s.engine->enable_rule_by_tag(s.options.disabled_rule_tags, false);
+	}
+
+	if(!s.options.enabled_rule_tags.empty())
+	{
+		// Since we only want to enable specific
+		// rules, first disable all rules.
+		s.engine->enable_rule(all_rules, false);
+		for(const auto &tag : s.options.enabled_rule_tags)
+		{
+			falco_logger::log(falco_logger::level::INFO, "Enabling rules with tag: " + tag + "\n");
+		}
+		s.engine->enable_rule_by_tag(s.options.enabled_rule_tags, true);
+	}
+
 	for(const auto& sel : s.config->m_rules_selection)
 	{
 		bool enable = sel.m_op == falco_configuration::rule_selection_operation::enable;

userspace/falco/app/actions/print_config_schema.cpp

@@ -1,31 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-/*
-Copyright (C) 2024 The Falco Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-#include "actions.h"
-
-using namespace falco::app;
-using namespace falco::app::actions;
-
-falco::app::run_result falco::app::actions::print_config_schema(falco::app::state &s)
-{
-	if(s.options.print_config_schema)
-	{
-		printf("%s", s.config->m_config_schema.dump(2).c_str());
-		return run_result::exit();
-	}
-	return run_result::ok();
-}

userspace/falco/app/actions/process_events.cpp

@@ -312,9 +312,7 @@ static falco::app::run_result do_inspect(
 		{
 			for(auto& rule_res : *res)
 			{
-				s.outputs->handle_event(
-					rule_res.evt, rule_res.rule, rule_res.source, rule_res.priority_num,
-					rule_res.format, rule_res.tags, rule_res.extra_output_fields);
+				s.outputs->handle_event(rule_res.evt, rule_res.rule, rule_res.source, rule_res.priority_num, rule_res.format, rule_res.tags);
 			}
 		}
 

userspace/falco/app/app.cpp

@@ -60,7 +60,6 @@ bool falco::app::run(falco::app::state& s, bool& restart, std::string& errstr)
 	// dependencies are honored (e.g. don't process events before
 	// loading plugins, opening inspector, etc.).
 	std::list<app_action> run_steps = {
-		falco::app::actions::print_config_schema,
 		falco::app::actions::load_config,
 		falco::app::actions::print_help,
 		falco::app::actions::print_kernel_version,

userspace/falco/app/options.cpp

@@ -95,6 +95,40 @@ bool options::parse(int argc, char **argv, std::string &errstr)
 		}
 	}
 
+	// Convert the vectors of enabled/disabled tags into sets to match falco engine API
+	if(m_cmdline_parsed.count("T") > 0)
+	{
+		falco_logger::log(falco_logger::level::WARNING, "The -T option is deprecated and will be removed in Falco 0.39.0. Use -o rules[].disable.tag=<tag> instead.");
+		for(auto &tag : m_cmdline_parsed["T"].as<std::vector<std::string>>())
+		{
+			disabled_rule_tags.insert(tag);
+		}
+	}
+
+	if(m_cmdline_parsed.count("t") > 0)
+	{
+		falco_logger::log(falco_logger::level::WARNING, "The -t option is deprecated and will be removed in Falco 0.39.0. Use -o rules[].disable.rule=* -o rules[].enable.tag=<tag> instead.");
+		for(auto &tag : m_cmdline_parsed["t"].as<std::vector<std::string>>())
+		{
+			enabled_rule_tags.insert(tag);
+		}
+	}
+
+	if(disabled_rule_substrings.size() > 0)
+	{
+		falco_logger::log(falco_logger::level::WARNING, "The -D option is deprecated and will be removed in Falco 0.39.0. Use -o rules[].disable.rule=<wildcard-pattern> instead.");
+	}
+
+	// Some combinations of arguments are not allowed.
+
+	// You can't both disable and enable rules
+	if((disabled_rule_substrings.size() + disabled_rule_tags.size() > 0) &&
+	   !enabled_rule_tags.empty())
+	{
+		errstr = std::string("You can not specify both disabled (-D/-T) and enabled (-t) rules");
+		return false;
+	}
+
 	list_fields = m_cmdline_parsed.count("list") > 0;
 
 	return true;
@@ -110,11 +144,10 @@ void options::define(cxxopts::Options& opts)
 	opts.add_options()
 		("h,help",                        "Print this help list and exit.", cxxopts::value(help)->default_value("false"))
 #ifdef BUILD_TYPE_RELEASE
-		("c",                             	"Configuration file. If not specified uses " FALCO_INSTALL_CONF_FILE ".", cxxopts::value(conf_filename), "<path>")
+		("c",                             "Configuration file. If not specified uses " FALCO_INSTALL_CONF_FILE ".", cxxopts::value(conf_filename), "<path>")
 #else
 		("c",                             "Configuration file. If not specified tries " FALCO_SOURCE_CONF_FILE ", " FALCO_INSTALL_CONF_FILE ".", cxxopts::value(conf_filename), "<path>")
 #endif
-		("config-schema",                 "Print the config json schema and exit.", cxxopts::value(print_config_schema)->default_value("false"))
 		("A",                             "Monitor all events supported by Falco and defined in rules and configs. Some events are ignored by default when -A is not specified (the -i option lists these events ignored). Using -A can impact performance. This option has no effect when reproducing events from a capture file.", cxxopts::value(all_events)->default_value("false"))
 		("b,print-base64",                "Print data buffers in base64. This is useful for encoding binary data that needs to be used over media designed to consume this format.")
 #if !defined(_WIN32) && !defined(__EMSCRIPTEN__) && !defined(MINIMAL_BUILD)
@@ -123,6 +156,7 @@ void options::define(cxxopts::Options& opts)
 #endif
 		("disable-source",                "Turn off a specific <event_source>. By default, all loaded sources get enabled. Available sources are 'syscall' plus all sources defined by loaded plugins supporting the event sourcing capability. This option can be passed multiple times, but turning off all event sources simultaneously is not permitted. This option can not be mixed with --enable-source. This option has no effect when reproducing events from a capture file.", cxxopts::value(disable_sources), "<event_source>")
 		("dry-run",                       "Run Falco without processing events. It can help check that the configuration and rules do not have any errors.", cxxopts::value(dry_run)->default_value("false"))
+		("D",                             "DEPRECATED: use -o rules[].disable.rule=<wildcard-pattern> instead. Turn off any rules with names having the substring <substring>. This option can be passed multiple times. It cannot be mixed with -t.", cxxopts::value(disabled_rule_substrings), "<substring>")
 		("enable-source",                 "Enable a specific <event_source>. By default, all loaded sources get enabled. Available sources are 'syscall' plus all sources defined by loaded plugins supporting the event sourcing capability. This option can be passed multiple times. When using this option, only the event sources specified by it will be enabled. This option can not be mixed with --disable-source. This option has no effect when reproducing events from a capture file.", cxxopts::value(enable_sources), "<event_source>")
 #ifdef HAS_GVISOR
 		("gvisor-generate-config",		  "Generate a configuration file that can be used for gVisor and exit. See --gvisor-config for more details.", cxxopts::value<std::string>(gvisor_generate_config_with_socket)->implicit_value("/run/falco/gvisor.sock"), "<socket_path>")
@@ -143,6 +177,8 @@ void options::define(cxxopts::Options& opts)
 		("r",                             "Rules file or directory to be loaded. This option can be passed multiple times. Falco defaults to the values in the configuration file when this option is not specified.", cxxopts::value<std::vector<std::string>>(), "<rules_file>")
 		("S,snaplen",                     "Collect only the first <len> bytes of each I/O buffer for 'syscall' events. By default, the first 80 bytes are collected by the driver and sent to the user space for processing. Use this option with caution since it can have a strong performance impact.", cxxopts::value(snaplen)->default_value("0"), "<len>")
 		("support",                       "Print support information, including version, rules files used, loaded configuration, etc., and exit. The output is in JSON format.", cxxopts::value(print_support)->default_value("false"))
+		("T",                             "DEPRECATED: use -o rules[].disable.tag=<tag> instead. Turn off any rules with a tag=<tag>. This option can be passed multiple times. This option can not be mixed with -t.", cxxopts::value<std::vector<std::string>>(), "<tag>")
+		("t",                             "DEPRECATED: use -o rules[].disable.rule=* -o rules[].enable.tag=<tag> instead. Only enable those rules with a tag=<tag>. This option can be passed multiple times. This option can not be mixed with -T/-D.", cxxopts::value<std::vector<std::string>>(), "<tag>")
 		("U,unbuffered",                  "Turn off output buffering for configured outputs. This causes every single line emitted by Falco to be flushed, which generates higher CPU usage but is useful when piping those outputs into another process or a script.", cxxopts::value(unbuffered_outputs)->default_value("false"))
 		("V,validate",                    "Read the contents of the specified <rules_file> file(s), validate the loaded rules, and exit. This option can be passed multiple times to validate multiple files.", cxxopts::value(validate_rules_filenames), "<rules_file>")
 		("v",                             "Enable verbose output.", cxxopts::value(verbose)->default_value("false"))

userspace/falco/app/options.h

@@ -40,13 +40,13 @@ public:
 
 	// Each of these maps directly to a command line option.
 	bool help = false;
-	bool print_config_schema = false;
 	std::string conf_filename;
 	bool all_events = false;
 	sinsp_evt::param_fmt event_buffer_format = sinsp_evt::PF_NORMAL;
 	std::vector<std::string> cri_socket_paths;
 	bool disable_cri_async = false;
 	std::vector<std::string> disable_sources;
+	std::vector<std::string> disabled_rule_substrings;
 	std::vector<std::string> enable_sources;
 	std::string gvisor_generate_config_with_socket;
 	bool describe_all_rules = false;
@@ -67,6 +67,8 @@ public:
 	std::list<std::string> rules_filenames;
 	uint64_t snaplen = 0;
 	bool print_support = false;
+	std::set<std::string> disabled_rule_tags;
+	std::set<std::string> enabled_rule_tags;
 	bool unbuffered_outputs = false;
 	std::vector<std::string> validate_rules_filenames;
 	bool verbose = false;

userspace/falco/configuration.h

@@ -47,9 +47,6 @@ enum class engine_kind_t : uint8_t
 	NODRIVER
 };
 
-// Map that holds { config filename | validation status } for each loaded config file.
-typedef std::map<std::string, std::string> config_loaded_res;
-
 class falco_configuration
 {
 public:
@@ -107,20 +104,11 @@ public:
 		std::string m_rule;
 	};
 
-	struct append_output_config {
-		std::string m_source;
-		std::string m_tag;
-		std::string m_rule;
-		std::string m_format;
-		std::unordered_map<std::string, std::string> m_formatted_fields;
-		std::set<std::string> m_raw_fields;
-	};
-
 	falco_configuration();
 	virtual ~falco_configuration() = default;
 
-	config_loaded_res init_from_file(const std::string& conf_filename, const std::vector<std::string>& cmdline_options);
-	config_loaded_res init_from_content(const std::string& config_content, const std::vector<std::string>& cmdline_options, const std::string& filename="default");
+	void init_from_file(const std::string& conf_filename, std::vector<std::string>& loaded_conf_files, const std::vector<std::string>& cmdline_options);
+	void init_from_content(const std::string& config_content, const std::vector<std::string>& cmdline_options, const std::string& filename="default");
 
 	std::string dump();
 
@@ -143,8 +131,6 @@ public:
 	std::list<std::string> m_loaded_rules_folders;
 	// Rule selection options passed by the user
 	std::vector<rule_selection_config> m_rules_selection;
-	// Append output configuration passed by the user
-	std::vector<append_output_config> m_append_output;
 
 	bool m_json_output;
 	bool m_json_include_output_property;
@@ -197,11 +183,6 @@ public:
 	bool m_metrics_include_empty_values;
 	std::vector<plugin_config> m_plugins;
 
-	// container engines
-	uint64_t m_container_engines_mask;
-	uint64_t m_container_engines_disable_cri_async;
-	std::vector<std::string> m_container_engines_cri_socket_paths;
-
 	// Falco engine
 	engine_kind_t m_engine_mode = engine_kind_t::KMOD;
 	kmod_config m_kmod = {};
@@ -211,11 +192,10 @@ public:
 	gvisor_config m_gvisor = {};
 
 	// Needed by tests
-	yaml_helper m_config;
-	nlohmann::json m_config_schema;
+	yaml_helper config;
 
 private:
-	void merge_config_files(const std::string& config_name, config_loaded_res &res);
+	void merge_config_files(const std::string& config_name, std::vector<std::string>& loaded_config_files);
 	void load_yaml(const std::string& config_name);
 	void init_logger();
 	void load_engine_config(const std::string& config_name);
@@ -230,114 +210,6 @@ private:
 };
 
 namespace YAML {
-	template<>
-	struct convert<falco_configuration::append_output_config> {
-		static Node encode(const falco_configuration::append_output_config & rhs) {
-			Node node;
-
-			if(rhs.m_source != "")
-			{
-				node["source"] = rhs.m_source;
-			}
-
-			if(rhs.m_rule != "")
-			{
-				node["rule"] = rhs.m_rule;
-			}
-
-			if(rhs.m_tag != "")
-			{
-				node["tag"] = rhs.m_tag;
-			}
-
-			if(rhs.m_format != "")
-			{
-				node["format"] = rhs.m_format;
-			}
-
-			for(auto const& field : rhs.m_formatted_fields)
-			{
-				YAML::Node field_node;
-				field_node[field.first] = field.second;
-				node["fields"].push_back(field_node);
-			}
-
-			for(auto const& field : rhs.m_raw_fields)
-			{
-				node["fields"].push_back(field);
-			}
-
-			return node;
-		}
-
-		static bool decode(const Node& node, falco_configuration::append_output_config & rhs) {
-			if(!node.IsMap())
-			{
-				return false;
-			}
-
-			if(node["source"])
-			{
-				rhs.m_source = node["source"].as<std::string>();
-			}
-
-			if(node["tag"])
-			{
-				rhs.m_tag = node["tag"].as<std::string>();
-			}
-
-			if(node["rule"])
-			{
-				rhs.m_rule = node["rule"].as<std::string>();
-			}
-
-			if(node["format"])
-			{
-				rhs.m_format = node["format"].as<std::string>();
-			}
-
-			if(node["fields"])
-			{
-				if(!node["fields"].IsSequence())
-				{
-					return false;
-				}
-
-				for(auto& field_definition : node["fields"])
-				{
-					if(field_definition.IsMap() && field_definition.size() == 1)
-					{
-						YAML::const_iterator def = field_definition.begin();
-						std::string key = def->first.as<std::string>();
-
-						// it is an error to redefine an existing key
-						if (rhs.m_formatted_fields.count(key) != 0 || rhs.m_raw_fields.count(key) != 0)
-						{
-							return false;
-						}
-
-						rhs.m_formatted_fields[key] = def->second.as<std::string>();
-					} else if (field_definition.IsScalar())
-					{
-						std::string key = field_definition.as<std::string>();
-
-						// it is an error to redefine an existing key
-						if (rhs.m_formatted_fields.count(key) != 0)
-						{
-							return false;
-						}
-
-						rhs.m_raw_fields.insert(key);
-					} else {
-						return false;
-					}
-				}
-			}
-
-			return true;
-		}
-	};
-
 	template<>
 	struct convert<falco_configuration::rule_selection_config> {
 		static Node encode(const falco_configuration::rule_selection_config & rhs) {

userspace/falco/falco_metrics.cpp

@@ -108,40 +108,6 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 			fs::path fs_path = item.first;
 			prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("falco_sha256_config_files", "falcosecurity", "falco", {{"file_name", fs_path.filename().stem()}, {"sha256", item.second}});
 		}
-
-		static std::string ifinfo_json_escaped;
-		auto ipv4list = inspector->get_ifaddr_list().get_ipv4_list();
-		auto ipv6list = inspector->get_ifaddr_list().get_ipv6_list();
-		nlohmann::json ipv4_json;
-		nlohmann::json ipv6_json;
-		if(ipv4list)
-		{
-			for (const auto& item : *ipv4list)
-			{
-				if(item.m_name == "lo")
-				{
-					continue;
-				}
-				ipv4_json[item.m_name] = item.addr_to_string();
-			}
-		}
-
-		if(ipv6list)
-		{
-			for (const auto& item : *ipv6list)
-			{
-				if(item.m_name == "lo")
-				{
-					continue;
-				}
-				ipv6_json[item.m_name] = item.addr_to_string();
-			}
-		}
-		nlohmann::json ifinfo_json;
-		ifinfo_json["ipv4"] = ipv4_json;
-		ifinfo_json["ipv6"] = ipv6_json;
-		ifinfo_json_escaped = ifinfo_json.dump();
-		prometheus_text += prometheus_metrics_converter.convert_metric_to_text_prometheus("host_ifinfo_json", "falcosecurity", "falco", {{"host_ifinfo_json", ifinfo_json_escaped}});
 #endif
 
 		for (const std::string& source: inspector->event_sources())
@@ -152,7 +118,7 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 
 		if (agent_info)
 		{
-			additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("start_ts",
+			additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("start_ts",
 												  METRICS_V2_MISC,
 												  METRIC_VALUE_TYPE_U64,
 												  METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
@@ -161,20 +127,20 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		}
 		if (machine_info)
 		{
-			additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("host_boot_ts",
+			additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_boot_ts",
 												  METRICS_V2_MISC,
 												  METRIC_VALUE_TYPE_U64,
 												  METRIC_VALUE_UNIT_TIME_TIMESTAMP_NS,
 												  METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
 												  machine_info->boot_ts_epoch));
-			additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("host_num_cpus",
+			additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("host_num_cpus",
 												  METRICS_V2_MISC,
 												  METRIC_VALUE_TYPE_U32,
 												  METRIC_VALUE_UNIT_COUNT,
 												  METRIC_VALUE_METRIC_TYPE_NON_MONOTONIC_CURRENT,
 												  machine_info->num_cpus));
 		}
-		additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("outputs_queue_num_drops",
+		additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("outputs_queue_num_drops",
 												METRICS_V2_MISC,
 												METRIC_VALUE_TYPE_U64,
 												METRIC_VALUE_UNIT_COUNT,
@@ -184,7 +150,7 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 		if (agent_info)
 		{
 			auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
-			additional_wrapper_metrics.emplace_back(libs::metrics::libsinsp_metrics::new_metric("duration_sec",
+			additional_wrapper_metrics.emplace_back(libs_metrics_collector.new_metric("duration_sec",
 												  METRICS_V2_MISC,
 												  METRIC_VALUE_TYPE_U64,
 												  METRIC_VALUE_UNIT_TIME_S_COUNT,
@@ -213,7 +179,7 @@ std::string falco_metrics::to_text(const falco::app::state& state)
 				auto count = rules_by_id[i]->load();
 				if (count > 0)
 				{
-					auto metric = libs::metrics::libsinsp_metrics::new_metric("rules_counters",
+					auto metric = libs_metrics_collector.new_metric("rules_counters",
 											METRICS_V2_RULE_COUNTERS,
 											METRIC_VALUE_TYPE_U64,
 											METRIC_VALUE_UNIT_COUNT,

userspace/falco/falco_outputs.cpp

@@ -127,8 +127,7 @@ void falco_outputs::add_output(const falco::outputs::config &oc)
 }
 
 void falco_outputs::handle_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
-				 falco_common::priority_type priority, const std::string &format, std::set<std::string> &tags,
-				 extra_output_field_t &extra_fields)
+				 falco_common::priority_type priority, const std::string &format, std::set<std::string> &tags)
 {
 	falco_outputs::ctrl_msg cmsg = {};
 	cmsg.ts = evt->get_ts();
@@ -158,30 +157,9 @@ void falco_outputs::handle_event(sinsp_evt *evt, const std::string &rule, const
 	}
 
 	cmsg.msg = m_formats->format_event(
-		evt, rule, source, falco_common::format_priority(priority), sformat, tags, m_hostname, extra_fields
+		evt, rule, source, falco_common::format_priority(priority), sformat, tags, m_hostname
 	);
-
-	auto fields = m_formats->get_field_values(evt, source, sformat);
-	for (auto const& ef : extra_fields)
-	{
-		// when formatting for the control message we always want strings,
-		// so we can simply format raw fields as string
-		std::string fformat = ef.second.first;
-		if (fformat.size() == 0)
-		{
-			continue;
-		}
-
-		if (!(fformat[0] == '*'))
-		{
-			fformat = "*" + fformat;
-		}
-
-		fields[ef.first] = m_formats->format_string(evt, fformat, source);
-	}
-
-	cmsg.fields = fields;
-
+	cmsg.fields = m_formats->get_field_values(evt, source, sformat);
 	cmsg.tags.insert(tags.begin(), tags.end());
 
 	cmsg.type = ctrl_msg_type::CTRL_MSG_OUTPUT;

userspace/falco/falco_outputs.h

@@ -59,8 +59,7 @@ public:
 		is an event that has matched some rule).
 	*/
 	void handle_event(sinsp_evt *evt, const std::string &rule, const std::string &source,
-			falco_common::priority_type priority, const std::string &format, std::set<std::string> &tags,
-			extra_output_field_t &extra_fields);
+			  falco_common::priority_type priority, const std::string &format, std::set<std::string> &tags);
 
 	/*!
 		\brief Format then send a generic message to all outputs.

userspace/falco/stats_writer.cpp

@@ -357,40 +357,6 @@ void stats_writer::collector::get_metrics_output_fields_wrapper(
 		metric_name_file_sha256 = "falco.sha256_config_file." + falco::utils::sanitize_metric_name(metric_name_file_sha256);
 		output_fields[metric_name_file_sha256] = item.second;
 	}
-
-	auto ipv4list = inspector->get_ifaddr_list().get_ipv4_list();
-	auto ipv6list = inspector->get_ifaddr_list().get_ipv6_list();
-	nlohmann::json ipv4_json;
-	nlohmann::json ipv6_json;
-	if(ipv4list)
-	{
-		for (const auto& item : *ipv4list)
-		{
-			if(item.m_name == "lo")
-			{
-				continue;
-			}
-			ipv4_json[item.m_name] = item.addr_to_string();
-		}
-	}
-
-	if(ipv6list)
-	{
-		for (const auto& item : *ipv6list)
-		{
-			if(item.m_name == "lo")
-			{
-				continue;
-			}
-			ipv6_json[item.m_name] = item.addr_to_string();
-		}
-	}
-	nlohmann::json ifinfo_json;
-	ifinfo_json["ipv4"] = ipv4_json;
-	ifinfo_json["ipv6"] = ipv6_json;
-	m_ifinfo_json_escaped = ifinfo_json.dump();
-	output_fields["falco.host_ifinfo_json"] = m_ifinfo_json_escaped;
-
 #endif
 	output_fields["evt.source"] = src;
 	for (size_t i = 0; i < sizeof(all_driver_engines) / sizeof(const char*); i++)

userspace/falco/stats_writer.h

@@ -80,7 +80,6 @@ public:
 		uint64_t m_last_n_evts = 0;
 		uint64_t m_last_n_drops = 0;
 		uint64_t m_last_num_evts = 0;
-		std::string m_ifinfo_json_escaped;
 	};
 
 	stats_writer(const stats_writer&) = delete;

userspace/falco/yaml_helper.h

@@ -32,14 +32,6 @@ limitations under the License.
 #include <iostream>
 #include <fstream>
 #include <filesystem>
-#include <numeric>
-
-#include <nlohmann/json.hpp>
-#include <valijson/adapters/nlohmann_json_adapter.hpp>
-#include <valijson/adapters/yaml_cpp_adapter.hpp>
-#include <valijson/schema.hpp>
-#include <valijson/schema_parser.hpp>
-#include <valijson/validator.hpp>
 
 #include "config_falco.h"
 
@@ -87,42 +79,27 @@ class yaml_helper
 {
 public:
 	inline static const std::string configs_key = "config_files";
-	inline static const std::string validation_ok = "ok";
-	inline static const std::string validation_failed = "failed";
-	inline static const std::string validation_none = "schema not provided";
 
 	/**
 	* Load the YAML document represented by the input string.
 	*/
-	void load_from_string(const std::string& input, const nlohmann::json& schema={}, std::string *validation=nullptr)
+	void load_from_string(const std::string& input)
 	{
 		m_root = YAML::Load(input);
 		pre_process_env_vars(m_root);
-
-		if (validation)
-		{
-			if(!schema.empty())
-			{
-				*validation = validate_node(m_root, schema);
-			}
-			else
-			{
-				*validation = validation_none;
-			}
-		}
 	}
 
 	/**
 	* Load the YAML document from the given file path.
 	*/
-	void load_from_file(const std::string& path, const nlohmann::json& schema={}, std::string *validation=nullptr)
+	void load_from_file(const std::string& path)
 	{
-		m_root = load_from_file_int(path, schema, validation);
+		m_root = load_from_file_int(path);
 	}
 
-	void include_config_file(const std::string& include_file_path, const nlohmann::json& schema={}, std::string *validation=nullptr)
+	void include_config_file(const std::string& include_file_path)
 	{
-		auto loaded_nodes = load_from_file_int(include_file_path, schema, validation);
+		auto loaded_nodes = load_from_file_int(include_file_path);
 		for(auto n : loaded_nodes)
 		{
 			/*
@@ -177,16 +154,6 @@ public:
 		node = value;
 	}
 
-	/**
-	 * Set the node identified by key to an object value
-	 */
-	void set_object(const std::string& key, const YAML::Node& value)
-	{
-		YAML::Node node;
-		get_node(node, key, true);
-		node = value;
-	}
-
 	/**
 	* Get the sequence value from the node identified by key.
 	*/
@@ -218,52 +185,13 @@ public:
 private:
 	YAML::Node m_root;
 
-	YAML::Node load_from_file_int(const std::string& path, const nlohmann::json& schema={}, std::string *validation=nullptr)
+	YAML::Node load_from_file_int(const std::string& path)
 	{
 		auto root = YAML::LoadFile(path);
 		pre_process_env_vars(root);
-
-		if (validation)
-		{
-			if(!schema.empty())
-			{
-				*validation = validate_node(root, schema);
-			}
-			else
-			{
-				*validation = validation_none;
-			}
-		}
 		return root;
 	}
 
-	std::string validate_node(const YAML::Node &node, const nlohmann::json& schema={})
-	{
-		// Validate the yaml against our json schema
-		valijson::Schema schemaDef;
-		valijson::SchemaParser schemaParser;
-		valijson::Validator validator(valijson::Validator::kWeakTypes);
-		valijson::ValidationResults validationResults;
-		valijson::adapters::YamlCppAdapter configAdapter(node);
-		valijson::adapters::NlohmannJsonAdapter schemaAdapter(schema);
-		schemaParser.populateSchema(schemaAdapter, schemaDef);
-
-		if (!validator.validate(schemaDef, configAdapter, &validationResults))
-		{
-			valijson::ValidationResults::Error error;
-			// report only the top-most error
-			if (validationResults.popError(error))
-			{
-				return std::string(validation_failed + " for ")
-				       + std::accumulate(error.context.begin(), error.context.end(), std::string(""))
-				       + ": "
-				       + error.description;
-			}
-			return validation_failed;
-		}
-		return validation_ok;
-	}
-
 	/*
 	 * When loading a yaml file,
 	 * we immediately pre process all scalar values through a visitor private API,
@@ -492,6 +420,5 @@ namespace YAML {
 
 			return true;
 		}
-		// The "encode" function is not needed here, in fact you can simply YAML::load any json string.
 	};
 }