tmp

Signed-off-by: Andrea Terzolo <andreaterzolo3@gmail.com>

.github/workflows/release.yaml

@@ -147,7 +147,7 @@ jobs:
           sed -i s/FALCOVER/${{ github.event.release.tag_name }}/g release-body.md
           
       - name: Generate release notes
-        uses: leodido/rn2md@0669e5f3b21492c11c2db43cd6e267566f5880f3
+        uses: leodido/rn2md@1378404a058ecf86701f3ab533d487333fc675a7
         with:
           milestone: ${{ github.event.release.tag_name }}
           output: ./notes.md

.github/workflows/reusable_test_packages.yaml

@@ -19,7 +19,7 @@ on:
 jobs:
   test-packages:
     # See https://github.com/actions/runner/issues/409#issuecomment-1158849936
-    runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-latest' }}
+    runs-on: ${{ (inputs.arch == 'aarch64' && 'actuated-arm64-8cpu-16gb') || 'ubuntu-22.04' }}
     steps:
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
@@ -43,26 +43,21 @@ jobs:
           tar -xvf $(ls falco-*.tar.gz)
           cd falco-${{ inputs.version }}-${{ inputs.arch }}
           sudo cp -r * /
-
+      
+      # Note: most probably the plugin related tests should be moved to the plugin repo sooner or later.
+      - name: Install needed artifacts using falcoctl
+        if: ${{ inputs.static == false }}
+        run: |
+          sudo mkdir -p /usr/share/falco/plugins
+          sudo falcoctl artifact install k8saudit-rules
+          sudo falcoctl artifact install cloudtrail-rules
+     
+      # We only run driver loader tests on x86_64
       - name: Install dependencies for falco-driver-loader tests
+        if: ${{ inputs.arch == 'x86_64' }}
         run: |
           sudo apt update -y
-          sudo apt install -y --no-install-recommends build-essential clang make llvm gcc dkms
-
-      - name: Install kernel headers (workaround)
-        if: inputs.arch == 'aarch64'
-        run: |
-          sudo mkdir -p /usr/src
-          sudo git clone --depth 1 --branch v$(uname -r) git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git /usr/src/linux
-          sudo rm /lib/modules/$(uname -r)/build
-          sudo ln -s /usr/src/linux-headers-$(uname -r)/ /lib/modules/$(uname -r)/build
-          sudo rm /lib/modules/$(uname -r)/source
-          sudo ln -s /usr/src/linux-headers-$(uname -r)/ /lib/modules/$(uname -r)/source
-
-      - name: Install kernel headers
-        if: inputs.arch == 'x86_64'
-        run: |
-          sudo apt install -y --no-install-recommends linux-headers-$(uname -r)
+          sudo apt install -y --no-install-recommends build-essential clang make llvm gcc dkms linux-headers-$(uname -r)
 
       - name: Install go-junit-report
         run: |
@@ -86,7 +81,9 @@ jobs:
           if ${{ inputs.static && 'false' || 'true' }}; then
             ./build/falcoctl.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
             ./build/k8saudit.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
-            sudo ./build/falco-driver-loader.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
+            if ${{ inputs.arch == 'x86_64' && 'true' || 'false' }}; then
+                sudo ./build/falco-driver-loader.test -test.timeout=90s -test.v >> ./report.txt 2>&1 || true
+            fi    
           fi
           cat ./report.txt | go-junit-report -set-exit-code > report.xml
           popd

CMakeLists.txt

@@ -180,7 +180,6 @@ include(static-analysis)
 # Shared build variables
 set(FALCO_SINSP_LIBRARY sinsp)
 set(FALCO_SHARE_DIR share/falco)
-set(FALCO_PLUGINS_DIR ${FALCO_SHARE_DIR}/plugins)
 set(FALCO_ABSOLUTE_SHARE_DIR "${CMAKE_INSTALL_PREFIX}/${FALCO_SHARE_DIR}")
 set(FALCO_BIN_DIR bin)
 
@@ -189,7 +188,6 @@ add_subdirectory(userspace/engine)
 add_subdirectory(userspace/falco)
 
 if(NOT WIN32 AND NOT APPLE AND NOT EMSCRIPTEN AND NOT MUSL_OPTIMIZED_BUILD)
-  include(plugins)
   include(falcoctl)
 endif()
 

cmake/modules/CPackConfig.cmake

@@ -44,11 +44,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux") # only Linux has drivers
     "${CMAKE_CURRENT_BINARY_DIR};${DRIVER_COMPONENT_NAME};${DRIVER_COMPONENT_NAME};/")
 endif()
 
-if(NOT WIN32 AND NOT APPLE AND NOT EMSCRIPTEN AND NOT MUSL_OPTIMIZED_BUILD) # static builds do not have plugins
-  list(APPEND CPACK_INSTALL_CMAKE_PROJECTS
-    "${CMAKE_CURRENT_BINARY_DIR};${PLUGINS_COMPONENT_NAME};${PLUGINS_COMPONENT_NAME};/")
-endif()
-
 if(NOT CPACK_GENERATOR)
   if (CMAKE_SYSTEM_NAME MATCHES "Linux")
     set(CPACK_GENERATOR DEB RPM TGZ)

cmake/modules/driver.cmake

@@ -34,8 +34,8 @@ else()
   # In case you want to test against another driver version (or branch, or commit) just pass the variable -
   # ie., `cmake -DDRIVER_VERSION=dev ..`
   if(NOT DRIVER_VERSION)
-    set(DRIVER_VERSION "ea23169c13df2ce5d8ba7d6faecdaa65f36140cb")
-    set(DRIVER_CHECKSUM "SHA256=c84c80a9a2241667e1c0be7a611f071c9b0264ac81b98103d2272b939337d02f")
+    set(DRIVER_VERSION "7.0.0+driver")
+    set(DRIVER_CHECKSUM "SHA256=9f2a0f14827c0d9d1c3d1abe45b8f074dea531ebeca9859363a92f0d2475757e")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

cmake/modules/falcoctl.cmake

@@ -16,14 +16,14 @@ include(ExternalProject)
 
 string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} FALCOCTL_SYSTEM_NAME)
 
-set(FALCOCTL_VERSION "0.7.0-beta5")
+set(FALCOCTL_VERSION "0.7.1")
 
 if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
     set(FALCOCTL_SYSTEM_PROC_GO "amd64")
-    set(FALCOCTL_HASH "e08cdd3bed96bda2e45f54d86aec0f1ad986963ff30624578283b218829df225")
+    set(FALCOCTL_HASH "f142507c0e2b1e7dc03fd0b1ec36b479eb171f1f58c17f90d2d8edeb00668ef5")
 else() # aarch64
     set(FALCOCTL_SYSTEM_PROC_GO "arm64")
-    set(FALCOCTL_HASH "582b6b73f77cfdf66dbcddadaa1073fa1802f24bd1670c1cf578e524fd3e8486")
+    set(FALCOCTL_HASH "93e4800b68e21057da82c8c7aafa0970598594d62cd9929ebb9b38a9c02159a6")
 endif()
 
 ExternalProject_Add(

cmake/modules/falcosecurity-libs.cmake

@@ -35,8 +35,8 @@ else()
   # In case you want to test against another falcosecurity/libs version (or branch, or commit) just pass the variable -
   # ie., `cmake -DFALCOSECURITY_LIBS_VERSION=dev ..`
   if(NOT FALCOSECURITY_LIBS_VERSION)
-    set(FALCOSECURITY_LIBS_VERSION "ea23169c13df2ce5d8ba7d6faecdaa65f36140cb")
-    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=c84c80a9a2241667e1c0be7a611f071c9b0264ac81b98103d2272b939337d02f")
+    set(FALCOSECURITY_LIBS_VERSION "0.14.1")
+    set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=defdea24bf3b176c63f10900d3716fe4373151965cc09d3fe67a31a3a9af0b13")
   endif()
 
   # cd /path/to/build && cmake /path/to/source

cmake/modules/njson.cmake

@@ -19,8 +19,9 @@ if(USE_BUNDLED_NLOHMANN_JSON)
         CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${PROJECT_BINARY_DIR}/njson-prefix -DJSON_BuildTests=OFF -DBUILD_TESTING=OFF
     )
 
-    set(nlohmann_json_DIR ${PROJECT_BINARY_DIR}/njson-prefix/include)
+    set(nlohmann_json_INCLUDE_DIRS ${PROJECT_BINARY_DIR}/njson-prefix/include)
 else()
     find_package(nlohmann_json CONFIG REQUIRED)
+    get_target_property(nlohmann_json_INCLUDE_DIRS nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES)
     add_custom_target(njson)
 endif()

cmake/modules/plugins.cmake

@@ -1,98 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-#
-# Copyright (C) 2023 The Falco Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-#
-
-include(ExternalProject)
-
-# 'stable' or 'dev'
-set(PLUGINS_DOWNLOAD_BUCKET "stable")
-
-string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} PLUGINS_SYSTEM_NAME)
-
-if(NOT DEFINED PLUGINS_COMPONENT_NAME)
-    set(PLUGINS_COMPONENT_NAME "${CMAKE_PROJECT_NAME}-plugins")
-endif()
-
-# k8saudit
-set(PLUGIN_K8S_AUDIT_VERSION "0.6.1")
-if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
-    set(PLUGIN_K8S_AUDIT_HASH "e2908ebf2c03feecd26307ceab55aec9cae1cbc63d6aa05e147d8786e7670fb0")
-else() # aarch64
-    set(PLUGIN_K8S_AUDIT_HASH "8987a995fa09518aebc488ba549448166d605596c2d6478c10415a9d9f5f05dd")
-endif()
-
-ExternalProject_Add(
-  k8saudit-plugin
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/k8saudit-${PLUGIN_K8S_AUDIT_VERSION}-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
-  URL_HASH "SHA256=${PLUGIN_K8S_AUDIT_HASH}"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/k8saudit-plugin-prefix/src/k8saudit-plugin/libk8saudit.so" DESTINATION "${FALCO_PLUGINS_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")
-
-ExternalProject_Add(
-  k8saudit-rules
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/k8saudit-rules-${PLUGIN_K8S_AUDIT_VERSION}.tar.gz"
-  URL_HASH "SHA256=36321b3f1d7969926073a4d40bbbb7b4b28805b038c067f140795210ab641161"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/k8saudit-rules-prefix/src/k8saudit-rules/k8s_audit_rules.yaml" DESTINATION "${FALCO_ETC_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")
-
-# cloudtrail
-set(PLUGIN_CLOUDTRAIL_VERSION "0.9.0")
-if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
-    set(PLUGIN_CLOUDTRAIL_HASH "c8dc8ea5337aa9475042e6441320a5188bbf76977e3a69dd34a49a6251f8e9ad")
-else() # aarch64
-    set(PLUGIN_CLOUDTRAIL_HASH "bea12e81409c3df5698f7ab6a740ee9698b9dd1275b5985810daf70ac505c810")
-endif()
-
-ExternalProject_Add(
-  cloudtrail-plugin
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/cloudtrail-${PLUGIN_CLOUDTRAIL_VERSION}-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
-  URL_HASH "SHA256=${PLUGIN_CLOUDTRAIL_HASH}"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so" DESTINATION "${FALCO_PLUGINS_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")
-
-ExternalProject_Add(
-  cloudtrail-rules
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/cloudtrail-rules-${PLUGIN_CLOUDTRAIL_VERSION}.tar.gz"
-  URL_HASH "SHA256=b0c2b6c78d61cc3e7fb66445bcd8f763d15eb4a24f518385377e704aacec6b3f"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/cloudtrail-rules-prefix/src/cloudtrail-rules/aws_cloudtrail_rules.yaml" DESTINATION "${FALCO_ETC_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")
-
-# json
-set(PLUGIN_JSON_VERSION "0.7.1")
-if(${CMAKE_HOST_SYSTEM_PROCESSOR} STREQUAL "x86_64")
-    set(PLUGIN_JSON_HASH "3177fd667b384df2ffd2ae3260bda867c407c09d3fbcae841af204b82c1341c1")
-else() # aarch64
-    set(PLUGIN_JSON_HASH "3b5d0a9190bfd08e21915f997f88ca314f2027564a022eb88eef80ff4e2c77fa")
-endif()
-
-ExternalProject_Add(
-  json-plugin
-  URL "https://download.falco.org/plugins/${PLUGINS_DOWNLOAD_BUCKET}/json-${PLUGIN_JSON_VERSION}-${PLUGINS_SYSTEM_NAME}-${CMAKE_HOST_SYSTEM_PROCESSOR}.tar.gz"
-  URL_HASH "SHA256=${PLUGIN_JSON_HASH}"
-  CONFIGURE_COMMAND ""
-  BUILD_COMMAND ""
-  INSTALL_COMMAND "")
-
-install(FILES "${PROJECT_BINARY_DIR}/json-plugin-prefix/src/json-plugin/libjson.so" DESTINATION "${FALCO_PLUGINS_DIR}" COMPONENT "${PLUGINS_COMPONENT_NAME}")

cmake/modules/rules.cmake

@@ -16,8 +16,8 @@ include(GNUInstallDirs)
 include(ExternalProject)
 
 # falco_rules.yaml
-set(FALCOSECURITY_RULES_FALCO_VERSION "falco-rules-2.0.0")
-set(FALCOSECURITY_RULES_FALCO_CHECKSUM "SHA256=48b6c5ae7a619a320eb51dbe036d1bc78622ab692956c9493390678874757b32")
+set(FALCOSECURITY_RULES_FALCO_VERSION "falco-rules-3.0.0-rc1")
+set(FALCOSECURITY_RULES_FALCO_CHECKSUM "SHA256=2e91799fee49c2daf58fb482e47410a21433eb116e02cde18206f7af87449ddb")
 set(FALCOSECURITY_RULES_FALCO_PATH "${PROJECT_BINARY_DIR}/falcosecurity-rules-falco-prefix/src/falcosecurity-rules-falco/falco_rules.yaml")
 ExternalProject_Add(
   falcosecurity-rules-falco

falco.yaml

@@ -41,7 +41,6 @@
 #     json_include_output_property
 #     json_include_tags_property
 #     buffered_outputs
-#     outputs (throttling)
 #     rule_matching
 #     outputs_queue
 # Falco outputs channels
@@ -62,13 +61,13 @@
 # Falco logging / alerting / metrics related to software functioning (advanced)
 #     output_timeout
 #     syscall_event_timeouts
-#     syscall_event_drops
+#     syscall_event_drops -> [CHANGE NOTICE] Automatic notifications will be simplified in Falco 0.38! If you depend on the detailed drop counters payload, use 'metrics.output_rule' along with 'metrics.kernel_event_counters_enabled' instead
 #     metrics
 # Falco performance tuning (advanced)
-#     syscall_buf_size_preset [DEPRECATED]
-#     syscall_drop_failed_exit [DEPRECATED]
+#     syscall_buf_size_preset [DEPRECATED] -> Replaced by `engine.<driver>.buf_size_preset` starting Falco 0.38!
+#     syscall_drop_failed_exit [DEPRECATED] -> Replaced by `engine.<driver>.drop_failed_exit` starting Falco 0.38! 
 #     base_syscalls
-#     modern_bpf.cpus_for_each_syscall_buffer [DEPRECATED]
+#     modern_bpf.cpus_for_each_syscall_buffer [DEPRECATED] -> Replaced by `engine.modern_ebpf.cpus_for_each_buffer` starting Falco 0.38! 
 
 
 ################################
@@ -80,9 +79,9 @@
 # configuration options from this config file as command-line arguments by using
 # the `-o` flag followed by the option name and value. In the following example,
 # three config options (`json_output`, `log_level`, and
-# `modern_bpf.cpus_for_each_syscall_buffer`) are passed as command-line
+# `engine.kind`) are passed as command-line
 # arguments with their corresponding values: falco -o "json_output=true"
-# -o "log_level=debug" -o "modern_bpf.cpus_for_each_syscall_buffer=4" 
+# -o "log_level=debug" -o "engine.kind=kmod" 
 # Please note that command-line arguments take precedence over the options
 # specified in this config file.
 
@@ -93,16 +92,32 @@
 
 # Customize Falco settings using environment variables:
 #
-# - "HOST_ROOT": Specifies the prefix to the underlying host `/proc` filesystem
+# - HOST_ROOT: Specifies the prefix to the underlying host `/proc` filesystem
 #   when deploying Falco over a container with read-only host mounts instead of
 #   directly on the host. Defaults to "/host". 
-# - "FALCO_BPF_PROBE": DEPRECATED. Specify a custom path to the BPF object code file (`bpf`
+#
+# - !!! [DEPRECATED] FALCO_BPF_PROBE: Specify a custom path to the BPF object code file (`bpf`
 #   driver). This is not needed for the modern_bpf driver. 
-# - "FALCO_HOSTNAME": Customize the hostname output field logged by Falco by
+#   -> Replaced by `engine.kind: ebpf` and `engine.ebpf` starting Falco 0.38!
+#
+# - FALCO_HOSTNAME: Customize the hostname output field logged by Falco by
 #   setting the "FALCO_HOSTNAME" environment variable.
-# - "FALCO_CGROUP_MEM_PATH": Specifies the file path holding the container
+#
+# - FALCO_CGROUP_MEM_PATH: Specifies the file path holding the container
 #   memory usage metric for the `metrics` feature. Defaults to
 #   "/sys/fs/cgroup/memory/memory.usage_in_bytes" (Kubernetes).
+#
+# - SKIP_DRIVER_LOADER is used by the Falco fat image to skip the driver loading part.
+#
+# - FALCO_FRONTEND is useful when set to noninteractive to skip the dialog choice during 
+# the installation of Falco deb/rpm packages. This setting is somewhat similar to DEBIAN_FRONTEND.
+#
+# - FALCO_DRIVER_CHOICE is useful when set to kmod, ebpf, or modern_ebpf (matching the names 
+# used in engine.kind in the Falco config) during the installation of Falco deb/rpm packages. 
+# It skips the dialog choice but retains the driver configuration.
+#
+# - FALCOCTL_ENABLED is useful when set to 'no' during the installation of Falco deb/rpm packages, 
+# disabling the automatic artifacts followed by falcoctl.
 
 
 #####################
@@ -164,7 +179,8 @@ rules_file:
 # - `modern_ebpf`: Modern eBPF (CO-RE eBPF probe)
 # - `gvisor`: gVisor (gVisor sandbox)
 # - `replay`: Replay a scap trace file
-# - `none`: No event producer loaded, useful to run with plugins.
+# - `nodriver`: No driver is injected into the system.
+#   This is useful to debug and to run plugins with 'syscall' source.
 #
 # Only one engine can be specified in the `kind` key.
 # Moreover, for each engine multiple options might be available,
@@ -482,7 +498,7 @@ buffered_outputs: false
 # deploying it in production.
 rule_matching: first
 
-# [Experimental] `outputs_queue`
+# [Stable] `outputs_queue`
 #
 # Falco utilizes tbb::concurrent_bounded_queue for handling outputs, and this parameter
 # allows you to customize the queue capacity. Please refer to the official documentation:
@@ -777,7 +793,7 @@ output_timeout: 2000
 syscall_event_timeouts:
   max_consecutives: 1000
 
-# [Stable] `syscall_event_drops`
+# [Stable] `syscall_event_drops` -> [CHANGE NOTICE] Automatic notifications will be simplified in Falco 0.38! If you depend on the detailed drop counters payload, use 'metrics.output_rule' along with 'metrics.kernel_event_counters_enabled' instead
 #
 # Generates "Falco internal: syscall event drop" rule output when `priority=debug` at minimum
 #
@@ -977,7 +993,7 @@ metrics:
 # Falco performance tuning (advanced) #
 #######################################
 
-# [DEPRECATED] `syscall_buf_size_preset`
+# [DEPRECATED] `syscall_buf_size_preset` -> Replaced by `engine.<driver>.buf_size_preset` starting Falco 0.38!
 #
 # Deprecated in favor of engine.{kmod,ebpf,modern_ebpf}.buf_size_preset.
 # This config is evaluated only if the default `engine` config block is not changed,
@@ -1033,7 +1049,7 @@ metrics:
 # if the default size is not suitable for your use case.
 syscall_buf_size_preset: 4
 
-# [DEPRECATED] `syscall_drop_failed_exit`
+# [DEPRECATED] `syscall_drop_failed_exit` -> Replaced by `engine.<driver>.drop_failed_exit` starting Falco 0.38! 
 #
 # Deprecated in favor of engine.{kmod,ebpf,modern_ebpf}.drop_failed_exit.
 # This config is evaluated only if the default `engine` config block is not changed,
@@ -1162,7 +1178,7 @@ base_syscalls:
   custom_set: []
   repair: false
 
-# [DEPRECATED] `modern_bpf.cpus_for_each_syscall_buffer`, modern_bpf only
+# [DEPRECATED] `modern_bpf.cpus_for_each_syscall_buffer`, modern_bpf only -> Replaced by `engine.modern_ebpf.cpus_for_each_buffer` starting Falco 0.38! 
 #
 # Deprecated in favor of engine.modern_ebpf.cpus_for_each_buffer.
 # This config is evaluated only if the default `engine` config block is not changed,

unit_tests/engine/test_add_source.cpp

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2023 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+
+#include <falco_engine.h>
+#include <evttype_index_ruleset.h>
+
+static std::string syscall_source_name = "syscall";
+
+// A variant of evttype_index_ruleset_factory that uses a singleton
+// for the underlying ruleset. This allows testing of
+// ruleset_for_source
+
+namespace
+{
+class test_ruleset_factory : public evttype_index_ruleset_factory
+{
+public:
+	test_ruleset_factory(std::shared_ptr<gen_event_filter_factory> factory):
+		evttype_index_ruleset_factory(factory)
+	{
+		ruleset = evttype_index_ruleset_factory::new_ruleset();
+	}
+
+	virtual ~test_ruleset_factory() = default;
+
+	inline std::shared_ptr<filter_ruleset> new_ruleset() override
+	{
+		return ruleset;
+	}
+
+	std::shared_ptr<filter_ruleset> ruleset;
+};
+}; // namespace
+
+TEST(AddSource, basic)
+{
+	falco_engine engine;
+	sinsp inspector;
+	sinsp_filter_check_list filterchecks;
+
+	auto filter_factory = std::shared_ptr<gen_event_filter_factory>(
+		new sinsp_filter_factory(&inspector, filterchecks));
+	auto formatter_factory = std::shared_ptr<gen_event_formatter_factory>(
+		new sinsp_evt_formatter_factory(&inspector, filterchecks));
+	test_ruleset_factory *test_factory = new test_ruleset_factory(filter_factory);
+	auto ruleset_factory = std::shared_ptr<filter_ruleset_factory>(test_factory);
+
+	falco_source syscall_source;
+	syscall_source.name = syscall_source_name;
+	syscall_source.ruleset = ruleset_factory->new_ruleset();
+	syscall_source.ruleset_factory = ruleset_factory;
+	syscall_source.filter_factory = filter_factory;
+	syscall_source.formatter_factory = formatter_factory;
+
+	size_t source_idx = engine.add_source(syscall_source_name,
+					      filter_factory,
+					      formatter_factory,
+					      ruleset_factory);
+
+	ASSERT_TRUE(engine.is_source_valid(syscall_source_name));
+
+	ASSERT_EQ(engine.filter_factory_for_source(syscall_source_name), filter_factory);
+	ASSERT_EQ(engine.filter_factory_for_source(source_idx), filter_factory);
+
+	ASSERT_EQ(engine.formatter_factory_for_source(syscall_source_name), formatter_factory);
+	ASSERT_EQ(engine.formatter_factory_for_source(source_idx), formatter_factory);
+
+	ASSERT_EQ(engine.ruleset_factory_for_source(syscall_source_name), ruleset_factory);
+	ASSERT_EQ(engine.ruleset_factory_for_source(source_idx), ruleset_factory);
+
+	ASSERT_EQ(engine.ruleset_for_source(syscall_source_name), test_factory->ruleset);
+	ASSERT_EQ(engine.ruleset_for_source(source_idx), test_factory->ruleset);
+}

unit_tests/engine/test_rule_loader.cpp

@@ -0,0 +1,970 @@
+#include <gtest/gtest.h>
+
+#include "falco_engine.h"
+#include "rule_loader_reader.h"
+#include "rule_loader_compiler.h"
+#include "rule_loading_messages.h"
+
+class engine_loader_test : public ::testing::Test {
+protected:
+	void SetUp() override
+	{
+		m_sample_ruleset = "sample-ruleset";
+		m_sample_source = falco_common::syscall_source;
+
+		// create a falco engine ready to load the ruleset
+		m_inspector.reset(new sinsp());
+		m_engine.reset(new falco_engine());
+		m_filter_factory = std::shared_ptr<gen_event_filter_factory>(
+			new sinsp_filter_factory(m_inspector.get(), m_filterlist));
+		m_formatter_factory = std::shared_ptr<gen_event_formatter_factory>(
+			new sinsp_evt_formatter_factory(m_inspector.get(), m_filterlist));
+		m_engine->add_source(m_sample_source, m_filter_factory, m_formatter_factory);
+	}
+
+	void TearDown() override
+	{
+
+	}
+
+	bool load_rules(std::string rules_content, std::string rules_filename)
+	{
+		bool ret = false;
+		falco::load_result::rules_contents_t rc = {{rules_filename, rules_content}};
+		m_load_result = m_engine->load_rules(rules_content, rules_filename);
+		m_load_result_string = m_load_result->as_string(true, rc);
+		m_load_result_json = m_load_result->as_json(rc);
+		ret = m_load_result->successful();
+
+		if (ret)
+		{
+			m_engine->enable_rule("", true, m_sample_ruleset);
+		}
+
+		return ret;
+	}
+
+	// This must be kept in line with the (private) falco_engine::s_default_ruleset
+	uint64_t num_rules_for_ruleset(std::string ruleset = "falco-default-ruleset")
+	{
+		return m_engine->num_rules_for_ruleset(ruleset);
+	}
+
+	bool has_warnings()
+	{
+		return m_load_result->has_warnings();
+	}
+
+	bool check_warning_message(std::string warning_msg)
+	{
+		if(!m_load_result->has_warnings())
+		{
+			return false;
+		}
+
+		for(auto &warn : m_load_result_json["warnings"])
+		{
+			std::string msg = warn["message"];
+			// Debug:
+			// printf("msg: %s\n", msg.c_str());
+			if(msg.find(warning_msg) != std::string::npos)
+			{
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	bool check_error_message(std::string error_msg)
+	{
+		// if the loading is successful there are no errors
+		if(m_load_result->successful())
+		{
+			return false;
+		}
+
+		for(auto &err : m_load_result_json["errors"])
+		{
+			std::string msg = err["message"];
+			// Debug:
+			// printf("msg: %s\n", msg.c_str());
+			if(msg.find(error_msg) != std::string::npos)
+			{
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	std::string get_compiled_rule_condition(std::string rule_name = "")
+	{
+		auto rule_description = m_engine->describe_rule(&rule_name, {});
+		return rule_description["rules"][0]["details"]["condition_compiled"].template get<std::string>();
+	}
+
+	std::string m_sample_ruleset;
+	std::string m_sample_source;
+	sinsp_filter_check_list m_filterlist;
+	std::shared_ptr<gen_event_filter_factory> m_filter_factory;
+	std::shared_ptr<gen_event_formatter_factory> m_formatter_factory;
+	std::unique_ptr<falco_engine> m_engine;
+	std::unique_ptr<falco::load_result> m_load_result;
+	std::string m_load_result_string;
+	nlohmann::json m_load_result_json;
+	std::unique_ptr<sinsp> m_inspector;
+};
+
+std::string s_sample_ruleset = "sample-ruleset";
+std::string s_sample_source = falco_common::syscall_source;
+
+TEST_F(engine_loader_test, list_append)
+{
+    std::string rules_content = R"END(
+- list: shell_binaries
+  items: [ash, bash, csh, ksh, sh, tcsh, zsh, dash]
+
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open and proc.name in (shell_binaries)
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- list: shell_binaries
+  items: [pwsh]
+  override:
+    items: append
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+	ASSERT_EQ(get_compiled_rule_condition("legit_rule"),"(evt.type = open and proc.name in (ash, bash, csh, ksh, sh, tcsh, zsh, dash, pwsh))");
+}
+
+TEST_F(engine_loader_test, condition_append)
+{
+    std::string rules_content = R"END(
+- macro: interactive
+  condition: >
+    ((proc.aname=sshd and proc.name != sshd) or
+    proc.name=systemd-logind or proc.name=login)
+
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open and interactive
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- macro: interactive
+  condition: or proc.name = ssh
+  override:
+    condition: append
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+	ASSERT_EQ(get_compiled_rule_condition("legit_rule"),"(evt.type = open and (((proc.aname = sshd and proc.name != sshd) or proc.name = systemd-logind or proc.name = login) or proc.name = ssh))");
+}
+
+TEST_F(engine_loader_test, rule_override_append)
+{
+    std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: legit_rule
+  desc: with append
+  condition: and proc.name = cat
+  output: proc=%proc.name
+  override:
+    desc: append
+    condition: append
+    output: append
+)END";
+
+	std::string rule_name = "legit_rule";
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	// Here we don't use the deprecated `append` flag, so we don't expect the warning.
+	ASSERT_FALSE(check_warning_message(WARNING_APPEND));
+
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	ASSERT_EQ(rule_description["rules"][0]["info"]["condition"].template get<std::string>(),
+	 	"evt.type=open and proc.name = cat");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["output"].template get<std::string>(),
+	 	"user=%user.name command=%proc.cmdline file=%fd.name proc=%proc.name");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["description"].template get<std::string>(),
+	 	"legit rule description with append");
+}
+
+TEST_F(engine_loader_test, rule_append)
+{
+    std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: legit_rule
+  condition: and proc.name = cat
+  append: true
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	// We should have at least one warning because the 'append' flag is deprecated.
+	ASSERT_TRUE(check_warning_message(WARNING_APPEND));
+
+	ASSERT_EQ(get_compiled_rule_condition("legit_rule"),"(evt.type = open and proc.name = cat)");
+}
+
+TEST_F(engine_loader_test, rule_override_replace)
+{
+    std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: legit_rule
+  desc: a replaced legit description
+  condition: evt.type = close
+  override:
+    desc: replace
+    condition: replace
+)END";
+
+	std::string rule_name = "legit_rule";
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	ASSERT_EQ(rule_description["rules"][0]["info"]["condition"].template get<std::string>(),
+	 	"evt.type = close");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["output"].template get<std::string>(),
+	 	"user=%user.name command=%proc.cmdline file=%fd.name");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["description"].template get<std::string>(),
+	 	"a replaced legit description");
+}
+
+TEST_F(engine_loader_test, rule_override_append_replace)
+{
+    std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: legit_rule
+  desc: a replaced legit description
+  condition: and proc.name = cat
+  priority: WARNING
+  override:
+    desc: replace
+    condition: append
+    priority: replace
+)END";
+
+	std::string rule_name = "legit_rule";
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	ASSERT_EQ(rule_description["rules"][0]["info"]["condition"].template get<std::string>(),
+	 	"evt.type = close and proc.name = cat");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["output"].template get<std::string>(),
+	 	"user=%user.name command=%proc.cmdline file=%fd.name");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["description"].template get<std::string>(),
+	 	"a replaced legit description");
+
+	ASSERT_EQ(rule_description["rules"][0]["info"]["priority"].template get<std::string>(),
+	 	"Warning");
+}
+
+TEST_F(engine_loader_test, rule_incorrect_override_type)
+{
+    std::string rules_content = R"END(
+- rule: failing_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: failing_rule
+  desc: an appended incorrect field
+  condition: and proc.name = cat
+  priority: WARNING
+  override:
+    desc: replace
+    condition: append
+    priority: append
+)END";
+
+	std::string rule_name = "failing_rule";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("Key 'priority' cannot be appended to, use 'replace' instead"));
+	ASSERT_TRUE(std::string(m_load_result_json["errors"][0]["context"]["snippet"]).find("priority: append") != std::string::npos);
+}
+
+TEST_F(engine_loader_test, rule_incorrect_append_override)
+{
+    std::string rules_content = R"END(
+- rule: failing_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: failing_rule
+  desc: an appended incorrect field
+  condition: and proc.name = cat
+  append: true
+  override:
+    desc: replace
+    condition: append
+)END";
+
+	std::string rule_name = "failing_rule";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	
+	// We should have at least one warning because the 'append' flag is deprecated.
+	ASSERT_TRUE(check_warning_message(WARNING_APPEND));
+	
+	ASSERT_TRUE(check_error_message(ERROR_OVERRIDE_APPEND));
+}
+
+TEST_F(engine_loader_test, macro_override_append_before_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  override:
+    condition: append
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a macro override before the macro definition.
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_MACRO));
+}
+
+TEST_F(engine_loader_test, macro_override_replace_before_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  override:
+    condition: replace
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// The first override defines a macro that is overridden by the second macro definition
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"evt.type in (open, openat)");	
+}
+
+TEST_F(engine_loader_test, macro_append_before_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  append: true
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a macro override before the macro definition.
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_MACRO));
+}
+
+TEST_F(engine_loader_test, macro_override_append_after_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  override:
+    condition: append
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a macro override before the macro definition.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) or evt.type = openat2)");
+}
+
+TEST_F(engine_loader_test, macro_append_after_macro_definition)
+{
+    std::string rules_content = R"END(
+
+- macro: open_simple
+  condition: evt.type in (open,openat)
+
+- macro: open_simple
+  condition: or evt.type = openat2
+  append: true
+
+- rule: test_rule
+  desc: simple rule
+  condition: open_simple
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a macro override before the macro definition.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) or evt.type = openat2)");
+}
+
+TEST_F(engine_loader_test, rule_override_append_before_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  condition: and proc.name = cat
+  override:
+    condition: append
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_RULE_APPEND));
+}
+
+TEST_F(engine_loader_test, rule_override_replace_before_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  condition: and proc.name = cat
+  override:
+    condition: replace
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_RULE_REPLACE));
+}
+
+TEST_F(engine_loader_test, rule_append_before_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  condition: and proc.name = cat
+  append: true
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_RULE_APPEND));
+}
+
+TEST_F(engine_loader_test, rule_override_append_after_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+- rule: test_rule
+  condition: and proc.name = cat
+  override:
+    condition: append
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) and proc.name = cat)");
+}
+
+TEST_F(engine_loader_test, rule_append_after_rule_definition)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type in (open,openat)
+  output: command=%proc.cmdline
+  priority: INFO
+
+- rule: test_rule
+  condition: and proc.name = cat
+  append: true
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type in (open, openat) and proc.name = cat)");
+}
+
+TEST_F(engine_loader_test, list_override_append_wrong_key)
+{
+	// todo: maybe we want to manage some non-existent keys
+	// Please note how the non-existent key 'non-existent keys' is ignored.
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  override_written_wrong:
+    items: append
+
+- list: dev_creation_binaries
+  items: [blkid]
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// Since there is a wrong key in the first list definition the `override` is not
+	// considered. so in this situation, we are defining the list 2 times. The 
+	// second one overrides the first one.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid))");
+}
+
+TEST_F(engine_loader_test, list_override_append_before_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  override:
+    items: append
+
+- list: dev_creation_binaries
+  items: [blkid]
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a list override before the list definition.
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_LIST));
+}
+
+TEST_F(engine_loader_test, list_override_replace_before_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  override:
+    items: replace
+
+- list: dev_creation_binaries
+  items: [blkid]
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// With override replace we define a first list that then is overridden by the second one.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid))");
+}
+
+TEST_F(engine_loader_test, list_append_before_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  append: true
+
+- list: dev_creation_binaries
+  items: [blkid]
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	// We cannot define a list append before the list definition.
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message(ERROR_NO_PREVIOUS_LIST));
+}
+
+TEST_F(engine_loader_test, list_override_append_after_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: [blkid]
+
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  override:
+    items: append
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid, csi-provisioner, csi-attacher))");
+}
+
+TEST_F(engine_loader_test, list_append_after_list_definition)
+{
+    std::string rules_content = R"END(
+- list: dev_creation_binaries
+  items: [blkid]
+
+- list: dev_creation_binaries
+  items: ["csi-provisioner", "csi-attacher"]
+  append: true
+
+- rule: test_rule
+  desc: simple rule
+  condition: evt.type = execve and proc.name in (dev_creation_binaries)
+  output: command=%proc.cmdline
+  priority: INFO
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"(evt.type = execve and proc.name in (blkid, csi-provisioner, csi-attacher))");
+}
+
+TEST_F(engine_loader_test, rule_override_without_field)
+{
+    std::string rules_content = R"END(
+- rule: failing_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: failing_rule
+  desc: an appended incorrect field
+  override:
+    desc: replace
+    condition: append
+)END";
+
+	std::string rule_name = "failing_rule";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("An append override for 'condition' was specified but 'condition' is not defined"));
+}
+
+TEST_F(engine_loader_test, rule_override_extra_field)
+{
+    std::string rules_content = R"END(
+- rule: failing_rule
+  desc: legit rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+
+- rule: failing_rule
+  desc: an appended incorrect field
+  condition: and proc.name = cat
+  priority: WARNING
+  override:
+    desc: replace
+    condition: append
+)END";
+
+	std::string rule_name = "failing_rule";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("Unexpected key 'priority'"));
+}
+
+TEST_F(engine_loader_test, missing_enabled_key_with_override)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  desc: missing enabled key
+  condition: and proc.name = cat
+  override:
+    desc: replace
+    condition: append
+    enabled: replace
+)END";
+
+	// In the rule override we miss `enabled: true`
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("'enabled' was specified but 'enabled' is not defined"));
+}
+
+TEST_F(engine_loader_test, rule_override_with_enabled)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  desc: correct override
+  condition: and proc.name = cat
+  enabled: true
+  override:
+    desc: replace
+    condition: append
+    enabled: replace
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_FALSE(has_warnings());
+	// The rule should be enabled at the end.
+	EXPECT_EQ(num_rules_for_ruleset(), 1);
+}
+
+TEST_F(engine_loader_test, rule_not_enabled)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: rule not enabled
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_FALSE(has_warnings());
+	EXPECT_EQ(num_rules_for_ruleset(), 0);
+}
+
+TEST_F(engine_loader_test, rule_enabled_warning)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  enabled: true
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_warning_message(WARNING_ENABLED));
+	// The rule should be enabled at the end.
+	EXPECT_EQ(num_rules_for_ruleset(), 1);
+}
+
+// todo!: Probably we shouldn't allow this syntax
+TEST_F(engine_loader_test, rule_enabled_is_ignored_by_append)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  condition: and proc.name = cat
+  append: true
+  enabled: true
+)END";
+
+	// 'enabled' is ignored by the append, this syntax is not supported
+	// so the rule is not enabled.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	EXPECT_EQ(num_rules_for_ruleset(), 0);
+}
+
+// todo!: Probably we shouldn't allow this syntax
+TEST_F(engine_loader_test, rewrite_rule)
+{
+    std::string rules_content = R"END(
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+- rule: test_rule
+  desc: redefined rule syntax
+  condition: proc.name = cat
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: WARNING
+  enabled: true
+)END";
+
+	// The above syntax is not supported, we cannot override the content
+	// of a rule in this way.
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	// In this case the rule is completely overridden but this syntax is not supported.
+	EXPECT_EQ(num_rules_for_ruleset(), 1);
+	ASSERT_EQ(get_compiled_rule_condition("test_rule"),"proc.name = cat");
+}
+
+TEST_F(engine_loader_test, required_engine_version_semver)
+{
+    std::string rules_content = R"END(
+- required_engine_version: 0.26.0
+
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_FALSE(has_warnings());
+}
+
+TEST_F(engine_loader_test, required_engine_version_not_semver)
+{
+    std::string rules_content = R"END(
+- required_engine_version: 26
+
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_FALSE(has_warnings());
+}
+
+TEST_F(engine_loader_test, required_engine_version_invalid)
+{
+    std::string rules_content = R"END(
+- required_engine_version: seven
+
+- rule: test_rule
+  desc: test rule description
+  condition: evt.type = close
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+  enabled: false
+
+)END";
+
+	ASSERT_FALSE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(check_error_message("Unable to parse engine version"));
+}
+
+// checks for issue described in https://github.com/falcosecurity/falco/pull/3028
+TEST_F(engine_loader_test, list_value_with_escaping)
+{
+    std::string rules_content = R"END(
+- list: my_list
+  items: [non_escaped_val, "escaped val"]
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "rules.yaml"));
+	ASSERT_TRUE(m_load_result->successful());
+  ASSERT_TRUE(m_load_result->has_warnings()); // a warning for the unused list
+
+  auto rule_description = m_engine->describe_rule(nullptr, {});
+  ASSERT_TRUE(m_load_result->successful());
+  ASSERT_EQ(rule_description["rules"].size(), 0);
+  ASSERT_EQ(rule_description["macros"].size(), 0);
+  ASSERT_EQ(rule_description["lists"].size(), 1);
+
+  // escaped values must not be interpreted as list refs by mistake
+  ASSERT_EQ(rule_description["lists"][0]["details"]["lists"].size(), 0);
+
+  // values should be escaped correctly
+  ASSERT_EQ(rule_description["lists"][0]["details"]["items_compiled"].size(), 2);
+  ASSERT_EQ(rule_description["lists"][0]["details"]["items_compiled"][0].template get<std::string>(), "non_escaped_val");
+  ASSERT_EQ(rule_description["lists"][0]["details"]["items_compiled"][1].template get<std::string>(), "escaped val");
+}

userspace/engine/CMakeLists.txt

@@ -39,7 +39,7 @@ PUBLIC
     ${LIBSCAP_INCLUDE_DIRS}
     ${LIBSINSP_INCLUDE_DIRS}
     ${PROJECT_BINARY_DIR}/userspace/engine
-    ${nlohmann_json_DIR}
+    ${nlohmann_json_INCLUDE_DIRS}
     ${TBB_INCLUDE_DIR}
     ${YAMLCPP_INCLUDE_DIR}
 )

userspace/engine/falco_engine.cpp

@@ -79,26 +79,6 @@ sinsp_version falco_engine::engine_version()
 	return sinsp_version(FALCO_ENGINE_VERSION);
 }
 
-const falco_source* falco_engine::find_source(const std::string& name) const
-{
-	auto ret = m_sources.at(name);
-	if(!ret)
-	{
-		throw falco_exception("Unknown event source " + name);
-	}
-	return ret;
-}
-
-const falco_source* falco_engine::find_source(std::size_t index) const
-{
-	auto ret = m_sources.at(index);
-	if(!ret)
-	{
-		throw falco_exception("Unknown event source index " + std::to_string(index));
-	}
-	return ret;
-}
-
 // Return a key that uniquely represents a field class.
 // For now, we assume name + shortdesc is unique.
 static std::string fieldclass_key(const gen_event_filter_factory::filter_fieldclass_info &fld_info)
@@ -177,15 +157,6 @@ void falco_engine::list_fields(std::string &source, bool verbose, bool names_onl
 	}
 }
 
-void falco_engine::load_rules(const std::string &rules_content, bool verbose, bool all_events)
-{
-	static const std::string no_name = "N/A";
-
-	std::unique_ptr<load_result> res = load_rules(rules_content, no_name);
-
-	interpret_load_result(res, no_name, rules_content, verbose);
-}
-
 std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_content, const std::string &name)
 {
 	rule_loader::configuration cfg(rules_content, m_sources, name);
@@ -257,44 +228,6 @@ std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_c
 	return std::move(cfg.res);
 }
 
-void falco_engine::load_rules_file(const std::string &rules_filename, bool verbose, bool all_events)
-{
-	std::string rules_content;
-
-	read_file(rules_filename, rules_content);
-
-	std::unique_ptr<load_result> res = load_rules(rules_content, rules_filename);
-
-	interpret_load_result(res, rules_filename, rules_content, verbose);
-}
-
-std::unique_ptr<load_result> falco_engine::load_rules_file(const std::string &rules_filename)
-{
-	std::string rules_content;
-
-	try {
-		read_file(rules_filename, rules_content);
-	}
-	catch (falco_exception &e)
-	{
-		rule_loader::context ctx(rules_filename);
-
-		std::unique_ptr<rule_loader::result> res(new rule_loader::result(rules_filename));
-
-		res->add_error(load_result::LOAD_ERR_FILE_READ, e.what(), ctx);
-
-// Old gcc versions (e.g. 4.8.3) won't allow move elision but newer versions
-// (e.g. 10.2.1) would complain about the redundant move.
-#if __GNUC__ > 4
-		return res;
-#else
-		return std::move(res);
-#endif
-	}
-
-	return load_rules(rules_content, rules_filename);
-}
-
 void falco_engine::enable_rule(const std::string &substring, bool enabled, const std::string &ruleset)
 {
 	uint16_t ruleset_id = find_ruleset_id(ruleset);
@@ -422,21 +355,7 @@ std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_ev
 	// source_idx, which means that at any time each filter_ruleset will only
 	// be accessed by a single thread.
 
-	const falco_source *source;
-
-	if(source_idx == m_syscall_source_idx)
-	{
-		if(m_syscall_source == NULL)
-		{
-			m_syscall_source = find_source(m_syscall_source_idx);
-		}
-
-		source = m_syscall_source;
-	}
-	else
-	{
-		source = find_source(source_idx);
-	}
+	const falco_source *source = find_source(source_idx);
 
 	if(should_drop_evt() || !source)
 	{
@@ -538,7 +457,7 @@ nlohmann::json falco_engine::describe_rule(std::string *rule, const std::vector<
 	// output of rules, macros, and lists.
 	if (m_last_compile_output == nullptr)
 	{
-		throw falco_exception("rules most be loaded before describing them");
+		throw falco_exception("rules must be loaded before describing them");
 	}
 
 	// use collected and compiled info to print a json output
@@ -941,6 +860,50 @@ bool falco_engine::is_source_valid(const std::string &source) const
 	return m_sources.at(source) != nullptr;
 }
 
+std::shared_ptr<gen_event_filter_factory> falco_engine::filter_factory_for_source(const std::string& source)
+{
+	return find_source(source)->filter_factory;
+}
+
+std::shared_ptr<gen_event_filter_factory> falco_engine::filter_factory_for_source(std::size_t source_idx)
+{
+	return find_source(source_idx)->filter_factory;
+}
+
+std::shared_ptr<gen_event_formatter_factory> falco_engine::formatter_factory_for_source(const std::string& source)
+{
+	return find_source(source)->formatter_factory;
+}
+
+std::shared_ptr<gen_event_formatter_factory> falco_engine::formatter_factory_for_source(std::size_t source_idx)
+{
+	return find_source(source_idx)->formatter_factory;
+}
+
+std::shared_ptr<filter_ruleset_factory> falco_engine::ruleset_factory_for_source(const std::string& source)
+{
+	return find_source(source)->ruleset_factory;
+}
+
+std::shared_ptr<filter_ruleset_factory> falco_engine::ruleset_factory_for_source(std::size_t source_idx)
+{
+	return find_source(source_idx)->ruleset_factory;
+}
+
+std::shared_ptr<filter_ruleset> falco_engine::ruleset_for_source(const std::string& source_name)
+{
+	const falco_source *source = find_source(source_name);
+
+	return source->ruleset;
+}
+
+std::shared_ptr<filter_ruleset> falco_engine::ruleset_for_source(std::size_t source_idx)
+{
+	const falco_source *source = find_source(source_idx);
+
+	return source->ruleset;
+}
+
 void falco_engine::read_file(const std::string& filename, std::string& contents)
 {
 	std::ifstream is;
@@ -955,29 +918,6 @@ void falco_engine::read_file(const std::string& filename, std::string& contents)
 			std::istreambuf_iterator<char>());
 }
 
-void falco_engine::interpret_load_result(std::unique_ptr<load_result>& res,
-					 const std::string& rules_filename,
-					 const std::string& rules_content,
-					 bool verbose)
-{
-	falco::load_result::rules_contents_t rc = {{rules_filename, rules_content}};
-
-	if(!res->successful())
-	{
-		// The output here is always the full e.g. "verbose" output.
-		throw falco_exception(res->as_string(true, rc).c_str());
-	}
-
-	if(verbose && res->has_warnings())
-	{
-		// Here, verbose controls whether to additionally
-		// "log" e.g. print to stderr. What's logged is always
-		// non-verbose so it fits on a single line.
-		// todo(jasondellaluce): introduce a logging callback in Falco
-		fprintf(stderr, "%s\n", res->as_string(false, rc).c_str());
-	}
-}
-
 static bool check_plugin_requirement_alternatives(
 		const std::vector<falco_engine::plugin_version_requirement>& plugins,
 		const rule_loader::plugin_version_info::requirement_alternatives& alternatives,

userspace/engine/falco_engine.h

@@ -74,15 +74,8 @@ public:
 	void list_fields(std::string &source, bool verbose, bool names_only, bool markdown) const;
 
 	//
-	// Load rules either directly or from a filename.
+	// Load rules and returns a result object.
 	//
-	void load_rules_file(const std::string &rules_filename, bool verbose, bool all_events);
-	void load_rules(const std::string &rules_content, bool verbose, bool all_events);
-
-	//
-	// Identical to above, but returns a result object instead of
-	// throwing exceptions on error.
-	std::unique_ptr<falco::load_result> load_rules_file(const std::string &rules_filename);
 	std::unique_ptr<falco::load_result> load_rules(const std::string &rules_content, const std::string &name);
 
 	//
@@ -242,6 +235,31 @@ public:
 	// factory for this source.
 	bool is_source_valid(const std::string &source) const;
 
+	//
+	// Given a source, return a formatter factory that can create
+	// filters for events of that source.
+	//
+	std::shared_ptr<gen_event_filter_factory> filter_factory_for_source(const std::string& source);
+	std::shared_ptr<gen_event_filter_factory> filter_factory_for_source(std::size_t source_idx);
+
+	//
+	// Given a source, return a formatter factory that can create
+	// formatters for an event.
+	//
+	std::shared_ptr<gen_event_formatter_factory> formatter_factory_for_source(const std::string& source);
+	std::shared_ptr<gen_event_formatter_factory> formatter_factory_for_source(std::size_t source_idx);
+
+	//
+	// Given a source, return a ruleset factory that can create
+	// rulesets for that source.
+	//
+	std::shared_ptr<filter_ruleset_factory> ruleset_factory_for_source(const std::string& source);
+	std::shared_ptr<filter_ruleset_factory> ruleset_factory_for_source(std::size_t source_idx);
+
+	// Return the filter_ruleset used for a given source.
+	std::shared_ptr<filter_ruleset> ruleset_for_source(const std::string& source);
+	std::shared_ptr<filter_ruleset> ruleset_for_source(std::size_t source_idx);
+
 	//
 	// Given an event source and ruleset, fill in a bitset
 	// containing the event types for which this ruleset can run.
@@ -296,17 +314,46 @@ private:
 	// Throws falco_exception if the file can not be read
 	void read_file(const std::string& filename, std::string& contents);
 
-	// For load_rules methods that throw exceptions on error,
-	// interpret a load_result and throw an exception if needed.
-	void interpret_load_result(std::unique_ptr<falco::load_result>& res,
-				   const std::string& rules_filename,
-				   const std::string& rules_content,
-				   bool verbose);
-
 	indexed_vector<falco_source> m_sources;
 
-	const falco_source* find_source(std::size_t index) const;
-	const falco_source* find_source(const std::string& name) const;
+	inline const falco_source* find_source(std::size_t index)
+	{
+		const falco_source *source;
+
+		if(index == m_syscall_source_idx)
+		{
+			if(m_syscall_source == NULL)
+			{
+				m_syscall_source = m_sources.at(m_syscall_source_idx);
+				if(!m_syscall_source)
+				{
+					throw falco_exception("Unknown event source index " + std::to_string(index));
+				}
+			}
+
+			source = m_syscall_source;
+		}
+		else
+		{
+			source = m_sources.at(index);
+			if(!source)
+			{
+				throw falco_exception("Unknown event source index " + std::to_string(index));
+			}
+		}
+
+		return source;
+	}
+
+	inline const falco_source* find_source(const std::string& name) const
+	{
+		auto ret = m_sources.at(name);
+		if(!ret)
+		{
+			throw falco_exception("Unknown event source " + name);
+		}
+		return ret;
+	}
 
 	// To allow the engine to be extremely fast for syscalls (can
 	// be > 1M events/sec), we save the syscall source/source_idx
@@ -387,4 +434,3 @@ private:
 	std::string m_extra;
 	bool m_replace_container_info;
 };
-

userspace/engine/falco_engine_version.h

@@ -20,7 +20,7 @@ limitations under the License.
 
 // The version of this Falco engine
 #define FALCO_ENGINE_VERSION_MAJOR 0
-#define FALCO_ENGINE_VERSION_MINOR 29
+#define FALCO_ENGINE_VERSION_MINOR 31
 #define FALCO_ENGINE_VERSION_PATCH 0
 
 #define FALCO_ENGINE_VERSION \
@@ -34,4 +34,4 @@ limitations under the License.
 // It represents the fields supported by this version of Falco,
 // the event types, and the underlying driverevent schema. It's used to
 // detetect changes in engine version in our CI jobs.
-#define FALCO_ENGINE_CHECKSUM "30bd8b1c09eab71e14416f04b617d4d20bc7c7358bce91748ce9bd8007786c4d"
+#define FALCO_ENGINE_CHECKSUM "7c512927c89f594f024f2ff181077c780c4fe6e9dd4cee3f20a9ef208a356e4e"

userspace/engine/falco_load_result.cpp

@@ -66,7 +66,8 @@ static const std::string warning_codes[] = {
 	"LOAD_UNKNOWN_FILTER",
 	"LOAD_UNUSED_MACRO",
 	"LOAD_UNUSED_LIST",
-	"LOAD_UNKNOWN_ITEM"
+	"LOAD_UNKNOWN_ITEM",
+	"LOAD_DEPRECATED_ITEM"
 };
 
 const std::string& falco::load_result::warning_code_str(warning_code wc)
@@ -81,7 +82,8 @@ static const std::string warning_strings[] = {
 	"Unknown field or event-type in condition or output",
 	"Unused macro",
 	"Unused list",
-	"Unknown rules file item"
+	"Unknown rules file item",
+	"Used deprecated item"
 };
 
 const std::string& falco::load_result::warning_str(warning_code wc)
@@ -96,7 +98,8 @@ static const std::string warning_descs[] = {
 	"A rule condition or output refers to a field or evt.type that does not exist. This is normally an error, but if a rule has a skip-if-unknown-filter property, the error is downgraded to a warning.",
 	"A macro is defined in the rules content but is not used by any other macro or rule.",
 	"A list is defined in the rules content but is not used by any other list, macro, or rule.",
-	"An unknown top-level object is in the rules content. It will be ignored."
+	"An unknown top-level object is in the rules content. It will be ignored.",
+	"A deprecated item is employed by lists, macros, or rules."
 };
 
 const std::string& falco::load_result::warning_desc(warning_code wc)

userspace/engine/falco_load_result.h

@@ -54,7 +54,8 @@ public:
 		LOAD_UNKNOWN_FILTER,
 		LOAD_UNUSED_MACRO,
 		LOAD_UNUSED_LIST,
-		LOAD_UNKNOWN_ITEM
+		LOAD_UNKNOWN_ITEM,
+		LOAD_DEPRECATED_ITEM
 	};
 
 	virtual ~load_result() = default;

userspace/engine/rule_loader.cpp

@@ -41,7 +41,8 @@ static const std::string item_type_strings[] = {
 	"condition expression",
 	"rule output",
 	"rule output expression",
-	"rule priority"
+	"rule priority",
+	"overrides"
 };
 
 const std::string& rule_loader::context::item_type_as_string(enum item_type it)
@@ -553,6 +554,11 @@ rule_loader::rule_info::rule_info(context &ctx)
 {
 }
 
+rule_loader::rule_update_info::rule_update_info(context &ctx)
+	: ctx(ctx), cond_ctx(ctx)
+{
+}
+
 rule_loader::rule_load_exception::rule_load_exception(falco::load_result::error_code ec, const std::string& msg, const context& ctx)
 	: ec(ec), msg(msg), ctx(ctx)
 {

userspace/engine/rule_loader.h

@@ -19,6 +19,7 @@ limitations under the License.
 
 #include <string>
 #include <vector>
+#include <optional>
 #include <yaml-cpp/yaml.h>
 #include <nlohmann/json.hpp>
 #include "falco_source.h"
@@ -56,7 +57,8 @@ namespace rule_loader
 			CONDITION_EXPRESSION,
 			RULE_OUTPUT,
 			RULE_OUTPUT_EXPRESSION,
-			RULE_PRIORITY
+			RULE_PRIORITY,
+			OVERRIDE
 		};
 
 		static const std::string& item_type_as_string(enum item_type it);
@@ -451,4 +453,38 @@ namespace rule_loader
 		bool warn_evttypes;
 		bool skip_if_unknown_filter;
 	};
+
+	/*!
+		\brief Represents infos about a rule update (append or replace) request
+	*/
+
+	struct rule_update_info
+	{
+		rule_update_info(context &ctx);
+		~rule_update_info() = default;
+		rule_update_info(rule_update_info&&) = default;
+		rule_update_info& operator = (rule_update_info&&) = default;
+		rule_update_info(const rule_update_info&) = default;
+		rule_update_info& operator = (const rule_update_info&) = default;
+
+		bool has_any_value()
+		{
+			return cond.has_value() || output.has_value() || desc.has_value() || tags.has_value() ||
+				   exceptions.has_value() || priority.has_value() || enabled.has_value() ||
+				   warn_evttypes.has_value() || skip_if_unknown_filter.has_value();
+		}
+
+		context ctx;
+		context cond_ctx;
+		std::string name;
+		std::optional<std::string> cond;
+		std::optional<std::string> output;
+		std::optional<std::string> desc;
+		std::optional<std::set<std::string>> tags;
+		std::optional<std::vector<rule_exception_info>> exceptions;
+		std::optional<falco_common::priority_type> priority;
+		std::optional<bool> enabled;
+		std::optional<bool> warn_evttypes;
+		std::optional<bool> skip_if_unknown_filter;
+	};
 };

userspace/engine/rule_loader_collector.cpp

@@ -20,6 +20,7 @@ limitations under the License.
 
 #include "falco_engine.h"
 #include "rule_loader_collector.h"
+#include "rule_loading_messages.h"
 
 #define THROW(cond, err, ctx)    { if ((cond)) { throw rule_loader::rule_load_exception(falco::load_result::LOAD_ERR_VALIDATE, (err), (ctx)); } }
 
@@ -48,8 +49,14 @@ static inline void define_info(indexed_vector<T>& infos, T& info, uint32_t id)
 	}
 }
 
-template <typename T>
-static inline void append_info(T* prev, T& info, uint32_t id)
+template <typename T, typename U>
+static inline void append_info(T* prev, U& info, uint32_t id)
+{
+	prev->visibility = id;
+}
+
+template <typename T, typename U>
+static inline void replace_info(T* prev, U& info, uint32_t id)
 {
 	prev->visibility = id;
 }
@@ -184,9 +191,7 @@ void rule_loader::collector::define(configuration& cfg, list_info& info)
 void rule_loader::collector::append(configuration& cfg, list_info& info)
 {
 	auto prev = m_list_infos.at(info.name);
-	THROW(!prev,
-	       "List has 'append' key but no list by that name already exists",
-	       info.ctx);
+	THROW(!prev, ERROR_NO_PREVIOUS_LIST, info.ctx);
 	prev->items.insert(prev->items.end(), info.items.begin(), info.items.end());
 	append_info(prev, info, m_cur_index++);
 }
@@ -199,9 +204,7 @@ void rule_loader::collector::define(configuration& cfg, macro_info& info)
 void rule_loader::collector::append(configuration& cfg, macro_info& info)
 {
 	auto prev = m_macro_infos.at(info.name);
-	THROW(!prev,
-	       "Macro has 'append' key but no macro by that name already exists",
-	       info.ctx);
+	THROW(!prev, ERROR_NO_PREVIOUS_MACRO, info.ctx);
 	prev->cond += " ";
 	prev->cond += info.cond;
 	append_info(prev, info, m_cur_index++);
@@ -233,15 +236,14 @@ void rule_loader::collector::define(configuration& cfg, rule_info& info)
 	define_info(m_rule_infos, info, m_cur_index++);
 }
 
-void rule_loader::collector::append(configuration& cfg, rule_info& info)
+void rule_loader::collector::append(configuration& cfg, rule_update_info& info)
 {
 	auto prev = m_rule_infos.at(info.name);
 
-	THROW(!prev,
-	       "Rule has 'append' key but no rule by that name already exists",
-	       info.ctx);
-	THROW(info.cond.empty() && info.exceptions.empty(),
+	THROW(!prev, ERROR_NO_PREVIOUS_RULE_APPEND, info.ctx);
+	THROW(!info.has_any_value(),
 	       "Appended rule must have exceptions or condition property",
+	       // "Appended rule must have at least one field that can be appended to", // TODO replace with this and update testing
 	       info.ctx);
 
 	// note: source can be nullptr in case we've collected a
@@ -256,43 +258,136 @@ void rule_loader::collector::append(configuration& cfg, rule_info& info)
 	    	info.ctx);
 	}
 
-	if (!info.cond.empty())
+	if (info.cond.has_value() && !info.cond->empty())
 	{
 		prev->cond += " ";
-		prev->cond += info.cond;
+		prev->cond += *info.cond;
 	}
 
-	for (auto &ex : info.exceptions)
+	if (info.output.has_value() && !info.output->empty())
 	{
-		auto prev_ex = find_if(prev->exceptions.begin(), prev->exceptions.end(),
-			[&ex](const rule_loader::rule_exception_info& i)
-				{ return i.name == ex.name; });
-		if (prev_ex == prev->exceptions.end())
+		prev->output += " ";
+		prev->output += *info.output;
+	}
+
+	if (info.desc.has_value() && !info.desc->empty())
+	{
+		prev->desc += " ";
+		prev->desc += *info.desc;
+	}
+
+	if (info.tags.has_value())
+	{
+		for (auto &tag: *info.tags)
 		{
-			THROW(!ex.fields.is_valid(),
-			       "Rule exception must have fields property with a list of fields",
-			       ex.ctx);
-			THROW(ex.values.empty(),
-			       "Rule exception must have values property with a list of values",
-			       ex.ctx);
-			validate_exception_info(source, ex);
-			prev->exceptions.push_back(ex);
-		}
-		else
-		{
-			THROW(ex.fields.is_valid(),
-			       "Can not append exception fields to existing exception, only values",
-			       ex.ctx);
-			THROW(ex.comps.is_valid(),
-			       "Can not append exception comps to existing exception, only values",
-			       ex.ctx);
-			prev_ex->values.insert(
-				prev_ex->values.end(), ex.values.begin(), ex.values.end());
+			prev->tags.insert(tag);
 		}
 	}
+
+	if (info.exceptions.has_value())
+	{
+		for (auto &ex : *info.exceptions)
+		{
+			auto prev_ex = find_if(prev->exceptions.begin(), prev->exceptions.end(),
+				[&ex](const rule_loader::rule_exception_info& i)
+					{ return i.name == ex.name; });
+			if (prev_ex == prev->exceptions.end())
+			{
+				THROW(!ex.fields.is_valid(),
+					"Rule exception must have fields property with a list of fields",
+					ex.ctx);
+				THROW(ex.values.empty(),
+					"Rule exception must have values property with a list of values",
+					ex.ctx);
+				validate_exception_info(source, ex);
+				prev->exceptions.push_back(ex);
+			}
+			else
+			{
+				THROW(ex.fields.is_valid(),
+					"Can not append exception fields to existing exception, only values",
+					ex.ctx);
+				THROW(ex.comps.is_valid(),
+					"Can not append exception comps to existing exception, only values",
+					ex.ctx);
+				prev_ex->values.insert(
+					prev_ex->values.end(), ex.values.begin(), ex.values.end());
+			}
+		}
+	}
+
 	append_info(prev, info, m_cur_index++);
 }
 
+void rule_loader::collector::selective_replace(configuration& cfg, rule_update_info& info)
+{
+	auto prev = m_rule_infos.at(info.name);
+
+	THROW(!prev, ERROR_NO_PREVIOUS_RULE_REPLACE, info.ctx);
+	THROW(!info.has_any_value(),
+	       "The rule must have at least one field that can be replaced",
+	       info.ctx);
+
+	// note: source can be nullptr in case we've collected a
+	// rule for which the source is unknown
+	falco_source* source = nullptr;
+	if (!prev->unknown_source)
+	{
+		// note: if the source is not unknown, this should not return nullptr
+		source = cfg.sources.at(prev->source);
+		THROW(!source,
+	    	std::string("Unknown source ") + prev->source,
+	    	info.ctx);
+	}
+
+	if (info.cond.has_value())
+	{
+		prev->cond = *info.cond;
+	}
+
+	if (info.output.has_value())
+	{
+		prev->output = *info.output;
+	}
+
+	if (info.desc.has_value())
+	{
+		prev->desc = *info.desc;
+	}
+
+	if (info.tags.has_value())
+	{
+		prev->tags = *info.tags;
+	}
+
+	if (info.exceptions.has_value())
+	{
+		prev->exceptions = *info.exceptions;
+	}
+
+	if (info.priority.has_value())
+	{
+		prev->priority = *info.priority;
+	}
+
+	if (info.enabled.has_value())
+	{
+		prev->enabled = *info.enabled;
+	}
+
+	if (info.warn_evttypes.has_value())
+	{
+		prev->warn_evttypes = *info.warn_evttypes;
+	}
+
+	if (info.skip_if_unknown_filter.has_value())
+	{
+		prev->skip_if_unknown_filter = *info.skip_if_unknown_filter;
+	}
+
+	replace_info(prev, info, m_cur_index++);
+}
+
 void rule_loader::collector::enable(configuration& cfg, rule_info& info)
 {
 	auto prev = m_rule_infos.at(info.name);

userspace/engine/rule_loader_collector.h

@@ -85,13 +85,18 @@ public:
 	*/
 	virtual void append(configuration& cfg, list_info& info);
 	virtual void append(configuration& cfg, macro_info& info);
-	virtual void append(configuration& cfg, rule_info& info);
+	virtual void append(configuration& cfg, rule_update_info& info);
 
 	/*!
 		\brief Updates the 'enabled' flag of an existing definition
 	*/
 	virtual void enable(configuration& cfg, rule_info& info);
 
+	/*!
+		\brief Selectively replaces some fields of an existing definition
+	*/
+	virtual void selective_replace(configuration& cfg, rule_update_info& info);
+
 private:
 	uint32_t m_cur_index;
 	indexed_vector<rule_info> m_rule_infos;

userspace/engine/rule_loader_compiler.cpp

@@ -181,6 +181,7 @@ static bool resolve_list(std::string& cnd, const falco_list& list)
 {
 	static std::string blanks = " \t\n\r";
 	static std::string delims = blanks + "(),=";
+	std::string tmp;
 	std::string new_cnd;
 	size_t start, end;
 	bool used = false;
@@ -212,7 +213,9 @@ static bool resolve_list(std::string& cnd, const falco_list& list)
 				{
 					sub += ", ";
 				}
-				sub += v;
+				tmp = v;
+				quote_item(tmp);
+				sub += tmp;
 			}
 			// if substituted list is empty, we need to
 			// remove a comma from the left or the right
@@ -339,7 +342,6 @@ void rule_loader::compiler::compile_list_infos(
 		const collector& col,
 		indexed_vector<falco_list>& out) const
 {
-	std::string tmp;
 	std::list<std::string> used;
 	falco_list v;
 	for (const auto &list : col.lists())
@@ -352,17 +354,14 @@ void rule_loader::compiler::compile_list_infos(
 			if (ref && ref->index < list.visibility)
 			{
 				used.push_back(ref->name);
-				for (auto val : ref->items)
+				for (const auto &val : ref->items)
 				{
-					quote_item(val);
 					v.items.push_back(val);
 				}
 			}
 			else
 			{
-				tmp = item;
-				quote_item(tmp);
-				v.items.push_back(tmp);
+				v.items.push_back(item);
 			}
 		}
 		v.used = false;

userspace/engine/rule_loader_reader.cpp

@@ -17,10 +17,12 @@ limitations under the License.
 
 #include <string>
 #include <vector>
+#include <set>
+#include <sstream>
 
 #include "rule_loader_reader.h"
 #include "falco_engine_version.h"
-#include "logger.h"
+#include "rule_loading_messages.h"
 
 #define THROW(cond, err, ctx)    { if ((cond)) { throw rule_loader::rule_load_exception(falco::load_result::LOAD_ERR_YAML_VALIDATE, (err), (ctx)); } }
 
@@ -45,6 +47,14 @@ static void decode_val_generic(const YAML::Node& item, const char *key, T& out,
 	THROW(!YAML::convert<T>::decode(val, out), "Can't decode YAML scalar value", valctx);
 }
 
+template <typename T>
+static void decode_val_generic(const YAML::Node& item, const char *key, std::optional<T>& out, const rule_loader::context& ctx, bool optional)
+{
+	T decoded;
+	decode_val_generic(item, key, decoded, ctx, optional);
+	out = decoded;
+}
+
 template <typename T>
 static void decode_val(const YAML::Node& item, const char *key, T& out, const rule_loader::context& ctx)
 {
@@ -115,6 +125,73 @@ static void decode_tags(const YAML::Node& item, std::set<T>& out,
 	decode_seq(item, "tags", inserter, ctx, optional);
 }
 
+template <typename T>
+static void decode_tags(const YAML::Node& item, std::optional<std::set<T>>& out,
+		       const rule_loader::context& ctx)
+{
+	std::set<T> decoded;
+	decode_tags(item, decoded, ctx);
+	out = decoded;
+}
+
+static void decode_overrides(const YAML::Node& item,
+				std::set<std::string>& overridable_append,
+				std::set<std::string>& overridable_replace,
+				std::set<std::string>& out_append,
+				std::set<std::string>& out_replace,
+				const rule_loader::context& ctx)
+{
+	const YAML::Node& val = item["override"];
+
+	if(!val.IsDefined())
+	{
+		return;
+	}
+
+	rule_loader::context overridectx(item, rule_loader::context::OVERRIDE, "", ctx);
+
+	for(YAML::const_iterator it=val.begin();it!=val.end();++it)
+	{
+		std::string key = it->first.as<std::string>();
+		std::string operation = it->second.as<std::string>();
+
+		bool is_overridable_append = overridable_append.find(it->first.as<std::string>()) != overridable_append.end();
+		bool is_overridable_replace = overridable_replace.find(it->first.as<std::string>()) != overridable_replace.end();
+
+		if (operation == "append")
+		{
+			rule_loader::context keyctx(it->first, rule_loader::context::OVERRIDE, key, overridectx);
+			THROW(!is_overridable_append, std::string("Key '") + key + std::string("' cannot be appended to, use 'replace' instead"), keyctx);
+
+			out_append.insert(key);
+		}
+		else if (operation == "replace")
+		{
+			rule_loader::context keyctx(it->first, rule_loader::context::OVERRIDE, key, overridectx);
+			THROW(!is_overridable_replace, std::string("Key '") + key + std::string("' cannot be replaced"), keyctx);
+
+			out_replace.insert(key);
+		}
+		else
+		{
+			rule_loader::context operationctx(it->second, rule_loader::context::VALUE_FOR, key, overridectx);
+			std::stringstream err_ss;
+			err_ss << "Invalid override operation for key '" << key << "': '" << operation << "'. "
+				   << "Allowed values are: ";
+			if (is_overridable_append)
+			{
+				err_ss << "append ";
+			}
+			if (is_overridable_replace)
+			{
+				err_ss << "replace ";
+			}
+
+			THROW(true, err_ss.str(), operationctx);
+		}
+	}
+}
+
 // Don't call this directly, call decode_exception_{fields,comps,values} instead
 static void decode_exception_info_entry(
 	const YAML::Node& item,
@@ -187,7 +264,7 @@ static void decode_exception_values(
 
 static void read_rule_exceptions(
 	const YAML::Node& item,
-	rule_loader::rule_info& v,
+	std::vector<rule_loader::rule_exception_info>& exceptions,
 	const rule_loader::context& parent,
 	bool append)
 {
@@ -239,10 +316,36 @@ static void read_rule_exceptions(
 				v_ex.values.push_back(v_ex_val);
 			}
 		}
-		v.exceptions.push_back(v_ex);
+		exceptions.push_back(v_ex);
 	}
 }
 
+static void read_rule_exceptions(
+	const YAML::Node& item,
+	std::optional<std::vector<rule_loader::rule_exception_info>>& exceptions,
+	const rule_loader::context& parent,
+	bool append)
+{
+	std::vector<rule_loader::rule_exception_info> decoded;
+	read_rule_exceptions(item, decoded, parent, append);
+	exceptions = decoded;
+}
+
+inline static bool check_update_expected(std::set<std::string>& expected_keys, const std::set<std::string>& overrides, const std::string& override_type, const std::string& key, const rule_loader::context& ctx)
+{
+	if (overrides.find(key) == overrides.end())
+	{
+		return false;
+	}
+	
+	THROW(expected_keys.find(key) == expected_keys.end(),
+		std::string("An ") + override_type + " override for '" + key + "' was specified but '" + key + "' is not defined", ctx);
+
+	expected_keys.erase(key);
+
+	return true;
+}
+
 static void read_item(
 	rule_loader::configuration& cfg,
 	rule_loader::collector& collector,
@@ -337,6 +440,21 @@ static void read_item(
 		decode_items(item, v.items, ctx);
 
 		decode_optional_val(item, "append", append, ctx);
+		if(append)
+		{
+			cfg.res->add_warning(falco::load_result::LOAD_DEPRECATED_ITEM, WARNING_APPEND, ctx);
+		}
+
+		std::set<std::string> override_append, override_replace;
+		std::set<std::string> overridable {"items"};
+		decode_overrides(item, overridable, overridable, override_append, override_replace, ctx);
+		bool has_overrides = !override_append.empty() || !override_replace.empty();
+
+		THROW(append && has_overrides, ERROR_OVERRIDE_APPEND, ctx);
+
+		// Since a list only has items, if we have chosen to append them we can append the entire object
+		// otherwise we just want to redefine the list.
+		append |= override_append.find("items") != override_append.end();
 
 		if(append)
 		{
@@ -365,6 +483,21 @@ static void read_item(
 		v.cond_ctx = rule_loader::context(item["condition"], rule_loader::context::MACRO_CONDITION, "", ctx);
 
 		decode_optional_val(item, "append", append, ctx);
+		if(append)
+		{
+			cfg.res->add_warning(falco::load_result::LOAD_DEPRECATED_ITEM, WARNING_APPEND, ctx);
+		}
+
+		std::set<std::string> override_append, override_replace;
+		std::set<std::string> overridable {"condition"};
+		decode_overrides(item, overridable, overridable, override_append, override_replace, ctx);
+		bool has_overrides = !override_append.empty() || !override_replace.empty();
+
+		THROW((append && has_overrides), ERROR_OVERRIDE_APPEND, ctx);
+
+		// Since a macro only has a condition, if we have chosen to append to it we can append the entire object
+		// otherwise we just want to redefine the macro.
+		append |= override_append.find("condition") != override_append.end();
 
 		if(append)
 		{
@@ -384,28 +517,172 @@ static void read_item(
 		decode_val(item, "rule", name, tmp);
 
 		rule_loader::context ctx(item, rule_loader::context::RULE, name, parent);
-		rule_loader::rule_info v(ctx);
-		v.name = name;
 
-		bool append = false;
-		v.enabled = true;
-		v.warn_evttypes = true;
-		v.skip_if_unknown_filter = false;
-
-		decode_optional_val(item, "append", append, ctx);
-
-		if(append)
+		bool has_append_flag = false;
+		decode_optional_val(item, "append", has_append_flag, ctx);
+		if(has_append_flag)
 		{
-			decode_optional_val(item, "condition", v.cond, ctx);
+			cfg.res->add_warning(falco::load_result::LOAD_DEPRECATED_ITEM, WARNING_APPEND, ctx);
+		}
+
+		std::set<std::string> override_append, override_replace;
+		std::set<std::string> overridable_append {"condition", "output", "desc", "tags", "exceptions"};
+		std::set<std::string> overridable_replace {
+			"condition", "output", "desc", "priority", "tags", "exceptions", "enabled", "warn_evttypes", "skip-if-unknown-filter"};
+		decode_overrides(item, overridable_append, overridable_replace, override_append, override_replace, ctx);
+		bool has_overrides_append = !override_append.empty();
+		bool has_overrides_replace = !override_replace.empty();
+		bool has_overrides = has_overrides_append || has_overrides_replace;
+
+		THROW((has_append_flag && has_overrides), ERROR_OVERRIDE_APPEND, ctx);
+
+		if(has_overrides)
+		{
+			std::set<std::string> expected_keys;
+			for (auto& key : overridable_append)
+			{
+				if (item[key].IsDefined())
+				{
+					expected_keys.insert(key);
+				}
+			}
+
+			for (auto& key : overridable_replace)
+			{
+				if (item[key].IsDefined())
+				{
+					expected_keys.insert(key);
+				}
+			}
+
+			// expected_keys is (appendable U replaceable) ^ (defined)
+			
+			if (has_overrides_append)
+			{
+				rule_loader::rule_update_info v(ctx);
+				v.name = name;
+				if (check_update_expected(expected_keys, override_append, "append", "condition", ctx))
+				{
+					decode_val(item, "condition", v.cond, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_append, "append", "exceptions", ctx))
+				{
+					read_rule_exceptions(item, v.exceptions, ctx, true);
+				}
+
+				if (check_update_expected(expected_keys, override_append, "append", "output", ctx))
+				{
+					decode_val(item, "output", v.output, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_append, "append", "desc", ctx))
+				{
+					decode_val(item, "desc", v.desc, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_append, "append", "tags", ctx))
+				{
+					decode_tags(item, v.tags, ctx);
+				}
+				
+				collector.append(cfg, v);
+			}
+
+			if (has_overrides_replace)
+			{
+				rule_loader::rule_update_info v(ctx);
+				v.name = name;
+				if (check_update_expected(expected_keys, override_replace, "replace", "condition", ctx))
+				{
+					decode_val(item, "condition", v.cond, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "exceptions", ctx))
+				{
+					read_rule_exceptions(item, v.exceptions, ctx, true);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "output", ctx))
+				{
+					decode_val(item, "output", v.output, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "desc", ctx))
+				{
+					decode_val(item, "desc", v.desc, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "tags", ctx))
+				{
+					decode_tags(item, v.tags, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "priority", ctx))
+				{
+					std::string priority;
+					decode_val(item, "priority", priority, ctx);
+					rule_loader::context prictx(item["priority"], rule_loader::context::RULE_PRIORITY, "", ctx);
+					falco_common::priority_type parsed_priority;
+					THROW(!falco_common::parse_priority(priority, parsed_priority), "Invalid priority", prictx);
+					v.priority = parsed_priority;
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "enabled", ctx))
+				{
+					decode_val(item, "enabled", v.enabled, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "warn_evttypes", ctx))
+				{
+					decode_val(item, "warn_evttypes", v.warn_evttypes, ctx);
+				}
+
+				if (check_update_expected(expected_keys, override_replace, "replace", "skip-if-unknown-filter", ctx))
+				{
+					decode_val(item, "skip-if-unknown-filter", v.skip_if_unknown_filter, ctx);
+				}
+
+				collector.selective_replace(cfg, v);
+			}
+
+			// if any expected key has not been defined throw an error
+			for (auto &key : expected_keys) {
+				rule_loader::context keyctx(item[key], rule_loader::context::OVERRIDE, key, ctx);
+				THROW(true, "Unexpected key '" + key + "': no corresponding entry under 'override' is defined.", keyctx);
+			}
+		}
+		else if(has_append_flag)
+		{
+			rule_loader::rule_update_info v(ctx);
+			v.name = name;
+
 			if(item["condition"].IsDefined())
 			{
 				v.cond_ctx = rule_loader::context(item["condition"], rule_loader::context::RULE_CONDITION, "", ctx);
+				decode_val(item, "condition", v.cond, ctx);
 			}
-			read_rule_exceptions(item, v, ctx, append);
+
+			if(item["exceptions"].IsDefined())
+			{
+				read_rule_exceptions(item, v.exceptions, ctx, true);
+			}
+
+			// TODO restore this error and update testing
+			//THROW((!v.cond.has_value() && !v.exceptions.has_value()),
+			//       "Appended rule must have exceptions or condition property",
+			//       v.ctx);
+
 			collector.append(cfg, v);
 		}
 		else
 		{
+			rule_loader::rule_info v(ctx);
+			v.name = name;
+			v.enabled = true;
+			v.warn_evttypes = true;
+			v.skip_if_unknown_filter = false;
+
 			// If the rule does *not* have any of
 			// condition/output/desc/priority, it *must*
 			// have an enabled property. Use the enabled
@@ -417,6 +694,7 @@ static void read_item(
 			    !item["priority"].IsDefined())
 			{
 				decode_val(item, "enabled", v.enabled, ctx);
+				cfg.res->add_warning(falco::load_result::LOAD_DEPRECATED_ITEM, WARNING_ENABLED, ctx);
 				collector.enable(cfg, v);
 			}
 			else
@@ -443,7 +721,7 @@ static void read_item(
 				decode_optional_val(item, "warn_evttypes", v.warn_evttypes, ctx);
 				decode_optional_val(item, "skip-if-unknown-filter", v.skip_if_unknown_filter, ctx);
 				decode_tags(item, v.tags, ctx);
-				read_rule_exceptions(item, v, ctx, append);
+				read_rule_exceptions(item, v.exceptions, ctx, has_append_flag);
 				collector.define(cfg, v);
 			}
 		}

userspace/engine/rule_loading_messages.h

@@ -0,0 +1,23 @@
+#pragma once
+
+////////////////
+// Warnings
+////////////////
+
+#define WARNING_APPEND "'append' key is deprecated. Add an 'append' entry (e.g. 'condition: append') under 'override' instead."
+
+#define WARNING_ENABLED "The standalone 'enabled' key usage is deprecated. The correct approach requires also a 'replace' entry under the 'override' key (i.e. 'enabled: replace')."
+
+////////////////
+// Errors
+////////////////
+
+#define ERROR_OVERRIDE_APPEND "Keys 'override' and 'append: true' cannot be used together. Add an 'append' entry (e.g. 'condition: append') under 'override' instead."
+
+#define ERROR_NO_PREVIOUS_MACRO "Macro uses 'append' or 'override.condition: append' but no macro by that name already exists"
+
+#define ERROR_NO_PREVIOUS_LIST "List uses 'append' or 'override.items: append' but no list by that name already exists"
+
+#define ERROR_NO_PREVIOUS_RULE_APPEND "Rule uses 'append' or 'override.<key>: append' but no rule by that name already exists"
+
+#define ERROR_NO_PREVIOUS_RULE_REPLACE "An 'override.<key>: replace' to a rule was requested but no rule by that name already exists"

userspace/falco/app/actions/load_config.cpp

@@ -79,17 +79,17 @@ static falco::app::run_result apply_deprecated_options(falco::app::state& s)
 	if(s.config->m_syscall_drop_failed_exit != DEFAULT_DROP_FAILED_EXIT)
 	{
 		falco_logger::log(falco_logger::level::WARNING,
-				"DEPRECATION NOTICE: 'syscall_drop_failed_exit' config is deprecated and will be removed in Falco 0.38! Use `engine.<driver>.drop_failed_exit' config instead\n");
+				"DEPRECATION NOTICE: 'syscall_drop_failed_exit' config is deprecated and will be removed in Falco 0.38! Use 'engine.<driver>.drop_failed_exit' config instead\n");
 	}
 	if(s.config->m_syscall_buf_size_preset != DEFAULT_BUF_SIZE_PRESET)
 	{
 		falco_logger::log(falco_logger::level::WARNING,
-				"DEPRECATION NOTICE: 'syscall_buf_size_preset' config is deprecated and will be removed in Falco 0.38! Use `engine.<driver>.buf_size_preset' config instead\n");
+				"DEPRECATION NOTICE: 'syscall_buf_size_preset' config is deprecated and will be removed in Falco 0.38! Use 'engine.<driver>.buf_size_preset' config instead\n");
 	}
 	if(s.config->m_cpus_for_each_syscall_buffer != DEFAULT_CPUS_FOR_EACH_SYSCALL_BUFFER)
 	{
 		falco_logger::log(falco_logger::level::WARNING,
-				"DEPRECATION NOTICE: 'modern_bpf.cpus_for_each_syscall_buffer' config is deprecated and will be removed in Falco 0.38! Use `engine.modern_ebpf.cpus_for_each_buffer' config instead\n");
+				"DEPRECATION NOTICE: 'modern_bpf.cpus_for_each_syscall_buffer' config is deprecated and will be removed in Falco 0.38! Use 'engine.modern_ebpf.cpus_for_each_buffer' config instead\n");
 	}
 
 	// Replace the kmod default values in case the engine was open with the kmod.
@@ -102,7 +102,7 @@ static falco::app::run_result apply_deprecated_options(falco::app::state& s)
 	// use the requested driver.
 	if (getenv(FALCO_BPF_ENV_VARIABLE))
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the 'FALCO_BPF_PROBE' environment variable is deprecated and will be removed in Falco 0.38!\n");
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the 'FALCO_BPF_PROBE' environment variable is deprecated and will be removed in Falco 0.38! Set 'engine.kind: ebpf' and use 'engine.ebpf' config instead in falco.yaml\n");
 		s.config->m_engine_mode = engine_kind_t::EBPF;
 		s.config->m_ebpf.m_probe_path = getenv(FALCO_BPF_ENV_VARIABLE);
 		s.config->m_ebpf.m_drop_failed_exit = s.config->m_syscall_drop_failed_exit;
@@ -110,7 +110,7 @@ static falco::app::run_result apply_deprecated_options(falco::app::state& s)
 	}
 	else if (s.options.modern_bpf)
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '--modern-bpf' command line option is deprecated and will be removed in Falco 0.38!\n");
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '--modern-bpf' command line option is deprecated and will be removed in Falco 0.38! Set 'engine.kind: modern_ebpf' and use 'engine.modern_ebpf' config instead in falco.yaml\n");
 		s.config->m_engine_mode = engine_kind_t::MODERN_EBPF;
 		s.config->m_modern_ebpf.m_drop_failed_exit = s.config->m_syscall_drop_failed_exit;
 		s.config->m_modern_ebpf.m_buf_size_preset = s.config->m_syscall_buf_size_preset;
@@ -118,19 +118,19 @@ static falco::app::run_result apply_deprecated_options(falco::app::state& s)
 	}
 	if (!s.options.gvisor_config.empty())
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '-g,--gvisor-config' command line option is deprecated and will be removed in Falco 0.38!\n");
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '-g,--gvisor-config' command line option is deprecated and will be removed in Falco 0.38! Set 'engine.kind: gvisor' and use 'engine.gvisor' config instead in falco.yaml\n");
 		s.config->m_engine_mode =  engine_kind_t::GVISOR;
 		s.config->m_gvisor.m_config = s.options.gvisor_config;
 		s.config->m_gvisor.m_root = s.options.gvisor_root;
 	}
 	if (s.options.nodriver)
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '--nodriver' command line option is deprecated and will be removed in Falco 0.38!\n");
-		s.config->m_engine_mode =  engine_kind_t::NONE;
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '--nodriver' command line option is deprecated and will be removed in Falco 0.38! Set 'engine.kind: nodriver' instead in falco.yaml\n");
+		s.config->m_engine_mode =  engine_kind_t::NODRIVER;
 	}
 	if (!s.options.capture_file.empty())
 	{
-		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '-e' command line option is deprecated and will be removed in Falco 0.38!\n");
+		falco_logger::log(falco_logger::level::WARNING, "DEPRECATION NOTICE: the '-e' command line option is deprecated and will be removed in Falco 0.38! Set 'engine.kind: replay' and use 'engine.replay' config instead in falco.yaml\n");
 		s.config->m_engine_mode = engine_kind_t::REPLAY;
 		s.config->m_replay.m_capture_file = s.options.capture_file;
 	}

userspace/falco/app/actions/load_rules_files.cpp

@@ -79,10 +79,9 @@ falco::app::run_result falco::app::actions::load_rules_files(falco::app::state&
 			break;
 		}
 
-		// If verbose is true, also print any warnings
-		if(s.options.verbose && res->has_warnings())
+		if(res->has_warnings())
 		{
-			fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
+			falco_logger::log(falco_logger::level::WARNING,res->as_string(true, rc) + "\n");
 		}
 	}
 

userspace/falco/app/actions/validate_rules_files.cpp

@@ -114,12 +114,7 @@ falco::app::run_result falco::app::actions::validate_rules_files(falco::app::sta
 				// file was ok with warnings, without actually
 				// printing the warnings.
 				summary += filename + ": Ok, with warnings";
-
-				// If verbose is true, print the warnings now.
-				if(s.options.verbose)
-				{
-					fprintf(stderr, "%s\n", res->as_string(true, rc).c_str());
-				}
+				falco_logger::log(falco_logger::level::WARNING, res->as_string(true, rc) + "\n");
 			}
 		}
 

userspace/falco/app/state.h

@@ -168,7 +168,7 @@ struct state
 
     inline bool is_nodriver() const
     {
-        return config->m_engine_mode == engine_kind_t::NONE;
+        return config->m_engine_mode == engine_kind_t::NODRIVER;
     }
 
     inline bool is_source_enabled(const std::string& src) const

userspace/falco/configuration.cpp

@@ -109,7 +109,7 @@ void falco_configuration::load_engine_config(const std::string& config_name, con
 		{"modern_ebpf",engine_kind_t::MODERN_EBPF},
 		{"replay",engine_kind_t::REPLAY},
 		{"gvisor",engine_kind_t::GVISOR},
-		{"none",engine_kind_t::NONE},
+		{"nodriver",engine_kind_t::NODRIVER},
 	};
 
 	auto driver_mode_str = config.get_scalar<std::string>("engine.kind", "kmod");
@@ -172,7 +172,7 @@ void falco_configuration::load_engine_config(const std::string& config_name, con
 		}
 		m_gvisor.m_root = config.get_scalar<std::string>("engine.gvisor.root", "");
 		break;
-	case engine_kind_t::NONE:
+	case engine_kind_t::NODRIVER:
 	default:
 		break;
 	}

userspace/falco/configuration.h

@@ -49,7 +49,7 @@ enum class engine_kind_t : uint8_t
 	MODERN_EBPF,
 	REPLAY,
 	GVISOR,
-	NONE
+	NODRIVER
 };
 
 class falco_configuration

userspace/falco/stats_writer.cpp

@@ -366,9 +366,16 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 	sinsp_thread_manager* thread_manager = inspector->m_thread_manager;
 	const scap_stats_v2* sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2, buffer, &nstats, &rc);
 
+	uint32_t base_stat = 0;
+	// todo @incertum this needs to become better with the next proper stats refactor in libs 0.15.0
+	if ((flags & PPM_SCAP_STATS_STATE_COUNTERS) && !(flags & PPM_SCAP_STATS_RESOURCE_UTILIZATION))
+	{
+		base_stat = SINSP_STATS_V2_N_THREADS;
+	}
+
 	if (sinsp_stats_v2_snapshot && rc == 0 && nstats > 0)
 	{
-		for(uint32_t stat = 0; stat < nstats; stat++)
+		for(uint32_t stat = base_stat; stat < nstats; stat++)
 		{
 			if (sinsp_stats_v2_snapshot[stat].name[0] == '\0')
 			{
@@ -376,6 +383,12 @@ void stats_writer::collector::get_metrics_output_fields_additional(
 			}
 			char metric_name[STATS_NAME_MAX] = "falco.";
 			strlcat(metric_name, sinsp_stats_v2_snapshot[stat].name, sizeof(metric_name));
+			// todo @incertum temporary fix for n_fds and n_threads, type assignment was missed in libs, will be fixed in libs 0.15.0
+			if (strncmp(sinsp_stats_v2_snapshot[stat].name, "n_fds", 6) == 0 || strncmp(sinsp_stats_v2_snapshot[stat].name, "n_threads", 10) == 0)
+			{
+				output_fields[metric_name] = sinsp_stats_v2_snapshot[stat].value.u64;
+			}
+
 			switch(sinsp_stats_v2_snapshot[stat].type)
 			{
 			case STATS_VALUE_TYPE_U64: