From c86615f68c307d5f76261e36247df5918b1c6ed9 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 6 Jan 2022 14:29:32 -0800 Subject: [PATCH] Embed .lua files into falco executable Instead of having .lua files external to the program responsible for loading rules, embed the contents of those files into the executable and load them as strings instead of as files: Add a cmake custom command below userspace/engine/lua that calls a bash script lua-to-cpp.sh to generate falco_engine_lua_files.{cpp,hh} that are compiled into the falco engine library. The script creates a .cpp file that has const char * symbols for each file, as well as lists of files that should be loaded when the falco engine is loaded. There are actually two lists: - lua_module_strings: these are loaded and also added to the lua runtime package.preload table, so they are available when lua code require()s them. - lua_code_strings: these are loaded *and* evaluated, so the functions in them are availble to be called from C++. This simplifies some of the falco_common methods, as there's no need to keep track of a "main" lua file to load or paths from which the lua loader should find files for modules, and there's no need to keep track of an "alternate" lua directory that occurs for debug builds. Also, there's no need to include any .lua files in the installed packages, as they're built into the falco binary. Signed-off-by: Mark Stemm --- userspace/engine/CMakeLists.txt | 24 +++------ userspace/engine/falco_common.cpp | 82 +++++++++------------------- userspace/engine/falco_common.h | 5 +- userspace/engine/falco_engine.cpp | 2 +- userspace/engine/falco_engine.h | 1 - userspace/engine/lua/CMakeLists.txt | 21 ++++++++ userspace/engine/lua/lua-to-cpp.sh | 84 +++++++++++++++++++++++++++++ 7 files changed, 138 insertions(+), 81 deletions(-) create mode 100644 userspace/engine/lua/CMakeLists.txt create mode 100644 userspace/engine/lua/lua-to-cpp.sh diff --git a/userspace/engine/CMakeLists.txt b/userspace/engine/CMakeLists.txt index ab569ea6..28beaae6 100644 --- a/userspace/engine/CMakeLists.txt +++ b/userspace/engine/CMakeLists.txt @@ -10,6 +10,8 @@ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the # specific language governing permissions and limitations under the License. +add_subdirectory(lua) + set(FALCO_ENGINE_SOURCE_FILES rules.cpp falco_common.cpp @@ -36,7 +38,8 @@ if(MINIMAL_BUILD) "${STRING_VIEW_LITE_INCLUDE}" "${LIBSCAP_INCLUDE_DIRS}" "${LIBSINSP_INCLUDE_DIRS}" - "${PROJECT_BINARY_DIR}/userspace/engine") + "${PROJECT_BINARY_DIR}/userspace/engine" + "${PROJECT_BINARY_DIR}/userspace/engine/lua") else() target_include_directories( falco_engine @@ -48,24 +51,11 @@ else() "${STRING_VIEW_LITE_INCLUDE}" "${LIBSCAP_INCLUDE_DIRS}" "${LIBSINSP_INCLUDE_DIRS}" - "${PROJECT_BINARY_DIR}/userspace/engine") + "${PROJECT_BINARY_DIR}/userspace/engine" + "${PROJECT_BINARY_DIR}/userspace/engine/lua") endif() -target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${LPEG_LIB}" "${LYAML_LIB}" "${LIBYAML_LIB}") +target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${LPEG_LIB}" "${LYAML_LIB}" "${LIBYAML_LIB}" luafiles) configure_file(config_falco_engine.h.in config_falco_engine.h) -if(DEFINED FALCO_COMPONENT) - install( - DIRECTORY lua - DESTINATION "${FALCO_SHARE_DIR}" - COMPONENT "${FALCO_COMPONENT}" - FILES_MATCHING - PATTERN *.lua) -else() - install( - DIRECTORY lua - DESTINATION "${FALCO_SHARE_DIR}" - FILES_MATCHING - PATTERN *.lua) -endif() diff --git a/userspace/engine/falco_common.cpp b/userspace/engine/falco_common.cpp index 69068f65..0e74b8e7 100644 --- a/userspace/engine/falco_common.cpp +++ b/userspace/engine/falco_common.cpp @@ -19,6 +19,7 @@ limitations under the License. #include "config_falco_engine.h" #include "falco_common.h" #include "banned.h" // This raises a compilation error when certain functions are used +#include "falco_engine_lua_files.hh" std::vector falco_common::priority_names = { "Emergency", @@ -48,68 +49,33 @@ falco_common::~falco_common() } } -void falco_common::init(const char *lua_main_filename, const char *alternate_lua_dir) +void falco_common::init() { - ifstream is; - string lua_dir = FALCO_ENGINE_LUA_DIR; - string lua_main_path = lua_dir + lua_main_filename; - - is.open(lua_main_path); - if (!is.is_open()) + // Strings in the list lua_module_strings need to be loaded as + // lua modules, which also involves adding them to the + // package.module table. + for(const auto &pair : lua_module_strings) { - lua_dir = alternate_lua_dir; - lua_main_path = lua_dir + lua_main_filename; + lua_getglobal(m_ls, "package"); + lua_getfield(m_ls, -1, "preload"); - is.open(lua_main_path); - if (!is.is_open()) + if(luaL_loadstring(m_ls, pair.first)) { - throw falco_exception("Could not find Falco Lua entrypoint (tried " + - string(FALCO_ENGINE_LUA_DIR) + lua_main_filename + ", " + - string(alternate_lua_dir) + lua_main_filename + ")"); + throw falco_exception("Failed to load embedded lua code " + + string(pair.second) + ": " + lua_tostring(m_ls, -1)); + } + + lua_setfield(m_ls, -2, pair.second); + } + + // Strings in the list lua_code_strings need to be loaded and + // evaluated so any public functions can be directly called. + for(const auto &str : lua_code_strings) + { + if(luaL_loadstring(m_ls, str) || lua_pcall(m_ls, 0, 0, 0)) + { + throw falco_exception("Failed to load + evaluate embedded lua code " + + string(str) + ": " + lua_tostring(m_ls, -1)); } } - - // Initialize Lua interpreter - add_lua_path(lua_dir); - - // Load the main program, which defines all the available functions. - string scriptstr((istreambuf_iterator(is)), - istreambuf_iterator()); - - if(luaL_loadstring(m_ls, scriptstr.c_str()) || lua_pcall(m_ls, 0, 0, 0)) - { - throw falco_exception("Failed to load script " + - lua_main_path + ": " + lua_tostring(m_ls, -1)); - } -} - -void falco_common::add_lua_path(string &path) -{ - string cpath = string(path); - path += "?.lua"; - cpath += "?.so"; - - lua_getglobal(m_ls, "package"); - - lua_getfield(m_ls, -1, "path"); - string cur_path = lua_tostring(m_ls, -1 ); - cur_path += ';'; - lua_pop(m_ls, 1); - - cur_path.append(path.c_str()); - - lua_pushstring(m_ls, cur_path.c_str()); - lua_setfield(m_ls, -2, "path"); - - lua_getfield(m_ls, -1, "cpath"); - string cur_cpath = lua_tostring(m_ls, -1 ); - cur_cpath += ';'; - lua_pop(m_ls, 1); - - cur_cpath.append(cpath.c_str()); - - lua_pushstring(m_ls, cur_cpath.c_str()); - lua_setfield(m_ls, -2, "cpath"); - - lua_pop(m_ls, 1); } diff --git a/userspace/engine/falco_common.h b/userspace/engine/falco_common.h index 494072a8..a3a3ea0e 100644 --- a/userspace/engine/falco_common.h +++ b/userspace/engine/falco_common.h @@ -69,7 +69,7 @@ public: falco_common(); virtual ~falco_common(); - void init(const char *lua_main_filename, const char *alternate_lua_dir); + void init(); // Priority levels, as a vector of strings static std::vector priority_names; @@ -91,7 +91,4 @@ protected: lua_State *m_ls; std::mutex m_ls_semaphore; - -private: - void add_lua_path(std::string &path); }; diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index b3228e99..a413a278 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -53,7 +53,7 @@ falco_engine::falco_engine(bool seed_rng, const std::string& alternate_lua_dir) luaopen_lpeg(m_ls); luaopen_yaml(m_ls); - falco_common::init(m_lua_main_filename.c_str(), alternate_lua_dir.c_str()); + falco_common::init(); falco_rules::init(m_ls); m_required_plugin_versions.clear(); diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index d3ba3728..149c5724 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -265,7 +265,6 @@ private: uint32_t m_sampling_ratio; double m_sampling_multiplier; - std::string m_lua_main_filename = "rule_loader.lua"; static const std::string s_default_ruleset; uint32_t m_default_ruleset_id; diff --git a/userspace/engine/lua/CMakeLists.txt b/userspace/engine/lua/CMakeLists.txt new file mode 100644 index 00000000..ac5a3dc1 --- /dev/null +++ b/userspace/engine/lua/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# Copyright (C) 2021 The Falco Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +file(GLOB_RECURSE lua_module_files ${CMAKE_CURRENT_SOURCE_DIR} *.lua) + +add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/falco_engine_lua_files.cpp + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/lua-to-cpp.sh ${CMAKE_CURRENT_SOURCE_DIR} ${LYAML_LUA_DIR} ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${lua_files} ${CMAKE_CURRENT_SOURCE_DIR}/lua-to-cpp.sh lyaml) + +add_library(luafiles falco_engine_lua_files.cpp) + +target_include_directories(luafiles PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/userspace/engine/lua/lua-to-cpp.sh b/userspace/engine/lua/lua-to-cpp.sh new file mode 100644 index 00000000..87c858ff --- /dev/null +++ b/userspace/engine/lua/lua-to-cpp.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +set -euo pipefail + +LUA_FILE_DIR=$1 +LYAML_LUA_DIR=$2 +OUTPUT_DIR=$3 + +MODULE_SYMS=() +CODE_SYMS=() + +function add_lua_file { + filename=$1 + is_module=$2 + + # Take the basename of the file + BASE_NAME=$(basename ${file} .lua) + SYMBOL_NAME="${BASE_NAME}_lua_file_contents" + FILE_CONTENTS=$(<${file}) + + # Add a symbol to the .cc file containing the contents of the file + echo "const char *${SYMBOL_NAME}=R\"LUAFILE(${FILE_CONTENTS})LUAFILE\";" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp + + # Add an extern reference to the .hh file + echo "extern const char *${SYMBOL_NAME};" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh + + if [[ "${is_module}" == "true" ]]; then + # Determine the module name for the file + if [[ "${file}" == *"/"* ]]; then + MODULE_NAME=$(echo ${file} | tr / . | sed -e 's/.lua//') + else + MODULE_NAME=$(basename ${file} .lua) + fi + + # Add the pair (string contents, module name) to MODULE_SYMS + PAIR=$(echo "{${SYMBOL_NAME},\"${MODULE_NAME}\"}") + MODULE_SYMS+=(${PAIR}) + else + # Add the string to CODE_SYMS + CODE_SYMS+=(${SYMBOL_NAME}) + fi +} + +cat < ${OUTPUT_DIR}/falco_engine_lua_files.cpp +// Automatically generated. Do not edit +#include "falco_engine_lua_files.hh" +EOF + +cat < ${OUTPUT_DIR}/falco_engine_lua_files.hh +#pragma once +// Automatically generated. Do not edit +#include +#include +EOF + +# lyaml and any files in the "modules" subdirectory are treated as lua +# modules. +pushd ${LYAML_LUA_DIR} +for file in *.lua */*.lua; do + add_lua_file $file "true" +done +popd + +pushd ${LUA_FILE_DIR}/modules +for file in *.lua; do + add_lua_file $file "true" +done +popd + +# Any .lua files in this directory are treated as code with functions +# to execute. +pushd ${LUA_FILE_DIR} +for file in ${LUA_FILE_DIR}/*.lua; do + add_lua_file $file "false" +done +popd + +# Create a list of lua module (string, module name) pairs from MODULE_SYMS +echo "extern std::list> lua_module_strings;" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh +echo "std::list> lua_module_strings = {$(IFS=, ; echo "${MODULE_SYMS[*]}")};" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp + +# Create a list of lua code strings from CODE_SYMS +echo "extern std::list lua_code_strings;" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh +echo "std::list lua_code_strings = {$(IFS=, ; echo "${CODE_SYMS[*]}")};" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp