Embed .lua files into falco executable

Instead of having .lua files external to the program responsible for
loading rules, embed the contents of those files into the executable
and load them as strings instead of as files:

Add a cmake custom command below userspace/engine/lua that calls a
bash script lua-to-cpp.sh to generate falco_engine_lua_files.{cpp,hh}
that are compiled into the falco engine library.

The script creates a .cpp file that has const char * symbols for each
file, as well as lists of files that should be loaded when the falco
engine is loaded. There are actually two lists:

- lua_module_strings: these are loaded and also added to the lua
  runtime package.preload table, so they are available when lua code
  require()s them.

- lua_code_strings: these are loaded *and* evaluated, so the functions
  in them are availble to be called from C++.

This simplifies some of the falco_common methods, as there's no need
to keep track of a "main" lua file to load or paths from which the lua
loader should find files for modules, and there's no need to keep
track of an "alternate" lua directory that occurs for debug builds.

Also, there's no need to include any .lua files in the installed
packages, as they're built into the falco binary.

Signed-off-by: Mark Stemm <mark.stemm@gmail.com>

cmake/modules/lyaml.cmake

@@ -11,9 +11,10 @@
 # specific language governing permissions and limitations under the License.
 #
 
-set(LYAML_SRC "${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/ext/yaml")
-set(LYAML_LIB "${LYAML_SRC}/.libs/yaml.a")
-message(STATUS "Using bundled lyaml in '${LYAML_SRC}'")
+set(LYAML_ROOT "${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml")
+set(LYAML_LIB "${LYAML_ROOT}/ext/yaml/.libs/yaml.a")
+set(LYAML_LUA_DIR "${LYAML_ROOT}/lib")
+message(STATUS "Using bundled lyaml in '${LYAML_ROOT}'")
 externalproject_add(
   lyaml
   DEPENDS luajit libyaml
@@ -22,7 +23,6 @@ externalproject_add(
   BUILD_COMMAND ${CMD_MAKE}
   BUILD_IN_SOURCE 1
   BUILD_BYPRODUCTS ${LYAML_LIB}
+  INSTALL_COMMAND ""
   CONFIGURE_COMMAND ./configure --enable-static CFLAGS=-I${LIBYAML_INSTALL_DIR}/include CPPFLAGS=-I${LIBYAML_INSTALL_DIR}/include LDFLAGS=-L${LIBYAML_INSTALL_DIR}/lib LIBS=-lyaml LUA=${LUAJIT_SRC}/luajit LUA_INCLUDE=-I${LUAJIT_INCLUDE}
-  INSTALL_COMMAND sh -c
-  "cp -R ${PROJECT_BINARY_DIR}/lyaml-prefix/src/lyaml/lib/* ${PROJECT_SOURCE_DIR}/userspace/engine/lua"
 )

userspace/engine/CMakeLists.txt

@@ -10,6 +10,8 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations under the License.
 
+add_subdirectory(lua)
+
 set(FALCO_ENGINE_SOURCE_FILES
     rules.cpp
     falco_common.cpp
@@ -36,7 +38,8 @@ if(MINIMAL_BUILD)
       "${STRING_VIEW_LITE_INCLUDE}"
       "${LIBSCAP_INCLUDE_DIRS}"
       "${LIBSINSP_INCLUDE_DIRS}"
-      "${PROJECT_BINARY_DIR}/userspace/engine")
+      "${PROJECT_BINARY_DIR}/userspace/engine"
+      "${PROJECT_BINARY_DIR}/userspace/engine/lua")
 else()
   target_include_directories(
     falco_engine
@@ -48,24 +51,11 @@ else()
       "${STRING_VIEW_LITE_INCLUDE}"
       "${LIBSCAP_INCLUDE_DIRS}"
       "${LIBSINSP_INCLUDE_DIRS}"
-      "${PROJECT_BINARY_DIR}/userspace/engine")
+      "${PROJECT_BINARY_DIR}/userspace/engine"
+      "${PROJECT_BINARY_DIR}/userspace/engine/lua")
 endif()
 
-target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${LPEG_LIB}" "${LYAML_LIB}" "${LIBYAML_LIB}")
+target_link_libraries(falco_engine "${FALCO_SINSP_LIBRARY}" "${LPEG_LIB}" "${LYAML_LIB}" "${LIBYAML_LIB}" luafiles)
 
 configure_file(config_falco_engine.h.in config_falco_engine.h)
 
-if(DEFINED FALCO_COMPONENT)
-  install(
-    DIRECTORY lua
-    DESTINATION "${FALCO_SHARE_DIR}"
-    COMPONENT "${FALCO_COMPONENT}"
-    FILES_MATCHING
-    PATTERN *.lua)
-else()
-  install(
-    DIRECTORY lua
-    DESTINATION "${FALCO_SHARE_DIR}"
-    FILES_MATCHING
-    PATTERN *.lua)
-endif()

userspace/engine/falco_common.cpp

@@ -19,6 +19,7 @@ limitations under the License.
 #include "config_falco_engine.h"
 #include "falco_common.h"
 #include "banned.h" // This raises a compilation error when certain functions are used
+#include "falco_engine_lua_files.hh"
 
 std::vector<std::string> falco_common::priority_names = {
 	"Emergency",
@@ -48,68 +49,33 @@ falco_common::~falco_common()
 	}
 }
 
-void falco_common::init(const char *lua_main_filename, const char *alternate_lua_dir)
+void falco_common::init()
 {
-	ifstream is;
-	string lua_dir = FALCO_ENGINE_LUA_DIR;
-	string lua_main_path = lua_dir + lua_main_filename;
-
-	is.open(lua_main_path);
-	if (!is.is_open())
+	// Strings in the list lua_module_strings need to be loaded as
+	// lua modules, which also involves adding them to the
+	// package.module table.
+	for(const auto &pair : lua_module_strings)
 	{
-		lua_dir = alternate_lua_dir;
-		lua_main_path = lua_dir + lua_main_filename;
+		lua_getglobal(m_ls, "package");
+		lua_getfield(m_ls, -1, "preload");
 
-		is.open(lua_main_path);
-		if (!is.is_open())
+		if(luaL_loadstring(m_ls, pair.first))
 		{
-			throw falco_exception("Could not find Falco Lua entrypoint (tried " +
-					      string(FALCO_ENGINE_LUA_DIR) + lua_main_filename + ", " +
-					      string(alternate_lua_dir) + lua_main_filename + ")");
+			throw falco_exception("Failed to load embedded lua code " +
+					      string(pair.second) + ": " + lua_tostring(m_ls, -1));
+		}
+
+		lua_setfield(m_ls, -2, pair.second);
+	}
+
+	// Strings in the list lua_code_strings need to be loaded and
+	// evaluated so any public functions can be directly called.
+	for(const auto &str : lua_code_strings)
+	{
+		if(luaL_loadstring(m_ls, str) || lua_pcall(m_ls, 0, 0, 0))
+		{
+			throw falco_exception("Failed to load + evaluate embedded lua code " +
+					      string(str) + ": " + lua_tostring(m_ls, -1));
 		}
 	}
-
-	// Initialize Lua interpreter
-	add_lua_path(lua_dir);
-
-	// Load the main program, which defines all the available functions.
-	string scriptstr((istreambuf_iterator<char>(is)),
-			 istreambuf_iterator<char>());
-
-	if(luaL_loadstring(m_ls, scriptstr.c_str()) || lua_pcall(m_ls, 0, 0, 0))
-	{
-		throw falco_exception("Failed to load script " +
-			lua_main_path + ": " + lua_tostring(m_ls, -1));
-	}
-}
-
-void falco_common::add_lua_path(string &path)
-{
-	string cpath = string(path);
-	path += "?.lua";
-	cpath += "?.so";
-
-	lua_getglobal(m_ls, "package");
-
-	lua_getfield(m_ls, -1, "path");
-	string cur_path = lua_tostring(m_ls, -1 );
-	cur_path += ';';
-	lua_pop(m_ls, 1);
-
-	cur_path.append(path.c_str());
-
-	lua_pushstring(m_ls, cur_path.c_str());
-	lua_setfield(m_ls, -2, "path");
-
-	lua_getfield(m_ls, -1, "cpath");
-	string cur_cpath = lua_tostring(m_ls, -1 );
-	cur_cpath += ';';
-	lua_pop(m_ls, 1);
-
-	cur_cpath.append(cpath.c_str());
-
-	lua_pushstring(m_ls, cur_cpath.c_str());
-	lua_setfield(m_ls, -2, "cpath");
-
-	lua_pop(m_ls, 1);
 }

userspace/engine/falco_common.h

@@ -69,7 +69,7 @@ public:
 	falco_common();
 	virtual ~falco_common();
 
-	void init(const char *lua_main_filename, const char *alternate_lua_dir);
+	void init();
 
     // Priority levels, as a vector of strings
 	static std::vector<std::string> priority_names;
@@ -91,7 +91,4 @@ protected:
 	lua_State *m_ls;
 
 	std::mutex m_ls_semaphore;
-
-private:
-	void add_lua_path(std::string &path);
 };

userspace/engine/falco_engine.cpp

@@ -53,7 +53,7 @@ falco_engine::falco_engine(bool seed_rng, const std::string& alternate_lua_dir)
 	luaopen_lpeg(m_ls);
 	luaopen_yaml(m_ls);
 
-	falco_common::init(m_lua_main_filename.c_str(), alternate_lua_dir.c_str());
+	falco_common::init();
 	falco_rules::init(m_ls);
 
 	m_required_plugin_versions.clear();

userspace/engine/falco_engine.h

@@ -265,7 +265,6 @@ private:
 	uint32_t m_sampling_ratio;
 	double m_sampling_multiplier;
 
-	std::string m_lua_main_filename = "rule_loader.lua";
 	static const std::string s_default_ruleset;
 	uint32_t m_default_ruleset_id;
 

userspace/engine/lua/CMakeLists.txt

@@ -0,0 +1,21 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+file(GLOB_RECURSE lua_module_files ${CMAKE_CURRENT_SOURCE_DIR} *.lua)
+
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/falco_engine_lua_files.cpp
+	COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/lua-to-cpp.sh ${CMAKE_CURRENT_SOURCE_DIR} ${LYAML_LUA_DIR} ${CMAKE_CURRENT_BINARY_DIR}
+	DEPENDS ${lua_files} ${CMAKE_CURRENT_SOURCE_DIR}/lua-to-cpp.sh lyaml)
+
+add_library(luafiles falco_engine_lua_files.cpp)
+
+target_include_directories(luafiles PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

userspace/engine/lua/lua-to-cpp.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -euo pipefail
+
+LUA_FILE_DIR=$1
+LYAML_LUA_DIR=$2
+OUTPUT_DIR=$3
+
+MODULE_SYMS=()
+CODE_SYMS=()
+
+function add_lua_file {
+    filename=$1
+    is_module=$2
+
+    # Take the basename of the file
+    BASE_NAME=$(basename ${file} .lua)
+    SYMBOL_NAME="${BASE_NAME}_lua_file_contents"
+    FILE_CONTENTS=$(<${file})
+
+    # Add a symbol to the .cc file containing the contents of the file
+    echo "const char *${SYMBOL_NAME}=R\"LUAFILE(${FILE_CONTENTS})LUAFILE\";" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp
+
+    # Add an extern reference to the .hh file
+    echo "extern const char *${SYMBOL_NAME};" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh
+
+    if [[ "${is_module}" == "true" ]]; then
+	# Determine the module name for the file
+	if [[ "${file}" == *"/"* ]]; then
+	    MODULE_NAME=$(echo ${file} | tr / . | sed -e 's/.lua//')
+	else
+	    MODULE_NAME=$(basename ${file} .lua)
+	fi
+
+	# Add the pair (string contents, module name) to MODULE_SYMS
+	PAIR=$(echo "{${SYMBOL_NAME},\"${MODULE_NAME}\"}")
+	MODULE_SYMS+=(${PAIR})
+    else
+	# Add the string to CODE_SYMS
+	CODE_SYMS+=(${SYMBOL_NAME})
+    fi
+}
+
+cat <<EOF > ${OUTPUT_DIR}/falco_engine_lua_files.cpp
+// Automatically generated. Do not edit
+#include "falco_engine_lua_files.hh"
+EOF
+
+cat <<EOF > ${OUTPUT_DIR}/falco_engine_lua_files.hh
+#pragma once
+// Automatically generated. Do not edit
+#include <list>
+#include <utility>
+EOF
+
+# lyaml and any files in the "modules" subdirectory are treated as lua
+# modules.
+pushd ${LYAML_LUA_DIR}
+for file in *.lua */*.lua; do
+    add_lua_file $file "true"
+done
+popd
+
+pushd ${LUA_FILE_DIR}/modules
+for file in *.lua; do
+    add_lua_file $file "true"
+done
+popd
+
+# Any .lua files in this directory are treated as code with functions
+# to execute.
+pushd ${LUA_FILE_DIR}
+for file in ${LUA_FILE_DIR}/*.lua; do
+    add_lua_file $file "false"
+done
+popd
+
+# Create a list of lua module (string, module name) pairs from MODULE_SYMS
+echo "extern std::list<std::pair<const char *,const char *>> lua_module_strings;" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh
+echo "std::list<std::pair<const char *,const char *>> lua_module_strings = {$(IFS=, ; echo "${MODULE_SYMS[*]}")};" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp
+
+# Create a list of lua code strings from CODE_SYMS
+echo "extern std::list<const char *> lua_code_strings;" >> ${OUTPUT_DIR}/falco_engine_lua_files.hh
+echo "std::list<const char *> lua_code_strings = {$(IFS=, ; echo "${CODE_SYMS[*]}")};" >> ${OUTPUT_DIR}/falco_engine_lua_files.cpp