Merge branch 'dev' into agent-master

GitHub uses a library called Licensee to identify a project's license
type. It shows this information in the status bar and via the API if it
can unambiguously identify the license.

This commit updates the COPYING file so that it contains only the full
text of the GPL 2.0 license. The info that pertains to OpenSSL has now
been moved to the "License Terms" section in the README.

Collectively, these changes allow Licensee to successfully identify the
license type of Falco as GPL 2.0.

falco-CLA-1.0-signed-off-by: Andrea Kao <eirinikos@gmail.com>

.travis.yml

@@ -10,7 +10,7 @@ before_install:
     - sudo apt-get update
 install:
     - sudo apt-get --force-yes install g++-4.8
-    - sudo apt-get install rpm linux-headers-$(uname -r)
+    - sudo apt-get install rpm linux-headers-$(uname -r) libelf-dev
     - git clone https://github.com/draios/sysdig.git ../sysdig
     - sudo apt-get install -y python-pip libvirt-dev jq dkms
     - cd ..

COPYING

@@ -277,18 +277,6 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
 PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGES.
 
-* In addition, as a special exception, the copyright holders give
-* permission to link the code of portions of this program with the
-* OpenSSL library under certain conditions as described in each
-* individual source file, and distribute linked combinations
-* including the two.
-* You must obey the GNU General Public License in all respects
-* for all of the code used other than OpenSSL.  If you modify
-* file(s) with this exception, you may extend this exception to your
-* version of the file(s), but you are not obligated to do so.  If you
-* do not wish to do so, delete this exception statement from your
-* version.
-
                      END OF TERMS AND CONDITIONS
 
             How to Apply These Terms to Your New Programs

README.md

@@ -41,6 +41,10 @@ License Terms
 ---
 Falco is licensed to you under the [GPL 2.0](./COPYING) open source license.
 
+In addition, as a special exception, the copyright holders give permission to link the code of portions of this program with the OpenSSL library under certain conditions as described in each individual source file, and distribute linked combinations including the two.
+
+You must obey the GNU General Public License in all respects for all of the code used other than OpenSSL.  If you modify file(s) with this exception, you may extend this exception to your version of the file(s), but you are not obligated to do so.  If you do not wish to do so, delete this exception statement from your version.
+
 Contributor License Agreements
 ---
 ### Background

docker/dev/Dockerfile

@@ -17,18 +17,30 @@ ADD http://download.draios.com/apt-draios-priority /etc/apt/preferences.d/
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
 	bash-completion \
-	curl \
-	jq \
-	gnupg2 \
+	bc \
+	clang-7 \
 	ca-certificates \
+	curl \
+	gnupg2 \
 	gcc \
 	gcc-5 \
-	gdb && rm -rf /var/lib/apt/lists/*
+	gcc-6 \
+	gdb \
+	jq \
+	libc6-dev \
+	libelf-dev \
+	llvm-7 \
+ && rm -rf /var/lib/apt/lists/*
 
 # Since our base Debian image ships with GCC 7 which breaks older kernels, revert the
 # default to gcc-5.
 RUN rm -rf /usr/bin/gcc && ln -s /usr/bin/gcc-5 /usr/bin/gcc
 
+RUN rm -rf /usr/bin/clang \
+ && rm -rf /usr/bin/llc \
+ && ln -s /usr/bin/clang-7 /usr/bin/clang \
+ && ln -s /usr/bin/llc-7 /usr/bin/llc
+
 RUN curl -s https://s3.amazonaws.com/download.draios.com/DRAIOS-GPG-KEY.public | apt-key add - \
  && curl -s -o /etc/apt/sources.list.d/draios.list http://download.draios.com/$FALCO_REPOSITORY/deb/draios.list \
  && apt-get update \

docker/local/Dockerfile

@@ -17,18 +17,30 @@ ADD http://download.draios.com/apt-draios-priority /etc/apt/preferences.d/
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
 	bash-completion \
-	curl \
-	jq \
-	gnupg2 \
+	bc \
+	clang-7 \
 	ca-certificates \
+	curl \
+	dkms \
+	gnupg2 \
 	gcc \
 	gcc-5 \
-	dkms && rm -rf /var/lib/apt/lists/*
+	gcc-6 \
+	jq \
+	libc6-dev \
+	libelf-dev \
+	llvm-7 \
+ && rm -rf /var/lib/apt/lists/*
 
 # Since our base Debian image ships with GCC 7 which breaks older kernels, revert the
 # default to gcc-5.
 RUN rm -rf /usr/bin/gcc && ln -s /usr/bin/gcc-5 /usr/bin/gcc
 
+RUN rm -rf /usr/bin/clang \
+ && rm -rf /usr/bin/llc \
+ && ln -s /usr/bin/clang-7 /usr/bin/clang \
+ && ln -s /usr/bin/llc-7 /usr/bin/llc
+
 RUN ln -s $SYSDIG_HOST_ROOT/lib/modules /lib/modules
 
 ADD falco-${FALCO_VERSION}-x86_64.deb /

docker/stable/Dockerfile

@@ -17,17 +17,29 @@ ADD http://download.draios.com/apt-draios-priority /etc/apt/preferences.d/
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
 	bash-completion \
-	curl \
-	jq \
+	bc \
+	clang-7 \
 	ca-certificates \
+	curl \
 	gnupg2 \
 	gcc \
-	gcc-5 && rm -rf /var/lib/apt/lists/*
+	gcc-5 \
+	gcc-6 \
+	jq \
+	libc6-dev \
+	libelf-dev \
+	llvm-7 \
+ && rm -rf /var/lib/apt/lists/*
 
 # Since our base Debian image ships with GCC 7 which breaks older kernels, revert the
 # default to gcc-5.
 RUN rm -rf /usr/bin/gcc && ln -s /usr/bin/gcc-5 /usr/bin/gcc
 
+RUN rm -rf /usr/bin/clang \
+ && rm -rf /usr/bin/llc \
+ && ln -s /usr/bin/clang-7 /usr/bin/clang \
+ && ln -s /usr/bin/llc-7 /usr/bin/llc
+
 RUN curl -s https://s3.amazonaws.com/download.draios.com/DRAIOS-GPG-KEY.public | apt-key add - \
  && curl -s -o /etc/apt/sources.list.d/draios.list http://download.draios.com/$FALCO_REPOSITORY/deb/draios.list \
  && apt-get update \

test/falco_tests.yaml

@@ -713,3 +713,30 @@ trace_files: !mux
       - open_dev_null: 1
         dev_null: 0
     trace_file: trace_files/cat_write.scap
+
+  skip_unknown_noevt:
+    detect: False
+    stdout_contains: Skipping rule "Contains Unknown Event And Skipping" that contains unknown filter proc.nobody
+    rules_file:
+      - rules/skip_unknown_evt.yaml
+    trace_file: trace_files/cat_write.scap
+
+  skip_unknown_prefix:
+    detect: False
+    rules_file:
+      - rules/skip_unknown_prefix.yaml
+    trace_file: trace_files/cat_write.scap
+
+  skip_unknown_error:
+    exit_status: 1
+    stderr_contains: Rule "Contains Unknown Event And Not Skipping" contains unknown filter proc.nobody. Exiting.
+    rules_file:
+      - rules/skip_unknown_error.yaml
+    trace_file: trace_files/cat_write.scap
+
+  skip_unknown_unspec_error:
+    exit_status: 1
+    stderr_contains: Rule "Contains Unknown Event And Unspecified" contains unknown filter proc.nobody. Exiting.
+    rules_file:
+      - rules/skip_unknown_unspec.yaml
+    trace_file: trace_files/cat_write.scap

test/rules/skip_unknown_error.yaml

@@ -0,0 +1,6 @@
+- rule: Contains Unknown Event And Not Skipping
+  desc: Contains an unknown event
+  condition: proc.nobody=cat
+  output: Never
+  skip-if-unknown-filter: false
+  priority: INFO

test/rules/skip_unknown_evt.yaml

@@ -0,0 +1,6 @@
+- rule: Contains Unknown Event And Skipping
+  desc: Contains an unknown event
+  condition: evt.type=open and proc.nobody=cat
+  output: Never
+  skip-if-unknown-filter: true
+  priority: INFO

test/rules/skip_unknown_prefix.yaml

@@ -0,0 +1,8 @@
+- rule: Contains Prefix of Filter
+  desc: Testing matching filter prefixes
+  condition: >
+    evt.type=open and evt.arg.path="foo" and evt.arg[0]="foo"
+    and proc.aname="ls" and proc.aname[1]="ls"
+    and proc.apid=10 and proc.apid[1]=10
+  output: Never
+  priority: INFO

test/rules/skip_unknown_unspec.yaml

@@ -0,0 +1,5 @@
+- rule: Contains Unknown Event And Unspecified
+  desc: Contains an unknown event
+  condition: proc.nobody=cat
+  output: Never
+  priority: INFO

userspace/engine/CMakeLists.txt

@@ -3,6 +3,7 @@ include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap")
 include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp")
 include_directories("${PROJECT_BINARY_DIR}/userspace/engine")
 include_directories("${LUAJIT_INCLUDE}")
+include_directories("${CURL_INCLUDE_DIR}")
 
 add_library(falco_engine STATIC rules.cpp falco_common.cpp falco_engine.cpp token_bucket.cpp formats.cpp)
 

userspace/engine/lua/compiler.lua

@@ -322,6 +322,21 @@ function get_evttypes_syscalls(name, ast, source, warn_evttypes)
    return evttypes, syscallnums
 end
 
+function get_filters(ast)
+
+   local filters = {}
+
+   function cb(node)
+      if node.type == "FieldName" then
+	 filters[node.value] = 1
+      end
+   end
+
+   parser.traverse_ast(ast.filter.value, {FieldName=1} , cb)
+
+   return filters
+end
+
 function compiler.expand_lists_in(source, list_defs)
 
    for name, def in pairs(list_defs) do
@@ -408,7 +423,9 @@ function compiler.compile_filter(name, source, macro_defs, list_defs, warn_evtty
 
    evttypes, syscallnums = get_evttypes_syscalls(name, ast, source, warn_evttypes)
 
-   return ast, evttypes, syscallnums
+   filters = get_filters(ast)
+
+   return ast, evttypes, syscallnums, filters
 end
 
 

userspace/engine/lua/rule_loader.lua

@@ -275,6 +275,12 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
 	    error ("Missing name in rule")
 	 end
 
+	 -- By default, if a rule's condition refers to an unknown
+	 -- filter like evt.type, etc the loader throws an error.
+	 if v['skip-if-unknown-filter'] == nil then
+	    v['skip-if-unknown-filter'] = false
+	 end
+
 	 -- Possibly append to the condition field of an existing rule
 	 append = false
 
@@ -378,9 +384,34 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
 	 warn_evttypes = v['warn_evttypes']
       end
 
-      local filter_ast, evttypes, syscallnums = compiler.compile_filter(v['rule'], v['condition'],
-									state.macros, state.lists,
-									warn_evttypes)
+      local filter_ast, evttypes, syscallnums, filters = compiler.compile_filter(v['rule'], v['condition'],
+										 state.macros, state.lists,
+										 warn_evttypes)
+
+      -- If a filter in the rule doesn't exist, either skip the rule
+      -- or raise an error, depending on the value of
+      -- skip-if-unknown-filter.
+      for filter, _ in pairs(filters) do
+	 found = false
+
+	 for pat, _ in pairs(defined_filters) do
+	    if string.match(filter, pat) ~= nil then
+	       found = true
+	       break
+	    end
+	 end
+
+	 if not found then
+	    if v['skip-if-unknown-filter'] then
+	       if verbose then
+		  print("Skipping rule \""..v['rule'].."\" that contains unknown filter "..filter)
+	       end
+	       goto next_rule
+	    else
+	       error("Rule \""..v['rule'].."\" contains unknown filter "..filter)
+	    end
+	 end
+      end
 
       if (filter_ast.type == "Rule") then
 	 state.n_rules = state.n_rules + 1
@@ -418,6 +449,8 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
 
 	 if (v['enabled'] == false) then
 	    falco_rules.enable_rule(rules_mgr, v['rule'], 0)
+	 else
+	    falco_rules.enable_rule(rules_mgr, v['rule'], 1)
 	 end
 
 	 -- If the format string contains %container.info, replace it
@@ -452,6 +485,8 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
       else
 	 error ("Unexpected type in load_rule: "..filter_ast.type)
       end
+
+      ::next_rule::
    end
 
    if verbose then

userspace/engine/rules.cpp

@@ -258,6 +258,63 @@ void falco_rules::load_rules(const string &rules_content,
 
 		lua_setglobal(m_ls, m_lua_ignored_syscalls.c_str());
 
+		// Create a table containing all filtercheck names.
+		lua_newtable(m_ls);
+
+		vector<const filter_check_info*> fc_plugins;
+		sinsp::get_filtercheck_fields_info(&fc_plugins);
+
+		for(uint32_t j = 0; j < fc_plugins.size(); j++)
+		{
+			const filter_check_info* fci = fc_plugins[j];
+
+			if(fci->m_flags & filter_check_info::FL_HIDDEN)
+			{
+				continue;
+			}
+
+			for(int32_t k = 0; k < fci->m_nfields; k++)
+			{
+				const filtercheck_field_info* fld = &fci->m_fields[k];
+
+				if(fld->m_flags & EPF_TABLE_ONLY ||
+				   fld->m_flags & EPF_PRINT_ONLY)
+				{
+					continue;
+				}
+
+				// Some filters can work with or without an argument
+				std::set<string> flexible_filters = {
+					"^proc.aname",
+					"^proc.apid"
+				};
+
+				std::list<string> fields;
+				std::string field_base = string("^") + fld->m_name;
+
+				if(fld->m_flags & EPF_REQUIRES_ARGUMENT ||
+				   flexible_filters.find(field_base) != flexible_filters.end())
+				{
+					fields.push_back(field_base + "[%[%.]");
+				}
+
+				if(!(fld->m_flags & EPF_REQUIRES_ARGUMENT) ||
+				   flexible_filters.find(field_base) != flexible_filters.end())
+				{
+					fields.push_back(field_base + "$");
+				}
+
+				for(auto &field : fields)
+				{
+					lua_pushstring(m_ls, field.c_str());
+					lua_pushnumber(m_ls, 1);
+					lua_settable(m_ls, -3);
+				}
+			}
+		}
+
+		lua_setglobal(m_ls, m_lua_defined_filters.c_str());
+
 		lua_pushstring(m_ls, rules_content.c_str());
 		lua_pushlightuserdata(m_ls, this);
 		lua_pushboolean(m_ls, (verbose ? 1 : 0));

userspace/engine/rules.h

@@ -56,6 +56,7 @@ class falco_rules
 	string m_lua_load_rules = "load_rules";
 	string m_lua_ignored_syscalls = "ignored_syscalls";
 	string m_lua_ignored_events = "ignored_events";
+	string m_lua_defined_filters = "defined_filters";
 	string m_lua_events = "events";
 	string m_lua_syscalls = "syscalls";
 	string m_lua_describe_rule = "describe_rule";

userspace/falco/falco.cpp

@@ -158,7 +158,7 @@ uint64_t do_inspect(falco_engine *engine,
 		    bool all_events)
 {
 	uint64_t num_evts = 0;
-	int32_t res;
+	int32_t rc;
 	sinsp_evt* ev;
 	StatsFileWriter writer;
 	uint64_t duration_start = 0;
@@ -179,7 +179,7 @@ uint64_t do_inspect(falco_engine *engine,
 	while(1)
 	{
 
-		res = inspector->next(&ev);
+		rc = inspector->next(&ev);
 
 		writer.handle();
 
@@ -193,21 +193,21 @@ uint64_t do_inspect(falco_engine *engine,
 		{
 			break;
 		}
-		else if(res == SCAP_TIMEOUT)
+		else if(rc == SCAP_TIMEOUT)
 		{
 			continue;
 		}
-		else if(res == SCAP_EOF)
+		else if(rc == SCAP_EOF)
 		{
 			break;
 		}
-		else if(res != SCAP_SUCCESS)
+		else if(rc != SCAP_SUCCESS)
 		{
 			//
 			// Event read error.
 			// Notify the chisels that we're exiting, and then die with an error.
 			//
-			cerr << "res = " << res << endl;
+			cerr << "rc = " << rc << endl;
 			throw sinsp_exception(inspector->getlasterr().c_str());
 		}