Merge branch 'dev' into agent-master

* Add alternatives as a binary dir writer

It can set symlinks below binary dirs.

* Let userhelper read sens.files/write below /etc

Part of usermode package, can be used by oVirt.

* Let package mgmt progs urlgrabber pki files

Some package management programs run urlgrabber-ext-{down} to update pki
files.

* Add additional root directory

for Jupyter-notebook

* Let brandbot write to /etc/os-release

Used on centos

* Add an additional veritas conf directory.

Also /etc/opt/VRTS...

* Let appdynamics spawn shells

Java, so we look at parent cmdline.

* Add more ancestors to output

In an attempt to track down the source of some additional shell
spawners, add additional parents.

* Let chef write below bin dirs/rpm database

Rename an existing macro chef_running_yum_dump to python_running_chef
and add additional variants.

Also add chef-client as a package management binary.

* Remove dangling macro.

No longer in use.

* Add additional volume mgmt progs

Add pvscan as a volume management program and add an additional
directory below /etc. Also rename the macro to make it more generic.

* Let openldap write below /etc/openldap

Only program is run-openldap.sh for now.

* Add additional veritas directory

Also /etc/vom.

* Let sed write /etc/sedXXXXX files

These are often seen in install scrips for rpm/deb packages. The test
only checks for /etc/sed, as we don't have anything like a regex match
or glob operator.

* Let dse (DataStax Search) write to /root

Only file is /root/tmp__.

* Add additional mysql programs and directories

Add run-mysqld and /etc/my.cnf.d directory.

* Let redis write its config below /etc.

* Let id program open network connections

Seen using port 111 (sun-rpc, but really user lookups).

* Opt-in rule for protecting tomcat shell spawns

Some users want to consider any shell spawned by tomcat suspect for
example, protecting against the famous apache struts attack
CVE-2017-5638, while others do not.

Split the difference by adding a macro
possibly_parent_java_running_tomcat, but disabling it by default.

*  added ossec-syscheckd to read_sensitive_file_binaries

* Add "Write below monitored directory"

Take the technique used by "Write below binary dir", and make it more
general, expanding to a list of "monitored directories". This contains
common directories like /boot, /lib, etc.

It has a small workaround to look for home ssh directories without using
the glob operator, which has a pending fix in
https://github.com/draios/sysdig/pull/1153.

* Fix FPs

Move monitored_dir to after evt type checks and allow mkinitramfs to
write below /boot

* Addl boot writers.

.travis.yml

@@ -10,7 +10,7 @@ before_install:
     - sudo apt-get update
 install:
     - sudo apt-get --force-yes install g++-4.8
-    - sudo apt-get install rpm linux-headers-$(uname -r)
+    - sudo apt-get install rpm linux-headers-$(uname -r) libelf-dev
     - git clone https://github.com/draios/sysdig.git ../sysdig
     - sudo apt-get install -y python-pip libvirt-dev jq dkms
     - cd ..

COPYING

@@ -277,18 +277,6 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
 PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGES.
 
-* In addition, as a special exception, the copyright holders give
-* permission to link the code of portions of this program with the
-* OpenSSL library under certain conditions as described in each
-* individual source file, and distribute linked combinations
-* including the two.
-* You must obey the GNU General Public License in all respects
-* for all of the code used other than OpenSSL.  If you modify
-* file(s) with this exception, you may extend this exception to your
-* version of the file(s), but you are not obligated to do so.  If you
-* do not wish to do so, delete this exception statement from your
-* version.
-
                      END OF TERMS AND CONDITIONS
 
             How to Apply These Terms to Your New Programs

README.md

@@ -41,6 +41,10 @@ License Terms
 ---
 Falco is licensed to you under the [GPL 2.0](./COPYING) open source license.
 
+In addition, as a special exception, the copyright holders give permission to link the code of portions of this program with the OpenSSL library under certain conditions as described in each individual source file, and distribute linked combinations including the two.
+
+You must obey the GNU General Public License in all respects for all of the code used other than OpenSSL.  If you modify file(s) with this exception, you may extend this exception to your version of the file(s), but you are not obligated to do so.  If you do not wish to do so, delete this exception statement from your version.
+
 Contributor License Agreements
 ---
 ### Background

docker/dev/Dockerfile

@@ -17,21 +17,30 @@ ADD http://download.draios.com/apt-draios-priority /etc/apt/preferences.d/
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
 	bash-completion \
+	bc \
+	clang-7 \
 	ca-certificates \
 	curl \
 	gnupg2 \
 	gcc \
 	gcc-5 \
+	gcc-6 \
 	gdb \
 	jq \
 	libc6-dev \
 	libelf-dev \
+	llvm-7 \
  && rm -rf /var/lib/apt/lists/*
 
 # Since our base Debian image ships with GCC 7 which breaks older kernels, revert the
 # default to gcc-5.
 RUN rm -rf /usr/bin/gcc && ln -s /usr/bin/gcc-5 /usr/bin/gcc
 
+RUN rm -rf /usr/bin/clang \
+ && rm -rf /usr/bin/llc \
+ && ln -s /usr/bin/clang-7 /usr/bin/clang \
+ && ln -s /usr/bin/llc-7 /usr/bin/llc
+
 RUN curl -s https://s3.amazonaws.com/download.draios.com/DRAIOS-GPG-KEY.public | apt-key add - \
  && curl -s -o /etc/apt/sources.list.d/draios.list http://download.draios.com/$FALCO_REPOSITORY/deb/draios.list \
  && apt-get update \

docker/local/Dockerfile

@@ -17,21 +17,30 @@ ADD http://download.draios.com/apt-draios-priority /etc/apt/preferences.d/
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
 	bash-completion \
+	bc \
+	clang-7 \
 	ca-certificates \
 	curl \
 	dkms \
 	gnupg2 \
 	gcc \
 	gcc-5 \
+	gcc-6 \
 	jq \
 	libc6-dev \
 	libelf-dev \
+	llvm-7 \
  && rm -rf /var/lib/apt/lists/*
 
 # Since our base Debian image ships with GCC 7 which breaks older kernels, revert the
 # default to gcc-5.
 RUN rm -rf /usr/bin/gcc && ln -s /usr/bin/gcc-5 /usr/bin/gcc
 
+RUN rm -rf /usr/bin/clang \
+ && rm -rf /usr/bin/llc \
+ && ln -s /usr/bin/clang-7 /usr/bin/clang \
+ && ln -s /usr/bin/llc-7 /usr/bin/llc
+
 RUN ln -s $SYSDIG_HOST_ROOT/lib/modules /lib/modules
 
 ADD falco-${FALCO_VERSION}-x86_64.deb /

docker/stable/Dockerfile

@@ -17,20 +17,29 @@ ADD http://download.draios.com/apt-draios-priority /etc/apt/preferences.d/
 RUN apt-get update \
  && apt-get install -y --no-install-recommends \
 	bash-completion \
+	bc \
+	clang-7 \
 	ca-certificates \
 	curl \
 	gnupg2 \
 	gcc \
 	gcc-5 \
+	gcc-6 \
 	jq \
 	libc6-dev \
 	libelf-dev \
+	llvm-7 \
  && rm -rf /var/lib/apt/lists/*
 
 # Since our base Debian image ships with GCC 7 which breaks older kernels, revert the
 # default to gcc-5.
 RUN rm -rf /usr/bin/gcc && ln -s /usr/bin/gcc-5 /usr/bin/gcc
 
+RUN rm -rf /usr/bin/clang \
+ && rm -rf /usr/bin/llc \
+ && ln -s /usr/bin/clang-7 /usr/bin/clang \
+ && ln -s /usr/bin/llc-7 /usr/bin/llc
+
 RUN curl -s https://s3.amazonaws.com/download.draios.com/DRAIOS-GPG-KEY.public | apt-key add - \
  && curl -s -o /etc/apt/sources.list.d/draios.list http://download.draios.com/$FALCO_REPOSITORY/deb/draios.list \
  && apt-get update \

rules/falco_rules.yaml

@@ -166,7 +166,7 @@
 # The truncated dpkg-preconfigu is intentional, process names are
 # truncated at the sysdig level.
 - list: package_mgmt_binaries
-  items: [rpm_binaries, deb_binaries, update-alternat, gem, pip, sane-utils.post]
+  items: [rpm_binaries, deb_binaries, update-alternat, gem, pip, sane-utils.post, alternatives, chef-client]
 
 - macro: package_mgmt_procs
   condition: proc.name in (package_mgmt_binaries)
@@ -375,6 +375,9 @@
      (proc.cmdline startswith "sed -ri" or proc.cmdline startswith "sed -i") and
      (fd.name startswith /etc/httpd/conf.d/ or fd.name startswith /etc/httpd/conf))
 
+- macro: userhelper_writing_etc_security
+  condition: (proc.name=userhelper and fd.name startswith /etc/security)
+
 - macro: parent_Xvfb_running_xkbcomp
   condition: (proc.pname=Xvfb and proc.cmdline startswith 'sh -c "/usr/bin/xkbcomp"')
 
@@ -395,18 +398,13 @@
 - list: known_shell_spawn_binaries
   items: []
 
-- macro: shell_spawning_containers
-  condition: (container.image startswith jenkins or
-              container.image startswith gitlab/gitlab-ce or
-              container.image startswith gitlab/gitlab-ee)
-
 ## End Deprecated
 
 - macro: ansible_running_python
   condition: (proc.name in (python, pypy) and proc.cmdline contains ansible)
 
-- macro: chef_running_yum_dump
-  condition: (proc.name=python and proc.cmdline contains yum-dump.py)
+- macro: python_running_chef
+  condition: (proc.name=python and (proc.cmdline contains yum-dump.py or proc.cmdline="python /usr/bin/chef-monitor.py"))
 
 - macro: python_running_denyhosts
   condition: >
@@ -490,9 +488,13 @@
 - macro: htpasswd_writing_passwd
   condition: (proc.name=htpasswd and fd.name=/etc/nginx/.htpasswd)
 
-- macro: lvprogs_writing_lvm_archive
-  condition: (proc.name in (dmeventd,lvcreate) and (fd.name startswith /etc/lvm/archive or
-                                      fd.name startswith /etc/lvm/backup))
+- macro: lvprogs_writing_conf
+  condition: >
+    (proc.name in (dmeventd,lvcreate,pvscan) and
+     (fd.name startswith /etc/lvm/archive or
+      fd.name startswith /etc/lvm/backup or
+      fd.name startswith /etc/lvm/cache))
+
 - macro: ovsdb_writing_openvswitch
   condition: (proc.name=ovsdb-server and fd.directory=/etc/openvswitch)
 
@@ -541,6 +543,9 @@
 - macro: slapadd_writing_conf
   condition: (proc.name=slapadd and fd.name startswith /etc/ldap)
 
+- macro: openldap_writing_conf
+  condition: (proc.pname=run-openldap.sh and fd.name startswith /etc/openldap)
+
 - macro: symantec_writing_conf
   condition: >
     ((proc.name=symcfgd and fd.name startswith /etc/symantec) or
@@ -554,6 +559,14 @@
     (proc.name=urlgrabber-ext- and proc.aname[3]=sosreport and
      (fd.name startswith /etc/pkt/nssdb or fd.name startswith /etc/pki/nssdb))
 
+- macro: pkgmgmt_progs_writing_pki
+  condition: >
+    (proc.name=urlgrabber-ext- and proc.pname in (yum, yum-cron, repoquery) and
+     (fd.name startswith /etc/pkt/nssdb or fd.name startswith /etc/pki/nssdb))
+
+- macro: brandbot_writing_os_release
+  condition: proc.name=brandbot and fd.name=/etc/os-release
+
 - macro: selinux_writing_conf
   condition: (proc.name in (semodule,genhomedircon,sefcontext_comp) and fd.name startswith /etc/selinux)
 
@@ -567,7 +580,7 @@
   condition: (proc.name in (veritas_binaries) or veritas_driver_script)
 
 - macro: veritas_writing_config
-  condition: (veritas_progs and fd.name startswith /etc/vx)
+  condition: (veritas_progs and (fd.name startswith /etc/vx or fd.name startswith /etc/opt/VRTS or fd.name startswith /etc/vom))
 
 - macro: nginx_writing_conf
   condition: (proc.name=nginx and fd.name startswith /etc/nginx)
@@ -593,6 +606,11 @@
 - macro: exe_running_docker_save
   condition: (proc.cmdline startswith "exe /var/lib/docker" and proc.pname in (dockerd, docker))
 
+# Ideally we'd have a length check here as well but sysdig
+# filterchecks don't have operators like len()
+- macro: sed_temporary_file
+  condition: (proc.name=sed and fd.name startswith "/etc/sed")
+
 - macro: python_running_get_pip
   condition: (proc.cmdline startswith "python get-pip.py")
 
@@ -602,6 +620,9 @@
 - macro: gugent_writing_guestagent_log
   condition: (proc.name=gugent and fd.name=GuestAgent.log)
 
+- macro: dse_writing_tmp
+  condition: (proc.name=dse-entrypoint and fd.name=/root/tmp__)
+
 - rule: Write below binary dir
   desc: an attempt to write to any file below a set of binary directories
   condition: >
@@ -616,6 +637,44 @@
   priority: ERROR
   tags: [filesystem]
 
+# If you'd like to generally monitor a wider set of directories on top
+# of the ones covered by the rule Write below binary dir, you can use
+# the following rule and lists.
+
+- list: monitored_directories
+  items: [/boot, /lib, /lib64, /usr/lib, /usr/local/lib, /usr/local/sbin, /usr/local/bin, /root/.ssh, /etc/cardserver]
+
+# Until https://github.com/draios/sysdig/pull/1153, which fixes
+# https://github.com/draios/sysdig/issues/1152, is widely available,
+# we can't use glob operators to match pathnames. Until then, we do a
+# looser check to match ssh directories.
+# When fixed, we will use "fd.name glob '/home/*/.ssh/*'"
+- macro: user_ssh_directory
+  condition: (fd.name startswith '/home' and fd.name contains '.ssh')
+
+- macro: mkinitramfs_writing_boot
+  condition: (proc.pname in (mkinitramfs, update-initramf) and fd.directory=/boot)
+
+- macro: monitored_dir
+  condition: >
+    (fd.directory in (monitored_directories)
+     or user_ssh_directory)
+    and not mkinitramfs_writing_boot
+
+- rule: Write below monitored dir
+  desc: an attempt to write to any file below a set of binary directories
+  condition: >
+    evt.dir = < and open_write and monitored_dir
+    and not package_mgmt_procs
+    and not exe_running_docker_save
+    and not python_running_get_pip
+    and not python_running_ms_oms
+  output: >
+    File below a monitored directory opened for writing (user=%user.name
+    command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline gparent=%proc.aname[2])
+  priority: ERROR
+  tags: [filesystem]
+
 - list: safe_etc_dirs
   items: [/etc/cassandra, /etc/ssl/certs/java, /etc/logstash, /etc/nginx/conf.d, /etc/container_environment, /etc/hrmconfig]
 
@@ -677,7 +736,13 @@
   condition: (proc.name=httpd and fd.name startswith /etc/httpd/)
 
 - macro: mysql_writing_conf
-  condition: ((proc.name=start-mysql.sh or proc.pname=start-mysql.sh) and fd.name startswith /etc/mysql)
+  condition: >
+    ((proc.name in (start-mysql.sh, run-mysqld) or proc.pname=start-mysql.sh) and
+     (fd.name startswith /etc/mysql or fd.directory=/etc/my.cnf.d))
+
+- macro: redis_writing_conf
+  condition: >
+    (proc.name=run-redis and fd.name=/etc/redis.conf)
 
 - macro: openvpn_writing_conf
   condition: (proc.name in (openvpn,openvpn-entrypo) and fd.name startswith /etc/openvpn)
@@ -733,6 +798,7 @@
     and not proc.pname in (sysdigcloud_binaries, mail_config_binaries, hddtemp.postins, sshkit_script_binaries, locales.postins, deb_binaries, dhcp_binaries)
     and not fd.name pmatch (safe_etc_dirs)
     and not fd.name in (/etc/container_environment.sh, /etc/container_environment.json, /etc/motd, /etc/motd.svc)
+    and not sed_temporary_file
     and not exe_running_docker_save
     and not ansible_running_python
     and not python_running_denyhosts
@@ -754,7 +820,7 @@
     and not supervise_writing_status
     and not pki_realm_writing_realms
     and not htpasswd_writing_passwd
-    and not lvprogs_writing_lvm_archive
+    and not lvprogs_writing_conf
     and not ovsdb_writing_openvswitch
     and not datadog_writing_conf
     and not curl_writing_pki_db
@@ -791,6 +857,11 @@
     and not cockpit_writing_conf
     and not ipsec_writing_conf
     and not httpd_writing_ssl_conf
+    and not userhelper_writing_etc_security
+    and not pkgmgmt_progs_writing_pki
+    and not brandbot_writing_os_release
+    and not redis_writing_conf
+    and not openldap_writing_conf
 
 - rule: Write below etc
   desc: an attempt to write to any file below /etc
@@ -837,7 +908,8 @@
               or fd.name startswith /root/.dbus
               or fd.name startswith /root/.composer
               or fd.name startswith /root/.gconf
-              or fd.name startswith /root/.nv)
+              or fd.name startswith /root/.nv
+              or fd.name startswith /root/.local/share/jupyter)
 
 - rule: Write below root
   desc: an attempt to write to any file directly below / or /root
@@ -847,6 +919,7 @@
     and not fd.directory in (known_root_directories)
     and not exe_running_docker_save
     and not gugent_writing_guestagent_log
+    and not dse_writing_tmp
     and not known_root_conditions
   output: "File below / or /root opened for writing (user=%user.name command=%proc.cmdline parent=%proc.pname file=%fd.name program=%proc.name)"
   priority: ERROR
@@ -872,7 +945,7 @@
     iptables, ps, lsb_release, check-new-relea, dumpe2fs, accounts-daemon, sshd,
     vsftpd, systemd, mysql_install_d, psql, screen, debconf-show, sa-update,
     pam-auth-update, /usr/sbin/spamd, polkit-agent-he, lsattr, file, sosreport,
-    scxcimservera, adclient, rtvscand, cockpit-session
+    scxcimservera, adclient, rtvscand, cockpit-session, userhelper, ossec-syscheckd
     ]
 
 # Add conditions to this macro (probably in a separate file,
@@ -918,7 +991,7 @@
 # Only let rpm-related programs write to the rpm database
 - rule: Write below rpm database
   desc: an attempt to write to the rpm database by any non-rpm related program
-  condition: fd.name startswith /var/lib/rpm and open_write and not rpm_procs and not ansible_running_python and not chef_running_yum_dump
+  condition: fd.name startswith /var/lib/rpm and open_write and not rpm_procs and not ansible_running_python and not python_running_chef
   output: "Rpm database opened for writing by a non-rpm program (command=%proc.cmdline file=%fd.name)"
   priority: ERROR
   tags: [filesystem, software_mgmt]
@@ -940,6 +1013,9 @@
 - macro: rabbitmqctl_running_scripts
   condition: (proc.aname[2]=rabbitmqctl and proc.cmdline startswith "sh -c ")
 
+- macro: run_by_appdynamics
+  condition: (proc.pname=java and proc.pcmdline startswith "java -jar -Dappdynamics")
+
 - rule: DB program spawned process
   desc: >
     a database-server related program spawned a new process other than itself.
@@ -1072,6 +1148,17 @@
 - macro: possibly_node_in_container
   condition: (never_true and (proc.pname=node and proc.aname[3]=docker-containe))
 
+# Similarly, you may want to consider any shell spawned by apache
+# tomcat as suspect. The famous apache struts attack (CVE-2017-5638)
+# could be exploited to do things like spawn shells.
+#
+# However, many applications *do* use tomcat to run arbitrary shells,
+# as a part of build pipelines, etc.
+#
+# Like for node, we make this case opt-in.
+- macro: possibly_parent_java_running_tomcat
+  condition: (never_true and proc.pname=java and proc.pcmdline contains org.apache.catalina.startup.Bootstrap)
+
 - macro: protected_shell_spawner
   condition: >
     (proc.aname in (protected_shell_spawning_binaries)
@@ -1084,6 +1171,7 @@
     or parent_java_running_glassfish
     or parent_java_running_hadoop
     or parent_java_running_datastax
+    or possibly_parent_java_running_tomcat
     or possibly_node_in_container)
 
 - list: mesos_shell_binaries
@@ -1122,11 +1210,12 @@
     and not redis_running_prepost_scripts
     and not rabbitmq_running_scripts
     and not rabbitmqctl_running_scripts
+    and not run_by_appdynamics
     and not user_shell_container_exclusions
   output: >
     Shell spawned by untrusted binary (user=%user.name shell=%proc.name parent=%proc.pname
     cmdline=%proc.cmdline pcmdline=%proc.pcmdline gparent=%proc.aname[2] ggparent=%proc.aname[3]
-    gggparent=%proc.aname[4] ggggparent=%proc.aname[5])
+    aname[4]=%proc.aname[4] aname[5]=%proc.aname[5] aname[6]=%proc.aname[6] aname[7]=%proc.aname[7])
   priority: DEBUG
   tags: [shell]
 
@@ -1330,7 +1419,7 @@
   condition: >
     (fd.sockfamily = ip and system_procs)
     and (inbound_outbound)
-    and not proc.name in (systemd, hostid)
+    and not proc.name in (systemd, hostid, id)
     and not login_doing_dns_lookup
   output: >
     Known system binary sent/received network traffic

userspace/engine/CMakeLists.txt

@@ -3,6 +3,7 @@ include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libscap")
 include_directories("${PROJECT_SOURCE_DIR}/../sysdig/userspace/libsinsp")
 include_directories("${PROJECT_BINARY_DIR}/userspace/engine")
 include_directories("${LUAJIT_INCLUDE}")
+include_directories("${CURL_INCLUDE_DIR}")
 
 add_library(falco_engine STATIC rules.cpp falco_common.cpp falco_engine.cpp token_bucket.cpp formats.cpp)
 

userspace/engine/lua/rule_loader.lua

@@ -449,6 +449,8 @@ function load_rules(rules_content, rules_mgr, verbose, all_events, extra, replac
 
 	 if (v['enabled'] == false) then
 	    falco_rules.enable_rule(rules_mgr, v['rule'], 0)
+	 else
+	    falco_rules.enable_rule(rules_mgr, v['rule'], 1)
 	 end
 
 	 -- If the format string contains %container.info, replace it

userspace/falco/falco.cpp

@@ -158,7 +158,7 @@ uint64_t do_inspect(falco_engine *engine,
 		    bool all_events)
 {
 	uint64_t num_evts = 0;
-	int32_t res;
+	int32_t rc;
 	sinsp_evt* ev;
 	StatsFileWriter writer;
 	uint64_t duration_start = 0;
@@ -179,7 +179,7 @@ uint64_t do_inspect(falco_engine *engine,
 	while(1)
 	{
 
-		res = inspector->next(&ev);
+		rc = inspector->next(&ev);
 
 		writer.handle();
 
@@ -193,21 +193,21 @@ uint64_t do_inspect(falco_engine *engine,
 		{
 			break;
 		}
-		else if(res == SCAP_TIMEOUT)
+		else if(rc == SCAP_TIMEOUT)
 		{
 			continue;
 		}
-		else if(res == SCAP_EOF)
+		else if(rc == SCAP_EOF)
 		{
 			break;
 		}
-		else if(res != SCAP_SUCCESS)
+		else if(rc != SCAP_SUCCESS)
 		{
 			//
 			// Event read error.
 			// Notify the chisels that we're exiting, and then die with an error.
 			//
-			cerr << "res = " << res << endl;
+			cerr << "rc = " << rc << endl;
 			throw sinsp_exception(inspector->getlasterr().c_str());
 		}