From 0e1c436d14fb468dd1530115ed19e43faa0a8331 Mon Sep 17 00:00:00 2001
From: Mark Stemm <mark.stemm@gmail.com>
Date: Fri, 26 Apr 2019 12:24:15 -0700
Subject: [PATCH] Add jenkins checks (#584)

* Supporting files to build/test via jenkins

Changes to build/test via jenkins, which also means running all tests in
a container instead of directly on the host:

- Jenkinsfile controls the stages, build.sh does the build and
  run-tests.sh does the regression tests.

- Create a new container falcosecurity/falco-tester that includes the
  dependencies required to run the regression tests. This is a different
  image than falco-builder because it doesn't need to be centos 6 based,
  doesn't install any compiler/etc, and installs the test running
  framework we use (avocado). We now use a newer version of avocado,
  which resulted in some small changes to how it is run and how yaml
  options are parsed.

- Modify run_regression_tests.sh to download trace files to the build
  directory and only if not present. Also honor BUILD_TYPE/BUILD_DIR,
  which is provided via the docker run cmd.

- The package tests are now moved to a separate falco_tests_package.yaml
  file. They will use rpm installs by default instead of debian
  packages. Also add the ability to install rpms in addition to debian
  packages.

- Automate the process of creating the docker local package by: 1)
  Adding CMake rules to copy the Dockerfile, entrypoint to the build
  directory and 2) Copy test trace files and rules into the build
  directory. This allows running the docker build command from
  build/docker/local instead of the source directory.

- Modify the way the container test is run a bit to use the trace
  files/rules copied into the container directly instead of host-mounted
  trace files.

* Use container builder + tester for travis

We'll probably be using jenkins soon, but this will allow switching back
to travis later if we want.

* Use download.draios.com for binutils packages

That way we won't be dependent on snapshot.debian.org.
---
 .travis.yml                                 | 23 +------
 CMakeLists.txt                              |  2 +
 docker/CMakeLists.txt                       |  1 +
 docker/dev/Dockerfile                       | 11 ++--
 docker/local/CMakeLists.txt                 | 17 +++++
 docker/local/Dockerfile                     | 19 ++++--
 docker/local/rules/CMakeLists.txt           | 13 ++++
 docker/local/traces/CMakeLists.txt          | 13 ++++
 docker/stable/Dockerfile                    | 11 ++--
 docker/tester/Dockerfile                    | 17 +++++
 docker/tester/entrypoint.sh                 | 24 +++++++
 scripts/jenkins/build-pipeline/Jenkinsfile  | 71 +++++++++++++++++++++
 scripts/jenkins/build-pipeline/build.sh     | 11 ++++
 scripts/jenkins/build-pipeline/run-tests.sh | 10 +++
 test/CMakeLists.txt                         |  1 +
 test/falco_test.py                          | 57 +++++++++++------
 test/falco_tests.yaml                       | 33 ----------
 test/falco_tests_package.yaml               | 34 ++++++++++
 test/run_regression_tests.sh                | 31 ++++++---
 test/trace_files/CMakeLists.txt             | 13 ++++
 test/trace_files/k8s_audit/CMakeLists.txt   | 12 ++++
 21 files changed, 326 insertions(+), 98 deletions(-)
 create mode 100644 docker/CMakeLists.txt
 create mode 100644 docker/local/CMakeLists.txt
 create mode 100644 docker/local/rules/CMakeLists.txt
 create mode 100644 docker/local/traces/CMakeLists.txt
 create mode 100644 docker/tester/Dockerfile
 create mode 100755 docker/tester/entrypoint.sh
 create mode 100644 scripts/jenkins/build-pipeline/Jenkinsfile
 create mode 100755 scripts/jenkins/build-pipeline/build.sh
 create mode 100755 scripts/jenkins/build-pipeline/run-tests.sh
 create mode 100644 test/CMakeLists.txt
 create mode 100644 test/falco_tests_package.yaml
 create mode 100644 test/trace_files/CMakeLists.txt
 create mode 100644 test/trace_files/k8s_audit/CMakeLists.txt

diff --git a/.travis.yml b/.travis.yml
index d78fd141..fb38328e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -31,29 +31,12 @@ install:
     - git clone https://github.com/draios/sysdig.git ../sysdig
     # if available, use the branch with the same name in sysdig
     - pushd ../sysdig && (git checkout "${BRANCH}" || exit 0) && echo "Using sysdig branch:" $(git rev-parse --abbrev-ref HEAD) && popd
-    - sudo apt-get install -y python-pip libvirt-dev jq dkms
-    - cd ..
-    - curl -Lo avocado-36.0-tar.gz https://github.com/avocado-framework/avocado/archive/36.0lts.tar.gz
-    - tar -zxvf avocado-36.0-tar.gz
-    - cd avocado-36.0lts
-    - sed -e 's/libvirt-python>=1.2.9/libvirt-python>=1.2.9,<4.1.0/' < requirements.txt  > /tmp/requirements.txt && mv /tmp/requirements.txt ./requirements.txt
-    - sudo -H pip install -r requirements.txt
-    - sudo python setup.py install
-    - cd ../falco
-before_script:
-    - export KERNELDIR=/lib/modules/$(uname -r)/build
 script:
-    - set -e
     - mkdir build
     - cd build
-    - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DDRAIOS_DEBUG_FLAGS="-D_DEBUG -DNDEBUG"
-    - make VERBOSE=1
-    - make package
-    - cp falco*.deb ../docker/local
-    - cd ../docker/local
-    - docker build -t falcosecurity/falco:test .
-    - cd ../..
-    - sudo test/run_regression_tests.sh $TRAVIS_BRANCH
+    - docker run --user $(id -u):$(id -g) -v /etc/passwd:/etc/passwd:ro -e MAKE_JOBS=4 -v $TRAVIS_BUILD_DIR/..:/source -v $TRAVIS_BUILD_DIR/build:/build falcosecurity/falco-builder cmake
+    - docker run --user $(id -u):$(id -g) -v /etc/passwd:/etc/passwd:ro -e MAKE_JOBS=4 -v $TRAVIS_BUILD_DIR/..:/source -v $TRAVIS_BUILD_DIR/build:/build falcosecurity/falco-builder package
+    - docker run -v /boot:/boot:ro -v /var/run/docker.sock:/var/run/docker.sock -v /etc/passwd:/etc/passwd:ro -e MAKE_JOBS=4 -v $TRAVIS_BUILD_DIR/..:/source -v $TRAVIS_BUILD_DIR/build:/build falcosecurity/falco-tester
 notifications:
   webhooks:
     urls:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c2c6f546..bc22d703 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -599,7 +599,9 @@ endif()
 install(FILES falco.yaml
 	DESTINATION "${FALCO_ETC_DIR}")
 
+add_subdirectory(test)
 add_subdirectory(rules)
+add_subdirectory(docker)
 
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")
 	add_subdirectory("${SYSDIG_DIR}/driver" "${PROJECT_BINARY_DIR}/driver")
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
new file mode 100644
index 00000000..0c647998
--- /dev/null
+++ b/docker/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(local)
diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile
index c5aa6156..f96a167d 100644
--- a/docker/dev/Dockerfile
+++ b/docker/dev/Dockerfile
@@ -96,11 +96,12 @@ RUN rm -df /lib/modules \
 # debian:unstable head contains binutils 2.31, which generates
 # binaries that are incompatible with kernels < 4.16. So manually
 # forcibly install binutils 2.30-22 instead.
-RUN curl -s -o binutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils_2.30-22_amd64.deb \
- && curl -s -o libbinutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/libbinutils_2.30-22_amd64.deb \
- && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
- && curl -s -o binutils-common_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-common_2.30-22_amd64.deb \
- && dpkg -i *binutils*.deb
+RUN curl -s -o binutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils_2.30-22_amd64.deb \
+ && curl -s -o libbinutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/libbinutils_2.30-22_amd64.deb \
+ && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
+ && curl -s -o binutils-common_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-common_2.30-22_amd64.deb \
+ && dpkg -i *binutils*.deb \
+ && rm -f *binutils*.deb
 
 COPY ./docker-entrypoint.sh /
 
diff --git a/docker/local/CMakeLists.txt b/docker/local/CMakeLists.txt
new file mode 100644
index 00000000..614a527d
--- /dev/null
+++ b/docker/local/CMakeLists.txt
@@ -0,0 +1,17 @@
+add_subdirectory(traces)
+add_subdirectory(rules)
+
+add_custom_target(local-Dockerfile ALL
+	DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/Dockerfile)
+
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/Dockerfile
+	COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/Dockerfile ${CMAKE_CURRENT_BINARY_DIR}/Dockerfile
+	DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Dockerfile)
+
+add_custom_target(local-docker-entrypoint ALL
+	DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/docker-entrypoint)
+
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/docker-entrypoint
+	COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/docker-entrypoint.sh ${CMAKE_CURRENT_BINARY_DIR}/docker-entrypoint.sh
+	DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/docker-entrypoint.sh)
+
diff --git a/docker/local/Dockerfile b/docker/local/Dockerfile
index ac48e845..dc4f8fe1 100644
--- a/docker/local/Dockerfile
+++ b/docker/local/Dockerfile
@@ -2,7 +2,8 @@ FROM debian:unstable
 
 LABEL maintainer="Sysdig <support@sysdig.com>"
 
-ENV FALCO_VERSION 0.1.1dev
+ARG FALCO_VERSION=0.1.1dev
+ENV FALCO_VERSION ${FALCO_VERSION}
 
 LABEL RUN="docker run -i -t -v /var/run/docker.sock:/host/var/run/docker.sock -v /dev:/host/dev -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro --name NAME IMAGE"
 
@@ -91,11 +92,17 @@ RUN sed -e 's/time_format_iso_8601: false/time_format_iso_8601: true/' < /etc/fa
 # debian:unstable head contains binutils 2.31, which generates
 # binaries that are incompatible with kernels < 4.16. So manually
 # forcibly install binutils 2.30-22 instead.
-RUN curl -s -o binutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils_2.30-22_amd64.deb \
- && curl -s -o libbinutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/libbinutils_2.30-22_amd64.deb \
- && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
- && curl -s -o binutils-common_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-common_2.30-22_amd64.deb \
- && dpkg -i *binutils*.deb
+RUN curl -s -o binutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils_2.30-22_amd64.deb \
+ && curl -s -o libbinutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/libbinutils_2.30-22_amd64.deb \
+ && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
+ && curl -s -o binutils-common_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-common_2.30-22_amd64.deb \
+ && dpkg -i *binutils*.deb \
+ && rm -f *binutils*.deb
+
+# The local container also copies some test trace files and
+# corresponding rules that are used when running regression tests.
+COPY rules/*.yaml /rules/
+COPY traces/*.scap /traces/
 
 COPY ./docker-entrypoint.sh /
 
diff --git a/docker/local/rules/CMakeLists.txt b/docker/local/rules/CMakeLists.txt
new file mode 100644
index 00000000..f6b934c1
--- /dev/null
+++ b/docker/local/rules/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Note: list of rules is created at cmake time, not build time
+file(GLOB test_rule_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/../../../test/rules/*.yaml")
+
+foreach(rule_file_path ${test_rule_files})
+	get_filename_component(rule_file ${rule_file_path} NAME)
+	add_custom_target(docker-local-rule-${rule_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${rule_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${rule_file}
+		COMMAND ${CMAKE_COMMAND} -E copy ${rule_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${rule_file}
+		DEPENDS ${rule_file_path})
+endforeach()
+
diff --git a/docker/local/traces/CMakeLists.txt b/docker/local/traces/CMakeLists.txt
new file mode 100644
index 00000000..87cd3356
--- /dev/null
+++ b/docker/local/traces/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Note: list of traces is created at cmake time, not build time
+file(GLOB test_trace_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/../../../test/trace_files/*.scap")
+
+foreach(trace_file_path ${test_trace_files})
+	get_filename_component(trace_file ${trace_file_path} NAME)
+	add_custom_target(docker-local-trace-${trace_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${trace_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		DEPENDS ${trace_file_path})
+endforeach()
+
diff --git a/docker/stable/Dockerfile b/docker/stable/Dockerfile
index bf15c909..9ceab73a 100644
--- a/docker/stable/Dockerfile
+++ b/docker/stable/Dockerfile
@@ -95,11 +95,12 @@ RUN rm -df /lib/modules \
 # debian:unstable head contains binutils 2.31, which generates
 # binaries that are incompatible with kernels < 4.16. So manually
 # forcibly install binutils 2.30-22 instead.
-RUN curl -s -o binutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils_2.30-22_amd64.deb \
- && curl -s -o libbinutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/libbinutils_2.30-22_amd64.deb \
- && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
- && curl -s -o binutils-common_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-common_2.30-22_amd64.deb \
- && dpkg -i *binutils*.deb
+RUN curl -s -o binutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils_2.30-22_amd64.deb \
+ && curl -s -o libbinutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/libbinutils_2.30-22_amd64.deb \
+ && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
+ && curl -s -o binutils-common_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-common_2.30-22_amd64.deb \
+ && dpkg -i *binutils*.deb \
+ && rm -f *binutils*.deb
 
 COPY ./docker-entrypoint.sh /
 
diff --git a/docker/tester/Dockerfile b/docker/tester/Dockerfile
new file mode 100644
index 00000000..20b47ce6
--- /dev/null
+++ b/docker/tester/Dockerfile
@@ -0,0 +1,17 @@
+FROM centos:7
+
+ENV FALCO_VERSION 0.1.1dev
+ENV BUILD_TYPE Release
+
+RUN yum -y install epel-release && \
+    yum -y install \
+      python-pip \
+      docker \
+      jq \
+      unzip
+
+RUN pip install avocado-framework avocado-framework-plugin-varianter-yaml-to-mux
+
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/tester/entrypoint.sh b/docker/tester/entrypoint.sh
new file mode 100755
index 00000000..1a1c2626
--- /dev/null
+++ b/docker/tester/entrypoint.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+SOURCE_DIR=/source
+BUILD_DIR=/build
+TASK=${1:-test}
+
+if [ $TASK == "test" ]; then
+    echo "Building local docker image falcosecurity/falco:test from latest debian package..."
+    cp $BUILD_DIR/$BUILD_TYPE/falco*.deb $BUILD_DIR/$BUILD_TYPE/docker/local
+    cd $BUILD_DIR/$BUILD_TYPE/docker/local && docker build --build-arg FALCO_VERSION=${FALCO_VERSION} -t falcosecurity/falco:test .
+
+    echo "Running regression tests"
+    cd $SOURCE_DIR/falco/test
+    bash run_regression_tests.sh $BUILD_DIR/$BUILD_TYPE
+
+    docker rmi falcosecurity/falco:test || true
+    exit 0
+fi
+
+if [ $TASK == "bash" ]; then
+    exec /bin/bash
+fi
diff --git a/scripts/jenkins/build-pipeline/Jenkinsfile b/scripts/jenkins/build-pipeline/Jenkinsfile
new file mode 100644
index 00000000..232fa850
--- /dev/null
+++ b/scripts/jenkins/build-pipeline/Jenkinsfile
@@ -0,0 +1,71 @@
+void setBuildStatus(String context, String message, String state) {
+    step([
+	  $class: "GitHubCommitStatusSetter",
+	  reposSource: [$class: "ManuallyEnteredRepositorySource", url: "https://github.com/falcosecurity/falco"],
+	  contextSource: [$class: "ManuallyEnteredCommitContextSource", context: context],
+	  errorHandlers: [[$class: "ChangingBuildStatusErrorHandler", result: "UNSTABLE"]],
+	  statusResultSource: [ $class: "ConditionalStatusResultSource", results: [[$class: "AnyBuildResult", message: message, state: state]] ]
+	  ]);
+}
+
+pipeline {
+	agent { label "agent-docker-builder" }
+	stages {
+	    stage("Check out dependencies") {
+		steps {
+		    dir("falco") {
+			checkout([$class: "GitSCM",
+				  branches: [[name: "refs/heads/"+env.BRANCH_NAME]],
+				  doGenerateSubmoduleConfigurations: false,
+				  extensions: [],
+				  submoduleCfg: [],
+				  userRemoteConfigs: [[credentialsId: "github-jenkins-user-token", url: "https://github.com/draios/falco"]]])
+			    }
+		    dir("sysdig") {
+			checkout([$class: "GitSCM",
+				  branches: [[name: "dev"]],
+				  doGenerateSubmoduleConfigurations: false,
+				  extensions: [],
+				  submoduleCfg: [],
+				  userRemoteConfigs: [[credentialsId: "github-jenkins-user-token", url: "https://github.com/draios/sysdig"]]])
+			    }
+		}
+	    }
+	    stage("Build") {
+		steps {
+		    script{
+			sh("./falco/scripts/jenkins/build-pipeline/build.sh")
+		    }
+		}
+		post {
+		    success {
+			setBuildStatus("Build", "Build Successful", "SUCCESS")
+		    }
+		    failure {
+			setBuildStatus("Build", "Build Failed", "FAILURE")
+		    }
+		}
+	    }
+	    stage("Run tests") {
+		steps {
+		    script{
+			sh("./falco/scripts/jenkins/build-pipeline/run-tests.sh")
+		    }
+		}
+		post {
+		    success {
+			setBuildStatus("Run tests", "All tests passed", "SUCCESS")
+		    }
+		    failure {
+			setBuildStatus("Run tests", "One or more tests failed", "FAILURE")
+		    }
+		}
+	    }
+	}
+	post {
+		always {
+			cleanWs()
+		}
+	}
+}
+
diff --git a/scripts/jenkins/build-pipeline/build.sh b/scripts/jenkins/build-pipeline/build.sh
new file mode 100755
index 00000000..b1e38ed9
--- /dev/null
+++ b/scripts/jenkins/build-pipeline/build.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+export FALCO_VERSION=0.1.$((2700+BUILD_NUMBER))dev
+
+rm -rf ${WORKSPACE}/build
+mkdir ${WORKSPACE}/build
+
+docker run --user $(id -u):$(id -g) -v /etc/passwd:/etc/passwd:ro -e FALCO_VERSION=${FALCO_VERSION} -e MAKE_JOBS=4 -v ${WORKSPACE}:/source -v ${WORKSPACE}/build:/build falcosecurity/falco-builder cmake
+docker run --user $(id -u):$(id -g) -v /etc/passwd:/etc/passwd:ro -e FALCO_VERSION=${FALCO_VERSION} -e MAKE_JOBS=4 -v ${WORKSPACE}:/source -v ${WORKSPACE}/build:/build falcosecurity/falco-builder package
diff --git a/scripts/jenkins/build-pipeline/run-tests.sh b/scripts/jenkins/build-pipeline/run-tests.sh
new file mode 100755
index 00000000..cfeeab6e
--- /dev/null
+++ b/scripts/jenkins/build-pipeline/run-tests.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+export FALCO_VERSION=0.1.$((2700+BUILD_NUMBER))dev
+
+docker pull falcosecurity/falco-tester
+docker run -v /boot:/boot:ro -v /var/run/docker.sock:/var/run/docker.sock -v /etc/passwd:/etc/passwd:ro -e FALCO_VERSION=${FALCO_VERSION} -v ${WORKSPACE}:/source -v ${WORKSPACE}/build:/build falcosecurity/falco-tester
+
+exit 0
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
new file mode 100644
index 00000000..207e9795
--- /dev/null
+++ b/test/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(trace_files)
diff --git a/test/falco_test.py b/test/falco_test.py
index 8e73e9ee..99082979 100644
--- a/test/falco_test.py
+++ b/test/falco_test.py
@@ -34,7 +34,12 @@ class FalcoTest(Test):
         """
         Load the sysdig kernel module if not already loaded.
         """
-        self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build'))
+        build_type = "Release"
+        if 'BUILD_TYPE' in os.environ:
+            build_type = os.environ['BUILD_TYPE']
+
+        build_dir = os.path.join('/build', build_type)
+        self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, build_dir))
 
         self.stdout_contains = self.params.get('stdout_contains', '*', default='')
 
@@ -67,7 +72,7 @@ class FalcoTest(Test):
         self.trace_file = self.params.get('trace_file', '*', default='')
 
         if self.trace_file and not os.path.isabs(self.trace_file):
-            self.trace_file = os.path.join(self.basedir, self.trace_file)
+            self.trace_file = os.path.join(build_dir, "test", self.trace_file)
 
         self.json_output = self.params.get('json_output', '*', default=False)
         self.json_include_output_property = self.params.get('json_include_output_property', '*', default=True)
@@ -110,8 +115,8 @@ class FalcoTest(Test):
         else:
             detect_counts = {}
             for item in self.detect_counts:
-                for item2 in item:
-                    detect_counts[item2[0]] = item2[1]
+                for key, value in item.items():
+                    detect_counts[key] = value
             self.detect_counts = detect_counts
 
         self.rules_warning = self.params.get('rules_warning', '*', default=False)
@@ -139,15 +144,6 @@ class FalcoTest(Test):
 
         self.package = self.params.get('package', '*', default='None')
 
-        if self.package == 'None':
-            # Doing this in 2 steps instead of simply using
-            # module_is_loaded to avoid logging lsmod output to the log.
-            lsmod_output = process.system_output("lsmod", verbose=False)
-
-            if linux_modules.parse_lsmod_for_module(lsmod_output, 'falco_probe') == {}:
-                self.log.debug("Loading falco kernel module")
-                process.run('insmod {}/driver/falco-probe.ko'.format(self.falcodir), sudo=True)
-
         self.addl_docker_run_args = self.params.get('addl_docker_run_args', '*', default='')
 
         self.copy_local_driver = self.params.get('copy_local_driver', '*', default=False)
@@ -162,10 +158,10 @@ class FalcoTest(Test):
         else:
             outputs = []
             for item in self.outputs:
-                for item2 in item:
+                for key, value in item.items():
                     output = {}
-                    output['file'] = item2[0]
-                    output['line'] = item2[1]
+                    output['file'] = key
+                    output['line'] = value
                     outputs.append(output)
                     filedir = os.path.dirname(output['file'])
                     # Create the parent directory for the trace file if it doesn't exist.
@@ -309,16 +305,15 @@ class FalcoTest(Test):
             # Remove an existing falco-test container first. Note we don't check the output--docker rm
             # doesn't have an -i equivalent.
             res = process.run("docker rm falco-test", ignore_status=True)
+
             rules_dir = os.path.abspath(os.path.join(self.basedir, "./rules"))
             conf_dir = os.path.abspath(os.path.join(self.basedir, "../"))
             traces_dir = os.path.abspath(os.path.join(self.basedir, "./trace_files"))
-            self.falco_binary_path = "docker run -i -t --name falco-test --privileged " \
-                                     "-v {}:/host/rules -v {}:/host/conf -v {}:/host/traces " \
+            self.falco_binary_path = "docker run --rm --name falco-test --privileged " \
                                      "-v /var/run/docker.sock:/host/var/run/docker.sock " \
                                      "-v /dev:/host/dev -v /proc:/host/proc:ro -v /boot:/host/boot:ro " \
                                      "-v /lib/modules:/host/lib/modules:ro -v {}:/root/.sysdig:ro -v " \
                                      "/usr:/host/usr:ro {} {} falco".format(
-                                         rules_dir, conf_dir, traces_dir,
                                          self.module_dir, self.addl_docker_run_args, image)
 
         elif self.package.endswith(".deb"):
@@ -337,11 +332,31 @@ class FalcoTest(Test):
             self.log.debug("Installing debian package via \"{}\"".format(cmdline))
             res = process.run(cmdline, timeout=120, sudo=True)
 
+        elif self.package.endswith(".rpm"):
+            self.falco_binary_path = '/usr/bin/falco';
+
+            package_glob = "{}/{}".format(self.falcodir, self.package)
+
+            matches = glob.glob(package_glob)
+
+            if len(matches) != 1:
+                self.fail("Package path {} did not match exactly 1 file. Instead it matched: {}", package_glob, ",".join(matches))
+
+            package_path = matches[0]
+
+            cmdline = "rpm -i --nodeps --noscripts {}".format(package_path)
+            self.log.debug("Installing centos package via \"{}\"".format(cmdline))
+            res = process.run(cmdline, timeout=120, sudo=True)
+
     def uninstall_package(self):
 
         if self.package.startswith("docker:"):
-            # Remove the falco-test image. Here we *do* check the return value
-            res = process.run("docker rm falco-test")
+            self.log.debug("Nothing to do, docker run with --rm")
+
+        elif self.package.endswith(".rpm"):
+            cmdline = "rpm -e --noscripts --nodeps falco"
+            self.log.debug("Uninstalling centos package via \"{}\"".format(cmdline))
+            res = process.run(cmdline, timeout=120, sudo=True)
 
         elif self.package.endswith(".deb"):
             cmdline = "dpkg --purge falco"
diff --git a/test/falco_tests.yaml b/test/falco_tests.yaml
index 4293bcf8..5a7c0598 100644
--- a/test/falco_tests.yaml
+++ b/test/falco_tests.yaml
@@ -17,39 +17,6 @@
 #
 trace_files: !mux
 
-  docker_package:
-    package: docker:falcosecurity/falco:test
-    detect: True
-    detect_level: WARNING
-    rules_file: /host/rules/rule_names_with_spaces.yaml
-    trace_file: /host/traces/cat_write.scap
-    conf_file: /host/conf/falco.yaml
-
-  # This uses a volume mount to overwrite and prevent /usr/sbin/dkms
-  # from being run. As a result, it will force falco-probe-loader to
-  # fall back to loading the driver from ~/.sysdig. Setting
-  # copy_local_driver to True copied the driver to ~/.sysdig, so it
-  # will be available. In this case, we're running live for 5 seconds
-  # just to see if falco can load the driver.
-
-  docker_package_local_driver:
-    package: docker:falcosecurity/falco:test
-    addl_docker_run_args: -v /dev/null:/usr/sbin/dkms
-    copy_local_driver: True
-    detect: False
-    detect_level: WARNING
-    rules_file: /host/rules/tagged_rules.yaml
-    conf_file: /host/conf/falco.yaml
-    run_duration: 5
-
-  debian_package:
-    package: falco*.deb
-    detect: True
-    detect_level: WARNING
-    rules_file:
-      - rules/rule_names_with_spaces.yaml
-    trace_file: trace_files/cat_write.scap
-
   builtin_rules_no_warnings:
     detect: False
     trace_file: trace_files/empty.scap
diff --git a/test/falco_tests_package.yaml b/test/falco_tests_package.yaml
new file mode 100644
index 00000000..7cc9e464
--- /dev/null
+++ b/test/falco_tests_package.yaml
@@ -0,0 +1,34 @@
+#
+# Copyright (C) 2016-2018 Draios Inc dba Sysdig.
+#
+# This file is part of falco.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+trace_files: !mux
+
+  docker_package:
+    package: docker:falcosecurity/falco:test
+    detect: True
+    detect_level: WARNING
+    rules_file: /rules/rule_names_with_spaces.yaml
+    trace_file: /traces/cat_write.scap
+    conf_file: /etc/falco/falco.yaml
+
+  centos_package:
+    package: falco*.rpm
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/rule_names_with_spaces.yaml
+    trace_file: trace_files/cat_write.scap
diff --git a/test/run_regression_tests.sh b/test/run_regression_tests.sh
index 964f3848..8fd6866a 100755
--- a/test/run_regression_tests.sh
+++ b/test/run_regression_tests.sh
@@ -16,17 +16,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+set -euo pipefail
+
 SCRIPT=$(readlink -f $0)
 SCRIPTDIR=$(dirname $SCRIPT)
-BRANCH=$1
+BUILD_DIR=$1
+BRANCH=${2:-none}
+
+TRACE_DIR=$BUILD_DIR/test
+
+mkdir -p $TRACE_DIR
 
 function download_trace_files() {
     echo "branch=$BRANCH"
     for TRACE in traces-positive traces-negative traces-info ; do
-	rm -rf $SCRIPTDIR/$TRACE
-	curl -fso $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE-$BRANCH.zip || curl -fso $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip &&
-	unzip -d $SCRIPTDIR $SCRIPTDIR/$TRACE.zip &&
-	rm -rf $SCRIPTDIR/$TRACE.zip
+	if [ ! -e $TRACE_DIR/$TRACE ]; then
+	    if [ $BRANCH != "none" ]; then
+		curl -fso $TRACE_DIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE-$BRANCH.zip
+	    else
+		curl -fso $TRACE_DIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip
+	    fi
+	    unzip -d $TRACE_DIR $TRACE_DIR/$TRACE.zip
+	    rm -rf $TRACE_DIR/$TRACE.zip
+	fi
     done
 }
 
@@ -35,7 +47,7 @@ function prepare_multiplex_fileset() {
     dir=$1
     detect=$2
 
-    for trace in $SCRIPTDIR/$dir/*.scap ; do
+    for trace in $TRACE_DIR/$dir/*.scap ; do
 	[ -e "$trace" ] || continue
 	NAME=`basename $trace .scap`
 
@@ -73,9 +85,12 @@ function print_test_failure_details() {
 function run_tests() {
     rm -rf /tmp/falco_outputs
     mkdir /tmp/falco_outputs
+    # If we got this far, we can undo set -e, as we're watching the
+    # return status when running avocado.
+    set +e
     TEST_RC=0
-    for mult in $SCRIPTDIR/falco_traces.yaml $SCRIPTDIR/falco_tests.yaml $SCRIPTDIR/falco_k8s_audit_tests.yaml; do
-	CMD="avocado run --multiplex $mult --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
+    for mult in $SCRIPTDIR/falco_traces.yaml $SCRIPTDIR/falco_tests.yaml $SCRIPTDIR/falco_tests_package.yaml $SCRIPTDIR/falco_k8s_audit_tests.yaml; do
+	CMD="avocado run --mux-yaml $mult --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
 	echo "Running: $CMD"
 	$CMD
 	RC=$?
diff --git a/test/trace_files/CMakeLists.txt b/test/trace_files/CMakeLists.txt
new file mode 100644
index 00000000..b6ede4ec
--- /dev/null
+++ b/test/trace_files/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_subdirectory(k8s_audit)
+# Note: list of traces is created at cmake time, not build time
+file(GLOB test_trace_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/*.scap")
+
+foreach(trace_file_path ${test_trace_files})
+	get_filename_component(trace_file ${trace_file_path} NAME)
+	add_custom_target(test-trace-${trace_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${trace_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		DEPENDS ${trace_file_path})
+endforeach()
diff --git a/test/trace_files/k8s_audit/CMakeLists.txt b/test/trace_files/k8s_audit/CMakeLists.txt
new file mode 100644
index 00000000..5be3d6aa
--- /dev/null
+++ b/test/trace_files/k8s_audit/CMakeLists.txt
@@ -0,0 +1,12 @@
+# Note: list of traces is created at cmake time, not build time
+file(GLOB test_trace_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/*.json")
+
+foreach(trace_file_path ${test_trace_files})
+	get_filename_component(trace_file ${trace_file_path} NAME)
+	add_custom_target(test-trace-${trace_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${trace_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		DEPENDS ${trace_file_path})
+endforeach()