Add jenkins checks (#584)

* Supporting files to build/test via jenkins

Changes to build/test via jenkins, which also means running all tests in
a container instead of directly on the host:

- Jenkinsfile controls the stages, build.sh does the build and
  run-tests.sh does the regression tests.

- Create a new container falcosecurity/falco-tester that includes the
  dependencies required to run the regression tests. This is a different
  image than falco-builder because it doesn't need to be centos 6 based,
  doesn't install any compiler/etc, and installs the test running
  framework we use (avocado). We now use a newer version of avocado,
  which resulted in some small changes to how it is run and how yaml
  options are parsed.

- Modify run_regression_tests.sh to download trace files to the build
  directory and only if not present. Also honor BUILD_TYPE/BUILD_DIR,
  which is provided via the docker run cmd.

- The package tests are now moved to a separate falco_tests_package.yaml
  file. They will use rpm installs by default instead of debian
  packages. Also add the ability to install rpms in addition to debian
  packages.

- Automate the process of creating the docker local package by: 1)
  Adding CMake rules to copy the Dockerfile, entrypoint to the build
  directory and 2) Copy test trace files and rules into the build
  directory. This allows running the docker build command from
  build/docker/local instead of the source directory.

- Modify the way the container test is run a bit to use the trace
  files/rules copied into the container directly instead of host-mounted
  trace files.

* Use container builder + tester for travis

We'll probably be using jenkins soon, but this will allow switching back
to travis later if we want.

* Use download.draios.com for binutils packages

That way we won't be dependent on snapshot.debian.org.

.travis.yml

@@ -31,29 +31,12 @@ install:
     - git clone https://github.com/draios/sysdig.git ../sysdig
     # if available, use the branch with the same name in sysdig
     - pushd ../sysdig && (git checkout "${BRANCH}" || exit 0) && echo "Using sysdig branch:" $(git rev-parse --abbrev-ref HEAD) && popd
-    - sudo apt-get install -y python-pip libvirt-dev jq dkms
-    - cd ..
-    - curl -Lo avocado-36.0-tar.gz https://github.com/avocado-framework/avocado/archive/36.0lts.tar.gz
-    - tar -zxvf avocado-36.0-tar.gz
-    - cd avocado-36.0lts
-    - sed -e 's/libvirt-python>=1.2.9/libvirt-python>=1.2.9,<4.1.0/' < requirements.txt  > /tmp/requirements.txt && mv /tmp/requirements.txt ./requirements.txt
-    - sudo -H pip install -r requirements.txt
-    - sudo python setup.py install
-    - cd ../falco
-before_script:
-    - export KERNELDIR=/lib/modules/$(uname -r)/build
 script:
-    - set -e
     - mkdir build
     - cd build
-    - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DDRAIOS_DEBUG_FLAGS="-D_DEBUG -DNDEBUG"
-    - make VERBOSE=1
-    - make package
-    - cp falco*.deb ../docker/local
-    - cd ../docker/local
-    - docker build -t falcosecurity/falco:test .
-    - cd ../..
-    - sudo test/run_regression_tests.sh $TRAVIS_BRANCH
+    - docker run --user $(id -u):$(id -g) -v /etc/passwd:/etc/passwd:ro -e MAKE_JOBS=4 -v $TRAVIS_BUILD_DIR/..:/source -v $TRAVIS_BUILD_DIR/build:/build falcosecurity/falco-builder cmake
+    - docker run --user $(id -u):$(id -g) -v /etc/passwd:/etc/passwd:ro -e MAKE_JOBS=4 -v $TRAVIS_BUILD_DIR/..:/source -v $TRAVIS_BUILD_DIR/build:/build falcosecurity/falco-builder package
+    - docker run -v /boot:/boot:ro -v /var/run/docker.sock:/var/run/docker.sock -v /etc/passwd:/etc/passwd:ro -e MAKE_JOBS=4 -v $TRAVIS_BUILD_DIR/..:/source -v $TRAVIS_BUILD_DIR/build:/build falcosecurity/falco-tester
 notifications:
   webhooks:
     urls:

CMakeLists.txt

@@ -599,7 +599,9 @@ endif()
 install(FILES falco.yaml
 	DESTINATION "${FALCO_ETC_DIR}")
 
+add_subdirectory(test)
 add_subdirectory(rules)
+add_subdirectory(docker)
 
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")
 	add_subdirectory("${SYSDIG_DIR}/driver" "${PROJECT_BINARY_DIR}/driver")

docker/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(local)

docker/dev/Dockerfile

@@ -96,11 +96,12 @@ RUN rm -df /lib/modules \
 # debian:unstable head contains binutils 2.31, which generates
 # binaries that are incompatible with kernels < 4.16. So manually
 # forcibly install binutils 2.30-22 instead.
-RUN curl -s -o binutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils_2.30-22_amd64.deb \
- && curl -s -o libbinutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/libbinutils_2.30-22_amd64.deb \
- && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
- && curl -s -o binutils-common_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-common_2.30-22_amd64.deb \
- && dpkg -i *binutils*.deb
+RUN curl -s -o binutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils_2.30-22_amd64.deb \
+ && curl -s -o libbinutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/libbinutils_2.30-22_amd64.deb \
+ && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
+ && curl -s -o binutils-common_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-common_2.30-22_amd64.deb \
+ && dpkg -i *binutils*.deb \
+ && rm -f *binutils*.deb
 
 COPY ./docker-entrypoint.sh /
 

docker/local/CMakeLists.txt

@@ -0,0 +1,17 @@
+add_subdirectory(traces)
+add_subdirectory(rules)
+
+add_custom_target(local-Dockerfile ALL
+	DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/Dockerfile)
+
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/Dockerfile
+	COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/Dockerfile ${CMAKE_CURRENT_BINARY_DIR}/Dockerfile
+	DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Dockerfile)
+
+add_custom_target(local-docker-entrypoint ALL
+	DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/docker-entrypoint)
+
+add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/docker-entrypoint
+	COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/docker-entrypoint.sh ${CMAKE_CURRENT_BINARY_DIR}/docker-entrypoint.sh
+	DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/docker-entrypoint.sh)
+

docker/local/Dockerfile

@@ -2,7 +2,8 @@ FROM debian:unstable
 
 LABEL maintainer="Sysdig <support@sysdig.com>"
 
-ENV FALCO_VERSION 0.1.1dev
+ARG FALCO_VERSION=0.1.1dev
+ENV FALCO_VERSION ${FALCO_VERSION}
 
 LABEL RUN="docker run -i -t -v /var/run/docker.sock:/host/var/run/docker.sock -v /dev:/host/dev -v /proc:/host/proc:ro -v /boot:/host/boot:ro -v /lib/modules:/host/lib/modules:ro -v /usr:/host/usr:ro --name NAME IMAGE"
 
@@ -91,11 +92,17 @@ RUN sed -e 's/time_format_iso_8601: false/time_format_iso_8601: true/' < /etc/fa
 # debian:unstable head contains binutils 2.31, which generates
 # binaries that are incompatible with kernels < 4.16. So manually
 # forcibly install binutils 2.30-22 instead.
-RUN curl -s -o binutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils_2.30-22_amd64.deb \
- && curl -s -o libbinutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/libbinutils_2.30-22_amd64.deb \
- && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
- && curl -s -o binutils-common_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-common_2.30-22_amd64.deb \
- && dpkg -i *binutils*.deb
+RUN curl -s -o binutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils_2.30-22_amd64.deb \
+ && curl -s -o libbinutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/libbinutils_2.30-22_amd64.deb \
+ && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
+ && curl -s -o binutils-common_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-common_2.30-22_amd64.deb \
+ && dpkg -i *binutils*.deb \
+ && rm -f *binutils*.deb
+
+# The local container also copies some test trace files and
+# corresponding rules that are used when running regression tests.
+COPY rules/*.yaml /rules/
+COPY traces/*.scap /traces/
 
 COPY ./docker-entrypoint.sh /
 

docker/local/rules/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Note: list of rules is created at cmake time, not build time
+file(GLOB test_rule_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/../../../test/rules/*.yaml")
+
+foreach(rule_file_path ${test_rule_files})
+	get_filename_component(rule_file ${rule_file_path} NAME)
+	add_custom_target(docker-local-rule-${rule_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${rule_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${rule_file}
+		COMMAND ${CMAKE_COMMAND} -E copy ${rule_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${rule_file}
+		DEPENDS ${rule_file_path})
+endforeach()
+

docker/local/traces/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Note: list of traces is created at cmake time, not build time
+file(GLOB test_trace_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/../../../test/trace_files/*.scap")
+
+foreach(trace_file_path ${test_trace_files})
+	get_filename_component(trace_file ${trace_file_path} NAME)
+	add_custom_target(docker-local-trace-${trace_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${trace_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		DEPENDS ${trace_file_path})
+endforeach()
+

docker/stable/Dockerfile

@@ -95,11 +95,12 @@ RUN rm -df /lib/modules \
 # debian:unstable head contains binutils 2.31, which generates
 # binaries that are incompatible with kernels < 4.16. So manually
 # forcibly install binutils 2.30-22 instead.
-RUN curl -s -o binutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils_2.30-22_amd64.deb \
- && curl -s -o libbinutils_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/libbinutils_2.30-22_amd64.deb \
- && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
- && curl -s -o binutils-common_2.30-22_amd64.deb http://snapshot.debian.org/archive/debian/20180622T211149Z/pool/main/b/binutils/binutils-common_2.30-22_amd64.deb \
- && dpkg -i *binutils*.deb
+RUN curl -s -o binutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils_2.30-22_amd64.deb \
+ && curl -s -o libbinutils_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/libbinutils_2.30-22_amd64.deb \
+ && curl -s -o binutils-x86-64-linux-gnu_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-x86-64-linux-gnu_2.30-22_amd64.deb \
+ && curl -s -o binutils-common_2.30-22_amd64.deb https://s3.amazonaws.com/download.draios.com/dependencies/binutils-common_2.30-22_amd64.deb \
+ && dpkg -i *binutils*.deb \
+ && rm -f *binutils*.deb
 
 COPY ./docker-entrypoint.sh /
 

docker/tester/Dockerfile

@@ -0,0 +1,17 @@
+FROM centos:7
+
+ENV FALCO_VERSION 0.1.1dev
+ENV BUILD_TYPE Release
+
+RUN yum -y install epel-release && \
+    yum -y install \
+      python-pip \
+      docker \
+      jq \
+      unzip
+
+RUN pip install avocado-framework avocado-framework-plugin-varianter-yaml-to-mux
+
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/tester/entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+SOURCE_DIR=/source
+BUILD_DIR=/build
+TASK=${1:-test}
+
+if [ $TASK == "test" ]; then
+    echo "Building local docker image falcosecurity/falco:test from latest debian package..."
+    cp $BUILD_DIR/$BUILD_TYPE/falco*.deb $BUILD_DIR/$BUILD_TYPE/docker/local
+    cd $BUILD_DIR/$BUILD_TYPE/docker/local && docker build --build-arg FALCO_VERSION=${FALCO_VERSION} -t falcosecurity/falco:test .
+
+    echo "Running regression tests"
+    cd $SOURCE_DIR/falco/test
+    bash run_regression_tests.sh $BUILD_DIR/$BUILD_TYPE
+
+    docker rmi falcosecurity/falco:test || true
+    exit 0
+fi
+
+if [ $TASK == "bash" ]; then
+    exec /bin/bash
+fi

scripts/jenkins/build-pipeline/Jenkinsfile

@@ -0,0 +1,71 @@
+void setBuildStatus(String context, String message, String state) {
+    step([
+	  $class: "GitHubCommitStatusSetter",
+	  reposSource: [$class: "ManuallyEnteredRepositorySource", url: "https://github.com/falcosecurity/falco"],
+	  contextSource: [$class: "ManuallyEnteredCommitContextSource", context: context],
+	  errorHandlers: [[$class: "ChangingBuildStatusErrorHandler", result: "UNSTABLE"]],
+	  statusResultSource: [ $class: "ConditionalStatusResultSource", results: [[$class: "AnyBuildResult", message: message, state: state]] ]
+	  ]);
+}
+
+pipeline {
+	agent { label "agent-docker-builder" }
+	stages {
+	    stage("Check out dependencies") {
+		steps {
+		    dir("falco") {
+			checkout([$class: "GitSCM",
+				  branches: [[name: "refs/heads/"+env.BRANCH_NAME]],
+				  doGenerateSubmoduleConfigurations: false,
+				  extensions: [],
+				  submoduleCfg: [],
+				  userRemoteConfigs: [[credentialsId: "github-jenkins-user-token", url: "https://github.com/draios/falco"]]])
+			    }
+		    dir("sysdig") {
+			checkout([$class: "GitSCM",
+				  branches: [[name: "dev"]],
+				  doGenerateSubmoduleConfigurations: false,
+				  extensions: [],
+				  submoduleCfg: [],
+				  userRemoteConfigs: [[credentialsId: "github-jenkins-user-token", url: "https://github.com/draios/sysdig"]]])
+			    }
+		}
+	    }
+	    stage("Build") {
+		steps {
+		    script{
+			sh("./falco/scripts/jenkins/build-pipeline/build.sh")
+		    }
+		}
+		post {
+		    success {
+			setBuildStatus("Build", "Build Successful", "SUCCESS")
+		    }
+		    failure {
+			setBuildStatus("Build", "Build Failed", "FAILURE")
+		    }
+		}
+	    }
+	    stage("Run tests") {
+		steps {
+		    script{
+			sh("./falco/scripts/jenkins/build-pipeline/run-tests.sh")
+		    }
+		}
+		post {
+		    success {
+			setBuildStatus("Run tests", "All tests passed", "SUCCESS")
+		    }
+		    failure {
+			setBuildStatus("Run tests", "One or more tests failed", "FAILURE")
+		    }
+		}
+	    }
+	}
+	post {
+		always {
+			cleanWs()
+		}
+	}
+}
+

scripts/jenkins/build-pipeline/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+export FALCO_VERSION=0.1.$((2700+BUILD_NUMBER))dev
+
+rm -rf ${WORKSPACE}/build
+mkdir ${WORKSPACE}/build
+
+docker run --user $(id -u):$(id -g) -v /etc/passwd:/etc/passwd:ro -e FALCO_VERSION=${FALCO_VERSION} -e MAKE_JOBS=4 -v ${WORKSPACE}:/source -v ${WORKSPACE}/build:/build falcosecurity/falco-builder cmake
+docker run --user $(id -u):$(id -g) -v /etc/passwd:/etc/passwd:ro -e FALCO_VERSION=${FALCO_VERSION} -e MAKE_JOBS=4 -v ${WORKSPACE}:/source -v ${WORKSPACE}/build:/build falcosecurity/falco-builder package

scripts/jenkins/build-pipeline/run-tests.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+export FALCO_VERSION=0.1.$((2700+BUILD_NUMBER))dev
+
+docker pull falcosecurity/falco-tester
+docker run -v /boot:/boot:ro -v /var/run/docker.sock:/var/run/docker.sock -v /etc/passwd:/etc/passwd:ro -e FALCO_VERSION=${FALCO_VERSION} -v ${WORKSPACE}:/source -v ${WORKSPACE}/build:/build falcosecurity/falco-tester
+
+exit 0

test/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(trace_files)

test/falco_test.py

@@ -34,7 +34,12 @@ class FalcoTest(Test):
         """
         Load the sysdig kernel module if not already loaded.
         """
-        self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, '../build'))
+        build_type = "Release"
+        if 'BUILD_TYPE' in os.environ:
+            build_type = os.environ['BUILD_TYPE']
+
+        build_dir = os.path.join('/build', build_type)
+        self.falcodir = self.params.get('falcodir', '/', default=os.path.join(self.basedir, build_dir))
 
         self.stdout_contains = self.params.get('stdout_contains', '*', default='')
 
@@ -67,7 +72,7 @@ class FalcoTest(Test):
         self.trace_file = self.params.get('trace_file', '*', default='')
 
         if self.trace_file and not os.path.isabs(self.trace_file):
-            self.trace_file = os.path.join(self.basedir, self.trace_file)
+            self.trace_file = os.path.join(build_dir, "test", self.trace_file)
 
         self.json_output = self.params.get('json_output', '*', default=False)
         self.json_include_output_property = self.params.get('json_include_output_property', '*', default=True)
@@ -110,8 +115,8 @@ class FalcoTest(Test):
         else:
             detect_counts = {}
             for item in self.detect_counts:
-                for item2 in item:
-                    detect_counts[item2[0]] = item2[1]
+                for key, value in item.items():
+                    detect_counts[key] = value
             self.detect_counts = detect_counts
 
         self.rules_warning = self.params.get('rules_warning', '*', default=False)
@@ -139,15 +144,6 @@ class FalcoTest(Test):
 
         self.package = self.params.get('package', '*', default='None')
 
-        if self.package == 'None':
-            # Doing this in 2 steps instead of simply using
-            # module_is_loaded to avoid logging lsmod output to the log.
-            lsmod_output = process.system_output("lsmod", verbose=False)
-
-            if linux_modules.parse_lsmod_for_module(lsmod_output, 'falco_probe') == {}:
-                self.log.debug("Loading falco kernel module")
-                process.run('insmod {}/driver/falco-probe.ko'.format(self.falcodir), sudo=True)
-
         self.addl_docker_run_args = self.params.get('addl_docker_run_args', '*', default='')
 
         self.copy_local_driver = self.params.get('copy_local_driver', '*', default=False)
@@ -162,10 +158,10 @@ class FalcoTest(Test):
         else:
             outputs = []
             for item in self.outputs:
-                for item2 in item:
+                for key, value in item.items():
                     output = {}
-                    output['file'] = item2[0]
-                    output['line'] = item2[1]
+                    output['file'] = key
+                    output['line'] = value
                     outputs.append(output)
                     filedir = os.path.dirname(output['file'])
                     # Create the parent directory for the trace file if it doesn't exist.
@@ -309,16 +305,15 @@ class FalcoTest(Test):
             # Remove an existing falco-test container first. Note we don't check the output--docker rm
             # doesn't have an -i equivalent.
             res = process.run("docker rm falco-test", ignore_status=True)
+
             rules_dir = os.path.abspath(os.path.join(self.basedir, "./rules"))
             conf_dir = os.path.abspath(os.path.join(self.basedir, "../"))
             traces_dir = os.path.abspath(os.path.join(self.basedir, "./trace_files"))
-            self.falco_binary_path = "docker run -i -t --name falco-test --privileged " \
-                                     "-v {}:/host/rules -v {}:/host/conf -v {}:/host/traces " \
+            self.falco_binary_path = "docker run --rm --name falco-test --privileged " \
                                      "-v /var/run/docker.sock:/host/var/run/docker.sock " \
                                      "-v /dev:/host/dev -v /proc:/host/proc:ro -v /boot:/host/boot:ro " \
                                      "-v /lib/modules:/host/lib/modules:ro -v {}:/root/.sysdig:ro -v " \
                                      "/usr:/host/usr:ro {} {} falco".format(
-                                         rules_dir, conf_dir, traces_dir,
                                          self.module_dir, self.addl_docker_run_args, image)
 
         elif self.package.endswith(".deb"):
@@ -337,11 +332,31 @@ class FalcoTest(Test):
             self.log.debug("Installing debian package via \"{}\"".format(cmdline))
             res = process.run(cmdline, timeout=120, sudo=True)
 
+        elif self.package.endswith(".rpm"):
+            self.falco_binary_path = '/usr/bin/falco';
+
+            package_glob = "{}/{}".format(self.falcodir, self.package)
+
+            matches = glob.glob(package_glob)
+
+            if len(matches) != 1:
+                self.fail("Package path {} did not match exactly 1 file. Instead it matched: {}", package_glob, ",".join(matches))
+
+            package_path = matches[0]
+
+            cmdline = "rpm -i --nodeps --noscripts {}".format(package_path)
+            self.log.debug("Installing centos package via \"{}\"".format(cmdline))
+            res = process.run(cmdline, timeout=120, sudo=True)
+
     def uninstall_package(self):
 
         if self.package.startswith("docker:"):
-            # Remove the falco-test image. Here we *do* check the return value
-            res = process.run("docker rm falco-test")
+            self.log.debug("Nothing to do, docker run with --rm")
+
+        elif self.package.endswith(".rpm"):
+            cmdline = "rpm -e --noscripts --nodeps falco"
+            self.log.debug("Uninstalling centos package via \"{}\"".format(cmdline))
+            res = process.run(cmdline, timeout=120, sudo=True)
 
         elif self.package.endswith(".deb"):
             cmdline = "dpkg --purge falco"

test/falco_tests.yaml

@@ -17,39 +17,6 @@
 #
 trace_files: !mux
 
-  docker_package:
-    package: docker:falcosecurity/falco:test
-    detect: True
-    detect_level: WARNING
-    rules_file: /host/rules/rule_names_with_spaces.yaml
-    trace_file: /host/traces/cat_write.scap
-    conf_file: /host/conf/falco.yaml
-
-  # This uses a volume mount to overwrite and prevent /usr/sbin/dkms
-  # from being run. As a result, it will force falco-probe-loader to
-  # fall back to loading the driver from ~/.sysdig. Setting
-  # copy_local_driver to True copied the driver to ~/.sysdig, so it
-  # will be available. In this case, we're running live for 5 seconds
-  # just to see if falco can load the driver.
-
-  docker_package_local_driver:
-    package: docker:falcosecurity/falco:test
-    addl_docker_run_args: -v /dev/null:/usr/sbin/dkms
-    copy_local_driver: True
-    detect: False
-    detect_level: WARNING
-    rules_file: /host/rules/tagged_rules.yaml
-    conf_file: /host/conf/falco.yaml
-    run_duration: 5
-
-  debian_package:
-    package: falco*.deb
-    detect: True
-    detect_level: WARNING
-    rules_file:
-      - rules/rule_names_with_spaces.yaml
-    trace_file: trace_files/cat_write.scap
-
   builtin_rules_no_warnings:
     detect: False
     trace_file: trace_files/empty.scap

test/falco_tests_package.yaml

@@ -0,0 +1,34 @@
+#
+# Copyright (C) 2016-2018 Draios Inc dba Sysdig.
+#
+# This file is part of falco.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+trace_files: !mux
+
+  docker_package:
+    package: docker:falcosecurity/falco:test
+    detect: True
+    detect_level: WARNING
+    rules_file: /rules/rule_names_with_spaces.yaml
+    trace_file: /traces/cat_write.scap
+    conf_file: /etc/falco/falco.yaml
+
+  centos_package:
+    package: falco*.rpm
+    detect: True
+    detect_level: WARNING
+    rules_file:
+      - rules/rule_names_with_spaces.yaml
+    trace_file: trace_files/cat_write.scap

test/run_regression_tests.sh

@@ -16,17 +16,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+set -euo pipefail
+
 SCRIPT=$(readlink -f $0)
 SCRIPTDIR=$(dirname $SCRIPT)
-BRANCH=$1
+BUILD_DIR=$1
+BRANCH=${2:-none}
+
+TRACE_DIR=$BUILD_DIR/test
+
+mkdir -p $TRACE_DIR
 
 function download_trace_files() {
     echo "branch=$BRANCH"
     for TRACE in traces-positive traces-negative traces-info ; do
-	rm -rf $SCRIPTDIR/$TRACE
-	curl -fso $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE-$BRANCH.zip || curl -fso $SCRIPTDIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip &&
-	unzip -d $SCRIPTDIR $SCRIPTDIR/$TRACE.zip &&
-	rm -rf $SCRIPTDIR/$TRACE.zip
+	if [ ! -e $TRACE_DIR/$TRACE ]; then
+	    if [ $BRANCH != "none" ]; then
+		curl -fso $TRACE_DIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE-$BRANCH.zip
+	    else
+		curl -fso $TRACE_DIR/$TRACE.zip https://s3.amazonaws.com/download.draios.com/falco-tests/$TRACE.zip
+	    fi
+	    unzip -d $TRACE_DIR $TRACE_DIR/$TRACE.zip
+	    rm -rf $TRACE_DIR/$TRACE.zip
+	fi
     done
 }
 
@@ -35,7 +47,7 @@ function prepare_multiplex_fileset() {
     dir=$1
     detect=$2
 
-    for trace in $SCRIPTDIR/$dir/*.scap ; do
+    for trace in $TRACE_DIR/$dir/*.scap ; do
 	[ -e "$trace" ] || continue
 	NAME=`basename $trace .scap`
 
@@ -73,9 +85,12 @@ function print_test_failure_details() {
 function run_tests() {
     rm -rf /tmp/falco_outputs
     mkdir /tmp/falco_outputs
+    # If we got this far, we can undo set -e, as we're watching the
+    # return status when running avocado.
+    set +e
     TEST_RC=0
-    for mult in $SCRIPTDIR/falco_traces.yaml $SCRIPTDIR/falco_tests.yaml $SCRIPTDIR/falco_k8s_audit_tests.yaml; do
-	CMD="avocado run --multiplex $mult --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
+    for mult in $SCRIPTDIR/falco_traces.yaml $SCRIPTDIR/falco_tests.yaml $SCRIPTDIR/falco_tests_package.yaml $SCRIPTDIR/falco_k8s_audit_tests.yaml; do
+	CMD="avocado run --mux-yaml $mult --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py"
 	echo "Running: $CMD"
 	$CMD
 	RC=$?

test/trace_files/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_subdirectory(k8s_audit)
+# Note: list of traces is created at cmake time, not build time
+file(GLOB test_trace_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/*.scap")
+
+foreach(trace_file_path ${test_trace_files})
+	get_filename_component(trace_file ${trace_file_path} NAME)
+	add_custom_target(test-trace-${trace_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${trace_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		DEPENDS ${trace_file_path})
+endforeach()

test/trace_files/k8s_audit/CMakeLists.txt

@@ -0,0 +1,12 @@
+# Note: list of traces is created at cmake time, not build time
+file(GLOB test_trace_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/*.json")
+
+foreach(trace_file_path ${test_trace_files})
+	get_filename_component(trace_file ${trace_file_path} NAME)
+	add_custom_target(test-trace-${trace_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${trace_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		COMMAND ${CMAKE_COMMAND} -E copy ${trace_file_path} ${CMAKE_CURRENT_BINARY_DIR}/${trace_file}
+		DEPENDS ${trace_file_path})
+endforeach()