diff --git a/test/falco_tests.yaml.in b/test/falco_tests.yaml similarity index 100% rename from test/falco_tests.yaml.in rename to test/falco_tests.yaml diff --git a/test/falco_traces.yaml.in b/test/falco_traces.yaml.in new file mode 100644 index 00000000..2c581e9b --- /dev/null +++ b/test/falco_traces.yaml.in @@ -0,0 +1,203 @@ +has_json_output: !mux + yes: + json_output: True + no: + json_output: False + +traces: !mux + change-thread-namespace: + trace_file: traces-positive/change-thread-namespace.scap + detect: True + detect_level: NOTICE + detect_counts: + - "Change thread namespace": 2 + + container-privileged: + trace_file: traces-positive/container-privileged.scap + detect: True + detect_level: INFO + detect_counts: + - "File Open by Privileged Container": 19 + + container-sensitive-mount: + trace_file: traces-positive/container-sensitive-mount.scap + detect: True + detect_level: INFO + detect_counts: + - "Sensitive Mount by Container": 19 + + create-files-below-dev: + trace_file: traces-positive/create-files-below-dev.scap + detect: True + detect_level: ERROR + detect_counts: + - "Create files below dev": 1 + + db-program-spawned-process: + trace_file: traces-positive/db-program-spawned-process.scap + detect: True + detect_level: NOTICE + detect_counts: + - "DB program spawned process": 1 + + falco-event-generator: + trace_file: traces-positive/falco-event-generator.scap + detect: True + detect_level: [ERROR, WARNING, INFO, NOTICE] + detect_counts: + - "Write below binary dir": 1 + - "Read sensitive file untrusted": 3 + - "Run shell in container": 1 + - "Write below rpm database": 1 + - "Write below etc": 1 + - "System procs network activity": 1 + - "Mkdir binary dirs": 1 + - "System user interactive": 1 + - "DB program spawned process": 1 + - "Non sudo setuid": 1 + - "Create files below dev": 1 + - "Modify binary dirs": 2 + - "Change thread namespace": 2 + + installer-fbash-manages-service: + trace_file: traces-info/installer-fbash-manages-service.scap + detect: True + detect_level: INFO + detect_counts: + - "Installer bash manages service": 4 + + installer-bash-non-https-connection: + trace_file: traces-positive/installer-bash-non-https-connection.scap + detect: True + detect_level: NOTICE + detect_counts: + - "Installer bash non https connection": 1 + + installer-fbash-runs-pkgmgmt: + trace_file: traces-info/installer-fbash-runs-pkgmgmt.scap + detect: True + detect_level: [NOTICE, INFO] + detect_counts: + - "Installer bash runs pkgmgmt program": 4 + - "Installer bash non https connection": 4 + + installer-bash-starts-network-server: + trace_file: traces-positive/installer-bash-starts-network-server.scap + detect: True + detect_level: NOTICE + detect_counts: + - "Installer bash starts network server": 2 + - "Installer bash non https connection": 3 + + installer-bash-starts-session: + trace_file: traces-positive/installer-bash-starts-session.scap + detect: True + detect_level: NOTICE + detect_counts: + - "Installer bash starts session": 1 + - "Installer bash non https connection": 3 + + mkdir-binary-dirs: + trace_file: traces-positive/mkdir-binary-dirs.scap + detect: True + detect_level: ERROR + detect_counts: + - "Mkdir binary dirs": 1 + + modify-binary-dirs: + trace_file: traces-positive/modify-binary-dirs.scap + detect: True + detect_level: ERROR + detect_counts: + - "Modify binary dirs": 1 + + modify-package-repo-list-installer: + trace_file: traces-info/modify-package-repo-list-installer.scap + detect: True + detect_level: INFO + detect_counts: + - "Write below etc in installer": 1 + + non-sudo-setuid: + trace_file: traces-positive/non-sudo-setuid.scap + detect: True + detect_level: NOTICE + detect_counts: + - "Non sudo setuid": 1 + + read-sensitive-file-after-startup: + trace_file: traces-positive/read-sensitive-file-after-startup.scap + detect: True + detect_level: WARNING + detect_counts: + - "Read sensitive file untrusted": 1 + + read-sensitive-file-untrusted: + trace_file: traces-positive/read-sensitive-file-untrusted.scap + detect: True + detect_level: WARNING + detect_counts: + - "Read sensitive file untrusted": 1 + + run-shell-untrusted: + trace_file: traces-positive/run-shell-untrusted.scap + detect: True + detect_level: DEBUG + detect_counts: + - "Run shell untrusted": 1 + + shell-in-container: + trace_file: traces-positive/shell-in-container.scap + detect: True + detect_level: NOTICE + detect_counts: + - "Run shell in container": 1 + + system-binaries-network-activity: + trace_file: traces-positive/system-binaries-network-activity.scap + detect: True + detect_level: NOTICE + detect_counts: + - "System procs network activity": 1 + + system-user-interactive: + trace_file: traces-positive/system-user-interactive.scap + detect: True + detect_level: INFO + detect_counts: + - "System user interactive": 1 + + user-mgmt-binaries: + trace_file: traces-positive/user-mgmt-binaries.scap + detect: True + detect_level: NOTICE + detect_counts: + - "User mgmt binaries": 1 + + write-binary-dir: + trace_file: traces-positive/write-binary-dir.scap + detect: True + detect_level: ERROR + detect_counts: + - "Write below binary dir": 4 + + write-etc: + trace_file: traces-positive/write-etc.scap + detect: True + detect_level: ERROR + detect_counts: + - "Write below etc": 1 + + write-etc-installer: + trace_file: traces-info/write-etc-installer.scap + detect: True + detect_level: INFO + detect_counts: + - "Write below etc in installer": 1 + + write-rpm-database: + trace_file: traces-positive/write-rpm-database.scap + detect: True + detect_level: ERROR + detect_counts: + - "Write below rpm database": 1 diff --git a/test/run_regression_tests.sh b/test/run_regression_tests.sh index 2b707fe5..006397b7 100755 --- a/test/run_regression_tests.sh +++ b/test/run_regression_tests.sh @@ -2,7 +2,6 @@ SCRIPT=$(readlink -f $0) SCRIPTDIR=$(dirname $SCRIPT) -MULT_FILE=$SCRIPTDIR/falco_tests.yaml BRANCH=$1 function download_trace_files() { @@ -19,56 +18,59 @@ function prepare_multiplex_fileset() { dir=$1 detect=$2 - detect_level=$3 - json_output=$4 for trace in $SCRIPTDIR/$dir/*.scap ; do [ -e "$trace" ] || continue NAME=`basename $trace .scap` - cat << EOF >> $MULT_FILE - $NAME-detect-$detect-json-$json_output: + + # falco_traces.yaml might already have an entry for this trace + # file, with specific detection levels and counts. If so, skip + # it. Otherwise, add a generic entry showing whether or not to + # detect anything. + grep -q "$NAME:" $SCRIPTDIR/falco_traces.yaml && continue + + cat << EOF >> $SCRIPTDIR/falco_traces.yaml + $NAME: detect: $detect - detect_level: $detect_level + detect_level: WARNING trace_file: $trace - json_output: $json_output EOF done } function prepare_multiplex_file() { - cp $SCRIPTDIR/falco_tests.yaml.in $MULT_FILE + cp $SCRIPTDIR/falco_traces.yaml.in $SCRIPTDIR/falco_traces.yaml - prepare_multiplex_fileset traces-positive True WARNING False - prepare_multiplex_fileset traces-negative False WARNING True - prepare_multiplex_fileset traces-info True INFO False + prepare_multiplex_fileset traces-positive True + prepare_multiplex_fileset traces-negative False + prepare_multiplex_fileset traces-info True - prepare_multiplex_fileset traces-positive True WARNING True - prepare_multiplex_fileset traces-info True INFO True - - echo "Contents of $MULT_FILE:" - cat $MULT_FILE + echo "Contents of $SCRIPTDIR/falco_traces.yaml:" + cat $SCRIPTDIR/falco_traces.yaml } -function run_tests() { - rm -rf /tmp/falco_outputs - mkdir /tmp/falco_outputs - CMD="avocado run --multiplex $MULT_FILE --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py" - echo "Running: $CMD" - $CMD - TEST_RC=$? -} - - function print_test_failure_details() { echo "Showing full job logs for any tests that failed:" jq '.tests[] | select(.status != "PASS") | .logfile' $SCRIPTDIR/job-results/latest/results.json | xargs cat } +function run_tests() { + rm -rf /tmp/falco_outputs + mkdir /tmp/falco_outputs + TEST_RC=0 + for mult in $SCRIPTDIR/falco_traces.yaml $SCRIPTDIR/falco_tests.yaml; do + CMD="avocado run --multiplex $mult --job-results-dir $SCRIPTDIR/job-results -- $SCRIPTDIR/falco_test.py" + echo "Running: $CMD" + $CMD + RC=$? + TEST_RC=$((TEST_RC+$RC)) + if [ $RC -ne 0 ]; then + print_test_failure_details + fi + done +} + download_trace_files prepare_multiplex_file run_tests -if [ $TEST_RC -ne 0 ]; then - print_test_failure_details -fi - exit $TEST_RC