diff --git a/test/cpu_monitor.sh b/test/cpu_monitor.sh deleted file mode 100644 index ead977fc..00000000 --- a/test/cpu_monitor.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2019 The Falco Authors. -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -SUBJ_PID=$1 -BENCHMARK=$2 -VARIANT=$3 -RESULTS_FILE=$4 -CPU_INTERVAL=$5 - -top -d $CPU_INTERVAL -b -p $SUBJ_PID | grep -E '(falco|sysdig|dragent|test_mm)' --line-buffered | awk -v benchmark=$BENCHMARK -v variant=$VARIANT '{printf("{\"time\": \"%s\", \"sample\": %d, \"benchmark\": \"%s\", \"variant\": \"%s\", \"cpu_usage\": %s},\n", strftime("%Y-%m-%d %H:%M:%S", systime(), 1), NR, benchmark, variant, $9); fflush();}' >> $RESULTS_FILE diff --git a/test/plot-live.r b/test/plot-live.r deleted file mode 100644 index faa909db..00000000 --- a/test/plot-live.r +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright (C) 2019 The Falco Authors. -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require(jsonlite) -library(ggplot2) -library(GetoptLong) - -initial.options <- commandArgs(trailingOnly = FALSE) -file.arg.name <- "--file=" -script.name <- sub(file.arg.name, "", initial.options[grep(file.arg.name, initial.options)]) -script.basename <- dirname(script.name) - -if (substr(script.basename, 1, 1) != '/') { - script.basename = paste(getwd(), script.basename, sep='/') -} - -results = paste(script.basename, "results.json", sep='/') -output = "./output.png" -metric = "cpu" - -GetoptLong( - "results=s", "Path to results file", - "benchmark=s", "Benchmark from results file to graph", - "variant=s@", "Variant(s) to include in graph. Can be specified multiple times", - "output=s", "Output graph file", - "metric=s", "Metric to graph. Can be one of (cpu|drops)" -) - -if (metric == "cpu") { - data_metric="cpu_usage" - yaxis_label="CPU Usage (%)" - title="Falco/Sysdig/Multimatch CPU Usage: %s" -} else if (metric == "drops") { - data_metric="drop_pct" - yaxis_label="Event Drops (%)" - title="Falco/Sysdig/Multimatch Event Drops: %s" -} - -res <- fromJSON(results, flatten=TRUE) - -res2 = res[res$benchmark == benchmark & res$variant %in% variant,] - -plot <- ggplot(data=res2, aes(x=sample, y=get(data_metric), group=variant, colour=variant)) + - geom_line() + - ylab(yaxis_label) + - xlab("Time") + - ggtitle(sprintf(title, benchmark)) - theme(legend.position=c(.2, .88)); - -print(paste("Writing graph to", output, sep=" ")) -ggsave(file=output) - - - - diff --git a/test/plot-traces.r b/test/plot-traces.r deleted file mode 100644 index 03d5188c..00000000 --- a/test/plot-traces.r +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright (C) 2019 The Falco Authors. -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -require(jsonlite) -library(ggplot2) -library(reshape) - -res <- fromJSON("/home/mstemm/results.txt", flatten=TRUE) - -plot <- ggplot(data=res, aes(x=config, y=elapsed.real)) + - geom_bar(stat = "summary", fun.y = "mean") + - coord_flip() + - facet_grid(shortfile ~ .) + - ylab("Wall Clock Time (sec)") + - xlab("Trace File/Program") - - -ggsave(file="/mnt/sf_mstemm/res-real.png") - -plot <- ggplot(data=res, aes(x=config, y=elapsed.user)) + - geom_bar(stat = "summary", fun.y = "mean") + - coord_flip() + - facet_grid(shortfile ~ .) + - ylab("User Time (sec)") + - xlab("Trace File/Program") - - -ggsave(file="/mnt/sf_mstemm/res-user.png") - -res2 <- melt(res, id.vars = c("config", "shortfile"), measure.vars = c("elapsed.sys", "elapsed.user")) -plot <- ggplot(data=res2, aes(x=config, y=value, fill=variable, order=variable)) + - geom_bar(stat = "summary", fun.y = "mean") + - coord_flip() + - facet_grid(shortfile ~ .) + - ylab("User/System Time (sec)") + - xlab("Trace File/Program") - -ggsave(file="/mnt/sf_mstemm/res-sys-user.png") diff --git a/test/run_performance_tests.sh b/test/run_performance_tests.sh deleted file mode 100755 index a1e1d6c4..00000000 --- a/test/run_performance_tests.sh +++ /dev/null @@ -1,444 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (C) 2019 The Falco Authors. -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#set -x - -trap "cleanup; exit" SIGHUP SIGINT SIGTERM - -TRACE_FILES_BASE_URL=${TRACE_FILES_BASE_URL:-"https://download.falco.org/fixtures/trace-files/"} - -function download_trace_files() { - - (mkdir -p $TRACEDIR && rm -rf $TRACEDIR/traces-perf && curl -fo $TRACEDIR/traces-perf.zip "${TRACE_FILES_BASE_URL}traces-perf.zip" && unzip -d $TRACEDIR $TRACEDIR/traces-perf.zip && rm -f $TRACEDIR/traces-perf.zip) || exit 1 - -} - -function time_cmd() { - cmd="$1" - file="$2" - - benchmark=`basename $file .scap` - - echo -n "$benchmark: " - for i in `seq 1 5`; do - echo -n "$i " - time=`date --iso-8601=sec` - /usr/bin/time -a -o $RESULTS_FILE --format "{\"time\": \"$time\", \"benchmark\": \"$benchmark\", \"file\": \"$file\", \"variant\": \"$VARIANT\", \"elapsed\": {\"real\": %e, \"user\": %U, \"sys\": %S}}," $cmd >> $OUTPUT_FILE 2>&1 - done - echo "" -} - -function run_falco_on() { - file="$1" - - if [ -z $RULES_FILE ]; then - RULES_FILE=$SOURCE/rules/falco_rules.yaml - fi - - cmd="$ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -r $SOURCE/rules/falco_rules.yaml --option=stdout_output.enabled=false -e $file -A" - - time_cmd "$cmd" "$file" -} - -function run_sysdig_on() { - file="$1" - - cmd="$ROOT/userspace/sysdig/sysdig -N -z -r $file evt.type=none" - - time_cmd "$cmd" "$file" -} - -function write_agent_config() { - cat > $ROOT/userspace/dragent/dragent.yaml <> $ROOT/userspace/dragent/dragent.yaml <> $ROOT/userspace/dragent/dragent.yaml <> $ROOT/userspace/dragent/dragent.yaml <> $ROOT/userspace/dragent/dragent.yaml < ./prog-output.txt 2>&1 & - elif [[ $ROOT == *"falco"* ]]; then - echo " starting falco..." - if [ -z $RULES_FILE ]; then - RULES_FILE=$SOURCE/rules/falco_rules.yaml - fi - sudo FALCO_STATS_EXTRA_variant=$VARIANT FALCO_STATS_EXTRA_benchmark=$live_test $ROOT/userspace/falco/falco -c $SOURCE/falco.yaml -s $STATS_FILE -r $RULES_FILE --option=stdout_output.enabled=false > ./prog-output.txt -A 2>&1 & - elif [[ $ROOT == *"sysdig"* ]]; then - echo " starting sysdig..." - sudo $ROOT/userspace/sysdig/sysdig -N -z evt.type=none & - else - echo " starting agent..." - write_agent_config - pushd $ROOT/userspace/dragent - sudo ./dragent > ./prog-output.txt 2>&1 & - popd - fi - - SUDO_PID=$! - sleep 5 - if [[ $ROOT == *"agent"* ]]; then - # The agent spawns several processes all below a main monitor - # process. We want the child with the lowest pid. - MON_PID=`ps -h -o pid --ppid $SUDO_PID` - SUBJ_PID=`ps -h -o pid --ppid $MON_PID | head -1` - else - SUBJ_PID=`ps -h -o pid --ppid $SUDO_PID` - fi - - if [ -z $SUBJ_PID ]; then - echo "Could not find pid of subject program--did it start successfully? Not continuing." - exit 1 - fi -} - -function run_htop() { - screen -S htop-screen -d -m /usr/bin/htop -d2 - sleep 90 - screen -X -S htop-screen quit -} - -function run_juttle_examples() { - pushd $SCRIPTDIR/../../juttle-engine/examples - docker-compose -f dc-juttle-engine.yml -f aws-cloudwatch/dc-aws-cloudwatch.yml -f elastic-newstracker/dc-elastic.yml -f github-tutorial/dc-elastic.yml -f nginx_logs/dc-nginx-logs.yml -f postgres-diskstats/dc-postgres.yml -f cadvisor-influx/dc-cadvisor-influx.yml up -d - sleep 120 - docker-compose -f dc-juttle-engine.yml -f aws-cloudwatch/dc-aws-cloudwatch.yml -f elastic-newstracker/dc-elastic.yml -f github-tutorial/dc-elastic.yml -f nginx_logs/dc-nginx-logs.yml -f postgres-diskstats/dc-postgres.yml -f cadvisor-influx/dc-cadvisor-influx.yml stop - docker-compose -f dc-juttle-engine.yml -f aws-cloudwatch/dc-aws-cloudwatch.yml -f elastic-newstracker/dc-elastic.yml -f github-tutorial/dc-elastic.yml -f nginx_logs/dc-nginx-logs.yml -f postgres-diskstats/dc-postgres.yml -f cadvisor-influx/dc-cadvisor-influx.yml rm -fv - popd -} - -function run_kubernetes_demo() { - pushd $SCRIPTDIR/../../infrastructure/test-infrastructures/kubernetes-demo - sudo bash run-local.sh - sudo bash init.sh - sleep 600 - docker stop $(docker ps -qa) - docker rm -fv $(docker ps -qa) - popd -} - -function run_live_test() { - - live_test="$1" - - echo "Running live test $live_test" - - case "$live_test" in - htop ) CPU_INTERVAL=2;; - * ) CPU_INTERVAL=10;; - esac - - start_subject_prog - start_monitor_cpu_usage - - echo " starting live program and waiting for it to finish" - case "$live_test" in - htop ) run_htop ;; - juttle-examples ) run_juttle_examples ;; - kube-demo ) run_kubernetes_demo ;; - * ) usage; cleanup; exit 1 ;; - esac - - cleanup - -} - -function cleanup() { - - if [ -n "$SUBJ_PID" ] ; then - echo " stopping falco/sysdig program $SUBJ_PID" - sudo kill $SUBJ_PID - fi - - if [ -n "$CPU_PID" ] ; then - echo " stopping cpu monitor program $CPU_PID" - kill -- -$CPU_PID - fi -} - -run_live_tests() { - test="$1" - - if [ $test == "all" ]; then - tests="htop juttle-examples kube-demo" - else - tests=$test - fi - - for test in $tests; do - run_live_test $test - done -} - -function run_phoronix_test() { - - live_test="$1" - - case "$live_test" in - pts/aio-stress | pts/fs-mark | pts/iozone | pts/network-loopback | pts/nginx | pts/pybench | pts/redis | pts/sqlite | pts/unpack-linux ) CPU_INTERVAL=2;; - * ) CPU_INTERVAL=10;; - esac - - echo "Running phoronix test $live_test" - - start_subject_prog - start_monitor_cpu_usage - - echo " starting phoronix test and waiting for it to finish" - - TEST_RESULTS_NAME=$VARIANT FORCE_TIMES_TO_RUN=1 phoronix-test-suite default-run $live_test - - cleanup - -} - -# To install and configure phoronix: -# (redhat instructions, adapt as necessary for ubuntu or other distros) -# - install phoronix: yum install phoronix-test-suite.noarch -# - install dependencies not handled by phoronix: yum install libaio-devel pcre-devel popt-devel glibc-static zlib-devel nc bc -# - fix trivial bugs in tests: -# - edit ~/.phoronix-test-suite/installed-tests/pts/network-loopback-1.0.1/network-loopback line "nc -d -l 9999 > /dev/null &" to "nc -d -l 9999 > /dev/null &" -# - edit ~/.phoronix-test-suite/test-profiles/pts/nginx-1.1.0/test-definition.xml line "-n 500000 -c 100 http://localhost:8088/test.html" to "-n 500000 -c 100 http://127.0.0.1:8088/test.html" -# - phoronix batch-install - -function run_phoronix_tests() { - - test="$1" - - if [ $test == "all" ]; then - tests="pts/aio-stress pts/apache pts/blogbench pts/compilebench pts/dbench pts/fio pts/fs-mark pts/iozone pts/network-loopback pts/nginx pts/pgbench pts/phpbench pts/postmark pts/pybench pts/redis pts/sqlite pts/unpack-linux" - else - tests=$test - fi - - for test in $tests; do - run_phoronix_test $test - done -} - -run_tests() { - - IFS=':' read -ra PARTS <<< "$TEST" - - case "${PARTS[0]}" in - trace ) run_trace "${PARTS[1]}" ;; - live ) run_live_tests "${PARTS[1]}" ;; - phoronix ) run_phoronix_tests "${PARTS[1]}" ;; - * ) usage; exit 1 ;; - esac -} - -usage() { - echo "Usage: $0 [options]" - echo "" - echo "Options:" - echo " -h/--help: show this help" - echo " -v/--variant: a variant name to attach to this set of test results" - echo " -r/--root: root directory containing falco/sysdig binaries (i.e. where you ran 'cmake')" - echo " -s/--source: root directory containing falco/sysdig source code" - echo " -R/--results: append test results to this file" - echo " -S/--stats: append capture statistics to this file (only works for falco/test_mm)" - echo " -o/--output: append program output to this file" - echo " -U/--rules: path to rules file (only applicable for falco/test_mm)" - echo " -t/--test: test to run. Argument has the following format:" - echo " trace:: read the specified trace file." - echo " trace:all means run all traces" - echo " live:: run the specified live test." - echo " live:all means run all live tests." - echo " possible live tests:" - echo " live:htop: run htop -d2" - echo " live:kube-demo: run kubernetes demo from infrastructure repo" - echo " live:juttle-examples: run a juttle demo environment based on docker-compose" - echo " phoronix:: run the specified phoronix test." - echo " if is not 'all', it is passed directly to the command line of \"phoronix-test-suite run \"" - echo " if is 'all', a built-in set of phoronix tests will be chosen and run" - echo " -T/--tracedir: Look for trace files in this directory. If doesn't exist, will download trace files from s3" - echo " -A/--agent-autodrop: When running an agent, whether or not to enable autodrop" - echo " -F/--falco-agent: When running an agent, whether or not to enable falco" -} - -OPTS=`getopt -o hv:r:s:R:S:o:U:t:T: --long help,variant:,root:,source:,results:,stats:,output:,rules:,test:,tracedir:,agent-autodrop:,falco-agent: -n $0 -- "$@"` - -if [ $? != 0 ]; then - echo "Exiting" >&2 - exit 1 -fi - -eval set -- "$OPTS" - -VARIANT="falco" -ROOT=`dirname $0`/../build -SOURCE=$ROOT -SCRIPTDIR=`dirname $0` -RESULTS_FILE=`dirname $0`/results.json -STATS_FILE=`dirname $0`/capture_stats.json -OUTPUT_FILE=`dirname $0`/program-output.txt -RULES_FILE= -TEST=trace:all -TRACEDIR=/tmp/falco-perf-traces.$USER -CPU_INTERVAL=10 -AGENT_AUTODROP=1 -FALCO_AGENT=1 - -while true; do - case "$1" in - -h | --help ) usage; exit 1;; - -v | --variant ) VARIANT="$2"; shift 2;; - -r | --root ) ROOT="$2"; shift 2;; - -s | --source ) SOURCE="$2"; shift 2;; - -R | --results ) RESULTS_FILE="$2"; shift 2;; - -S | --stats ) STATS_FILE="$2"; shift 2;; - -o | --output ) OUTPUT_FILE="$2"; shift 2;; - -U | --rules ) RULES_FILE="$2"; shift 2;; - -t | --test ) TEST="$2"; shift 2;; - -T | --tracedir ) TRACEDIR="$2"; shift 2;; - -A | --agent-autodrop ) AGENT_AUTODROP="$2"; shift 2;; - -F | --falco-agent ) FALCO_AGENT="$2"; shift 2;; - * ) break;; - esac -done - -if [ -z $VARIANT ]; then - echo "A test variant name must be provided. Not continuing." - exit 1 -fi - -if [ -z $ROOT ]; then - echo "A root directory containing a falco/sysdig binary must be provided. Not continuing." - exit 1 -fi - -ROOT=`realpath $ROOT` - -if [ -z $SOURCE ]; then - echo "A source directory containing falco/sysdig source code. Not continuing." - exit 1 -fi - -SOURCE=`realpath $SOURCE` - -if [ -z $RESULTS_FILE ]; then - echo "An output file for test results must be provided. Not continuing." - exit 1 -fi - -if [ -z $STATS_FILE ]; then - echo "An output file for capture statistics must be provided. Not continuing." - exit 1 -fi - -if [ -z $OUTPUT_FILE ]; then - echo "An file for program output must be provided. Not continuing." - exit 1 -fi - -if [ -z $TEST ]; then - echo "A test must be provided. Not continuing." - exit 1 -fi - -run_tests