diff --git a/benching/tools/errors.sh b/benching/tools/errors.sh index 09f636d..4e5407e 100755 --- a/benching/tools/errors.sh +++ b/benching/tools/errors.sh @@ -3,11 +3,14 @@ source "$(dirname $0)/common.sh" TMP_FILE=$(mktemp) +if [ -z "$EH_ELFS_NAME" ]; then + EH_ELFS_NAME="eh_elfs" +fi function get_perf_output { envtype=$1 source $ENV_APPLY "$envtype" "dbg" - LD_LIBRARY_PATH="$BENCH_DIR/eh_elfs:$LD_LIBRARY_PATH" \ + LD_LIBRARY_PATH="$BENCH_DIR/$EH_ELFS_NAME:$LD_LIBRARY_PATH" \ UNW_DEBUG_LEVEL=15 \ perf report -i "$BENCH_DIR/perf.data" 2>$TMP_FILE >/dev/null deactivate diff --git a/benching/tools/gen_perf_stats.py b/benching/tools/gen_perf_stats.py index 34f42da..76dd441 100644 --- a/benching/tools/gen_perf_stats.py +++ b/benching/tools/gen_perf_stats.py @@ -3,29 +3,23 @@ """ Generates performance statistics for the eh_elf vs vanilla libunwind unwinding, based on time series generated beforehand -First run -```bash -for i in $(seq 1 100); do - perf report 2>&1 >/dev/null | tail -n 1 \ - | python ../hackbench/to_report_fmt.py \ - | sed 's/^.* & .* & \([0-9]*\) & .*$/\1/g' -done > $SOME_PLACE/$FLAVOUR_times -``` - -for each flavour (eh_elf, vanilla) - -Then run this script, with `$SOME_PLACE` as argument. +Intended to be run from `statistics.sh` """ +from collections import namedtuple import numpy as np import sys import os +Datapoint = namedtuple("Datapoint", ["nb_frames", "total_time", "avg_time"]) + + def read_series(path): with open(path, "r") as handle: for line in handle: - yield int(line.strip()) + nb_frames, total_time, avg_time = map(int, line.strip().split()) + yield Datapoint(nb_frames, total_time, avg_time) FLAVOURS = ["eh_elf", "vanilla"] @@ -36,14 +30,32 @@ if "WITH_NOCACHE" in os.environ: FLAVOURS.append("vanilla-nocache") path_format = os.path.join(sys.argv[1], "{}_times") -times = {} +datapoints = {} +avg_times = {} +total_times = {} +avgs_total = {} avgs = {} std_deviations = {} +unwound_frames = {} for flv in FLAVOURS: - times[flv] = list(read_series(path_format.format(flv))) - avgs[flv] = sum(times[flv]) / len(times[flv]) - std_deviations[flv] = np.sqrt(np.var(times[flv])) + datapoints[flv] = list(read_series(path_format.format(flv))) + avg_times[flv] = list(map(lambda x: x.avg_time, datapoints[flv])) + total_times[flv] = list(map(lambda x: x.total_time, datapoints[flv])) + avgs[flv] = sum(avg_times[flv]) / len(avg_times[flv]) + avgs_total[flv] = sum(total_times[flv]) / len(total_times[flv]) + std_deviations[flv] = np.sqrt(np.var(avg_times[flv])) + + cur_unwound_frames = list(map(lambda x: x.nb_frames, datapoints[flv])) + unwound_frames[flv] = cur_unwound_frames[0] + for run_id, unw_frames in enumerate(cur_unwound_frames[1:]): + if unw_frames != unwound_frames[flv]: + print( + "{}, run {}: unwound {} frames, reference unwound {}".format( + flv, run_id + 1, unw_frames, unwound_frames[flv] + ), + file=sys.stderr, + ) avg_ratio = avgs["vanilla"] / avgs["eh_elf"] ratio_uncertainty = ( @@ -56,11 +68,12 @@ ratio_uncertainty = ( ) -def format_flv(flv_dict, formatter): +def format_flv(flv_dict, formatter, alterator=None): out = "" for flv in FLAVOURS: val = flv_dict[flv] - out += "* {}: {}\n".format(flv, formatter.format(val)) + altered = alterator(val) if alterator else val + out += "* {}: {}\n".format(flv, formatter.format(altered)) return out @@ -77,10 +90,14 @@ def get_ratios(avgs): print( - "Average time:\n{}\n" + "Unwound frames:\n{}\n" + "Average whole unwinding time (one run):\n{}\n" + "Average time to unwind one frame:\n{}\n" "Standard deviation:\n{}\n" "Average ratio: {}\n" "Ratio uncertainty: {}".format( + format_flv(unwound_frames, "{}"), + format_flv(avgs_total, "{} μs", alterator=lambda x: x // 1000), format_flv(avgs, "{} ns"), format_flv(std_deviations, "{}"), get_ratios(avgs), diff --git a/benching/tools/statistics.sh b/benching/tools/statistics.sh index 67fbdc0..6e50292 100755 --- a/benching/tools/statistics.sh +++ b/benching/tools/statistics.sh @@ -12,7 +12,7 @@ function collect_perf_time_data { perf report -i "$BENCH_DIR/perf.data" 2>&1 >/dev/null \ | tail -n 1 \ | python "$(dirname $0)/to_report_fmt.py" \ - | sed 's/^.* & .* & \([0-9]*\) & .*$/\1/g' + | sed 's/^\([0-9]*\) & \([0-9]*\) & \([0-9]*\) & .*$/\1 \2 \3/g' deactivate } @@ -32,6 +32,11 @@ vanilla_data="$TEMP_DIR/vanilla_times" collect_perf_time_data_runs "eh_elf" "$eh_elf_data" collect_perf_time_data_runs "vanilla" "$vanilla_data" +if [ -n "$WITH_NOCACHE" ]; then + vanilla_nocache_data="$TEMP_DIR/vanilla-nocache_times" + collect_perf_time_data_runs "vanilla-nocache" "$vanilla_nocache_data" +fi + status_report "benchmark statistics" python "$(dirname "$0")/gen_perf_stats.py" "$TEMP_DIR"