Enhance statistics generation

This commit is contained in:
Théophile Bastian 2019-07-15 21:34:50 +02:00
parent 2e449a9822
commit 4846775529
3 changed files with 47 additions and 22 deletions

View file

@ -3,11 +3,14 @@
source "$(dirname $0)/common.sh" source "$(dirname $0)/common.sh"
TMP_FILE=$(mktemp) TMP_FILE=$(mktemp)
if [ -z "$EH_ELFS_NAME" ]; then
EH_ELFS_NAME="eh_elfs"
fi
function get_perf_output { function get_perf_output {
envtype=$1 envtype=$1
source $ENV_APPLY "$envtype" "dbg" source $ENV_APPLY "$envtype" "dbg"
LD_LIBRARY_PATH="$BENCH_DIR/eh_elfs:$LD_LIBRARY_PATH" \ LD_LIBRARY_PATH="$BENCH_DIR/$EH_ELFS_NAME:$LD_LIBRARY_PATH" \
UNW_DEBUG_LEVEL=15 \ UNW_DEBUG_LEVEL=15 \
perf report -i "$BENCH_DIR/perf.data" 2>$TMP_FILE >/dev/null perf report -i "$BENCH_DIR/perf.data" 2>$TMP_FILE >/dev/null
deactivate deactivate

View file

@ -3,29 +3,23 @@
""" Generates performance statistics for the eh_elf vs vanilla libunwind unwinding, """ Generates performance statistics for the eh_elf vs vanilla libunwind unwinding,
based on time series generated beforehand based on time series generated beforehand
First run Intended to be run from `statistics.sh`
```bash
for i in $(seq 1 100); do
perf report 2>&1 >/dev/null | tail -n 1 \
| python ../hackbench/to_report_fmt.py \
| sed 's/^.* & .* & \([0-9]*\) & .*$/\1/g'
done > $SOME_PLACE/$FLAVOUR_times
```
for each flavour (eh_elf, vanilla)
Then run this script, with `$SOME_PLACE` as argument.
""" """
from collections import namedtuple
import numpy as np import numpy as np
import sys import sys
import os import os
Datapoint = namedtuple("Datapoint", ["nb_frames", "total_time", "avg_time"])
def read_series(path): def read_series(path):
with open(path, "r") as handle: with open(path, "r") as handle:
for line in handle: for line in handle:
yield int(line.strip()) nb_frames, total_time, avg_time = map(int, line.strip().split())
yield Datapoint(nb_frames, total_time, avg_time)
FLAVOURS = ["eh_elf", "vanilla"] FLAVOURS = ["eh_elf", "vanilla"]
@ -36,14 +30,32 @@ if "WITH_NOCACHE" in os.environ:
FLAVOURS.append("vanilla-nocache") FLAVOURS.append("vanilla-nocache")
path_format = os.path.join(sys.argv[1], "{}_times") path_format = os.path.join(sys.argv[1], "{}_times")
times = {} datapoints = {}
avg_times = {}
total_times = {}
avgs_total = {}
avgs = {} avgs = {}
std_deviations = {} std_deviations = {}
unwound_frames = {}
for flv in FLAVOURS: for flv in FLAVOURS:
times[flv] = list(read_series(path_format.format(flv))) datapoints[flv] = list(read_series(path_format.format(flv)))
avgs[flv] = sum(times[flv]) / len(times[flv]) avg_times[flv] = list(map(lambda x: x.avg_time, datapoints[flv]))
std_deviations[flv] = np.sqrt(np.var(times[flv])) total_times[flv] = list(map(lambda x: x.total_time, datapoints[flv]))
avgs[flv] = sum(avg_times[flv]) / len(avg_times[flv])
avgs_total[flv] = sum(total_times[flv]) / len(total_times[flv])
std_deviations[flv] = np.sqrt(np.var(avg_times[flv]))
cur_unwound_frames = list(map(lambda x: x.nb_frames, datapoints[flv]))
unwound_frames[flv] = cur_unwound_frames[0]
for run_id, unw_frames in enumerate(cur_unwound_frames[1:]):
if unw_frames != unwound_frames[flv]:
print(
"{}, run {}: unwound {} frames, reference unwound {}".format(
flv, run_id + 1, unw_frames, unwound_frames[flv]
),
file=sys.stderr,
)
avg_ratio = avgs["vanilla"] / avgs["eh_elf"] avg_ratio = avgs["vanilla"] / avgs["eh_elf"]
ratio_uncertainty = ( ratio_uncertainty = (
@ -56,11 +68,12 @@ ratio_uncertainty = (
) )
def format_flv(flv_dict, formatter): def format_flv(flv_dict, formatter, alterator=None):
out = "" out = ""
for flv in FLAVOURS: for flv in FLAVOURS:
val = flv_dict[flv] val = flv_dict[flv]
out += "* {}: {}\n".format(flv, formatter.format(val)) altered = alterator(val) if alterator else val
out += "* {}: {}\n".format(flv, formatter.format(altered))
return out return out
@ -77,10 +90,14 @@ def get_ratios(avgs):
print( print(
"Average time:\n{}\n" "Unwound frames:\n{}\n"
"Average whole unwinding time (one run):\n{}\n"
"Average time to unwind one frame:\n{}\n"
"Standard deviation:\n{}\n" "Standard deviation:\n{}\n"
"Average ratio: {}\n" "Average ratio: {}\n"
"Ratio uncertainty: {}".format( "Ratio uncertainty: {}".format(
format_flv(unwound_frames, "{}"),
format_flv(avgs_total, "{} μs", alterator=lambda x: x // 1000),
format_flv(avgs, "{} ns"), format_flv(avgs, "{} ns"),
format_flv(std_deviations, "{}"), format_flv(std_deviations, "{}"),
get_ratios(avgs), get_ratios(avgs),

View file

@ -12,7 +12,7 @@ function collect_perf_time_data {
perf report -i "$BENCH_DIR/perf.data" 2>&1 >/dev/null \ perf report -i "$BENCH_DIR/perf.data" 2>&1 >/dev/null \
| tail -n 1 \ | tail -n 1 \
| python "$(dirname $0)/to_report_fmt.py" \ | python "$(dirname $0)/to_report_fmt.py" \
| sed 's/^.* & .* & \([0-9]*\) & .*$/\1/g' | sed 's/^\([0-9]*\) & \([0-9]*\) & \([0-9]*\) & .*$/\1 \2 \3/g'
deactivate deactivate
} }
@ -32,6 +32,11 @@ vanilla_data="$TEMP_DIR/vanilla_times"
collect_perf_time_data_runs "eh_elf" "$eh_elf_data" collect_perf_time_data_runs "eh_elf" "$eh_elf_data"
collect_perf_time_data_runs "vanilla" "$vanilla_data" collect_perf_time_data_runs "vanilla" "$vanilla_data"
if [ -n "$WITH_NOCACHE" ]; then
vanilla_nocache_data="$TEMP_DIR/vanilla-nocache_times"
collect_perf_time_data_runs "vanilla-nocache" "$vanilla_nocache_data"
fi
status_report "benchmark statistics" status_report "benchmark statistics"
python "$(dirname "$0")/gen_perf_stats.py" "$TEMP_DIR" python "$(dirname "$0")/gen_perf_stats.py" "$TEMP_DIR"