Enhance statistics generation

benching/tools/errors.sh

@@ -3,11 +3,14 @@
 source "$(dirname $0)/common.sh"
 
 TMP_FILE=$(mktemp)
+if [ -z "$EH_ELFS_NAME" ]; then
+    EH_ELFS_NAME="eh_elfs"
+fi
 
 function get_perf_output {
     envtype=$1
     source $ENV_APPLY "$envtype" "dbg"
-    LD_LIBRARY_PATH="$BENCH_DIR/eh_elfs:$LD_LIBRARY_PATH" \
+    LD_LIBRARY_PATH="$BENCH_DIR/$EH_ELFS_NAME:$LD_LIBRARY_PATH" \
         UNW_DEBUG_LEVEL=15 \
         perf report -i "$BENCH_DIR/perf.data" 2>$TMP_FILE >/dev/null
     deactivate

benching/tools/gen_perf_stats.py

@@ -3,29 +3,23 @@
 """ Generates performance statistics for the eh_elf vs vanilla libunwind unwinding,
 based on time series generated beforehand
 
-First run
+Intended to be run from `statistics.sh`
-```bash
-for i in $(seq 1 100); do
-  perf report 2>&1 >/dev/null | tail -n 1 \
-      | python ../hackbench/to_report_fmt.py \
-          | sed 's/^.* & .* & \([0-9]*\) & .*$/\1/g'
-done > $SOME_PLACE/$FLAVOUR_times
-```
-
-for each flavour (eh_elf, vanilla)
-
-Then run this script, with `$SOME_PLACE` as argument.
 """
 
+from collections import namedtuple
 import numpy as np
 import sys
 import os
 
 
+Datapoint = namedtuple("Datapoint", ["nb_frames", "total_time", "avg_time"])
+
+
 def read_series(path):
     with open(path, "r") as handle:
         for line in handle:
-            yield int(line.strip())
+            nb_frames, total_time, avg_time = map(int, line.strip().split())
+            yield Datapoint(nb_frames, total_time, avg_time)
 
 
 FLAVOURS = ["eh_elf", "vanilla"]
@@ -36,14 +30,32 @@ if "WITH_NOCACHE" in os.environ:
     FLAVOURS.append("vanilla-nocache")
 
 path_format = os.path.join(sys.argv[1], "{}_times")
-times = {}
+datapoints = {}
+avg_times = {}
+total_times = {}
+avgs_total = {}
 avgs = {}
 std_deviations = {}
+unwound_frames = {}
 
 for flv in FLAVOURS:
-    times[flv] = list(read_series(path_format.format(flv)))
+    datapoints[flv] = list(read_series(path_format.format(flv)))
-    avgs[flv] = sum(times[flv]) / len(times[flv])
+    avg_times[flv] = list(map(lambda x: x.avg_time, datapoints[flv]))
-    std_deviations[flv] = np.sqrt(np.var(times[flv]))
+    total_times[flv] = list(map(lambda x: x.total_time, datapoints[flv]))
+    avgs[flv] = sum(avg_times[flv]) / len(avg_times[flv])
+    avgs_total[flv] = sum(total_times[flv]) / len(total_times[flv])
+    std_deviations[flv] = np.sqrt(np.var(avg_times[flv]))
+
+    cur_unwound_frames = list(map(lambda x: x.nb_frames, datapoints[flv]))
+    unwound_frames[flv] = cur_unwound_frames[0]
+    for run_id, unw_frames in enumerate(cur_unwound_frames[1:]):
+        if unw_frames != unwound_frames[flv]:
+            print(
+                "{}, run {}: unwound {} frames, reference unwound {}".format(
+                    flv, run_id + 1, unw_frames, unwound_frames[flv]
+                ),
+                file=sys.stderr,
+            )
 
 avg_ratio = avgs["vanilla"] / avgs["eh_elf"]
 ratio_uncertainty = (
@@ -56,11 +68,12 @@ ratio_uncertainty = (
 )
 
 
-def format_flv(flv_dict, formatter):
+def format_flv(flv_dict, formatter, alterator=None):
     out = ""
     for flv in FLAVOURS:
         val = flv_dict[flv]
-        out += "* {}: {}\n".format(flv, formatter.format(val))
+        altered = alterator(val) if alterator else val
+        out += "* {}: {}\n".format(flv, formatter.format(altered))
     return out
 
 
@@ -77,10 +90,14 @@ def get_ratios(avgs):
 
 
 print(
-    "Average time:\n{}\n"
+    "Unwound frames:\n{}\n"
+    "Average whole unwinding time (one run):\n{}\n"
+    "Average time to unwind one frame:\n{}\n"
     "Standard deviation:\n{}\n"
     "Average ratio: {}\n"
     "Ratio uncertainty: {}".format(
+        format_flv(unwound_frames, "{}"),
+        format_flv(avgs_total, "{} μs", alterator=lambda x: x // 1000),
         format_flv(avgs, "{} ns"),
         format_flv(std_deviations, "{}"),
         get_ratios(avgs),

benching/tools/statistics.sh

@@ -12,7 +12,7 @@ function collect_perf_time_data {
         perf report -i "$BENCH_DIR/perf.data" 2>&1 >/dev/null \
         | tail -n 1 \
         | python "$(dirname $0)/to_report_fmt.py" \
-        | sed 's/^.* & .* & \([0-9]*\) & .*$/\1/g'
+        | sed 's/^\([0-9]*\) & \([0-9]*\) & \([0-9]*\) & .*$/\1 \2 \3/g'
     deactivate
 }
 
@@ -32,6 +32,11 @@ vanilla_data="$TEMP_DIR/vanilla_times"
 collect_perf_time_data_runs "eh_elf" "$eh_elf_data"
 collect_perf_time_data_runs "vanilla" "$vanilla_data"
 
+if [ -n "$WITH_NOCACHE" ]; then
+    vanilla_nocache_data="$TEMP_DIR/vanilla-nocache_times"
+    collect_perf_time_data_runs "vanilla-nocache" "$vanilla_nocache_data"
+fi
+
 status_report "benchmark statistics"
 python "$(dirname "$0")/gen_perf_stats.py" "$TEMP_DIR"