From 2561d3ed4999096cef39ac2b33b957a0e8e72d0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Th=C3=A9ophile=20Bastian?= <contact@tobast.fr>
Date: Mon, 15 Jul 2019 16:23:44 +0200
Subject: [PATCH] Add/enhance benchmarking tools

---
 benching/tools/gen_evals.sh      | 27 +++++++++++++++++++++++++++
 benching/tools/gen_perf_stats.py | 19 ++++++++++++++++++-
 benching/tools/to_report_fmt.py  | 21 +++++++++++++++++++++
 3 files changed, 66 insertions(+), 1 deletion(-)
 create mode 100755 benching/tools/gen_evals.sh
 create mode 100755 benching/tools/to_report_fmt.py

diff --git a/benching/tools/gen_evals.sh b/benching/tools/gen_evals.sh
new file mode 100755
index 0000000..8c8bce1
--- /dev/null
+++ b/benching/tools/gen_evals.sh
@@ -0,0 +1,27 @@
+OUTPUT="$1"
+NB_ITER=10
+
+if [ "$#" -lt 1 ] ; then
+    >&2 echo "Missing argument: output directory."
+    exit 1
+fi
+
+if [ -z "$EH_ELFS" ]; then
+    >&2 echo "Missing environment: EH_ELFS. Aborting."
+    exit 1
+fi
+
+mkdir -p "$OUTPUT"
+
+for flavour in 'eh_elf' 'vanilla' 'vanilla-nocache'; do
+    >&2 echo "$flavour..."
+    source "$(dirname "$0")/../../env/apply" "$flavour" release
+    for iter in $(seq 1 $NB_ITER); do
+        >&2 echo -e "\t$iter..."
+        LD_LIBRARY_PATH="$EH_ELFS:$LD_LIBRARY_PATH" \
+            perf report 2>&1 >/dev/null | tail -n 1 \
+            | python "$(dirname $0)/to_report_fmt.py" \
+            | sed 's/^.* & .* & \([0-9]*\) & .*$/\1/g'
+    done > "$OUTPUT/${flavour}_times"
+    deactivate
+done
diff --git a/benching/tools/gen_perf_stats.py b/benching/tools/gen_perf_stats.py
index ce79bfe..ef8140e 100644
--- a/benching/tools/gen_perf_stats.py
+++ b/benching/tools/gen_perf_stats.py
@@ -29,6 +29,11 @@ def read_series(path):
 
 
 FLAVOURS = ["eh_elf", "vanilla"]
+WITH_NOCACHE = False
+
+if "WITH_NOCACHE" in os.environ:
+    WITH_NOCACHE = True
+    FLAVOURS.append("vanilla-nocache")
 
 path_format = os.path.join(sys.argv[1], "{}_times")
 times = {}
@@ -59,6 +64,18 @@ def format_flv(flv_dict, formatter):
     return out
 
 
+def get_ratios(avgs):
+    def avg_of(flavour):
+        return avgs[flavour] / avgs["eh_elf"]
+
+    if WITH_NOCACHE:
+        return "\n\tcached: {}\n\tuncached: {}".format(
+            avg_of("vanilla"), avg_of("vanilla-nocache")
+        )
+    else:
+        return avg_of("vanilla-nocache")
+
+
 print(
     "Average time:\n{}\n"
     "Standard deviation:\n{}\n"
@@ -66,7 +83,7 @@ print(
     "Ratio uncertainty: {}".format(
         format_flv(avgs, "{} ns"),
         format_flv(std_deviations, "{}"),
-        avg_ratio,
+        get_ratios(avgs),
         ratio_uncertainty,
     )
 )
diff --git a/benching/tools/to_report_fmt.py b/benching/tools/to_report_fmt.py
new file mode 100755
index 0000000..9500714
--- /dev/null
+++ b/benching/tools/to_report_fmt.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+
+line = input()
+regex = \
+    re.compile(r'Total unwind time: ([0-9]*) s ([0-9]*) ns, ([0-9]*) calls')
+
+match = regex.match(line.strip())
+if not match:
+    print('Badly formatted line', file=sys.stderr)
+    sys.exit(1)
+
+sec = int(match.group(1))
+ns = int(match.group(2))
+calls = int(match.group(3))
+
+time = sec * 10**9 + ns
+
+print("{} & {} & {} & ??".format(calls, time, time // calls))