From d4e0272e9ea0dba2d2864560507a712d1633dce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Bastian?= Date: Mon, 18 Sep 2023 17:02:34 +0200 Subject: [PATCH] Palmed: add results --- manuscrit/30_palmed/40-1_results_fig.tex | 302 +++++++++++----------- manuscrit/30_palmed/40_palmed_results.tex | 49 ++-- 2 files changed, 186 insertions(+), 165 deletions(-) diff --git a/manuscrit/30_palmed/40-1_results_fig.tex b/manuscrit/30_palmed/40-1_results_fig.tex index d27051b..7793a6c 100644 --- a/manuscrit/30_palmed/40-1_results_fig.tex +++ b/manuscrit/30_palmed/40-1_results_fig.tex @@ -1,76 +1,75 @@ %%% Macros specific to this figure \newcommand{\na}{{\color{gray}N/A}} \newcommand{\centerheader}[1]{\multicolumn{1}{c}{#1}} -% Table-specific \newcommand{\figfiverow}[2]{ - \begin{minipage}[c]{0.03\linewidth} - \figrowlegend{#1} - \end{minipage}\figspaceleft{} - \begin{minipage}[c]{0.13\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-palmed.png} - \end{minipage}\figspacemid{} - \begin{minipage}[c]{0.13\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-UOPS.png} - \end{minipage}\figspacemid{} - \begin{minipage}[c]{0.13\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-PMEvo.png} - \end{minipage}\figspacemid{} - \begin{minipage}[c]{0.13\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-IACA.png} - \end{minipage}\figspacemid{} - \begin{minipage}[c]{0.13\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-LLVM_MCA.png} - \end{minipage}\figspaceright{} - \begin{minipage}[c]{0.035\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-legend.png} - \end{minipage} - } + \begin{minipage}[c]{0.03\linewidth} + \figrowlegend{#1} + \end{minipage}\figspaceleft{} + \begin{minipage}[c]{0.13\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-palmed.png} + \end{minipage}\figspacemid{} + \begin{minipage}[c]{0.13\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-UOPS.png} + \end{minipage}\figspacemid{} + \begin{minipage}[c]{0.13\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-PMEvo.png} + \end{minipage}\figspacemid{} + \begin{minipage}[c]{0.13\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-IACA.png} + \end{minipage}\figspacemid{} + \begin{minipage}[c]{0.13\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-LLVM_MCA.png} + \end{minipage}\figspaceright{} + \begin{minipage}[c]{0.035\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-legend.png} + \end{minipage} +} \newcommand{\figthreerow}[2]{ - \begin{minipage}[c]{0.03\linewidth} - \figrowlegend{#1} - \end{minipage}\figspaceleft{} - \begin{minipage}[c]{0.15\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-palmed.png} - \end{minipage}\figspacezenmid{} - \begin{minipage}[c]{0.15\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-PMEvo.png} - \end{minipage}\figspacezenmid{} - \begin{minipage}[c]{0.15\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-LLVM_MCA.png} - \end{minipage}\figspaceright{} - \begin{minipage}[c]{0.035\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-legend.png} - \end{minipage} - } + \begin{minipage}[c]{0.03\linewidth} + \figrowlegend{#1} + \end{minipage}\figspaceleft{} + \begin{minipage}[c]{0.15\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-palmed.png} + \end{minipage}\figspacezenmid{} + \begin{minipage}[c]{0.15\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-PMEvo.png} + \end{minipage}\figspacezenmid{} + \begin{minipage}[c]{0.15\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-LLVM_MCA.png} + \end{minipage}\figspaceright{} + \begin{minipage}[c]{0.035\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-legend.png} + \end{minipage} +} \newcommand{\figthreerowlegend}[2]{ - \begin{minipage}[c]{0.03\linewidth} - \figrowlegend{#1} - \end{minipage}\figspaceleft{} - \begin{minipage}[c]{0.15\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-palmed.png} - \end{minipage}\figspacezenmid{} - \begin{minipage}[c]{0.12\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmap_legend.svg} - \end{minipage}\figspacezenmid{} - \begin{minipage}[c]{0.15\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-PMEvo.png} - \end{minipage}\figspacezenmid{} - \begin{minipage}[c]{0.12\linewidth} - ~ - \end{minipage}\figspacezenmid{} - \begin{minipage}[c]{0.15\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-LLVM_MCA.png} - \end{minipage}\figspaceright{} - \begin{minipage}[c]{0.035\linewidth} - \includegraphics[width=\linewidth]{ipc_heatmaps/#2-legend.png} - \end{minipage} - } + \begin{minipage}[c]{0.03\linewidth} + \figrowlegend{#1} + \end{minipage}\figspaceleft{} + \begin{minipage}[c]{0.15\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-palmed.png} + \end{minipage}\figspacezenmid{} + \begin{minipage}[c]{0.12\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmap_legend.svg} + \end{minipage}\figspacezenmid{} + \begin{minipage}[c]{0.15\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-PMEvo.png} + \end{minipage}\figspacezenmid{} + \begin{minipage}[c]{0.12\linewidth} + ~ + \end{minipage}\figspacezenmid{} + \begin{minipage}[c]{0.15\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-LLVM_MCA.png} + \end{minipage}\figspaceright{} + \begin{minipage}[c]{0.035\linewidth} + \includegraphics[width=\linewidth]{ipc_heatmaps/#2-legend.png} + \end{minipage} +} \newenvironment{figleftlabel}[1]{ - \begin{minipage}[c]{0.02\linewidth} - \figrowlegend{#1} - \end{minipage}\figspaceleft{} - \begin{minipage}[c]{0.94\linewidth}} - {\end{minipage}} + \begin{minipage}[c]{0.02\linewidth} + \figrowlegend{#1} + \end{minipage}\figspaceleft{} +\begin{minipage}[c]{0.94\linewidth}} +{\end{minipage}} \newcommand{\figspaceleft}{\hspace{1em}} \newcommand{\figspaceright}{\hspace{1em}} \newcommand{\figspacemid}{\hfill} @@ -79,12 +78,12 @@ \newcommand{\figrowlegend}[1]{\rotatebox{90}{\small\textbf{#1}}} \newcommand{\figspacerow} -\newcommand{\tabcoverage}{\centerheader{Cov.}} -\newcommand{\taberror}{\centerheader{Err.}} -\newcommand{\tabkendall}{\centerheader{$\tau_K$}} -\newcommand{\tabcoverageunit}{\centerheader{(\%)}} -\newcommand{\taberrorunit}{\centerheader{(\%)}} -\newcommand{\tabkendallunit}{\centerheader{$(1)$}} +\newcommand{\tabcoverage}{Cov.} +\newcommand{\taberror}{Err.} +\newcommand{\tabkendall}{$\tau_K$} +\newcommand{\tabcoverageunit}{(\%)} +\newcommand{\taberrorunit}{(\%)} +\newcommand{\tabkendallunit}{$(1)$} \definecolor{tabcovbg}{RGB}{235,255,255} \definecolor{taberrbg}{RGB}{255,240,255} \definecolor{tabktaubg}{RGB}{239,250,235} @@ -92,92 +91,97 @@ \newcolumntype{E}{>{\columncolor{taberrbg}}r} \newcolumntype{K}{>{\columncolor{tabktaubg}}r} +\newcommand{\covrow}{\tabcoverage{} & \tabcoverageunit{}} +\newcommand{\errrow}{\taberror{} & \taberrorunit{}} +\newcommand{\taurow}{\tabkendall{} & \tabkendallunit{}} + %%% Taken directly from Palmed paper \begin{figure} - \begin{subfigure}[c]{\linewidth} - \centering - \begin{figleftlabel}{} - \begin{minipage}[c]{0.03\linewidth}~\end{minipage}\figspaceleft{} - \begin{minipage}[c]{0.13\linewidth} - \figcollegend{\palmed{}} - \end{minipage}\figspacemid{} - \begin{minipage}[c]{0.13\linewidth} - \figcollegend{uops.info} - \end{minipage}\figspacemid{} - \begin{minipage}[c]{0.13\linewidth} - \figcollegend{PMEvo} - \end{minipage}\figspacemid{} - \begin{minipage}[c]{0.13\linewidth} - \figcollegend{IACA} - \end{minipage}\figspacemid{} - \begin{minipage}[c]{0.13\linewidth} - \figcollegend{llvm-mca} - \end{minipage}\figspaceright{} - \begin{minipage}[c]{0.035\linewidth}~\end{minipage} - \end{figleftlabel} + \centering + \begin{figleftlabel}{} + \begin{minipage}[c]{0.03\linewidth}~\end{minipage}\figspaceleft{} + \begin{minipage}[c]{0.13\linewidth} + \figcollegend{\palmed{}} + \end{minipage}\figspacemid{} + \begin{minipage}[c]{0.13\linewidth} + \figcollegend{uops.info} + \end{minipage}\figspacemid{} + \begin{minipage}[c]{0.13\linewidth} + \figcollegend{PMEvo} + \end{minipage}\figspacemid{} + \begin{minipage}[c]{0.13\linewidth} + \figcollegend{IACA} + \end{minipage}\figspacemid{} + \begin{minipage}[c]{0.13\linewidth} + \figcollegend{llvm-mca} + \end{minipage}\figspaceright{} + \begin{minipage}[c]{0.035\linewidth}~\end{minipage} + \end{figleftlabel} - \begin{figleftlabel}{SKL-SP} - \figfiverow{SPEC 17}{spec-W-skx} + \begin{figleftlabel}{SKL-SP} + \figfiverow{SPEC 17}{spec-W-skx} - \figspacerow + \figspacerow - \figfiverow{Polybench}{polybench-W-skx} - \end{figleftlabel} + \figfiverow{Polybench}{polybench-W-skx} + \end{figleftlabel} - \figspacerow{} + \figspacerow{} - \begin{figleftlabel}{ZEN1} - \figthreerow{SPEC 17}{spec-W-zen} + \begin{figleftlabel}{ZEN1} + \figthreerow{SPEC 17}{spec-W-zen} - \figspacerow + \figspacerow - \figthreerowlegend{Polybench}{polybench-W-zen} - \end{figleftlabel} + \figthreerowlegend{Polybench}{polybench-W-zen} + \end{figleftlabel} - \caption{IPC prediction profile heatmaps~--~predictions closer to the - red line are more accurate. Predicted IPC ratio (Y) against native - IPC (X)} - \label{fig:realcpu_heatmap} - \end{subfigure} - - \vspace{1em} - - \begin{subtable}[c]{\linewidth} - { - \centering - \caption{Translation block coverage (Cov.), root-mean-square error on - IPC predictions (Err.) and Kendall's tau correlation coefficient - ($\tau_K$) compared to native execution} - \footnotesize -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Table generated automatically using palmed's `tools/gen_results_table.py` -% on 2021-08-27 17:22:33.207721 -% by tobast@patate -\label{tab:realcpu_results}\begin{tabular}{c c | C E K | C E K | C E K | C E K | C E K} - \toprule - & & \multicolumn{3}{c}{\palmed} & \multicolumn{3}{c}{\uopsinfo} & \multicolumn{3}{c}{\pmevo} & \multicolumn{3}{c}{\iaca} & \multicolumn{3}{c}{\llvmmca}\\ - & & \tabcoverage & \taberror & \tabkendall & \tabcoverage & \taberror & \tabkendall & \tabcoverage & \taberror & \tabkendall & \tabcoverage & \taberror & \tabkendall & \tabcoverage & \taberror & \tabkendall\\ - & Unit & \tabcoverageunit & \taberrorunit & \tabkendallunit & \tabcoverageunit & \taberrorunit & \tabkendallunit & \tabcoverageunit & \taberrorunit & \tabkendallunit & \tabcoverageunit & \taberrorunit & \tabkendallunit & \tabcoverageunit & \taberrorunit & \tabkendallunit\\ - \midrule - % Generated from /home/tobast/src/palmed/results/data/scw-skx.2021-08-26.benchs.spec17 - \multirow{2}{*}{\textbf{SKL-SP}} & \textbf{SPEC2017} & \na & 7.8 & 0.90 & 99.9 & 40.3 & 0.71 & 71.3 & 28.1 & 0.47 & 100.0 & 8.7 & 0.80 & 96.8 & 20.1 & 0.73\\ - % Generated from /home/tobast/src/palmed/results/data/scw-skx.2021-08-26.benchs.polybench - & \textbf{Polybench} & \na & 24.4 & 0.78 & 100.0 & 68.1 & 0.29 & 66.8 & 46.7 & 0.14 & 100.0 & 15.1 & 0.67 & 99.5 & 15.3 & 0.65\\ - % Generated from /home/tobast/src/palmed/results/data/scw-zen.2021-08-24.benchs.spec17 - \multirow{2}{*}{\textbf{ZEN1}} & \textbf{SPEC2017} & \na & 29.9 & 0.68 & \na & \na & \na & 71.3 & 36.5 & 0.43 & \na & \na & \na & 96.8 & 33.4 & 0.75\\ - % Generated from /home/tobast/src/palmed/results/data/scw-zen.2021-08-24.benchs.polybench - & \textbf{Polybench} & \na & 32.6 & 0.46 & \na & \na & \na & 66.8 & 38.5 & 0.11 & \na & \na & \na & 99.5 & 28.6 & 0.40\\ - \bottomrule -\end{tabular} -% End automatically generated table -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - } - \end{subtable} - - \caption{Accuracy of IPC predictions compared to native execution of - \palmed{} versus uops.info, PMEvo, IACA and llvm-mca on SPEC CPU2017 and -PolyBench/C 4.2}\label{fig:palmed_results} + \begin{figleftlabel}{A72 (ARM)} + \todo{PLOTS} + \end{figleftlabel} + \caption{IPC prediction profile heatmaps~--~predictions closer to the + red line are more accurate. Predicted IPC ratio (Y) against native + IPC (X)} + \label{fig:palmed_heatmaps} \end{figure} + +\begin{table} + \centering + \begin{tabular}{l l c r r r r} + \toprule + & & & \multicolumn{2}{c}{SKL-SP} & \multicolumn{2}{c}{ZEN1} \\ + & & & \centerheader{SPEC2017} & \centerheader{Polybench} & + \centerheader{SPEC2017} & \centerheader{Polybench} \\ + \midrule + \multirow{3}{*}{\palmed} + & \covrow{} & \na{}{} & \na{} & \na{} & \na{} \\ + & \errrow{} & 7.8 & 24.4 & 29.9 & 32.6 \\ + & \taurow{} & 0.90 & 0.78 & 0.68 & 0.46 \\ + \midrule{} + \multirow{3}{*}{\uopsinfo} + & \covrow{} & 99.9 & 100.0 & \na{} & \na{} \\ + & \errrow{} & 40.3 & 68.1 & \na{} & \na{} \\ + & \taurow{} & 0.71 & 0.29 & \na{} & \na{} \\ + \midrule{} + \multirow{3}{*}{\pmevo} + & \covrow{} & 71.3 & 66.8 & 71.3 & 66.8 \\ + & \errrow{} & 28.1 & 46.7 & 36.5 & 38.5 \\ + & \taurow{} & 0.47 & 0.14 & 0.43 & 0.11 \\ + \midrule{} + \multirow{3}{*}{\iaca} + & \covrow{} & 100.0 & 100.0 & \na{} & \na{} \\ + & \errrow{} & 8.7 & 15.1 & \na{} & \na{} \\ + & \taurow{} & 0.80 & 0.67 & \na{} & \na{} \\ + \midrule{} + \multirow{3}{*}{\llvmmca} + & \covrow{} & 96.8 & 99.5 & 96.8 & 99.5 \\ + & \errrow{} & 20.1 & 15.3 & 33.4 & 28.6 \\ + & \taurow{} & 0.73 & 0.65 & 0.75 & 0.40 \\ + \bottomrule + \end{tabular} + + \todo{A72} + \caption{Accuracy of IPC predictions compared to \pipedream{}-based native + executions on SPEC 2017 and Polybench}\label{table:palmed_eval} +\end{table} diff --git a/manuscrit/30_palmed/40_palmed_results.tex b/manuscrit/30_palmed/40_palmed_results.tex index c9f4baf..41ab5f9 100644 --- a/manuscrit/30_palmed/40_palmed_results.tex +++ b/manuscrit/30_palmed/40_palmed_results.tex @@ -52,14 +52,20 @@ In the case of SPEC, there is no clear kernel available for each benchmark; extracting basic blocks to evaluate \palmed{} is not trivial. We manually extract the relevant basic blocks using a profiling-based approach with Linux \perf{}, as the \qemu{}-based solution used for Polybench would be too costly -for SPEC\@. We automatize and describe this method in detail later in -\qtodo{ref}. +for SPEC\@. \bigskip{} Altogether, this method generates, for x86-64 processors, 13\,778 SPEC-based and 2\,664 polybench-based basic blocks. +\medskip{} + +We automate and describe in detail the \perf{}-based method later in +\qtodo{ref}; however, this automation allows us to also extract basic blocks +from these same benchmark suites, compiled for ARMv8a. + + \subsection{Evaluation harness} We implement into \palmed{} an evaluation harness to evaluate it both against @@ -103,20 +109,6 @@ To evaluate \palmed{}, the same kernel is run: The raw results are saved (as a Python \pymodule{pickle} file) for reuse and archival. -\medskip{} - -The evaluation is run on two different machines: -\begin{itemize} - \item{} an Intel \texttt{SKL-SP}-based machine, with two Intel Xeon Silver - 4114 CPU, totalling 20 cores; - \item{} an AMD \texttt{ZEN1}-based machine, with a single AMD EPYC 7401P - CPU with 24 cores. -\end{itemize} - -As \iaca{} only supports Intel CPUs, and \uopsinfo{} gives only very rough -information for \texttt{ZEN} architectures ---~without port mapping~---, these -two tools were not tested on the \texttt{ZEN1} machine. - \subsection{Metrics extracted} As \palmed{} internally works with Instructions Per Cycle (IPC) metrics, and as @@ -173,4 +165,29 @@ instructions found in the original basic blocks. \subsection{Results} +We run the evaluation harness on three different machines: +\begin{itemize} + \item{} an x86-64 Intel \texttt{SKL-SP}-based machine, with two Intel Xeon Silver + 4114 CPU, totalling 20 cores; + \item{} an x86-64 AMD \texttt{ZEN1}-based machine, with a single AMD EPYC 7401P + CPU with 24 cores; + \item{} an ARMv8a Raspberry Pi 4 with 4 Cortex A72 cores. +\end{itemize} + +As \iaca{} only supports Intel CPUs, and \uopsinfo{} only supports x86-64 +machines and gives only very rough +information for \texttt{ZEN} architectures ---~without port mapping~---, these +two tools were only tested on the \texttt{SKL-SP} machine. + +\medskip{} + +The evaluation metrics for all three architecture and all five tools are +presented in \autoref{table:palmed_eval}. We further represent IPC prediction +accuracy as heatmaps in \autoref{fig:palmed_heatmaps}. A dark area at +coordinate $(x, y)$ means that the selected tool has a prediction accuracy of +$y$ for a significant number of microkernels with a measured IPC of $x$. The +closer a prediction is to the red horizontal line, the more accurate it is. + +These results are analyzed in the full article~\cite{palmed}. + \input{40-1_results_fig.tex}