forked from m2-internship/talk-2018-11-uppsala
Cut out a large part of the content
This commit is contained in:
parent
a22b55e086
commit
7cd581fbd0
399
slides.tex
399
slides.tex
|
|
@ -66,17 +66,14 @@
|
|||
{\lstinline[language=C, keepspaces=true, basicstyle=\ttfamily]}
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
\title[\sectionline] {Growing the DWARF tougher:\\
|
||||
synthesis, validation and compilation}
|
||||
\author[\slidecountline]{\textbf{Théophile Bastian} \\ \vspace{0.5em}
|
||||
{{\footnotesize{}Based on work done with}\\
|
||||
\textbf{Francesco Zappa Nardelli},
|
||||
\textbf{Stephen Kell},
|
||||
\textbf{Simon Ser}}}
|
||||
\title[\sectionline] {Reliable and Fast DWARF-based Stack Unwinding}
|
||||
\author[\slidecountline]{\textbf{Théophile Bastian},\\
|
||||
\textbf{Stephen Kell}, \\
|
||||
\textbf{Francesco Zappa Nardelli}}
|
||||
\date{}
|
||||
%\subject{}
|
||||
%\logo{}
|
||||
\institute{ENS Paris, INRIA}
|
||||
\institute{ENS Paris, University of Kent, Inria}
|
||||
|
||||
\begin{document}
|
||||
|
||||
|
|
@ -87,7 +84,7 @@
|
|||
\vspace{-2em}
|
||||
\begin{center}
|
||||
\begin{align*}
|
||||
\text{Slides: } &\text{\url{https://tobast.fr/files/oracle18.pdf}} \\
|
||||
\text{Slides: } &\text{\todo{add URL for this PDF}} \\
|
||||
\end{align*}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
|
@ -264,31 +261,6 @@ $1 = 84
|
|||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{Difficult to generate}
|
||||
Major concern with DWARF\@: it is \alert{difficult to generate}
|
||||
(correctly).
|
||||
|
||||
\begin{itemize}
|
||||
\item{} \alert{Hard to generate}: each compiler pass must keep it in
|
||||
sync
|
||||
\item{} Most of it is \alert{seldom used} (\eg{} unwinding data of
|
||||
dusty code), and thus \alert{seldom tested}
|
||||
\end{itemize}
|
||||
\vspace{1em}
|
||||
|
||||
Yields to
|
||||
\begin{itemize}
|
||||
\item{} unreliable DWARF\@: can cause headaches when debugging
|
||||
\item{} or not generated at all (eg. OCaml until recently)
|
||||
\end{itemize}
|
||||
|
||||
\vspace{1em}
|
||||
\begin{center}
|
||||
\Large\bf
|
||||
$\leadsto$ Complex, buggy, untested
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\newcommand{\LinusMailOne}{
|
||||
``Sorry, but last time was too f\dots painful. The whole (and
|
||||
only) point of unwinders is to make debugging easy
|
||||
|
|
@ -309,18 +281,29 @@ $1 = 84
|
|||
\begin{frame}{A debugging hell: Linux kernel}
|
||||
\LinusMailOne{}
|
||||
|
||||
\vspace{1em}
|
||||
\LinusSource{}
|
||||
\only<1-2>{
|
||||
\vspace{1em}
|
||||
\LinusSource{}
|
||||
}
|
||||
|
||||
\pause{}\vspace{1em}
|
||||
\begin{center}
|
||||
\Large\bf
|
||||
\alert{This is where we still are!}
|
||||
\end{center}
|
||||
\vspace{1em}
|
||||
\only<2>{
|
||||
\begin{center}
|
||||
\Large\bf
|
||||
\alert{This is where we still are!}
|
||||
\end{center}
|
||||
}
|
||||
|
||||
\only<3>{
|
||||
\LinusMailTwo{}
|
||||
|
||||
\vspace{1em}
|
||||
\LinusSource{}
|
||||
}
|
||||
\end{frame}
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
\section{Unwinding data validation}
|
||||
\section{Unwinding data as an abstract state}
|
||||
|
||||
\newcommand{\tblrowval}[4]{#1 & #2 & \only<2->{#3} & \only<2->{#4} \\}
|
||||
\newcommand{\blknote}[1]
|
||||
|
|
@ -335,7 +318,7 @@ $1 = 84
|
|||
\end{block}}
|
||||
\newcommand{\tblhl}{\rowcolor{Tan}}
|
||||
|
||||
\begin{frame}{Validating an example}
|
||||
\begin{frame}{Working on an example}
|
||||
\newcommand{\firsttblrows}{
|
||||
\tblrowval{\hspace{-2ex}<{\bf foo}>:}{}{\textbf{CFA}}{\textbf{ra}}
|
||||
\rowonly<3>{\tblhl{}} \tblrowval{push}{\%r15}{rsp+8}{c-8}
|
||||
|
|
@ -346,7 +329,6 @@ $1 = 84
|
|||
\tblrowval{push}{\%rbp}{rsp+40}{c-8}
|
||||
\tblrowval{push}{\%rbx}{rsp+48}{c-8}
|
||||
\tblrowval{sub}{\$0x68,\%rsp}{rsp+56}{c-8}
|
||||
\rowonly<9>{\tblhl{}} \tblrowval{cmp}{\$0x1,\%edi}{rsp+160}{c-8}%
|
||||
}
|
||||
|
||||
\only<-8>{
|
||||
|
|
@ -377,154 +359,10 @@ $1 = 84
|
|||
\only<7>{The unwinding table can actually be seen as\\
|
||||
an \alert{abstract interpretation} of the code\ldots}
|
||||
\only<8>{\ldots and thus, for a given run, be
|
||||
\alert{re-computed for verification}}
|
||||
\alert{re-computed from scratch}}
|
||||
\end{overlayarea}
|
||||
}
|
||||
}
|
||||
\only<9->{
|
||||
\begin{table}
|
||||
\ttfamily\large
|
||||
\begin{tabularx}{0.9\linewidth}{
|
||||
l
|
||||
b
|
||||
>{\columncolor{SkyBlue}}s
|
||||
>{\columncolor{SkyBlue}}s
|
||||
}
|
||||
\firsttblrows{}%
|
||||
\end{tabularx}
|
||||
\end{table}
|
||||
|
||||
\vspace{-0.8em}
|
||||
\only<9>{\blklnote{If, within an execution,
|
||||
\begin{itemize}
|
||||
\item ra = \texttt{*(0xFFFF1098)}
|
||||
\item \reg{rsp} = \texttt{0xFFFF1000}
|
||||
\end{itemize}
|
||||
We can \alert{evaluate both expressions} and \alert{compare}
|
||||
}}
|
||||
}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{Dynamic validation}
|
||||
\textbf{Abstract state}
|
||||
\begin{itemize}
|
||||
\item \alert{Stack} of actual \alert{addresses} where
|
||||
\alert{return addresses} are stored
|
||||
\end{itemize}
|
||||
|
||||
\vspace{2em}\pause{}
|
||||
|
||||
\textbf{Abstract instruction semantics}
|
||||
\begin{itemize}
|
||||
\item[\alert{\texttt{call}}] push \alert{\reg{rsp}} on the stack
|
||||
\item[\alert{\texttt{ret}}] pop from the stack
|
||||
\end{itemize}
|
||||
|
||||
\vspace{2em}\pause{}
|
||||
|
||||
\textbf{Validation of each instruction}
|
||||
\begin{itemize}
|
||||
\item Evaluate the return address provided by DWARF
|
||||
\item Compare it with the value at the top of the stack
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{In practice: \texttt{eh\_frame\_check}}
|
||||
Strategy implemented and working: \alert{\texttt{eh\_frame\_check}}
|
||||
|
||||
\begin{itemize}
|
||||
\item \texttt{gdb} allows for Python instrumentation
|
||||
\vspace{1em}\pause{}
|
||||
\item Parse ELF and DWARF data (\texttt{pyelftools})
|
||||
\item Run the binary inside gdb
|
||||
\item Pause at each (assembly) step
|
||||
\item Jointly evaluate DWARF data and the abstract stack
|
||||
\item Report upon error
|
||||
\end{itemize}
|
||||
|
||||
\vspace{1em}
|
||||
Works, but… \alert{Python is slow}!
|
||||
|
||||
A few thousand of ASM instructions/second (good enough)
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{A real bug!}
|
||||
\begin{columns}[c]
|
||||
\begin{column}{0.65\textwidth}
|
||||
\lstinputlisting[language=C]{src/llvm_bug.c}
|
||||
\end{column} \begin{column}{0.35\textwidth}
|
||||
\textbf{\texttt{CSmith}\\+ \texttt{Creduce}\\+
|
||||
\texttt{eh\_frame\_check}}
|
||||
|
||||
\vspace{2em}$\leadsto$ \alert{\bf LLVM (3.8) bug!}
|
||||
\end{column}
|
||||
\end{columns}
|
||||
\end{frame}
|
||||
|
||||
\renewcommand{\tblrowval}[5]{#1 & #2 & #3 & #4 & #5 \\}
|
||||
\begin{frame}{A real bug!}
|
||||
|
||||
\begin{columns}[c]
|
||||
\column{0.7\textwidth}
|
||||
\begin{align*}
|
||||
\onslide<2->{\textbf{Abstract state} \qquad &
|
||||
\left[\texttt{0xFFFF1000}\right]} \\
|
||||
\onslide<3->{\reg{rsp} \qquad & %
|
||||
~\,\texttt{%
|
||||
\only<3-4>{0xFFFF1000}%
|
||||
\only<5-8>{0xFFFF0FF8}%
|
||||
\only<9->{0xFFFF1000}%
|
||||
}
|
||||
}
|
||||
\end{align*}
|
||||
|
||||
\column{0.3\textwidth}
|
||||
{\vspace{-4mm}\bf \fontsize{2cm}{5.5cm}\selectfont %
|
||||
\only<4>{\cmark}%
|
||||
\only<6>{\cmark}%
|
||||
\only<8>{\cmark}%
|
||||
\only<10->{\xmark}%
|
||||
}
|
||||
\end{columns}
|
||||
|
||||
|
||||
\vspace{1em}
|
||||
|
||||
\begin{table}
|
||||
\ttfamily\large
|
||||
\begin{tabularx}{0.95\linewidth}{
|
||||
l
|
||||
l
|
||||
b
|
||||
>{\columncolor{SkyBlue}}s
|
||||
>{\columncolor{SkyBlue}}s
|
||||
}
|
||||
\tblrowval{\hspace{-2ex}<{\bf foo}>:}{}{}{\textbf{CFA}}{\textbf{ra}}
|
||||
%\rowonly<3>{\tblhl{}} \tblrowval{4004e0}{push}{\%rbx}{rsp+8}{c-8}
|
||||
\rowonly<2-4>{\tblhl{}} \tblrowval{4004e0}{push}{\%rbx}{rsp+8}{c-8}
|
||||
\rowonly<5-6>{\tblhl{}} \tblrowval{}{}{}{rsp+16}{c-8}
|
||||
\tblrowval{}{[\ldots]}{}{}{}
|
||||
\tblrowval{}{}{}{}{}
|
||||
\rowonly<7-8>{\tblhl{}} \tblrowval{40061d}{pop}{\%rbx}{rsp+16}{c-8}
|
||||
\rowonly<9->{\tblhl{}} \tblrowval{40061e}{retq}{}{rsp+16}{c-8}
|
||||
\end{tabularx}
|
||||
\end{table}
|
||||
|
||||
\begin{center}
|
||||
\bf\Large %
|
||||
\onslide<11>{$\leadsto$ LLVM bug \#13161}
|
||||
\end{center}
|
||||
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{What for, in the end?}
|
||||
\begin{itemize}[<+->]
|
||||
\item We can \alert{find bugs} in compilers
|
||||
\item We can \alert{validate DWARF tables}!
|
||||
\item \ldots{}well, only along \alert{one execution path}\ldots
|
||||
\item but mostly we are close to a working \alert{algorithm} to
|
||||
\alert{synthesize unwinding data from binaries}!
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
|
@ -575,11 +413,11 @@ $1 = 84
|
|||
\begin{frame}{\cfa{} expression}
|
||||
Two possibilities:
|
||||
\begin{itemize}
|
||||
\item Either \reg{rbp} is used as base pointer \pause{}
|
||||
\item Or we must track \cfa{} wrt. \reg{rsp}
|
||||
\item Either we track \cfa{} wrt. \reg{rsp}
|
||||
\begin{itemize}
|
||||
\item And update it after each instruction if needed
|
||||
\end{itemize}
|
||||
\item and update it after each instruction if needed
|
||||
\end{itemize} \pause{}
|
||||
\item Or \reg{rbp} is used as base pointer: easy
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
|
|
@ -600,7 +438,7 @@ $1 = 84
|
|||
|
||||
\begin{itemize}
|
||||
\item \alert{Upon split} (eg. \texttt{X})\alert{:} nothing special,
|
||||
propagate end state of X to children nodes A and B
|
||||
propagate end state of X to child nodes A and B
|
||||
\item \alert{Upon join} (eg. \texttt{while\_end})\alert{:} check
|
||||
consistency of both input states
|
||||
\begin{itemize}
|
||||
|
|
@ -611,44 +449,6 @@ $1 = 84
|
|||
|
||||
\end{frame}
|
||||
|
||||
\renewcommand{\tblrowval}[4]{#1 & #2 & #3 & #4 \\}
|
||||
\begin{frame}{Trust the compiler to avoid tricky unwinding}
|
||||
\begin{columns}[c]
|
||||
\column{0.55\textwidth}
|
||||
\lstinputlisting[language=C,firstline=3,lastline=7]
|
||||
{src/use_rbp/use_rbp.c}
|
||||
|
||||
\pause{}
|
||||
\begin{itemize}
|
||||
\item At each loop cycle, \alert{\texttt{y} is larger} and
|
||||
\alert{allocated on the stack}
|
||||
\item Thus, \reg{rsp} is \alert{farther from \cfa} at each cycle: no
|
||||
constant rule $\cfa = \reg{rsp} + k$.
|
||||
\item A complex DWARF expression is possible, but
|
||||
\alert{the compiler won't}.
|
||||
\end{itemize}
|
||||
|
||||
\pause{}
|
||||
\column{0.45\textwidth}
|
||||
\lstbash{\$ gcc -O0 -g -c src.c -fomit-frame-pointer}
|
||||
|
||||
\vspace{1em}
|
||||
|
||||
\begin{center}
|
||||
\tt
|
||||
\begin{tabular}{l l l l}
|
||||
\tblrowval{LOC} {CFA} {rbp} {ra}
|
||||
\tblrowval{000} {rsp+8} {u} {c-8}
|
||||
\tblrowval{001} {rsp+16} {c-16} {c-8}
|
||||
\tblrowval{004} {\alert{rbp}+16} {c-16} {c-8}
|
||||
\tblrowval{010} {\alert{rbp}+16} {c-16} {c-8}
|
||||
\tblrowval{0ce} {rsp+8} {c-16} {c-8}
|
||||
\end{tabular}
|
||||
\end{center}
|
||||
|
||||
\end{columns}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{}
|
||||
\vfill
|
||||
\centering
|
||||
|
|
@ -699,10 +499,6 @@ $1 = 84
|
|||
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
\begin{frame}{Compilation example: original C, DWARF}
|
||||
\lstinputlisting[language=C]{src/fib7/fib7.cfde}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}[shrink]{Compilation example: generated C}
|
||||
\lstinputlisting[language=C]{src/fib7/fib7.eh_elf_basic.c}
|
||||
\end{frame}
|
||||
|
|
@ -710,28 +506,7 @@ $1 = 84
|
|||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
\begin{frame}{Compilation choices}
|
||||
\textbf{In order to keep the compiler \alert{simple} and \alert{easily
|
||||
testable}, the whole DWARF5 instruction set is not supported.}
|
||||
|
||||
\begin{itemize}
|
||||
\item Focus on \alert{x86\_64}
|
||||
\item Focus on unwinding return address \\
|
||||
\vspace{0.3ex}
|
||||
$\leadsto$ \textit{Allows building a backtrace}
|
||||
\begin{itemize}
|
||||
\item \alert{suitable for perf, not for gdb}
|
||||
\item Only supports \alert{unwinding registers}: \reg{rip}, \reg{rsp},
|
||||
\reg{rbp}, \reg{rbx}
|
||||
\item Supports the \alert{wide majority} ($> 99.9\%$) of instructions
|
||||
used
|
||||
\item Among \alert{4000} randomly sampled filed, only \alert{24}
|
||||
containing unsupported instructions
|
||||
\end{itemize}
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{Interface: libunwind}
|
||||
\begin{frame}{Mostly plug-and-play: libunwind interface}
|
||||
\begin{itemize}
|
||||
\item \alert{libunwind}: \textit{de facto} standard library for
|
||||
unwinding
|
||||
|
|
@ -752,79 +527,11 @@ $1 = 84
|
|||
\end{frame}
|
||||
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
\begin{frame}{Size optimisation: outlining}
|
||||
\begin{itemize}
|
||||
\item Most of the rows boil down to \alert{a few common rows}.
|
||||
\begin{itemize}
|
||||
\item[$\leadsto$] \textbf{\emph{outline} them!}
|
||||
\end{itemize}
|
||||
|
||||
\pause{}
|
||||
|
||||
\item On libc, $20\,827$ rows $\rightarrow$ $302$ outlined ($1.5\,\%$)
|
||||
\item Turn the big switch into a binary search \alert{if/else tree}
|
||||
\end{itemize}
|
||||
|
||||
\pause{}
|
||||
|
||||
\bigskip{}
|
||||
\begin{center}
|
||||
$\leadsto$ only \textbf{2.5 times bigger than DWARF}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}{Example with outlining}
|
||||
\lstinputlisting[language=C]{src/fib7/fib7.eh_elf_outline.c}
|
||||
\end{frame}
|
||||
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
\subsection{Benchmarking}
|
||||
|
||||
\begin{frame}{Benchmarking requirements}
|
||||
\begin{enumerate}
|
||||
\item Thousands of samples (single unwind: $10\,\mu{}s$)
|
||||
\item Interesting enough program to unwind: nested functions, complex
|
||||
FDEs
|
||||
\item Mitigate caching: don't always unwind from the \emph{same} point
|
||||
\item Yet be fair: don't always unwind from totally different places
|
||||
\item Distribute evenly: if possible, also from within libraries
|
||||
\end{enumerate}
|
||||
\end{frame}
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
\begin{frame}{perf instrumentation}
|
||||
\textbf{\alert{perf} is a state-of-the-art polling profiler for Linux.}
|
||||
\begin{itemize}
|
||||
\item{} used to get readings of the time spent in each function
|
||||
\item{} works by regularly stopping the program, unwinding its stack,
|
||||
then aggregating the gathered data
|
||||
\end{itemize}
|
||||
|
||||
\pause{}\bigskip{}
|
||||
\textbf{Instrumenting perf matches all the requirements!}
|
||||
|
||||
\begin{itemize}
|
||||
\item{} \alert{Plug \ehelfs{} into perf}: use \ehelfs{} instead of
|
||||
DWARF to unwind the stack
|
||||
\item{} Implement \alert{unwinding performance counters} inside perf
|
||||
\bigskip{}
|
||||
|
||||
\item{} Use perf on \alert{hackbench}, a kernel stress-test program
|
||||
\begin{itemize}
|
||||
\item Small program
|
||||
\item Lots of calls
|
||||
\item Relies on libc, libpthread
|
||||
\end{itemize}
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
\subsection{Results}
|
||||
|
||||
\begin{frame}{Time performance}
|
||||
\todo{UPDATE}
|
||||
\small
|
||||
\centering
|
||||
\begin{tabular}{l r r r r r}
|
||||
|
|
@ -857,6 +564,7 @@ $1 = 84
|
|||
\end{frame}
|
||||
|
||||
\begin{frame}{Space performance}
|
||||
\todo{UPDATE}
|
||||
\begin{center}
|
||||
\begin{tabular}{r r r r r r}
|
||||
\toprule
|
||||
|
|
@ -884,40 +592,23 @@ $1 = 84
|
|||
\section*{Conclusion}
|
||||
\setcounter{section}{0}
|
||||
|
||||
\begin{frame}{}
|
||||
\LinusMailOne{}
|
||||
\begin{frame}{A fragment of our article}
|
||||
The original article \textbf{Reliable and Fast DWARF-based Stack Unwinding}
|
||||
contains
|
||||
|
||||
\only<1>{\LinusSource{}}
|
||||
\only<2->{\vspace{1em}}
|
||||
\pause{}
|
||||
\vspace{1em}
|
||||
|
||||
\LinusMailTwo{}
|
||||
\begin{itemize}
|
||||
\item{} DWARF unwinding tables validation;
|
||||
\item{} DWARF unwinding tables synthesis;
|
||||
\item{} DWARF-based unwinding speedup.
|
||||
\end{itemize}
|
||||
|
||||
\LinusSource{}
|
||||
|
||||
\pause{}
|
||||
\vspace{1em}
|
||||
|
||||
\begin{center}
|
||||
\bf
|
||||
Give us a few months: we will make Linus reconsider \texttt{;)}
|
||||
Come and chat if interested! \texttt{:)}
|
||||
\end{center}
|
||||
|
||||
\end{frame}
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
\begin{frame}
|
||||
\vspace{5mm}
|
||||
\includegraphics[width=\linewidth]{img/keep_breathing}
|
||||
\vspace{-1cm}
|
||||
|
||||
\begin{center}
|
||||
\large
|
||||
\begin{align*}
|
||||
\textbf{Slides: } &\text{\url{https://tobast.fr/files/oracle18.pdf}} \\
|
||||
\end{align*}
|
||||
\end{center}
|
||||
|
||||
\end{frame}
|
||||
|
||||
\end{document}
|
||||
|
|
|
|||
Loading…
Reference in New Issue