From 789e79613c3b016fd6c8936f8233f6cd76d2d489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Bastian?= Date: Wed, 9 May 2018 17:55:57 +0200 Subject: [PATCH] Refactor python shared snippets --- .gitignore | 1 + compare_sizes.py | 6 +-- generate_eh_elf.py | 99 +++----------------------------------- shared_python.py | 116 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 96 deletions(-) create mode 100644 shared_python.py diff --git a/.gitignore b/.gitignore index f68817c..cd1f905 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ *.app dwarf-assembly +__pycache__ diff --git a/compare_sizes.py b/compare_sizes.py index 0b4f76c..dea8faf 100755 --- a/compare_sizes.py +++ b/compare_sizes.py @@ -9,7 +9,7 @@ import os import subprocess from collections import namedtuple -from generate_eh_elf import elf_so_deps +from shared_python import elf_so_deps ''' An ELF object, including the path to the ELF itself, and the path to its @@ -41,11 +41,11 @@ def section_size(elf_loc, section_name): except subprocess.CalledProcessError as exn: raise Exception(("Cannot obtain section {} size for {}: objdump " "terminated with exit code {}.").format( - section_name, elf_loc)) + section_name, elf_loc, exn.returncode)) for line in objdump_out.split('\n'): line = line.strip() - if not line or not ('0' <= line[0] <= '9'): # not a section line + if not line or not '0' <= line[0] <= '9': # not a section line continue spl = line.split() diff --git a/generate_eh_elf.py b/generate_eh_elf.py index 258ec84..68cf577 100755 --- a/generate_eh_elf.py +++ b/generate_eh_elf.py @@ -9,10 +9,10 @@ all the shared objects it depends upon. import os import sys import subprocess -import re import tempfile import argparse -from collections import namedtuple + +from shared_python import elf_so_deps, do_remote, is_newer DWARF_ASSEMBLY_BIN = os.path.join( @@ -23,33 +23,6 @@ C_BIN = ( else os.environ['C']) -def elf_so_deps(path): - ''' Get the list of shared objects dependencies of the given ELF object. - This is obtained by running `ldd`. ''' - - deps_list = [] - - try: - ldd_output = subprocess.check_output(['/usr/bin/ldd', path]) \ - .decode('utf-8') - ldd_re = re.compile(r'^.* => (.*) \(0x[0-9a-fA-F]*\)$') - - ldd_lines = ldd_output.strip().split('\n') - for line in ldd_lines: - line = line.strip() - match = ldd_re.match(line) - if match is None: - continue # Just ignore that line — it might be eg. linux-vdso - deps_list.append(match.group(1)) - - return deps_list - - except subprocess.CalledProcessError as e: - raise Exception( - ("Cannot get dependencies for {}: ldd terminated with exit code " - "{}.").format(path, e.returncode)) - - def gen_dw_asm_c(obj_path, out_path, dwarf_assembly_args): ''' Generate the C code produced by dwarf-assembly from `obj_path`, saving it as `out_path` ''' @@ -61,66 +34,13 @@ def gen_dw_asm_c(obj_path, out_path, dwarf_assembly_args): [DWARF_ASSEMBLY_BIN, obj_path] + dwarf_assembly_args) \ .decode('utf-8') out_handle.write(dw_asm_output) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError as exn: raise Exception( ("Cannot generate C code from object file {} using {}: process " "terminated with exit code {}.").format( obj_path, DWARF_ASSEMBLY_BIN, - e.returncode)) - - -def do_remote(remote, command, send_files=None, retr_files=None): - def ssh_do(cmd_args, working_directory=None): - try: - cmd = ['ssh', remote] - if working_directory: - cmd += ['cd', working_directory, '&&'] - cmd += cmd_args - - return subprocess.check_output(cmd).decode('utf-8').strip() - except subprocess.CalledProcessError as e: - return None - - def ssh_copy(what, where, is_upload): - if is_upload: - where = '{}:{}'.format(remote, where) - else: - what = '{}:{}'.format(remote, what) - - subprocess.check_output(['scp', what, where]) - - TransferredFile = namedtuple('TransferredFile', 'local remote') - - def interpret_transferred_file(descr): - if type(descr) == type(''): - return TransferredFile(descr, descr) - if os.path.isdir(descr[1]): - to = os.path.join(descr[1], descr[0]) - else: - to = descr[1] - return TransferredFile(descr[0], to) - - # Create temp dir - tmp_dir = ssh_do(['mktemp', '-d']) - - # Upload `send_files` - for f in send_files: - dest = tmp_dir+'/' - ssh_copy(f, dest, is_upload=True) - - # Do whatever must be done - output = ssh_do(command, working_directory=tmp_dir) - - # Download `retr_files` - for f in map(interpret_transferred_file, retr_files): - src = os.path.join(tmp_dir, f.local) - ssh_copy(src, f.remote, is_upload=False) - - # Remove temp dir - ssh_do(['rm', '-rf', tmp_dir]) - - return output + exn.returncode)) def gen_eh_elf(obj_path, args, dwarf_assembly_args=None): @@ -141,15 +61,8 @@ def gen_eh_elf(obj_path, args, dwarf_assembly_args=None): out_base_name = os.path.basename(obj_path) + '.eh_elf' out_so_path = os.path.join(out_dir, (out_base_name + '.so')) - try: - so_mtime = os.path.getmtime(out_so_path) - obj_mtime = os.path.getmtime(obj_path) - - if obj_mtime < so_mtime: - return # The object is recent enough, no need to recreate it - except OSError: - pass - + if is_newer(out_so_path, obj_path): + return # The object is recent enough, no need to recreate it with tempfile.TemporaryDirectory() as compile_dir: # Generate the C source file diff --git a/shared_python.py b/shared_python.py new file mode 100644 index 0000000..00b29e1 --- /dev/null +++ b/shared_python.py @@ -0,0 +1,116 @@ +""" Shared snippets between the various python scripts of this project """ + +import subprocess +import re +import os +from collections import namedtuple + + +def is_newer(file1, file2): + ''' Returns True iff file1 is newer than file2 ''' + try: + f1_mtime = os.path.getmtime(file1) + f2_mtime = os.path.getmtime(file2) + + if f1_mtime > f2_mtime: + return True + except OSError: + pass + + return False + + +def elf_so_deps(path): + ''' Get the list of shared objects dependencies of the given ELF object. + This is obtained by running `ldd`. ''' + + deps_list = [] + + try: + ldd_output = subprocess.check_output(['/usr/bin/ldd', path]) \ + .decode('utf-8') + ldd_re = re.compile(r'^.* => (.*) \(0x[0-9a-fA-F]*\)$') + + ldd_lines = ldd_output.strip().split('\n') + for line in ldd_lines: + line = line.strip() + match = ldd_re.match(line) + if match is None: + continue # Just ignore that line — it might be eg. linux-vdso + deps_list.append(match.group(1)) + + return deps_list + + except subprocess.CalledProcessError as exn: + raise Exception( + ("Cannot get dependencies for {}: ldd terminated with exit code " + "{}.").format(path, exn.returncode)) + + +def do_remote(remote, command, send_files=None, retr_files=None): + ''' Execute remotely (via ssh) a given command + + The command is executed on the machine described by `remote` (see ssh(1)). + + send_files is a list of file paths that must be first copied at the root of + a temporary directory on `remote` before running the command. Consider + yourself jailed in that directory. + + retr_files is a list of files that will be copied to the local machine + after the command is executed. Each list item can either be a string, which + is both the path on the remote and the local machine; or a pair + `(file_name, local_path)`. In the latter case, `file_name` is copied as + `local_path/file_name` if `local_path` is a directory, or as `local_path` + otherwise, on the local machine. + ''' + + def ssh_do(cmd_args, working_directory=None): + try: + cmd = ['ssh', remote] + if working_directory: + cmd += ['cd', working_directory, '&&'] + cmd += cmd_args + + return subprocess.check_output(cmd).decode('utf-8').strip() + except subprocess.CalledProcessError as e: + return None + + def ssh_copy(what, where, is_upload): + if is_upload: + where = '{}:{}'.format(remote, where) + else: + what = '{}:{}'.format(remote, what) + + subprocess.check_output(['scp', what, where]) + + TransferredFile = namedtuple('TransferredFile', 'local remote') + + def interpret_transferred_file(descr): + if isinstance(descr, type('')): + return TransferredFile(descr, descr) + if os.path.isdir(descr[1]): + to = os.path.join(descr[1], descr[0]) + else: + to = descr[1] + return TransferredFile(descr[0], to) + + # Create temp dir + tmp_dir = ssh_do(['mktemp', '-d']) + + # Upload `send_files` + for f in send_files: + dest = tmp_dir+'/' + ssh_copy(f, dest, is_upload=True) + + # Do whatever must be done + output = ssh_do(command, working_directory=tmp_dir) + + # Download `retr_files` + for f in map(interpret_transferred_file, retr_files): + src = os.path.join(tmp_dir, f.local) + ssh_copy(src, f.remote, is_upload=False) + + # Remove temp dir + ssh_do(['rm', '-rf', tmp_dir]) + + return output