#!/usr/bin/env python #===- lib/asan/scripts/asan_symbolize.py -----------------------------------===# # # The LLVM Compiler Infrastructure # # This file is distributed under the University of Illinois Open Source # License. See https://github.com/llvm-mirror/compiler-rt/blob/master/LICENSE.TXT # for details. # #===------------------------------------------------------------------------===# import argparse import array import binascii import bisect import os import re import subprocess import sys symbolizers = {} DEBUG = False demangle = False binutils_prefix = None sysroot_path = None binary_name_filter = None fix_filename_patterns = None logfile = sys.stdin separate_debug_dir_list = [] prelink_offset = None # FIXME: merge the code that calls fix_filename(). def fix_filename(file_name): if fix_filename_patterns: for path_to_cut in fix_filename_patterns: file_name = re.sub('.*' + path_to_cut, '', file_name) file_name = re.sub('.*asan_[a-z_]*.cc:[0-9]*', '_asan_rtl_', file_name) file_name = re.sub('.*crtstuff.c:0', '???:0', file_name) return file_name def sysroot_path_filter(binary_name): return sysroot_path + binary_name def use_binutils_prefix(tool_name): if binutils_prefix: tool_name = binutils_prefix + tool_name return tool_name def print_error_message(message): print >> sys.stderr, 'Error occured during symbolizisation: ' + message class DebugInfoHandler(object): def __init__(self, binary_name_filter=None): self.binary_name_filter = binary_name_filter self.global_debug_dir_list = [ self.use_name_filter("/usr/lib/debug") ] if separate_debug_dir_list: self.global_debug_dir_list = separate_debug_dir_list def use_name_filter(self, binary_name): if self.binary_name_filter: binary_name = self.binary_name_filter(binary_name) return binary_name def calc_crc32(self, filename): buf = open(filename,'rb').read() buf = (binascii.crc32(buf, 0) & 0xFFFFFFFF) return "%08X" % buf def readelf_binary(self, options, binary): cmd = [use_binutils_prefix('readelf'), options,'-W', binary] try: process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=open(os.devnull, 'w')) except: print_error_message('the following command failed:\n''"' + ' '.join(cmd) + '"') return (readelf_out, _) = process.communicate() if process.returncode == 0: return readelf_out def has_debuginfo(self, binary): readelf_out = self.readelf_binary('-S', binary) return readelf_out and (".debug_" in readelf_out) def get_buildid(self, binary): readelf_out = self.readelf_binary('-nw', binary) ''' Build ID is 40-length hex value following after "Build ID:". e.g.: Notes at offset 0x00000274 with length 0x00000024: Owner Data size Description GNU 0x00000014 NT_GNU_BUILD_ID (unique build ID bitstring) Build ID: 977b1d1375ba6791d5f6dd38d8d55b95f9fca33a ''' if readelf_out: readelf_lines = readelf_out.split("\n"); buildid_lines = filter(re.compile('Build ID:').search, readelf_lines) for line in buildid_lines: match = re.search('[a-f0-9]{40}', line) if match: return match.group(0) else: print_error_message('failed to read Build ID value from ' + binary) def get_debuglink_name(self, binary): readelf_out = self.readelf_binary('--string-dump=.gnu_debuglink', binary) ''' "debug link" is the last word in the first line of dump. e.g.: String dump of section '.gnu_debuglink': [ 0] HeapOutOfBounds.out.debug ''' if readelf_out: readelf_lines = filter(None, readelf_out.split("\n")); headline ="String dump of section '.gnu_debuglink':" try: debuglink_line_idx = readelf_lines.index(headline) + 1 except ValueError: # There is no gnu_debuglink section in this binary return if len(readelf_lines) > debuglink_line_idx: return readelf_lines[debuglink_line_idx].split()[-1] else: print_error_message('failed to read debuglink value from ' + binary) def get_debuglink_crc(self, binary): readelf_out = self.readelf_binary('--hex-dump=.gnu_debuglink', binary) ''' crc is the last hex group (before string representation) in the last line of dump. e.g. (crc is f89f21c2) : Hex dump of section '.gnu_debuglink': 0x00000000 48656170 4f75744f 66426f75 6e64732e HeapOutOfBounds. 0x00000010 6f75742e 64656275 67000000 f89f21c2 out.debug.....!. ''' if readelf_out: # get last non-empty line crc_line = filter(None, readelf_out.split("\n"))[-1] # remove last 16 characters (string dump) from line crc_line = crc_line[:-16] # crc is last word in string crc = crc_line.split()[-1] match = re.match('[a-f0-9]{8}', crc) if match: if sys.byteorder == 'little': crc = array.array('i', binascii.unhexlify(crc) ) crc.byteswap() crc = binascii.hexlify(crc) return crc else: print_error_message('failed to get crc checksum from debuglink in ' + binary) def is_prelinked(self, binary): readelf_out = self.readelf_binary('-S', binary) return readelf_out and ".gnu.prelink_undo" in readelf_out def get_load_address(self, binary): readelf_out = self.readelf_binary('-l', binary) ''' Readelf program headers output example: Elf file type is DYN (Shared object file) Entry point 0xb1160668 There are 10 program headers, starting at offset 52 Program Headers: Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align EXIDX 0x124754 0xb126c754 0xb126c754 0x04498 0x04498 R 0x4 [...] LOAD 0x000000 0xb1148000 0xb1148000 0x12be9c 0x12be9c R E 0x8000 ''' readelf_lines = readelf_out.split("\n"); load_lines = filter(re.compile( '[\s]*LOAD[\s]+0x[0]+[\s]+0x[a-fA-f\d]+').match, readelf_lines) if load_lines: return load_lines[0].split()[2] else: print_error_message('failed to get load address in ' + binary) def get_prelink_offset(self, orig, prelinked): if not self.is_prelinked(prelinked): return orig_load_addr = self.get_load_address(orig) prelinked_load_addr = self.get_load_address(prelinked) return int(prelinked_load_addr, 16) - int(orig_load_addr, 16) def locate_in_orig_dir(self, debuglink_name, orig_binary_name): debuginfo = os.path.join(os.path.dirname(orig_binary_name), debuglink_name) debuginfo = self.use_name_filter(debuginfo) return [debuginfo] def locate_in_debug_subdir(self, debuglink_name, orig_binary_name): debuginfo = os.path.join(os.path.dirname(orig_binary_name), '.debug', debuglink_name) debuginfo = self.use_name_filter(debuginfo) return [debuginfo] def locate_in_global_debug_dir(self, debuglink_name, orig_binary_name): debuginfo_list = [] for global_debug_dir in self.global_debug_dir_list: debuginfo = global_debug_dir + os.path.join(os.path.dirname(orig_binary_name), debuglink_name) debuginfo_list.append(debuginfo) return debuginfo_list def locate_in_buildid_dir(self, debuglink_name, orig_binary_name): dir_name = debuglink_name[0:2] file_name = debuglink_name[2:] + ".debug" debuginfo_list = [] for global_debug_dir in self.global_debug_dir_list: debuginfo = os.path.join(global_debug_dir, ".build-id", dir_name, file_name) debuginfo_list.append(debuginfo) return debuginfo_list def get_debuginfo(self,binary_name): global prelink_offset prelink_offset = None orig_binary_name = binary_name # First apply sysroot path if defined to get real binary real_binary = self.use_name_filter(binary_name) if self.has_debuginfo(real_binary): return real_binary # Look for debuginfo file according to GDB rules # 1) "build-id" method buildid_name = self.get_buildid(real_binary) debugfile_list = [] if buildid_name: debugfile_list = self.locate_in_buildid_dir(buildid_name, orig_binary_name) for debugfile in debugfile_list: if os.path.isfile(debugfile): prelink_offset = self.get_prelink_offset(debugfile, real_binary) return debugfile # 2) "debug link" method debuglink_name = self.get_debuglink_name(real_binary) debuglink_crc = self.get_debuglink_crc(real_binary) debugfile_list = [] if debuglink_name and debuglink_crc: debuglink_locate_list = [ self.locate_in_orig_dir, self.locate_in_debug_subdir, self.locate_in_global_debug_dir ] for debuglink_locate_function in debuglink_locate_list: debugfile_list = debuglink_locate_function(debuglink_name, orig_binary_name) for debugfile in debugfile_list: if os.path.isfile(debugfile): debugfile_crc = self.calc_crc32(debugfile) if int(debuglink_crc,16) == int(debugfile_crc, 16): prelink_offset = self.get_prelink_offset(debugfile, real_binary) return debugfile return real_binary def guess_arch(addr): # Guess which arch we're running. 10 = len('0x') + 8 hex digits. if len(addr) > 10: return 'x86_64' else: return 'i386' class Symbolizer(object): def __init__(self): pass def symbolize(self, addr, binary, offset): """Symbolize the given address (pair of binary and offset). Overriden in subclasses. Args: addr: virtual address of an instruction. binary: path to executable/shared object containing this instruction. offset: instruction offset in the @binary. Returns: list of strings (one string for each inlined frame) describing the code locations for this instruction (that is, function name, file name, line and column numbers). """ return None class LLVMSymbolizer(Symbolizer): def __init__(self, symbolizer_path, default_arch, system, dsym_hints=[]): super(LLVMSymbolizer, self).__init__() self.symbolizer_path = symbolizer_path self.default_arch = default_arch self.system = system self.dsym_hints = dsym_hints self.pipe = self.open_llvm_symbolizer() def open_llvm_symbolizer(self): cmd = [self.symbolizer_path, '--use-symbol-table=true', '--demangle=%s' % demangle, '--functions=short', '--inlining=true', '--default-arch=%s' % self.default_arch] if self.system == 'Darwin': for hint in self.dsym_hints: cmd.append('--dsym-hint=%s' % hint) if DEBUG: print ' '.join(cmd) try: result = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) except OSError: result = None return result def symbolize(self, addr, binary, offset): """Overrides Symbolizer.symbolize.""" if not self.pipe: return None result = [] try: symbolizer_input = '"%s" %s' % (binary, offset) if DEBUG: print symbolizer_input print >> self.pipe.stdin, symbolizer_input while True: function_name = self.pipe.stdout.readline().rstrip() if not function_name: break file_name = self.pipe.stdout.readline().rstrip() file_name = fix_filename(file_name) if (not function_name.startswith('??') or not file_name.startswith('??')): # Append only non-trivial frames. result.append('%s in %s %s' % (addr, function_name, file_name)) except Exception: result = [] if not result: result = None return result def LLVMSymbolizerFactory(system, default_arch, dsym_hints=[]): symbolizer_path = os.getenv('LLVM_SYMBOLIZER_PATH') if not symbolizer_path: symbolizer_path = os.getenv('ASAN_SYMBOLIZER_PATH') if not symbolizer_path: # Assume llvm-symbolizer is in PATH. symbolizer_path = 'llvm-symbolizer' return LLVMSymbolizer(symbolizer_path, default_arch, system, dsym_hints) class Addr2LineSymbolizer(Symbolizer): def __init__(self, binary): super(Addr2LineSymbolizer, self).__init__() self.binary = binary def symbolize(self, addr, binary, offset): """Overrides Symbolizer.symbolize.""" cmd = [use_binutils_prefix('addr2line'), '-fi'] if demangle: cmd += ['--demangle'] cmd += ['-e', self.binary, offset] if DEBUG: print ' '.join(cmd) self.pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE) result = [] if self.binary != binary: return None try: lines = self.pipe.stdout.readlines() except Exception: lines = [] if not lines: lines.append('??') lines.append('??:?') for i in range(0, len(lines), 2): function_name = lines[i].rstrip() file_name = fix_filename(lines[i+1].rstrip()) result.append('%s in %s %s' % (addr, function_name, file_name)) return result class UnbufferedLineConverter(object): """ Wrap a child process that responds to each line of input with one line of output. Uses pty to trick the child into providing unbuffered output. """ def __init__(self, args, close_stderr=False): # Local imports so that the script can start on Windows. import pty import termios pid, fd = pty.fork() if pid == 0: # We're the child. Transfer control to command. if close_stderr: dev_null = os.open('/dev/null', 0) os.dup2(dev_null, 2) os.execvp(args[0], args) else: # Disable echoing. attr = termios.tcgetattr(fd) attr[3] = attr[3] & ~termios.ECHO termios.tcsetattr(fd, termios.TCSANOW, attr) # Set up a file()-like interface to the child process self.r = os.fdopen(fd, "r", 1) self.w = os.fdopen(os.dup(fd), "w", 1) def convert(self, line): self.w.write(line + "\n") return self.readline() def readline(self): return self.r.readline().rstrip() class DarwinSymbolizer(Symbolizer): def __init__(self, addr, binary): super(DarwinSymbolizer, self).__init__() self.binary = binary self.arch = guess_arch(addr) self.open_atos() def open_atos(self): if DEBUG: print 'atos -o %s -arch %s' % (self.binary, self.arch) cmdline = ['atos', '-o', self.binary, '-arch', self.arch] self.atos = UnbufferedLineConverter(cmdline, close_stderr=True) def symbolize(self, addr, binary, offset): """Overrides Symbolizer.symbolize.""" if self.binary != binary: return None atos_line = self.atos.convert('0x%x' % int(offset, 16)) while "got symbolicator for" in atos_line: atos_line = self.atos.readline() # A well-formed atos response looks like this: # foo(type1, type2) (in object.name) (filename.cc:80) match = re.match('^(.*) \(in (.*)\) \((.*:\d*)\)$', atos_line) if DEBUG: print 'atos_line: ', atos_line if match: function_name = match.group(1) function_name = re.sub('\(.*?\)', '', function_name) file_name = fix_filename(match.group(3)) return ['%s in %s %s' % (addr, function_name, file_name)] else: return ['%s in %s' % (addr, atos_line)] # Chain several symbolizers so that if one symbolizer fails, we fall back # to the next symbolizer in chain. class ChainSymbolizer(Symbolizer): def __init__(self, symbolizer_list): super(ChainSymbolizer, self).__init__() self.symbolizer_list = symbolizer_list def symbolize(self, addr, binary, offset): """Overrides Symbolizer.symbolize.""" for symbolizer in self.symbolizer_list: if symbolizer: result = symbolizer.symbolize(addr, binary, offset) if result: return result return None def append_symbolizer(self, symbolizer): self.symbolizer_list.append(symbolizer) def BreakpadSymbolizerFactory(binary): suffix = os.getenv('BREAKPAD_SUFFIX') if suffix: filename = binary + suffix if os.access(filename, os.F_OK): return BreakpadSymbolizer(filename) return None def SystemSymbolizerFactory(system, addr, binary): if system == 'Darwin': return DarwinSymbolizer(addr, binary) elif system == 'Linux': return Addr2LineSymbolizer(binary) class BreakpadSymbolizer(Symbolizer): def __init__(self, filename): super(BreakpadSymbolizer, self).__init__() self.filename = filename lines = file(filename).readlines() self.files = [] self.symbols = {} self.address_list = [] self.addresses = {} # MODULE mac x86_64 A7001116478B33F18FF9BEDE9F615F190 t fragments = lines[0].rstrip().split() self.arch = fragments[2] self.debug_id = fragments[3] self.binary = ' '.join(fragments[4:]) self.parse_lines(lines[1:]) def parse_lines(self, lines): cur_function_addr = '' for line in lines: fragments = line.split() if fragments[0] == 'FILE': assert int(fragments[1]) == len(self.files) self.files.append(' '.join(fragments[2:])) elif fragments[0] == 'PUBLIC': self.symbols[int(fragments[1], 16)] = ' '.join(fragments[3:]) elif fragments[0] in ['CFI', 'STACK']: pass elif fragments[0] == 'FUNC': cur_function_addr = int(fragments[1], 16) if not cur_function_addr in self.symbols.keys(): self.symbols[cur_function_addr] = ' '.join(fragments[4:]) else: # Line starting with an address. addr = int(fragments[0], 16) self.address_list.append(addr) # Tuple of symbol address, size, line, file number. self.addresses[addr] = (cur_function_addr, int(fragments[1], 16), int(fragments[2]), int(fragments[3])) self.address_list.sort() def get_sym_file_line(self, addr): key = None if addr in self.addresses.keys(): key = addr else: index = bisect.bisect_left(self.address_list, addr) if index == 0: return None else: key = self.address_list[index - 1] sym_id, size, line_no, file_no = self.addresses[key] symbol = self.symbols[sym_id] filename = self.files[file_no] if addr < key + size: return symbol, filename, line_no else: return None def symbolize(self, addr, binary, offset): if self.binary != binary: return None res = self.get_sym_file_line(int(offset, 16)) if res: function_name, file_name, line_no = res result = ['%s in %s %s:%d' % ( addr, function_name, file_name, line_no)] print result return result else: return None class SymbolizationLoop(object): def __init__(self, binary_name_filter=None, dsym_hint_producer=None): if sys.platform == 'win32': # ASan on Windows uses dbghelp.dll to symbolize in-process, which works # even in sandboxed processes. Nothing needs to be done here. self.process_line = self.process_line_echo else: # Used by clients who may want to supply a different binary name. # E.g. in Chrome several binaries may share a single .dSYM. self.binary_name_filter = binary_name_filter self.dsym_hint_producer = dsym_hint_producer self.system = os.uname()[0] if self.system not in ['Linux', 'Darwin', 'FreeBSD']: raise Exception('Unknown system') self.llvm_symbolizers = {} self.last_llvm_symbolizer = None self.dsym_hints = set([]) self.frame_no = 0 self.process_line = self.process_line_posix def symbolize_address(self, addr, binary, offset): # On non-Darwin (i.e. on platforms without .dSYM debug info) always use # a single symbolizer binary. # On Darwin, if the dsym hint producer is present: # 1. check whether we've seen this binary already; if so, # use |llvm_symbolizers[binary]|, which has already loaded the debug # info for this binary (might not be the case for # |last_llvm_symbolizer|); # 2. otherwise check if we've seen all the hints for this binary already; # if so, reuse |last_llvm_symbolizer| which has the full set of hints; # 3. otherwise create a new symbolizer and pass all currently known # .dSYM hints to it. if not binary in self.llvm_symbolizers: use_new_symbolizer = True if self.system == 'Darwin' and self.dsym_hint_producer: dsym_hints_for_binary = set(self.dsym_hint_producer(binary)) use_new_symbolizer = bool(dsym_hints_for_binary - self.dsym_hints) self.dsym_hints |= dsym_hints_for_binary if self.last_llvm_symbolizer and not use_new_symbolizer: self.llvm_symbolizers[binary] = self.last_llvm_symbolizer else: self.last_llvm_symbolizer = LLVMSymbolizerFactory( self.system, guess_arch(addr), self.dsym_hints) self.llvm_symbolizers[binary] = self.last_llvm_symbolizer # Use the chain of symbolizers: # Breakpad symbolizer -> LLVM symbolizer -> addr2line/atos # (fall back to next symbolizer if the previous one fails). if not binary in symbolizers: symbolizers[binary] = ChainSymbolizer( [BreakpadSymbolizerFactory(binary), self.llvm_symbolizers[binary]]) result = symbolizers[binary].symbolize(addr, binary, offset) if result is None: # Initialize system symbolizer only if other symbolizers failed. symbolizers[binary].append_symbolizer( SystemSymbolizerFactory(self.system, addr, binary)) result = symbolizers[binary].symbolize(addr, binary, offset) # The system symbolizer must produce some result. assert result return result def get_symbolized_lines(self, symbolized_lines): if not symbolized_lines: return [self.current_line] else: result = [] for symbolized_frame in symbolized_lines: if '?' in symbolized_frame: symbolized_frame += " " + re.search('\(.*?\)',self.current_line).group(0) result.append(' #%s %s' % (str(self.frame_no), symbolized_frame.rstrip())) self.frame_no += 1 return result def process_logfile(self): self.frame_no = 0 for line in logfile: processed = self.process_line(line) print '\n'.join(processed) def process_line_echo(self, line): return [line.rstrip()] def have_line_to_symbolize(self, line): #0 0x7f6e35cf2e45 in func (/blah/foo.so+0x11fe45) stack_trace_line_format = ( '^( *#([0-9]+) *)(0x[0-9a-f]+)( *in [^/]+)? *\((.*)\+(0x[0-9a-f]+)\)') match = re.match(stack_trace_line_format, line) if not match: # If already symbolized (format below) - fix given frame number #0 0x7f6e35cf2e45 in func foo:46 stack_trace_line_format_symbolized = ( '^( *#([0-9]+) *)(0x[0-9a-f]+)( *in [^/]+)? *(.*)\:([0-9]+)') match_symbolized = re.match(stack_trace_line_format_symbolized, line); if match_symbolized: # Frame number from line match_frame_number = match_symbolized.group(2) if (self.frame_no != int(match_frame_number)): self.current_line = re.sub(match_frame_number, str(self.frame_no), line, count=1) self.frame_no += 1 return match def process_line_posix(self, line): self.current_line = line.rstrip() match = self.have_line_to_symbolize(line) if not match: return [self.current_line] if DEBUG: print line _, frameno_str, addr, func, binary, offset = match.groups() if frameno_str == '0': # Assume that frame #0 is the first frame of new stack trace. self.frame_no = 0 original_binary = binary if self.binary_name_filter: binary = self.binary_name_filter(binary) # Correct offset from backtrace if the binary was prelinked # and printed address considers the prelink offset: if prelink_offset: real_offset = int(offset,16) if real_offset > prelink_offset: #FIXME: Need to check that offset fits section size offset = hex(real_offset - prelink_offset) if DEBUG: print 'real address: ' + offset symbolized_line = self.symbolize_address(addr, binary, offset) if not symbolized_line: if original_binary != binary: symbolized_line = self.symbolize_address(addr, binary, offset) return self.get_symbolized_lines(symbolized_line) if __name__ == '__main__': parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description='ASan symbolization script', epilog='Example of use:\n' 'asan_symbolize.py -c "$HOME/opt/cross/bin/armv7l-tizen-linux-gnueabi-" ' '-s "$HOME/SymbolFiles" < asan.log') parser.add_argument('path_to_cut', nargs='*', help='pattern to be cut from the result file path ') parser.add_argument('-d','--demangle', action='store_true', help='demangle function names') parser.add_argument('-s', metavar='SYSROOT', help='set path to sysroot for sanitized binaries') parser.add_argument('-c', metavar='CROSS_COMPILE', help='set prefix for binutils') parser.add_argument('-l','--logfile', default=sys.stdin, type=argparse.FileType('r'), help='set log file name to parse, default is stdin') parser.add_argument('-y', '--debug-file-directory', metavar='DEBUGDIR', help='The directories for separate debug information \ files. Multiple path components can be set concatenating \ them by a path separator.') args = parser.parse_args() if args.path_to_cut: fix_filename_patterns = args.path_to_cut if args.demangle: demangle = True if args.s: binary_name_filter = sysroot_path_filter sysroot_path = args.s if args.c: binutils_prefix = args.c if args.logfile: logfile = args.logfile else: logfile = sys.stdin if args.debug_file_directory: separate_debug_dir_list = args.debug_file_directory.split(":") if os.uname()[0] == 'Linux': debug_info_handler = DebugInfoHandler(binary_name_filter) binary_name_filter = debug_info_handler.get_debuginfo loop = SymbolizationLoop(binary_name_filter) loop.process_logfile()