tombstones.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
  1. #!/usr/bin/env vpython3
  2. #
  3. # Copyright 2013 The Chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. #
  7. # Find the most recent tombstone file(s) on all connected devices
  8. # and prints their stacks.
  9. #
  10. # Assumes tombstone file was created with current symbols.
  11. import argparse
  12. import datetime
  13. import logging
  14. import os
  15. import sys
  16. from multiprocessing.pool import ThreadPool
  17. import devil_chromium
  18. from devil.android import device_denylist
  19. from devil.android import device_errors
  20. from devil.android import device_utils
  21. from devil.utils import run_tests_helper
  22. from pylib import constants
  23. from pylib.symbols import stack_symbolizer
  24. _TZ_UTC = {'TZ': 'UTC'}
  25. def _ListTombstones(device):
  26. """List the tombstone files on the device.
  27. Args:
  28. device: An instance of DeviceUtils.
  29. Yields:
  30. Tuples of (tombstone filename, date time of file on device).
  31. """
  32. try:
  33. if not device.PathExists('/data/tombstones', as_root=True):
  34. return
  35. entries = device.StatDirectory('/data/tombstones', as_root=True)
  36. for entry in entries:
  37. if 'tombstone' in entry['filename']:
  38. yield (entry['filename'],
  39. datetime.datetime.fromtimestamp(entry['st_mtime']))
  40. except device_errors.CommandFailedError:
  41. logging.exception('Could not retrieve tombstones.')
  42. except device_errors.DeviceUnreachableError:
  43. logging.exception('Device unreachable retrieving tombstones.')
  44. except device_errors.CommandTimeoutError:
  45. logging.exception('Timed out retrieving tombstones.')
  46. def _GetDeviceDateTime(device):
  47. """Determine the date time on the device.
  48. Args:
  49. device: An instance of DeviceUtils.
  50. Returns:
  51. A datetime instance.
  52. """
  53. device_now_string = device.RunShellCommand(
  54. ['date'], check_return=True, env=_TZ_UTC)
  55. return datetime.datetime.strptime(
  56. device_now_string[0], '%a %b %d %H:%M:%S %Z %Y')
  57. def _GetTombstoneData(device, tombstone_file):
  58. """Retrieve the tombstone data from the device
  59. Args:
  60. device: An instance of DeviceUtils.
  61. tombstone_file: the tombstone to retrieve
  62. Returns:
  63. A list of lines
  64. """
  65. return device.ReadFile(
  66. '/data/tombstones/' + tombstone_file, as_root=True).splitlines()
  67. def _EraseTombstone(device, tombstone_file):
  68. """Deletes a tombstone from the device.
  69. Args:
  70. device: An instance of DeviceUtils.
  71. tombstone_file: the tombstone to delete.
  72. """
  73. return device.RunShellCommand(
  74. ['rm', '/data/tombstones/' + tombstone_file],
  75. as_root=True, check_return=True)
  76. def _ResolveTombstone(args):
  77. tombstone = args[0]
  78. tombstone_symbolizer = args[1]
  79. lines = []
  80. lines += [tombstone['file'] + ' created on ' + str(tombstone['time']) +
  81. ', about this long ago: ' +
  82. (str(tombstone['device_now'] - tombstone['time']) +
  83. ' Device: ' + tombstone['serial'])]
  84. logging.info('\n'.join(lines))
  85. logging.info('Resolving...')
  86. lines += tombstone_symbolizer.ExtractAndResolveNativeStackTraces(
  87. tombstone['data'],
  88. tombstone['device_abi'],
  89. tombstone['stack'])
  90. return lines
  91. def _ResolveTombstones(jobs, tombstones, tombstone_symbolizer):
  92. """Resolve a list of tombstones.
  93. Args:
  94. jobs: the number of jobs to use with multithread.
  95. tombstones: a list of tombstones.
  96. """
  97. if not tombstones:
  98. logging.warning('No tombstones to resolve.')
  99. return []
  100. if len(tombstones) == 1:
  101. data = [_ResolveTombstone([tombstones[0], tombstone_symbolizer])]
  102. else:
  103. pool = ThreadPool(jobs)
  104. data = pool.map(
  105. _ResolveTombstone,
  106. [[tombstone, tombstone_symbolizer] for tombstone in tombstones])
  107. pool.close()
  108. pool.join()
  109. resolved_tombstones = []
  110. for tombstone in data:
  111. resolved_tombstones.extend(tombstone)
  112. return resolved_tombstones
  113. def _GetTombstonesForDevice(device, resolve_all_tombstones,
  114. include_stack_symbols,
  115. wipe_tombstones):
  116. """Returns a list of tombstones on a given device.
  117. Args:
  118. device: An instance of DeviceUtils.
  119. resolve_all_tombstone: Whether to resolve every tombstone.
  120. include_stack_symbols: Whether to include symbols for stack data.
  121. wipe_tombstones: Whether to wipe tombstones.
  122. """
  123. ret = []
  124. all_tombstones = list(_ListTombstones(device))
  125. if not all_tombstones:
  126. logging.warning('No tombstones.')
  127. return ret
  128. # Sort the tombstones in date order, descending
  129. all_tombstones.sort(key=lambda a: a[1], reverse=True)
  130. # Only resolve the most recent unless --all-tombstones given.
  131. tombstones = all_tombstones if resolve_all_tombstones else [all_tombstones[0]]
  132. device_now = _GetDeviceDateTime(device)
  133. try:
  134. for tombstone_file, tombstone_time in tombstones:
  135. ret += [{'serial': str(device),
  136. 'device_abi': device.product_cpu_abi,
  137. 'device_now': device_now,
  138. 'time': tombstone_time,
  139. 'file': tombstone_file,
  140. 'stack': include_stack_symbols,
  141. 'data': _GetTombstoneData(device, tombstone_file)}]
  142. except device_errors.CommandFailedError:
  143. for entry in device.StatDirectory(
  144. '/data/tombstones', as_root=True, timeout=60):
  145. logging.info('%s: %s', str(device), entry)
  146. raise
  147. # Erase all the tombstones if desired.
  148. if wipe_tombstones:
  149. for tombstone_file, _ in all_tombstones:
  150. _EraseTombstone(device, tombstone_file)
  151. return ret
  152. def ClearAllTombstones(device):
  153. """Clear all tombstones in the device.
  154. Args:
  155. device: An instance of DeviceUtils.
  156. """
  157. all_tombstones = list(_ListTombstones(device))
  158. if not all_tombstones:
  159. logging.warning('No tombstones to clear.')
  160. for tombstone_file, _ in all_tombstones:
  161. _EraseTombstone(device, tombstone_file)
  162. def ResolveTombstones(device, resolve_all_tombstones, include_stack_symbols,
  163. wipe_tombstones, jobs=4, apk_under_test=None,
  164. tombstone_symbolizer=None):
  165. """Resolve tombstones in the device.
  166. Args:
  167. device: An instance of DeviceUtils.
  168. resolve_all_tombstone: Whether to resolve every tombstone.
  169. include_stack_symbols: Whether to include symbols for stack data.
  170. wipe_tombstones: Whether to wipe tombstones.
  171. jobs: Number of jobs to use when processing multiple crash stacks.
  172. Returns:
  173. A list of resolved tombstones.
  174. """
  175. return _ResolveTombstones(jobs,
  176. _GetTombstonesForDevice(device,
  177. resolve_all_tombstones,
  178. include_stack_symbols,
  179. wipe_tombstones),
  180. (tombstone_symbolizer
  181. or stack_symbolizer.Symbolizer(apk_under_test)))
  182. def main():
  183. custom_handler = logging.StreamHandler(sys.stdout)
  184. custom_handler.setFormatter(run_tests_helper.CustomFormatter())
  185. logging.getLogger().addHandler(custom_handler)
  186. logging.getLogger().setLevel(logging.INFO)
  187. parser = argparse.ArgumentParser()
  188. parser.add_argument('--device',
  189. help='The serial number of the device. If not specified '
  190. 'will use all devices.')
  191. parser.add_argument('--denylist-file', help='Device denylist JSON file.')
  192. parser.add_argument('-a', '--all-tombstones', action='store_true',
  193. help='Resolve symbols for all tombstones, rather than '
  194. 'just the most recent.')
  195. parser.add_argument('-s', '--stack', action='store_true',
  196. help='Also include symbols for stack data')
  197. parser.add_argument('-w', '--wipe-tombstones', action='store_true',
  198. help='Erase all tombstones from device after processing')
  199. parser.add_argument('-j', '--jobs', type=int,
  200. default=4,
  201. help='Number of jobs to use when processing multiple '
  202. 'crash stacks.')
  203. parser.add_argument('--output-directory',
  204. help='Path to the root build directory.')
  205. parser.add_argument('--adb-path', type=os.path.abspath,
  206. help='Path to the adb binary.')
  207. args = parser.parse_args()
  208. if args.output_directory:
  209. constants.SetOutputDirectory(args.output_directory)
  210. devil_chromium.Initialize(output_directory=constants.GetOutDirectory(),
  211. adb_path=args.adb_path)
  212. denylist = (device_denylist.Denylist(args.denylist_file)
  213. if args.denylist_file else None)
  214. if args.device:
  215. devices = [device_utils.DeviceUtils(args.device)]
  216. else:
  217. devices = device_utils.DeviceUtils.HealthyDevices(denylist)
  218. # This must be done serially because strptime can hit a race condition if
  219. # used for the first time in a multithreaded environment.
  220. # http://bugs.python.org/issue7980
  221. for device in devices:
  222. resolved_tombstones = ResolveTombstones(
  223. device, args.all_tombstones,
  224. args.stack, args.wipe_tombstones, args.jobs)
  225. for line in resolved_tombstones:
  226. logging.info(line)
  227. if __name__ == '__main__':
  228. sys.exit(main())