convert_dex_profile.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569
  1. #!/usr/bin/env vpython3
  2. #
  3. # Copyright 2018 The Chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. import argparse
  7. import collections
  8. import functools
  9. import logging
  10. import re
  11. import subprocess
  12. import sys
  13. DEX_CLASS_NAME_RE = re.compile(r'\'L(?P<class_name>[^;]+);\'')
  14. DEX_METHOD_NAME_RE = re.compile(r'\'(?P<method_name>[^\']+)\'')
  15. DEX_METHOD_TYPE_RE = re.compile( # type descriptor method signature re
  16. r'\''
  17. r'\('
  18. r'(?P<method_params>[^)]*)'
  19. r'\)'
  20. r'(?P<method_return_type>[^\']+)'
  21. r'\'')
  22. DEX_METHOD_LINE_NR_RE = re.compile(r'line=(?P<line_number>\d+)')
  23. PROFILE_METHOD_RE = re.compile(
  24. r'(?P<tags>[HSP]+)' # tags such as H/S/P
  25. r'(?P<class_name>L[^;]+;)' # class name in type descriptor format
  26. r'->(?P<method_name>[^(]+)'
  27. r'\((?P<method_params>[^)]*)\)'
  28. r'(?P<method_return_type>.+)')
  29. PROGUARD_CLASS_MAPPING_RE = re.compile(
  30. r'(?P<original_name>[^ ]+)'
  31. r' -> '
  32. r'(?P<obfuscated_name>[^:]+):')
  33. PROGUARD_METHOD_MAPPING_RE = re.compile(
  34. # line_start:line_end: (optional)
  35. r'((?P<line_start>\d+):(?P<line_end>\d+):)?'
  36. r'(?P<return_type>[^ ]+)' # original method return type
  37. # original method class name (if exists)
  38. r' (?:(?P<original_method_class>[a-zA-Z_\d.$]+)\.)?'
  39. r'(?P<original_method_name>[^.\(]+)'
  40. r'\((?P<params>[^\)]*)\)' # original method params
  41. r'(?:[^ ]*)' # original method line numbers (ignored)
  42. r' -> '
  43. r'(?P<obfuscated_name>.+)') # obfuscated method name
  44. TYPE_DESCRIPTOR_RE = re.compile(
  45. r'(?P<brackets>\[*)'
  46. r'(?:'
  47. r'(?P<class_name>L[^;]+;)'
  48. r'|'
  49. r'[VZBSCIJFD]'
  50. r')')
  51. DOT_NOTATION_MAP = {
  52. '': '',
  53. 'boolean': 'Z',
  54. 'byte': 'B',
  55. 'void': 'V',
  56. 'short': 'S',
  57. 'char': 'C',
  58. 'int': 'I',
  59. 'long': 'J',
  60. 'float': 'F',
  61. 'double': 'D'
  62. }
  63. @functools.total_ordering
  64. class Method:
  65. def __init__(self, name, class_name, param_types=None, return_type=None):
  66. self.name = name
  67. self.class_name = class_name
  68. self.param_types = param_types
  69. self.return_type = return_type
  70. def __str__(self):
  71. return '{}->{}({}){}'.format(self.class_name, self.name,
  72. self.param_types or '', self.return_type or '')
  73. def __repr__(self):
  74. return 'Method<{}->{}({}){}>'.format(self.class_name, self.name,
  75. self.param_types or '', self.return_type or '')
  76. @staticmethod
  77. def serialize(method):
  78. return (method.class_name, method.name, method.param_types,
  79. method.return_type)
  80. def __eq__(self, other):
  81. return self.serialize(self) == self.serialize(other)
  82. def __lt__(self, other):
  83. return self.serialize(self) < self.serialize(other)
  84. def __hash__(self):
  85. # only hash name and class_name since other fields may not be set yet.
  86. return hash((self.name, self.class_name))
  87. class Class:
  88. def __init__(self, name):
  89. self.name = name
  90. self._methods = []
  91. def AddMethod(self, method, line_numbers):
  92. self._methods.append((method, set(line_numbers)))
  93. def FindMethodsAtLine(self, method_name, line_start, line_end=None):
  94. """Searches through dex class for a method given a name and line numbers
  95. The dex maps methods to line numbers, this method, given the a method name
  96. in this class as well as a start line and an optional end line (which act as
  97. hints as to which function in the class is being looked for), returns a list
  98. of possible matches (or none if none are found).
  99. Args:
  100. method_name: name of method being searched for
  101. line_start: start of hint range for lines in this method
  102. line_end: end of hint range for lines in this method (optional)
  103. Returns:
  104. A list of Method objects that could match the hints given, or None if no
  105. method is found.
  106. """
  107. found_methods = []
  108. if line_end is None:
  109. hint_lines = set([line_start])
  110. else:
  111. hint_lines = set(range(line_start, line_end+1))
  112. named_methods = [(method, l) for method, l in self._methods
  113. if method.name == method_name]
  114. if len(named_methods) == 1:
  115. return [method for method, l in named_methods]
  116. if len(named_methods) == 0:
  117. return None
  118. for method, line_numbers in named_methods:
  119. if not hint_lines.isdisjoint(line_numbers):
  120. found_methods.append(method)
  121. if len(found_methods) > 0:
  122. if len(found_methods) > 1:
  123. logging.warning('ambigous methods in dex %s at lines %s in class "%s"',
  124. found_methods, hint_lines, self.name)
  125. return found_methods
  126. for method, line_numbers in named_methods:
  127. if (max(hint_lines) >= min(line_numbers)
  128. and min(hint_lines) <= max(line_numbers)):
  129. found_methods.append(method)
  130. if len(found_methods) > 0:
  131. if len(found_methods) > 1:
  132. logging.warning('ambigous methods in dex %s at lines %s in class "%s"',
  133. found_methods, hint_lines, self.name)
  134. return found_methods
  135. logging.warning(
  136. 'No method named "%s" in class "%s" is '
  137. 'mapped to lines %s', method_name, self.name, hint_lines)
  138. return None
  139. class Profile:
  140. def __init__(self):
  141. # {Method: set(char)}
  142. self._methods = collections.defaultdict(set)
  143. self._classes = []
  144. def AddMethod(self, method, tags):
  145. for tag in tags:
  146. self._methods[method].add(tag)
  147. def AddClass(self, cls):
  148. self._classes.append(cls)
  149. def WriteToFile(self, path):
  150. with open(path, 'w') as output_profile:
  151. for cls in sorted(self._classes):
  152. output_profile.write(cls + '\n')
  153. for method in sorted(self._methods):
  154. tags = sorted(self._methods[method])
  155. line = '{}{}\n'.format(''.join(tags), str(method))
  156. output_profile.write(line)
  157. class ProguardMapping:
  158. def __init__(self):
  159. # {Method: set(Method)}
  160. self._method_mapping = collections.defaultdict(set)
  161. # {String: String} String is class name in type descriptor format
  162. self._class_mapping = dict()
  163. def AddMethodMapping(self, from_method, to_method):
  164. self._method_mapping[from_method].add(to_method)
  165. def AddClassMapping(self, from_class, to_class):
  166. self._class_mapping[from_class] = to_class
  167. def GetMethodMapping(self, from_method):
  168. return self._method_mapping.get(from_method)
  169. def GetClassMapping(self, from_class):
  170. return self._class_mapping.get(from_class, from_class)
  171. def MapTypeDescriptor(self, type_descriptor):
  172. match = TYPE_DESCRIPTOR_RE.search(type_descriptor)
  173. assert match is not None
  174. class_name = match.group('class_name')
  175. if class_name is not None:
  176. return match.group('brackets') + self.GetClassMapping(class_name)
  177. # just a native type, return as is
  178. return match.group()
  179. def MapTypeDescriptorList(self, type_descriptor_list):
  180. return TYPE_DESCRIPTOR_RE.sub(
  181. lambda match: self.MapTypeDescriptor(match.group()),
  182. type_descriptor_list)
  183. class MalformedLineException(Exception):
  184. def __init__(self, message, line_number):
  185. super().__init__(message)
  186. self.message = message
  187. self.line_number = line_number
  188. def __str__(self):
  189. return self.message + ' at line {}'.format(self.line_number)
  190. class MalformedProguardMappingException(MalformedLineException):
  191. pass
  192. class MalformedProfileException(MalformedLineException):
  193. pass
  194. def _RunDexDump(dexdump_path, dex_file_path):
  195. return subprocess.check_output([dexdump_path,
  196. dex_file_path]).decode('utf-8').splitlines()
  197. def _ReadFile(file_path):
  198. with open(file_path, 'r') as f:
  199. return f.readlines()
  200. def _ToTypeDescriptor(dot_notation):
  201. """Parses a dot notation type and returns it in type descriptor format
  202. eg:
  203. org.chromium.browser.ChromeActivity -> Lorg/chromium/browser/ChromeActivity;
  204. boolean -> Z
  205. int[] -> [I
  206. Args:
  207. dot_notation: trimmed string with a single type in dot notation format
  208. Returns:
  209. A string with the type in type descriptor format
  210. """
  211. dot_notation = dot_notation.strip()
  212. prefix = ''
  213. while dot_notation.endswith('[]'):
  214. prefix += '['
  215. dot_notation = dot_notation[:-2]
  216. if dot_notation in DOT_NOTATION_MAP:
  217. return prefix + DOT_NOTATION_MAP[dot_notation]
  218. return prefix + 'L' + dot_notation.replace('.', '/') + ';'
  219. def _DotNotationListToTypeDescriptorList(dot_notation_list_string):
  220. """Parses a param list of dot notation format and returns it in type
  221. descriptor format
  222. eg:
  223. org.chromium.browser.ChromeActivity,boolean,int[] ->
  224. Lorg/chromium/browser/ChromeActivity;Z[I
  225. Args:
  226. dot_notation_list_string: single string with multiple comma separated types
  227. in dot notation format
  228. Returns:
  229. A string with the param list in type descriptor format
  230. """
  231. return ''.join(_ToTypeDescriptor(param) for param in
  232. dot_notation_list_string.split(','))
  233. def ProcessDex(dex_dump):
  234. """Parses dexdump output returning a dict of class names to Class objects
  235. Parses output of the dexdump command on a dex file and extracts information
  236. about classes and their respective methods and which line numbers a method is
  237. mapped to.
  238. Methods that are not mapped to any line number are ignored and not listed
  239. inside their respective Class objects.
  240. Args:
  241. dex_dump: An array of lines of dexdump output
  242. Returns:
  243. A dict that maps from class names in type descriptor format (but without the
  244. surrounding 'L' and ';') to Class objects.
  245. """
  246. # class_name: Class
  247. classes_by_name = {}
  248. current_class = None
  249. current_method = None
  250. reading_positions = False
  251. reading_methods = False
  252. method_line_numbers = []
  253. for line in dex_dump:
  254. line = line.strip()
  255. if line.startswith('Class descriptor'):
  256. # New class started, no longer reading methods.
  257. reading_methods = False
  258. current_class = Class(DEX_CLASS_NAME_RE.search(line).group('class_name'))
  259. classes_by_name[current_class.name] = current_class
  260. elif (line.startswith('Direct methods')
  261. or line.startswith('Virtual methods')):
  262. reading_methods = True
  263. elif reading_methods and line.startswith('name'):
  264. assert current_class is not None
  265. current_method = Method(
  266. DEX_METHOD_NAME_RE.search(line).group('method_name'),
  267. "L" + current_class.name + ";")
  268. elif reading_methods and line.startswith('type'):
  269. assert current_method is not None
  270. match = DEX_METHOD_TYPE_RE.search(line)
  271. current_method.param_types = match.group('method_params')
  272. current_method.return_type = match.group('method_return_type')
  273. elif line.startswith('positions'):
  274. assert reading_methods
  275. reading_positions = True
  276. method_line_numbers = []
  277. elif reading_positions and line.startswith('0x'):
  278. line_number = DEX_METHOD_LINE_NR_RE.search(line).group('line_number')
  279. method_line_numbers.append(int(line_number))
  280. elif reading_positions and line.startswith('locals'):
  281. if len(method_line_numbers) > 0:
  282. current_class.AddMethod(current_method, method_line_numbers)
  283. # finished reading method line numbers
  284. reading_positions = False
  285. return classes_by_name
  286. def ProcessProguardMapping(proguard_mapping_lines, dex):
  287. """Parses a proguard mapping file
  288. This takes proguard mapping file lines and then uses the obfuscated dex to
  289. create a mapping of unobfuscated methods to obfuscated ones and vice versa.
  290. The dex is used because the proguard mapping file only has the name of the
  291. obfuscated methods but not their signature, thus the dex is read to look up
  292. which method with a specific name was mapped to the lines mentioned in the
  293. proguard mapping file.
  294. Args:
  295. proguard_mapping_lines: Array of strings, each is a line from the proguard
  296. mapping file (in order).
  297. dex: a dict of class name (in type descriptor format but without the
  298. enclosing 'L' and ';') to a Class object.
  299. Returns:
  300. Two dicts the first maps from obfuscated methods to a set of non-obfuscated
  301. ones. It also maps the obfuscated class names to original class names, both
  302. in type descriptor format (with the enclosing 'L' and ';')
  303. """
  304. mapping = ProguardMapping()
  305. reverse_mapping = ProguardMapping()
  306. to_be_obfuscated = []
  307. current_class_orig = None
  308. current_class_obfs = None
  309. for index, line in enumerate(proguard_mapping_lines):
  310. if line.strip() == '':
  311. continue
  312. if not line.startswith(' '):
  313. match = PROGUARD_CLASS_MAPPING_RE.search(line)
  314. if match is None:
  315. raise MalformedProguardMappingException(
  316. 'Malformed class mapping', index)
  317. current_class_orig = match.group('original_name')
  318. current_class_obfs = match.group('obfuscated_name')
  319. mapping.AddClassMapping(_ToTypeDescriptor(current_class_obfs),
  320. _ToTypeDescriptor(current_class_orig))
  321. reverse_mapping.AddClassMapping(_ToTypeDescriptor(current_class_orig),
  322. _ToTypeDescriptor(current_class_obfs))
  323. continue
  324. assert current_class_orig is not None
  325. assert current_class_obfs is not None
  326. line = line.strip()
  327. match = PROGUARD_METHOD_MAPPING_RE.search(line)
  328. # check if is a method mapping (we ignore field mappings)
  329. if match is not None:
  330. # check if this line is an inlining by reading ahead 1 line.
  331. if index + 1 < len(proguard_mapping_lines):
  332. next_match = PROGUARD_METHOD_MAPPING_RE.search(
  333. proguard_mapping_lines[index+1].strip())
  334. if (next_match and match.group('line_start') is not None
  335. and next_match.group('line_start') == match.group('line_start')
  336. and next_match.group('line_end') == match.group('line_end')):
  337. continue # This is an inlining, skip
  338. original_method = Method(
  339. match.group('original_method_name'),
  340. _ToTypeDescriptor(
  341. match.group('original_method_class') or current_class_orig),
  342. _DotNotationListToTypeDescriptorList(match.group('params')),
  343. _ToTypeDescriptor(match.group('return_type')))
  344. if match.group('line_start') is not None:
  345. obfs_methods = (dex[current_class_obfs.replace('.', '/')]
  346. .FindMethodsAtLine(
  347. match.group('obfuscated_name'),
  348. int(match.group('line_start')),
  349. int(match.group('line_end'))))
  350. if obfs_methods is None:
  351. continue
  352. for obfs_method in obfs_methods:
  353. mapping.AddMethodMapping(obfs_method, original_method)
  354. reverse_mapping.AddMethodMapping(original_method, obfs_method)
  355. else:
  356. to_be_obfuscated.append(
  357. (original_method, match.group('obfuscated_name')))
  358. for original_method, obfuscated_name in to_be_obfuscated:
  359. obfuscated_method = Method(
  360. obfuscated_name,
  361. reverse_mapping.GetClassMapping(original_method.class_name),
  362. reverse_mapping.MapTypeDescriptorList(original_method.param_types),
  363. reverse_mapping.MapTypeDescriptor(original_method.return_type))
  364. mapping.AddMethodMapping(obfuscated_method, original_method)
  365. reverse_mapping.AddMethodMapping(original_method, obfuscated_method)
  366. return mapping, reverse_mapping
  367. def ProcessProfile(input_profile, proguard_mapping):
  368. """Parses an android profile and uses the proguard mapping to (de)obfuscate it
  369. This takes the android profile lines and for each method or class for the
  370. profile, it uses the mapping to either obfuscate or deobfuscate (based on the
  371. provided mapping) and returns a Profile object that stores this information.
  372. Args:
  373. input_profile: array of lines of the input profile
  374. proguard_mapping: a proguard mapping that would map from the classes and
  375. methods in the input profile to the classes and methods
  376. that should be in the output profile.
  377. Returns:
  378. A Profile object that stores the information (ie list of mapped classes and
  379. methods + tags)
  380. """
  381. profile = Profile()
  382. for index, line in enumerate(input_profile):
  383. line = line.strip()
  384. if line.startswith('L'):
  385. profile.AddClass(proguard_mapping.GetClassMapping(line))
  386. continue
  387. match = PROFILE_METHOD_RE.search(line)
  388. if not match:
  389. raise MalformedProfileException("Malformed line", index)
  390. method = Method(
  391. match.group('method_name'),
  392. match.group('class_name'),
  393. match.group('method_params'),
  394. match.group('method_return_type'))
  395. mapped_methods = proguard_mapping.GetMethodMapping(method)
  396. if mapped_methods is None:
  397. logging.warning('No method matching "%s" has been found in the proguard '
  398. 'mapping file', method)
  399. continue
  400. for original_method in mapped_methods:
  401. profile.AddMethod(original_method, match.group('tags'))
  402. return profile
  403. def ObfuscateProfile(nonobfuscated_profile, dex_file, proguard_mapping,
  404. dexdump_path, output_filename):
  405. """Helper method for obfuscating a profile.
  406. Args:
  407. nonobfuscated_profile: a profile with nonobfuscated symbols.
  408. dex_file: path to the dex file matching the mapping.
  409. proguard_mapping: a mapping from nonobfuscated to obfuscated symbols used
  410. in the dex file.
  411. dexdump_path: path to the dexdump utility.
  412. output_filename: output filename in which to write the obfuscated profile.
  413. """
  414. dexinfo = ProcessDex(_RunDexDump(dexdump_path, dex_file))
  415. _, reverse_mapping = ProcessProguardMapping(
  416. _ReadFile(proguard_mapping), dexinfo)
  417. obfuscated_profile = ProcessProfile(
  418. _ReadFile(nonobfuscated_profile), reverse_mapping)
  419. obfuscated_profile.WriteToFile(output_filename)
  420. def main(args):
  421. parser = argparse.ArgumentParser()
  422. parser.add_argument(
  423. '--dexdump-path',
  424. required=True,
  425. help='Path to dexdump binary.')
  426. parser.add_argument(
  427. '--dex-path',
  428. required=True,
  429. help='Path to dex file corresponding to the proguard mapping file.')
  430. parser.add_argument(
  431. '--proguard-mapping-path',
  432. required=True,
  433. help='Path to input proguard mapping file corresponding to the dex file.')
  434. parser.add_argument(
  435. '--output-profile-path',
  436. required=True,
  437. help='Path to output profile.')
  438. parser.add_argument(
  439. '--input-profile-path',
  440. required=True,
  441. help='Path to output profile.')
  442. parser.add_argument(
  443. '--verbose',
  444. action='store_true',
  445. default=False,
  446. help='Print verbose output.')
  447. obfuscation = parser.add_mutually_exclusive_group(required=True)
  448. obfuscation.add_argument('--obfuscate', action='store_true',
  449. help='Indicates to output an obfuscated profile given a deobfuscated '
  450. 'one.')
  451. obfuscation.add_argument('--deobfuscate', dest='obfuscate',
  452. action='store_false', help='Indicates to output a deobfuscated profile '
  453. 'given an obfuscated one.')
  454. options = parser.parse_args(args)
  455. if options.verbose:
  456. log_level = logging.WARNING
  457. else:
  458. log_level = logging.ERROR
  459. logging.basicConfig(format='%(levelname)s: %(message)s', level=log_level)
  460. dex = ProcessDex(_RunDexDump(options.dexdump_path, options.dex_path))
  461. proguard_mapping, reverse_proguard_mapping = ProcessProguardMapping(
  462. _ReadFile(options.proguard_mapping_path), dex)
  463. if options.obfuscate:
  464. profile = ProcessProfile(
  465. _ReadFile(options.input_profile_path),
  466. reverse_proguard_mapping)
  467. else:
  468. profile = ProcessProfile(
  469. _ReadFile(options.input_profile_path),
  470. proguard_mapping)
  471. profile.WriteToFile(options.output_profile_path)
  472. if __name__ == '__main__':
  473. main(sys.argv[1:])