trybot_commit_size_checker.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489
  1. #!/usr/bin/env python3
  2. # Copyright 2018 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Creates several files used by the size trybot to monitor size regressions.
  6. To test locally:
  7. 1. Run diagnose_bloat.py to create some entries in out/binary-size-results
  8. 2. Run this script with:
  9. HASH1=some hash within out/binary-size-results
  10. HASH2=some hash within out/binary-size-results
  11. mkdir tmp
  12. tools/binary_size/trybot_commit_size_checker.py \
  13. --author Batman \
  14. --review-subject "Testing 123" \
  15. --review-url "https://google.com" \
  16. --size-config-json-name \
  17. out/binary-size-build/config/Trichrome_size_config.json \
  18. --before-dir out/binary-size-results/$HASH1 \
  19. --after-dir out/binary-size-results/$HASH2 \
  20. --results-path output.json \
  21. --staging-dir tmp \
  22. --local-test \
  23. -v
  24. """
  25. import argparse
  26. import collections
  27. import json
  28. import logging
  29. import os
  30. import pathlib
  31. import re
  32. import sys
  33. sys.path.append(str(pathlib.Path(__file__).parent / 'libsupersize'))
  34. import archive
  35. import diagnose_bloat
  36. import diff
  37. import describe
  38. import dex_disassembly
  39. import file_format
  40. import models
  41. import native_disassembly
  42. _RESOURCE_SIZES_LOG = 'resource_sizes_log'
  43. _BASE_RESOURCE_SIZES_LOG = 'base_resource_sizes_log'
  44. _MUTABLE_CONSTANTS_LOG = 'mutable_contstants_log'
  45. _FOR_TESTING_LOG = 'for_test_log'
  46. _DEX_SYMBOLS_LOG = 'dex_symbols_log'
  47. _SIZEDIFF_FILENAME = 'supersize_diff.sizediff'
  48. _HTML_REPORT_URL = (
  49. 'https://chrome-supersize.firebaseapp.com/viewer.html?load_url={{' +
  50. _SIZEDIFF_FILENAME + '}}')
  51. _MAX_DEX_METHOD_COUNT_INCREASE = 50
  52. _MAX_NORMALIZED_INCREASE = 16 * 1024
  53. _MAX_PAK_INCREASE = 1024
  54. _PROGUARD_CLASS_MAPPING_RE = re.compile(r'(?P<original_name>[^ ]+)'
  55. r' -> '
  56. r'(?P<obfuscated_name>[^:]+):')
  57. _PROGUARD_FIELD_MAPPING_RE = re.compile(r'(?P<type>[^ ]+) '
  58. r'(?P<original_name>[^ (]+)'
  59. r' -> '
  60. r'(?P<obfuscated_name>[^:]+)')
  61. _PROGUARD_METHOD_MAPPING_RE = re.compile(
  62. # line_start:line_end: (optional)
  63. r'((?P<line_start>\d+):(?P<line_end>\d+):)?'
  64. r'(?P<return_type>[^ ]+)' # original method return type
  65. # original method class name (if exists)
  66. r' (?:(?P<original_method_class>[a-zA-Z_\d.$]+)\.)?'
  67. r'(?P<original_method_name>[^.\(]+)'
  68. r'\((?P<params>[^\)]*)\)' # original method params
  69. r'(?:[^ ]*)' # original method line numbers (ignored)
  70. r' -> '
  71. r'(?P<obfuscated_name>.+)') # obfuscated method name
  72. class _SizeDelta(collections.namedtuple(
  73. 'SizeDelta', ['name', 'units', 'expected', 'actual'])):
  74. @property
  75. def explanation(self):
  76. ret = '{}: {} {} (max is {} {})'.format(
  77. self.name, self.actual, self.units, self.expected, self.units)
  78. return ret
  79. def IsAllowable(self):
  80. return self.actual <= self.expected
  81. def IsLargeImprovement(self):
  82. return (self.actual * -1) >= self.expected
  83. def __lt__(self, other):
  84. return self.name < other.name
  85. def _SymbolDiffHelper(title_fragment, symbols):
  86. added = symbols.WhereDiffStatusIs(models.DIFF_STATUS_ADDED)
  87. removed = symbols.WhereDiffStatusIs(models.DIFF_STATUS_REMOVED)
  88. both = (added + removed).SortedByName()
  89. lines = []
  90. if len(both) > 0:
  91. for group in both.GroupedByContainer():
  92. counts = group.CountsByDiffStatus()
  93. lines += [
  94. '===== {} Added & Removed ({}) ====='.format(
  95. title_fragment, group.full_name),
  96. 'Added: {}'.format(counts[models.DIFF_STATUS_ADDED]),
  97. 'Removed: {}'.format(counts[models.DIFF_STATUS_REMOVED]),
  98. ''
  99. ]
  100. lines.extend(describe.GenerateLines(group, summarize=False))
  101. lines += ['']
  102. return lines, len(added) - len(removed)
  103. def _CreateMutableConstantsDelta(symbols):
  104. symbols = symbols.WhereInSection('d').WhereNameMatches(r'\bk[A-Z]|\b[A-Z_]+$')
  105. lines, net_added = _SymbolDiffHelper('Mutable Constants', symbols)
  106. return lines, _SizeDelta('Mutable Constants', 'symbols', 0, net_added)
  107. def _CreateMethodCountDelta(symbols):
  108. symbols = symbols.WhereIsOnDemand(False)
  109. method_symbols = symbols.WhereInSection(models.SECTION_DEX_METHOD)
  110. method_lines, net_method_added = _SymbolDiffHelper('Methods', method_symbols)
  111. class_symbols = symbols.WhereInSection(
  112. models.SECTION_DEX).WhereNameMatches('#').Inverted()
  113. class_lines, _ = _SymbolDiffHelper('Classes', class_symbols)
  114. lines = []
  115. if class_lines:
  116. lines.extend(class_lines)
  117. lines.extend(['', '']) # empty lines added for clarity
  118. if method_lines:
  119. lines.extend(method_lines)
  120. return lines, _SizeDelta('Dex Methods Count', 'methods',
  121. _MAX_DEX_METHOD_COUNT_INCREASE, net_method_added)
  122. def _CreateResourceSizesDelta(before_dir, after_dir):
  123. sizes_diff = diagnose_bloat.ResourceSizesDiff()
  124. sizes_diff.ProduceDiff(before_dir, after_dir)
  125. return sizes_diff.Summary(), _SizeDelta(
  126. 'Normalized APK Size', 'bytes', _MAX_NORMALIZED_INCREASE,
  127. sizes_diff.summary_stat.value)
  128. def _CreateBaseModuleResourceSizesDelta(before_dir, after_dir):
  129. sizes_diff = diagnose_bloat.ResourceSizesDiff(include_sections=['base'])
  130. sizes_diff.ProduceDiff(before_dir, after_dir)
  131. return sizes_diff.DetailedResults(), _SizeDelta(
  132. 'Base Module Size', 'bytes', _MAX_NORMALIZED_INCREASE,
  133. sizes_diff.CombinedSizeChangeForSection('base'))
  134. def _CreateSupersizeDiff(before_size_path, after_size_path, review_subject,
  135. review_url):
  136. before = archive.LoadAndPostProcessSizeInfo(before_size_path)
  137. after = archive.LoadAndPostProcessSizeInfo(after_size_path)
  138. if review_subject:
  139. after.build_config[models.BUILD_CONFIG_TITLE] = review_subject
  140. if review_url:
  141. after.build_config[models.BUILD_CONFIG_URL] = review_url
  142. delta_size_info = diff.Diff(before, after, sort=True)
  143. lines = list(describe.GenerateLines(delta_size_info))
  144. return lines, delta_size_info
  145. def _CreateUncompressedPakSizeDeltas(symbols):
  146. pak_symbols = symbols.Filter(lambda s:
  147. s.size > 0 and
  148. bool(s.flags & models.FLAG_UNCOMPRESSED) and
  149. s.section_name == models.SECTION_PAK_NONTRANSLATED)
  150. return [
  151. _SizeDelta('Uncompressed Pak Entry "{}"'.format(pak.full_name), 'bytes',
  152. _MAX_PAK_INCREASE, pak.after_symbol.size)
  153. for pak in pak_symbols
  154. ]
  155. def _ExtractForTestingSymbolsFromSingleMapping(mapping_path):
  156. with open(mapping_path) as f:
  157. proguard_mapping_lines = f.readlines()
  158. current_class_orig = None
  159. for line in proguard_mapping_lines:
  160. if line.isspace() or '#' in line:
  161. continue
  162. if not line.startswith(' '):
  163. match = _PROGUARD_CLASS_MAPPING_RE.search(line)
  164. if match is None:
  165. raise Exception('Malformed class mapping')
  166. current_class_orig = match.group('original_name')
  167. continue
  168. assert current_class_orig is not None
  169. line = line.strip()
  170. match = _PROGUARD_METHOD_MAPPING_RE.search(line)
  171. if (match is not None
  172. and match.group('original_method_name').find('ForTest') > -1):
  173. method_symbol = '{}#{}'.format(
  174. match.group('original_method_class') or current_class_orig,
  175. match.group('original_method_name'))
  176. yield method_symbol
  177. match = _PROGUARD_FIELD_MAPPING_RE.search(line)
  178. if (match is not None
  179. and match.group('original_name').find('ForTest') > -1):
  180. field_symbol = '{}#{}'.format(current_class_orig,
  181. match.group('original_name'))
  182. yield field_symbol
  183. def _ExtractForTestingSymbolsFromMappings(mapping_paths):
  184. symbols = set()
  185. for mapping_path in mapping_paths:
  186. symbols.update(_ExtractForTestingSymbolsFromSingleMapping(mapping_path))
  187. return symbols
  188. def _CreateTestingSymbolsDeltas(before_mapping_paths, after_mapping_paths):
  189. before_symbols = _ExtractForTestingSymbolsFromMappings(before_mapping_paths)
  190. after_symbols = _ExtractForTestingSymbolsFromMappings(after_mapping_paths)
  191. added_symbols = list(after_symbols.difference(before_symbols))
  192. removed_symbols = list(before_symbols.difference(after_symbols))
  193. lines = []
  194. if added_symbols:
  195. lines.append('Added Symbols Named "ForTest"')
  196. lines.extend(added_symbols)
  197. lines.extend(['', '']) # empty lines added for clarity
  198. if removed_symbols:
  199. lines.append('Removed Symbols Named "ForTest"')
  200. lines.extend(removed_symbols)
  201. lines.extend(['', '']) # empty lines added for clarity
  202. return lines, _SizeDelta('Added symbols named "ForTest"', 'symbols', 0,
  203. len(added_symbols) - len(removed_symbols))
  204. def _GenerateBinarySizePluginDetails(metrics):
  205. binary_size_listings = []
  206. for delta, log_name in metrics:
  207. # Only show the base module delta if it is significant.
  208. if (log_name == _BASE_RESOURCE_SIZES_LOG and delta.IsAllowable()
  209. and not delta.IsLargeImprovement()):
  210. continue
  211. listing = {
  212. 'name': delta.name,
  213. 'delta': '{} {}'.format(_FormatNumber(delta.actual), delta.units),
  214. 'limit': '{} {}'.format(_FormatNumber(delta.expected), delta.units),
  215. 'log_name': log_name,
  216. 'allowed': delta.IsAllowable(),
  217. 'large_improvement': delta.IsLargeImprovement(),
  218. }
  219. if log_name == _RESOURCE_SIZES_LOG:
  220. listing['name'] = 'Android Binary Size'
  221. binary_size_listings.insert(0, listing)
  222. continue
  223. # The main 'binary size' delta is always shown even if unchanged.
  224. if delta.actual == 0:
  225. continue
  226. binary_size_listings.append(listing)
  227. binary_size_extras = [
  228. {
  229. 'text': 'APK Breakdown',
  230. 'url': _HTML_REPORT_URL
  231. },
  232. ]
  233. return {
  234. 'listings': binary_size_listings,
  235. 'extras': binary_size_extras,
  236. }
  237. def _FormatNumber(number):
  238. # Adds a sign for positive numbers and puts commas in large numbers
  239. return '{:+,}'.format(number)
  240. def main():
  241. parser = argparse.ArgumentParser()
  242. parser.add_argument('--author', required=True, help='CL author')
  243. parser.add_argument('--review-subject', help='Review subject')
  244. parser.add_argument('--review-url', help='Review URL')
  245. parser.add_argument('--size-config-json-name',
  246. required=True,
  247. help='Filename of JSON with configs for '
  248. 'binary size measurement.')
  249. parser.add_argument(
  250. '--before-dir',
  251. required=True,
  252. help='Directory containing the APK from reference build.')
  253. parser.add_argument(
  254. '--after-dir',
  255. required=True,
  256. help='Directory containing APK for the new build.')
  257. parser.add_argument(
  258. '--results-path',
  259. required=True,
  260. help='Output path for the trybot result .json file.')
  261. parser.add_argument(
  262. '--staging-dir',
  263. required=True,
  264. help='Directory to write summary files to.')
  265. parser.add_argument(
  266. '--local-test',
  267. action='store_true',
  268. help='Allow input directories to be diagnose_bloat.py ones.')
  269. parser.add_argument('-v', '--verbose', action='store_true')
  270. args = parser.parse_args()
  271. if args.verbose:
  272. logging.basicConfig(level=logging.INFO)
  273. before_path = pathlib.Path(args.before_dir)
  274. after_path = pathlib.Path(args.after_dir)
  275. before_path_resolver = lambda p: str(before_path / os.path.basename(p))
  276. after_path_resolver = lambda p: str(after_path / os.path.basename(p))
  277. if args.local_test:
  278. config_path = args.size_config_json_name
  279. else:
  280. config_path = after_path_resolver(args.size_config_json_name)
  281. with open(config_path, 'rt') as fh:
  282. config = json.load(fh)
  283. if args.local_test:
  284. size_filename = 'Trichrome.minimal.apks.size'
  285. else:
  286. size_filename = config['supersize_input_file'] + '.size'
  287. before_mapping_paths = [
  288. before_path_resolver(f) for f in config['mapping_files']
  289. ]
  290. after_mapping_paths = [
  291. after_path_resolver(f) for f in config['mapping_files']
  292. ]
  293. logging.info('Creating Supersize diff')
  294. supersize_diff_lines, delta_size_info = _CreateSupersizeDiff(
  295. before_path_resolver(size_filename), after_path_resolver(size_filename),
  296. args.review_subject, args.review_url)
  297. changed_symbols = delta_size_info.raw_symbols.WhereDiffStatusIs(
  298. models.DIFF_STATUS_UNCHANGED).Inverted()
  299. # Monitor dex method count since the "multidex limit" is a thing.
  300. logging.info('Checking dex symbols')
  301. dex_delta_lines, dex_delta = _CreateMethodCountDelta(changed_symbols)
  302. size_deltas = {dex_delta}
  303. metrics = {(dex_delta, _DEX_SYMBOLS_LOG)}
  304. # Look for native symbols called "kConstant" that are not actually constants.
  305. # C++ syntax makes this an easy mistake, and having symbols in .data uses more
  306. # RAM than symbols in .rodata (at least for multi-process apps).
  307. logging.info('Checking for mutable constants in native symbols')
  308. mutable_constants_lines, mutable_constants_delta = (
  309. _CreateMutableConstantsDelta(changed_symbols))
  310. size_deltas.add(mutable_constants_delta)
  311. metrics.add((mutable_constants_delta, _MUTABLE_CONSTANTS_LOG))
  312. # Look for symbols with 'ForTest' in their name.
  313. logging.info('Checking for DEX symbols named "ForTest"')
  314. testing_symbols_lines, test_symbols_delta = _CreateTestingSymbolsDeltas(
  315. before_mapping_paths, after_mapping_paths)
  316. size_deltas.add(test_symbols_delta)
  317. metrics.add((test_symbols_delta, _FOR_TESTING_LOG))
  318. # Check for uncompressed .pak file entries being added to avoid unnecessary
  319. # bloat.
  320. logging.info('Checking pak symbols')
  321. size_deltas.update(_CreateUncompressedPakSizeDeltas(changed_symbols))
  322. # Normalized APK Size is the main metric we use to monitor binary size.
  323. logging.info('Creating sizes diff')
  324. resource_sizes_lines, resource_sizes_delta = (_CreateResourceSizesDelta(
  325. args.before_dir, args.after_dir))
  326. size_deltas.add(resource_sizes_delta)
  327. metrics.add((resource_sizes_delta, _RESOURCE_SIZES_LOG))
  328. logging.info('Creating base module sizes diff')
  329. base_resource_sizes_lines, base_resource_sizes_delta = (
  330. _CreateBaseModuleResourceSizesDelta(args.before_dir, args.after_dir))
  331. size_deltas.add(base_resource_sizes_delta)
  332. metrics.add((base_resource_sizes_delta, _BASE_RESOURCE_SIZES_LOG))
  333. logging.info('Adding disassembly to dex symbols')
  334. dex_disassembly.AddDisassembly(delta_size_info, before_path_resolver,
  335. after_path_resolver)
  336. logging.info('Adding disassembly to native symbols')
  337. native_disassembly.AddDisassembly(delta_size_info, before_path_resolver,
  338. after_path_resolver)
  339. # .sizediff can be consumed by the html viewer.
  340. logging.info('Creating HTML Report')
  341. sizediff_path = os.path.join(args.staging_dir, _SIZEDIFF_FILENAME)
  342. file_format.SaveDeltaSizeInfo(delta_size_info, sizediff_path)
  343. passing_deltas = set(d for d in size_deltas if d.IsAllowable())
  344. failing_deltas = size_deltas - passing_deltas
  345. is_roller = '-autoroll' in args.author
  346. failing_checks_text = '\n'.join(d.explanation for d in sorted(failing_deltas))
  347. passing_checks_text = '\n'.join(d.explanation for d in sorted(passing_deltas))
  348. checks_text = """\
  349. FAILING Checks:
  350. {}
  351. PASSING Checks:
  352. {}
  353. To understand what those checks are and how to pass them, see:
  354. https://chromium.googlesource.com/chromium/src/+/main/docs/speed/binary_size/android_binary_size_trybot.md
  355. """.format(failing_checks_text, passing_checks_text)
  356. status_code = int(bool(failing_deltas))
  357. # Give rollers a free pass, except for mutable constants.
  358. # Mutable constants are rare, and other regressions are generally noticed in
  359. # size graphs and can be investigated after-the-fact.
  360. if is_roller and mutable_constants_delta not in failing_deltas:
  361. status_code = 0
  362. summary = '<br>' + checks_text.replace('\n', '<br>')
  363. links_json = [
  364. {
  365. 'name': 'Binary Size Details',
  366. 'lines': resource_sizes_lines,
  367. 'log_name': _RESOURCE_SIZES_LOG,
  368. },
  369. {
  370. 'name': 'Base Module Binary Size Details',
  371. 'lines': base_resource_sizes_lines,
  372. 'log_name': _BASE_RESOURCE_SIZES_LOG,
  373. },
  374. {
  375. 'name': 'Mutable Constants Diff',
  376. 'lines': mutable_constants_lines,
  377. 'log_name': _MUTABLE_CONSTANTS_LOG,
  378. },
  379. {
  380. 'name': 'ForTest Symbols Diff',
  381. 'lines': testing_symbols_lines,
  382. 'log_name': _FOR_TESTING_LOG,
  383. },
  384. {
  385. 'name': 'Dex Class and Method Diff',
  386. 'lines': dex_delta_lines,
  387. 'log_name': _DEX_SYMBOLS_LOG,
  388. },
  389. {
  390. 'name': 'SuperSize Text Diff',
  391. 'lines': supersize_diff_lines,
  392. },
  393. {
  394. 'name': 'SuperSize HTML Diff',
  395. 'url': _HTML_REPORT_URL,
  396. },
  397. ]
  398. # Remove empty diffs (Mutable Constants, Dex Method, ...).
  399. links_json = [o for o in links_json if o.get('lines') or o.get('url')]
  400. binary_size_plugin_json = _GenerateBinarySizePluginDetails(metrics)
  401. results_json = {
  402. 'status_code': status_code,
  403. 'summary': summary,
  404. 'archive_filenames': [_SIZEDIFF_FILENAME],
  405. 'links': links_json,
  406. 'gerrit_plugin_details': binary_size_plugin_json,
  407. }
  408. with open(args.results_path, 'w') as f:
  409. json.dump(results_json, f)
  410. if __name__ == '__main__':
  411. main()