diagnose_bloat.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971
  1. #!/usr/bin/env python3
  2. # Copyright 2017 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Tool for finding the cause of binary size bloat.
  6. See //tools/binary_size/README.md for example usage.
  7. Note: this tool will perform gclient sync/git checkout on your local repo.
  8. """
  9. import argparse
  10. import atexit
  11. import collections
  12. from contextlib import contextmanager
  13. import json
  14. import logging
  15. import os
  16. import re
  17. import shutil
  18. import subprocess
  19. import sys
  20. _COMMIT_COUNT_WARN_THRESHOLD = 15
  21. _ALLOWED_CONSECUTIVE_FAILURES = 2
  22. _SRC_ROOT = os.path.abspath(
  23. os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
  24. _DEFAULT_ARCHIVE_DIR = os.path.join(_SRC_ROOT, 'out', 'binary-size-results')
  25. _DEFAULT_OUT_DIR = os.path.join(_SRC_ROOT, 'out', 'binary-size-build')
  26. _SUPERSIZE_PATH = os.path.join(_SRC_ROOT, 'tools', 'binary_size', 'supersize')
  27. _RESOURCE_SIZES_PATH = os.path.join(
  28. _SRC_ROOT, 'build', 'android', 'resource_sizes.py')
  29. _GN_PATH = os.path.join(_SRC_ROOT, 'third_party', 'depot_tools', 'gn')
  30. _LLVM_TOOLS_DIR = os.path.join(_SRC_ROOT, 'third_party', 'llvm-build',
  31. 'Release+Asserts', 'bin')
  32. _CLANG_UPDATE_PATH = os.path.join(_SRC_ROOT, 'tools', 'clang', 'scripts',
  33. 'update.py')
  34. _DiffResult = collections.namedtuple('DiffResult', ['name', 'value', 'units'])
  35. class BaseDiff:
  36. """Base class capturing binary size diffs."""
  37. def __init__(self, name):
  38. self.name = name
  39. self.banner = '\n' + '*' * 30 + name + '*' * 30
  40. def AppendResults(self, logfiles):
  41. """Print and write diff results to an open |logfile|."""
  42. full, short = logfiles
  43. _WriteToFile(full, self.banner)
  44. _WriteToFile(short, self.banner)
  45. for s in self.Summary():
  46. _WriteToFile(short, s)
  47. _WriteToFile(short, '')
  48. for s in self.DetailedResults():
  49. full.write(s + '\n')
  50. @property
  51. def summary_stat(self):
  52. """Returns a tuple of (name, value, units) for the most important metric."""
  53. raise NotImplementedError()
  54. def Summary(self):
  55. """A short description that summarizes the source of binary size bloat."""
  56. raise NotImplementedError()
  57. def DetailedResults(self):
  58. """An iterable description of the cause of binary size bloat."""
  59. raise NotImplementedError()
  60. def ProduceDiff(self, before_dir, after_dir):
  61. """Prepare a binary size diff with ready to print results."""
  62. raise NotImplementedError()
  63. def RunDiff(self, logfiles, before_dir, after_dir):
  64. logging.info('Creating: %s', self.name)
  65. self.ProduceDiff(before_dir, after_dir)
  66. self.AppendResults(logfiles)
  67. class NativeDiff(BaseDiff):
  68. # E.g.: Section Sizes (Total=1.2 kb (1222 bytes)):
  69. _RE_SUMMARY_STAT = re.compile(
  70. r'Section Sizes \(Total=(?P<value>-?[0-9\.]+) ?(?P<units>\w+)')
  71. _SUMMARY_STAT_NAME = 'Native Library Delta'
  72. def __init__(self, size_name):
  73. self._size_name = size_name
  74. self._diff = []
  75. super().__init__('Native Diff')
  76. @property
  77. def summary_stat(self):
  78. m = NativeDiff._RE_SUMMARY_STAT.search(self._diff)
  79. if m:
  80. return _DiffResult(
  81. NativeDiff._SUMMARY_STAT_NAME, m.group('value'), m.group('units'))
  82. raise Exception('Could not extract total from:\n' + self._diff)
  83. def DetailedResults(self):
  84. return self._diff.splitlines()
  85. def Summary(self):
  86. return self.DetailedResults()[:100]
  87. def ProduceDiff(self, before_dir, after_dir):
  88. before_size = os.path.join(before_dir, self._size_name)
  89. after_size = os.path.join(after_dir, self._size_name)
  90. cmd = [_SUPERSIZE_PATH, 'diff', before_size, after_size]
  91. self._diff = _RunCmd(cmd)[0].replace('{', '{{').replace('}', '}}')
  92. class ResourceSizesDiff(BaseDiff):
  93. # Ordered by output appearance.
  94. _SUMMARY_SECTIONS = (
  95. 'Specifics', 'InstallSize', 'InstallBreakdown', 'Dex')
  96. # Sections where it makes sense to sum subsections into a section total.
  97. _AGGREGATE_SECTIONS = (
  98. 'InstallBreakdown', 'Breakdown', 'MainLibInfo', 'Uncompressed')
  99. def __init__(self, filename='results-chart.json', include_sections=None):
  100. self._diff = None # Set by |ProduceDiff()|
  101. self._filename = filename
  102. self._include_sections = include_sections
  103. super().__init__('Resource Sizes Diff')
  104. @property
  105. def summary_stat(self):
  106. items = []
  107. for section_name, results in self._diff.items():
  108. for subsection_name, value, units in results:
  109. if 'normalized' in subsection_name:
  110. items.append([section_name, subsection_name, value, units])
  111. if len(items) > 1: # Handle Trichrome.
  112. items = [item for item in items if 'Combined_normalized' in item[1]]
  113. if len(items) == 1:
  114. [section_name, subsection_name, value, units] = items[0]
  115. full_name = '{} {}'.format(section_name, subsection_name)
  116. return _DiffResult(full_name, value, units)
  117. raise Exception('Could not find canonical "normalized" in: %r' % self._diff)
  118. def CombinedSizeChangeForSection(self, section):
  119. for subsection_name, value, _ in self._diff[section]:
  120. if 'Combined' in subsection_name:
  121. return value
  122. raise Exception('Could not find "Combined" in: ' +
  123. repr(self._diff[section]))
  124. def DetailedResults(self):
  125. return self._ResultLines()
  126. def Summary(self):
  127. footer_lines = [
  128. '',
  129. 'For an explanation of these metrics, see:',
  130. ('https://chromium.googlesource.com/chromium/src/+/main/docs/speed/'
  131. 'binary_size/metrics.md#Metrics-for-Android')]
  132. return self._ResultLines(
  133. include_sections=ResourceSizesDiff._SUMMARY_SECTIONS) + footer_lines
  134. def ProduceDiff(self, before_dir, after_dir):
  135. before = self._LoadResults(before_dir)
  136. after = self._LoadResults(after_dir)
  137. self._diff = collections.defaultdict(list)
  138. for section, section_dict in after.items():
  139. if self._include_sections and section not in self._include_sections:
  140. continue
  141. for subsection, v in section_dict.items():
  142. # Ignore entries when resource_sizes.py chartjson format has changed.
  143. if (section not in before or
  144. subsection not in before[section] or
  145. v['units'] != before[section][subsection]['units']):
  146. logging.warning(
  147. 'Found differing dict structures for resource_sizes.py, '
  148. 'skipping %s %s', section, subsection)
  149. else:
  150. self._diff[section].append(_DiffResult(
  151. subsection,
  152. v['value'] - before[section][subsection]['value'],
  153. v['units']))
  154. def _ResultLines(self, include_sections=None):
  155. """Generates diff lines for the specified sections (defaults to all)."""
  156. section_lines = collections.defaultdict(list)
  157. for section_name, section_results in self._diff.items():
  158. if not include_sections or section_name in include_sections:
  159. subsection_lines = []
  160. section_sum = 0
  161. units = ''
  162. for name, value, units in section_results:
  163. # Omit subsections with no changes for summaries.
  164. if value == 0 and include_sections:
  165. continue
  166. section_sum += value
  167. subsection_lines.append('{:>+14,} {} {}'.format(value, units, name))
  168. section_header = section_name
  169. if section_name in ResourceSizesDiff._AGGREGATE_SECTIONS:
  170. section_header += ' ({:+,} {})'.format(section_sum, units)
  171. section_header += ':'
  172. # Omit sections with empty subsections.
  173. if subsection_lines:
  174. section_lines[section_name].append(section_header)
  175. section_lines[section_name].extend(subsection_lines)
  176. if not section_lines:
  177. return ['Empty ' + self.name]
  178. ret = []
  179. for k in include_sections or sorted(section_lines):
  180. ret.extend(section_lines[k])
  181. return ret
  182. def _LoadResults(self, archive_dir):
  183. chartjson_file = os.path.join(archive_dir, self._filename)
  184. with open(chartjson_file) as f:
  185. chartjson = json.load(f)
  186. charts = chartjson['charts']
  187. # Older versions of resource_sizes.py prefixed the apk onto section names.
  188. ret = {}
  189. for section, section_dict in charts.items():
  190. section_no_target = re.sub(r'^.*_', '', section)
  191. ret[section_no_target] = section_dict
  192. return ret
  193. class _BuildHelper:
  194. """Helper class for generating and building targets."""
  195. def __init__(self, args):
  196. self.clean = args.clean
  197. self.enable_chrome_android_internal = args.enable_chrome_android_internal
  198. self.extra_gn_args_str = args.gn_args
  199. self.apply_patch = args.extra_rev
  200. self.output_directory = args.output_directory
  201. self.target = args.target
  202. self.target_os = args.target_os
  203. self.use_goma = args.use_goma
  204. self._SetDefaults()
  205. self.is_bundle = 'minimal' in self.target
  206. def _MaybeAddGoogleSuffix(self, path):
  207. if self.IsTrichrome() and '_google' in self.target:
  208. return path.replace('.', 'Google.', 1)
  209. return path
  210. @property
  211. def abs_apk_paths(self):
  212. return [os.path.join(self.output_directory, x) for x in self.apk_paths]
  213. @property
  214. def abs_mapping_paths(self):
  215. def to_mapping_path(p):
  216. return p.replace('.minimal.apks', '.aab') + '.mapping'
  217. return [to_mapping_path(x) for x in self.abs_apk_paths]
  218. @property
  219. def apk_name(self):
  220. # my_great_apk -> MyGreat.apk
  221. apk_name = ''.join(s.title() for s in self.target.split('_')[:-1]) + '.apk'
  222. if self.is_bundle:
  223. # trichrome_minimal_apks->TrichromeMinimal.apk->Trichrome.minimal.apks
  224. apk_name = apk_name.replace('Minimal.apk', '.minimal.apks')
  225. return apk_name.replace('Webview', 'WebView')
  226. @property
  227. def supersize_input(self):
  228. if self.IsTrichrome():
  229. return self._MaybeAddGoogleSuffix(
  230. os.path.join(self.output_directory, 'apks', 'Trichrome.ssargs'))
  231. return self.abs_apk_paths[0]
  232. @property
  233. def apk_paths(self):
  234. if self.IsTrichrome():
  235. ret = [
  236. os.path.join('apks', 'TrichromeChrome.minimal.apks'),
  237. os.path.join('apks', 'TrichromeWebView.minimal.apks'),
  238. os.path.join('apks', 'TrichromeLibrary.apk'),
  239. ]
  240. return [self._MaybeAddGoogleSuffix(x) for x in ret]
  241. return [os.path.join('apks', self.apk_name)]
  242. @property
  243. def main_lib_path(self):
  244. # TODO(agrieve): Could maybe extract from .apk or GN?
  245. if self.IsLinux():
  246. return 'chrome'
  247. if 'monochrome' in self.target or 'trichrome' in self.target:
  248. ret = 'lib.unstripped/libmonochrome.so'
  249. elif 'webview' in self.target:
  250. ret = 'lib.unstripped/libwebviewchromium.so'
  251. else:
  252. ret = 'lib.unstripped/libchrome.so'
  253. return ret
  254. @property
  255. def abs_main_lib_path(self):
  256. return os.path.join(self.output_directory, self.main_lib_path)
  257. @property
  258. def map_file_path(self):
  259. return self.main_lib_path + '.map.gz'
  260. @property
  261. def size_name(self):
  262. if self.IsLinux():
  263. return os.path.basename(self.main_lib_path) + '.size'
  264. return self.apk_name + '.size'
  265. def _SetDefaults(self):
  266. if self.use_goma:
  267. try:
  268. goma_is_running = not subprocess.call(['goma_ctl', 'status'],
  269. stdout=subprocess.DEVNULL,
  270. stderr=subprocess.DEVNULL)
  271. self.use_goma = self.use_goma and goma_is_running
  272. except Exception:
  273. # goma_ctl not in PATH.
  274. self.use_goma = False
  275. if not self.use_goma:
  276. logging.warning('GOMA not running. Setting use_goma=false.')
  277. has_internal = os.path.exists(
  278. os.path.join(os.path.dirname(_SRC_ROOT), 'src-internal'))
  279. if has_internal:
  280. self.extra_gn_args_str = (
  281. 'is_chrome_branded=true ' + self.extra_gn_args_str)
  282. else:
  283. self.extra_gn_args_str = (
  284. 'ffmpeg_branding="Chrome" proprietary_codecs=true' +
  285. self.extra_gn_args_str)
  286. if self.IsLinux():
  287. self.extra_gn_args_str = (
  288. 'is_cfi=false generate_linker_map=true ' + self.extra_gn_args_str)
  289. self.extra_gn_args_str = ' ' + self.extra_gn_args_str.strip()
  290. if not self.target:
  291. if self.IsLinux():
  292. self.target = 'chrome'
  293. elif self.enable_chrome_android_internal:
  294. self.target = 'trichrome_google_minimal_apks'
  295. else:
  296. self.target = 'trichrome_minimal_apks'
  297. def _GenGnCmd(self):
  298. gn_args = 'is_official_build=true'
  299. gn_args += ' android_channel="stable"'
  300. # Variables often become unused when experimenting with macros to reduce
  301. # size, so don't fail on warnings.
  302. gn_args += ' treat_warnings_as_errors=false'
  303. # Speed things up a bit by skipping lint & errorprone.
  304. gn_args += ' disable_android_lint=true'
  305. # Down from default of 2 to speed up compile and use less disk.
  306. # Compiles need at least symbol_level=1 for pak allowlist to work.
  307. gn_args += ' symbol_level=1'
  308. gn_args += ' use_errorprone_java_compiler=false'
  309. gn_args += ' use_goma=%s' % str(self.use_goma).lower()
  310. gn_args += ' target_os="%s"' % self.target_os
  311. if self.IsAndroid():
  312. gn_args += (' enable_chrome_android_internal=%s' %
  313. str(self.enable_chrome_android_internal).lower())
  314. gn_args += self.extra_gn_args_str
  315. return [_GN_PATH, 'gen', self.output_directory, '--args=%s' % gn_args]
  316. def _GenNinjaCmd(self):
  317. cmd = ['autoninja', '-C', self.output_directory]
  318. cmd += [self.target]
  319. return cmd
  320. def Run(self):
  321. """Run GN gen/ninja build and return the process returncode."""
  322. logging.info('Building %s within %s (this might take a while).',
  323. self.target, os.path.relpath(self.output_directory))
  324. if self.clean:
  325. _RunCmd([_GN_PATH, 'clean', self.output_directory], cwd=_SRC_ROOT)
  326. retcode = _RunCmd(self._GenGnCmd(),
  327. cwd=_SRC_ROOT,
  328. verbose=True,
  329. exit_on_failure=False)[1]
  330. if retcode:
  331. return retcode
  332. return _RunCmd(
  333. self._GenNinjaCmd(), verbose=True, exit_on_failure=False)[1]
  334. def IsAndroid(self):
  335. return self.target_os == 'android'
  336. def IsTrichrome(self):
  337. return 'trichrome' in self.target
  338. def IsLinux(self):
  339. return self.target_os == 'linux'
  340. class _BuildArchive:
  341. """Class for managing a directory with build results and build metadata."""
  342. def __init__(self, rev, base_archive_dir, build, subrepo, save_unstripped):
  343. self.build = build
  344. self.dir = os.path.join(base_archive_dir, rev)
  345. metadata_path = os.path.join(self.dir, 'metadata.txt')
  346. self.rev = rev
  347. self.metadata = _Metadata([self], build, metadata_path, subrepo)
  348. self._save_unstripped = save_unstripped
  349. def ArchiveBuildResults(self):
  350. """Save build artifacts necessary for diffing."""
  351. logging.info('Saving build results to: %s', self.dir)
  352. _EnsureDirsExist(self.dir)
  353. if self.build.IsAndroid():
  354. for path in self.build.abs_apk_paths:
  355. self._ArchiveFile(path)
  356. for path in self.build.abs_mapping_paths:
  357. # TrichromeLibrary has no .mapping file.
  358. if 'TrichromeLibrary' not in path:
  359. self._ArchiveFile(path)
  360. self._ArchiveResourceSizes()
  361. self._ArchiveSizeFile()
  362. if self._save_unstripped:
  363. self._ArchiveFile(self.build.abs_main_lib_path)
  364. self.metadata.Write()
  365. assert self.Exists()
  366. def Exists(self):
  367. ret = self.metadata.Exists() and os.path.exists(self.archived_size_path)
  368. if self._save_unstripped:
  369. ret = ret and os.path.exists(self.archived_unstripped_path)
  370. return ret
  371. @property
  372. def archived_unstripped_path(self):
  373. return os.path.join(self.dir, os.path.basename(self.build.main_lib_path))
  374. @property
  375. def archived_size_path(self):
  376. return os.path.join(self.dir, self.build.size_name)
  377. def _ArchiveResourceSizes(self):
  378. cmd = [
  379. _RESOURCE_SIZES_PATH, '--output-dir', self.dir, '--chartjson',
  380. '--chromium-output-dir', self.build.output_directory
  381. ]
  382. if self.build.IsTrichrome():
  383. get_apk = lambda t: next(x for x in self.build.abs_apk_paths if t in x)
  384. cmd += ['--trichrome-chrome', get_apk('Chrome')]
  385. cmd += ['--trichrome-webview', get_apk('WebView')]
  386. cmd += ['--trichrome-library', get_apk('Library')]
  387. cmd += [self.build.apk_name]
  388. else:
  389. cmd += [self.build.abs_apk_paths[0]]
  390. _RunCmd(cmd)
  391. def _ArchiveFile(self, filename):
  392. if not os.path.exists(filename):
  393. _Die('missing expected file: %s', filename)
  394. shutil.copy(filename, self.dir)
  395. def _ArchiveSizeFile(self):
  396. supersize_cmd = [_SUPERSIZE_PATH, 'archive', self.archived_size_path]
  397. if self.build.IsAndroid():
  398. supersize_cmd += [
  399. '-f', self.build.supersize_input, '--aux-elf-file',
  400. self.build.abs_main_lib_path
  401. ]
  402. else:
  403. supersize_cmd += ['--elf-file', self.build.abs_main_lib_path]
  404. supersize_cmd += ['--output-directory', self.build.output_directory]
  405. logging.info('Creating .size file')
  406. _RunCmd(supersize_cmd)
  407. class _DiffArchiveManager:
  408. """Class for maintaining BuildArchives and their related diff artifacts."""
  409. def __init__(self, revs, archive_dir, diffs, build, subrepo, save_unstripped):
  410. self.archive_dir = archive_dir
  411. self.build = build
  412. self.build_archives = [
  413. _BuildArchive(rev, archive_dir, build, subrepo, save_unstripped)
  414. for rev in revs
  415. ]
  416. self.diffs = diffs
  417. self.subrepo = subrepo
  418. self._summary_stats = []
  419. def MaybeDiff(self, before_id, after_id):
  420. """Perform diffs given two build archives."""
  421. before = self.build_archives[before_id]
  422. after = self.build_archives[after_id]
  423. diff_path, short_diff_path = self._DiffFilePaths(before, after)
  424. if not self._CanDiff(before, after):
  425. logging.info(
  426. 'Skipping diff for %s due to missing build archives.', diff_path)
  427. return
  428. metadata_path = self._DiffMetadataPath(before, after)
  429. metadata = _Metadata(
  430. [before, after], self.build, metadata_path, self.subrepo)
  431. if metadata.Exists():
  432. logging.info(
  433. 'Skipping diff for %s and %s. Matching diff already exists: %s',
  434. before.rev, after.rev, diff_path)
  435. else:
  436. with open(diff_path, 'w') as diff_file, \
  437. open(short_diff_path, 'w') as summary_file:
  438. for d in self.diffs:
  439. d.RunDiff((diff_file, summary_file), before.dir, after.dir)
  440. metadata.Write()
  441. self._AddDiffSummaryStat(before, after)
  442. if os.path.exists(short_diff_path):
  443. _PrintFile(short_diff_path)
  444. logging.info('See detailed diff results here: %s',
  445. os.path.relpath(diff_path))
  446. def GenerateHtmlReport(self, before_id, after_id, is_internal=False):
  447. """Generate HTML report given two build archives."""
  448. before = self.build_archives[before_id]
  449. after = self.build_archives[after_id]
  450. diff_path = self._DiffDir(before, after)
  451. if not self._CanDiff(before, after):
  452. logging.info(
  453. 'Skipping HTML report for %s due to missing build archives.',
  454. diff_path)
  455. return
  456. report_path = os.path.join(diff_path, 'diff.sizediff')
  457. supersize_cmd = [
  458. _SUPERSIZE_PATH, 'save_diff', before.archived_size_path,
  459. after.archived_size_path, report_path
  460. ]
  461. logging.info('Creating .sizediff')
  462. _RunCmd(supersize_cmd)
  463. oneoffs_dir = 'oneoffs'
  464. visibility = '-a public-read '
  465. if is_internal:
  466. oneoffs_dir = 'private-oneoffs'
  467. visibility = ''
  468. unique_name = '{}_{}.sizediff'.format(before.rev, after.rev)
  469. msg = (
  470. '\n=====================\n'
  471. 'Saved locally to {local}. To view, upload to '
  472. 'https://chrome-supersize.firebaseapp.com/viewer.html.\n'
  473. 'To share, run:\n'
  474. '> gsutil.py cp {visibility}{local} '
  475. 'gs://chrome-supersize/{oneoffs_dir}/{unique_name}\n\n'
  476. 'Then view it at https://chrome-supersize.firebaseapp.com/viewer.html'
  477. '?load_url=https://storage.googleapis.com/chrome-supersize/'
  478. '{oneoffs_dir}/{unique_name}'
  479. '\n=====================\n')
  480. msg = msg.format(local=os.path.relpath(report_path),
  481. unique_name=unique_name,
  482. visibility=visibility,
  483. oneoffs_dir=oneoffs_dir)
  484. logging.info(msg)
  485. def Summarize(self):
  486. path = os.path.join(self.archive_dir, 'last_diff_summary.txt')
  487. if self._summary_stats:
  488. with open(path, 'w') as f:
  489. stats = sorted(
  490. self._summary_stats, key=lambda x: x[0].value, reverse=True)
  491. _WriteToFile(f, '\nDiff Summary')
  492. for s, before, after in stats:
  493. _WriteToFile(f, '{:>+10} {} {} for range: {}..{}',
  494. s.value, s.units, s.name, before, after)
  495. # Print cached file if all builds were cached.
  496. num_archives = len(self.build_archives)
  497. if os.path.exists(path) and num_archives > 1:
  498. _PrintFile(path)
  499. if num_archives <= 2:
  500. if not all(a.Exists() for a in self.build_archives):
  501. return
  502. size2 = ''
  503. if num_archives == 2:
  504. size2 = os.path.relpath(self.build_archives[-1].archived_size_path)
  505. logging.info('Enter supersize console via: %s console %s %s',
  506. os.path.relpath(_SUPERSIZE_PATH),
  507. os.path.relpath(self.build_archives[0].archived_size_path),
  508. size2)
  509. def _AddDiffSummaryStat(self, before, after):
  510. stat = None
  511. if self.build.IsAndroid():
  512. summary_diff_type = ResourceSizesDiff
  513. else:
  514. summary_diff_type = NativeDiff
  515. for d in self.diffs:
  516. if isinstance(d, summary_diff_type):
  517. stat = d.summary_stat
  518. if stat:
  519. self._summary_stats.append((stat, before.rev, after.rev))
  520. def _CanDiff(self, before, after):
  521. return before.Exists() and after.Exists()
  522. def _DiffFilePaths(self, before, after):
  523. ret = os.path.join(self._DiffDir(before, after), 'diff_results')
  524. return ret + '.txt', ret + '.short.txt'
  525. def _DiffMetadataPath(self, before, after):
  526. return os.path.join(self._DiffDir(before, after), 'metadata.txt')
  527. def _DiffDir(self, before, after):
  528. archive_range = '%s..%s' % (before.rev, after.rev)
  529. diff_path = os.path.join(self.archive_dir, 'diffs', archive_range)
  530. _EnsureDirsExist(diff_path)
  531. return diff_path
  532. class _Metadata:
  533. def __init__(self, archives, build, path, subrepo):
  534. self.data = {
  535. 'revs': [a.rev for a in archives],
  536. 'apply_patch': build.apply_patch,
  537. 'archive_dirs': [a.dir for a in archives],
  538. 'target': build.target,
  539. 'target_os': build.target_os,
  540. 'subrepo': subrepo,
  541. 'path': path,
  542. 'gn_args': {
  543. 'extra_gn_args_str': build.extra_gn_args_str,
  544. 'enable_chrome_android_internal': build.enable_chrome_android_internal,
  545. }
  546. }
  547. def Exists(self):
  548. path = self.data['path']
  549. if os.path.exists(path):
  550. with open(path, 'r') as f:
  551. return self.data == json.load(f)
  552. return False
  553. def Write(self):
  554. with open(self.data['path'], 'w') as f:
  555. json.dump(self.data, f)
  556. def _EnsureDirsExist(path):
  557. if not os.path.exists(path):
  558. os.makedirs(path)
  559. def _RunCmd(cmd, cwd=None, verbose=False, exit_on_failure=True):
  560. """Convenience function for running commands.
  561. Args:
  562. cmd: the command to run.
  563. verbose: if this is True, then the stdout and stderr of the process will be
  564. printed. If it's false, the stdout will be returned.
  565. exit_on_failure: die if an error occurs when this is True.
  566. Returns:
  567. Tuple of (process stdout, process returncode).
  568. """
  569. assert not (verbose and exit_on_failure)
  570. cmd_str = ' '.join(c for c in cmd)
  571. logging.debug('Running: %s', cmd_str)
  572. proc_stdout = proc_stderr = subprocess.PIPE
  573. if verbose:
  574. proc_stdout, proc_stderr = sys.stdout, subprocess.STDOUT
  575. # pylint: disable=unexpected-keyword-arg
  576. proc = subprocess.Popen(cmd,
  577. cwd=cwd,
  578. stdout=proc_stdout,
  579. stderr=proc_stderr,
  580. encoding='utf-8')
  581. stdout, stderr = proc.communicate()
  582. if proc.returncode and exit_on_failure:
  583. _Die('command failed: %s\nstderr:\n%s', cmd_str, stderr)
  584. stdout = stdout.strip() if stdout else ''
  585. return stdout, proc.returncode
  586. def _GitCmd(args, subrepo):
  587. return _RunCmd(['git', '-C', subrepo] + args)[0]
  588. def _GclientSyncCmd(rev, subrepo):
  589. cwd = os.getcwd()
  590. os.chdir(subrepo)
  591. _, retcode = _RunCmd(['gclient', 'sync', '-r', 'src@' + rev],
  592. verbose=True, exit_on_failure=False)
  593. os.chdir(cwd)
  594. return retcode
  595. def _SyncAndBuild(archive, build, subrepo, no_gclient, extra_rev):
  596. """Sync, build and return non 0 if any commands failed."""
  597. # Simply do a checkout if subrepo is used.
  598. if _CurrentGitHash(subrepo) == archive.rev:
  599. if subrepo != _SRC_ROOT:
  600. logging.info('Skipping git checkout since already at desired rev')
  601. else:
  602. logging.info('Skipping gclient sync since already at desired rev')
  603. elif subrepo != _SRC_ROOT or no_gclient:
  604. _GitCmd(['checkout', archive.rev], subrepo)
  605. else:
  606. # Move to a detached state since gclient sync doesn't work with local
  607. # commits on a branch.
  608. _GitCmd(['checkout', '--detach'], subrepo)
  609. logging.info('Syncing to %s', archive.rev)
  610. ret = _GclientSyncCmd(archive.rev, subrepo)
  611. if ret:
  612. return ret
  613. with _ApplyPatch(extra_rev, subrepo):
  614. return build.Run()
  615. @contextmanager
  616. def _ApplyPatch(rev, subrepo):
  617. if not rev:
  618. yield
  619. else:
  620. restore_func = _GenRestoreFunc(subrepo)
  621. try:
  622. _GitCmd(['cherry-pick', rev, '--strategy-option', 'theirs'], subrepo)
  623. yield
  624. finally:
  625. restore_func()
  626. def _GenerateRevList(rev, reference_rev, all_in_range, subrepo, step):
  627. """Normalize and optionally generate a list of commits in the given range.
  628. Returns:
  629. A list of revisions ordered from oldest to newest.
  630. """
  631. rev_seq = '%s^..%s' % (reference_rev, rev)
  632. stdout = _GitCmd(['rev-list', rev_seq], subrepo)
  633. all_revs = stdout.splitlines()[::-1]
  634. if all_in_range or len(all_revs) < 2 or step:
  635. revs = all_revs
  636. if step:
  637. revs = revs[::step]
  638. else:
  639. revs = [all_revs[0], all_revs[-1]]
  640. num_revs = len(revs)
  641. if num_revs >= _COMMIT_COUNT_WARN_THRESHOLD:
  642. _VerifyUserAccepts(
  643. 'You\'ve provided a commit range that contains %d commits.' % num_revs)
  644. logging.info('Processing %d commits', num_revs)
  645. return revs
  646. def _ValidateRevs(rev, reference_rev, subrepo, extra_rev):
  647. def git_fatal(args, message):
  648. devnull = open(os.devnull, 'wb')
  649. retcode = subprocess.call(
  650. ['git', '-C', subrepo] + args, stdout=devnull, stderr=subprocess.STDOUT)
  651. if retcode:
  652. _Die(message)
  653. no_obj_message = ('%s either doesn\'t exist or your local repo is out of '
  654. 'date, try "git fetch origin master"')
  655. git_fatal(['cat-file', '-e', rev], no_obj_message % rev)
  656. git_fatal(['cat-file', '-e', reference_rev], no_obj_message % reference_rev)
  657. if extra_rev:
  658. git_fatal(['cat-file', '-e', extra_rev], no_obj_message % extra_rev)
  659. git_fatal(['merge-base', '--is-ancestor', reference_rev, rev],
  660. f'reference-rev ({reference_rev}) is not an ancestor of '
  661. f'rev ({rev})')
  662. def _VerifyUserAccepts(message):
  663. print(message + ' Do you want to proceed? [y/n]')
  664. if input('> ').lower() != 'y':
  665. sys.exit()
  666. def _EnsureDirectoryClean(subrepo):
  667. logging.info('Checking source directory')
  668. stdout = _GitCmd(['status', '--porcelain'], subrepo)
  669. # Ignore untracked files.
  670. if stdout and stdout[:2] != '??':
  671. logging.error('Failure: please ensure working directory is clean.')
  672. sys.exit()
  673. def _Die(s, *args):
  674. logging.error('Failure: ' + s, *args)
  675. sys.exit(1)
  676. def _WriteToFile(logfile, s, *args, **kwargs):
  677. if isinstance(s, str):
  678. data = s.format(*args, **kwargs) + '\n'
  679. else:
  680. data = '\n'.join(s) + '\n'
  681. logfile.write(data)
  682. def _PrintFile(path):
  683. with open(path) as f:
  684. sys.stdout.write(f.read())
  685. def _CurrentGitHash(subrepo):
  686. return _GitCmd(['rev-parse', 'HEAD'], subrepo)
  687. def _GenRestoreFunc(subrepo):
  688. branch = _GitCmd(['rev-parse', '--abbrev-ref', 'HEAD'], subrepo)
  689. # Happens when the repo didn't start on a named branch.
  690. if branch == 'HEAD':
  691. branch = _GitCmd(['rev-parse', 'HEAD'], subrepo)
  692. def _RestoreFunc():
  693. logging.warning('Restoring original git checkout')
  694. _GitCmd(['checkout', branch], subrepo)
  695. return _RestoreFunc
  696. def _SetRestoreFunc(subrepo):
  697. atexit.register(_GenRestoreFunc(subrepo))
  698. def main():
  699. parser = argparse.ArgumentParser(
  700. description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
  701. parser.add_argument('rev',
  702. help='Find binary size bloat for this commit.')
  703. parser.add_argument('--archive-directory',
  704. default=_DEFAULT_ARCHIVE_DIR,
  705. help='Where results are stored.')
  706. parser.add_argument('--reference-rev',
  707. help='Older rev to diff against. If not supplied, '
  708. 'the previous commit to rev will be used.')
  709. parser.add_argument('--all',
  710. action='store_true',
  711. help='Build/download all revs from --reference-rev to '
  712. 'rev and diff the contiguous revisions.')
  713. parser.add_argument('--single',
  714. action='store_true',
  715. help='Sets --reference-rev=rev.')
  716. parser.add_argument('--unstripped',
  717. action='store_true',
  718. help='Save the unstripped native library when archiving.')
  719. parser.add_argument(
  720. '--subrepo',
  721. help='Specify a subrepo directory to use. Implies '
  722. '--no-gclient. All git commands will be executed '
  723. 'from the subrepo directory.')
  724. parser.add_argument('--no-gclient',
  725. action='store_true',
  726. help='Do not perform gclient sync steps.')
  727. parser.add_argument('--apply-patch', dest='extra_rev',
  728. help='A local commit to cherry-pick before each build. '
  729. 'This can leave your repo in a broken state if '
  730. 'the cherry-pick fails.')
  731. parser.add_argument('--step', type=int,
  732. help='Assumes --all and only builds/downloads every '
  733. '--step\'th revision.')
  734. parser.add_argument('-v',
  735. '--verbose',
  736. action='store_true',
  737. help='Show commands executed, extra debugging output'
  738. ', and Ninja/GN output.')
  739. build_group = parser.add_argument_group('build arguments')
  740. build_group.add_argument('--no-goma',
  741. action='store_false',
  742. dest='use_goma',
  743. default=True,
  744. help='Do not use goma when building with ninja.')
  745. build_group.add_argument('--clean',
  746. action='store_true',
  747. help='Do a clean build for each revision.')
  748. build_group.add_argument('--gn-args',
  749. default='',
  750. help='Extra GN args to set.')
  751. build_group.add_argument('--target-os',
  752. default='android',
  753. choices=['android', 'linux'],
  754. help='target_os gn arg. Default: android.')
  755. build_group.add_argument('--output-directory',
  756. default=_DEFAULT_OUT_DIR,
  757. help='ninja output directory. '
  758. 'Default: %s.' % _DEFAULT_OUT_DIR)
  759. build_group.add_argument('--enable-chrome-android-internal',
  760. action='store_true',
  761. help='Allow downstream targets to be built.')
  762. build_group.add_argument('--target',
  763. help='GN target to build. Linux default: chrome. '
  764. 'Android default: trichrome_minimal_apks or '
  765. 'trichrome_google_minimal_apks (depending on '
  766. '--enable-chrome-android-internal).')
  767. if len(sys.argv) == 1:
  768. parser.print_help()
  769. return 1
  770. args = parser.parse_args()
  771. log_level = logging.DEBUG if args.verbose else logging.INFO
  772. logging.basicConfig(level=log_level,
  773. format='%(levelname).1s %(relativeCreated)6d %(message)s')
  774. if args.target and args.target.endswith('_bundle'):
  775. parser.error('Bundle targets must use _minimal_apks variants')
  776. build = _BuildHelper(args)
  777. subrepo = args.subrepo or _SRC_ROOT
  778. _EnsureDirectoryClean(subrepo)
  779. _SetRestoreFunc(subrepo)
  780. if build.IsLinux():
  781. _VerifyUserAccepts('Linux diffs have known deficiencies (crbug/717550).')
  782. # llvm-objdump always exists for android checkouts, which run it as DEPS hook,
  783. # but not for linux.
  784. if not os.path.exists(os.path.join(_LLVM_TOOLS_DIR, 'llvm-objdump')):
  785. _RunCmd([_CLANG_UPDATE_PATH, '--package=objdump'])
  786. reference_rev = args.reference_rev or args.rev + '^'
  787. if args.single:
  788. reference_rev = args.rev
  789. _ValidateRevs(args.rev, reference_rev, subrepo, args.extra_rev)
  790. revs = _GenerateRevList(args.rev, reference_rev, args.all, subrepo, args.step)
  791. diffs = [NativeDiff(build.size_name)]
  792. if build.IsAndroid():
  793. diffs += [ResourceSizesDiff()]
  794. diff_mngr = _DiffArchiveManager(revs, args.archive_directory, diffs, build,
  795. subrepo, args.unstripped)
  796. consecutive_failures = 0
  797. i = 0
  798. for i, archive in enumerate(diff_mngr.build_archives):
  799. if archive.Exists():
  800. logging.info('Found matching metadata for %s, skipping build step.',
  801. archive.rev)
  802. else:
  803. build_failure = _SyncAndBuild(archive, build, subrepo, args.no_gclient,
  804. args.extra_rev)
  805. if build_failure:
  806. logging.info(
  807. 'Build failed for %s, diffs using this rev will be skipped.',
  808. archive.rev)
  809. consecutive_failures += 1
  810. if len(diff_mngr.build_archives) <= 2:
  811. _Die('Stopping due to build failure.')
  812. elif consecutive_failures > _ALLOWED_CONSECUTIVE_FAILURES:
  813. _Die('%d builds failed in a row, last failure was %s.',
  814. consecutive_failures, archive.rev)
  815. else:
  816. archive.ArchiveBuildResults()
  817. consecutive_failures = 0
  818. if i != 0:
  819. diff_mngr.MaybeDiff(i - 1, i)
  820. diff_mngr.GenerateHtmlReport(0,
  821. i,
  822. is_internal=args.enable_chrome_android_internal)
  823. diff_mngr.Summarize()
  824. return 0
  825. if __name__ == '__main__':
  826. sys.exit(main())