resource_sizes.py 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932
  1. #!/usr/bin/env vpython3
  2. # Copyright (c) 2011 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Reports binary size metrics for an APK.
  6. More information at //docs/speed/binary_size/metrics.md.
  7. """
  8. from __future__ import print_function
  9. import argparse
  10. import collections
  11. from contextlib import contextmanager
  12. import json
  13. import logging
  14. import os
  15. import posixpath
  16. import re
  17. import struct
  18. import sys
  19. import tempfile
  20. import zipfile
  21. import zlib
  22. import devil_chromium
  23. from devil.android.sdk import build_tools
  24. from devil.utils import cmd_helper
  25. from devil.utils import lazy
  26. import method_count
  27. from pylib import constants
  28. from pylib.constants import host_paths
  29. _AAPT_PATH = lazy.WeakConstant(lambda: build_tools.GetPath('aapt'))
  30. _ANDROID_UTILS_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build',
  31. 'android', 'gyp')
  32. _BUILD_UTILS_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build', 'util')
  33. _READOBJ_PATH = os.path.join(constants.ANDROID_NDK_ROOT, 'toolchains', 'llvm',
  34. 'prebuilt', 'linux-x86_64', 'bin', 'llvm-readobj')
  35. with host_paths.SysPath(host_paths.BUILD_COMMON_PATH):
  36. import perf_tests_results_helper # pylint: disable=import-error
  37. with host_paths.SysPath(host_paths.TRACING_PATH):
  38. from tracing.value import convert_chart_json # pylint: disable=import-error
  39. with host_paths.SysPath(_ANDROID_UTILS_PATH, 0):
  40. from util import build_utils # pylint: disable=import-error
  41. from util import zipalign # pylint: disable=import-error
  42. with host_paths.SysPath(_BUILD_UTILS_PATH, 0):
  43. from lib.results import result_sink # pylint: disable=import-error
  44. from lib.results import result_types # pylint: disable=import-error
  45. zipalign.ApplyZipFileZipAlignFix()
  46. # Captures an entire config from aapt output.
  47. _AAPT_CONFIG_PATTERN = r'config %s:(.*?)config [a-zA-Z-]+:'
  48. # Matches string resource entries from aapt output.
  49. _AAPT_ENTRY_RE = re.compile(
  50. r'resource (?P<id>\w{10}) [\w\.]+:string/.*?"(?P<val>.+?)"', re.DOTALL)
  51. _BASE_CHART = {
  52. 'format_version': '0.1',
  53. 'benchmark_name': 'resource_sizes',
  54. 'benchmark_description': 'APK resource size information.',
  55. 'trace_rerun_options': [],
  56. 'charts': {}
  57. }
  58. # Macro definitions look like (something, 123) when
  59. # enable_resource_allowlist_generation=true.
  60. _RC_HEADER_RE = re.compile(r'^#define (?P<name>\w+).* (?P<id>\d+)\)?$')
  61. _RE_NON_LANGUAGE_PAK = re.compile(r'^assets/.*(resources|percent)\.pak$')
  62. _READELF_SIZES_METRICS = {
  63. 'text': ['.text'],
  64. 'data': ['.data', '.rodata', '.data.rel.ro', '.data.rel.ro.local'],
  65. 'relocations': ['.rel.dyn', '.rel.plt', '.rela.dyn', '.rela.plt'],
  66. 'unwind': [
  67. '.ARM.extab', '.ARM.exidx', '.eh_frame', '.eh_frame_hdr',
  68. '.ARM.exidxsentinel_section_after_text'
  69. ],
  70. 'symbols': [
  71. '.dynsym', '.dynstr', '.dynamic', '.shstrtab', '.got', '.plt',
  72. '.got.plt', '.hash', '.gnu.hash'
  73. ],
  74. 'other': [
  75. '.init_array', '.preinit_array', '.ctors', '.fini_array', '.comment',
  76. '.note.gnu.gold-version', '.note.crashpad.info', '.note.android.ident',
  77. '.ARM.attributes', '.note.gnu.build-id', '.gnu.version',
  78. '.gnu.version_d', '.gnu.version_r', '.interp', '.gcc_except_table'
  79. ]
  80. }
  81. class _AccumulatingReporter:
  82. def __init__(self):
  83. self._combined_metrics = collections.defaultdict(int)
  84. def __call__(self, graph_title, trace_title, value, units):
  85. self._combined_metrics[(graph_title, trace_title, units)] += value
  86. def DumpReports(self, report_func):
  87. for (graph_title, trace_title,
  88. units), value in sorted(self._combined_metrics.items()):
  89. report_func(graph_title, trace_title, value, units)
  90. class _ChartJsonReporter(_AccumulatingReporter):
  91. def __init__(self, chartjson):
  92. super().__init__()
  93. self._chartjson = chartjson
  94. self.trace_title_prefix = ''
  95. def __call__(self, graph_title, trace_title, value, units):
  96. super().__call__(graph_title, trace_title, value, units)
  97. perf_tests_results_helper.ReportPerfResult(
  98. self._chartjson, graph_title, self.trace_title_prefix + trace_title,
  99. value, units)
  100. def SynthesizeTotals(self, unique_method_count):
  101. for tup, value in sorted(self._combined_metrics.items()):
  102. graph_title, trace_title, units = tup
  103. if trace_title == 'unique methods':
  104. value = unique_method_count
  105. perf_tests_results_helper.ReportPerfResult(self._chartjson, graph_title,
  106. 'Combined_' + trace_title,
  107. value, units)
  108. def _PercentageDifference(a, b):
  109. if a == 0:
  110. return 0
  111. return float(b - a) / a
  112. def _ReadZipInfoExtraFieldLength(zip_file, zip_info):
  113. """Reads the value of |extraLength| from |zip_info|'s local file header.
  114. |zip_info| has an |extra| field, but it's read from the central directory.
  115. Android's zipalign tool sets the extra field only in local file headers.
  116. """
  117. # Refer to https://en.wikipedia.org/wiki/Zip_(file_format)#File_headers
  118. zip_file.fp.seek(zip_info.header_offset + 28)
  119. return struct.unpack('<H', zip_file.fp.read(2))[0]
  120. def _MeasureApkSignatureBlock(zip_file):
  121. """Measures the size of the v2 / v3 signing block.
  122. Refer to: https://source.android.com/security/apksigning/v2
  123. """
  124. # Seek to "end of central directory" struct.
  125. eocd_offset_from_end = -22 - len(zip_file.comment)
  126. zip_file.fp.seek(eocd_offset_from_end, os.SEEK_END)
  127. assert zip_file.fp.read(4) == b'PK\005\006', (
  128. 'failed to find end-of-central-directory')
  129. # Read out the "start of central directory" offset.
  130. zip_file.fp.seek(eocd_offset_from_end + 16, os.SEEK_END)
  131. start_of_central_directory = struct.unpack('<I', zip_file.fp.read(4))[0]
  132. # Compute the offset after the last zip entry.
  133. last_info = max(zip_file.infolist(), key=lambda i: i.header_offset)
  134. last_header_size = (30 + len(last_info.filename) +
  135. _ReadZipInfoExtraFieldLength(zip_file, last_info))
  136. end_of_last_file = (last_info.header_offset + last_header_size +
  137. last_info.compress_size)
  138. return start_of_central_directory - end_of_last_file
  139. def _RunReadobj(so_path, options):
  140. return cmd_helper.GetCmdOutput([_READOBJ_PATH, '--elf-output-style=GNU'] +
  141. options + [so_path])
  142. def _ExtractLibSectionSizesFromApk(apk_path, lib_path):
  143. with Unzip(apk_path, filename=lib_path) as extracted_lib_path:
  144. grouped_section_sizes = collections.defaultdict(int)
  145. no_bits_section_sizes, section_sizes = _CreateSectionNameSizeMap(
  146. extracted_lib_path)
  147. for group_name, section_names in _READELF_SIZES_METRICS.items():
  148. for section_name in section_names:
  149. if section_name in section_sizes:
  150. grouped_section_sizes[group_name] += section_sizes.pop(section_name)
  151. # Consider all NOBITS sections as .bss.
  152. grouped_section_sizes['bss'] = sum(no_bits_section_sizes.values())
  153. # Group any unknown section headers into the "other" group.
  154. for section_header, section_size in section_sizes.items():
  155. sys.stderr.write('Unknown elf section header: %s\n' % section_header)
  156. grouped_section_sizes['other'] += section_size
  157. return grouped_section_sizes
  158. def _CreateSectionNameSizeMap(so_path):
  159. stdout = _RunReadobj(so_path, ['-S', '--wide'])
  160. section_sizes = {}
  161. no_bits_section_sizes = {}
  162. # Matches [ 2] .hash HASH 00000000006681f0 0001f0 003154 04 A 3 0 8
  163. for match in re.finditer(r'\[[\s\d]+\] (\..*)$', stdout, re.MULTILINE):
  164. items = match.group(1).split()
  165. target = no_bits_section_sizes if items[1] == 'NOBITS' else section_sizes
  166. target[items[0]] = int(items[4], 16)
  167. return no_bits_section_sizes, section_sizes
  168. def _ParseManifestAttributes(apk_path):
  169. # Check if the manifest specifies whether or not to extract native libs.
  170. output = cmd_helper.GetCmdOutput([
  171. _AAPT_PATH.read(), 'd', 'xmltree', apk_path, 'AndroidManifest.xml'])
  172. def parse_attr(name):
  173. # android:extractNativeLibs(0x010104ea)=(type 0x12)0x0
  174. # android:extractNativeLibs(0x010104ea)=(type 0x12)0xffffffff
  175. # dist:onDemand=(type 0x12)0xffffffff
  176. m = re.search(name + r'(?:\(.*?\))?=\(type .*?\)(\w+)', output)
  177. return m and int(m.group(1), 16)
  178. skip_extract_lib = bool(parse_attr('android:extractNativeLibs'))
  179. sdk_version = parse_attr('android:minSdkVersion')
  180. is_feature_split = parse_attr('android:isFeatureSplit')
  181. # Can use <dist:on-demand>, or <module dist:onDemand="true">.
  182. on_demand = parse_attr('dist:onDemand') or 'dist:on-demand' in output
  183. on_demand = bool(on_demand and is_feature_split)
  184. return sdk_version, skip_extract_lib, on_demand
  185. def _NormalizeLanguagePaks(translations, factor):
  186. english_pak = translations.FindByPattern(r'.*/en[-_][Uu][Ss]\.l?pak')
  187. num_translations = translations.GetNumEntries()
  188. ret = 0
  189. if english_pak:
  190. ret -= translations.ComputeZippedSize()
  191. ret += int(english_pak.compress_size * num_translations * factor)
  192. return ret
  193. def _NormalizeResourcesArsc(apk_path, num_arsc_files, num_translations,
  194. out_dir):
  195. """Estimates the expected overhead of untranslated strings in resources.arsc.
  196. See http://crbug.com/677966 for why this is necessary.
  197. """
  198. # If there are multiple .arsc files, use the resource packaged APK instead.
  199. if num_arsc_files > 1:
  200. if not out_dir:
  201. return -float('inf')
  202. ap_name = os.path.basename(apk_path).replace('.apk', '.ap_')
  203. ap_path = os.path.join(out_dir, 'arsc/apks', ap_name)
  204. if not os.path.exists(ap_path):
  205. raise Exception('Missing expected file: %s, try rebuilding.' % ap_path)
  206. apk_path = ap_path
  207. aapt_output = _RunAaptDumpResources(apk_path)
  208. # en-rUS is in the default config and may be cluttered with non-translatable
  209. # strings, so en-rGB is a better baseline for finding missing translations.
  210. en_strings = _CreateResourceIdValueMap(aapt_output, 'en-rGB')
  211. fr_strings = _CreateResourceIdValueMap(aapt_output, 'fr')
  212. # en-US and en-GB will never be translated.
  213. config_count = num_translations - 2
  214. size = 0
  215. for res_id, string_val in en_strings.items():
  216. if string_val == fr_strings[res_id]:
  217. string_size = len(string_val)
  218. # 7 bytes is the per-entry overhead (not specific to any string). See
  219. # https://android.googlesource.com/platform/frameworks/base.git/+/android-4.2.2_r1/tools/aapt/StringPool.cpp#414.
  220. # The 1.5 factor was determined experimentally and is meant to account for
  221. # other languages generally having longer strings than english.
  222. size += config_count * (7 + string_size * 1.5)
  223. return int(size)
  224. def _CreateResourceIdValueMap(aapt_output, lang):
  225. """Return a map of resource ids to string values for the given |lang|."""
  226. config_re = _AAPT_CONFIG_PATTERN % lang
  227. return {entry.group('id'): entry.group('val')
  228. for config_section in re.finditer(config_re, aapt_output, re.DOTALL)
  229. for entry in re.finditer(_AAPT_ENTRY_RE, config_section.group(0))}
  230. def _RunAaptDumpResources(apk_path):
  231. cmd = [_AAPT_PATH.read(), 'dump', '--values', 'resources', apk_path]
  232. status, output = cmd_helper.GetCmdStatusAndOutput(cmd)
  233. if status != 0:
  234. raise Exception('Failed running aapt command: "%s" with output "%s".' %
  235. (' '.join(cmd), output))
  236. return output
  237. class _FileGroup:
  238. """Represents a category that apk files can fall into."""
  239. def __init__(self, name):
  240. self.name = name
  241. self._zip_infos = []
  242. self._extracted_multipliers = []
  243. def AddZipInfo(self, zip_info, extracted_multiplier=0):
  244. self._zip_infos.append(zip_info)
  245. self._extracted_multipliers.append(extracted_multiplier)
  246. def AllEntries(self):
  247. return iter(self._zip_infos)
  248. def GetNumEntries(self):
  249. return len(self._zip_infos)
  250. def FindByPattern(self, pattern):
  251. return next((i for i in self._zip_infos if re.match(pattern, i.filename)),
  252. None)
  253. def FindLargest(self):
  254. if not self._zip_infos:
  255. return None
  256. return max(self._zip_infos, key=lambda i: i.file_size)
  257. def ComputeZippedSize(self):
  258. return sum(i.compress_size for i in self._zip_infos)
  259. def ComputeUncompressedSize(self):
  260. return sum(i.file_size for i in self._zip_infos)
  261. def ComputeExtractedSize(self):
  262. ret = 0
  263. for zi, multiplier in zip(self._zip_infos, self._extracted_multipliers):
  264. ret += zi.file_size * multiplier
  265. return ret
  266. def ComputeInstallSize(self):
  267. return self.ComputeExtractedSize() + self.ComputeZippedSize()
  268. def _AnalyzeInternal(apk_path,
  269. sdk_version,
  270. report_func,
  271. dex_stats_collector,
  272. out_dir,
  273. apks_path=None,
  274. split_name=None):
  275. """Analyse APK to determine size contributions of different file classes.
  276. Returns: Normalized APK size.
  277. """
  278. dex_stats_collector.CollectFromZip(split_name or '', apk_path)
  279. file_groups = []
  280. def make_group(name):
  281. group = _FileGroup(name)
  282. file_groups.append(group)
  283. return group
  284. def has_no_extension(filename):
  285. return os.path.splitext(filename)[1] == ''
  286. native_code = make_group('Native code')
  287. java_code = make_group('Java code')
  288. native_resources_no_translations = make_group('Native resources (no l10n)')
  289. translations = make_group('Native resources (l10n)')
  290. stored_translations = make_group('Native resources stored (l10n)')
  291. icu_data = make_group('ICU (i18n library) data')
  292. v8_snapshots = make_group('V8 Snapshots')
  293. png_drawables = make_group('PNG drawables')
  294. res_directory = make_group('Non-compiled Android resources')
  295. arsc = make_group('Compiled Android resources')
  296. metadata = make_group('Package metadata')
  297. unknown = make_group('Unknown files')
  298. notices = make_group('licenses.notice file')
  299. unwind_cfi = make_group('unwind_cfi (dev and canary only)')
  300. with zipfile.ZipFile(apk_path, 'r') as apk:
  301. apk_contents = apk.infolist()
  302. # Account for zipalign overhead that exists in local file header.
  303. zipalign_overhead = sum(
  304. _ReadZipInfoExtraFieldLength(apk, i) for i in apk_contents)
  305. # Account for zipalign overhead that exists in central directory header.
  306. # Happens when python aligns entries in apkbuilder.py, but does not
  307. # exist when using Android's zipalign. E.g. for bundle .apks files.
  308. zipalign_overhead += sum(len(i.extra) for i in apk_contents)
  309. signing_block_size = _MeasureApkSignatureBlock(apk)
  310. _, skip_extract_lib, _ = _ParseManifestAttributes(apk_path)
  311. # Pre-L: Dalvik - .odex file is simply decompressed/optimized dex file (~1x).
  312. # L, M: ART - .odex file is compiled version of the dex file (~4x).
  313. # N: ART - Uses Dalvik-like JIT for normal apps (~1x), full compilation for
  314. # shared apps (~4x).
  315. # Actual multipliers calculated using "apk_operations.py disk-usage".
  316. # Will need to update multipliers once apk obfuscation is enabled.
  317. # E.g. with obfuscation, the 4.04 changes to 4.46.
  318. speed_profile_dex_multiplier = 1.17
  319. orig_filename = apks_path or apk_path
  320. is_webview = 'WebView' in orig_filename
  321. is_monochrome = 'Monochrome' in orig_filename
  322. is_library = 'Library' in orig_filename
  323. is_trichrome = 'TrichromeChrome' in orig_filename
  324. # WebView is always a shared APK since other apps load it.
  325. # Library is always shared since it's used by chrome and webview
  326. # Chrome is always shared since renderers can't access dex otherwise
  327. # (see DexFixer).
  328. is_shared_apk = sdk_version >= 24 and (is_monochrome or is_webview
  329. or is_library or is_trichrome)
  330. # Dex decompression overhead varies by Android version.
  331. if sdk_version < 21:
  332. # JellyBean & KitKat
  333. dex_multiplier = 1.16
  334. elif sdk_version < 24:
  335. # Lollipop & Marshmallow
  336. dex_multiplier = 4.04
  337. elif is_shared_apk:
  338. # Oreo and above, compilation_filter=speed
  339. dex_multiplier = 4.04
  340. else:
  341. # Oreo and above, compilation_filter=speed-profile
  342. dex_multiplier = speed_profile_dex_multiplier
  343. total_apk_size = os.path.getsize(apk_path)
  344. for member in apk_contents:
  345. filename = member.filename
  346. if filename.endswith('/'):
  347. continue
  348. if filename.endswith('.so'):
  349. basename = posixpath.basename(filename)
  350. should_extract_lib = not skip_extract_lib and basename.startswith('lib')
  351. native_code.AddZipInfo(
  352. member, extracted_multiplier=int(should_extract_lib))
  353. elif filename.startswith('classes') and filename.endswith('.dex'):
  354. # Android P+, uncompressed dex does not need to be extracted.
  355. compressed = member.compress_type != zipfile.ZIP_STORED
  356. multiplier = dex_multiplier
  357. if not compressed and sdk_version >= 28:
  358. multiplier -= 1
  359. java_code.AddZipInfo(member, extracted_multiplier=multiplier)
  360. elif re.search(_RE_NON_LANGUAGE_PAK, filename):
  361. native_resources_no_translations.AddZipInfo(member)
  362. elif filename.endswith('.pak') or filename.endswith('.lpak'):
  363. compressed = member.compress_type != zipfile.ZIP_STORED
  364. bucket = translations if compressed else stored_translations
  365. extracted_multiplier = 0
  366. if compressed:
  367. extracted_multiplier = int('en_' in filename or 'en-' in filename)
  368. bucket.AddZipInfo(member, extracted_multiplier=extracted_multiplier)
  369. elif 'icu' in filename and filename.endswith('.dat'):
  370. icu_data.AddZipInfo(member)
  371. elif filename.endswith('.bin'):
  372. v8_snapshots.AddZipInfo(member)
  373. elif filename.startswith('res/'):
  374. if (filename.endswith('.png') or filename.endswith('.webp')
  375. or has_no_extension(filename)):
  376. png_drawables.AddZipInfo(member)
  377. else:
  378. res_directory.AddZipInfo(member)
  379. elif filename.endswith('.arsc'):
  380. arsc.AddZipInfo(member)
  381. elif filename.startswith('META-INF') or filename in (
  382. 'AndroidManifest.xml', 'assets/webapk_dex_version.txt'):
  383. metadata.AddZipInfo(member)
  384. elif filename.endswith('.notice'):
  385. notices.AddZipInfo(member)
  386. elif filename.startswith('assets/unwind_cfi'):
  387. unwind_cfi.AddZipInfo(member)
  388. else:
  389. unknown.AddZipInfo(member)
  390. if apks_path:
  391. # We're mostly focused on size of Chrome for non-English locales, so assume
  392. # Hindi (arbitrarily chosen) locale split is installed.
  393. with zipfile.ZipFile(apks_path) as z:
  394. subpath = 'splits/{}-hi.apk'.format(split_name)
  395. if subpath in z.namelist():
  396. hindi_apk_info = z.getinfo(subpath)
  397. total_apk_size += hindi_apk_info.file_size
  398. else:
  399. assert split_name != 'base', 'splits/base-hi.apk should always exist'
  400. total_install_size = total_apk_size
  401. total_install_size_android_go = total_apk_size
  402. zip_overhead = total_apk_size
  403. for group in file_groups:
  404. actual_size = group.ComputeZippedSize()
  405. install_size = group.ComputeInstallSize()
  406. uncompressed_size = group.ComputeUncompressedSize()
  407. extracted_size = group.ComputeExtractedSize()
  408. total_install_size += extracted_size
  409. zip_overhead -= actual_size
  410. report_func('Breakdown', group.name + ' size', actual_size, 'bytes')
  411. report_func('InstallBreakdown', group.name + ' size', int(install_size),
  412. 'bytes')
  413. # Only a few metrics are compressed in the first place.
  414. # To avoid over-reporting, track uncompressed size only for compressed
  415. # entries.
  416. if uncompressed_size != actual_size:
  417. report_func('Uncompressed', group.name + ' size', uncompressed_size,
  418. 'bytes')
  419. if group is java_code:
  420. # Updates are compiled using quicken, but system image uses speed-profile.
  421. multiplier = speed_profile_dex_multiplier
  422. # Android P+, uncompressed dex does not need to be extracted.
  423. compressed = uncompressed_size != actual_size
  424. if not compressed and sdk_version >= 28:
  425. multiplier -= 1
  426. extracted_size = int(uncompressed_size * multiplier)
  427. total_install_size_android_go += extracted_size
  428. report_func('InstallBreakdownGo', group.name + ' size',
  429. actual_size + extracted_size, 'bytes')
  430. elif group is translations and apks_path:
  431. # Assume Hindi rather than English (accounted for above in total_apk_size)
  432. total_install_size_android_go += actual_size
  433. else:
  434. total_install_size_android_go += extracted_size
  435. # Per-file zip overhead is caused by:
  436. # * 30 byte entry header + len(file name)
  437. # * 46 byte central directory entry + len(file name)
  438. # * 0-3 bytes for zipalign.
  439. report_func('Breakdown', 'Zip Overhead', zip_overhead, 'bytes')
  440. report_func('InstallSize', 'APK size', total_apk_size, 'bytes')
  441. report_func('InstallSize', 'Estimated installed size',
  442. int(total_install_size), 'bytes')
  443. report_func('InstallSize', 'Estimated installed size (Android Go)',
  444. int(total_install_size_android_go), 'bytes')
  445. transfer_size = _CalculateCompressedSize(apk_path)
  446. report_func('TransferSize', 'Transfer size (deflate)', transfer_size, 'bytes')
  447. # Size of main dex vs remaining.
  448. main_dex_info = java_code.FindByPattern('classes.dex')
  449. if main_dex_info:
  450. main_dex_size = main_dex_info.file_size
  451. report_func('Specifics', 'main dex size', main_dex_size, 'bytes')
  452. secondary_size = java_code.ComputeUncompressedSize() - main_dex_size
  453. report_func('Specifics', 'secondary dex size', secondary_size, 'bytes')
  454. main_lib_info = native_code.FindLargest()
  455. native_code_unaligned_size = 0
  456. for lib_info in native_code.AllEntries():
  457. section_sizes = _ExtractLibSectionSizesFromApk(apk_path, lib_info.filename)
  458. native_code_unaligned_size += sum(v for k, v in section_sizes.items()
  459. if k != 'bss')
  460. # Size of main .so vs remaining.
  461. if lib_info == main_lib_info:
  462. main_lib_size = lib_info.file_size
  463. report_func('Specifics', 'main lib size', main_lib_size, 'bytes')
  464. secondary_size = native_code.ComputeUncompressedSize() - main_lib_size
  465. report_func('Specifics', 'other lib size', secondary_size, 'bytes')
  466. for metric_name, size in section_sizes.items():
  467. report_func('MainLibInfo', metric_name, size, 'bytes')
  468. # Main metric that we want to monitor for jumps.
  469. normalized_apk_size = total_apk_size
  470. # unwind_cfi exists only in dev, canary, and non-channel builds.
  471. normalized_apk_size -= unwind_cfi.ComputeZippedSize()
  472. # Sections within .so files get 4kb aligned, so use section sizes rather than
  473. # file size. Also gets rid of compression.
  474. normalized_apk_size -= native_code.ComputeZippedSize()
  475. normalized_apk_size += native_code_unaligned_size
  476. # Normalized dex size: Size within the zip + size on disk for Android Go
  477. # devices running Android O (which ~= uncompressed dex size).
  478. # Use a constant compression factor to account for fluctuations.
  479. normalized_apk_size -= java_code.ComputeZippedSize()
  480. normalized_apk_size += java_code.ComputeUncompressedSize()
  481. # Don't include zipalign overhead in normalized size, since it effectively
  482. # causes size changes files that proceed aligned files to be rounded.
  483. # For APKs where classes.dex directly proceeds libchrome.so (the normal case),
  484. # this causes small dex size changes to disappear into libchrome.so alignment.
  485. normalized_apk_size -= zipalign_overhead
  486. # Don't include the size of the apk's signing block because it can fluctuate
  487. # by up to 4kb (from my non-scientific observations), presumably based on hash
  488. # sizes.
  489. normalized_apk_size -= signing_block_size
  490. # Unaligned size should be ~= uncompressed size or something is wrong.
  491. # As of now, padding_fraction ~= .007
  492. padding_fraction = -_PercentageDifference(
  493. native_code.ComputeUncompressedSize(), native_code_unaligned_size)
  494. # Ignore this check for small / no native code
  495. if native_code.ComputeUncompressedSize() > 1000000:
  496. assert 0 <= padding_fraction < .02, (
  497. 'Padding was: {} (file_size={}, sections_sum={})'.format(
  498. padding_fraction, native_code.ComputeUncompressedSize(),
  499. native_code_unaligned_size))
  500. if apks_path:
  501. # Locale normalization not needed when measuring only one locale.
  502. # E.g. a change that adds 300 chars of unstranslated strings would cause the
  503. # metric to be off by only 390 bytes (assuming a multiplier of 2.3 for
  504. # Hindi).
  505. pass
  506. else:
  507. # Avoid noise caused when strings change and translations haven't yet been
  508. # updated.
  509. num_translations = translations.GetNumEntries()
  510. num_stored_translations = stored_translations.GetNumEntries()
  511. if num_translations > 1:
  512. # Multipliers found by looking at MonochromePublic.apk and seeing how much
  513. # smaller en-US.pak is relative to the average locale.pak.
  514. normalized_apk_size += _NormalizeLanguagePaks(translations, 1.17)
  515. if num_stored_translations > 1:
  516. normalized_apk_size += _NormalizeLanguagePaks(stored_translations, 1.43)
  517. if num_translations + num_stored_translations > 1:
  518. if num_translations == 0:
  519. # WebView stores all locale paks uncompressed.
  520. num_arsc_translations = num_stored_translations
  521. else:
  522. # Monochrome has more configurations than Chrome since it includes
  523. # WebView (which supports more locales), but these should mostly be
  524. # empty so ignore them here.
  525. num_arsc_translations = num_translations
  526. normalized_apk_size += _NormalizeResourcesArsc(apk_path,
  527. arsc.GetNumEntries(),
  528. num_arsc_translations,
  529. out_dir)
  530. # It will be -Inf for .apk files with multiple .arsc files and no out_dir set.
  531. if normalized_apk_size < 0:
  532. sys.stderr.write('Skipping normalized_apk_size (no output directory set)\n')
  533. else:
  534. report_func('Specifics', 'normalized apk size', normalized_apk_size,
  535. 'bytes')
  536. # The "file count" metric cannot be grouped with any other metrics when the
  537. # end result is going to be uploaded to the perf dashboard in the HistogramSet
  538. # format due to mixed units (bytes vs. zip entries) causing malformed
  539. # summaries to be generated.
  540. # TODO(https://crbug.com/903970): Remove this workaround if unit mixing is
  541. # ever supported.
  542. report_func('FileCount', 'file count', len(apk_contents), 'zip entries')
  543. for info in unknown.AllEntries():
  544. sys.stderr.write(
  545. 'Unknown entry: %s %d\n' % (info.filename, info.compress_size))
  546. return normalized_apk_size
  547. def _CalculateCompressedSize(file_path):
  548. CHUNK_SIZE = 256 * 1024
  549. compressor = zlib.compressobj()
  550. total_size = 0
  551. with open(file_path, 'rb') as f:
  552. for chunk in iter(lambda: f.read(CHUNK_SIZE), b''):
  553. total_size += len(compressor.compress(chunk))
  554. total_size += len(compressor.flush())
  555. return total_size
  556. @contextmanager
  557. def Unzip(zip_file, filename=None):
  558. """Utility for temporary use of a single file in a zip archive."""
  559. with build_utils.TempDir() as unzipped_dir:
  560. unzipped_files = build_utils.ExtractAll(
  561. zip_file, unzipped_dir, True, pattern=filename)
  562. if len(unzipped_files) == 0:
  563. raise Exception(
  564. '%s not found in %s' % (filename, zip_file))
  565. yield unzipped_files[0]
  566. def _ConfigOutDir(out_dir):
  567. if out_dir:
  568. constants.SetOutputDirectory(out_dir)
  569. else:
  570. try:
  571. # Triggers auto-detection when CWD == output directory.
  572. constants.CheckOutputDirectory()
  573. out_dir = constants.GetOutDirectory()
  574. except Exception: # pylint: disable=broad-except
  575. pass
  576. return out_dir
  577. def _IterSplits(namelist):
  578. for subpath in namelist:
  579. # Looks for paths like splits/vr-master.apk, splits/vr-hi.apk.
  580. name_parts = subpath.split('/')
  581. if name_parts[0] == 'splits' and len(name_parts) == 2:
  582. name_parts = name_parts[1].split('-')
  583. if len(name_parts) == 2:
  584. split_name, config_name = name_parts
  585. if config_name == 'master.apk':
  586. yield subpath, split_name
  587. def _ExtractToTempFile(zip_obj, subpath, temp_file):
  588. temp_file.seek(0)
  589. temp_file.truncate()
  590. temp_file.write(zip_obj.read(subpath))
  591. temp_file.flush()
  592. def _AnalyzeApkOrApks(report_func, apk_path, out_dir):
  593. # Create DexStatsCollector here to track unique methods across base & chrome
  594. # modules.
  595. dex_stats_collector = method_count.DexStatsCollector()
  596. if apk_path.endswith('.apk'):
  597. sdk_version, _, _ = _ParseManifestAttributes(apk_path)
  598. _AnalyzeInternal(apk_path, sdk_version, report_func, dex_stats_collector,
  599. out_dir)
  600. elif apk_path.endswith('.apks'):
  601. with tempfile.NamedTemporaryFile(suffix='.apk') as f:
  602. with zipfile.ZipFile(apk_path) as z:
  603. # Currently bundletool is creating two apks when .apks is created
  604. # without specifying an sdkVersion. Always measure the one with an
  605. # uncompressed shared library.
  606. try:
  607. info = z.getinfo('splits/base-master_2.apk')
  608. except KeyError:
  609. info = z.getinfo('splits/base-master.apk')
  610. _ExtractToTempFile(z, info.filename, f)
  611. sdk_version, _, _ = _ParseManifestAttributes(f.name)
  612. orig_report_func = report_func
  613. report_func = _AccumulatingReporter()
  614. def do_measure(split_name, on_demand):
  615. logging.info('Measuring %s on_demand=%s', split_name, on_demand)
  616. # Use no-op reporting functions to get normalized size for DFMs.
  617. inner_report_func = report_func
  618. inner_dex_stats_collector = dex_stats_collector
  619. if on_demand:
  620. inner_report_func = lambda *_: None
  621. inner_dex_stats_collector = method_count.DexStatsCollector()
  622. size = _AnalyzeInternal(f.name,
  623. sdk_version,
  624. inner_report_func,
  625. inner_dex_stats_collector,
  626. out_dir,
  627. apks_path=apk_path,
  628. split_name=split_name)
  629. report_func('DFM_' + split_name, 'Size with hindi', size, 'bytes')
  630. # Measure base outside of the loop since we've already extracted it.
  631. do_measure('base', on_demand=False)
  632. for subpath, split_name in _IterSplits(z.namelist()):
  633. if split_name != 'base':
  634. _ExtractToTempFile(z, subpath, f)
  635. _, _, on_demand = _ParseManifestAttributes(f.name)
  636. do_measure(split_name, on_demand=on_demand)
  637. report_func.DumpReports(orig_report_func)
  638. report_func = orig_report_func
  639. else:
  640. raise Exception('Unknown file type: ' + apk_path)
  641. # Report dex stats outside of _AnalyzeInternal() so that the "unique methods"
  642. # metric is not just the sum of the base and chrome modules.
  643. for metric, count in dex_stats_collector.GetTotalCounts().items():
  644. report_func('Dex', metric, count, 'entries')
  645. report_func('Dex', 'unique methods',
  646. dex_stats_collector.GetUniqueMethodCount(), 'entries')
  647. report_func('DexCache', 'DexCache',
  648. dex_stats_collector.GetDexCacheSize(pre_oreo=sdk_version < 26),
  649. 'bytes')
  650. return dex_stats_collector
  651. def _ResourceSizes(args):
  652. chartjson = _BASE_CHART.copy() if args.output_format else None
  653. reporter = _ChartJsonReporter(chartjson)
  654. # Create DexStatsCollector here to track unique methods across trichrome APKs.
  655. dex_stats_collector = method_count.DexStatsCollector()
  656. specs = [
  657. ('Chrome_', args.trichrome_chrome),
  658. ('WebView_', args.trichrome_webview),
  659. ('Library_', args.trichrome_library),
  660. ]
  661. for prefix, path in specs:
  662. if path:
  663. reporter.trace_title_prefix = prefix
  664. child_dex_stats_collector = _AnalyzeApkOrApks(reporter, path,
  665. args.out_dir)
  666. dex_stats_collector.MergeFrom(prefix, child_dex_stats_collector)
  667. if any(path for _, path in specs):
  668. reporter.SynthesizeTotals(dex_stats_collector.GetUniqueMethodCount())
  669. else:
  670. _AnalyzeApkOrApks(reporter, args.input, args.out_dir)
  671. if chartjson:
  672. _DumpChartJson(args, chartjson)
  673. def _DumpChartJson(args, chartjson):
  674. if args.output_file == '-':
  675. json_file = sys.stdout
  676. elif args.output_file:
  677. json_file = open(args.output_file, 'w')
  678. else:
  679. results_path = os.path.join(args.output_dir, 'results-chart.json')
  680. logging.critical('Dumping chartjson to %s', results_path)
  681. json_file = open(results_path, 'w')
  682. json.dump(chartjson, json_file, indent=2)
  683. if json_file is not sys.stdout:
  684. json_file.close()
  685. # We would ideally generate a histogram set directly instead of generating
  686. # chartjson then converting. However, perf_tests_results_helper is in
  687. # //build, which doesn't seem to have any precedent for depending on
  688. # anything in Catapult. This can probably be fixed, but since this doesn't
  689. # need to be super fast or anything, converting is a good enough solution
  690. # for the time being.
  691. if args.output_format == 'histograms':
  692. histogram_result = convert_chart_json.ConvertChartJson(results_path)
  693. if histogram_result.returncode != 0:
  694. raise Exception('chartjson conversion failed with error: ' +
  695. histogram_result.stdout)
  696. histogram_path = os.path.join(args.output_dir, 'perf_results.json')
  697. logging.critical('Dumping histograms to %s', histogram_path)
  698. with open(histogram_path, 'wb') as json_file:
  699. json_file.write(histogram_result.stdout)
  700. def main():
  701. build_utils.InitLogging('RESOURCE_SIZES_DEBUG')
  702. argparser = argparse.ArgumentParser(description='Print APK size metrics.')
  703. argparser.add_argument(
  704. '--min-pak-resource-size',
  705. type=int,
  706. default=20 * 1024,
  707. help='Minimum byte size of displayed pak resources.')
  708. argparser.add_argument(
  709. '--chromium-output-directory',
  710. dest='out_dir',
  711. type=os.path.realpath,
  712. help='Location of the build artifacts.')
  713. argparser.add_argument(
  714. '--chartjson',
  715. action='store_true',
  716. help='DEPRECATED. Use --output-format=chartjson '
  717. 'instead.')
  718. argparser.add_argument(
  719. '--output-format',
  720. choices=['chartjson', 'histograms'],
  721. help='Output the results to a file in the given '
  722. 'format instead of printing the results.')
  723. argparser.add_argument('--loadable_module', help='Obsolete (ignored).')
  724. # Accepted to conform to the isolated script interface, but ignored.
  725. argparser.add_argument(
  726. '--isolated-script-test-filter', help=argparse.SUPPRESS)
  727. argparser.add_argument(
  728. '--isolated-script-test-perf-output',
  729. type=os.path.realpath,
  730. help=argparse.SUPPRESS)
  731. output_group = argparser.add_mutually_exclusive_group()
  732. output_group.add_argument(
  733. '--output-dir', default='.', help='Directory to save chartjson to.')
  734. output_group.add_argument(
  735. '--output-file',
  736. help='Path to output .json (replaces --output-dir). Works only for '
  737. '--output-format=chartjson')
  738. output_group.add_argument(
  739. '--isolated-script-test-output',
  740. type=os.path.realpath,
  741. help='File to which results will be written in the '
  742. 'simplified JSON output format.')
  743. argparser.add_argument('input', help='Path to .apk or .apks file to measure.')
  744. trichrome_group = argparser.add_argument_group(
  745. 'Trichrome inputs',
  746. description='When specified, |input| is used only as Test suite name.')
  747. trichrome_group.add_argument(
  748. '--trichrome-chrome', help='Path to Trichrome Chrome .apks')
  749. trichrome_group.add_argument(
  750. '--trichrome-webview', help='Path to Trichrome WebView .apk(s)')
  751. trichrome_group.add_argument(
  752. '--trichrome-library', help='Path to Trichrome Library .apk')
  753. args = argparser.parse_args()
  754. args.out_dir = _ConfigOutDir(args.out_dir)
  755. devil_chromium.Initialize(output_directory=args.out_dir)
  756. # TODO(bsheedy): Remove this once uses of --chartjson have been removed.
  757. if args.chartjson:
  758. args.output_format = 'chartjson'
  759. result_sink_client = result_sink.TryInitClient()
  760. isolated_script_output = {'valid': False, 'failures': []}
  761. test_name = 'resource_sizes (%s)' % os.path.basename(args.input)
  762. if args.isolated_script_test_output:
  763. args.output_dir = os.path.join(
  764. os.path.dirname(args.isolated_script_test_output), test_name)
  765. if not os.path.exists(args.output_dir):
  766. os.makedirs(args.output_dir)
  767. try:
  768. _ResourceSizes(args)
  769. isolated_script_output = {
  770. 'valid': True,
  771. 'failures': [],
  772. }
  773. finally:
  774. if args.isolated_script_test_output:
  775. results_path = os.path.join(args.output_dir, 'test_results.json')
  776. with open(results_path, 'w') as output_file:
  777. json.dump(isolated_script_output, output_file)
  778. with open(args.isolated_script_test_output, 'w') as output_file:
  779. json.dump(isolated_script_output, output_file)
  780. if result_sink_client:
  781. status = result_types.PASS
  782. if not isolated_script_output['valid']:
  783. status = result_types.UNKNOWN
  784. elif isolated_script_output['failures']:
  785. status = result_types.FAIL
  786. result_sink_client.Post(test_name, status, None, None, None)
  787. if __name__ == '__main__':
  788. main()