123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932 |
- #!/usr/bin/env vpython3
- # Copyright (c) 2011 The Chromium Authors. All rights reserved.
- # Use of this source code is governed by a BSD-style license that can be
- # found in the LICENSE file.
- """Reports binary size metrics for an APK.
- More information at //docs/speed/binary_size/metrics.md.
- """
- from __future__ import print_function
- import argparse
- import collections
- from contextlib import contextmanager
- import json
- import logging
- import os
- import posixpath
- import re
- import struct
- import sys
- import tempfile
- import zipfile
- import zlib
- import devil_chromium
- from devil.android.sdk import build_tools
- from devil.utils import cmd_helper
- from devil.utils import lazy
- import method_count
- from pylib import constants
- from pylib.constants import host_paths
- _AAPT_PATH = lazy.WeakConstant(lambda: build_tools.GetPath('aapt'))
- _ANDROID_UTILS_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build',
- 'android', 'gyp')
- _BUILD_UTILS_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'build', 'util')
- _READOBJ_PATH = os.path.join(constants.ANDROID_NDK_ROOT, 'toolchains', 'llvm',
- 'prebuilt', 'linux-x86_64', 'bin', 'llvm-readobj')
- with host_paths.SysPath(host_paths.BUILD_COMMON_PATH):
- import perf_tests_results_helper # pylint: disable=import-error
- with host_paths.SysPath(host_paths.TRACING_PATH):
- from tracing.value import convert_chart_json # pylint: disable=import-error
- with host_paths.SysPath(_ANDROID_UTILS_PATH, 0):
- from util import build_utils # pylint: disable=import-error
- from util import zipalign # pylint: disable=import-error
- with host_paths.SysPath(_BUILD_UTILS_PATH, 0):
- from lib.results import result_sink # pylint: disable=import-error
- from lib.results import result_types # pylint: disable=import-error
- zipalign.ApplyZipFileZipAlignFix()
- # Captures an entire config from aapt output.
- _AAPT_CONFIG_PATTERN = r'config %s:(.*?)config [a-zA-Z-]+:'
- # Matches string resource entries from aapt output.
- _AAPT_ENTRY_RE = re.compile(
- r'resource (?P<id>\w{10}) [\w\.]+:string/.*?"(?P<val>.+?)"', re.DOTALL)
- _BASE_CHART = {
- 'format_version': '0.1',
- 'benchmark_name': 'resource_sizes',
- 'benchmark_description': 'APK resource size information.',
- 'trace_rerun_options': [],
- 'charts': {}
- }
- # Macro definitions look like (something, 123) when
- # enable_resource_allowlist_generation=true.
- _RC_HEADER_RE = re.compile(r'^#define (?P<name>\w+).* (?P<id>\d+)\)?$')
- _RE_NON_LANGUAGE_PAK = re.compile(r'^assets/.*(resources|percent)\.pak$')
- _READELF_SIZES_METRICS = {
- 'text': ['.text'],
- 'data': ['.data', '.rodata', '.data.rel.ro', '.data.rel.ro.local'],
- 'relocations': ['.rel.dyn', '.rel.plt', '.rela.dyn', '.rela.plt'],
- 'unwind': [
- '.ARM.extab', '.ARM.exidx', '.eh_frame', '.eh_frame_hdr',
- '.ARM.exidxsentinel_section_after_text'
- ],
- 'symbols': [
- '.dynsym', '.dynstr', '.dynamic', '.shstrtab', '.got', '.plt',
- '.got.plt', '.hash', '.gnu.hash'
- ],
- 'other': [
- '.init_array', '.preinit_array', '.ctors', '.fini_array', '.comment',
- '.note.gnu.gold-version', '.note.crashpad.info', '.note.android.ident',
- '.ARM.attributes', '.note.gnu.build-id', '.gnu.version',
- '.gnu.version_d', '.gnu.version_r', '.interp', '.gcc_except_table'
- ]
- }
- class _AccumulatingReporter:
- def __init__(self):
- self._combined_metrics = collections.defaultdict(int)
- def __call__(self, graph_title, trace_title, value, units):
- self._combined_metrics[(graph_title, trace_title, units)] += value
- def DumpReports(self, report_func):
- for (graph_title, trace_title,
- units), value in sorted(self._combined_metrics.items()):
- report_func(graph_title, trace_title, value, units)
- class _ChartJsonReporter(_AccumulatingReporter):
- def __init__(self, chartjson):
- super().__init__()
- self._chartjson = chartjson
- self.trace_title_prefix = ''
- def __call__(self, graph_title, trace_title, value, units):
- super().__call__(graph_title, trace_title, value, units)
- perf_tests_results_helper.ReportPerfResult(
- self._chartjson, graph_title, self.trace_title_prefix + trace_title,
- value, units)
- def SynthesizeTotals(self, unique_method_count):
- for tup, value in sorted(self._combined_metrics.items()):
- graph_title, trace_title, units = tup
- if trace_title == 'unique methods':
- value = unique_method_count
- perf_tests_results_helper.ReportPerfResult(self._chartjson, graph_title,
- 'Combined_' + trace_title,
- value, units)
- def _PercentageDifference(a, b):
- if a == 0:
- return 0
- return float(b - a) / a
- def _ReadZipInfoExtraFieldLength(zip_file, zip_info):
- """Reads the value of |extraLength| from |zip_info|'s local file header.
- |zip_info| has an |extra| field, but it's read from the central directory.
- Android's zipalign tool sets the extra field only in local file headers.
- """
- # Refer to https://en.wikipedia.org/wiki/Zip_(file_format)#File_headers
- zip_file.fp.seek(zip_info.header_offset + 28)
- return struct.unpack('<H', zip_file.fp.read(2))[0]
- def _MeasureApkSignatureBlock(zip_file):
- """Measures the size of the v2 / v3 signing block.
- Refer to: https://source.android.com/security/apksigning/v2
- """
- # Seek to "end of central directory" struct.
- eocd_offset_from_end = -22 - len(zip_file.comment)
- zip_file.fp.seek(eocd_offset_from_end, os.SEEK_END)
- assert zip_file.fp.read(4) == b'PK\005\006', (
- 'failed to find end-of-central-directory')
- # Read out the "start of central directory" offset.
- zip_file.fp.seek(eocd_offset_from_end + 16, os.SEEK_END)
- start_of_central_directory = struct.unpack('<I', zip_file.fp.read(4))[0]
- # Compute the offset after the last zip entry.
- last_info = max(zip_file.infolist(), key=lambda i: i.header_offset)
- last_header_size = (30 + len(last_info.filename) +
- _ReadZipInfoExtraFieldLength(zip_file, last_info))
- end_of_last_file = (last_info.header_offset + last_header_size +
- last_info.compress_size)
- return start_of_central_directory - end_of_last_file
- def _RunReadobj(so_path, options):
- return cmd_helper.GetCmdOutput([_READOBJ_PATH, '--elf-output-style=GNU'] +
- options + [so_path])
- def _ExtractLibSectionSizesFromApk(apk_path, lib_path):
- with Unzip(apk_path, filename=lib_path) as extracted_lib_path:
- grouped_section_sizes = collections.defaultdict(int)
- no_bits_section_sizes, section_sizes = _CreateSectionNameSizeMap(
- extracted_lib_path)
- for group_name, section_names in _READELF_SIZES_METRICS.items():
- for section_name in section_names:
- if section_name in section_sizes:
- grouped_section_sizes[group_name] += section_sizes.pop(section_name)
- # Consider all NOBITS sections as .bss.
- grouped_section_sizes['bss'] = sum(no_bits_section_sizes.values())
- # Group any unknown section headers into the "other" group.
- for section_header, section_size in section_sizes.items():
- sys.stderr.write('Unknown elf section header: %s\n' % section_header)
- grouped_section_sizes['other'] += section_size
- return grouped_section_sizes
- def _CreateSectionNameSizeMap(so_path):
- stdout = _RunReadobj(so_path, ['-S', '--wide'])
- section_sizes = {}
- no_bits_section_sizes = {}
- # Matches [ 2] .hash HASH 00000000006681f0 0001f0 003154 04 A 3 0 8
- for match in re.finditer(r'\[[\s\d]+\] (\..*)$', stdout, re.MULTILINE):
- items = match.group(1).split()
- target = no_bits_section_sizes if items[1] == 'NOBITS' else section_sizes
- target[items[0]] = int(items[4], 16)
- return no_bits_section_sizes, section_sizes
- def _ParseManifestAttributes(apk_path):
- # Check if the manifest specifies whether or not to extract native libs.
- output = cmd_helper.GetCmdOutput([
- _AAPT_PATH.read(), 'd', 'xmltree', apk_path, 'AndroidManifest.xml'])
- def parse_attr(name):
- # android:extractNativeLibs(0x010104ea)=(type 0x12)0x0
- # android:extractNativeLibs(0x010104ea)=(type 0x12)0xffffffff
- # dist:onDemand=(type 0x12)0xffffffff
- m = re.search(name + r'(?:\(.*?\))?=\(type .*?\)(\w+)', output)
- return m and int(m.group(1), 16)
- skip_extract_lib = bool(parse_attr('android:extractNativeLibs'))
- sdk_version = parse_attr('android:minSdkVersion')
- is_feature_split = parse_attr('android:isFeatureSplit')
- # Can use <dist:on-demand>, or <module dist:onDemand="true">.
- on_demand = parse_attr('dist:onDemand') or 'dist:on-demand' in output
- on_demand = bool(on_demand and is_feature_split)
- return sdk_version, skip_extract_lib, on_demand
- def _NormalizeLanguagePaks(translations, factor):
- english_pak = translations.FindByPattern(r'.*/en[-_][Uu][Ss]\.l?pak')
- num_translations = translations.GetNumEntries()
- ret = 0
- if english_pak:
- ret -= translations.ComputeZippedSize()
- ret += int(english_pak.compress_size * num_translations * factor)
- return ret
- def _NormalizeResourcesArsc(apk_path, num_arsc_files, num_translations,
- out_dir):
- """Estimates the expected overhead of untranslated strings in resources.arsc.
- See http://crbug.com/677966 for why this is necessary.
- """
- # If there are multiple .arsc files, use the resource packaged APK instead.
- if num_arsc_files > 1:
- if not out_dir:
- return -float('inf')
- ap_name = os.path.basename(apk_path).replace('.apk', '.ap_')
- ap_path = os.path.join(out_dir, 'arsc/apks', ap_name)
- if not os.path.exists(ap_path):
- raise Exception('Missing expected file: %s, try rebuilding.' % ap_path)
- apk_path = ap_path
- aapt_output = _RunAaptDumpResources(apk_path)
- # en-rUS is in the default config and may be cluttered with non-translatable
- # strings, so en-rGB is a better baseline for finding missing translations.
- en_strings = _CreateResourceIdValueMap(aapt_output, 'en-rGB')
- fr_strings = _CreateResourceIdValueMap(aapt_output, 'fr')
- # en-US and en-GB will never be translated.
- config_count = num_translations - 2
- size = 0
- for res_id, string_val in en_strings.items():
- if string_val == fr_strings[res_id]:
- string_size = len(string_val)
- # 7 bytes is the per-entry overhead (not specific to any string). See
- # https://android.googlesource.com/platform/frameworks/base.git/+/android-4.2.2_r1/tools/aapt/StringPool.cpp#414.
- # The 1.5 factor was determined experimentally and is meant to account for
- # other languages generally having longer strings than english.
- size += config_count * (7 + string_size * 1.5)
- return int(size)
- def _CreateResourceIdValueMap(aapt_output, lang):
- """Return a map of resource ids to string values for the given |lang|."""
- config_re = _AAPT_CONFIG_PATTERN % lang
- return {entry.group('id'): entry.group('val')
- for config_section in re.finditer(config_re, aapt_output, re.DOTALL)
- for entry in re.finditer(_AAPT_ENTRY_RE, config_section.group(0))}
- def _RunAaptDumpResources(apk_path):
- cmd = [_AAPT_PATH.read(), 'dump', '--values', 'resources', apk_path]
- status, output = cmd_helper.GetCmdStatusAndOutput(cmd)
- if status != 0:
- raise Exception('Failed running aapt command: "%s" with output "%s".' %
- (' '.join(cmd), output))
- return output
- class _FileGroup:
- """Represents a category that apk files can fall into."""
- def __init__(self, name):
- self.name = name
- self._zip_infos = []
- self._extracted_multipliers = []
- def AddZipInfo(self, zip_info, extracted_multiplier=0):
- self._zip_infos.append(zip_info)
- self._extracted_multipliers.append(extracted_multiplier)
- def AllEntries(self):
- return iter(self._zip_infos)
- def GetNumEntries(self):
- return len(self._zip_infos)
- def FindByPattern(self, pattern):
- return next((i for i in self._zip_infos if re.match(pattern, i.filename)),
- None)
- def FindLargest(self):
- if not self._zip_infos:
- return None
- return max(self._zip_infos, key=lambda i: i.file_size)
- def ComputeZippedSize(self):
- return sum(i.compress_size for i in self._zip_infos)
- def ComputeUncompressedSize(self):
- return sum(i.file_size for i in self._zip_infos)
- def ComputeExtractedSize(self):
- ret = 0
- for zi, multiplier in zip(self._zip_infos, self._extracted_multipliers):
- ret += zi.file_size * multiplier
- return ret
- def ComputeInstallSize(self):
- return self.ComputeExtractedSize() + self.ComputeZippedSize()
- def _AnalyzeInternal(apk_path,
- sdk_version,
- report_func,
- dex_stats_collector,
- out_dir,
- apks_path=None,
- split_name=None):
- """Analyse APK to determine size contributions of different file classes.
- Returns: Normalized APK size.
- """
- dex_stats_collector.CollectFromZip(split_name or '', apk_path)
- file_groups = []
- def make_group(name):
- group = _FileGroup(name)
- file_groups.append(group)
- return group
- def has_no_extension(filename):
- return os.path.splitext(filename)[1] == ''
- native_code = make_group('Native code')
- java_code = make_group('Java code')
- native_resources_no_translations = make_group('Native resources (no l10n)')
- translations = make_group('Native resources (l10n)')
- stored_translations = make_group('Native resources stored (l10n)')
- icu_data = make_group('ICU (i18n library) data')
- v8_snapshots = make_group('V8 Snapshots')
- png_drawables = make_group('PNG drawables')
- res_directory = make_group('Non-compiled Android resources')
- arsc = make_group('Compiled Android resources')
- metadata = make_group('Package metadata')
- unknown = make_group('Unknown files')
- notices = make_group('licenses.notice file')
- unwind_cfi = make_group('unwind_cfi (dev and canary only)')
- with zipfile.ZipFile(apk_path, 'r') as apk:
- apk_contents = apk.infolist()
- # Account for zipalign overhead that exists in local file header.
- zipalign_overhead = sum(
- _ReadZipInfoExtraFieldLength(apk, i) for i in apk_contents)
- # Account for zipalign overhead that exists in central directory header.
- # Happens when python aligns entries in apkbuilder.py, but does not
- # exist when using Android's zipalign. E.g. for bundle .apks files.
- zipalign_overhead += sum(len(i.extra) for i in apk_contents)
- signing_block_size = _MeasureApkSignatureBlock(apk)
- _, skip_extract_lib, _ = _ParseManifestAttributes(apk_path)
- # Pre-L: Dalvik - .odex file is simply decompressed/optimized dex file (~1x).
- # L, M: ART - .odex file is compiled version of the dex file (~4x).
- # N: ART - Uses Dalvik-like JIT for normal apps (~1x), full compilation for
- # shared apps (~4x).
- # Actual multipliers calculated using "apk_operations.py disk-usage".
- # Will need to update multipliers once apk obfuscation is enabled.
- # E.g. with obfuscation, the 4.04 changes to 4.46.
- speed_profile_dex_multiplier = 1.17
- orig_filename = apks_path or apk_path
- is_webview = 'WebView' in orig_filename
- is_monochrome = 'Monochrome' in orig_filename
- is_library = 'Library' in orig_filename
- is_trichrome = 'TrichromeChrome' in orig_filename
- # WebView is always a shared APK since other apps load it.
- # Library is always shared since it's used by chrome and webview
- # Chrome is always shared since renderers can't access dex otherwise
- # (see DexFixer).
- is_shared_apk = sdk_version >= 24 and (is_monochrome or is_webview
- or is_library or is_trichrome)
- # Dex decompression overhead varies by Android version.
- if sdk_version < 21:
- # JellyBean & KitKat
- dex_multiplier = 1.16
- elif sdk_version < 24:
- # Lollipop & Marshmallow
- dex_multiplier = 4.04
- elif is_shared_apk:
- # Oreo and above, compilation_filter=speed
- dex_multiplier = 4.04
- else:
- # Oreo and above, compilation_filter=speed-profile
- dex_multiplier = speed_profile_dex_multiplier
- total_apk_size = os.path.getsize(apk_path)
- for member in apk_contents:
- filename = member.filename
- if filename.endswith('/'):
- continue
- if filename.endswith('.so'):
- basename = posixpath.basename(filename)
- should_extract_lib = not skip_extract_lib and basename.startswith('lib')
- native_code.AddZipInfo(
- member, extracted_multiplier=int(should_extract_lib))
- elif filename.startswith('classes') and filename.endswith('.dex'):
- # Android P+, uncompressed dex does not need to be extracted.
- compressed = member.compress_type != zipfile.ZIP_STORED
- multiplier = dex_multiplier
- if not compressed and sdk_version >= 28:
- multiplier -= 1
- java_code.AddZipInfo(member, extracted_multiplier=multiplier)
- elif re.search(_RE_NON_LANGUAGE_PAK, filename):
- native_resources_no_translations.AddZipInfo(member)
- elif filename.endswith('.pak') or filename.endswith('.lpak'):
- compressed = member.compress_type != zipfile.ZIP_STORED
- bucket = translations if compressed else stored_translations
- extracted_multiplier = 0
- if compressed:
- extracted_multiplier = int('en_' in filename or 'en-' in filename)
- bucket.AddZipInfo(member, extracted_multiplier=extracted_multiplier)
- elif 'icu' in filename and filename.endswith('.dat'):
- icu_data.AddZipInfo(member)
- elif filename.endswith('.bin'):
- v8_snapshots.AddZipInfo(member)
- elif filename.startswith('res/'):
- if (filename.endswith('.png') or filename.endswith('.webp')
- or has_no_extension(filename)):
- png_drawables.AddZipInfo(member)
- else:
- res_directory.AddZipInfo(member)
- elif filename.endswith('.arsc'):
- arsc.AddZipInfo(member)
- elif filename.startswith('META-INF') or filename in (
- 'AndroidManifest.xml', 'assets/webapk_dex_version.txt'):
- metadata.AddZipInfo(member)
- elif filename.endswith('.notice'):
- notices.AddZipInfo(member)
- elif filename.startswith('assets/unwind_cfi'):
- unwind_cfi.AddZipInfo(member)
- else:
- unknown.AddZipInfo(member)
- if apks_path:
- # We're mostly focused on size of Chrome for non-English locales, so assume
- # Hindi (arbitrarily chosen) locale split is installed.
- with zipfile.ZipFile(apks_path) as z:
- subpath = 'splits/{}-hi.apk'.format(split_name)
- if subpath in z.namelist():
- hindi_apk_info = z.getinfo(subpath)
- total_apk_size += hindi_apk_info.file_size
- else:
- assert split_name != 'base', 'splits/base-hi.apk should always exist'
- total_install_size = total_apk_size
- total_install_size_android_go = total_apk_size
- zip_overhead = total_apk_size
- for group in file_groups:
- actual_size = group.ComputeZippedSize()
- install_size = group.ComputeInstallSize()
- uncompressed_size = group.ComputeUncompressedSize()
- extracted_size = group.ComputeExtractedSize()
- total_install_size += extracted_size
- zip_overhead -= actual_size
- report_func('Breakdown', group.name + ' size', actual_size, 'bytes')
- report_func('InstallBreakdown', group.name + ' size', int(install_size),
- 'bytes')
- # Only a few metrics are compressed in the first place.
- # To avoid over-reporting, track uncompressed size only for compressed
- # entries.
- if uncompressed_size != actual_size:
- report_func('Uncompressed', group.name + ' size', uncompressed_size,
- 'bytes')
- if group is java_code:
- # Updates are compiled using quicken, but system image uses speed-profile.
- multiplier = speed_profile_dex_multiplier
- # Android P+, uncompressed dex does not need to be extracted.
- compressed = uncompressed_size != actual_size
- if not compressed and sdk_version >= 28:
- multiplier -= 1
- extracted_size = int(uncompressed_size * multiplier)
- total_install_size_android_go += extracted_size
- report_func('InstallBreakdownGo', group.name + ' size',
- actual_size + extracted_size, 'bytes')
- elif group is translations and apks_path:
- # Assume Hindi rather than English (accounted for above in total_apk_size)
- total_install_size_android_go += actual_size
- else:
- total_install_size_android_go += extracted_size
- # Per-file zip overhead is caused by:
- # * 30 byte entry header + len(file name)
- # * 46 byte central directory entry + len(file name)
- # * 0-3 bytes for zipalign.
- report_func('Breakdown', 'Zip Overhead', zip_overhead, 'bytes')
- report_func('InstallSize', 'APK size', total_apk_size, 'bytes')
- report_func('InstallSize', 'Estimated installed size',
- int(total_install_size), 'bytes')
- report_func('InstallSize', 'Estimated installed size (Android Go)',
- int(total_install_size_android_go), 'bytes')
- transfer_size = _CalculateCompressedSize(apk_path)
- report_func('TransferSize', 'Transfer size (deflate)', transfer_size, 'bytes')
- # Size of main dex vs remaining.
- main_dex_info = java_code.FindByPattern('classes.dex')
- if main_dex_info:
- main_dex_size = main_dex_info.file_size
- report_func('Specifics', 'main dex size', main_dex_size, 'bytes')
- secondary_size = java_code.ComputeUncompressedSize() - main_dex_size
- report_func('Specifics', 'secondary dex size', secondary_size, 'bytes')
- main_lib_info = native_code.FindLargest()
- native_code_unaligned_size = 0
- for lib_info in native_code.AllEntries():
- section_sizes = _ExtractLibSectionSizesFromApk(apk_path, lib_info.filename)
- native_code_unaligned_size += sum(v for k, v in section_sizes.items()
- if k != 'bss')
- # Size of main .so vs remaining.
- if lib_info == main_lib_info:
- main_lib_size = lib_info.file_size
- report_func('Specifics', 'main lib size', main_lib_size, 'bytes')
- secondary_size = native_code.ComputeUncompressedSize() - main_lib_size
- report_func('Specifics', 'other lib size', secondary_size, 'bytes')
- for metric_name, size in section_sizes.items():
- report_func('MainLibInfo', metric_name, size, 'bytes')
- # Main metric that we want to monitor for jumps.
- normalized_apk_size = total_apk_size
- # unwind_cfi exists only in dev, canary, and non-channel builds.
- normalized_apk_size -= unwind_cfi.ComputeZippedSize()
- # Sections within .so files get 4kb aligned, so use section sizes rather than
- # file size. Also gets rid of compression.
- normalized_apk_size -= native_code.ComputeZippedSize()
- normalized_apk_size += native_code_unaligned_size
- # Normalized dex size: Size within the zip + size on disk for Android Go
- # devices running Android O (which ~= uncompressed dex size).
- # Use a constant compression factor to account for fluctuations.
- normalized_apk_size -= java_code.ComputeZippedSize()
- normalized_apk_size += java_code.ComputeUncompressedSize()
- # Don't include zipalign overhead in normalized size, since it effectively
- # causes size changes files that proceed aligned files to be rounded.
- # For APKs where classes.dex directly proceeds libchrome.so (the normal case),
- # this causes small dex size changes to disappear into libchrome.so alignment.
- normalized_apk_size -= zipalign_overhead
- # Don't include the size of the apk's signing block because it can fluctuate
- # by up to 4kb (from my non-scientific observations), presumably based on hash
- # sizes.
- normalized_apk_size -= signing_block_size
- # Unaligned size should be ~= uncompressed size or something is wrong.
- # As of now, padding_fraction ~= .007
- padding_fraction = -_PercentageDifference(
- native_code.ComputeUncompressedSize(), native_code_unaligned_size)
- # Ignore this check for small / no native code
- if native_code.ComputeUncompressedSize() > 1000000:
- assert 0 <= padding_fraction < .02, (
- 'Padding was: {} (file_size={}, sections_sum={})'.format(
- padding_fraction, native_code.ComputeUncompressedSize(),
- native_code_unaligned_size))
- if apks_path:
- # Locale normalization not needed when measuring only one locale.
- # E.g. a change that adds 300 chars of unstranslated strings would cause the
- # metric to be off by only 390 bytes (assuming a multiplier of 2.3 for
- # Hindi).
- pass
- else:
- # Avoid noise caused when strings change and translations haven't yet been
- # updated.
- num_translations = translations.GetNumEntries()
- num_stored_translations = stored_translations.GetNumEntries()
- if num_translations > 1:
- # Multipliers found by looking at MonochromePublic.apk and seeing how much
- # smaller en-US.pak is relative to the average locale.pak.
- normalized_apk_size += _NormalizeLanguagePaks(translations, 1.17)
- if num_stored_translations > 1:
- normalized_apk_size += _NormalizeLanguagePaks(stored_translations, 1.43)
- if num_translations + num_stored_translations > 1:
- if num_translations == 0:
- # WebView stores all locale paks uncompressed.
- num_arsc_translations = num_stored_translations
- else:
- # Monochrome has more configurations than Chrome since it includes
- # WebView (which supports more locales), but these should mostly be
- # empty so ignore them here.
- num_arsc_translations = num_translations
- normalized_apk_size += _NormalizeResourcesArsc(apk_path,
- arsc.GetNumEntries(),
- num_arsc_translations,
- out_dir)
- # It will be -Inf for .apk files with multiple .arsc files and no out_dir set.
- if normalized_apk_size < 0:
- sys.stderr.write('Skipping normalized_apk_size (no output directory set)\n')
- else:
- report_func('Specifics', 'normalized apk size', normalized_apk_size,
- 'bytes')
- # The "file count" metric cannot be grouped with any other metrics when the
- # end result is going to be uploaded to the perf dashboard in the HistogramSet
- # format due to mixed units (bytes vs. zip entries) causing malformed
- # summaries to be generated.
- # TODO(https://crbug.com/903970): Remove this workaround if unit mixing is
- # ever supported.
- report_func('FileCount', 'file count', len(apk_contents), 'zip entries')
- for info in unknown.AllEntries():
- sys.stderr.write(
- 'Unknown entry: %s %d\n' % (info.filename, info.compress_size))
- return normalized_apk_size
- def _CalculateCompressedSize(file_path):
- CHUNK_SIZE = 256 * 1024
- compressor = zlib.compressobj()
- total_size = 0
- with open(file_path, 'rb') as f:
- for chunk in iter(lambda: f.read(CHUNK_SIZE), b''):
- total_size += len(compressor.compress(chunk))
- total_size += len(compressor.flush())
- return total_size
- @contextmanager
- def Unzip(zip_file, filename=None):
- """Utility for temporary use of a single file in a zip archive."""
- with build_utils.TempDir() as unzipped_dir:
- unzipped_files = build_utils.ExtractAll(
- zip_file, unzipped_dir, True, pattern=filename)
- if len(unzipped_files) == 0:
- raise Exception(
- '%s not found in %s' % (filename, zip_file))
- yield unzipped_files[0]
- def _ConfigOutDir(out_dir):
- if out_dir:
- constants.SetOutputDirectory(out_dir)
- else:
- try:
- # Triggers auto-detection when CWD == output directory.
- constants.CheckOutputDirectory()
- out_dir = constants.GetOutDirectory()
- except Exception: # pylint: disable=broad-except
- pass
- return out_dir
- def _IterSplits(namelist):
- for subpath in namelist:
- # Looks for paths like splits/vr-master.apk, splits/vr-hi.apk.
- name_parts = subpath.split('/')
- if name_parts[0] == 'splits' and len(name_parts) == 2:
- name_parts = name_parts[1].split('-')
- if len(name_parts) == 2:
- split_name, config_name = name_parts
- if config_name == 'master.apk':
- yield subpath, split_name
- def _ExtractToTempFile(zip_obj, subpath, temp_file):
- temp_file.seek(0)
- temp_file.truncate()
- temp_file.write(zip_obj.read(subpath))
- temp_file.flush()
- def _AnalyzeApkOrApks(report_func, apk_path, out_dir):
- # Create DexStatsCollector here to track unique methods across base & chrome
- # modules.
- dex_stats_collector = method_count.DexStatsCollector()
- if apk_path.endswith('.apk'):
- sdk_version, _, _ = _ParseManifestAttributes(apk_path)
- _AnalyzeInternal(apk_path, sdk_version, report_func, dex_stats_collector,
- out_dir)
- elif apk_path.endswith('.apks'):
- with tempfile.NamedTemporaryFile(suffix='.apk') as f:
- with zipfile.ZipFile(apk_path) as z:
- # Currently bundletool is creating two apks when .apks is created
- # without specifying an sdkVersion. Always measure the one with an
- # uncompressed shared library.
- try:
- info = z.getinfo('splits/base-master_2.apk')
- except KeyError:
- info = z.getinfo('splits/base-master.apk')
- _ExtractToTempFile(z, info.filename, f)
- sdk_version, _, _ = _ParseManifestAttributes(f.name)
- orig_report_func = report_func
- report_func = _AccumulatingReporter()
- def do_measure(split_name, on_demand):
- logging.info('Measuring %s on_demand=%s', split_name, on_demand)
- # Use no-op reporting functions to get normalized size for DFMs.
- inner_report_func = report_func
- inner_dex_stats_collector = dex_stats_collector
- if on_demand:
- inner_report_func = lambda *_: None
- inner_dex_stats_collector = method_count.DexStatsCollector()
- size = _AnalyzeInternal(f.name,
- sdk_version,
- inner_report_func,
- inner_dex_stats_collector,
- out_dir,
- apks_path=apk_path,
- split_name=split_name)
- report_func('DFM_' + split_name, 'Size with hindi', size, 'bytes')
- # Measure base outside of the loop since we've already extracted it.
- do_measure('base', on_demand=False)
- for subpath, split_name in _IterSplits(z.namelist()):
- if split_name != 'base':
- _ExtractToTempFile(z, subpath, f)
- _, _, on_demand = _ParseManifestAttributes(f.name)
- do_measure(split_name, on_demand=on_demand)
- report_func.DumpReports(orig_report_func)
- report_func = orig_report_func
- else:
- raise Exception('Unknown file type: ' + apk_path)
- # Report dex stats outside of _AnalyzeInternal() so that the "unique methods"
- # metric is not just the sum of the base and chrome modules.
- for metric, count in dex_stats_collector.GetTotalCounts().items():
- report_func('Dex', metric, count, 'entries')
- report_func('Dex', 'unique methods',
- dex_stats_collector.GetUniqueMethodCount(), 'entries')
- report_func('DexCache', 'DexCache',
- dex_stats_collector.GetDexCacheSize(pre_oreo=sdk_version < 26),
- 'bytes')
- return dex_stats_collector
- def _ResourceSizes(args):
- chartjson = _BASE_CHART.copy() if args.output_format else None
- reporter = _ChartJsonReporter(chartjson)
- # Create DexStatsCollector here to track unique methods across trichrome APKs.
- dex_stats_collector = method_count.DexStatsCollector()
- specs = [
- ('Chrome_', args.trichrome_chrome),
- ('WebView_', args.trichrome_webview),
- ('Library_', args.trichrome_library),
- ]
- for prefix, path in specs:
- if path:
- reporter.trace_title_prefix = prefix
- child_dex_stats_collector = _AnalyzeApkOrApks(reporter, path,
- args.out_dir)
- dex_stats_collector.MergeFrom(prefix, child_dex_stats_collector)
- if any(path for _, path in specs):
- reporter.SynthesizeTotals(dex_stats_collector.GetUniqueMethodCount())
- else:
- _AnalyzeApkOrApks(reporter, args.input, args.out_dir)
- if chartjson:
- _DumpChartJson(args, chartjson)
- def _DumpChartJson(args, chartjson):
- if args.output_file == '-':
- json_file = sys.stdout
- elif args.output_file:
- json_file = open(args.output_file, 'w')
- else:
- results_path = os.path.join(args.output_dir, 'results-chart.json')
- logging.critical('Dumping chartjson to %s', results_path)
- json_file = open(results_path, 'w')
- json.dump(chartjson, json_file, indent=2)
- if json_file is not sys.stdout:
- json_file.close()
- # We would ideally generate a histogram set directly instead of generating
- # chartjson then converting. However, perf_tests_results_helper is in
- # //build, which doesn't seem to have any precedent for depending on
- # anything in Catapult. This can probably be fixed, but since this doesn't
- # need to be super fast or anything, converting is a good enough solution
- # for the time being.
- if args.output_format == 'histograms':
- histogram_result = convert_chart_json.ConvertChartJson(results_path)
- if histogram_result.returncode != 0:
- raise Exception('chartjson conversion failed with error: ' +
- histogram_result.stdout)
- histogram_path = os.path.join(args.output_dir, 'perf_results.json')
- logging.critical('Dumping histograms to %s', histogram_path)
- with open(histogram_path, 'wb') as json_file:
- json_file.write(histogram_result.stdout)
- def main():
- build_utils.InitLogging('RESOURCE_SIZES_DEBUG')
- argparser = argparse.ArgumentParser(description='Print APK size metrics.')
- argparser.add_argument(
- '--min-pak-resource-size',
- type=int,
- default=20 * 1024,
- help='Minimum byte size of displayed pak resources.')
- argparser.add_argument(
- '--chromium-output-directory',
- dest='out_dir',
- type=os.path.realpath,
- help='Location of the build artifacts.')
- argparser.add_argument(
- '--chartjson',
- action='store_true',
- help='DEPRECATED. Use --output-format=chartjson '
- 'instead.')
- argparser.add_argument(
- '--output-format',
- choices=['chartjson', 'histograms'],
- help='Output the results to a file in the given '
- 'format instead of printing the results.')
- argparser.add_argument('--loadable_module', help='Obsolete (ignored).')
- # Accepted to conform to the isolated script interface, but ignored.
- argparser.add_argument(
- '--isolated-script-test-filter', help=argparse.SUPPRESS)
- argparser.add_argument(
- '--isolated-script-test-perf-output',
- type=os.path.realpath,
- help=argparse.SUPPRESS)
- output_group = argparser.add_mutually_exclusive_group()
- output_group.add_argument(
- '--output-dir', default='.', help='Directory to save chartjson to.')
- output_group.add_argument(
- '--output-file',
- help='Path to output .json (replaces --output-dir). Works only for '
- '--output-format=chartjson')
- output_group.add_argument(
- '--isolated-script-test-output',
- type=os.path.realpath,
- help='File to which results will be written in the '
- 'simplified JSON output format.')
- argparser.add_argument('input', help='Path to .apk or .apks file to measure.')
- trichrome_group = argparser.add_argument_group(
- 'Trichrome inputs',
- description='When specified, |input| is used only as Test suite name.')
- trichrome_group.add_argument(
- '--trichrome-chrome', help='Path to Trichrome Chrome .apks')
- trichrome_group.add_argument(
- '--trichrome-webview', help='Path to Trichrome WebView .apk(s)')
- trichrome_group.add_argument(
- '--trichrome-library', help='Path to Trichrome Library .apk')
- args = argparser.parse_args()
- args.out_dir = _ConfigOutDir(args.out_dir)
- devil_chromium.Initialize(output_directory=args.out_dir)
- # TODO(bsheedy): Remove this once uses of --chartjson have been removed.
- if args.chartjson:
- args.output_format = 'chartjson'
- result_sink_client = result_sink.TryInitClient()
- isolated_script_output = {'valid': False, 'failures': []}
- test_name = 'resource_sizes (%s)' % os.path.basename(args.input)
- if args.isolated_script_test_output:
- args.output_dir = os.path.join(
- os.path.dirname(args.isolated_script_test_output), test_name)
- if not os.path.exists(args.output_dir):
- os.makedirs(args.output_dir)
- try:
- _ResourceSizes(args)
- isolated_script_output = {
- 'valid': True,
- 'failures': [],
- }
- finally:
- if args.isolated_script_test_output:
- results_path = os.path.join(args.output_dir, 'test_results.json')
- with open(results_path, 'w') as output_file:
- json.dump(isolated_script_output, output_file)
- with open(args.isolated_script_test_output, 'w') as output_file:
- json.dump(isolated_script_output, output_file)
- if result_sink_client:
- status = result_types.PASS
- if not isolated_script_output['valid']:
- status = result_types.UNKNOWN
- elif isolated_script_output['failures']:
- status = result_types.FAIL
- result_sink_client.Post(test_name, status, None, None, None)
- if __name__ == '__main__':
- main()
|