run_cts.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512
  1. #!/usr/bin/env vpython3
  2. #
  3. # Copyright 2016 The Chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """Runs the CTS test APKs stored in CIPD."""
  7. import argparse
  8. import contextlib
  9. import json
  10. import logging
  11. import os
  12. import shutil
  13. import sys
  14. import tempfile
  15. import zipfile
  16. sys.path.append(os.path.join(
  17. os.path.dirname(__file__), os.pardir, os.pardir, 'build', 'android'))
  18. # pylint: disable=wrong-import-position,import-error
  19. import devil_chromium # pylint: disable=unused-import
  20. from devil.android.ndk import abis
  21. from devil.android.sdk import version_codes
  22. from devil.android.tools import script_common
  23. from devil.utils import cmd_helper
  24. from devil.utils import logging_common
  25. from pylib.local.emulator import avd
  26. from pylib.utils import test_filter
  27. # cts test archives for all platforms are stored in this bucket
  28. # contents need to be updated if there is an important fix to any of
  29. # the tests
  30. _APP_MODE_FULL = 'full'
  31. _APP_MODE_INSTANT = 'instant'
  32. _TEST_RUNNER_PATH = os.path.join(
  33. os.path.dirname(__file__), os.pardir, os.pardir,
  34. 'build', 'android', 'test_runner.py')
  35. _WEBVIEW_CTS_GCS_PATH_FILE = os.path.join(
  36. os.path.dirname(__file__), 'cts_config', 'webview_cts_gcs_path.json')
  37. _ARCH_SPECIFIC_CTS_INFO = ["filename", "unzip_dir", "_origin"]
  38. _CTS_ARCHIVE_DIR = os.path.join(os.path.dirname(__file__), 'cts_archive')
  39. _CTS_WEBKIT_PACKAGES = ["com.android.cts.webkit", "android.webkit.cts"]
  40. _TEST_APK_AS_INSTANT_ARG = '--test-apk-as-instant'
  41. SDK_PLATFORM_DICT = {
  42. version_codes.MARSHMALLOW: 'M',
  43. version_codes.NOUGAT: 'N',
  44. version_codes.NOUGAT_MR1: 'N',
  45. version_codes.OREO: 'O',
  46. version_codes.OREO_MR1: 'O',
  47. version_codes.PIE: 'P',
  48. version_codes.Q: 'Q',
  49. version_codes.R: 'R',
  50. version_codes.S: 'S',
  51. version_codes.S_V2: 'S',
  52. }
  53. # The test apks are apparently compatible across all architectures, the
  54. # arm vs x86 split is to match the current cts releases and in case things
  55. # start to diverge in the future. Keeping the arm64 (instead of arm) dict
  56. # key to avoid breaking the bots that specify --arch arm64 to invoke the tests.
  57. _SUPPORTED_ARCH_DICT = {
  58. # TODO(aluo): Investigate how to force WebView abi on platforms supporting
  59. # multiple abis.
  60. # The test apks under 'arm64' support both arm and arm64 devices.
  61. abis.ARM: 'arm64',
  62. abis.ARM_64: 'arm64',
  63. # The test apks under 'x86' support both x86 and x86_64 devices.
  64. abis.X86: 'x86',
  65. abis.X86_64: 'x86'
  66. }
  67. TEST_FILTER_OPT = '--test-filter'
  68. def GetCtsInfo(arch, cts_release, item):
  69. """Gets contents of CTS Info for arch and cts_release.
  70. See _WEBVIEW_CTS_GCS_PATH_FILE
  71. """
  72. with open(_WEBVIEW_CTS_GCS_PATH_FILE) as f:
  73. cts_gcs_path_info = json.load(f)
  74. try:
  75. if item in _ARCH_SPECIFIC_CTS_INFO:
  76. return cts_gcs_path_info[cts_release]['arch'][arch][item]
  77. return cts_gcs_path_info[cts_release][item]
  78. except KeyError:
  79. # pylint: disable=raise-missing-from
  80. # This script is executed with python2, and cannot use 'from'.
  81. raise Exception('No %s info available for arch:%s, android:%s' %
  82. (item, arch, cts_release))
  83. def GetCTSModuleNames(arch, cts_release):
  84. """Gets the module apk name of the arch and cts_release"""
  85. test_runs = GetCtsInfo(arch, cts_release, 'test_runs')
  86. return [os.path.basename(r['apk']) for r in test_runs]
  87. def GetTestRunFilterArg(args, test_run, test_app_mode=None, arch=None):
  88. """ Merges json file filters with cmdline filters using
  89. test_filter.InitializeFilterFromArgs
  90. """
  91. test_app_mode = test_app_mode or _APP_MODE_FULL
  92. # Convert cmdline filters to test-filter style
  93. filter_string = test_filter.InitializeFilterFromArgs(args)
  94. # Get all the filters for either include or exclude patterns
  95. # and filter where an architecture is provided and does not match
  96. # The architecture is used when tests only fail on one architecture
  97. def getTestRunFilters(key):
  98. filters = test_run.get(key, [])
  99. return [
  100. filter_["match"] for filter_ in filters
  101. if 'arch' not in filter_ or filter_['arch'] == arch
  102. if 'mode' not in filter_ or filter_['mode'] == test_app_mode
  103. ]
  104. # Only add inclusion filters if there's not already one specified, since
  105. # they would conflict, see test_filter.ConflictingPositiveFiltersException.
  106. if not test_filter.HasPositivePatterns(filter_string):
  107. patterns = getTestRunFilters("includes")
  108. filter_string = test_filter.AppendPatternsToFilter(
  109. filter_string, positive_patterns=patterns)
  110. if args.skip_expected_failures:
  111. patterns = getTestRunFilters("excludes")
  112. filter_string = test_filter.AppendPatternsToFilter(
  113. filter_string, negative_patterns=patterns)
  114. if filter_string:
  115. return [TEST_FILTER_OPT + '=' + filter_string]
  116. return []
  117. def RunCTS(
  118. test_runner_args,
  119. local_cts_dir,
  120. apk,
  121. *, # Optional parameters must be passed by keyword (PEP 3102)
  122. voice_service=None,
  123. additional_apks=None,
  124. test_app_mode=None,
  125. json_results_file=None):
  126. """Run tests in apk using test_runner script at _TEST_RUNNER_PATH.
  127. Returns the script result code, test results will be stored in
  128. the json_results_file file if specified.
  129. """
  130. test_app_mode = test_app_mode or _APP_MODE_FULL
  131. local_test_runner_args = test_runner_args + ['--test-apk',
  132. os.path.join(local_cts_dir, apk)]
  133. if voice_service:
  134. local_test_runner_args += ['--use-voice-interaction-service', voice_service]
  135. if additional_apks:
  136. for additional_apk in additional_apks:
  137. additional_apk_tmp = os.path.join(local_cts_dir, additional_apk['apk'])
  138. local_test_runner_args += ['--additional-apk', additional_apk_tmp]
  139. if additional_apk.get('forced_queryable', False):
  140. local_test_runner_args += [
  141. '--forced-queryable-additional-apk', additional_apk_tmp
  142. ]
  143. if test_app_mode == _APP_MODE_INSTANT and not additional_apk.get(
  144. 'force_full_mode', False):
  145. local_test_runner_args += [
  146. '--instant-additional-apk', additional_apk_tmp
  147. ]
  148. if json_results_file:
  149. local_test_runner_args += ['--json-results-file=%s' %
  150. json_results_file]
  151. return cmd_helper.RunCmd(
  152. [_TEST_RUNNER_PATH, 'instrumentation'] + local_test_runner_args)
  153. def MergeTestResults(existing_results_json, additional_results_json):
  154. """Appends results in additional_results_json to existing_results_json."""
  155. for k, v in additional_results_json.items():
  156. if k not in existing_results_json:
  157. existing_results_json[k] = v
  158. else:
  159. if isinstance(v, dict):
  160. if not isinstance(existing_results_json[k], dict):
  161. raise NotImplementedError(
  162. "Can't merge results field %s of different types" % v)
  163. existing_results_json[k].update(v)
  164. elif isinstance(v, list):
  165. if not isinstance(existing_results_json[k], list):
  166. raise NotImplementedError(
  167. "Can't merge results field %s of different types" % v)
  168. existing_results_json[k].extend(v)
  169. else:
  170. raise NotImplementedError(
  171. "Can't merge results field %s that is not a list or dict" % v)
  172. def ExtractCTSZip(args, arch, cts_release):
  173. """Extract the CTS tests for cts_release.
  174. Extract the CTS zip file from _CTS_ARCHIVE_DIR to
  175. apk_dir if specified, or a new temporary directory if not.
  176. Returns following tuple (local_cts_dir, base_cts_dir, delete_cts_dir):
  177. local_cts_dir - CTS extraction location for current arch and cts_release
  178. base_cts_dir - Root directory for all the arches and platforms
  179. delete_cts_dir - Set if the base_cts_dir was created as a temporary
  180. directory
  181. """
  182. base_cts_dir = None
  183. delete_cts_dir = False
  184. relative_cts_zip_path = GetCtsInfo(arch, cts_release, 'filename')
  185. if args.apk_dir:
  186. base_cts_dir = args.apk_dir
  187. else:
  188. base_cts_dir = tempfile.mkdtemp()
  189. delete_cts_dir = True
  190. cts_zip_path = os.path.join(_CTS_ARCHIVE_DIR, relative_cts_zip_path)
  191. local_cts_dir = os.path.join(base_cts_dir,
  192. GetCtsInfo(arch, cts_release,
  193. 'unzip_dir')
  194. )
  195. zf = zipfile.ZipFile(cts_zip_path, 'r')
  196. zf.extractall(local_cts_dir)
  197. return (local_cts_dir, base_cts_dir, delete_cts_dir)
  198. def RunAllCTSTests(args, arch, cts_release, test_runner_args):
  199. """Run CTS tests downloaded from _CTS_BUCKET.
  200. Downloads CTS tests from bucket, runs them for the
  201. specified cts_release+arch, then creates a single
  202. results json file (if specified)
  203. Returns 0 if all tests passed, otherwise
  204. returns the failure code of the last failing
  205. test.
  206. """
  207. local_cts_dir, base_cts_dir, delete_cts_dir = ExtractCTSZip(args, arch,
  208. cts_release)
  209. cts_result = 0
  210. json_results_file = args.json_results_file
  211. try:
  212. cts_test_runs = GetCtsInfo(arch, cts_release, 'test_runs')
  213. cts_results_json = {}
  214. for cts_test_run in cts_test_runs:
  215. iteration_cts_result = 0
  216. test_apk = cts_test_run['apk']
  217. voice_service = cts_test_run.get('voice_service')
  218. # Some tests need additional APKs that providing mocking
  219. # services to run
  220. additional_apks = cts_test_run.get('additional_apks')
  221. test_app_mode = (_APP_MODE_INSTANT
  222. if args.test_apk_as_instant else _APP_MODE_FULL)
  223. # If --module-apk is specified then skip tests in all other modules
  224. if args.module_apk and os.path.basename(test_apk) != args.module_apk:
  225. continue
  226. iter_test_runner_args = test_runner_args + GetTestRunFilterArg(
  227. args, cts_test_run, test_app_mode, arch)
  228. if json_results_file:
  229. with tempfile.NamedTemporaryFile() as iteration_json_file:
  230. iteration_cts_result = RunCTS(
  231. test_runner_args=iter_test_runner_args,
  232. local_cts_dir=local_cts_dir,
  233. apk=test_apk,
  234. voice_service=voice_service,
  235. additional_apks=additional_apks,
  236. test_app_mode=test_app_mode,
  237. json_results_file=iteration_json_file.name)
  238. with open(iteration_json_file.name) as f:
  239. additional_results_json = json.load(f)
  240. MergeTestResults(cts_results_json, additional_results_json)
  241. else:
  242. iteration_cts_result = RunCTS(test_runner_args=iter_test_runner_args,
  243. local_cts_dir=local_cts_dir,
  244. apk=test_apk,
  245. voice_service=voice_service,
  246. additional_apks=additional_apks,
  247. test_app_mode=test_app_mode)
  248. if iteration_cts_result:
  249. cts_result = iteration_cts_result
  250. if json_results_file:
  251. with open(json_results_file, 'w') as f:
  252. json.dump(cts_results_json, f, indent=2)
  253. finally:
  254. if delete_cts_dir and base_cts_dir:
  255. shutil.rmtree(base_cts_dir)
  256. return cts_result
  257. def DetermineCtsRelease(device):
  258. """Determines the CTS release based on the Android SDK level
  259. Args:
  260. device: The DeviceUtils instance
  261. Returns:
  262. The first letter of the cts_release in uppercase.
  263. Raises:
  264. Exception: if we don't have the CTS tests for the device platform saved in
  265. CIPD already.
  266. """
  267. cts_release = SDK_PLATFORM_DICT.get(device.build_version_sdk)
  268. if not cts_release:
  269. # Check if we're above the supported version range.
  270. max_supported_sdk = max(SDK_PLATFORM_DICT.keys())
  271. if device.build_version_sdk > max_supported_sdk:
  272. raise Exception("We don't have tests for API level {api_level}, try "
  273. "running the {release} tests with `--cts-release "
  274. "{release}`".format(
  275. api_level=device.build_version_sdk,
  276. release=SDK_PLATFORM_DICT.get(max_supported_sdk),
  277. ))
  278. # Otherwise, we must be below the supported version range.
  279. min_supported_sdk = min(SDK_PLATFORM_DICT.keys())
  280. raise Exception("We don't support running CTS tests on platforms less "
  281. "than {release}".format(
  282. release=SDK_PLATFORM_DICT.get(min_supported_sdk), ))
  283. logging.info(('Using test APKs from CTS release=%s because '
  284. 'build.version.sdk=%s'),
  285. cts_release, device.build_version_sdk)
  286. return cts_release
  287. def DetermineArch(device):
  288. """Determines which architecture to use based on the device properties
  289. Args:
  290. device: The DeviceUtils instance
  291. Returns:
  292. The formatted arch string (as expected by CIPD)
  293. Raises:
  294. Exception: if device architecture is not currently supported by this script.
  295. """
  296. arch = _SUPPORTED_ARCH_DICT.get(device.product_cpu_abi)
  297. if not arch:
  298. raise Exception('Could not find CIPD bucket for your device arch (' +
  299. device.product_cpu_abi +
  300. '), please specify with --arch')
  301. logging.info('Guessing arch=%s because product.cpu.abi=%s', arch,
  302. device.product_cpu_abi)
  303. return arch
  304. def UninstallAnyCtsWebkitPackages(device):
  305. for package in _CTS_WEBKIT_PACKAGES:
  306. device.Uninstall(package)
  307. def ForwardArgsToTestRunner(known_args):
  308. """Convert any args that should be forwarded to test_runner.py"""
  309. forwarded_args = []
  310. if known_args.devices:
  311. # test_runner.py parses --device as nargs instead of append args
  312. forwarded_args.extend(['--device'] + known_args.devices)
  313. if known_args.denylist_file:
  314. forwarded_args.extend(['--denylist-file', known_args.denylist_file])
  315. if known_args.test_apk_as_instant:
  316. forwarded_args.extend([_TEST_APK_AS_INSTANT_ARG])
  317. if known_args.verbose:
  318. forwarded_args.extend(['-' + 'v' * known_args.verbose])
  319. #TODO: Pass quiet to test runner when it becomes supported
  320. return forwarded_args
  321. @contextlib.contextmanager
  322. def GetDevice(args):
  323. try:
  324. emulator_instance = None
  325. if args.avd_config:
  326. avd_config = avd.AvdConfig(args.avd_config)
  327. avd_config.Install()
  328. emulator_instance = avd_config.CreateInstance()
  329. # Start the emulator w/ -writable-system s.t. we can remount the system
  330. # partition r/w and install our own webview provider.
  331. emulator_instance.Start(writable_system=True)
  332. devices = script_common.GetDevices(args.devices, args.denylist_file)
  333. device = devices[0]
  334. if len(devices) > 1:
  335. logging.warning('Detection of arch and cts-release will use 1st of %d '
  336. 'devices: %s', len(devices), device.serial)
  337. yield device
  338. finally:
  339. if emulator_instance:
  340. emulator_instance.Stop()
  341. def main():
  342. parser = argparse.ArgumentParser()
  343. parser.add_argument(
  344. '--arch',
  345. choices=list(set(_SUPPORTED_ARCH_DICT.values())),
  346. default=None,
  347. type=str,
  348. help=('Architecture to for CTS tests. Will auto-determine based on '
  349. 'the device ro.product.cpu.abi property.'))
  350. parser.add_argument(
  351. '--cts-release',
  352. # TODO(aluo): --platform is deprecated (the meaning is unclear).
  353. '--platform',
  354. choices=sorted(set(SDK_PLATFORM_DICT.values())),
  355. required=False,
  356. default=None,
  357. help='Which CTS release to use for the run. This should generally be <= '
  358. 'device OS level (otherwise, the newer tests will fail). If '
  359. 'unspecified, the script will auto-determine the release based on '
  360. 'device OS level.')
  361. parser.add_argument(
  362. '--skip-expected-failures',
  363. action='store_true',
  364. help="Option to skip all tests that are expected to fail. Can't be used "
  365. "with test filters.")
  366. parser.add_argument(
  367. '--apk-dir',
  368. help='Directory to extract CTS APKs to. '
  369. 'Will use temp directory by default.')
  370. parser.add_argument(
  371. '--test-launcher-summary-output',
  372. '--json-results-file',
  373. '--write-full-results-to',
  374. '--isolated-script-test-output',
  375. dest='json_results_file', type=os.path.realpath,
  376. help='If set, will dump results in JSON form to the specified file. '
  377. 'Note that this will also trigger saving per-test logcats to '
  378. 'logdog.')
  379. parser.add_argument(
  380. '-m',
  381. '--module-apk',
  382. dest='module_apk',
  383. help='CTS module apk name in ' + _WEBVIEW_CTS_GCS_PATH_FILE +
  384. ' file, without the path prefix.')
  385. parser.add_argument(
  386. '--avd-config',
  387. type=os.path.realpath,
  388. help='Path to the avd config textpb. '
  389. '(See //tools/android/avd/proto for message definition'
  390. ' and existing textpb files.)')
  391. # We are re-using this argument that is used by our test runner
  392. # to detect if we are testing against an instant app
  393. # This allows us to know if we should filter tests based off the app
  394. # app mode or not
  395. # We are adding this filter directly instead of calling a function like
  396. # we do with test_filter.AddFilterOptions below because this would make
  397. # test-apk a required argument which would make this script fail
  398. # because we provide this later programmatically in this script
  399. parser.add_argument(
  400. _TEST_APK_AS_INSTANT_ARG,
  401. action='store_true',
  402. help='Run CTS tests in instant app mode. '
  403. 'Instant apps run in a more restrictive execution environment.')
  404. test_filter.AddFilterOptions(parser)
  405. script_common.AddDeviceArguments(parser)
  406. logging_common.AddLoggingArguments(parser)
  407. args, test_runner_args = parser.parse_known_args()
  408. logging_common.InitializeLogging(args)
  409. devil_chromium.Initialize()
  410. test_runner_args.extend(ForwardArgsToTestRunner(args))
  411. with GetDevice(args) as device:
  412. arch = args.arch or DetermineArch(device)
  413. cts_release = args.cts_release or DetermineCtsRelease(device)
  414. if (args.test_filter_files or args.test_filter
  415. or args.isolated_script_test_filter):
  416. # TODO(aluo): auto-determine the module based on the test filter and the
  417. # available tests in each module
  418. if not args.module_apk:
  419. args.module_apk = 'CtsWebkitTestCases.apk'
  420. platform_modules = GetCTSModuleNames(arch, cts_release)
  421. if args.module_apk and args.module_apk not in platform_modules:
  422. raise Exception('--module-apk for arch==' + arch + 'and cts_release=='
  423. + cts_release + ' must be one of: '
  424. + ', '.join(platform_modules))
  425. # Need to uninstall all previous cts webkit packages so that the
  426. # MockContentProvider names won't conflict with a previously installed
  427. # one under a different package name. This is due to CtsWebkitTestCases's
  428. # package name change from M to N versions of the tests while keeping the
  429. # MockContentProvider's authority string the same.
  430. UninstallAnyCtsWebkitPackages(device)
  431. return RunAllCTSTests(args, arch, cts_release, test_runner_args)
  432. if __name__ == '__main__':
  433. sys.exit(main())