test_runner.py 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345
  1. #!/usr/bin/env vpython3
  2. #
  3. # Copyright 2013 The Chromium Authors. All rights reserved.
  4. # Use of this source code is governed by a BSD-style license that can be
  5. # found in the LICENSE file.
  6. """Runs all types of tests from one unified interface."""
  7. from __future__ import absolute_import
  8. import argparse
  9. import collections
  10. import contextlib
  11. import io
  12. import itertools
  13. import logging
  14. import os
  15. import re
  16. import shlex
  17. import shutil
  18. import signal
  19. import sys
  20. import tempfile
  21. import threading
  22. import traceback
  23. import unittest
  24. # Import _strptime before threaded code. datetime.datetime.strptime is
  25. # threadsafe except for the initial import of the _strptime module.
  26. # See http://crbug.com/724524 and https://bugs.python.org/issue7980.
  27. import _strptime # pylint: disable=unused-import
  28. # pylint: disable=ungrouped-imports
  29. from pylib.constants import host_paths
  30. if host_paths.DEVIL_PATH not in sys.path:
  31. sys.path.append(host_paths.DEVIL_PATH)
  32. from devil import base_error
  33. from devil.utils import reraiser_thread
  34. from devil.utils import run_tests_helper
  35. from pylib import constants
  36. from pylib.base import base_test_result
  37. from pylib.base import environment_factory
  38. from pylib.base import output_manager
  39. from pylib.base import output_manager_factory
  40. from pylib.base import test_instance_factory
  41. from pylib.base import test_run_factory
  42. from pylib.results import json_results
  43. from pylib.results import report_results
  44. from pylib.results.presentation import test_results_presentation
  45. from pylib.utils import local_utils
  46. from pylib.utils import logdog_helper
  47. from pylib.utils import logging_utils
  48. from pylib.utils import test_filter
  49. from py_utils import contextlib_ext
  50. from lib.results import result_sink # pylint: disable=import-error
  51. _DEVIL_STATIC_CONFIG_FILE = os.path.abspath(os.path.join(
  52. host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'devil_config.json'))
  53. _RERUN_FAILED_TESTS_FILE = 'rerun_failed_tests.filter'
  54. def _RealPath(arg):
  55. if arg.startswith('//'):
  56. arg = os.path.abspath(os.path.join(host_paths.DIR_SOURCE_ROOT,
  57. arg[2:].replace('/', os.sep)))
  58. return os.path.realpath(arg)
  59. def AddTestLauncherOptions(parser):
  60. """Adds arguments mirroring //base/test/launcher.
  61. Args:
  62. parser: The parser to which arguments should be added.
  63. Returns:
  64. The given parser.
  65. """
  66. parser.add_argument(
  67. '--test-launcher-retry-limit',
  68. '--test_launcher_retry_limit',
  69. '--num_retries', '--num-retries',
  70. '--isolated-script-test-launcher-retry-limit',
  71. dest='num_retries', type=int, default=2,
  72. help='Number of retries for a test before '
  73. 'giving up (default: %(default)s).')
  74. parser.add_argument(
  75. '--test-launcher-summary-output',
  76. '--json-results-file',
  77. dest='json_results_file', type=os.path.realpath,
  78. help='If set, will dump results in JSON form to the specified file. '
  79. 'Note that this will also trigger saving per-test logcats to '
  80. 'logdog.')
  81. parser.add_argument(
  82. '--test-launcher-shard-index',
  83. type=int, default=os.environ.get('GTEST_SHARD_INDEX', 0),
  84. help='Index of the external shard to run.')
  85. parser.add_argument(
  86. '--test-launcher-total-shards',
  87. type=int, default=os.environ.get('GTEST_TOTAL_SHARDS', 1),
  88. help='Total number of external shards.')
  89. test_filter.AddFilterOptions(parser)
  90. return parser
  91. def AddCommandLineOptions(parser):
  92. """Adds arguments to support passing command-line flags to the device."""
  93. parser.add_argument(
  94. '--device-flags-file',
  95. type=os.path.realpath,
  96. help='The relative filepath to a file containing '
  97. 'command-line flags to set on the device')
  98. parser.add_argument(
  99. '--use-apk-under-test-flags-file',
  100. action='store_true',
  101. help='Wether to use the flags file for the apk under test. If set, '
  102. "the filename will be looked up in the APK's PackageInfo.")
  103. parser.set_defaults(allow_unknown=True)
  104. parser.set_defaults(command_line_flags=None)
  105. def AddTracingOptions(parser):
  106. # TODO(shenghuazhang): Move this into AddCommonOptions once it's supported
  107. # for all test types.
  108. parser.add_argument(
  109. '--trace-output',
  110. metavar='FILENAME', type=os.path.realpath,
  111. help='Path to save test_runner trace json output to.')
  112. parser.add_argument(
  113. '--trace-all',
  114. action='store_true',
  115. help='Whether to trace all function calls.')
  116. def AddCommonOptions(parser):
  117. """Adds all common options to |parser|."""
  118. default_build_type = os.environ.get('BUILDTYPE', 'Debug')
  119. debug_or_release_group = parser.add_mutually_exclusive_group()
  120. debug_or_release_group.add_argument(
  121. '--debug',
  122. action='store_const', const='Debug', dest='build_type',
  123. default=default_build_type,
  124. help='If set, run test suites under out/Debug. '
  125. 'Default is env var BUILDTYPE or Debug.')
  126. debug_or_release_group.add_argument(
  127. '--release',
  128. action='store_const', const='Release', dest='build_type',
  129. help='If set, run test suites under out/Release. '
  130. 'Default is env var BUILDTYPE or Debug.')
  131. parser.add_argument(
  132. '--break-on-failure', '--break_on_failure',
  133. dest='break_on_failure', action='store_true',
  134. help='Whether to break on failure.')
  135. # TODO(jbudorick): Remove this once everything has switched to platform
  136. # mode.
  137. parser.add_argument(
  138. '--enable-platform-mode',
  139. action='store_true',
  140. help='Run the test scripts in platform mode, which '
  141. 'conceptually separates the test runner from the '
  142. '"device" (local or remote, real or emulated) on '
  143. 'which the tests are running. [experimental]')
  144. parser.add_argument(
  145. '-e', '--environment',
  146. default='local', choices=constants.VALID_ENVIRONMENTS,
  147. help='Test environment to run in (default: %(default)s).')
  148. parser.add_argument(
  149. '--local-output',
  150. action='store_true',
  151. help='Whether to archive test output locally and generate '
  152. 'a local results detail page.')
  153. parser.add_argument('--list-tests',
  154. action='store_true',
  155. help='List available tests and exit.')
  156. parser.add_argument('--wrapper-script-args',
  157. help='A string of args that were passed to the wrapper '
  158. 'script. This should probably not be edited by a '
  159. 'user as it is passed by the wrapper itself.')
  160. class FastLocalDevAction(argparse.Action):
  161. def __call__(self, parser, namespace, values, option_string=None):
  162. namespace.enable_concurrent_adb = True
  163. namespace.enable_device_cache = True
  164. namespace.extract_test_list_from_filter = True
  165. namespace.local_output = True
  166. namespace.num_retries = 0
  167. namespace.skip_clear_data = True
  168. parser.add_argument(
  169. '--fast-local-dev',
  170. type=bool,
  171. nargs=0,
  172. action=FastLocalDevAction,
  173. help='Alias for: --num-retries=0 --enable-device-cache '
  174. '--enable-concurrent-adb --skip-clear-data '
  175. '--extract-test-list-from-filter --local-output')
  176. # TODO(jbudorick): Remove this once downstream bots have switched to
  177. # api.test_results.
  178. parser.add_argument(
  179. '--flakiness-dashboard-server',
  180. dest='flakiness_dashboard_server',
  181. help=argparse.SUPPRESS)
  182. parser.add_argument(
  183. '--gs-results-bucket',
  184. help='Google Storage bucket to upload results to.')
  185. parser.add_argument(
  186. '--output-directory',
  187. dest='output_directory', type=os.path.realpath,
  188. help='Path to the directory in which build files are'
  189. ' located (must include build type). This will take'
  190. ' precedence over --debug and --release')
  191. parser.add_argument(
  192. '-v', '--verbose',
  193. dest='verbose_count', default=0, action='count',
  194. help='Verbose level (multiple times for more)')
  195. parser.add_argument(
  196. '--repeat', '--gtest_repeat', '--gtest-repeat',
  197. '--isolated-script-test-repeat',
  198. dest='repeat', type=int, default=0,
  199. help='Number of times to repeat the specified set of tests.')
  200. # This is currently only implemented for gtests and instrumentation tests.
  201. parser.add_argument(
  202. '--gtest_also_run_disabled_tests', '--gtest-also-run-disabled-tests',
  203. '--isolated-script-test-also-run-disabled-tests',
  204. dest='run_disabled', action='store_true',
  205. help='Also run disabled tests if applicable.')
  206. # These are currently only implemented for gtests.
  207. parser.add_argument('--isolated-script-test-output',
  208. help='If present, store test results on this path.')
  209. parser.add_argument('--isolated-script-test-perf-output',
  210. help='If present, store chartjson results on this path.')
  211. AddTestLauncherOptions(parser)
  212. def ProcessCommonOptions(args):
  213. """Processes and handles all common options."""
  214. run_tests_helper.SetLogLevel(args.verbose_count, add_handler=False)
  215. if args.verbose_count > 0:
  216. handler = logging_utils.ColorStreamHandler()
  217. else:
  218. handler = logging.StreamHandler(sys.stdout)
  219. handler.setFormatter(run_tests_helper.CustomFormatter())
  220. logging.getLogger().addHandler(handler)
  221. constants.SetBuildType(args.build_type)
  222. if args.output_directory:
  223. constants.SetOutputDirectory(args.output_directory)
  224. def AddDeviceOptions(parser):
  225. """Adds device options to |parser|."""
  226. parser = parser.add_argument_group('device arguments')
  227. parser.add_argument(
  228. '--adb-path',
  229. type=os.path.realpath,
  230. help='Specify the absolute path of the adb binary that '
  231. 'should be used.')
  232. parser.add_argument('--denylist-file',
  233. type=os.path.realpath,
  234. help='Device denylist file.')
  235. parser.add_argument(
  236. '-d', '--device', nargs='+',
  237. dest='test_devices',
  238. help='Target device(s) for the test suite to run on.')
  239. parser.add_argument(
  240. '--enable-concurrent-adb',
  241. action='store_true',
  242. help='Run multiple adb commands at the same time, even '
  243. 'for the same device.')
  244. parser.add_argument(
  245. '--enable-device-cache',
  246. action='store_true',
  247. help='Cache device state to disk between runs')
  248. parser.add_argument(
  249. '--skip-clear-data',
  250. action='store_true',
  251. help='Do not wipe app data between tests. Use this to '
  252. 'speed up local development and never on bots '
  253. '(increases flakiness)')
  254. parser.add_argument(
  255. '--recover-devices',
  256. action='store_true',
  257. help='Attempt to recover devices prior to the final retry. Warning: '
  258. 'this will cause all devices to reboot.')
  259. parser.add_argument(
  260. '--tool',
  261. dest='tool',
  262. help='Run the test under a tool '
  263. '(use --tool help to list them)')
  264. parser.add_argument(
  265. '--upload-logcats-file',
  266. action='store_true',
  267. dest='upload_logcats_file',
  268. help='Whether to upload logcat file to logdog.')
  269. logcat_output_group = parser.add_mutually_exclusive_group()
  270. logcat_output_group.add_argument(
  271. '--logcat-output-dir', type=os.path.realpath,
  272. help='If set, will dump logcats recorded during test run to directory. '
  273. 'File names will be the device ids with timestamps.')
  274. logcat_output_group.add_argument(
  275. '--logcat-output-file', type=os.path.realpath,
  276. help='If set, will merge logcats recorded during test run and dump them '
  277. 'to the specified file.')
  278. def AddEmulatorOptions(parser):
  279. """Adds emulator-specific options to |parser|."""
  280. parser = parser.add_argument_group('emulator arguments')
  281. parser.add_argument(
  282. '--avd-config',
  283. type=os.path.realpath,
  284. help='Path to the avd config textpb. '
  285. '(See //tools/android/avd/proto/ for message definition'
  286. ' and existing textpb files.)')
  287. parser.add_argument(
  288. '--emulator-count',
  289. type=int,
  290. default=1,
  291. help='Number of emulators to use.')
  292. parser.add_argument(
  293. '--emulator-window',
  294. action='store_true',
  295. default=False,
  296. help='Enable graphical window display on the emulator.')
  297. def AddGTestOptions(parser):
  298. """Adds gtest options to |parser|."""
  299. parser = parser.add_argument_group('gtest arguments')
  300. parser.add_argument(
  301. '--app-data-file',
  302. action='append', dest='app_data_files',
  303. help='A file path relative to the app data directory '
  304. 'that should be saved to the host.')
  305. parser.add_argument(
  306. '--app-data-file-dir',
  307. help='Host directory to which app data files will be'
  308. ' saved. Used with --app-data-file.')
  309. parser.add_argument(
  310. '--enable-xml-result-parsing',
  311. action='store_true', help=argparse.SUPPRESS)
  312. parser.add_argument(
  313. '--executable-dist-dir',
  314. type=os.path.realpath,
  315. help="Path to executable's dist directory for native"
  316. " (non-apk) tests.")
  317. parser.add_argument(
  318. '--extract-test-list-from-filter',
  319. action='store_true',
  320. help='When a test filter is specified, and the list of '
  321. 'tests can be determined from it, skip querying the '
  322. 'device for the list of all tests. Speeds up local '
  323. 'development, but is not safe to use on bots ('
  324. 'http://crbug.com/549214')
  325. parser.add_argument(
  326. '--gs-test-artifacts-bucket',
  327. help=('If present, test artifacts will be uploaded to this Google '
  328. 'Storage bucket.'))
  329. parser.add_argument(
  330. '--render-test-output-dir',
  331. help='If present, store rendering artifacts in this path.')
  332. parser.add_argument(
  333. '--runtime-deps-path',
  334. dest='runtime_deps_path', type=os.path.realpath,
  335. help='Runtime data dependency file from GN.')
  336. parser.add_argument(
  337. '-t', '--shard-timeout',
  338. dest='shard_timeout', type=int, default=120,
  339. help='Timeout to wait for each test (default: %(default)s).')
  340. parser.add_argument(
  341. '--store-tombstones',
  342. dest='store_tombstones', action='store_true',
  343. help='Add tombstones in results if crash.')
  344. parser.add_argument(
  345. '-s', '--suite',
  346. dest='suite_name', nargs='+', metavar='SUITE_NAME', required=True,
  347. help='Executable name of the test suite to run.')
  348. parser.add_argument(
  349. '--test-apk-incremental-install-json',
  350. type=os.path.realpath,
  351. help='Path to install json for the test apk.')
  352. parser.add_argument('--test-launcher-batch-limit',
  353. dest='test_launcher_batch_limit',
  354. type=int,
  355. help='The max number of tests to run in a shard. '
  356. 'Ignores non-positive ints and those greater than '
  357. 'MAX_SHARDS')
  358. parser.add_argument(
  359. '-w', '--wait-for-java-debugger', action='store_true',
  360. help='Wait for java debugger to attach before running any application '
  361. 'code. Also disables test timeouts and sets retries=0.')
  362. parser.add_argument(
  363. '--coverage-dir',
  364. type=os.path.realpath,
  365. help='Directory in which to place all generated coverage files.')
  366. parser.add_argument(
  367. '--use-existing-test-data',
  368. action='store_true',
  369. help='Do not push new files to the device, instead using existing APK '
  370. 'and test data. Only use when running the same test for multiple '
  371. 'iterations.')
  372. def AddInstrumentationTestOptions(parser):
  373. """Adds Instrumentation test options to |parser|."""
  374. parser = parser.add_argument_group('instrumentation arguments')
  375. parser.add_argument('--additional-apex',
  376. action='append',
  377. dest='additional_apexs',
  378. default=[],
  379. type=_RealPath,
  380. help='Additional apex that must be installed on '
  381. 'the device when the tests are run')
  382. parser.add_argument(
  383. '--additional-apk',
  384. action='append', dest='additional_apks', default=[],
  385. type=_RealPath,
  386. help='Additional apk that must be installed on '
  387. 'the device when the tests are run')
  388. parser.add_argument('--forced-queryable-additional-apk',
  389. action='append',
  390. dest='forced_queryable_additional_apks',
  391. default=[],
  392. type=_RealPath,
  393. help='Configures an additional-apk to be forced '
  394. 'to be queryable by other APKs.')
  395. parser.add_argument('--instant-additional-apk',
  396. action='append',
  397. dest='instant_additional_apks',
  398. default=[],
  399. type=_RealPath,
  400. help='Configures an additional-apk to be an instant APK')
  401. parser.add_argument(
  402. '-A', '--annotation',
  403. dest='annotation_str',
  404. help='Comma-separated list of annotations. Run only tests with any of '
  405. 'the given annotations. An annotation can be either a key or a '
  406. 'key-values pair. A test that has no annotation is considered '
  407. '"SmallTest".')
  408. # TODO(jbudorick): Remove support for name-style APK specification once
  409. # bots are no longer doing it.
  410. parser.add_argument(
  411. '--apk-under-test',
  412. help='Path or name of the apk under test.')
  413. parser.add_argument(
  414. '--store-data-in-app-directory',
  415. action='store_true',
  416. help='Store test data in the application\'s data directory. By default '
  417. 'the test data is stored in the external storage folder.')
  418. parser.add_argument(
  419. '--module',
  420. action='append',
  421. dest='modules',
  422. help='Specify Android App Bundle modules to install in addition to the '
  423. 'base module.')
  424. parser.add_argument(
  425. '--fake-module',
  426. action='append',
  427. dest='fake_modules',
  428. help='Specify Android App Bundle modules to fake install in addition to '
  429. 'the real modules.')
  430. parser.add_argument(
  431. '--additional-locale',
  432. action='append',
  433. dest='additional_locales',
  434. help='Specify locales in addition to the device locale to install splits '
  435. 'for when --apk-under-test is an Android App Bundle.')
  436. parser.add_argument(
  437. '--coverage-dir',
  438. type=os.path.realpath,
  439. help='Directory in which to place all generated '
  440. 'Jacoco coverage files.')
  441. parser.add_argument(
  442. '--disable-dalvik-asserts',
  443. dest='set_asserts', action='store_false', default=True,
  444. help='Removes the dalvik.vm.enableassertions property')
  445. parser.add_argument(
  446. '--proguard-mapping-path',
  447. help='.mapping file to use to Deobfuscate java stack traces in test '
  448. 'output and logcat.')
  449. parser.add_argument(
  450. '-E', '--exclude-annotation',
  451. dest='exclude_annotation_str',
  452. help='Comma-separated list of annotations. Exclude tests with these '
  453. 'annotations.')
  454. parser.add_argument(
  455. '--enable-breakpad-dump',
  456. action='store_true',
  457. help='Stores any breakpad dumps till the end of the test.')
  458. parser.add_argument(
  459. '--replace-system-package',
  460. type=_RealPath,
  461. default=None,
  462. help='Use this apk to temporarily replace a system package with the same '
  463. 'package name.')
  464. parser.add_argument(
  465. '--remove-system-package',
  466. default=[],
  467. action='append',
  468. dest='system_packages_to_remove',
  469. help='Specifies a system package to remove before testing if it exists '
  470. 'on the system. WARNING: THIS WILL PERMANENTLY REMOVE THE SYSTEM APP. '
  471. 'Unlike --replace-system-package, the app will not be restored after '
  472. 'tests are finished.')
  473. parser.add_argument(
  474. '--use-voice-interaction-service',
  475. help='This can be used to update the voice interaction service to be a '
  476. 'custom one. This is useful for mocking assistants. eg: '
  477. 'android.assist.service/.MainInteractionService')
  478. parser.add_argument(
  479. '--use-webview-provider',
  480. type=_RealPath, default=None,
  481. help='Use this apk as the webview provider during test. '
  482. 'The original provider will be restored if possible, '
  483. "on Nougat the provider can't be determined and so "
  484. 'the system will choose the default provider.')
  485. parser.add_argument(
  486. '--runtime-deps-path',
  487. dest='runtime_deps_path', type=os.path.realpath,
  488. help='Runtime data dependency file from GN.')
  489. parser.add_argument(
  490. '--screenshot-directory',
  491. dest='screenshot_dir', type=os.path.realpath,
  492. help='Capture screenshots of test failures')
  493. parser.add_argument(
  494. '--shared-prefs-file',
  495. dest='shared_prefs_file', type=_RealPath,
  496. help='The relative path to a file containing JSON list of shared '
  497. 'preference files to edit and how to do so. Example list: '
  498. '[{'
  499. ' "package": "com.package.example",'
  500. ' "filename": "ExampleSettings.xml",'
  501. ' "set": {'
  502. ' "boolean_key_in_xml": true,'
  503. ' "string_key_in_xml": "string_value"'
  504. ' },'
  505. ' "remove": ['
  506. ' "key_in_xml_to_remove"'
  507. ' ]'
  508. '}]')
  509. parser.add_argument(
  510. '--store-tombstones',
  511. action='store_true', dest='store_tombstones',
  512. help='Add tombstones in results if crash.')
  513. parser.add_argument(
  514. '--strict-mode',
  515. dest='strict_mode', default='testing',
  516. help='StrictMode command-line flag set on the device, '
  517. 'death/testing to kill the process, off to stop '
  518. 'checking, flash to flash only. (default: %(default)s)')
  519. parser.add_argument(
  520. '--test-apk',
  521. required=True,
  522. help='Path or name of the apk containing the tests.')
  523. parser.add_argument(
  524. '--test-apk-as-instant',
  525. action='store_true',
  526. help='Install the test apk as an instant app. '
  527. 'Instant apps run in a more restrictive execution environment.')
  528. parser.add_argument(
  529. '--test-launcher-batch-limit',
  530. dest='test_launcher_batch_limit',
  531. type=int,
  532. help=('Not actually used for instrumentation tests, but can be used as '
  533. 'a proxy for determining if the current run is a retry without '
  534. 'patch.'))
  535. parser.add_argument(
  536. '--timeout-scale',
  537. type=float,
  538. help='Factor by which timeouts should be scaled.')
  539. parser.add_argument(
  540. '--is-unit-test',
  541. action='store_true',
  542. help=('Specify the test suite as composed of unit tests, blocking '
  543. 'certain operations.'))
  544. parser.add_argument(
  545. '-w', '--wait-for-java-debugger', action='store_true',
  546. help='Wait for java debugger to attach before running any application '
  547. 'code. Also disables test timeouts and sets retries=0.')
  548. # WPR record mode.
  549. parser.add_argument('--wpr-enable-record',
  550. action='store_true',
  551. default=False,
  552. help='If true, WPR server runs in record mode.'
  553. 'otherwise, runs in replay mode.')
  554. parser.add_argument(
  555. '--approve-app-links',
  556. help='Force enables Digital Asset Link verification for the provided '
  557. 'package and domain, example usage: --approve-app-links '
  558. 'com.android.package:www.example.com')
  559. # These arguments are suppressed from the help text because they should
  560. # only ever be specified by an intermediate script.
  561. parser.add_argument(
  562. '--apk-under-test-incremental-install-json',
  563. help=argparse.SUPPRESS)
  564. parser.add_argument(
  565. '--test-apk-incremental-install-json',
  566. type=os.path.realpath,
  567. help=argparse.SUPPRESS)
  568. def AddSkiaGoldTestOptions(parser):
  569. """Adds Skia Gold test options to |parser|."""
  570. parser = parser.add_argument_group("Skia Gold arguments")
  571. parser.add_argument(
  572. '--code-review-system',
  573. help='A non-default code review system to pass to pass to Gold, if '
  574. 'applicable')
  575. parser.add_argument(
  576. '--continuous-integration-system',
  577. help='A non-default continuous integration system to pass to Gold, if '
  578. 'applicable')
  579. parser.add_argument(
  580. '--git-revision', help='The git commit currently being tested.')
  581. parser.add_argument(
  582. '--gerrit-issue',
  583. help='The Gerrit issue this test is being run on, if applicable.')
  584. parser.add_argument(
  585. '--gerrit-patchset',
  586. help='The Gerrit patchset this test is being run on, if applicable.')
  587. parser.add_argument(
  588. '--buildbucket-id',
  589. help='The Buildbucket build ID that this test was triggered from, if '
  590. 'applicable.')
  591. local_group = parser.add_mutually_exclusive_group()
  592. local_group.add_argument(
  593. '--local-pixel-tests',
  594. action='store_true',
  595. default=None,
  596. help='Specifies to run the Skia Gold pixel tests in local mode. When run '
  597. 'in local mode, uploading to Gold is disabled and traditional '
  598. 'generated/golden/diff images are output instead of triage links. '
  599. 'Running in local mode also implies --no-luci-auth. If both this '
  600. 'and --no-local-pixel-tests are left unset, the test harness will '
  601. 'attempt to detect whether it is running on a workstation or not '
  602. 'and set the options accordingly.')
  603. local_group.add_argument(
  604. '--no-local-pixel-tests',
  605. action='store_false',
  606. dest='local_pixel_tests',
  607. help='Specifies to run the Skia Gold pixel tests in non-local (bot) '
  608. 'mode. When run in this mode, data is actually uploaded to Gold and '
  609. 'triage links are generated. If both this and --local-pixel-tests '
  610. 'are left unset, the test harness will attempt to detect whether '
  611. 'it is running on a workstation or not and set the options '
  612. 'accordingly.')
  613. parser.add_argument(
  614. '--no-luci-auth',
  615. action='store_true',
  616. default=False,
  617. help="Don't use the serve account provided by LUCI for authentication "
  618. 'with Skia Gold, instead relying on gsutil to be pre-authenticated. '
  619. 'Meant for testing locally instead of on the bots.')
  620. parser.add_argument(
  621. '--bypass-skia-gold-functionality',
  622. action='store_true',
  623. default=False,
  624. help='Bypass all interaction with Skia Gold, effectively disabling the '
  625. 'image comparison portion of any tests that use Gold. Only meant to be '
  626. 'used in case a Gold outage occurs and cannot be fixed quickly.')
  627. def AddJUnitTestOptions(parser):
  628. """Adds junit test options to |parser|."""
  629. parser = parser.add_argument_group('junit arguments')
  630. parser.add_argument(
  631. '--coverage-on-the-fly',
  632. action='store_true',
  633. help='Generate coverage data by Jacoco on-the-fly instrumentation.')
  634. parser.add_argument(
  635. '--coverage-dir', type=os.path.realpath,
  636. help='Directory to store coverage info.')
  637. parser.add_argument(
  638. '--package-filter',
  639. help='Filters tests by package.')
  640. parser.add_argument(
  641. '--runner-filter',
  642. help='Filters tests by runner class. Must be fully qualified.')
  643. parser.add_argument(
  644. '--shards',
  645. default=-1,
  646. type=int,
  647. help='Number of shards to run junit tests in parallel on. Only 1 shard '
  648. 'is supported when test-filter is specified. Values less than 1 will '
  649. 'use auto select.')
  650. parser.add_argument(
  651. '-s', '--test-suite', required=True,
  652. help='JUnit test suite to run.')
  653. debug_group = parser.add_mutually_exclusive_group()
  654. debug_group.add_argument(
  655. '-w', '--wait-for-java-debugger', action='store_const', const='8701',
  656. dest='debug_socket', help='Alias for --debug-socket=8701')
  657. debug_group.add_argument(
  658. '--debug-socket',
  659. help='Wait for java debugger to attach at specified socket address '
  660. 'before running any application code. Also disables test timeouts '
  661. 'and sets retries=0.')
  662. # These arguments are for Android Robolectric tests.
  663. parser.add_argument(
  664. '--robolectric-runtime-deps-dir',
  665. help='Path to runtime deps for Robolectric.')
  666. parser.add_argument(
  667. '--resource-apk',
  668. required=True,
  669. help='Path to .ap_ containing binary resources for Robolectric.')
  670. def AddLinkerTestOptions(parser):
  671. parser = parser.add_argument_group('linker arguments')
  672. parser.add_argument(
  673. '--test-apk',
  674. type=os.path.realpath,
  675. help='Path to the linker test APK.')
  676. def AddMonkeyTestOptions(parser):
  677. """Adds monkey test options to |parser|."""
  678. parser = parser.add_argument_group('monkey arguments')
  679. parser.add_argument('--browser',
  680. required=True,
  681. choices=list(constants.PACKAGE_INFO.keys()),
  682. metavar='BROWSER',
  683. help='Browser under test.')
  684. parser.add_argument(
  685. '--category',
  686. nargs='*', dest='categories', default=[],
  687. help='A list of allowed categories. Monkey will only visit activities '
  688. 'that are listed with one of the specified categories.')
  689. parser.add_argument(
  690. '--event-count',
  691. default=10000, type=int,
  692. help='Number of events to generate (default: %(default)s).')
  693. parser.add_argument(
  694. '--seed',
  695. type=int,
  696. help='Seed value for pseudo-random generator. Same seed value generates '
  697. 'the same sequence of events. Seed is randomized by default.')
  698. parser.add_argument(
  699. '--throttle',
  700. default=100, type=int,
  701. help='Delay between events (ms) (default: %(default)s). ')
  702. def AddPythonTestOptions(parser):
  703. parser = parser.add_argument_group('python arguments')
  704. parser.add_argument('-s',
  705. '--suite',
  706. dest='suite_name',
  707. metavar='SUITE_NAME',
  708. choices=list(constants.PYTHON_UNIT_TEST_SUITES.keys()),
  709. help='Name of the test suite to run.')
  710. def _CreateClassToFileNameDict(test_apk):
  711. """Creates a dict mapping classes to file names from size-info apk."""
  712. constants.CheckOutputDirectory()
  713. test_apk_size_info = os.path.join(constants.GetOutDirectory(), 'size-info',
  714. os.path.basename(test_apk) + '.jar.info')
  715. class_to_file_dict = {}
  716. # Some tests such as webview_cts_tests use a separately downloaded apk to run
  717. # tests. This means the apk may not have been built by the system and hence
  718. # no size info file exists.
  719. if not os.path.exists(test_apk_size_info):
  720. logging.debug('Apk size file not found. %s', test_apk_size_info)
  721. return class_to_file_dict
  722. with open(test_apk_size_info, 'r') as f:
  723. for line in f:
  724. file_class, file_name = line.rstrip().split(',', 1)
  725. # Only want files that are not prebuilt.
  726. if file_name.startswith('../../'):
  727. class_to_file_dict[file_class] = str(
  728. file_name.replace('../../', '//', 1))
  729. return class_to_file_dict
  730. def _RunPythonTests(args):
  731. """Subcommand of RunTestsCommand which runs python unit tests."""
  732. suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
  733. suite_path = suite_vars['path']
  734. suite_test_modules = suite_vars['test_modules']
  735. sys.path = [suite_path] + sys.path
  736. try:
  737. suite = unittest.TestSuite()
  738. suite.addTests(unittest.defaultTestLoader.loadTestsFromName(m)
  739. for m in suite_test_modules)
  740. runner = unittest.TextTestRunner(verbosity=1+args.verbose_count)
  741. return 0 if runner.run(suite).wasSuccessful() else 1
  742. finally:
  743. sys.path = sys.path[1:]
  744. _DEFAULT_PLATFORM_MODE_TESTS = [
  745. 'gtest', 'instrumentation', 'junit', 'linker', 'monkey'
  746. ]
  747. def RunTestsCommand(args, result_sink_client=None):
  748. """Checks test type and dispatches to the appropriate function.
  749. Args:
  750. args: argparse.Namespace object.
  751. result_sink_client: A ResultSinkClient object.
  752. Returns:
  753. Integer indicated exit code.
  754. Raises:
  755. Exception: Unknown command name passed in, or an exception from an
  756. individual test runner.
  757. """
  758. command = args.command
  759. ProcessCommonOptions(args)
  760. logging.info('command: %s', ' '.join(sys.argv))
  761. if args.enable_platform_mode or command in _DEFAULT_PLATFORM_MODE_TESTS:
  762. return RunTestsInPlatformMode(args, result_sink_client)
  763. if command == 'python':
  764. return _RunPythonTests(args)
  765. raise Exception('Unknown test type.')
  766. def _SinkTestResult(test_result, test_file_name, result_sink_client):
  767. """Upload test result to result_sink.
  768. Args:
  769. test_result: A BaseTestResult object
  770. test_file_name: A string representing the file location of the test
  771. result_sink_client: A ResultSinkClient object
  772. Returns:
  773. N/A
  774. """
  775. # Some tests put in non utf-8 char as part of the test
  776. # which breaks uploads, so need to decode and re-encode.
  777. log_decoded = test_result.GetLog()
  778. if isinstance(log_decoded, bytes):
  779. log_decoded = log_decoded.decode('utf-8', 'replace')
  780. html_artifact = ''
  781. https_artifacts = []
  782. for link_name, link_url in sorted(test_result.GetLinks().items()):
  783. if link_url.startswith('https:'):
  784. https_artifacts.append('<li><a target="_blank" href=%s>%s</a></li>' %
  785. (link_url, link_name))
  786. else:
  787. logging.info('Skipping non-https link %r (%s) for test %s.', link_name,
  788. link_url, test_result.GetName())
  789. if https_artifacts:
  790. html_artifact += '<ul>%s</ul>' % '\n'.join(https_artifacts)
  791. result_sink_client.Post(test_result.GetName(),
  792. test_result.GetType(),
  793. test_result.GetDuration(),
  794. log_decoded.encode('utf-8'),
  795. test_file_name,
  796. failure_reason=test_result.GetFailureReason(),
  797. html_artifact=html_artifact)
  798. _SUPPORTED_IN_PLATFORM_MODE = [
  799. # TODO(jbudorick): Add support for more test types.
  800. 'gtest',
  801. 'instrumentation',
  802. 'junit',
  803. 'linker',
  804. 'monkey',
  805. ]
  806. def RunTestsInPlatformMode(args, result_sink_client=None):
  807. def infra_error(message):
  808. logging.fatal(message)
  809. sys.exit(constants.INFRA_EXIT_CODE)
  810. if args.command not in _SUPPORTED_IN_PLATFORM_MODE:
  811. infra_error('%s is not yet supported in platform mode' % args.command)
  812. ### Set up sigterm handler.
  813. contexts_to_notify_on_sigterm = []
  814. def unexpected_sigterm(_signum, _frame):
  815. msg = [
  816. 'Received SIGTERM. Shutting down.',
  817. ]
  818. for live_thread in threading.enumerate():
  819. # pylint: disable=protected-access
  820. thread_stack = ''.join(traceback.format_stack(
  821. sys._current_frames()[live_thread.ident]))
  822. msg.extend([
  823. 'Thread "%s" (ident: %s) is currently running:' % (
  824. live_thread.name, live_thread.ident),
  825. thread_stack])
  826. for context in contexts_to_notify_on_sigterm:
  827. context.ReceivedSigterm()
  828. infra_error('\n'.join(msg))
  829. signal.signal(signal.SIGTERM, unexpected_sigterm)
  830. ### Set up results handling.
  831. # TODO(jbudorick): Rewrite results handling.
  832. # all_raw_results is a list of lists of
  833. # base_test_result.TestRunResults objects. Each instance of
  834. # TestRunResults contains all test results produced by a single try,
  835. # while each list of TestRunResults contains all tries in a single
  836. # iteration.
  837. all_raw_results = []
  838. # all_iteration_results is a list of base_test_result.TestRunResults
  839. # objects. Each instance of TestRunResults contains the last test
  840. # result for each test run in that iteration.
  841. all_iteration_results = []
  842. global_results_tags = set()
  843. json_file = tempfile.NamedTemporaryFile(delete=False)
  844. json_file.close()
  845. @contextlib.contextmanager
  846. def json_finalizer():
  847. try:
  848. yield
  849. finally:
  850. if args.json_results_file and os.path.exists(json_file.name):
  851. shutil.move(json_file.name, args.json_results_file)
  852. elif args.isolated_script_test_output and os.path.exists(json_file.name):
  853. shutil.move(json_file.name, args.isolated_script_test_output)
  854. else:
  855. os.remove(json_file.name)
  856. @contextlib.contextmanager
  857. def json_writer():
  858. try:
  859. yield
  860. except Exception:
  861. global_results_tags.add('UNRELIABLE_RESULTS')
  862. raise
  863. finally:
  864. if args.isolated_script_test_output:
  865. interrupted = 'UNRELIABLE_RESULTS' in global_results_tags
  866. json_results.GenerateJsonTestResultFormatFile(all_raw_results,
  867. interrupted,
  868. json_file.name,
  869. indent=2)
  870. else:
  871. json_results.GenerateJsonResultsFile(
  872. all_raw_results,
  873. json_file.name,
  874. global_tags=list(global_results_tags),
  875. indent=2)
  876. test_class_to_file_name_dict = {}
  877. # Test Location is only supported for instrumentation tests as it
  878. # requires the size-info file.
  879. if test_instance.TestType() == 'instrumentation':
  880. test_class_to_file_name_dict = _CreateClassToFileNameDict(args.test_apk)
  881. if result_sink_client:
  882. for run in all_raw_results:
  883. for results in run:
  884. for r in results.GetAll():
  885. # Matches chrome.page_info.PageInfoViewTest#testChromePage
  886. match = re.search(r'^(.+\..+)#', r.GetName())
  887. test_file_name = test_class_to_file_name_dict.get(
  888. match.group(1)) if match else None
  889. _SinkTestResult(r, test_file_name, result_sink_client)
  890. @contextlib.contextmanager
  891. def upload_logcats_file():
  892. try:
  893. yield
  894. finally:
  895. if not args.logcat_output_file:
  896. logging.critical('Cannot upload logcat file: no file specified.')
  897. elif not os.path.exists(args.logcat_output_file):
  898. logging.critical("Cannot upload logcat file: file doesn't exist.")
  899. else:
  900. with open(args.logcat_output_file) as src:
  901. dst = logdog_helper.open_text('unified_logcats')
  902. if dst:
  903. shutil.copyfileobj(src, dst)
  904. dst.close()
  905. logging.critical(
  906. 'Logcat: %s', logdog_helper.get_viewer_url('unified_logcats'))
  907. logcats_uploader = contextlib_ext.Optional(
  908. upload_logcats_file(),
  909. 'upload_logcats_file' in args and args.upload_logcats_file)
  910. save_detailed_results = (args.local_output or not local_utils.IsOnSwarming()
  911. ) and not args.isolated_script_test_output
  912. ### Set up test objects.
  913. out_manager = output_manager_factory.CreateOutputManager(args)
  914. env = environment_factory.CreateEnvironment(
  915. args, out_manager, infra_error)
  916. test_instance = test_instance_factory.CreateTestInstance(args, infra_error)
  917. test_run = test_run_factory.CreateTestRun(env, test_instance, infra_error)
  918. contexts_to_notify_on_sigterm.append(env)
  919. contexts_to_notify_on_sigterm.append(test_run)
  920. if args.list_tests:
  921. try:
  922. with out_manager, env, test_instance, test_run:
  923. test_names = test_run.GetTestsForListing()
  924. print('There are {} tests:'.format(len(test_names)))
  925. for n in test_names:
  926. print(n)
  927. return 0
  928. except NotImplementedError:
  929. sys.stderr.write('Test does not support --list-tests (type={}).\n'.format(
  930. args.command))
  931. return 1
  932. ### Run.
  933. with out_manager, json_finalizer():
  934. # |raw_logs_fh| is only used by Robolectric tests.
  935. raw_logs_fh = io.StringIO() if save_detailed_results else None
  936. with json_writer(), logcats_uploader, env, test_instance, test_run:
  937. repetitions = (range(args.repeat +
  938. 1) if args.repeat >= 0 else itertools.count())
  939. result_counts = collections.defaultdict(
  940. lambda: collections.defaultdict(int))
  941. iteration_count = 0
  942. for _ in repetitions:
  943. # raw_results will be populated with base_test_result.TestRunResults by
  944. # test_run.RunTests(). It is immediately added to all_raw_results so
  945. # that in the event of an exception, all_raw_results will already have
  946. # the up-to-date results and those can be written to disk.
  947. raw_results = []
  948. all_raw_results.append(raw_results)
  949. test_run.RunTests(raw_results, raw_logs_fh=raw_logs_fh)
  950. if not raw_results:
  951. all_raw_results.pop()
  952. continue
  953. iteration_results = base_test_result.TestRunResults()
  954. for r in reversed(raw_results):
  955. iteration_results.AddTestRunResults(r)
  956. all_iteration_results.append(iteration_results)
  957. iteration_count += 1
  958. for r in iteration_results.GetAll():
  959. result_counts[r.GetName()][r.GetType()] += 1
  960. report_results.LogFull(
  961. results=iteration_results,
  962. test_type=test_instance.TestType(),
  963. test_package=test_run.TestPackage(),
  964. annotation=getattr(args, 'annotations', None),
  965. flakiness_server=getattr(args, 'flakiness_dashboard_server',
  966. None))
  967. if iteration_results.GetNotPass():
  968. _LogRerunStatement(iteration_results.GetNotPass(),
  969. args.wrapper_script_args)
  970. if args.break_on_failure and not iteration_results.DidRunPass():
  971. break
  972. if iteration_count > 1:
  973. # display summary results
  974. # only display results for a test if at least one test did not pass
  975. all_pass = 0
  976. tot_tests = 0
  977. for test_name in result_counts:
  978. tot_tests += 1
  979. if any(result_counts[test_name][x] for x in (
  980. base_test_result.ResultType.FAIL,
  981. base_test_result.ResultType.CRASH,
  982. base_test_result.ResultType.TIMEOUT,
  983. base_test_result.ResultType.UNKNOWN)):
  984. logging.critical(
  985. '%s: %s',
  986. test_name,
  987. ', '.join('%s %s' % (str(result_counts[test_name][i]), i)
  988. for i in base_test_result.ResultType.GetTypes()))
  989. else:
  990. all_pass += 1
  991. logging.critical('%s of %s tests passed in all %s runs',
  992. str(all_pass),
  993. str(tot_tests),
  994. str(iteration_count))
  995. if save_detailed_results:
  996. assert raw_logs_fh
  997. raw_logs_fh.seek(0)
  998. raw_logs = raw_logs_fh.read()
  999. if raw_logs:
  1000. with out_manager.ArchivedTempfile(
  1001. 'raw_logs.txt', 'raw_logs',
  1002. output_manager.Datatype.TEXT) as raw_logs_file:
  1003. raw_logs_file.write(raw_logs)
  1004. logging.critical('RAW LOGS: %s', raw_logs_file.Link())
  1005. with out_manager.ArchivedTempfile(
  1006. 'test_results_presentation.html',
  1007. 'test_results_presentation',
  1008. output_manager.Datatype.HTML) as results_detail_file:
  1009. result_html_string, _, _ = test_results_presentation.result_details(
  1010. json_path=json_file.name,
  1011. test_name=args.command,
  1012. cs_base_url='http://cs.chromium.org',
  1013. local_output=True)
  1014. results_detail_file.write(result_html_string)
  1015. results_detail_file.flush()
  1016. logging.critical('TEST RESULTS: %s', results_detail_file.Link())
  1017. ui_screenshots = test_results_presentation.ui_screenshot_set(
  1018. json_file.name)
  1019. if ui_screenshots:
  1020. with out_manager.ArchivedTempfile(
  1021. 'ui_screenshots.json',
  1022. 'ui_capture',
  1023. output_manager.Datatype.JSON) as ui_screenshot_file:
  1024. ui_screenshot_file.write(ui_screenshots)
  1025. logging.critical('UI Screenshots: %s', ui_screenshot_file.Link())
  1026. return (0 if all(r.DidRunPass() for r in all_iteration_results)
  1027. else constants.ERROR_EXIT_CODE)
  1028. def _LogRerunStatement(failed_tests, wrapper_arg_str):
  1029. """Logs a message that can rerun the failed tests.
  1030. Logs a copy/pasteable message that filters tests so just the failing tests
  1031. are run.
  1032. Args:
  1033. failed_tests: A set of test results that did not pass.
  1034. wrapper_arg_str: A string of args that were passed to the called wrapper
  1035. script.
  1036. """
  1037. rerun_arg_list = []
  1038. try:
  1039. constants.CheckOutputDirectory()
  1040. # constants.CheckOutputDirectory throws bare exceptions.
  1041. except: # pylint: disable=bare-except
  1042. logging.exception('Output directory not found. Unable to generate failing '
  1043. 'test filter file.')
  1044. return
  1045. test_filter_file = os.path.join(os.path.relpath(constants.GetOutDirectory()),
  1046. _RERUN_FAILED_TESTS_FILE)
  1047. arg_list = shlex.split(wrapper_arg_str) if wrapper_arg_str else sys.argv
  1048. index = 0
  1049. while index < len(arg_list):
  1050. arg = arg_list[index]
  1051. # Skip adding the filter=<file> and/or the filter arg as we're replacing
  1052. # it with the new filter arg.
  1053. # This covers --test-filter=, --test-launcher-filter-file=, --gtest-filter=,
  1054. # --test-filter *Foobar.baz, -f *foobar, --package-filter <package>,
  1055. # --runner-filter <runner>.
  1056. if 'filter' in arg or arg == '-f':
  1057. index += 1 if '=' in arg else 2
  1058. continue
  1059. rerun_arg_list.append(arg)
  1060. index += 1
  1061. failed_test_list = [str(t) for t in failed_tests]
  1062. with open(test_filter_file, 'w') as fp:
  1063. for t in failed_test_list:
  1064. # Test result names can have # in them that don't match when applied as
  1065. # a test name filter.
  1066. fp.write('%s\n' % t.replace('#', '.'))
  1067. rerun_arg_list.append('--test-launcher-filter-file=%s' % test_filter_file)
  1068. msg = """
  1069. %d Test(s) failed.
  1070. Rerun failed tests with copy and pastable command:
  1071. %s
  1072. """
  1073. logging.critical(msg, len(failed_tests), shlex.join(rerun_arg_list))
  1074. def DumpThreadStacks(_signal, _frame):
  1075. for thread in threading.enumerate():
  1076. reraiser_thread.LogThreadStack(thread)
  1077. def main():
  1078. signal.signal(signal.SIGUSR1, DumpThreadStacks)
  1079. parser = argparse.ArgumentParser()
  1080. command_parsers = parser.add_subparsers(
  1081. title='test types', dest='command')
  1082. subp = command_parsers.add_parser(
  1083. 'gtest',
  1084. help='googletest-based C++ tests')
  1085. AddCommonOptions(subp)
  1086. AddDeviceOptions(subp)
  1087. AddEmulatorOptions(subp)
  1088. AddGTestOptions(subp)
  1089. AddTracingOptions(subp)
  1090. AddCommandLineOptions(subp)
  1091. subp = command_parsers.add_parser(
  1092. 'instrumentation',
  1093. help='InstrumentationTestCase-based Java tests')
  1094. AddCommonOptions(subp)
  1095. AddDeviceOptions(subp)
  1096. AddEmulatorOptions(subp)
  1097. AddInstrumentationTestOptions(subp)
  1098. AddSkiaGoldTestOptions(subp)
  1099. AddTracingOptions(subp)
  1100. AddCommandLineOptions(subp)
  1101. subp = command_parsers.add_parser(
  1102. 'junit',
  1103. help='JUnit4-based Java tests')
  1104. AddCommonOptions(subp)
  1105. AddJUnitTestOptions(subp)
  1106. subp = command_parsers.add_parser(
  1107. 'linker',
  1108. help='linker tests')
  1109. AddCommonOptions(subp)
  1110. AddDeviceOptions(subp)
  1111. AddEmulatorOptions(subp)
  1112. AddLinkerTestOptions(subp)
  1113. subp = command_parsers.add_parser(
  1114. 'monkey',
  1115. help="tests based on Android's monkey command")
  1116. AddCommonOptions(subp)
  1117. AddDeviceOptions(subp)
  1118. AddEmulatorOptions(subp)
  1119. AddMonkeyTestOptions(subp)
  1120. subp = command_parsers.add_parser(
  1121. 'python',
  1122. help='python tests based on unittest.TestCase')
  1123. AddCommonOptions(subp)
  1124. AddPythonTestOptions(subp)
  1125. args, unknown_args = parser.parse_known_args()
  1126. if unknown_args:
  1127. if hasattr(args, 'allow_unknown') and args.allow_unknown:
  1128. args.command_line_flags = unknown_args
  1129. else:
  1130. parser.error('unrecognized arguments: %s' % ' '.join(unknown_args))
  1131. # --replace-system-package/--remove-system-package has the potential to cause
  1132. # issues if --enable-concurrent-adb is set, so disallow that combination.
  1133. concurrent_adb_enabled = (hasattr(args, 'enable_concurrent_adb')
  1134. and args.enable_concurrent_adb)
  1135. replacing_system_packages = (hasattr(args, 'replace_system_package')
  1136. and args.replace_system_package)
  1137. removing_system_packages = (hasattr(args, 'system_packages_to_remove')
  1138. and args.system_packages_to_remove)
  1139. if (concurrent_adb_enabled
  1140. and (replacing_system_packages or removing_system_packages)):
  1141. parser.error('--enable-concurrent-adb cannot be used with either '
  1142. '--replace-system-package or --remove-system-package')
  1143. # --use-webview-provider has the potential to cause issues if
  1144. # --enable-concurrent-adb is set, so disallow that combination
  1145. if (hasattr(args, 'use_webview_provider') and
  1146. hasattr(args, 'enable_concurrent_adb') and args.use_webview_provider and
  1147. args.enable_concurrent_adb):
  1148. parser.error('--use-webview-provider and --enable-concurrent-adb cannot '
  1149. 'be used together')
  1150. if (getattr(args, 'coverage_on_the_fly', False)
  1151. and not getattr(args, 'coverage_dir', '')):
  1152. parser.error('--coverage-on-the-fly requires --coverage-dir')
  1153. if (hasattr(args, 'debug_socket') or
  1154. (hasattr(args, 'wait_for_java_debugger') and
  1155. args.wait_for_java_debugger)):
  1156. args.num_retries = 0
  1157. # Result-sink may not exist in the environment if rdb stream is not enabled.
  1158. result_sink_client = result_sink.TryInitClient()
  1159. try:
  1160. return RunTestsCommand(args, result_sink_client)
  1161. except base_error.BaseError as e:
  1162. logging.exception('Error occurred.')
  1163. if e.is_infra_error:
  1164. return constants.INFRA_EXIT_CODE
  1165. return constants.ERROR_EXIT_CODE
  1166. except: # pylint: disable=W0702
  1167. logging.exception('Unrecognized error occurred.')
  1168. return constants.ERROR_EXIT_CODE
  1169. if __name__ == '__main__':
  1170. sys.exit(main())